├── .coveragerc ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── feature_request.md │ ├── help-.md │ ├── installation-issue.md │ └── package-management.md ├── pull_request_template.md └── workflows │ ├── cd.yml │ ├── ci-release.yml │ ├── deploy-image.yml │ └── doc-tests.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── README.rst ├── auxdir └── CFIS │ ├── CFIS_cuts.txt │ ├── filter_curves │ ├── CFHT-MegaCam.r.dat │ └── info.rst │ ├── tiles_202007 │ ├── tiles_DR2_no_patch.txt │ ├── tiles_P1.txt │ ├── tiles_P2.txt │ ├── tiles_P3.txt │ ├── tiles_P3_wo_W3.txt │ ├── tiles_P4.txt │ ├── tiles_W3.txt │ ├── tiles_W3_m2.txt │ └── tiles_all_order.txt │ ├── tiles_202011 │ ├── tiles_W3_complete.txt │ ├── tiles_i.txt │ ├── tiles_r.txt │ ├── tiles_u.txt │ └── tiles_z.txt │ ├── tiles_202106 │ ├── tiles_P1.txt │ ├── tiles_P2.txt │ ├── tiles_P3.txt │ ├── tiles_P3_wo_W3.txt │ ├── tiles_P4.txt │ ├── tiles_P5.txt │ ├── tiles_P6.txt │ ├── tiles_P7.txt │ └── tiles_W3.txt │ └── tiles_202402 │ ├── tiles_COSMOS.txt │ └── tiles_all.txt ├── docs ├── _templates │ ├── module.rst_t │ ├── package.rst_t │ └── toc.rst_t └── source │ ├── about.md │ ├── basic_execution.md │ ├── canfar.md │ ├── conf.py │ ├── configuration.md │ ├── contributing.md │ ├── contributors.md │ ├── dependencies.md │ ├── img │ ├── 1_nb_nonmasked.png │ ├── 2_nb_stars.png │ ├── 5_mode_fhwm_star.png │ ├── ShapePipe-6.pdf │ ├── ShapePipe_v0.0.1.png │ ├── cosmostat_logo.jpg │ ├── diag_mask.png │ ├── fwhm_field-2113737-10.png │ ├── fwhm_field-2113737-35.png │ ├── hist_mag_stars-2113737-10.png │ ├── mag_star_field-2113737-10.png │ └── size_mag-2113737-10.png │ ├── index.rst │ ├── installation.md │ ├── module_develop.md │ ├── module_example.md │ ├── pipeline_canfar.md │ ├── pipeline_tutorial.md │ ├── post_processing.md │ ├── random_cat.md │ ├── refs.bib │ ├── testing.md │ ├── toc.rst │ ├── understanding_api.md │ ├── vos_retrieve.md │ └── z_ref.rst ├── environment-dev.yml ├── environment.yml ├── example ├── cfis │ ├── config_Fe.ini │ ├── config_Gie_symlink.ini │ ├── config_Gie_vos.ini │ ├── config_GitFeGie_symlink.ini │ ├── config_GitFeGie_vos.ini │ ├── config_GitFe_symlink.ini │ ├── config_Git_vos.ini │ ├── config_MCCD.ini │ ├── config_MaMa_onthefly.ini │ ├── config_MaMa_save.ini │ ├── config_MsPl_mccd.ini │ ├── config_MsPl_psfex.ini │ ├── config_MsPl_stars.ini │ ├── config_Ms_psfex.ini │ ├── config_Ms_psfex_conv.ini │ ├── config_Pl_mccd.ini │ ├── config_Pl_psfex.ini │ ├── config_Pl_psfex_conv.ini │ ├── config_Rc.ini │ ├── config_exp_Ma_onthefly.ini │ ├── config_exp_Mh.ini │ ├── config_exp_Pi.ini │ ├── config_exp_Sp.ini │ ├── config_exp_SpMh.ini │ ├── config_exp_mccd.ini │ ├── config_exp_psfex.ini │ ├── config_get_tiles_vos_headers.ini │ ├── config_make_cat_mccd.ini │ ├── config_make_cat_psfex.ini │ ├── config_make_cat_psfex_nosm.ini │ ├── config_merge_sep_cats_template.ini │ ├── config_onthefly.mask │ ├── config_save.mask │ ├── config_stats.ini │ ├── config_tile_Ma_onthefly.ini │ ├── config_tile_MiViSmVi.ini │ ├── config_tile_Ng_template.ini │ ├── config_tile_Ng_template_batch.ini │ ├── config_tile_PiViSmVi.ini │ ├── config_tile_PiViSmVi_canfar.ini │ ├── config_tile_PiViVi_canfar.ini │ ├── config_tile_Sx.ini │ ├── config_tile_Sx_exp_mccd.ini │ ├── config_tile_Sx_exp_psfex.ini │ ├── config_tile_Uz.ini │ ├── config_tile_onthefly.mask │ ├── config_tile_save.mask │ ├── config_valjoint_Pl_mccd.ini │ ├── default.conv │ ├── default.param │ ├── default.psfex │ ├── default_exp.sex │ ├── default_tile.sex │ ├── final_cat.param │ ├── mask_default │ │ ├── MEGAPRIME_star_i_13.8.reg │ │ ├── Messier_catalog.npy │ │ ├── Messier_catalog_updated.fits │ │ ├── default.ww │ │ ├── halo_mask.reg │ │ └── ngc_cat.fits │ └── star_selection.setools ├── cfis_simu │ ├── config.mask_simu │ ├── config_GitFeGie_symlink.ini │ ├── config_MCCD.ini │ ├── config_MaMa.ini │ ├── config_exp_SpMh.ini │ ├── config_make_cat_mccd.ini │ ├── config_merge_sep_cats_template.ini │ ├── config_tile.mask_simu │ ├── config_tile_MiViSmVi.ini │ ├── config_tile_Ng_template.ini │ ├── config_tile_Sx_exp_mccd.ini │ ├── config_tile_Sx_exp_psfex.ini │ ├── default.conv │ ├── default.param │ ├── default.psfex │ ├── default_exp.sex │ ├── default_tile.sex │ ├── job_sp_simu.bash │ ├── mask_default │ │ ├── MEGAPRIME_star_i_13.8.reg │ │ ├── Messier_catalog.npy │ │ ├── Messier_catalog_updated.fits │ │ ├── default.ww │ │ ├── halo_mask.reg │ │ └── ngc_cat.fits │ ├── readme.txt │ ├── star_selection.setools │ └── tile_numbers.txt ├── config.ini ├── data │ ├── letters │ │ ├── letters_1.txt │ │ ├── letters_2.txt │ │ ├── letters_3.txt │ │ ├── letters_4.txt │ │ └── letters_5.txt │ ├── number_list.txt │ └── numbers │ │ ├── hold_3.txt │ │ ├── numbers_1.txt │ │ ├── numbers_2.txt │ │ ├── numbers_4.txt │ │ └── numbers_5.txt ├── output │ └── .gitkeep ├── pbs │ ├── candide_mpi.sh │ ├── candide_smp.sh │ ├── cc_mpi.sh │ ├── cc_smp.sh │ ├── config_mpi.ini │ └── config_smp.ini └── unions_800 │ ├── cat_matched.param │ └── config_tile_match_ext_r.ini ├── install_shapepipe ├── nsswitch.conf ├── pyproject.toml ├── scripts ├── README.rst ├── jupyter │ ├── CFIS_Footprint.ipynb │ ├── plot_spectro_areas.ipynb │ ├── shapepipe_tutorial_exploration_notebook.ipynb │ ├── summary_run.ipynb │ └── wcs.ipynb ├── python │ ├── canfar_avail_results.py │ ├── canfar_run_analyse.py │ ├── cfis_field_select.py │ ├── check_nobj_ngmix_mc.py │ ├── convert_psf_pix2world.py │ ├── count_shdu_special.py │ ├── create_final_cat.py │ ├── create_sample_results.py │ ├── create_star_cat.py │ ├── get_ccds_with_psf.py │ ├── get_number_objects.py │ ├── link_to_exp_for_tile.py │ ├── merge_final_cat.py │ ├── stats_global.py │ ├── stats_headless_canfar.py │ ├── summary_params_pre_v2.py │ ├── summary_run.py │ ├── summary_tiles.py │ ├── update_runs_log_file.py │ └── vignets2image.py └── sh │ ├── canfar_download_results.bash │ ├── canfar_selection.sh │ ├── canfar_submit_selection.sh │ ├── combine_runs.bash │ ├── curl_canfar_local.sh │ ├── curl_canfar_monitor_local.sh │ ├── functions.sh │ ├── init_run_exclusive_canfar.sh │ ├── job_sp.bash │ ├── job_sp_canfar.bash │ ├── missing_unique.sh │ ├── post_proc_sp.bash │ ├── remove_duplicates.sh │ ├── remove_duplicates_tiles.sh │ ├── run_scratch_local.sh │ ├── stats_jobs_canfar.sh │ └── untar_results.bash ├── src └── shapepipe │ ├── __init__.py │ ├── info.py │ ├── modules │ ├── __init__.py │ ├── execute_example_runner.py │ ├── find_exposures_package │ │ ├── __init__.py │ │ └── find_exposures.py │ ├── find_exposures_runner.py │ ├── get_images_package │ │ ├── __init__.py │ │ └── get_images.py │ ├── get_images_runner.py │ ├── make_cat_package │ │ ├── __init__.py │ │ └── make_cat.py │ ├── make_cat_runner.py │ ├── mask_package │ │ ├── __init__.py │ │ └── mask.py │ ├── mask_runner.py │ ├── match_external_package │ │ ├── __init__.py │ │ └── match_external.py │ ├── match_external_runner.py │ ├── mccd_fit_runner.py │ ├── mccd_fit_val_runner.py │ ├── mccd_interp_runner.py │ ├── mccd_package │ │ ├── __init__.py │ │ ├── mccd_interpolation_script.py │ │ ├── mccd_plot_utilities.py │ │ └── shapepipe_auxiliary_mccd.py │ ├── mccd_plots_runner.py │ ├── mccd_preprocessing_runner.py │ ├── mccd_val_runner.py │ ├── merge_headers_package │ │ ├── __init__.py │ │ └── merge_headers.py │ ├── merge_headers_runner.py │ ├── merge_sep_cats_package │ │ ├── __init__.py │ │ └── merge_sep_cats.py │ ├── merge_sep_cats_runner.py │ ├── merge_starcat_package │ │ ├── __init__.py │ │ └── merge_starcat.py │ ├── merge_starcat_runner.py │ ├── module_decorator.py │ ├── module_runners.py │ ├── ngmix_package │ │ ├── __init__.py │ │ └── ngmix.py │ ├── ngmix_runner.py │ ├── pastecat_package │ │ ├── __init__.py │ │ └── pastecat.py │ ├── pastecat_runner.py │ ├── psfex_interp_package │ │ ├── __init__.py │ │ └── psfex_interp.py │ ├── psfex_interp_runner.py │ ├── psfex_package │ │ ├── __init__.py │ │ └── psfex_script.py │ ├── psfex_runner.py │ ├── python_example_package │ │ ├── __init__.py │ │ └── python_example.py │ ├── python_example_runner.py │ ├── random_cat_package │ │ ├── __init__.py │ │ └── random_cat.py │ ├── random_cat_runner.py │ ├── serial_example_runner.py │ ├── setools_package │ │ ├── __init__.py │ │ └── setools.py │ ├── setools_runner.py │ ├── sextractor_package │ │ ├── __init__.py │ │ └── sextractor_script.py │ ├── sextractor_runner.py │ ├── split_exp_package │ │ ├── __init__.py │ │ └── split_exp.py │ ├── split_exp_runner.py │ ├── spread_model_package │ │ ├── __init__.py │ │ └── spread_model.py │ ├── spread_model_runner.py │ ├── uncompress_fits_package │ │ ├── __init__.py │ │ └── uncompress_fits.py │ ├── uncompress_fits_runner.py │ ├── vignetmaker_package │ │ ├── __init__.py │ │ └── vignetmaker.py │ └── vignetmaker_runner.py │ ├── pipeline │ ├── __init__.py │ ├── args.py │ ├── config.py │ ├── dependency_handler.py │ ├── execute.py │ ├── file_handler.py │ ├── file_io.py │ ├── job_handler.py │ ├── mpi_run.py │ ├── run_log.py │ ├── shared.py │ ├── str_handler.py │ ├── timeout.py │ └── worker_handler.py │ ├── run.py │ ├── shapepipe_run.py │ ├── tests │ ├── test_get_images.py │ ├── test_pipeline.py │ └── test_utilities.py │ └── utilities │ ├── __init__.py │ ├── cfis.py │ ├── file_system.py │ ├── galaxy.py │ └── summary.py └── xterm-start.sh /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | show_missing=True 3 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "[BUG]" 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behaviour: 15 | 16 | **Expected behaviour** 17 | A clear and concise description of what you expected to happen. 18 | 19 | **Screenshots** 20 | If applicable, add screenshots to help explain your problem. 21 | 22 | **Desktop (please complete the following information):** 23 | - OS: [e.g. iOS] 24 | - Browser [e.g. chrome, safari] 25 | - Version [e.g. 22] 26 | 27 | **Additional context** 28 | Add any other context about the problem here. 29 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: "[NEW FEATURE]" 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/help-.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Help! 3 | about: General help needed 4 | title: "[HELP]" 5 | labels: help wanted 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Let us know how we can help** 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/installation-issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Installation Issue 3 | about: Problem with package installation 4 | title: "[INSTALL ISSUE]" 5 | labels: install issue 6 | assignees: '' 7 | 8 | --- 9 | 10 | Describe the problem 11 | A clear and concise description of the problem encountered. 12 | 13 | To Reproduce 14 | Steps to reproduce the behaviour: 15 | 16 | Screenshots 17 | If applicable, add screenshots to help explain your problem. 18 | 19 | Desktop (please complete the following information): 20 | 21 | OS: [e.g. iOS] 22 | Browser [e.g. chrome, safari] 23 | Version [e.g. 22] 24 | Additional context 25 | Add any other context about the problem here 26 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/package-management.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Package Management 3 | about: Issues concerning the overall management of the project 4 | title: "[Management]" 5 | labels: clean up 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Summary 2 | 3 | > Developers should provide a summary of the proposed changes here and include "closes #" if this addresses an open issue. 4 | 5 | ## Reviewer Checklist 6 | 7 | > Reviewers should tick the following boxes before approving and merging the PR. 8 | 9 | - [ ] The PR targets the `develop` branch 10 | - [ ] The PR is assigned to the developer 11 | - [ ] The PR has appropriate labels 12 | - [ ] The PR is included in appropriate projects and/or milestones 13 | - [ ] The PR includes a clear description of the proposed changes 14 | - [ ] If the PR addresses an open issue the description includes "closes #" 15 | - [ ] The code and documentation style match the current standards 16 | - [ ] Documentation has been added/updated consistently with the code 17 | - [ ] All CI tests are passing 18 | - [ ] API docs have been built and checked at least once (if relevant) 19 | - [ ] All changed files have been checked and comments provided to the developer 20 | - [ ] All of the reviewer's comments have been satisfactorily addressed by the developer 21 | -------------------------------------------------------------------------------- /.github/workflows/cd.yml: -------------------------------------------------------------------------------- 1 | name: Deploy API Docs 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | - main 8 | 9 | jobs: 10 | 11 | api: 12 | name: Deploy API Documentation 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | 18 | - name: Set up conda 19 | uses: conda-incubator/setup-miniconda@v2 20 | with: 21 | auto-update-conda: true 22 | python-version: 3.9 23 | auto-activate-base: true 24 | 25 | - name: Install dependencies 26 | shell: bash -l {0} 27 | run: | 28 | ./install_shapepipe --develop --no-exe --no-mpi 29 | conda activate shapepipe 30 | python -m pip install --upgrade importlib-metadata 31 | conda install -c conda-forge pandoc 32 | 33 | - name: Build API documentation 34 | shell: bash -l {0} 35 | run: | 36 | conda activate shapepipe 37 | sphinx-apidoc -t docs/_templates -feTMo docs/source shapepipe shapepipe/modules/*_runner.py 38 | sphinx-build -E docs/source docs/_build 39 | 40 | - name: Deploy API documentation 41 | uses: peaceiris/actions-gh-pages@v3.5.9 42 | with: 43 | github_token: ${{ secrets.GITHUB_TOKEN }} 44 | publish_dir: docs/_build 45 | -------------------------------------------------------------------------------- /.github/workflows/ci-release.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | - master 8 | 9 | jobs: 10 | 11 | test-full: 12 | name: Full Test Suite 13 | runs-on: ${{ matrix.os }} 14 | 15 | strategy: 16 | fail-fast: false 17 | matrix: 18 | os: [ubuntu-latest, macos-latest] 19 | python-version: [3.11] 20 | 21 | steps: 22 | - uses: actions/checkout@v4 23 | 24 | - name: Install Linux dependencies 25 | if: runner.os == 'Linux' 26 | run: | 27 | sudo apt-get install libopenblas-dev gcc-9 g++-9 28 | 29 | - name: Install macOS Dependencies 30 | shell: bash -l {0} 31 | if: runner.os == 'macOS' 32 | run: | 33 | brew tap sfarrens/sf 34 | brew install bigmac libomp 35 | brew reinstall gcc 36 | 37 | - name: Set up conda 38 | uses: conda-incubator/setup-miniconda@v3 39 | with: 40 | auto-update-conda: true 41 | python-version: ${{ matrix.python-version }} 42 | auto-activate-base: true 43 | miniforge-version: latest 44 | 45 | - name: Install package (Linux) 46 | shell: bash -l {0} 47 | env: 48 | CC: gcc-9 49 | CXX: g++-9 50 | if: runner.os == 'Linux' 51 | run: ./install_shapepipe --develop 52 | 53 | - name: Install package (macOS) 54 | shell: bash -l {0} 55 | if: runner.os == 'macOS' 56 | run: ./install_shapepipe --develop 57 | 58 | - name: Run tests 59 | shell: bash -l {0} 60 | run: | 61 | conda activate shapepipe-dev 62 | pytest 63 | shapepipe_run -c example/config.ini 64 | -------------------------------------------------------------------------------- /.github/workflows/deploy-image.yml: -------------------------------------------------------------------------------- 1 | name: Create and publish a Docker image 2 | 3 | on: [push, workflow_dispatch] 4 | 5 | env: 6 | REGISTRY: ghcr.io 7 | IMAGE_NAME: ${{ github.repository }} 8 | BRANCH: ${{ github.ref }} 9 | 10 | jobs: 11 | build-and-push-image: 12 | runs-on: 13 | - ubuntu-latest 14 | # - macos-latest 15 | permissions: 16 | contents: read 17 | packages: write 18 | 19 | steps: 20 | - name: Log in to the Container registry 21 | uses: docker/login-action@v3 22 | with: 23 | registry: ${{ env.REGISTRY }} 24 | username: ${{ github.actor }} 25 | password: ${{ secrets.GITHUB_TOKEN }} 26 | 27 | - name: Set up Docker Buildx 28 | uses: docker/setup-buildx-action@v3 29 | 30 | - name: Extract metadata (tags, labels) for Docker 31 | id: meta 32 | uses: docker/metadata-action@v5 33 | with: 34 | images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} 35 | 36 | - name: Build and export to Docker 37 | uses: docker/build-push-action@v6 38 | with: 39 | load: true 40 | tags: ${{ steps.meta.outputs.tags }} 41 | labels: ${{ steps.meta.outputs.labels }} 42 | 43 | - name: Test 44 | run: docker run --rm ${{ steps.meta.outputs.tags }} shapepipe_run -c /app/example/config.ini 45 | 46 | - name: Push 47 | uses: docker/build-push-action@v6 48 | with: 49 | push: true 50 | tags: ${{ steps.meta.outputs.tags }} 51 | labels: ${{ steps.meta.outputs.labels }} -------------------------------------------------------------------------------- /.github/workflows/doc-tests.yml: -------------------------------------------------------------------------------- 1 | name: Build API Doc 2 | 3 | on: [workflow_dispatch] 4 | 5 | jobs: 6 | 7 | Ubuntu: 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@v2 12 | 13 | - name: Set up conda 14 | uses: conda-incubator/setup-miniconda@v2 15 | with: 16 | auto-update-conda: true 17 | python-version: 3.8 18 | auto-activate-base: true 19 | 20 | - name: Install dependencies 21 | shell: bash -l {0} 22 | run: | 23 | ./install_shapepipe --develop --no-exe --no-mpi 24 | conda activate shapepipe 25 | python -m pip install --upgrade importlib-metadata 26 | conda install -c conda-forge pandoc 27 | 28 | - name: Build API documentation 29 | shell: bash -l {0} 30 | run: | 31 | conda activate shapepipe 32 | sphinx-apidoc -t docs/_templates -feTMo docs/source shapepipe shapepipe/modules/*_runner.py 33 | sphinx-build -E docs/source docs/_build 34 | 35 | - name: Archive API build 36 | uses: actions/upload-artifact@v2 37 | with: 38 | name: api-docs 39 | retention-days: 14 40 | path: | 41 | docs/_build 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | docs/build/ 74 | docs/source/shapepipe.* 75 | 76 | # PyBuilder 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # IPython 83 | profile_default/ 84 | ipython_config.py 85 | 86 | # pyenv 87 | .python-version 88 | 89 | # pipenv 90 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 91 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 92 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 93 | # install all needed dependencies. 94 | #Pipfile.lock 95 | 96 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 97 | __pypackages__/ 98 | 99 | # Celery stuff 100 | celerybeat-schedule 101 | celerybeat.pid 102 | 103 | # SageMath parsed files 104 | *.sage.py 105 | 106 | # Environments 107 | .env 108 | .venv 109 | env/ 110 | venv/ 111 | ENV/ 112 | env.bak/ 113 | venv.bak/ 114 | 115 | # Spyder project settings 116 | .spyderproject 117 | .spyproject 118 | 119 | # Rope project settings 120 | .ropeproject 121 | 122 | # mkdocs documentation 123 | /site 124 | 125 | # mypy 126 | .mypy_cache/ 127 | .dmypy.json 128 | dmypy.json 129 | 130 | # Pyre type checker 131 | .pyre/ 132 | 133 | # Ignore example output 134 | *shapepipe_run_* 135 | *shapepipe_runs* 136 | code 137 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 CosmoStat 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ShapePipe 2 | ========= 3 | 4 | |CI| |CD| |python39| |release| 5 | 6 | .. |CI| image:: https://github.com/CosmoStat/shapepipe/workflows/CI/badge.svg 7 | :target: https://github.com/CosmoStat/shapepipe/actions?query=workflow%3ACI 8 | 9 | .. |CD| image:: https://github.com/CosmoStat/shapepipe/actions/workflows/pages/pages-build-deployment/badge.svg 10 | :target: https://github.com/CosmoStat/shapepipe/actions/workflows/pages/pages-build-deployment 11 | 12 | .. |python39| image:: https://img.shields.io/badge/python-3.9-green.svg 13 | :target: https://www.python.org/‰ 14 | 15 | .. |release| image:: https://img.shields.io/github/v/release/CosmoStat/shapepipe 16 | :target: https://github.com/CosmoStat/shapepipe/releases/latest 17 | 18 | ShapePipe is a galaxy shape measurement pipeline developed within the 19 | CosmoStat lab at CEA Paris-Saclay. 20 | 21 | See the `documentation `_ for details 22 | on how to install and run ShapePipe. 23 | -------------------------------------------------------------------------------- /auxdir/CFIS/filter_curves/info.rst: -------------------------------------------------------------------------------- 1 | CFHT-MegaCam.r.dat Total transmission curve (QE + atmosphere + filter). 2 | From Jean-Charles Cuillandre (email from 09/11/2020). 3 | 4 | -------------------------------------------------------------------------------- /auxdir/CFIS/tiles_202402/tiles_COSMOS.txt: -------------------------------------------------------------------------------- 1 | 299.183 2 | 300.183 3 | 301.183 4 | 299.184 5 | 300.184 6 | 301.184 7 | 299.185 8 | 300.185 9 | 301.185 10 | -------------------------------------------------------------------------------- /docs/_templates/module.rst_t: -------------------------------------------------------------------------------- 1 | {%- if show_headings %} 2 | {{- basename | e | heading(2) }} 3 | 4 | {% endif -%} 5 | .. automodule:: {{ qualname }} 6 | {%- for option in automodule_options %} 7 | :{{ option }}: 8 | {%- endfor %} 9 | -------------------------------------------------------------------------------- /docs/_templates/package.rst_t: -------------------------------------------------------------------------------- 1 | {%- macro automodule(modname, options) -%} 2 | .. automodule:: {{ modname }} 3 | {%- for option in options %} 4 | :{{ option }}: 5 | {%- endfor %} 6 | {%- endmacro %} 7 | 8 | {%- macro toctree(docnames) -%} 9 | .. toctree:: 10 | :maxdepth: {{ maxdepth }} 11 | {% for docname in docnames %} 12 | {{ docname }} 13 | {%- endfor %} 14 | {%- endmacro %} 15 | 16 | {%- if is_namespace %} 17 | {{- [pkgname, "namespace"] | join(" ") | e | heading }} 18 | {% else %} 19 | {{- pkgname | e | heading }} 20 | 21 | {% endif %} 22 | 23 | {%- if modulefirst and not is_namespace %} 24 | {{ automodule(pkgname, automodule_options) }} 25 | {% endif %} 26 | 27 | {%- if subpackages %} 28 | Subpackages 29 | ----------- 30 | 31 | {{ toctree(subpackages) }} 32 | {% endif %} 33 | 34 | {%- if submodules %} 35 | Submodules 36 | ---------- 37 | {% if separatemodules %} 38 | {{ toctree(submodules) }} 39 | {% else %} 40 | {%- for submodule in submodules %} 41 | {% if show_headings %} 42 | {{- submodule | e | heading(2) }} 43 | {% endif %} 44 | {{ automodule(submodule, automodule_options) }} 45 | {% endfor %} 46 | {%- endif %} 47 | {%- endif %} 48 | 49 | {%- if not modulefirst and not is_namespace %} 50 | Module contents 51 | --------------- 52 | 53 | {{ automodule(pkgname, automodule_options) }} 54 | {% endif %} 55 | -------------------------------------------------------------------------------- /docs/_templates/toc.rst_t: -------------------------------------------------------------------------------- 1 | {{ header | heading }} 2 | 3 | .. toctree:: 4 | :maxdepth: {{ maxdepth }} 5 | {% for docname in docnames %} 6 | {{ docname }} 7 | {%- endfor %} 8 | 9 | -------------------------------------------------------------------------------- /docs/source/about.md: -------------------------------------------------------------------------------- 1 | # About 2 | 3 | ShapePipe an open-source and modular weak-lensing measurement, analysis and 4 | validation pipeline written in Python. 5 | 6 | The current version of ShapePipe starts with reduced survey images and ends by 7 | providing galaxy shape measurements along with all of the information required for 8 | shear calibration. It includes various validation tools and a novel point spread 9 | function (PSF) modelling technique. The code has been designed to facilitate 10 | the inclusion of new or improved processing steps to adapt to advances made 11 | in the coming years. 12 | 13 | ```{figure} https://www.skysurvey.cc/wp-content/uploads/2022/04/UNIONS-Logo-TextBlack-1K-300x82.png 14 | --- 15 | figclass: margin 16 | alt: UNIONS logo 17 | target: https://www.skysurvey.cc/ 18 | --- 19 | ``` 20 | 21 | The primary application of ShapePipe so far has been to the Ultraviolet 22 | Near-Infrared Optical Northern Survey (UNIONS) data (see {cite:t}`guinot:22` 23 | and Kilbinger et al., in prep). 24 | 25 | We plan to add features and find new applications for ShapePipe and will update 26 | this documentation accordingly. For more details on the first public release 27 | of ShapePipe please see {cite:t}`farrens:22b`. 28 | -------------------------------------------------------------------------------- /docs/source/basic_execution.md: -------------------------------------------------------------------------------- 1 | # Basic Execution 2 | 3 | ShapePipe pipelines are launched and managed via the `shapepipe_run` script. 4 | 5 | A list of command line arguments can be displayed using the `--help` 6 | option: 7 | 8 | ```{seealso} 9 | :class: margin 10 | The `shapepipe` environment will need to be built and activated in order to run this script (see [Installation](installation.md)). 11 | ``` 12 | ```bash 13 | shapepipe_run --help 14 | ``` 15 | 16 | The options for defining a pipeline are managed via a 17 | [configuration file](configuration.md). 18 | 19 | ## Running ShapePipe with Joblib 20 | 21 | By default ShapePipe is run using 22 | [Joblib](https://joblib.readthedocs.io/en/latest/) to manage parallel 23 | processes. The `shapepipe_run` script can run as follows 24 | 25 | ```bash 26 | shapepipe_run 27 | ``` 28 | 29 | which will by default look for a file called `config.ini` in the current 30 | directory. To specify the path to a given configuration file (with any name) 31 | you would run 32 | 33 | ```bash 34 | shapepipe_run -c 35 | ``` 36 | 37 | ## Running the Pipeline with MPI 38 | 39 | ShapePipe can also use [mpi4py](https://mpi4py.readthedocs.io/en/stable/) 40 | for managing parallel processes on clusters with multiple nodes. 41 | The `shapepipe_run` script can be run with MPI as follows 42 | 43 | ```bash 44 | mpiexec -n shapepipe_run 45 | ``` 46 | 47 | where `` is the number of cores to allocate to the run. 48 | -------------------------------------------------------------------------------- /docs/source/contributing.md: -------------------------------------------------------------------------------- 1 | # Contribution Guidelines 2 | 3 | ShapePipe is a fully open-source project and we welcome contributions. 4 | 5 | Pleas read our 6 | [contribution guidelines](https://github.com/CosmoStat/shapepipe/blob/develop/CONTRIBUTING.md). 7 | for details on how to contribute to the development of this package. 8 | 9 | All contributors are kindly asked to adhere to the 10 | [code of conduct](https://github.com/CosmoStat/shapepipe/blob/develop/CODE_OF_CONDUCT.md) 11 | at all times to ensure a safe and inclusive environment for everyone. 12 | -------------------------------------------------------------------------------- /docs/source/contributors.md: -------------------------------------------------------------------------------- 1 | # Contributors 2 | 3 | ShapePipe is an open source project and we welcome any 4 | [contributions](contributing.md) that could improve the quality or extend the 5 | possible applications of the package. 6 | 7 | Below will list the individuals who have contributed to the development of 8 | ShapePipe so far. 9 | 10 | ```{figure} https://avatars.githubusercontent.com/u/6851839?v=4 11 | --- 12 | figclass: margin 13 | alt: Samuel Farrens 14 | width: 25% 15 | target: https://github.com/sfarrens 16 | --- 17 | Samuel Farrens 18 | ``` 19 | ```{figure} https://avatars.githubusercontent.com/u/39480528?v=4 20 | --- 21 | figclass: margin 22 | alt: Axel Guinot 23 | width: 25% 24 | target: https://github.com/aguinot 25 | --- 26 | Axel Guinot 27 | ``` 28 | ```{figure} https://avatars.githubusercontent.com/u/4549196?v=4 29 | --- 30 | figclass: margin 31 | alt: Martin Kilbinger 32 | width: 25% 33 | target: https://github.com/martinkilbinger 34 | --- 35 | Martin Kilbinger 36 | ``` 37 | 38 | ## Maintainers 39 | 40 | - [Samuel Farrens](https://github.com/sfarrens) : Development lead, core package developer, performance and debugging 41 | - [Axel Guinot](https://github.com/aguinot) : Primary module developer, validation, performance and debugging 42 | - [Martin Kilbinger](https://github.com/martinkilbinger) : Scientific lead, module developer and debugging 43 | 44 | ## Principal Contributors 45 | 46 | - [Lucie Baumont](https://github.com/lbaumo) : Testing 47 | - [Jérôme Bonnin](https://github.com/jerome-bonnin) : PSF module developer 48 | - Marc Gentile : File IO, initial architecture 49 | - Xavier Jimenez : External catalogue matching module developer 50 | - [François Lanusse](https://github.com/eiffl) : Further contributors 51 | - [Tobias Liaudat](https://github.com/tobias-liaudat) : PSF module developer and validation 52 | - [Austin Peel](https://github.com/austinpeel) : Validation 53 | - [Arnau Pujol](https://github.com/arnaupujol) : Validation 54 | - [Morgan Schmitz](https://github.com/MorganSchmitz) : PSF module developer and validation 55 | - [Jean-Luc Starck](https://github.com/jstarck) : Further contributors 56 | - [Florent Sureau](https://github.com/florentsureau) : Further contributors 57 | - [André Zamorano Vitorelli](https://github.com/orgs/CosmoStat/people/andrevitorelli) : Further contributors 58 | -------------------------------------------------------------------------------- /docs/source/dependencies.md: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | 3 | All third-party software packages required by ShapePipe are installed 4 | automatically (see [Installation](installation.md)). Note that all packages 5 | are pinned to a given version for each release of ShapePipe to maximise the 6 | reproducibility of the results. Below we list the main packages used. 7 | 8 | ## Python Dependencies 9 | 10 | | Package Name | References | 11 | |--------------|---------------------------------------------------| 12 | | [Astropy](https://www.astropy.org/) | {cite:p}`astropy:2013,astropy:2018` | 13 | | [GalSim](https://github.com/GalSim-developers/GalSim) | {cite:p}`rowe:15` | 14 | | [Joblib](https://joblib.readthedocs.io/en/latest/) | {cite:p}`joblib:20` | 15 | | [Matplotlib](https://matplotlib.org/) | {cite:p}`hunter:07` | 16 | | [MCCD](https://github.com/CosmoStat/mccd) | {cite:p}`liaudat:21` | 17 | | [ModOpt](https://cea-cosmic.github.io/ModOpt/) | {cite:p}`farrens:20` | 18 | | [mpi4py](https://mpi4py.readthedocs.io/en/stable/) | {cite:p}`dalcin:05,dalcin:08,dalcin:11` | 19 | | [NGMIX](https://github.com/esheldon/ngmix) | {cite:p}`sheldon:15` | 20 | | [Numpy](https://numpy.org/) | {cite:p}`harris:20` | 21 | | [Pandas](https://pandas.pydata.org/) | {cite:p}`pandas:10,pandas:20` | 22 | | [sf_tools](https://github.com/sfarrens/sf_tools) | | 23 | | [sip_tpv](https://github.com/stargaser/sip_tpv) | {cite:p}`shupe:12` | 24 | | [sqlitedict](https://github.com/RaRe-Technologies/sqlitedict) | | 25 | | [TreeCorr](https://rmjarvis.github.io/TreeCorr/_build/html/index.html) | {cite:p}`jarvis:04` | 26 | 27 | ## Executable Dependencies 28 | 29 | | Package Name | References | 30 | |---------------|------------| 31 | | [CDSclient](http://cdsarc.u-strasbg.fr/doc/cdsclient.html) | | 32 | | [PSFEx](https://www.astromatic.net/software/psfex/) | {cite:p}`bertin:11` | 33 | | [SExtractor](https://www.astromatic.net/software/sextractor/) | {cite:p}`bertin:96` | 34 | | [WeightWatcher](https://www.astromatic.net/software/weightwatcher/) | {cite:p}`marmo:08` | 35 | -------------------------------------------------------------------------------- /docs/source/img/1_nb_nonmasked.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CosmoStat/shapepipe/43d3730f0a7766a6585bc085c8412811aa50563e/docs/source/img/1_nb_nonmasked.png -------------------------------------------------------------------------------- /docs/source/img/2_nb_stars.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CosmoStat/shapepipe/43d3730f0a7766a6585bc085c8412811aa50563e/docs/source/img/2_nb_stars.png -------------------------------------------------------------------------------- /docs/source/img/5_mode_fhwm_star.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CosmoStat/shapepipe/43d3730f0a7766a6585bc085c8412811aa50563e/docs/source/img/5_mode_fhwm_star.png -------------------------------------------------------------------------------- /docs/source/img/ShapePipe-6.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CosmoStat/shapepipe/43d3730f0a7766a6585bc085c8412811aa50563e/docs/source/img/ShapePipe-6.pdf -------------------------------------------------------------------------------- /docs/source/img/ShapePipe_v0.0.1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CosmoStat/shapepipe/43d3730f0a7766a6585bc085c8412811aa50563e/docs/source/img/ShapePipe_v0.0.1.png -------------------------------------------------------------------------------- /docs/source/img/cosmostat_logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CosmoStat/shapepipe/43d3730f0a7766a6585bc085c8412811aa50563e/docs/source/img/cosmostat_logo.jpg -------------------------------------------------------------------------------- /docs/source/img/diag_mask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CosmoStat/shapepipe/43d3730f0a7766a6585bc085c8412811aa50563e/docs/source/img/diag_mask.png -------------------------------------------------------------------------------- /docs/source/img/fwhm_field-2113737-10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CosmoStat/shapepipe/43d3730f0a7766a6585bc085c8412811aa50563e/docs/source/img/fwhm_field-2113737-10.png -------------------------------------------------------------------------------- /docs/source/img/fwhm_field-2113737-35.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CosmoStat/shapepipe/43d3730f0a7766a6585bc085c8412811aa50563e/docs/source/img/fwhm_field-2113737-35.png -------------------------------------------------------------------------------- /docs/source/img/hist_mag_stars-2113737-10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CosmoStat/shapepipe/43d3730f0a7766a6585bc085c8412811aa50563e/docs/source/img/hist_mag_stars-2113737-10.png -------------------------------------------------------------------------------- /docs/source/img/mag_star_field-2113737-10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CosmoStat/shapepipe/43d3730f0a7766a6585bc085c8412811aa50563e/docs/source/img/mag_star_field-2113737-10.png -------------------------------------------------------------------------------- /docs/source/img/size_mag-2113737-10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CosmoStat/shapepipe/43d3730f0a7766a6585bc085c8412811aa50563e/docs/source/img/size_mag-2113737-10.png -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | ShapePipe 2 | ========= 3 | 4 | .. Include table of contents 5 | .. include:: toc.rst 6 | 7 | .. figure:: img/cosmostat_logo.jpg 8 | :figclass: margin 9 | :width: 100% 10 | :alt: CosmoStat Logo 11 | :target: http://www.cosmostat.org/ 12 | 13 | ShapePipe is a galaxy shape measurement pipeline developed within the 14 | |link-to-cosmostat| lab at CEA Paris-Saclay. 15 | 16 | This documentation aims to provide all the information needed for installing 17 | and running ShapePipe. If you unable to find what you are looking for here, 18 | we invite you to |link-to-issues| on the GitHub repository an we will do our 19 | best to help you. 20 | 21 | We kindly request that if you use ShapePipe for your academic work that you 22 | cite :cite:t:`guinot:22` and :cite:t:`farrens:22b`. 23 | 24 | .. |link-to-cosmostat| raw:: html 25 | 26 | CosmoStat 27 | 28 | .. |link-to-issues| raw:: html 29 | 30 | open an issue 32 | -------------------------------------------------------------------------------- /docs/source/testing.md: -------------------------------------------------------------------------------- 1 | # Testing 2 | 3 | After reading through the [basic execution](basic_execution.md) and 4 | [configuration](configuration.md) instructions it may be helpful to run a few 5 | tests. 6 | 7 | ShapePipe includes some dummy modules for testing purposes 8 | 9 | - `python_example_runner` : an example of a module written entirely in Python 10 | - `serial_example_runner` : an example of a module that is run in serial mode 11 | - `execute_example_runner` : an example of a module that calls a system 12 | executable 13 | 14 | None of these modules do anything particularly interesting, but they can be run 15 | to test that ShapePipe is up and running. 16 | 17 | The [example](https://github.com/CosmoStat/shapepipe/tree/develop/example) 18 | directory contains an example config file called `config.ini` that runs all of 19 | these dummy modules. ShapePipe can be run with this config file as follows 20 | 21 | ```bash 22 | shapepipe_run -c ./example/config.ini 23 | ``` 24 | 25 | The output of this run will be saved to `./example/output`. If you want a 26 | more concrete understanding of what each of the configuration options does you 27 | can modify `config.ini` and see what happens. 28 | -------------------------------------------------------------------------------- /docs/source/toc.rst: -------------------------------------------------------------------------------- 1 | .. Toctrees define sidebar contents 2 | 3 | .. toctree:: 4 | :hidden: 5 | :titlesonly: 6 | :caption: About ShapePipe 7 | 8 | about 9 | contributors 10 | 11 | .. toctree:: 12 | :hidden: 13 | :titlesonly: 14 | :caption: Installing ShapePipe 15 | 16 | dependencies 17 | installation 18 | 19 | .. toctree:: 20 | :hidden: 21 | :titlesonly: 22 | :caption: Running ShapePipe 23 | 24 | basic_execution 25 | configuration 26 | testing 27 | pipeline_tutorial 28 | 29 | .. toctree:: 30 | :hidden: 31 | :titlesonly: 32 | :caption: Miscellaneous 33 | 34 | post_processing 35 | random_cat 36 | 37 | .. toctree:: 38 | :hidden: 39 | :titlesonly: 40 | :caption: API Documentation 41 | 42 | understanding_api 43 | shapepipe 44 | 45 | .. toctree:: 46 | :hidden: 47 | :titlesonly: 48 | :caption: Contributing to ShapePipe 49 | 50 | contributing 51 | module_develop 52 | module_example 53 | 54 | .. toctree:: 55 | :hidden: 56 | :titlesonly: 57 | :caption: References 58 | 59 | z_ref 60 | -------------------------------------------------------------------------------- /docs/source/z_ref.rst: -------------------------------------------------------------------------------- 1 | References 2 | ========== 3 | 4 | .. bibliography:: refs.bib 5 | -------------------------------------------------------------------------------- /environment-dev.yml: -------------------------------------------------------------------------------- 1 | # ShapePipe Development Environment 2 | # --------------------------------- 3 | # This environment should only be used for development and testing of 4 | # ShapePipe (i.e. nothing that could be published). Packages need only 5 | # specify the minimum compatible versions. This environment can updated 6 | # as needed. 7 | name: shapepipe-dev 8 | channels: 9 | - conda-forge 10 | dependencies: 11 | - python>=3.9.18 12 | - pip>=23.3.1 13 | - astropy>=5.1.1 14 | - autoconf>=2.71 15 | - automake>=1.16.5 16 | - cmake>=3.27.7 17 | - galsim>=2.5.1 18 | - joblib>=1.3.2 19 | - libtool>=2.4 20 | - matplotlib>=3.8.1 21 | - numba>=0.58.1 22 | - pandas>=2.1 23 | - pyqt5-sip>=12.13 24 | - pyqtgraph>=0.13.3 25 | - reproject>=0.12 26 | - sqlitedict>=2.1 27 | - termcolor>=2.3 28 | - tqdm>=4.66.1 29 | - treecorr>=4.3.3 30 | - pip: 31 | - cs_util==0.0.5 32 | - mccd==1.2.4 33 | - modopt==1.6.1 34 | - sip_tpv==1.1 35 | - sf_tools==2.0.4 36 | - git+https://github.com/CEA-COSMIC/pysap@develop 37 | - git+https://github.com/aguinot/ngmix@stable_version 38 | - git+https://github.com/tobias-liaudat/Stile@v0.1 39 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: shapepipe_v1.4 2 | # ShapePipe Production Environment 3 | # -------------------------------- 4 | # This environment should be used for any production runs of ShapePipe 5 | # (i.e. anything that could be published). All packages are pinned to 6 | # specific versions to maximise the reproducibility of the results. 7 | # This environment should be updated for each release of ShapePipe. 8 | channels: 9 | - conda-forge 10 | - defaults 11 | dependencies: 12 | - python=3.9 13 | - pip>=21.2.4 14 | - astromatic-psfex==3.21.1 15 | - astromatic-source-extractor==2.25.0 16 | - astropy==5.2 17 | - automake 18 | - autoconf 19 | - cmake 20 | - h5py 21 | - galsim 22 | - joblib 23 | - libtool 24 | - matplotlib==3.8.4 25 | - numba==0.58.1 26 | - numpy==1.26.4 27 | - pandas 28 | - pip: 29 | - cs_util==0.1 30 | - mccd==1.2.4 31 | - modopt 32 | - PyQt5 33 | - pyqtgraph 34 | - python-pysap==0.2.1 35 | - reproject 36 | - shear_psf_leakage 37 | - skaha 38 | - sip_tpv 39 | - sf_tools 40 | - sqlitedict 41 | - termcolor 42 | - tqdm 43 | - treecorr 44 | - vos 45 | - git+https://github.com/aguinot/ngmix@stable_version 46 | - git+https://github.com/tobias-liaudat/Stile@v0.1 47 | -------------------------------------------------------------------------------- /example/cfis/config_Fe.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe configuration file for: find exposures 2 | 3 | 4 | ## Default ShapePipe options 5 | [DEFAULT] 6 | 7 | # verbose mode (optional), default: True, print messages on terminal 8 | VERBOSE = False 9 | 10 | # Name of run (optional) default: shapepipe_run 11 | RUN_NAME = run_sp_Fe 12 | 13 | # Add date and time to RUN_NAME, optional, default: False 14 | RUN_DATETIME = True 15 | 16 | 17 | ## ShapePipe execution options 18 | [EXECUTION] 19 | 20 | # Module name, single string or comma-separated list of valid module runner names 21 | MODULE = find_exposures_runner 22 | 23 | # Parallel processing mode, SMP or MPI 24 | MODE = SMP 25 | 26 | 27 | ## ShapePipe file handling options 28 | [FILE] 29 | 30 | # Log file master name, optional, default: shapepipe 31 | LOG_NAME = log_sp 32 | 33 | # Runner log file name, optional, default: shapepipe_runs 34 | RUN_LOG_NAME = log_run_sp 35 | 36 | # Input directory, containing input files, single string or list of names 37 | INPUT_DIR = $SP_RUN 38 | 39 | # Output directory 40 | OUTPUT_DIR = $SP_RUN/output 41 | 42 | 43 | ## ShapePipe job handling options 44 | [JOB] 45 | 46 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 47 | SMP_BATCH_SIZE = 1 48 | 49 | # Timeout value (optional), default is None, i.e. no timeout limit applied 50 | TIMEOUT = 96:00:00 51 | 52 | 53 | ## Module options 54 | 55 | # Get tiles 56 | [FIND_EXPOSURES_RUNNER] 57 | 58 | INPUT_DIR = run_sp_Git:get_images_runner_run_1 59 | 60 | FILE_PATTERN = CFIS_image 61 | 62 | FILE_EXT = .fits 63 | 64 | # NUMBERING_SCHEME (optional) string with numbering pattern for input files 65 | NUMBERING_SCHEME = -000-000 66 | 67 | # Column number of exposure name in FITS header 68 | COLNUM = 3 69 | 70 | # Prefix to remove from exposure name 71 | EXP_PREFIX = p 72 | 73 | -------------------------------------------------------------------------------- /example/cfis/config_Gie_symlink.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe configuration file for: get images 2 | 3 | 4 | ## Default ShapePipe options 5 | [DEFAULT] 6 | 7 | # verbose mode (optional), default: True, print messages on terminal 8 | VERBOSE = False 9 | 10 | # Name of run (optional) default: shapepipe_run 11 | RUN_NAME = run_sp_Gie 12 | 13 | # Add date and time to RUN_NAME, optional, default: False 14 | RUN_DATETIME = True 15 | 16 | 17 | ## ShapePipe execution options 18 | [EXECUTION] 19 | 20 | # Module name, single string or comma-separated list of valid module runner names 21 | MODULE = get_images_runner 22 | 23 | # Parallel processing mode, SMP or MPI 24 | MODE = SMP 25 | 26 | 27 | ## ShapePipe file handling options 28 | [FILE] 29 | 30 | # Log file master name, optional, default: shapepipe 31 | LOG_NAME = log_sp 32 | 33 | # Runner log file name, optional, default: shapepipe_runs 34 | RUN_LOG_NAME = log_run_sp 35 | 36 | # Input directory, containing input files, single string or list of names 37 | INPUT_DIR = $SP_RUN 38 | 39 | # Output directory 40 | OUTPUT_DIR = $SP_RUN/output 41 | 42 | 43 | ## ShapePipe job handling options 44 | [JOB] 45 | 46 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 47 | SMP_BATCH_SIZE = 1 48 | 49 | # Timeout value (optional), default is None, i.e. no timeout limit applied 50 | TIMEOUT = 96:00:00 51 | 52 | 53 | ## Module options 54 | 55 | # Get exposures 56 | [GET_IMAGES_RUNNER] 57 | 58 | INPUT_DIR = last:find_exposures_runner 59 | 60 | FILE_PATTERN = exp_numbers 61 | 62 | FILE_EXT = .txt 63 | 64 | # NUMBERING_SCHEME (optional) string with numbering pattern for input files 65 | NUMBERING_SCHEME = -000-000 66 | 67 | 68 | # Paths 69 | 70 | # Output path (optional, default is [FILE]:OUTPUT_DIR 71 | # OUTPUT_PATH = input_images 72 | 73 | # Input path where original images are stored. Can be local path or vos url. 74 | # Single string or list of strings 75 | INPUT_PATH = $SP_RUN/data_exp, $SP_RUN/data_exp, $SP_RUN/data_exp 76 | 77 | # Input file pattern including tile number as dummy template 78 | INPUT_FILE_PATTERN = 000000, 000000.weight, 000000.flag 79 | 80 | # Input file extensions 81 | INPUT_FILE_EXT = .fits.fz, .fits.fz, .fits.fz 82 | 83 | # Input numbering scheme, python regexp 84 | INPUT_NUMBERING = \d{6} 85 | 86 | # Output file pattern without number 87 | OUTPUT_FILE_PATTERN = image-, weight-, flag- 88 | 89 | # Method to retrieve images, one in 'vos', 'symlink' 90 | RETRIEVE = symlink 91 | 92 | # If RETRIEVE=vos, number of attempts to download 93 | # Optional, default=3 94 | N_TRY = 3 95 | 96 | # Retrieve command options, optional 97 | RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem 98 | -------------------------------------------------------------------------------- /example/cfis/config_Git_vos.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe configuration file for: get images 2 | 3 | 4 | ## Default ShapePipe options 5 | [DEFAULT] 6 | 7 | # verbose mode (optional), default: True, print messages on terminal 8 | VERBOSE = False 9 | 10 | # Name of run (optional) default: shapepipe_run 11 | RUN_NAME = run_sp_Git 12 | 13 | # Add date and time to RUN_NAME, optional, default: False 14 | RUN_DATETIME = True 15 | 16 | 17 | ## ShapePipe execution options 18 | [EXECUTION] 19 | 20 | # Module name, single string or comma-separated list of valid module runner names 21 | MODULE = get_images_runner 22 | 23 | # Parallel processing mode, SMP or MPI 24 | MODE = SMP 25 | 26 | 27 | ## ShapePipe file handling options 28 | [FILE] 29 | 30 | # Log file master name, optional, default: shapepipe 31 | LOG_NAME = log_sp 32 | 33 | # Runner log file name, optional, default: shapepipe_runs 34 | RUN_LOG_NAME = log_run_sp 35 | 36 | # Input directory, containing input files, single string or list of names 37 | INPUT_DIR = $SP_RUN 38 | 39 | # Output directory 40 | OUTPUT_DIR = $SP_RUN/output 41 | 42 | 43 | ## ShapePipe job handling options 44 | [JOB] 45 | 46 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 47 | SMP_BATCH_SIZE = 1 48 | 49 | # Timeout value (optional), default is None, i.e. no timeout limit applied 50 | TIMEOUT = 96:00:00 51 | 52 | 53 | ## Module options 54 | 55 | # Get tiles 56 | [GET_IMAGES_RUNNER] 57 | 58 | FILE_PATTERN = tile_numbers 59 | 60 | FILE_EXT = .txt 61 | 62 | # NUMBERING_SCHEME (optional) string with numbering pattern for input files 63 | NUMBERING_SCHEME = 64 | 65 | # Paths 66 | 67 | # Input path where original images are stored. Can be local path or vos url. 68 | # Single string or list of strings 69 | INPUT_PATH = vos:cfis/tiles_DR5, vos:cfis/tiles_DR5 70 | 71 | # Input file pattern including tile number as dummy template 72 | INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight 73 | 74 | # Input file extensions 75 | INPUT_FILE_EXT = .fits, .fits.fz 76 | 77 | # Input numbering scheme, python regexp 78 | INPUT_NUMBERING = \d{3}\.\d{3} 79 | 80 | # Output file pattern without number 81 | OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- 82 | 83 | # Copy/download method, one in 'vos', 'symlink' 84 | RETRIEVE = vos 85 | 86 | # If RETRIEVE=vos, number of attempts to download 87 | # Optional, default=3 88 | N_TRY = 3 89 | 90 | # Copy command options, optional 91 | RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem 92 | 93 | CHECK_EXISTING_DIR = $SP_RUN/data_tiles 94 | -------------------------------------------------------------------------------- /example/cfis/config_Ms_psfex.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe configuration file for post-processing. 2 | # merge star cat. 3 | 4 | 5 | ## Default ShapePipe options 6 | [DEFAULT] 7 | 8 | # verbose mode (optional), default: True, print messages on terminal 9 | VERBOSE = True 10 | 11 | # Name of run (optional) default: shapepipe_run 12 | RUN_NAME = run_sp_Ms 13 | 14 | # Add date and time to RUN_NAME, optional, default: False 15 | RUN_DATETIME = False 16 | 17 | 18 | ## ShapePipe execution options 19 | [EXECUTION] 20 | 21 | # Module name, single string or comma-separated list of valid module runner names 22 | MODULE = merge_starcat_runner 23 | 24 | # Parallel processing mode, SMP or MPI 25 | MODE = SMP 26 | 27 | 28 | ## ShapePipe file handling options 29 | [FILE] 30 | 31 | # Log file master name, optional, default: shapepipe 32 | LOG_NAME = log_sp 33 | 34 | # Runner log file name, optional, default: shapepipe_runs 35 | RUN_LOG_NAME = log_run_sp 36 | 37 | # Input directory, containing input files, single string or list of names 38 | INPUT_DIR = $SP_RUN/output 39 | 40 | # Output directory 41 | OUTPUT_DIR = $SP_RUN/output 42 | 43 | 44 | ## ShapePipe job handling options 45 | [JOB] 46 | 47 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 48 | SMP_BATCH_SIZE = 4 49 | 50 | # Timeout value (optional), default is None, i.e. no timeout limit applied 51 | TIMEOUT = 96:00:00 52 | 53 | 54 | ## Module options 55 | [MERGE_STARCAT_RUNNER] 56 | 57 | INPUT_DIR = last:psfex_interp_runner 58 | 59 | PSF_MODEL = psfex 60 | 61 | NUMBERING_SCHEME = -0000000-0 62 | 63 | # Input file pattern(s), list of strings with length matching number of expected input file types 64 | # Cannot contain wild cards 65 | FILE_PATTERN = validation_psf 66 | 67 | # FILE_EXT (optional) list of string extensions to identify input files 68 | FILE_EXT = .fits 69 | 70 | HDU = 2 71 | -------------------------------------------------------------------------------- /example/cfis/config_Ms_psfex_conv.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe configuration file for post-processing. 2 | # merge star cat (converted to WCS) 3 | 4 | 5 | ## Default ShapePipe options 6 | [DEFAULT] 7 | 8 | # verbose mode (optional), default: True, print messages on terminal 9 | VERBOSE = True 10 | 11 | # Name of run (optional) default: shapepipe_run 12 | RUN_NAME = run_sp_Ms 13 | 14 | # Add date and time to RUN_NAME, optional, default: False 15 | RUN_DATETIME = False 16 | 17 | 18 | ## ShapePipe execution options 19 | [EXECUTION] 20 | 21 | # Module name, single string or comma-separated list of valid module runner names 22 | MODULE = merge_starcat_runner 23 | 24 | # Parallel processing mode, SMP or MPI 25 | MODE = SMP 26 | 27 | 28 | ## ShapePipe file handling options 29 | [FILE] 30 | 31 | # Log file master name, optional, default: shapepipe 32 | LOG_NAME = log_sp 33 | 34 | # Runner log file name, optional, default: shapepipe_runs 35 | RUN_LOG_NAME = log_run_sp 36 | 37 | # Input directory, containing input files, single string or list of names 38 | INPUT_DIR = $SP_RUN/output 39 | 40 | # Output directory 41 | OUTPUT_DIR = $SP_RUN/output 42 | 43 | 44 | ## ShapePipe job handling options 45 | [JOB] 46 | 47 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 48 | SMP_BATCH_SIZE = 4 49 | 50 | # Timeout value (optional), default is None, i.e. no timeout limit applied 51 | TIMEOUT = 96:00:00 52 | 53 | 54 | ## Module options 55 | [MERGE_STARCAT_RUNNER] 56 | 57 | INPUT_DIR = last:psfex_interp_runner 58 | 59 | PSF_MODEL = psfex 60 | 61 | NUMBERING_SCHEME = -0-0 62 | 63 | # Input file pattern(s), list of strings with length matching number of expected input file types 64 | # Cannot contain wild cards 65 | FILE_PATTERN = validation_psf_conv 66 | 67 | # Additional header key 68 | INPUT_CAT_TYPE = validation_psf_conf 69 | 70 | # FILE_EXT (optional) list of string extensions to identify input files 71 | FILE_EXT = .fits 72 | -------------------------------------------------------------------------------- /example/cfis/config_Pl_mccd.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe configuration file for post-processing. 2 | # merge star cat and PSF plots. 3 | 4 | 5 | ## Default ShapePipe options 6 | [DEFAULT] 7 | 8 | # verbose mode (optional), default: True, print messages on terminal 9 | VERBOSE = True 10 | 11 | # Name of run (optional) default: shapepipe_run 12 | RUN_NAME = run_sp_Pl 13 | 14 | # Add date and time to RUN_NAME, optional, default: False 15 | RUN_DATETIME = True 16 | 17 | 18 | ## ShapePipe execution options 19 | [EXECUTION] 20 | 21 | # Module name, single string or comma-separated list of valid module runner names 22 | MODULE = mccd_plots_runner 23 | 24 | # Parallel processing mode, SMP or MPI 25 | MODE = SMP 26 | 27 | 28 | ## ShapePipe file handling options 29 | [FILE] 30 | 31 | # Log file master name, optional, default: shapepipe 32 | LOG_NAME = log_sp 33 | 34 | # Runner log file name, optional, default: shapepipe_runs 35 | RUN_LOG_NAME = log_run_sp 36 | 37 | # Input directory, containing input files, single string or list of names 38 | INPUT_DIR = $SP_RUN/output 39 | 40 | # Output directory 41 | OUTPUT_DIR = $SP_RUN/output 42 | 43 | 44 | ## ShapePipe job handling options 45 | [JOB] 46 | 47 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 48 | SMP_BATCH_SIZE = 4 49 | 50 | # Timeout value (optional), default is None, i.e. no timeout limit applied 51 | TIMEOUT = 96:00:00 52 | 53 | 54 | ## Module options 55 | [MCCD_PLOTS_RUNNER] 56 | 57 | # NUMBERING_SCHEME (optional) string with numbering pattern for input files 58 | NUMBERING_SCHEME = -0000000 59 | 60 | FILE_PATTERN = full_starcat 61 | FILE_EXT = .fits 62 | 63 | PSF = mccd 64 | PLOT_MEANSHAPES = True 65 | X_GRID = 5 66 | Y_GRID = 10 67 | 68 | # Optional: max values for focal plan plots (meanshape) 69 | MAX_E = 0.1 70 | MAX_DE = 0.01 71 | MIN_R2 = 4.5 72 | MAX_R2 = 7 73 | MAX_DR2 = 0.03 74 | 75 | PLOT_HISTOGRAMS = True 76 | REMOVE_OUTLIERS = False 77 | 78 | # X_GRID, Y_GRID: correspond to the number of bins in each direction of each 79 | # CCD from the focal plane. Ex: each CCD will be binned in 5x10 regular grids. 80 | # 81 | # REMOVE_OUTLIERS: Remove validated stars that are outliers in terms of shape 82 | # before drawing the plots. 83 | 84 | PLOT_RHO_STATS = True 85 | RHO_STATS_STYLE = HSC 86 | 87 | RHO_STATS_YLIM_L = 1e-10, 4e-5 88 | RHO_STATS_YLIM_R = 1e-8, 4e-5 89 | -------------------------------------------------------------------------------- /example/cfis/config_Pl_psfex.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe configuration file for post-processing. 2 | # merge star cat and PSF plots. 3 | 4 | 5 | ## Default ShapePipe options 6 | [DEFAULT] 7 | 8 | # verbose mode (optional), default: True, print messages on terminal 9 | VERBOSE = True 10 | 11 | # Name of run (optional) default: shapepipe_run 12 | RUN_NAME = run_sp_Pl 13 | 14 | # Add date and time to RUN_NAME, optional, default: False 15 | RUN_DATETIME = False 16 | 17 | 18 | ## ShapePipe execution options 19 | [EXECUTION] 20 | 21 | # Module name, single string or comma-separated list of valid module runner names 22 | MODULE = mccd_plots_runner 23 | 24 | # Parallel processing mode, SMP or MPI 25 | MODE = SMP 26 | 27 | 28 | ## ShapePipe file handling options 29 | [FILE] 30 | 31 | # Log file master name, optional, default: shapepipe 32 | LOG_NAME = log_sp 33 | 34 | # Runner log file name, optional, default: shapepipe_runs 35 | RUN_LOG_NAME = log_run_sp 36 | 37 | # Input directory, containing input files, single string or list of names 38 | INPUT_DIR = $SP_RUN/output 39 | 40 | # Output directory 41 | OUTPUT_DIR = $SP_RUN/output 42 | 43 | 44 | ## ShapePipe job handling options 45 | [JOB] 46 | 47 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 48 | SMP_BATCH_SIZE = 4 49 | 50 | # Timeout value (optional), default is None, i.e. no timeout limit applied 51 | TIMEOUT = 96:00:00 52 | 53 | 54 | ## Module options 55 | [MCCD_PLOTS_RUNNER] 56 | 57 | # NUMBERING_SCHEME (optional) string with numbering pattern for input files 58 | NUMBERING_SCHEME = -0000000 59 | 60 | #FILE_PATTERN = test 61 | FILE_PATTERN = full_starcat 62 | FILE_EXT = .fits 63 | 64 | HDU = 1 65 | 66 | PSF = psfex 67 | 68 | # was True 69 | PLOT_MEANSHAPES = False 70 | X_GRID = 5 71 | Y_GRID = 10 72 | 73 | # Optional: max values for elliptity and residual ellipticities 74 | MAX_E = 0.1 75 | MAX_DE = 0.01 76 | MIN_R2 = 4.5 77 | MAX_R2 = 7 78 | MAX_DR2 = 0.03 79 | 80 | # was True 81 | PLOT_HISTOGRAMS = False 82 | REMOVE_OUTLIERS = False 83 | 84 | # X_GRID, Y_GRID: correspond to the number of bins in each direction of each 85 | # CCD from the focal plane. Ex: each CCD will be binned in 5x10 regular grids. 86 | # 87 | # REMOVE_OUTLIERS: Remove validated stars that are outliers in terms of shape 88 | # before drawing the plots. 89 | 90 | PLOT_RHO_STATS = True 91 | RHO_STATS_STYLE = UNIONS 92 | 93 | RHO_STATS_YLIM_L = 1e-10, 4e-5 94 | RHO_STATS_YLIM_R = 1e-8, 4e-5 95 | -------------------------------------------------------------------------------- /example/cfis/config_Pl_psfex_conv.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe configuration file for post-processing. 2 | # merge star cat and PSF plots. 3 | 4 | 5 | ## Default ShapePipe options 6 | [DEFAULT] 7 | 8 | # verbose mode (optional), default: True, print messages on terminal 9 | VERBOSE = True 10 | 11 | # Name of run (optional) default: shapepipe_run 12 | RUN_NAME = run_sp_Pl 13 | 14 | # Add date and time to RUN_NAME, optional, default: False 15 | RUN_DATETIME = False 16 | 17 | 18 | ## ShapePipe execution options 19 | [EXECUTION] 20 | 21 | # Module name, single string or comma-separated list of valid module runner names 22 | MODULE = mccd_plots_runner 23 | 24 | # Parallel processing mode, SMP or MPI 25 | MODE = SMP 26 | 27 | 28 | ## ShapePipe file handling options 29 | [FILE] 30 | 31 | # Log file master name, optional, default: shapepipe 32 | LOG_NAME = log_sp 33 | 34 | # Runner log file name, optional, default: shapepipe_runs 35 | RUN_LOG_NAME = log_run_sp 36 | 37 | # Input directory, containing input files, single string or list of names 38 | INPUT_DIR = $SP_RUN/output 39 | 40 | # Output directory 41 | OUTPUT_DIR = $SP_RUN/output 42 | 43 | 44 | ## ShapePipe job handling options 45 | [JOB] 46 | 47 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 48 | SMP_BATCH_SIZE = 4 49 | 50 | # Timeout value (optional), default is None, i.e. no timeout limit applied 51 | TIMEOUT = 96:00:00 52 | 53 | 54 | ## Module options 55 | [MCCD_PLOTS_RUNNER] 56 | 57 | # NUMBERING_SCHEME (optional) string with numbering pattern for input files 58 | NUMBERING_SCHEME = -0000000 59 | 60 | FILE_PATTERN = full_starcat 61 | FILE_EXT = .fits 62 | 63 | HDU = 1 64 | 65 | PSF = psfex 66 | 67 | # was True 68 | PLOT_MEANSHAPES = False 69 | X_GRID = 5 70 | Y_GRID = 10 71 | 72 | # Optional: max values for elliptity and residual ellipticities 73 | MAX_E = 0.1 74 | MAX_DE = 0.01 75 | MIN_R2 = 4.5 76 | MAX_R2 = 7 77 | MAX_DR2 = 0.03 78 | 79 | # was True 80 | PLOT_HISTOGRAMS = False 81 | REMOVE_OUTLIERS = False 82 | 83 | # X_GRID, Y_GRID: correspond to the number of bins in each direction of each 84 | # CCD from the focal plane. Ex: each CCD will be binned in 5x10 regular grids. 85 | # 86 | # REMOVE_OUTLIERS: Remove validated stars that are outliers in terms of shape 87 | # before drawing the plots. 88 | 89 | PLOT_RHO_STATS = True 90 | RHO_STATS_STYLE = UNIONS 91 | 92 | RHO_STATS_YLIM_L = 1e-10, 4e-5 93 | RHO_STATS_YLIM_R = 1e-8, 4e-5 94 | -------------------------------------------------------------------------------- /example/cfis/config_Rc.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe configuration file for: create random catalogue 2 | 3 | 4 | ## Default ShapePipe options 5 | [DEFAULT] 6 | 7 | # verbose mode (optional), default: True, print messages on terminal 8 | VERBOSE = True 9 | 10 | # Name of run (optional) default: shapepipe_run 11 | RUN_NAME = run_sp_Rc 12 | 13 | # Add date and time to RUN_NAME, optional, default: False 14 | RUN_DATETIME = False 15 | 16 | 17 | ## ShapePipe execution options 18 | [EXECUTION] 19 | 20 | # Module name, single string or comma-separated list of valid module runner names 21 | MODULE = random_cat_runner 22 | 23 | # Parallel processing mode, SMP or MPI 24 | MODE = SMP 25 | 26 | 27 | ## ShapePipe file handling options 28 | [FILE] 29 | 30 | # Log file master name, optional, default: shapepipe 31 | LOG_NAME = log_sp 32 | 33 | # Runner log file name, optional, default: shapepipe_runs 34 | RUN_LOG_NAME = log_run_sp 35 | 36 | # Input directory, containing input files, single string or list of names 37 | INPUT_DIR = . 38 | 39 | # Output directory 40 | OUTPUT_DIR = $SP_RUN/output 41 | 42 | 43 | ## ShapePipe job handling options 44 | [JOB] 45 | 46 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 47 | SMP_BATCH_SIZE = 24 48 | 49 | # Timeout value (optional), default is None, i.e. no timeout limit applied 50 | TIMEOUT = 96:00:00 51 | 52 | 53 | ## Module options 54 | [RANDOM_CAT_RUNNER] 55 | 56 | #INPUT_DIR = last:get_images_runner, last:mask_runner 57 | INPUT_DIR = last:get_images_runner, $SP_RUN/output/run_sp_combined_flag:mask_runner 58 | 59 | FILE_PATTERN = CFIS_image, pipeline_flag 60 | 61 | NUMBERING_SCHEME = 000-000 62 | 63 | # Number of random objects 64 | N_RANDOM = 50000 65 | 66 | # N_RANDOM is per square degrees if True 67 | DENSITY = True 68 | 69 | # Output healpix mask if True 70 | SAVE_MASK_AS_HEALPIX = True 71 | 72 | # Healpix mask file base name (used if SAVE_MASK_AS_HEALPIX is True) 73 | HEALPIX_OUT_FILE_BASE = mask_hp 74 | 75 | # Healpix mask nside (used if SAVE_MASK_AS_HEALPIX is True) 76 | HEALPIX_OUT_NSIDE = 1024 77 | -------------------------------------------------------------------------------- /example/cfis/config_exp_Ma_onthefly.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe configuration file for masking of exposures 2 | 3 | 4 | ## Default ShapePipe options 5 | [DEFAULT] 6 | 7 | # verbose mode (optional), default: True, print messages on terminal 8 | VERBOSE = True 9 | 10 | # Name of run (optional) default: shapepipe_run 11 | RUN_NAME = run_sp_exp_Ma 12 | 13 | # Add date and time to RUN_NAME, optional, default: False 14 | ; RUN_DATETIME = False 15 | 16 | 17 | ## ShapePipe execution options 18 | [EXECUTION] 19 | 20 | # Module name, single string or comma-separated list of valid module runner names 21 | MODULE = mask_runner 22 | 23 | # Parallel processing mode, SMP or MPI 24 | MODE = SMP 25 | 26 | 27 | ## ShapePipe file handling options 28 | [FILE] 29 | 30 | # Log file master name, optional, default: shapepipe 31 | LOG_NAME = log_sp 32 | 33 | # Runner log file name, optional, default: shapepipe_runs 34 | RUN_LOG_NAME = log_run_sp 35 | 36 | # Input directory, containing input files, single string or list of names 37 | INPUT_DIR = . 38 | 39 | # Output directory 40 | OUTPUT_DIR = $SP_RUN/output 41 | 42 | 43 | ## ShapePipe job handling options 44 | [JOB] 45 | 46 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 47 | SMP_BATCH_SIZE = 16 48 | 49 | # Timeout value (optional), default is None, i.e. no timeout limit applied 50 | TIMEOUT = 96:00:00 51 | 52 | 53 | ## Module options 54 | 55 | ### Mask exposures 56 | [MASK_RUNNER] 57 | 58 | # Parent module 59 | INPUT_DIR = last:split_exp_runner 60 | 61 | # Update numbering convention, accounting for HDU number of 62 | # single-exposure single-HDU files 63 | NUMBERING_SCHEME = -0000000-0 64 | 65 | # Path of mask config file 66 | MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask 67 | 68 | # External mask file flag, use if True, otherwise ignore 69 | USE_EXT_FLAG = True 70 | 71 | # External star catalogue flag, use external cat if True, 72 | # obtain from online catalogue if False 73 | USE_EXT_STAR = False 74 | 75 | # File name suffix for the output flag files (optional) 76 | PREFIX = pipeline 77 | 78 | # Path to check for existing output mask files 79 | CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_Ma_exp/mask_runner/output 80 | -------------------------------------------------------------------------------- /example/cfis/config_exp_Mh.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe configuration file for single-exposures, 2 | # merge headers 3 | 4 | 5 | ## Default ShapePipe options 6 | [DEFAULT] 7 | 8 | # verbose mode (optional), default: True, print messages on terminal 9 | VERBOSE = True 10 | 11 | # Name of run (optional) default: shapepipe_run 12 | RUN_NAME = run_sp_exp_Mh 13 | 14 | # Add date and time to RUN_NAME, optional, default: True 15 | ; RUN_DATETIME = False 16 | 17 | 18 | ## ShapePipe execution options 19 | [EXECUTION] 20 | 21 | # Module name, single string or comma-separated list of valid module runner names 22 | MODULE = merge_headers_runner 23 | 24 | # Run mode, SMP or MPI 25 | MODE = SMP 26 | 27 | 28 | ## ShapePipe file handling options 29 | [FILE] 30 | 31 | # Log file master name, optional, default: shapepipe 32 | LOG_NAME = log_sp 33 | 34 | # Runner log file name, optional, default: shapepipe_runs 35 | RUN_LOG_NAME = log_run_sp 36 | 37 | # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN 38 | INPUT_DIR = . 39 | 40 | # Output directory 41 | OUTPUT_DIR = $SP_RUN/output 42 | 43 | 44 | ## ShapePipe job handling options 45 | [JOB] 46 | 47 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 48 | SMP_BATCH_SIZE = 16 49 | 50 | # Timeout value (optional), default is None, i.e. no timeout limit applied 51 | TIMEOUT = 96:00:00 52 | 53 | 54 | ## Module options 55 | 56 | [MERGE_HEADERS_RUNNER] 57 | 58 | INPUT_DIR = last:split_exp_runner 59 | 60 | FILE_PATTERN = headers 61 | 62 | FILE_EXT = .npy 63 | 64 | # Single-exposure numbering scheme 65 | NUMBERING_SCHEME = -0000000 66 | 67 | OUTPUT_PATH = $SP_RUN/output 68 | -------------------------------------------------------------------------------- /example/cfis/config_exp_Pi.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe configuration file for single-HDU single exposure images.. 2 | # PSFex PSF model; interpolation for validation. 3 | 4 | 5 | ## Default ShapePipe options 6 | [DEFAULT] 7 | 8 | # verbose mode (optional), default: True, print messages on terminal 9 | VERBOSE = True 10 | 11 | # Name of run (optional) default: shapepipe_run 12 | RUN_NAME = run_sp_exp_Pi 13 | 14 | # Add date and time to RUN_NAME, optional, default: True 15 | ; RUN_DATETIME = False 16 | 17 | 18 | ## ShapePipe execution options 19 | [EXECUTION] 20 | 21 | # Module name, single string or comma-separated list of valid module runner names 22 | MODULE = psfex_interp_runner 23 | 24 | 25 | # Run mode, SMP or MPI 26 | MODE = SMP 27 | 28 | 29 | ## ShapePipe file handling options 30 | [FILE] 31 | 32 | # Log file master name, optional, default: shapepipe 33 | LOG_NAME = log_sp 34 | 35 | # Runner log file name, optional, default: shapepipe_runs 36 | RUN_LOG_NAME = log_run_sp 37 | 38 | # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN 39 | INPUT_DIR = . 40 | 41 | # Output directory 42 | OUTPUT_DIR = $SP_RUN/output 43 | 44 | 45 | ## ShapePipe job handling options 46 | [JOB] 47 | 48 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 49 | SMP_BATCH_SIZE = 1 50 | 51 | # Timeout value (optional), default is None, i.e. no timeout limit applied 52 | TIMEOUT = 96:00:00 53 | 54 | 55 | ## Module options 56 | 57 | [PSFEX_INTERP_RUNNER] 58 | 59 | # Use 20% sample for PSF validation 60 | FILE_PATTERN = star_split_ratio_80, star_split_ratio_20, psfex_cat 61 | 62 | FILE_EXT = .psf, .fits, .cat 63 | 64 | NUMBERING_SCHEME = -0000000-0 65 | 66 | # Run mode for psfex interpolation: 67 | # CLASSIC: 'classical' run, interpolate to object positions 68 | # MULTI-EPOCH: interpolate for multi-epoch images 69 | # VALIDATION: validation for single-epoch images 70 | MODE = VALIDATION 71 | 72 | # Column names of position parameters 73 | POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE 74 | 75 | # If True, measure and store ellipticity of the PSF (using moments) 76 | GET_SHAPES = True 77 | 78 | # Minimum number of stars per CCD for PSF model to be computed 79 | STAR_THRESH = 22 80 | 81 | # Maximum chi^2 for PSF model to be computed on CCD 82 | CHI2_THRESH = 2 83 | -------------------------------------------------------------------------------- /example/cfis/config_exp_Sp.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe configuration file for single-exposures, 2 | # split images, merge headers 3 | 4 | 5 | ## Default ShapePipe options 6 | [DEFAULT] 7 | 8 | # verbose mode (optional), default: True, print messages on terminal 9 | VERBOSE = True 10 | 11 | # Name of run (optional) default: shapepipe_run 12 | RUN_NAME = run_sp_exp_Sp 13 | 14 | # Add date and time to RUN_NAME, optional, default: True 15 | RUN_DATETIME = False 16 | 17 | 18 | ## ShapePipe execution options 19 | [EXECUTION] 20 | 21 | # Module name, single string or comma-separated list of valid module runner names 22 | MODULE = split_exp_runner 23 | 24 | # Run mode, SMP or MPI 25 | MODE = SMP 26 | 27 | 28 | ## ShapePipe file handling options 29 | [FILE] 30 | 31 | # Log file master name, optional, default: shapepipe 32 | LOG_NAME = log_sp 33 | 34 | # Runner log file name, optional, default: shapepipe_runs 35 | RUN_LOG_NAME = log_run_sp 36 | 37 | # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN 38 | INPUT_DIR = . 39 | 40 | # Output directory 41 | OUTPUT_DIR = $SP_RUN/output 42 | 43 | 44 | ## ShapePipe job handling options 45 | [JOB] 46 | 47 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 48 | SMP_BATCH_SIZE = 8 49 | 50 | # Timeout value (optional), default is None, i.e. no timeout limit applied 51 | TIMEOUT = 96:00:00 52 | 53 | 54 | ## Module options 55 | 56 | [SPLIT_EXP_RUNNER] 57 | 58 | INPUT_DIR = last:get_images_runner 59 | 60 | FILE_PATTERN = image, weight, flag 61 | 62 | # Matches compressed single-exposure files 63 | FILE_EXT = .fitsfz, .fitsfz, .fitsfz 64 | 65 | NUMBERING_SCHEME = -0000000 66 | 67 | # OUTPUT_SUFFIX, actually file name prefixes. 68 | # Expected keyword "flag" will lead to a behavior where the data are saved as int. 69 | # The code also expects the image data to use the "image" suffix 70 | # (default value in the pipeline). 71 | OUTPUT_SUFFIX = image, weight, flag 72 | 73 | # Number of HDUs/CCDs of mosaic 74 | N_HDU = 40 75 | -------------------------------------------------------------------------------- /example/cfis/config_exp_SpMh.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe configuration file for single-exposures, 2 | # split images, merge headers 3 | 4 | 5 | ## Default ShapePipe options 6 | [DEFAULT] 7 | 8 | # verbose mode (optional), default: True, print messages on terminal 9 | VERBOSE = True 10 | 11 | # Name of run (optional) default: shapepipe_run 12 | RUN_NAME = run_sp_exp_SpMh 13 | 14 | # Add date and time to RUN_NAME, optional, default: True 15 | RUN_DATETIME = False 16 | 17 | 18 | ## ShapePipe execution options 19 | [EXECUTION] 20 | 21 | # Module name, single string or comma-separated list of valid module runner names 22 | MODULE = split_exp_runner, merge_headers_runner 23 | 24 | # Run mode, SMP or MPI 25 | MODE = SMP 26 | 27 | 28 | ## ShapePipe file handling options 29 | [FILE] 30 | 31 | # Log file master name, optional, default: shapepipe 32 | LOG_NAME = log_sp 33 | 34 | # Runner log file name, optional, default: shapepipe_runs 35 | RUN_LOG_NAME = log_run_sp 36 | 37 | # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN 38 | INPUT_DIR = . 39 | 40 | # Output directory 41 | OUTPUT_DIR = $SP_RUN/output 42 | 43 | 44 | ## ShapePipe job handling options 45 | [JOB] 46 | 47 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 48 | SMP_BATCH_SIZE = 16 49 | 50 | # Timeout value (optional), default is None, i.e. no timeout limit applied 51 | TIMEOUT = 96:00:00 52 | 53 | 54 | ## Module options 55 | 56 | [SPLIT_EXP_RUNNER] 57 | 58 | INPUT_DIR = last:get_images_runner_run_2 59 | 60 | FILE_PATTERN = image, weight, flag 61 | 62 | # Matches compressed single-exposure files 63 | FILE_EXT = .fitsfz, .fitsfz, .fitsfz 64 | 65 | NUMBERING_SCHEME = -0000000 66 | 67 | # OUTPUT_SUFFIX, actually file name prefixes. 68 | # Expected keyword "flag" will lead to a behavior where the data are saved as int. 69 | # The code also expects the image data to use the "image" suffix 70 | # (default value in the pipeline). 71 | OUTPUT_SUFFIX = image, weight, flag 72 | 73 | # Number of HDUs/CCDs of mosaic 74 | N_HDU = 40 75 | 76 | 77 | [MERGE_HEADERS_RUNNER] 78 | 79 | FILE_PATTERN = headers 80 | 81 | FILE_EXT = .npy 82 | 83 | # Single-exposure numbering scheme 84 | NUMBERING_SCHEME = -0000000 85 | 86 | OUTPUT_PATH = $SP_RUN/output 87 | -------------------------------------------------------------------------------- /example/cfis/config_get_tiles_vos_headers.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe configuration file for: get images 2 | 3 | 4 | ## Default ShapePipe options 5 | [DEFAULT] 6 | 7 | # verbose mode (optional), default: True, print messages on terminal 8 | VERBOSE = False 9 | 10 | # Name of run (optional) default: shapepipe_run 11 | RUN_NAME = run_sp_Git 12 | 13 | # Add date and time to RUN_NAME, optional, default: False 14 | RUN_DATETIME = True 15 | 16 | 17 | ## ShapePipe execution options 18 | [EXECUTION] 19 | 20 | # Module name, single string or comma-separated list of valid module runner names 21 | MODULE = get_images_runner 22 | 23 | # Parallel processing mode, SMP or MPI 24 | MODE = SMP 25 | 26 | 27 | ## ShapePipe file handling options 28 | [FILE] 29 | 30 | # Log file master name, optional, default: shapepipe 31 | LOG_NAME = log_sp 32 | 33 | # Runner log file name, optional, default: shapepipe_runs 34 | RUN_LOG_NAME = log_run_sp 35 | 36 | # Input directory, containing input files, single string or list of names 37 | INPUT_DIR = $SP_RUN 38 | 39 | # Output directory 40 | OUTPUT_DIR = $SP_RUN/output 41 | 42 | 43 | ## ShapePipe job handling options 44 | [JOB] 45 | 46 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 47 | SMP_BATCH_SIZE = 1 48 | 49 | # Timeout value (optional), default is None, i.e. no timeout limit applied 50 | TIMEOUT = 96:00:00 51 | 52 | 53 | ## Module options 54 | [GET_IMAGES_RUNNER] 55 | 56 | FILE_PATTERN = tile_numbers 57 | 58 | FILE_EXT = .txt 59 | 60 | 61 | # NUMBERING_SCHEME (optional) string with numbering pattern for input files 62 | NUMBERING_SCHEME = 63 | 64 | 65 | # Paths 66 | 67 | # Output path (optional, default is [FILE]:OUTPUT_DIR 68 | # OUTPUT_PATH = input_images 69 | 70 | # Input path where original images are stored. Can be local path or vos url. 71 | # Single string or list of strings 72 | INPUT_PATH = vos:cfis/tiles_DR5 73 | 74 | # Input file pattern including tile number as dummy template 75 | INPUT_FILE_PATTERN = CFIS.000.000.r 76 | 77 | # Input file extensions 78 | INPUT_FILE_EXT = .fits 79 | 80 | # Input numbering scheme, python regexp 81 | INPUT_NUMBERING = \d{3}\.\d{3} 82 | 83 | # Output file pattern without number 84 | OUTPUT_FILE_PATTERN = CFIS_image- 85 | #, CFIS_weight- 86 | 87 | # Copy/download method, one in 'vos', 'symlink' 88 | RETRIEVE = vos 89 | 90 | # If RETRIEVE=vos, number of attempts to download 91 | # Optional, default=3 92 | N_TRY = 3 93 | 94 | # Copy command options, optional 95 | RETRIEVE_OPTIONS = --head 96 | -------------------------------------------------------------------------------- /example/cfis/config_make_cat_mccd.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe post-run configuration file: create final catalogs 2 | 3 | 4 | ## Default ShapePipe options 5 | [DEFAULT] 6 | 7 | # verbose mode (optional), default: True, print messages on terminal 8 | VERBOSE = True 9 | 10 | # Name of run (optional) default: shapepipe_run 11 | RUN_NAME = run_sp_Mc 12 | 13 | # Add date and time to RUN_NAME, optional, default: True 14 | ; RUN_DATETIME = False 15 | 16 | 17 | ## ShapePipe execution options 18 | [EXECUTION] 19 | 20 | # Module name, single string or comma-separated list of valid module runner names 21 | MODULE = make_cat_runner 22 | 23 | # Parallel processing mode, SMP or MPI 24 | MODE = SMP 25 | 26 | 27 | ## ShapePipe file handling options 28 | [FILE] 29 | 30 | # Log file master name, optional, default: shapepipe 31 | LOG_NAME = log_sp 32 | 33 | # Runner log file name, optional, default: shapepipe_runs 34 | RUN_LOG_NAME = log_run_sp 35 | 36 | # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN 37 | INPUT_DIR = last:sextractor_runner_run_1, last:spread_model_runner, last:mccd_interp_runner, last:merge_sep_cats_runner 38 | 39 | # Output directory 40 | OUTPUT_DIR = ./output 41 | 42 | 43 | ## ShapePipe job handling options 44 | [JOB] 45 | 46 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 47 | SMP_BATCH_SIZE = 8 48 | 49 | # Timeout value (optional), default is None, i.e. no timeout limit applied 50 | TIMEOUT = 96:00:00 51 | 52 | 53 | ## Module options 54 | 55 | [MAKE_CAT_RUNNER] 56 | 57 | # Input file pattern(s), list of strings with length matching number of expected input file types 58 | # Cannot contain wild cards 59 | FILE_PATTERN = sexcat, sexcat_sm, galaxy_psf, ngmix 60 | #, galsim 61 | 62 | # FILE_EXT (optional) list of string extensions to identify input files 63 | FILE_EXT = .fits, .fits, .sqlite, .fits 64 | #, .fits 65 | 66 | # Numbering convention, string that exemplifies a numbering pattern. 67 | # Matches input single exposures (with 'p' removed) 68 | # Needs to be given in this section, will be updated in module 69 | # sections below 70 | NUMBERING_SCHEME = -000-000 71 | 72 | SM_DO_CLASSIFICATION = True 73 | SM_STAR_THRESH = 0.003 74 | SM_GAL_THRESH = 0.01 75 | 76 | SHAPE_MEASUREMENT_TYPE = ngmix 77 | #, galsim 78 | -------------------------------------------------------------------------------- /example/cfis/config_make_cat_psfex.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe post-run configuration file: create final catalogs 2 | 3 | 4 | ## Default ShapePipe options 5 | [DEFAULT] 6 | 7 | # verbose mode (optional), default: True, print messages on terminal 8 | VERBOSE = True 9 | 10 | # Name of run (optional) default: shapepipe_run 11 | RUN_NAME = run_sp_Mc 12 | 13 | # Add date and time to RUN_NAME, optional, default: True 14 | ; RUN_DATETIME = False 15 | 16 | 17 | ## ShapePipe execution options 18 | [EXECUTION] 19 | 20 | # Module name, single string or comma-separated list of valid module runner names 21 | MODULE = make_cat_runner 22 | 23 | # Parallel processing mode, SMP or MPI 24 | MODE = SMP 25 | 26 | 27 | ## ShapePipe file handling options 28 | [FILE] 29 | 30 | # Log file master name, optional, default: shapepipe 31 | LOG_NAME = log_sp 32 | 33 | # Runner log file name, optional, default: shapepipe_runs 34 | RUN_LOG_NAME = log_run_sp 35 | 36 | # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN 37 | INPUT_DIR = . 38 | 39 | # Output directory 40 | OUTPUT_DIR = ./output 41 | 42 | 43 | ## ShapePipe job handling options 44 | [JOB] 45 | 46 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 47 | SMP_BATCH_SIZE = 8 48 | 49 | # Timeout value (optional), default is None, i.e. no timeout limit applied 50 | TIMEOUT = 96:00:00 51 | 52 | 53 | ## Module options 54 | 55 | [MAKE_CAT_RUNNER] 56 | 57 | # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN 58 | INPUT_DIR = run_sp_tile_Sx:sextractor_runner, last:spread_model_runner, last:psfex_interp_runner, last:merge_sep_cats_runner 59 | 60 | # Input file pattern(s), list of strings with length matching number of expected input file types 61 | # Cannot contain wild cards 62 | FILE_PATTERN = sexcat, sexcat_sm, galaxy_psf, ngmix 63 | 64 | # FILE_EXT (optional) list of string extensions to identify input files 65 | FILE_EXT = .fits, .fits, .sqlite, .fits 66 | 67 | # Numbering convention, string that exemplifies a numbering pattern. 68 | # Matches input single exposures (with 'p' removed) 69 | # Needs to be given in this section, will be updated in module 70 | # sections below 71 | NUMBERING_SCHEME = -000-000 72 | 73 | SM_DO_CLASSIFICATION = True 74 | SM_STAR_THRESH = 0.003 75 | SM_GAL_THRESH = 0.01 76 | 77 | SHAPE_MEASUREMENT_TYPE = ngmix 78 | -------------------------------------------------------------------------------- /example/cfis/config_make_cat_psfex_nosm.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe post-run configuration file: create final catalogs, with 2 | # no spread model on input 3 | 4 | 5 | ## Default ShapePipe options 6 | [DEFAULT] 7 | 8 | # verbose mode (optional), default: True, print messages on terminal 9 | VERBOSE = True 10 | 11 | # Name of run (optional) default: shapepipe_run 12 | RUN_NAME = run_sp_Mc 13 | 14 | # Add date and time to RUN_NAME, optional, default: True 15 | ; RUN_DATETIME = False 16 | 17 | 18 | ## ShapePipe execution options 19 | [EXECUTION] 20 | 21 | # Module name, single string or comma-separated list of valid module runner names 22 | MODULE = make_cat_runner 23 | 24 | # Parallel processing mode, SMP or MPI 25 | MODE = SMP 26 | 27 | 28 | ## ShapePipe file handling options 29 | [FILE] 30 | 31 | # Log file master name, optional, default: shapepipe 32 | LOG_NAME = log_sp 33 | 34 | # Runner log file name, optional, default: shapepipe_runs 35 | RUN_LOG_NAME = log_run_sp 36 | 37 | # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN 38 | INPUT_DIR = . 39 | 40 | # Output directory 41 | OUTPUT_DIR = ./output 42 | 43 | 44 | ## ShapePipe job handling options 45 | [JOB] 46 | 47 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 48 | SMP_BATCH_SIZE = 8 49 | 50 | # Timeout value (optional), default is None, i.e. no timeout limit applied 51 | TIMEOUT = 96:00:00 52 | 53 | 54 | ## Module options 55 | 56 | [MAKE_CAT_RUNNER] 57 | 58 | # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN 59 | INPUT_DIR = run_sp_tile_Sx:sextractor_runner, last:psfex_interp_runner, last:merge_sep_cats_runner 60 | 61 | # Input file pattern(s), list of strings with length matching number of expected input file types 62 | # Cannot contain wild cards 63 | FILE_PATTERN = sexcat, galaxy_psf, ngmix 64 | 65 | # FILE_EXT (optional) list of string extensions to identify input files 66 | FILE_EXT = .fits, .sqlite, .fits 67 | 68 | # Numbering convention, string that exemplifies a numbering pattern. 69 | # Matches input single exposures (with 'p' removed) 70 | # Needs to be given in this section, will be updated in module 71 | # sections below 72 | NUMBERING_SCHEME = -000-000 73 | 74 | SM_DO_CLASSIFICATION = False 75 | 76 | SHAPE_MEASUREMENT_TYPE = ngmix 77 | -------------------------------------------------------------------------------- /example/cfis/config_merge_sep_cats_template.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe post-run configuration file: merge separated catalogues 2 | 3 | 4 | ## Default ShapePipe options 5 | [DEFAULT] 6 | 7 | # verbose mode (optional), default: True, print messages on terminal 8 | VERBOSE = True 9 | 10 | # Name of run (optional) default: shapepipe_run 11 | RUN_NAME = run_sp_Ms 12 | 13 | # Add date and time to RUN_NAME, optional, default: True 14 | ; RUN_DATETIME = False 15 | 16 | 17 | ## ShapePipe execution options 18 | [EXECUTION] 19 | 20 | # Module name, single string or comma-separated list of valid module runner names 21 | MODULE = merge_sep_cats_runner 22 | 23 | # Parallel processing mode, SMP or MPI 24 | MODE = SMP 25 | 26 | 27 | ## ShapePipe file handling options 28 | [FILE] 29 | 30 | # Log file master name, optional, default: shapepipe 31 | LOG_NAME = log_sp 32 | 33 | # Runner log file name, optional, default: shapepipe_runs 34 | RUN_LOG_NAME = log_run_sp 35 | 36 | # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN 37 | INPUT_DIR = ./output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output 38 | 39 | # Output directory 40 | OUTPUT_DIR = ./output 41 | 42 | 43 | ## ShapePipe job handling options 44 | [JOB] 45 | 46 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 47 | SMP_BATCH_SIZE = 8 48 | 49 | # Timeout value (optional), default is None, i.e. no timeout limit applied 50 | TIMEOUT = 96:00:00 51 | 52 | 53 | ## Module options 54 | 55 | [MERGE_SEP_CATS_RUNNER] 56 | 57 | # Input file pattern(s), list of strings with length matching number of expected input file types 58 | # Cannot contain wild cards 59 | FILE_PATTERN = ngmix 60 | 61 | # FILE_EXT (optional) list of string extensions to identify input files 62 | FILE_EXT = .fits 63 | 64 | # Numbering convention, string that exemplifies a numbering pattern. 65 | NUMBERING_SCHEME = -000-000 66 | 67 | # WARNING (optional, default is 'error'). Use 'always'/'ignore' to 68 | # display/ignore warnings, and not raise error 69 | WARNING = always 70 | 71 | # Maximum number of separated catalogues per input 72 | N_SPLIT_MAX = X 73 | -------------------------------------------------------------------------------- /example/cfis/config_onthefly.mask: -------------------------------------------------------------------------------- 1 | # Mask module configuration file for single-exposure images 2 | 3 | ## Paths to executables 4 | [PROGRAM_PATH] 5 | 6 | WW_PATH = ww 7 | WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww 8 | 9 | # Indicate cds client executable if no external star catalogue is available 10 | # (e.g. no internet access on run nodes) 11 | CDSCLIENT_PATH = findgsc2.2 12 | 13 | 14 | ## Border mask 15 | [BORDER_PARAMETERS] 16 | 17 | BORDER_MAKE = True 18 | 19 | BORDER_WIDTH = 50 20 | BORDER_FLAG_VALUE = 4 21 | 22 | 23 | ## Halo mask 24 | [HALO_PARAMETERS] 25 | 26 | HALO_MAKE = True 27 | 28 | HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg 29 | HALO_MAG_LIM = 13. 30 | HALO_SCALE_FACTOR = 0.05 31 | HALO_MAG_PIVOT = 13.8 32 | HALO_FLAG_VALUE = 2 33 | HALO_REG_FILE = halo.reg 34 | 35 | 36 | ## Diffraction spike mask 37 | [SPIKE_PARAMETERS] 38 | 39 | SPIKE_MAKE = True 40 | 41 | SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg 42 | SPIKE_MAG_LIM = 18. 43 | SPIKE_SCALE_FACTOR = 0.3 44 | SPIKE_MAG_PIVOT = 13.8 45 | SPIKE_FLAG_VALUE = 128 46 | SPIKE_REG_FILE = spike.reg 47 | 48 | 49 | ## Messier mask 50 | [MESSIER_PARAMETERS] 51 | 52 | MESSIER_MAKE = True 53 | 54 | MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits 55 | MESSIER_SIZE_PLUS = 0. 56 | MESSIER_FLAG_VALUE = 16 57 | 58 | 59 | ## NGC mask 60 | [NGC_PARAMETERS] 61 | 62 | NGC_MAKE = True 63 | 64 | NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits 65 | NGC_SIZE_PLUS = 0. 66 | NGC_FLAG_VALUE = 32 67 | 68 | 69 | 70 | ## Missing data parameters 71 | [MD_PARAMETERS] 72 | 73 | MD_MAKE = False 74 | 75 | MD_THRESH_FLAG = 0.3 76 | MD_THRESH_REMOVE = 0.75 77 | MD_REMOVE = False 78 | 79 | 80 | ## Other parameters 81 | [OTHER] 82 | 83 | TEMP_DIRECTORY = .temp 84 | 85 | KEEP_REG_FILE = False 86 | KEEP_INDIVIDUAL_MASK = False 87 | -------------------------------------------------------------------------------- /example/cfis/config_save.mask: -------------------------------------------------------------------------------- 1 | # Mask module configuration file for single-exposure images 2 | 3 | ## Paths to executables 4 | [PROGRAM_PATH] 5 | 6 | WW_PATH = ww 7 | WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww 8 | 9 | # Indicate cds client executable if no external star catalogue is available 10 | # (e.g. no internet access on run nodes) 11 | #CDSCLIENT_PATH = findgsc2.2 12 | 13 | 14 | ## Border mask 15 | [BORDER_PARAMETERS] 16 | 17 | BORDER_MAKE = True 18 | 19 | BORDER_WIDTH = 50 20 | BORDER_FLAG_VALUE = 4 21 | 22 | 23 | ## Halo mask 24 | [HALO_PARAMETERS] 25 | 26 | HALO_MAKE = True 27 | 28 | HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg 29 | HALO_MAG_LIM = 13. 30 | HALO_SCALE_FACTOR = 0.05 31 | HALO_MAG_PIVOT = 13.8 32 | HALO_FLAG_VALUE = 2 33 | HALO_REG_FILE = halo.reg 34 | 35 | 36 | ## Diffraction spike mask 37 | [SPIKE_PARAMETERS] 38 | 39 | SPIKE_MAKE = True 40 | 41 | SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg 42 | SPIKE_MAG_LIM = 18. 43 | SPIKE_SCALE_FACTOR = 0.3 44 | SPIKE_MAG_PIVOT = 13.8 45 | SPIKE_FLAG_VALUE = 128 46 | SPIKE_REG_FILE = spike.reg 47 | 48 | 49 | ## Messier mask 50 | [MESSIER_PARAMETERS] 51 | 52 | MESSIER_MAKE = True 53 | 54 | MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits 55 | MESSIER_SIZE_PLUS = 0. 56 | MESSIER_FLAG_VALUE = 16 57 | 58 | 59 | ## NGC mask 60 | [NGC_PARAMETERS] 61 | 62 | NGC_MAKE = True 63 | 64 | NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits 65 | NGC_SIZE_PLUS = 0. 66 | NGC_FLAG_VALUE = 32 67 | 68 | 69 | 70 | ## Missing data parameters 71 | [MD_PARAMETERS] 72 | 73 | MD_MAKE = False 74 | 75 | MD_THRESH_FLAG = 0.3 76 | MD_THRESH_REMOVE = 0.75 77 | MD_REMOVE = False 78 | 79 | 80 | ## Other parameters 81 | [OTHER] 82 | 83 | TEMP_DIRECTORY = .temp 84 | 85 | KEEP_REG_FILE = False 86 | KEEP_INDIVIDUAL_MASK = False 87 | -------------------------------------------------------------------------------- /example/cfis/config_stats.ini: -------------------------------------------------------------------------------- 1 | [ALL] 2 | nbins = 50 3 | 4 | [0] 5 | # Nb objects full cat 6 | fname = 0_nb_obj 7 | xlabel = number / CCD 8 | xmin = 0 9 | xmax = 3000 10 | title = all objects 11 | 12 | [1] 13 | # Nb objects not masked 14 | fname = 1_nb_nonmasked 15 | xlabel = number / CCD 16 | xmax = 210 17 | title = non-masked objects 18 | 19 | [2] 20 | # Nb stars 21 | fname = 2_nb_stars 22 | xlabel = number / CCD 23 | xmax = 210 24 | title = stars 25 | 26 | [3] 27 | # stars/deg^2 28 | plot = False 29 | 30 | [4] 31 | # Mean star fwhm selected (arcsec) 32 | fname = 3_mean_fhwm_star 33 | xlabel = FHWM mean (arcsec) 34 | xmax = 1.1 35 | title = stars 36 | 37 | [5] 38 | # Standard deviation fwhm star selected (arcsec) 39 | fname = 4_std_fhwm_star 40 | xlabel = FHWM standard deviation (arcsec) 41 | title = stars 42 | 43 | [6] 44 | # Mode fwhm used (arcsec) 45 | fname = 5_mode_fhwm_star 46 | xlabel = FHWM mode (arcsec) 47 | xmax = 1.1 48 | title = stars 49 | 50 | [7] 51 | # Min fwhm cut (arcesec) 52 | plot = False 53 | 54 | [8] 55 | # Max fwhm cut (arcsec) 56 | plot = False 57 | -------------------------------------------------------------------------------- /example/cfis/config_tile_Ma_onthefly.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe configuration file for masking of tiles 2 | 3 | 4 | ## Default ShapePipe options 5 | [DEFAULT] 6 | 7 | # verbose mode (optional), default: True, print messages on terminal 8 | VERBOSE = True 9 | 10 | # Name of run (optional) default: shapepipe_run 11 | RUN_NAME = run_sp_tile_Ma 12 | 13 | # Add date and time to RUN_NAME, optional, default: False 14 | ; RUN_DATETIME = False 15 | 16 | 17 | ## ShapePipe execution options 18 | [EXECUTION] 19 | 20 | # Module name, single string or comma-separated list of valid module runner names 21 | MODULE = mask_runner 22 | 23 | # Parallel processing mode, SMP or MPI 24 | MODE = SMP 25 | 26 | 27 | ## ShapePipe file handling options 28 | [FILE] 29 | 30 | # Log file master name, optional, default: shapepipe 31 | LOG_NAME = log_sp 32 | 33 | # Runner log file name, optional, default: shapepipe_runs 34 | RUN_LOG_NAME = log_run_sp 35 | 36 | # Input directory, containing input files, single string or list of names 37 | INPUT_DIR = $SP_RUN/output 38 | 39 | # Output directory 40 | OUTPUT_DIR = $SP_RUN/output 41 | 42 | 43 | ## ShapePipe job handling options 44 | [JOB] 45 | 46 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 47 | SMP_BATCH_SIZE = 8 48 | 49 | # Timeout value (optional), default is None, i.e. no timeout limit applied 50 | TIMEOUT = 96:00:00 51 | 52 | 53 | ## Module options 54 | 55 | ### Mask tiles 56 | [MASK_RUNNER] 57 | 58 | # Input directory, containing input files, single string or list of names 59 | INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner 60 | 61 | # NUMBERING_SCHEME (optional) string with numbering pattern for input files 62 | NUMBERING_SCHEME = -000-000 63 | 64 | # Input file pattern(s), list of strings with length matching number of expected input file types 65 | # Cannot contain wild cards 66 | FILE_PATTERN = CFIS_image, CFIS_weight 67 | 68 | # FILE_EXT (optional) list of string extensions to identify input files 69 | FILE_EXT = .fits, .fits 70 | 71 | # Path of mask config file 72 | MASK_CONFIG_PATH = $SP_CONFIG/config_tile_onthefly.mask 73 | 74 | # External mask file flag, use if True, otherwise ignore 75 | USE_EXT_FLAG = False 76 | 77 | # External star catalogue flag, use external cat if True, 78 | # obtain from online catalogue if False 79 | USE_EXT_STAR = False 80 | 81 | # File name suffix for the output flag files (optional) 82 | PREFIX = pipeline 83 | -------------------------------------------------------------------------------- /example/cfis/config_tile_Ng_template.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe configuration file for tiles: ngmix + KSB 2 | 3 | 4 | ## Default ShapePipe options 5 | [DEFAULT] 6 | 7 | # verbose mode (optional), default: True, print messages on terminal 8 | VERBOSE = True 9 | 10 | # Name of run (optional) default: shapepipe_run 11 | RUN_NAME = run_sp_tile_ngmix_NgXu 12 | 13 | # Add date and time to RUN_NAME, optional, default: False 14 | RUN_DATETIME = False 15 | 16 | 17 | ## ShapePipe execution options 18 | [EXECUTION] 19 | 20 | # Module name, single string or comma-separated list of valid module runner names 21 | MODULE = ngmix_runner 22 | 23 | # Parallel processing mode, SMP or MPI 24 | MODE = SMP 25 | 26 | 27 | ## ShapePipe file handling options 28 | [FILE] 29 | 30 | # Log file master name, optional, default: shapepipe 31 | LOG_NAME = log_sp 32 | 33 | # Runner log file name, optional, default: shapepipe_runs 34 | RUN_LOG_NAME = log_run_sp 35 | 36 | # Input directory, containing input files, single string or list of names 37 | INPUT_DIR = . 38 | 39 | # Output directory 40 | OUTPUT_DIR = $SP_RUN/output 41 | 42 | 43 | ## ShapePipe job handling options 44 | [JOB] 45 | 46 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 47 | SMP_BATCH_SIZE = 1 48 | 49 | # Timeout value (optional), default is None, i.e. no timeout limit applied 50 | TIMEOUT = 96:00:00 51 | 52 | 53 | ## Module options 54 | 55 | # Model-fitting shapes with ngmix 56 | [NGMIX_RUNNER] 57 | 58 | INPUT_DIR = run_sp_tile_Sx:sextractor_runner,last:X_interp_runner,last:vignetmaker_runner_run_2 59 | 60 | FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet 61 | 62 | FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite 63 | 64 | # NUMBERING_SCHEME (optional) string with numbering pattern for input files 65 | NUMBERING_SCHEME = -000-000 66 | 67 | # Multi-epoch mode: Path to file with single-exposure WCS header information 68 | LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite 69 | 70 | # Magnitude zero-point 71 | MAG_ZP = 30.0 72 | 73 | # Pixel scale in arcsec 74 | PIXEL_SCALE = 0.186 75 | 76 | ID_OBJ_MIN = X 77 | ID_OBJ_MAX = X 78 | -------------------------------------------------------------------------------- /example/cfis/config_tile_Ng_template_batch.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe configuration file for tiles: ngmix + KSB 2 | 3 | 4 | ## Default ShapePipe options 5 | [DEFAULT] 6 | 7 | # verbose mode (optional), default: True, print messages on terminal 8 | VERBOSE = True 9 | 10 | # Name of run (optional) default: shapepipe_run 11 | RUN_NAME = run_sp_tile_ngmix_NgXu 12 | 13 | # Add date and time to RUN_NAME, optional, default: False 14 | RUN_DATETIME = False 15 | 16 | 17 | ## ShapePipe execution options 18 | [EXECUTION] 19 | 20 | # Module name, single string or comma-separated list of valid module runner names 21 | MODULE = ngmix_runner 22 | 23 | # Parallel processing mode, SMP or MPI 24 | MODE = SMP 25 | 26 | 27 | ## ShapePipe file handling options 28 | [FILE] 29 | 30 | # Log file master name, optional, default: shapepipe 31 | LOG_NAME = log_sp 32 | 33 | # Runner log file name, optional, default: shapepipe_runs 34 | RUN_LOG_NAME = log_run_sp 35 | 36 | # Input directory, containing input files, single string or list of names 37 | INPUT_DIR = . 38 | 39 | # Output directory 40 | OUTPUT_DIR = $SP_RUN/output 41 | 42 | 43 | ## ShapePipe job handling options 44 | [JOB] 45 | 46 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 47 | SMP_BATCH_SIZE = 1 48 | 49 | # Timeout value (optional), default is None, i.e. no timeout limit applied 50 | TIMEOUT = 96:00:00 51 | 52 | 53 | ## Module options 54 | 55 | # Model-fitting shapes with ngmix 56 | [NGMIX_RUNNER] 57 | 58 | INPUT_DIR = run_sp_tile_Sx:sextractor_runner,last:X_interp_runner,last:vignetmaker_runner_run_2 59 | 60 | FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet 61 | 62 | FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite 63 | 64 | # NUMBERING_SCHEME (optional) string with numbering pattern for input files 65 | NUMBERING_SCHEME = -000-000 66 | 67 | # Multi-epoch mode: Path to file with single-exposure WCS header information 68 | LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite 69 | 70 | # Directory of previous run, optional. 71 | CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_tile_ngmix_Ng1u_prev/ngmix_runner/output 72 | 73 | # Number of objects to batch save during processing, optional. Omit or set 74 | # to -1 for no batch saving 75 | SAVE_BATCH = 1000 76 | 77 | # Magnitude zero-point 78 | MAG_ZP = 30.0 79 | 80 | # Pixel scale in arcsec 81 | PIXEL_SCALE = 0.186 82 | 83 | ID_OBJ_MIN = X 84 | ID_OBJ_MAX = X 85 | -------------------------------------------------------------------------------- /example/cfis/config_tile_Uz.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe configuration file for: uncompress FITS image 2 | 3 | 4 | ## Default ShapePipe options 5 | [DEFAULT] 6 | 7 | # verbose mode (optional), default: True, print messages on terminal 8 | VERBOSE = True 9 | 10 | # Name of run (optional) default: shapepipe_run 11 | RUN_NAME = run_sp_Uz 12 | 13 | # Add date and time to RUN_NAME, optional, default: False 14 | RUN_DATETIME = True 15 | 16 | 17 | ## ShapePipe execution options 18 | [EXECUTION] 19 | 20 | # Module name, single string or comma-separated list of valid module runner names 21 | MODULE = uncompress_fits_runner 22 | 23 | # Parallel processing mode, SMP or MPI 24 | MODE = SMP 25 | 26 | 27 | ## ShapePipe file handling options 28 | [FILE] 29 | 30 | # Log file master name, optional, default: shapepipe 31 | LOG_NAME = log_sp 32 | 33 | # Runner log file name, optional, default: shapepipe_runs 34 | RUN_LOG_NAME = log_run_sp 35 | 36 | # Input directory, containing input files, single string or list of names 37 | INPUT_DIR = . 38 | 39 | # Output directory 40 | OUTPUT_DIR = $SP_RUN/output 41 | 42 | 43 | ## ShapePipe job handling options 44 | [JOB] 45 | 46 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 47 | SMP_BATCH_SIZE = 16 48 | 49 | # Timeout value (optional), default is None, i.e. no timeout limit applied 50 | TIMEOUT = 96:00:00 51 | 52 | 53 | ## Module options 54 | [UNCOMPRESS_FITS_RUNNER] 55 | 56 | INPUT_DIR = last:get_images_runner_run_1 57 | #INPUT_DIR = last:get_images_runner 58 | 59 | FILE_PATTERN = CFIS_weight 60 | 61 | FILE_EXT = .fitsfz 62 | 63 | # NUMBERING_SCHEME (optional) string with numbering pattern for input files 64 | NUMBERING_SCHEME = -000-000 65 | 66 | # Input HDU of image data, optional, default=0 67 | HDU_DATA = 1 68 | 69 | # Output file pattern 70 | OUTPUT_PATTERN = CFIS_weight 71 | -------------------------------------------------------------------------------- /example/cfis/config_tile_onthefly.mask: -------------------------------------------------------------------------------- 1 | # Mask module config file for tiles 2 | 3 | ## Paths to executables 4 | [PROGRAM_PATH] 5 | 6 | WW_PATH = ww 7 | WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww 8 | 9 | # Indicate cds client executable if no external star catalogue is available 10 | # (e.g. no internet access on run nodes) 11 | CDSCLIENT_PATH = findgsc2.2 12 | 13 | ## Border parameters 14 | [BORDER_PARAMETERS] 15 | 16 | BORDER_MAKE = False 17 | 18 | BORDER_WIDTH = 0 19 | BORDER_FLAG_VALUE = 4 20 | 21 | 22 | ## Halo parameters 23 | [HALO_PARAMETERS] 24 | 25 | HALO_MAKE = True 26 | 27 | HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg 28 | HALO_MAG_LIM = 13. 29 | HALO_SCALE_FACTOR = 0.05 30 | HALO_MAG_PIVOT = 13.8 31 | HALO_FLAG_VALUE = 2 32 | HALO_REG_FILE = halo.reg 33 | 34 | 35 | ## Diffraction pike parameters 36 | [SPIKE_PARAMETERS] 37 | 38 | SPIKE_MAKE = True 39 | 40 | SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg 41 | SPIKE_MAG_LIM = 18. 42 | SPIKE_SCALE_FACTOR = 0.3 43 | SPIKE_MAG_PIVOT = 13.8 44 | SPIKE_FLAG_VALUE = 128 45 | SPIKE_REG_FILE = spike.reg 46 | 47 | 48 | ## Messier parameters 49 | [MESSIER_PARAMETERS] 50 | 51 | MESSIER_MAKE = True 52 | 53 | MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits 54 | MESSIER_PIXEL_SCALE = 0.187 55 | MESSIER_SIZE_PLUS = 0. 56 | MESSIER_FLAG_VALUE = 16 57 | 58 | ## NGC mask 59 | [NGC_PARAMETERS] 60 | 61 | NGC_MAKE = True 62 | 63 | NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits 64 | NGC_SIZE_PLUS = 0. 65 | NGC_FLAG_VALUE = 32 66 | 67 | 68 | ## External flag 69 | [EXTERNAL_FLAG] 70 | 71 | EF_MAKE = False 72 | 73 | 74 | ## Missing data parameters 75 | [MD_PARAMETERS] 76 | 77 | MD_MAKE = False 78 | 79 | MD_THRESH_FLAG = 0.3 80 | MD_THRESH_REMOVE = 0.75 81 | MD_REMOVE = False 82 | 83 | 84 | ## Other parameters 85 | [OTHER] 86 | 87 | KEEP_REG_FILE = False 88 | KEEP_INDIVIDUAL_MASK = False 89 | 90 | TEMP_DIRECTORY = .temp_tiles 91 | -------------------------------------------------------------------------------- /example/cfis/config_tile_save.mask: -------------------------------------------------------------------------------- 1 | # Mask module config file for tiles 2 | 3 | ## Paths to executables 4 | [PROGRAM_PATH] 5 | 6 | WW_PATH = ww 7 | WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww 8 | 9 | # Indicate cds client executable if no external star catalogue is available 10 | # (e.g. no internet access on run nodes) 11 | #CDSCLIENT_PATH = findgsc2.2 12 | 13 | ## Border parameters 14 | [BORDER_PARAMETERS] 15 | 16 | BORDER_MAKE = False 17 | 18 | BORDER_WIDTH = 0 19 | BORDER_FLAG_VALUE = 4 20 | 21 | 22 | ## Halo parameters 23 | [HALO_PARAMETERS] 24 | 25 | HALO_MAKE = True 26 | 27 | HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg 28 | HALO_MAG_LIM = 13. 29 | HALO_SCALE_FACTOR = 0.05 30 | HALO_MAG_PIVOT = 13.8 31 | HALO_FLAG_VALUE = 2 32 | HALO_REG_FILE = halo.reg 33 | 34 | 35 | ## Diffraction pike parameters 36 | [SPIKE_PARAMETERS] 37 | 38 | SPIKE_MAKE = True 39 | 40 | SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg 41 | SPIKE_MAG_LIM = 18. 42 | SPIKE_SCALE_FACTOR = 0.3 43 | SPIKE_MAG_PIVOT = 13.8 44 | SPIKE_FLAG_VALUE = 128 45 | SPIKE_REG_FILE = spike.reg 46 | 47 | 48 | ## Messier parameters 49 | [MESSIER_PARAMETERS] 50 | 51 | MESSIER_MAKE = True 52 | 53 | MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits 54 | MESSIER_PIXEL_SCALE = 0.187 55 | MESSIER_SIZE_PLUS = 0. 56 | MESSIER_FLAG_VALUE = 16 57 | 58 | ## NGC mask 59 | [NGC_PARAMETERS] 60 | 61 | NGC_MAKE = True 62 | 63 | NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits 64 | NGC_SIZE_PLUS = 0. 65 | NGC_FLAG_VALUE = 32 66 | 67 | 68 | ## External flag 69 | [EXTERNAL_FLAG] 70 | 71 | EF_MAKE = False 72 | 73 | 74 | ## Missing data parameters 75 | [MD_PARAMETERS] 76 | 77 | MD_MAKE = False 78 | 79 | MD_THRESH_FLAG = 0.3 80 | MD_THRESH_REMOVE = 0.75 81 | MD_REMOVE = False 82 | 83 | 84 | ## Other parameters 85 | [OTHER] 86 | 87 | KEEP_REG_FILE = False 88 | KEEP_INDIVIDUAL_MASK = False 89 | 90 | TEMP_DIRECTORY = .temp_tiles 91 | -------------------------------------------------------------------------------- /example/cfis/default.conv: -------------------------------------------------------------------------------- 1 | CONV NORM 2 | # 3x3 ``all-ground'' convolution mask with FWHM = 2 pixels. 3 | 1 2 1 4 | 2 4 2 5 | 1 2 1 6 | -------------------------------------------------------------------------------- /example/cfis/final_cat.param: -------------------------------------------------------------------------------- 1 | # coordinates 2 | XWIN_WORLD 3 | YWIN_WORLD 4 | 5 | # tile ID, for plot of tile-dependent additive bias. 6 | # Can maybe be removed. 7 | TILE_ID 8 | 9 | # flags 10 | FLAGS 11 | IMAFLAGS_ISO 12 | NGMIX_MCAL_FLAGS 13 | 14 | # PSF ellipticity 15 | NGMIX_ELL_PSFo_NOSHEAR 16 | 17 | # spread class 18 | #SPREAD_CLASS 19 | 20 | # spread model flag and error 21 | #SPREAD_MODEL 22 | #SPREADERR_MODEL 23 | 24 | # Number of epochs (exposures) 25 | N_EPOCH 26 | NGMIX_N_EPOCH 27 | 28 | ## Shape measurement outputs 29 | ## Ngmix: model fitting 30 | 31 | # galaxy ellipticity 32 | NGMIX_ELL_1M 33 | NGMIX_ELL_1P 34 | NGMIX_ELL_2M 35 | NGMIX_ELL_2P 36 | NGMIX_ELL_NOSHEAR 37 | #NGMIX_ELL_ERR_1M 38 | #NGMIX_ELL_ERR_1P 39 | #NGMIX_ELL_ERR_2M 40 | #NGMIX_ELL_ERR_2P 41 | NGMIX_ELL_ERR_NOSHEAR 42 | 43 | # flags 44 | NGMIX_FLAGS_1M 45 | NGMIX_FLAGS_1P 46 | NGMIX_FLAGS_2M 47 | NGMIX_FLAGS_2P 48 | NGMIX_FLAGS_NOSHEAR 49 | 50 | # size and error 51 | NGMIX_T_1M 52 | NGMIX_T_1P 53 | NGMIX_T_2M 54 | NGMIX_T_2P 55 | NGMIX_T_NOSHEAR 56 | NGMIX_T_ERR_1M 57 | NGMIX_T_ERR_1P 58 | NGMIX_T_ERR_2M 59 | NGMIX_T_ERR_2P 60 | NGMIX_T_ERR_NOSHEAR 61 | NGMIX_Tpsf_1M 62 | NGMIX_Tpsf_1P 63 | NGMIX_Tpsf_2M 64 | NGMIX_Tpsf_2P 65 | NGMIX_Tpsf_NOSHEAR 66 | 67 | # flux and error 68 | NGMIX_FLUX_1M 69 | NGMIX_FLUX_1P 70 | NGMIX_FLUX_2M 71 | NGMIX_FLUX_2P 72 | NGMIX_FLUX_NOSHEAR 73 | NGMIX_FLUX_ERR_1M 74 | NGMIX_FLUX_ERR_1P 75 | NGMIX_FLUX_ERR_2M 76 | NGMIX_FLUX_ERR_2P 77 | NGMIX_FLUX_ERR_NOSHEAR 78 | 79 | # magnitudes 80 | MAG_AUTO 81 | MAGERR_AUTO 82 | MAG_WIN 83 | MAGERR_WIN 84 | FLUX_AUTO 85 | FLUXERR_AUTO 86 | FLUX_APER 87 | FLUXERR_APER 88 | FLUX_RADIUS 89 | 90 | # SNR from SExtractor 91 | SNR_WIN 92 | 93 | FWHM_IMAGE 94 | FWHM_WORLD 95 | 96 | # PSF size measured on original image 97 | NGMIX_T_PSFo_NOSHEAR 98 | 99 | # PSF size measured on reconvolved image 100 | # NGMIX_Tpsf_NOSHEAR 101 | 102 | # ngmix moment failure flag 103 | NGMIX_MOM_FAIL 104 | -------------------------------------------------------------------------------- /example/cfis/mask_default/MEGAPRIME_star_i_13.8.reg: -------------------------------------------------------------------------------- 1 | -11.5 68 2 | -6 186.5 3 | 7 188 4 | 10 64.5 5 | 31 55 6 | 50 38.5 7 | 56.5 11.5 8 | 188 8 9 | 192 -4 10 | 59.5 -11.5 11 | 45 -33 12 | 13.5 -64 13 | 5 -154 14 | -6 -155 15 | -11 -64.5 16 | -40 -44.5 17 | -51.5 -30.5 18 | -62.5 -22.5 19 | -68 -9.5 20 | -177 -2 21 | -176 3 22 | -78 12.5 23 | -67.5 14.5 24 | -38.5 50 25 | -------------------------------------------------------------------------------- /example/cfis/mask_default/Messier_catalog.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CosmoStat/shapepipe/43d3730f0a7766a6585bc085c8412811aa50563e/example/cfis/mask_default/Messier_catalog.npy -------------------------------------------------------------------------------- /example/cfis/mask_default/Messier_catalog_updated.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CosmoStat/shapepipe/43d3730f0a7766a6585bc085c8412811aa50563e/example/cfis/mask_default/Messier_catalog_updated.fits -------------------------------------------------------------------------------- /example/cfis/mask_default/default.ww: -------------------------------------------------------------------------------- 1 | #--------------------------------- Weights ------------------------------------ 2 | 3 | WEIGHT_NAMES weightin.fits # Filename(s) of the input WEIGHT map(s) 4 | 5 | WEIGHT_MIN 0. # Pixel below those thresholds will be flagged 6 | WEIGHT_MAX 1000. # Pixels above those thresholds will be flagged 7 | WEIGHT_OUTFLAGS 1 # FLAG values for thresholded pixels 8 | 9 | #---------------------------------- Flags ------------------------------------- 10 | 11 | FLAG_NAMES flagin.fits # Filename(s) of the input FLAG map(s) 12 | 13 | FLAG_WMASKS 0xff # Bits which will nullify the WEIGHT-map pixels 14 | FLAG_MASKS 0x01 # Bits which will be converted as output FLAGs 15 | FLAG_OUTFLAGS 2 # Translation of the FLAG_MASKS bits 16 | 17 | #---------------------------------- Polygons ---------------------------------- 18 | 19 | POLY_NAMES "" # Filename(s) of input DS9 regions 20 | POLY_OUTFLAGS # FLAG values for polygon masks 21 | POLY_OUTWEIGHTS 0.0 # Weight values for polygon masks 22 | POLY_INTERSECT Y # Use inclusive OR for polygon intersects (Y/N)? 23 | 24 | #---------------------------------- Output ------------------------------------ 25 | 26 | OUTWEIGHT_NAME "w.fits" # Output WEIGHT-map filename 27 | OUTFLAG_NAME flag.fits # Output FLAG-map filename 28 | 29 | #----------------------------- Miscellaneous --------------------------------- 30 | 31 | GETAREA N # Compute area for flags and weights (Y/N)? 32 | GETAREA_WEIGHT 0.0 # Weight threshold for area computation 33 | GETAREA_FLAGS 1 # Bit mask for flag pixels not counted in area 34 | MEMORY_BUFSIZE 256 # Buffer size in lines 35 | VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL 36 | WRITE_XML N # Write XML file (Y/N)? 37 | XML_NAME ww.xml # Filename for XML output 38 | XSL_URL file:///usr/local/share/weightwatcher/ww.xsl 39 | # Filename for XSL style-sheet 40 | NTHREADS 1 # 1 single thread -------------------------------------------------------------------------------- /example/cfis/mask_default/halo_mask.reg: -------------------------------------------------------------------------------- 1 | 274.66813 -1.25966 2 | 272.54579 32.47406 3 | 266.21222 65.67579 4 | 255.76731 97.82190 5 | 241.37579 128.40544 6 | 223.26462 156.94408 7 | 201.71942 182.98775 8 | 177.07997 206.12573 9 | 149.73486 225.99312 10 | 120.11532 242.27660 11 | 88.68848 254.71937 12 | 55.94996 263.12519 13 | 22.41606 267.36151 14 | -11.38436 267.36151 15 | -44.91826 263.12519 16 | -77.65678 254.71937 17 | -109.08362 242.27660 18 | -138.70315 225.99312 19 | -166.04827 206.12573 20 | -190.68772 182.98775 21 | -212.23292 156.94408 22 | -230.34409 128.40544 23 | -244.73561 97.82190 24 | -255.18052 65.67579 25 | -261.51409 32.47406 26 | -263.63643 -1.25966 27 | -261.51409 -34.99339 28 | -255.18052 -68.19511 29 | -244.73561 -100.34123 30 | -230.34409 -130.92476 31 | -212.23292 -159.46341 32 | -190.68772 -185.50708 33 | -166.04827 -208.64506 34 | -138.70315 -228.51245 35 | -109.08362 -244.79593 36 | -77.65678 -257.23870 37 | -44.91826 -265.64452 38 | -11.38436 -269.88084 39 | 22.41606 -269.88084 40 | 55.94996 -265.64452 41 | 88.68848 -257.23870 42 | 120.11532 -244.79593 43 | 149.73486 -228.51245 44 | 177.07997 -208.64506 45 | 201.71942 -185.50708 46 | 223.26462 -159.46341 47 | 241.37579 -130.92476 48 | 255.76731 -100.34123 49 | 266.21222 -68.19511 50 | 272.54579 -34.99339 51 | -------------------------------------------------------------------------------- /example/cfis/mask_default/ngc_cat.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CosmoStat/shapepipe/43d3730f0a7766a6585bc085c8412811aa50563e/example/cfis/mask_default/ngc_cat.fits -------------------------------------------------------------------------------- /example/cfis_simu/config.mask_simu: -------------------------------------------------------------------------------- 1 | # Mask module configuration file for single-exposure images 2 | 3 | ## Paths to executables 4 | [PROGRAM_PATH] 5 | 6 | WW_PATH = ww 7 | WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww 8 | 9 | # Indicate cds client executable if no external star catalogue is available 10 | # (e.g. no internet access on run nodes) 11 | CDSCLIENT_PATH = findgsc2.2 12 | 13 | 14 | ## Border mask 15 | [BORDER_PARAMETERS] 16 | 17 | BORDER_MAKE = True 18 | 19 | BORDER_WIDTH = 50 20 | BORDER_FLAG_VALUE = 4 21 | 22 | 23 | ## Halo mask 24 | [HALO_PARAMETERS] 25 | 26 | HALO_MAKE = False 27 | 28 | HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg 29 | HALO_MAG_LIM = 13. 30 | HALO_SCALE_FACTOR = 0.05 31 | HALO_MAG_PIVOT = 13.8 32 | HALO_FLAG_VALUE = 2 33 | HALO_REG_FILE = halo.reg 34 | 35 | 36 | ## Diffraction spike mask 37 | [SPIKE_PARAMETERS] 38 | 39 | SPIKE_MAKE = False 40 | 41 | SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg 42 | SPIKE_MAG_LIM = 18. 43 | SPIKE_SCALE_FACTOR = 0.3 44 | SPIKE_MAG_PIVOT = 13.8 45 | SPIKE_FLAG_VALUE = 128 46 | SPIKE_REG_FILE = spike.reg 47 | 48 | 49 | ## Messier mask 50 | [MESSIER_PARAMETERS] 51 | 52 | MESSIER_MAKE = False 53 | 54 | MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits 55 | MESSIER_SIZE_PLUS = 0. 56 | MESSIER_FLAG_VALUE = 16 57 | 58 | 59 | ## NGC mask 60 | [NGC_PARAMETERS] 61 | 62 | NGC_MAKE = False 63 | 64 | NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits 65 | NGC_SIZE_PLUS = 0. 66 | NGC_FLAG_VALUE = 32 67 | 68 | 69 | 70 | ## Missing data parameters 71 | [MD_PARAMETERS] 72 | 73 | MD_MAKE = False 74 | 75 | MD_THRESH_FLAG = 0.3 76 | MD_THRESH_REMOVE = 0.75 77 | MD_REMOVE = False 78 | 79 | 80 | ## Other parameters 81 | [OTHER] 82 | 83 | TEMP_DIRECTORY = .temp 84 | 85 | KEEP_REG_FILE = False 86 | KEEP_INDIVIDUAL_MASK = False 87 | -------------------------------------------------------------------------------- /example/cfis_simu/config_exp_SpMh.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe configuration file for single-exposures, 2 | # split images, merge headers 3 | 4 | 5 | ## Default ShapePipe options 6 | [DEFAULT] 7 | 8 | # verbose mode (optional), default: True, print messages on terminal 9 | VERBOSE = True 10 | 11 | # Name of run (optional) default: shapepipe_run 12 | RUN_NAME = run_sp_exp_SpMh 13 | 14 | # Add date and time to RUN_NAME, optional, default: True 15 | ; RUN_DATETIME = False 16 | 17 | 18 | ## ShapePipe execution options 19 | [EXECUTION] 20 | 21 | # Module name, single string or comma-separated list of valid module runner names 22 | MODULE = split_exp_runner, merge_headers_runner 23 | 24 | # Run mode, SMP or MPI 25 | MODE = SMP 26 | 27 | 28 | ## ShapePipe file handling options 29 | [FILE] 30 | 31 | # Log file master name, optional, default: shapepipe 32 | LOG_NAME = log_sp 33 | 34 | # Runner log file name, optional, default: shapepipe_runs 35 | RUN_LOG_NAME = log_run_sp 36 | 37 | # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN 38 | INPUT_DIR = . 39 | 40 | # Output directory 41 | OUTPUT_DIR = $SP_RUN/output 42 | 43 | 44 | ## ShapePipe job handling options 45 | [JOB] 46 | 47 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 48 | SMP_BATCH_SIZE = 16 49 | 50 | # Timeout value (optional), default is None, i.e. no timeout limit applied 51 | TIMEOUT = 96:00:00 52 | 53 | 54 | ## Module options 55 | 56 | [SPLIT_EXP_RUNNER] 57 | 58 | INPUT_DIR = last:get_images_runner_run_2 59 | 60 | # Matches compressed single-exposure files 61 | FILE_EXT = .fits, .fits, .fits 62 | 63 | NUMBERING_SCHEME = -0000000 64 | 65 | # OUTPUT_SUFFIX, actually file name prefixes. 66 | # Expected keyword "flag" will lead to a behavior where the data are save as int. 67 | # The code also expects the image data to use the "image" suffix 68 | # (default value in the pipeline). 69 | OUTPUT_SUFFIX = image, weight, flag 70 | 71 | # Number of HDUs/CCDs of mosaic 72 | N_HDU = 40 73 | 74 | 75 | [MERGE_HEADERS_RUNNER] 76 | 77 | FILE_PATTERN = headers 78 | 79 | FILE_EXT = .npy 80 | 81 | # Single-exposure numbering scheme 82 | NUMBERING_SCHEME = -0000000 83 | 84 | OUTPUT_PATH = $SP_RUN/output 85 | -------------------------------------------------------------------------------- /example/cfis_simu/config_make_cat_mccd.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe post-run configuration file: create final catalogs 2 | 3 | 4 | ## Default ShapePipe options 5 | [DEFAULT] 6 | 7 | # verbose mode (optional), default: True, print messages on terminal 8 | VERBOSE = True 9 | 10 | # Name of run (optional) default: shapepipe_run 11 | RUN_NAME = run_sp_Mc 12 | 13 | # Add date and time to RUN_NAME, optional, default: True 14 | ; RUN_DATETIME = False 15 | 16 | 17 | ## ShapePipe execution options 18 | [EXECUTION] 19 | 20 | # Module name, single string or comma-separated list of valid module runner names 21 | MODULE = make_cat_runner 22 | 23 | # Parallel processing mode, SMP or MPI 24 | MODE = SMP 25 | 26 | 27 | ## ShapePipe file handling options 28 | [FILE] 29 | 30 | # Log file master name, optional, default: shapepipe 31 | LOG_NAME = log_sp 32 | 33 | # Runner log file name, optional, default: shapepipe_runs 34 | RUN_LOG_NAME = log_run_sp 35 | 36 | # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN 37 | INPUT_DIR = last:sextractor_runner_run_1, last:spread_model_runner, last:mccd_interp_runner, last:merge_sep_cats_runner 38 | 39 | # Output directory 40 | OUTPUT_DIR = ./output 41 | 42 | 43 | ## ShapePipe job handling options 44 | [JOB] 45 | 46 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 47 | SMP_BATCH_SIZE = 8 48 | 49 | # Timeout value (optional), default is None, i.e. no timeout limit applied 50 | TIMEOUT = 96:00:00 51 | 52 | 53 | ## Module options 54 | 55 | [MAKE_CAT_RUNNER] 56 | 57 | # Input file pattern(s), list of strings with length matching number of expected input file types 58 | # Cannot contain wild cards 59 | FILE_PATTERN = sexcat, sexcat_sm, galaxy_psf, ngmix 60 | #, galsim 61 | 62 | # FILE_EXT (optional) list of string extensions to identify input files 63 | FILE_EXT = .fits, .fits, .sqlite, .fits 64 | #, .fits 65 | 66 | # Numbering convention, string that exemplifies a numbering pattern. 67 | # Matches input single exposures (with 'p' removed) 68 | # Needs to be given in this section, will be updated in module 69 | # sections below 70 | NUMBERING_SCHEME = -000-000 71 | 72 | SM_DO_CLASSIFICATION = True 73 | SM_STAR_THRESH = 0.003 74 | SM_GAL_THRESH = 0.01 75 | 76 | SHAPE_MEASUREMENT_TYPE = ngmix 77 | #, galsim 78 | -------------------------------------------------------------------------------- /example/cfis_simu/config_merge_sep_cats_template.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe post-run configuration file: merge separated catalogues 2 | 3 | 4 | ## Default ShapePipe options 5 | [DEFAULT] 6 | 7 | # verbose mode (optional), default: True, print messages on terminal 8 | VERBOSE = True 9 | 10 | # Name of run (optional) default: shapepipe_run 11 | RUN_NAME = run_sp_Ms 12 | 13 | # Add date and time to RUN_NAME, optional, default: True 14 | ; RUN_DATETIME = False 15 | 16 | 17 | ## ShapePipe execution options 18 | [EXECUTION] 19 | 20 | # Module name, single string or comma-separated list of valid module runner names 21 | MODULE = merge_sep_cats_runner 22 | 23 | # Parallel processing mode, SMP or MPI 24 | MODE = SMP 25 | 26 | 27 | ## ShapePipe file handling options 28 | [FILE] 29 | 30 | # Log file master name, optional, default: shapepipe 31 | LOG_NAME = log_sp 32 | 33 | # Runner log file name, optional, default: shapepipe_runs 34 | RUN_LOG_NAME = log_run_sp 35 | 36 | # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN 37 | INPUT_DIR = ./output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output 38 | #, ./output/run_sp_tile_ngmix_Ng1u/galsim_shapes_v2_runner/output 39 | 40 | # Output directory 41 | OUTPUT_DIR = ./output 42 | 43 | 44 | ## ShapePipe job handling options 45 | [JOB] 46 | 47 | # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial 48 | SMP_BATCH_SIZE = 8 49 | 50 | # Timeout value (optional), default is None, i.e. no timeout limit applied 51 | TIMEOUT = 96:00:00 52 | 53 | 54 | ## Module options 55 | 56 | [MERGE_SEP_CATS_RUNNER] 57 | 58 | # Input file pattern(s), list of strings with length matching number of expected input file types 59 | # Cannot contain wild cards 60 | FILE_PATTERN = ngmix 61 | #, galsim 62 | 63 | # FILE_EXT (optional) list of string extensions to identify input files 64 | FILE_EXT = .fits 65 | #, .fits 66 | 67 | # Numbering convention, string that exemplifies a numbering pattern. 68 | NUMBERING_SCHEME = -000-000 69 | 70 | # WARNING (optional, default is 'error'). Use 'always'/'ignore' to 71 | # display/ignore warnings, and not raise error 72 | WARNING = always 73 | 74 | # Maximum number of separated catalogues per input 75 | N_SPLIT_MAX = 0 76 | -------------------------------------------------------------------------------- /example/cfis_simu/config_tile.mask_simu: -------------------------------------------------------------------------------- 1 | # Mask module config file for tiles 2 | 3 | ## Paths to executables 4 | [PROGRAM_PATH] 5 | 6 | WW_PATH = ww 7 | WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww 8 | 9 | # Indicate cds client executable if no external star catalogue is available 10 | # (e.g. no internet access on run nodes) 11 | CDSCLIENT_PATH = findgsc2.2 12 | 13 | ## Border parameters 14 | [BORDER_PARAMETERS] 15 | 16 | BORDER_MAKE = True 17 | 18 | BORDER_WIDTH = 1 19 | BORDER_FLAG_VALUE = 4 20 | 21 | 22 | ## Halo parameters 23 | [HALO_PARAMETERS] 24 | 25 | HALO_MAKE = False 26 | 27 | HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg 28 | HALO_MAG_LIM = 13. 29 | HALO_SCALE_FACTOR = 0.05 30 | HALO_MAG_PIVOT = 13.8 31 | HALO_FLAG_VALUE = 2 32 | HALO_REG_FILE = halo.reg 33 | 34 | 35 | ## Diffraction pike parameters 36 | [SPIKE_PARAMETERS] 37 | 38 | SPIKE_MAKE = False 39 | 40 | SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg 41 | SPIKE_MAG_LIM = 18. 42 | SPIKE_SCALE_FACTOR = 0.3 43 | SPIKE_MAG_PIVOT = 13.8 44 | SPIKE_FLAG_VALUE = 128 45 | SPIKE_REG_FILE = spike.reg 46 | 47 | 48 | ## Messier parameters 49 | [MESSIER_PARAMETERS] 50 | 51 | MESSIER_MAKE = False 52 | 53 | MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits 54 | MESSIER_PIXEL_SCALE = 0.187 55 | MESSIER_SIZE_PLUS = 0. 56 | MESSIER_FLAG_VALUE = 16 57 | 58 | ## NGC mask 59 | [NGC_PARAMETERS] 60 | 61 | NGC_MAKE = False 62 | 63 | NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits 64 | NGC_SIZE_PLUS = 0. 65 | NGC_FLAG_VALUE = 32 66 | 67 | 68 | ## External flag 69 | [EXTERNAL_FLAG] 70 | 71 | EF_MAKE = False 72 | 73 | 74 | ## Missing data parameters 75 | [MD_PARAMETERS] 76 | 77 | MD_MAKE = False 78 | 79 | MD_THRESH_FLAG = 0.3 80 | MD_THRESH_REMOVE = 0.75 81 | MD_REMOVE = False 82 | 83 | 84 | ## Other parameters 85 | [OTHER] 86 | 87 | KEEP_REG_FILE = False 88 | KEEP_INDIVIDUAL_MASK = False 89 | 90 | TEMP_DIRECTORY = .temp_tiles 91 | -------------------------------------------------------------------------------- /example/cfis_simu/default.conv: -------------------------------------------------------------------------------- 1 | CONV NORM 2 | # 3x3 ``all-ground'' convolution mask with FWHM = 2 pixels. 3 | 1 2 1 4 | 2 4 2 5 | 1 2 1 6 | -------------------------------------------------------------------------------- /example/cfis_simu/mask_default/MEGAPRIME_star_i_13.8.reg: -------------------------------------------------------------------------------- 1 | -11.5 68 2 | -6 186.5 3 | 7 188 4 | 10 64.5 5 | 31 55 6 | 50 38.5 7 | 56.5 11.5 8 | 188 8 9 | 192 -4 10 | 59.5 -11.5 11 | 45 -33 12 | 13.5 -64 13 | 5 -154 14 | -6 -155 15 | -11 -64.5 16 | -40 -44.5 17 | -51.5 -30.5 18 | -62.5 -22.5 19 | -68 -9.5 20 | -177 -2 21 | -176 3 22 | -78 12.5 23 | -67.5 14.5 24 | -38.5 50 25 | -------------------------------------------------------------------------------- /example/cfis_simu/mask_default/Messier_catalog.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CosmoStat/shapepipe/43d3730f0a7766a6585bc085c8412811aa50563e/example/cfis_simu/mask_default/Messier_catalog.npy -------------------------------------------------------------------------------- /example/cfis_simu/mask_default/Messier_catalog_updated.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CosmoStat/shapepipe/43d3730f0a7766a6585bc085c8412811aa50563e/example/cfis_simu/mask_default/Messier_catalog_updated.fits -------------------------------------------------------------------------------- /example/cfis_simu/mask_default/default.ww: -------------------------------------------------------------------------------- 1 | #--------------------------------- Weights ------------------------------------ 2 | 3 | WEIGHT_NAMES weightin.fits # Filename(s) of the input WEIGHT map(s) 4 | 5 | WEIGHT_MIN 0. # Pixel below those thresholds will be flagged 6 | WEIGHT_MAX 1000. # Pixels above those thresholds will be flagged 7 | WEIGHT_OUTFLAGS 1 # FLAG values for thresholded pixels 8 | 9 | #---------------------------------- Flags ------------------------------------- 10 | 11 | FLAG_NAMES flagin.fits # Filename(s) of the input FLAG map(s) 12 | 13 | FLAG_WMASKS 0xff # Bits which will nullify the WEIGHT-map pixels 14 | FLAG_MASKS 0x01 # Bits which will be converted as output FLAGs 15 | FLAG_OUTFLAGS 2 # Translation of the FLAG_MASKS bits 16 | 17 | #---------------------------------- Polygons ---------------------------------- 18 | 19 | POLY_NAMES "" # Filename(s) of input DS9 regions 20 | POLY_OUTFLAGS # FLAG values for polygon masks 21 | POLY_OUTWEIGHTS 0.0 # Weight values for polygon masks 22 | POLY_INTERSECT Y # Use inclusive OR for polygon intersects (Y/N)? 23 | 24 | #---------------------------------- Output ------------------------------------ 25 | 26 | OUTWEIGHT_NAME "w.fits" # Output WEIGHT-map filename 27 | OUTFLAG_NAME flag.fits # Output FLAG-map filename 28 | 29 | #----------------------------- Miscellaneous --------------------------------- 30 | 31 | GETAREA N # Compute area for flags and weights (Y/N)? 32 | GETAREA_WEIGHT 0.0 # Weight threshold for area computation 33 | GETAREA_FLAGS 1 # Bit mask for flag pixels not counted in area 34 | MEMORY_BUFSIZE 256 # Buffer size in lines 35 | VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL 36 | WRITE_XML N # Write XML file (Y/N)? 37 | XML_NAME ww.xml # Filename for XML output 38 | XSL_URL file:///usr/local/share/weightwatcher/ww.xsl 39 | # Filename for XSL style-sheet 40 | NTHREADS 1 # 1 single thread -------------------------------------------------------------------------------- /example/cfis_simu/mask_default/halo_mask.reg: -------------------------------------------------------------------------------- 1 | 274.66813 -1.25966 2 | 272.54579 32.47406 3 | 266.21222 65.67579 4 | 255.76731 97.82190 5 | 241.37579 128.40544 6 | 223.26462 156.94408 7 | 201.71942 182.98775 8 | 177.07997 206.12573 9 | 149.73486 225.99312 10 | 120.11532 242.27660 11 | 88.68848 254.71937 12 | 55.94996 263.12519 13 | 22.41606 267.36151 14 | -11.38436 267.36151 15 | -44.91826 263.12519 16 | -77.65678 254.71937 17 | -109.08362 242.27660 18 | -138.70315 225.99312 19 | -166.04827 206.12573 20 | -190.68772 182.98775 21 | -212.23292 156.94408 22 | -230.34409 128.40544 23 | -244.73561 97.82190 24 | -255.18052 65.67579 25 | -261.51409 32.47406 26 | -263.63643 -1.25966 27 | -261.51409 -34.99339 28 | -255.18052 -68.19511 29 | -244.73561 -100.34123 30 | -230.34409 -130.92476 31 | -212.23292 -159.46341 32 | -190.68772 -185.50708 33 | -166.04827 -208.64506 34 | -138.70315 -228.51245 35 | -109.08362 -244.79593 36 | -77.65678 -257.23870 37 | -44.91826 -265.64452 38 | -11.38436 -269.88084 39 | 22.41606 -269.88084 40 | 55.94996 -265.64452 41 | 88.68848 -257.23870 42 | 120.11532 -244.79593 43 | 149.73486 -228.51245 44 | 177.07997 -208.64506 45 | 201.71942 -185.50708 46 | 223.26462 -159.46341 47 | 241.37579 -130.92476 48 | 255.76731 -100.34123 49 | 266.21222 -68.19511 50 | 272.54579 -34.99339 51 | -------------------------------------------------------------------------------- /example/cfis_simu/mask_default/ngc_cat.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CosmoStat/shapepipe/43d3730f0a7766a6585bc085c8412811aa50563e/example/cfis_simu/mask_default/ngc_cat.fits -------------------------------------------------------------------------------- /example/cfis_simu/readme.txt: -------------------------------------------------------------------------------- 1 | This folder contains the configuration files used to run 2 | shapepipe on simulated images. 3 | The $SP_RUN and $SP_CONFIG can point to this directory. 4 | A link to the exposures and tiles can be called "input_exp" 5 | and "input_tiles" and added to this directory. 6 | 7 | 8 | -------------------------------------------------------------------------------- /example/cfis_simu/tile_numbers.txt: -------------------------------------------------------------------------------- 1 | 224-295 2 | -------------------------------------------------------------------------------- /example/data/letters/letters_1.txt: -------------------------------------------------------------------------------- 1 | a 2 | -------------------------------------------------------------------------------- /example/data/letters/letters_2.txt: -------------------------------------------------------------------------------- 1 | b 2 | -------------------------------------------------------------------------------- /example/data/letters/letters_3.txt: -------------------------------------------------------------------------------- 1 | c 2 | -------------------------------------------------------------------------------- /example/data/letters/letters_4.txt: -------------------------------------------------------------------------------- 1 | d 2 | -------------------------------------------------------------------------------- /example/data/letters/letters_5.txt: -------------------------------------------------------------------------------- 1 | e 2 | -------------------------------------------------------------------------------- /example/data/number_list.txt: -------------------------------------------------------------------------------- 1 | _1 2 | _2 3 | -------------------------------------------------------------------------------- /example/data/numbers/hold_3.txt: -------------------------------------------------------------------------------- 1 | 4 2 | -------------------------------------------------------------------------------- /example/data/numbers/numbers_1.txt: -------------------------------------------------------------------------------- 1 | 1 2 | -------------------------------------------------------------------------------- /example/data/numbers/numbers_2.txt: -------------------------------------------------------------------------------- 1 | 2 2 | -------------------------------------------------------------------------------- /example/data/numbers/numbers_4.txt: -------------------------------------------------------------------------------- 1 | 8 2 | -------------------------------------------------------------------------------- /example/data/numbers/numbers_5.txt: -------------------------------------------------------------------------------- 1 | 16 2 | -------------------------------------------------------------------------------- /example/output/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CosmoStat/shapepipe/43d3730f0a7766a6585bc085c8412811aa50563e/example/output/.gitkeep -------------------------------------------------------------------------------- /example/pbs/candide_mpi.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ########################## 4 | # MPI Script for CANDIDE # 5 | ########################## 6 | 7 | # Receive email when job finishes or aborts 8 | ## #PBS -M @cea.fr 9 | ## #PBS -m ea 10 | 11 | # Set a name for the job 12 | #PBS -N shapepipe_mpi 13 | 14 | # Join output and errors in one file 15 | #PBS -j oe 16 | 17 | # Set maximum computing time (e.g. 5min) 18 | #PBS -l walltime=00:05:00 19 | 20 | # Request number of cores (e.g. 2 from 2 different machines) 21 | #PBS -l nodes=2:ppn=2 22 | 23 | # Full path to environment 24 | export SPENV="$HOME/.conda/envs/shapepipe" 25 | 26 | # Full path to example config file and input data 27 | export SPDIR="$HOME/shapepipe" 28 | 29 | # Load modules 30 | module load intelpython/3 31 | module load openmpi/4.0.5 32 | 33 | # Activate conda environment 34 | source activate $SPENV 35 | 36 | # Run ShapePipe using full paths to executables 37 | $SPENV/bin/mpiexec --map-by node $SPENV/bin/shapepipe_run -c $SPDIR/example/config_mpi.ini 38 | 39 | # Return exit code 40 | exit 0 41 | -------------------------------------------------------------------------------- /example/pbs/candide_smp.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ########################## 4 | # SMP Script for CANDIDE # 5 | ########################## 6 | 7 | # Receive email when job finishes or aborts 8 | #PBS -M @cea.fr 9 | #PBS -m ea 10 | # Set a name for the job 11 | #PBS -N shapepipe_smp 12 | # Join output and errors in one file 13 | #PBS -j oe 14 | # Set maximum computing time (e.g. 5min) 15 | #PBS -l walltime=00:05:00 16 | # Request number of cores 17 | #PBS -l nodes=4 18 | 19 | # Full path to environment 20 | export SPENV="$HOME/.conda/envs/shapepipe" 21 | export SPDIR="$HOME/shapepipe" 22 | 23 | # Activate conda environment 24 | module load intelpython/3 25 | source activate $SPENV 26 | 27 | # Run ShapePipe using full paths to executables 28 | $SPENV/bin/shapepipe_run -c $SPDIR/example/pbs/config_smp.ini 29 | 30 | # Return exit code 31 | exit 0 32 | -------------------------------------------------------------------------------- /example/pbs/cc_mpi.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ########################## 4 | # MPI Script for ccin2p3 # 5 | ########################## 6 | 7 | # Receive email when job finishes or aborts 8 | #$ -M @cea.fr 9 | #$ -m bea 10 | # Set a name for the job 11 | #$ -N shapepipe_mpi 12 | # Set a group for the job 13 | #$ -P P_euclid_sci 14 | # Join output and errors in one file 15 | #$ -j y 16 | # Set maximum computing time (e.g. 5min) 17 | #$ -l h_cpu=00:05:00 18 | # Request muliprocessing resources 19 | #$ -l os=cl7 20 | # Request number of cores 21 | #$ -pe openmpi 4 22 | 23 | # Full path to environment 24 | export SPENV="$HOME/.conda/envs/shapepipe" 25 | export SPDIR="$HOME/shapepipe" 26 | 27 | # Activate conda environment 28 | ccenv anaconda 29 | ccenv openmpi 30 | source activate $SPENV 31 | 32 | # Run ShapePipe 33 | $SPENV/bin/mpiexec -n $NSLOTS $SPENV/bin/shapepipe_run -c $SPDIR/example/pbs/config_mpi.ini 34 | 35 | # Return exit code 36 | exit 0 37 | -------------------------------------------------------------------------------- /example/pbs/cc_smp.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ########################## 4 | # SMP Script for ccin2p3 # 5 | ########################## 6 | 7 | # Receive email when job finishes or aborts 8 | #$ -M @cea.fr 9 | #$ -m bea 10 | # Set a name for the job 11 | #$ -N shapepipe_smp 12 | # Set a group for the job 13 | #$ -P P_euclid_sci 14 | # Join output and errors in one file 15 | #$ -j y 16 | # Set maximum computing time (e.g. 5min) 17 | #$ -l h_cpu=00:05:00 18 | # Request muliprocessing resources 19 | #$ -l os=cl7 20 | # Request number of cores 21 | #$ -pe multicores 4 22 | 23 | # Full path to environment 24 | export SPENV="$HOME/.conda/envs/shapepipe" 25 | export SPDIR="$HOME/shapepipe" 26 | 27 | # Activate conda environment 28 | ccenv anaconda 29 | source activate $SPENV 30 | 31 | # Run ShapePipe using full paths to executables 32 | $SPENV/bin/shapepipe_run -c $SPDIR/example/pbs/config_smp.ini 33 | 34 | # Return exit code 35 | exit 0 36 | -------------------------------------------------------------------------------- /example/pbs/config_mpi.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe Configuration File Example for MPI 2 | 3 | ## ShapePipe execution options 4 | [EXECUTION] 5 | MODULE = python_example, serial_example, execute_example 6 | MODE = mpi 7 | 8 | ## ShapePipe file handling options 9 | [FILE] 10 | INPUT_DIR = $SPDIR/example/data 11 | OUTPUT_DIR = $SPDIR/example/output 12 | 13 | ## ShapePipe job handling options 14 | [JOB] 15 | TIMEOUT = 00:01:35 16 | 17 | ## Module options 18 | [PYTHON_EXAMPLE] 19 | MESSAGE = The obtained value is: 20 | 21 | [SERIAL_EXAMPLE] 22 | ADD_INPUT_DIR = $SPDIR/example/data/numbers, $SPDIR/example/data/letters 23 | -------------------------------------------------------------------------------- /example/pbs/config_smp.ini: -------------------------------------------------------------------------------- 1 | # ShapePipe Configuration File Example for SMP 2 | 3 | ## ShapePipe execution options 4 | [EXECUTION] 5 | MODULE = python_example_runner, serial_example_runner, execute_example_runner 6 | MODE = smp 7 | 8 | ## ShapePipe file handling options 9 | [FILE] 10 | INPUT_DIR = $SPDIR/example/data 11 | OUTPUT_DIR = $SPDIR/example/output 12 | 13 | ## ShapePipe job handling options 14 | [JOB] 15 | SMP_BATCH_SIZE = 4 16 | TIMEOUT = 00:01:35 17 | 18 | ## Module options 19 | [PYTHON_EXAMPLE_RUNNER] 20 | MESSAGE = The obtained value is: 21 | 22 | [SERIAL_EXAMPLE_RUNNER] 23 | ADD_INPUT_DIR = $SPDIR/example/data/numbers, $SPDIR/example/data/letters 24 | -------------------------------------------------------------------------------- /example/unions_800/cat_matched.param: -------------------------------------------------------------------------------- 1 | # coordinates 2 | XWIN_WORLD 3 | YWIN_WORLD 4 | 5 | # tile ID, for plot of tile-dependent additive bias. 6 | # Can maybe be removed. 7 | TILE_ID 8 | 9 | # flags 10 | FLAGS 11 | IMAFLAGS_ISO 12 | NGMIX_MCAL_FLAGS 13 | 14 | # PSF ellipticity 15 | NGMIX_ELL_PSFo_NOSHEAR 16 | 17 | # spread class 18 | SPREAD_CLASS 19 | 20 | # spread model flag and error 21 | SPREAD_MODEL 22 | SPREADERR_MODEL 23 | 24 | # Number of epochs (exposures) 25 | N_EPOCH 26 | NGMIX_N_EPOCH 27 | 28 | ## Shape measurement outputs 29 | ## Ngmix: model fitting 30 | 31 | # galaxy ellipticity 32 | NGMIX_ELL_1M 33 | NGMIX_ELL_1P 34 | NGMIX_ELL_2M 35 | NGMIX_ELL_2P 36 | NGMIX_ELL_NOSHEAR 37 | #NGMIX_ELL_ERR_1M 38 | #NGMIX_ELL_ERR_1P 39 | #NGMIX_ELL_ERR_2M 40 | #NGMIX_ELL_ERR_2P 41 | NGMIX_ELL_ERR_NOSHEAR 42 | 43 | # flags 44 | NGMIX_FLAGS_1M 45 | NGMIX_FLAGS_1P 46 | NGMIX_FLAGS_2M 47 | NGMIX_FLAGS_2P 48 | NGMIX_FLAGS_NOSHEAR 49 | 50 | # size and error 51 | NGMIX_T_1M 52 | NGMIX_T_1P 53 | NGMIX_T_2M 54 | NGMIX_T_2P 55 | NGMIX_T_NOSHEAR 56 | NGMIX_T_ERR_1M 57 | NGMIX_T_ERR_1P 58 | NGMIX_T_ERR_2M 59 | NGMIX_T_ERR_2P 60 | NGMIX_T_ERR_NOSHEAR 61 | NGMIX_Tpsf_1M 62 | NGMIX_Tpsf_1P 63 | NGMIX_Tpsf_2M 64 | NGMIX_Tpsf_2P 65 | NGMIX_Tpsf_NOSHEAR 66 | 67 | # flux and error 68 | NGMIX_FLUX_1M 69 | NGMIX_FLUX_1P 70 | NGMIX_FLUX_2M 71 | NGMIX_FLUX_2P 72 | NGMIX_FLUX_NOSHEAR 73 | NGMIX_FLUX_ERR_1M 74 | NGMIX_FLUX_ERR_1P 75 | NGMIX_FLUX_ERR_2M 76 | NGMIX_FLUX_ERR_2P 77 | NGMIX_FLUX_ERR_NOSHEAR 78 | 79 | # Flag for multiple objects due to overlapping 80 | # tiles 81 | FLAG_TILING 82 | 83 | # magnitude, mainly for plots 84 | MAG_AUTO 85 | 86 | # SNR from SExtractor, used for cuts on GALSIM shapes 87 | SNR_WIN 88 | 89 | # PSF size measured on original image 90 | NGMIX_T_PSFo_NOSHEAR 91 | 92 | # PSF size measured on reconvolved image 93 | # NGMIX_Tpsf_NOSHEAR 94 | 95 | # ngmix moment failure flag 96 | NGMIX_MOM_FAIL 97 | 98 | # 800 deg^2 external cat parameters 99 | CFIS_ID 100 | RA 101 | Dec 102 | distance 103 | -------------------------------------------------------------------------------- /nsswitch.conf: -------------------------------------------------------------------------------- 1 | # 2 | # /etc/nsswitch.conf 3 | # 4 | # An example Name Service Switch config file. This file should be 5 | # sorted with the most-used services at the beginning. 6 | # 7 | # The entry '[NOTFOUND=return]' means that the search for an 8 | # entry should stop if the search in the previous entry turned 9 | # up nothing. Note that if the search failed due to some other reason 10 | # (like no NIS server responding) then the search continues with the 11 | # next entry. 12 | # 13 | # Valid entries include: 14 | # 15 | # nisplus Use NIS+ (NIS version 3) 16 | # nis Use NIS (NIS version 2), also called YP 17 | # dns Use DNS (Domain Name Service) 18 | # files Use the local files 19 | # db Use the local database (.db) files 20 | # compat Use NIS on compat mode 21 | # hesiod Use Hesiod for user lookups 22 | # [NOTFOUND=return] Stop searching if not found so far 23 | # 24 | 25 | # To use db, put the "db" in front of "files" for entries you want to be 26 | # looked up first in the databases 27 | # 28 | # Example: 29 | #passwd: db files nisplus nis 30 | #shadow: db files nisplus nis 31 | #group: db files nisplus nis 32 | 33 | passwd: sss files 34 | shadow: files sss 35 | group: sss files 36 | 37 | #hosts: db files nisplus nis dns 38 | hosts: files dns 39 | 40 | # Example - obey only what nisplus tells us... 41 | #services: nisplus [NOTFOUND=return] files 42 | #networks: nisplus [NOTFOUND=return] files 43 | #protocols: nisplus [NOTFOUND=return] files 44 | #rpc: nisplus [NOTFOUND=return] files 45 | #ethers: nisplus [NOTFOUND=return] files 46 | #netmasks: nisplus [NOTFOUND=return] files 47 | 48 | bootparams: nisplus [NOTFOUND=return] files 49 | 50 | ethers: files 51 | netmasks: files 52 | networks: files 53 | protocols: files 54 | rpc: files 55 | services: files 56 | 57 | netgroup: nisplus 58 | 59 | publickey: nisplus 60 | 61 | automount: files nisplus 62 | aliases: files nisplus 63 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "shapepipe" 3 | version = "1.1.0" 4 | description = "Galaxy shape measurement pipeline." 5 | readme = "README.rst" 6 | authors = [ 7 | { name = "Samuel Farrens", email = "samuel.farrens@cea.fr" }, 8 | { name = "Axel Guinot", email = "axel.guinot@cea.fr" }, 9 | { name = "Martin Kilbinger", email = "martin.kilbinger@cea.fr" } 10 | ] 11 | license = { "file" = "LICENSE" } 12 | requires-python = ">=3.11" 13 | dependencies = [ 14 | "joblib>=0.13", 15 | "modopt>=1.2", 16 | "numpy>=1.14" 17 | ] 18 | 19 | 20 | [project.optional-dependencies] 21 | doc = [ 22 | "myst-parser", 23 | "numpydoc", 24 | "sphinx", 25 | "sphinxcontrib-bibtex", 26 | "sphinx-book-theme" 27 | ] 28 | lint = [ 29 | "black", 30 | "isort" 31 | ] 32 | release = [ 33 | "build", 34 | "twine", 35 | ] 36 | test = [ 37 | "pytest", 38 | "pytest-cov", 39 | "pytest-pycodestyle", 40 | "pytest-pydocstyle" 41 | ] 42 | dev = ["shapepipe[doc,lint,release,test]"] 43 | 44 | [project.scripts] 45 | shapepipe_run = "shapepipe.shapepipe_run:main" 46 | 47 | [tool.pytest.ini_options] 48 | addopts = "--verbose --cov=shapepipe" 49 | testpaths = ["shapepipe"] -------------------------------------------------------------------------------- /scripts/README.rst: -------------------------------------------------------------------------------- 1 | Scripts directory 2 | ================= 3 | 4 | This directory contain scripts that are used on pipeline outputs or to prepare 5 | files. They are not run throught the pipeline framework. For more details on how 6 | to run each them see below. 7 | 8 | Python scripts 9 | ============== 10 | 11 | 1. `create_log_exp_headers`_ 12 | 2. `create_star_cat`_ 13 | 14 | create_log_exp_headers 15 | ====================== 16 | 17 | This as to run after the module `split_exp_runner` it will create a master log 18 | file containing all the WCS information for each CCDs of each single exposures. 19 | To run the script : 20 | `python create_log_exp_headers.py path/to/split_exp_runner/output path/to/srcipt/output_dir` 21 | 22 | create_star_cat 23 | =============== 24 | 25 | This script create all the star catalogs required to run the mask module for a 26 | computational node without internet access. 27 | To run the script : 28 | `python create_star_cat.py path/to/image_dir path/to/script/output_dir` 29 | -------------------------------------------------------------------------------- /scripts/python/count_shdu_special.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import re 5 | import sys 6 | 7 | ID = sys.argv[1] 8 | print(f"Checking tile ID {ID}...") 9 | 10 | # Base directory for the tile runs 11 | tile_runs_dir = "tile_runs" 12 | 13 | sum_dir = "summary" 14 | specials = ["special_job_32_setools_runner_0", "special_job_32_setools_runner_1"] 15 | 16 | 17 | shdu_IDs = [] 18 | print("Going through links...") 19 | #3for ID in os.listdir(tile_runs_dir): 20 | if True: 21 | id_path = os.path.join(tile_runs_dir, ID, "output") 22 | 23 | for filename in os.listdir(id_path): 24 | if ( 25 | filename.startswith("run_sp_exp_SxSePsf") 26 | and os.path.islink(os.path.join(id_path, filename)) 27 | ): 28 | symlink_path = os.path.join(id_path, filename) 29 | target_path = os.readlink(symlink_path) 30 | 31 | pattern = r"/exp_runs/([^/]+)/output/" 32 | 33 | match = re.search(pattern, target_path) 34 | if match: 35 | shdu_IDs.append(match.group(1)) 36 | else: 37 | print(f"{filename} -> No match in target path: {target_path}") 38 | 39 | if len(shdu_IDs) == 0: 40 | print("No shdus found, exiting") 41 | sys.exit(1) 42 | else: 43 | print(f"Found {len(shdu_IDs)} links in dir {id_path}") 44 | 45 | 46 | print("Going through special files...") 47 | count = 0 48 | for special in specials: 49 | print(f"special {special}...") 50 | path = f"{sum_dir}/{special}.txt" 51 | with open(path, "r") as f: 52 | content = f.read() 53 | for shdu_ID in shdu_IDs: 54 | if re.search(shdu_ID, content): 55 | count += 1 56 | 57 | print(f"Found {count} / {len(shdu_IDs)} = {count / len(shdu_IDs):.2%}") 58 | -------------------------------------------------------------------------------- /scripts/python/stats_headless_canfar.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Name: stats_headless_canfar.py 4 | 5 | # Caution: Does not show all running or pending 6 | # headless jobs, for some reason. 7 | 8 | import sys 9 | from skaha.session import Session 10 | 11 | 12 | def main(argv=None): 13 | 14 | print( 15 | "# Depreciated, does not show pending jobs; use stats_jobs_canfar.sh", 16 | file=sys.stderr, 17 | ) 18 | 19 | session = Session() 20 | 21 | n_headless = session.stats()["instances"]["headless"] 22 | 23 | print(n_headless) 24 | 25 | return 0 26 | 27 | 28 | if __name__ == "__main__": 29 | sys.exit(main(sys.argv)) 30 | -------------------------------------------------------------------------------- /scripts/python/summary_run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import os 5 | 6 | from shapepipe.utilities.summary import * 7 | 8 | from summary_params_pre_v2 import * 9 | 10 | 11 | def main(argv=None): 12 | 13 | patch = argv[1] 14 | 15 | if len(argv) == 3: 16 | job_exclusive = argv[2] 17 | else: 18 | job_exclusive = None 19 | 20 | verbose = False 21 | 22 | jobs, list_tile_IDs_dot = set_jobs_v2_pre_v2(patch, verbose) 23 | 24 | list_tile_IDs = job_data.replace_dot_dash(list_tile_IDs_dot) 25 | 26 | # Numbers updated at runtime 27 | par_runtime = init_par_runtime(list_tile_IDs) 28 | 29 | job_data.print_stats_header() 30 | 31 | exp_IDs_path = "exp_numbers.txt" 32 | if os.path.exists(exp_IDs_path): 33 | # Read exposure ID list if file exists 34 | all_exposures = get_IDs_from_file(exp_IDs_path) 35 | par_runtime = update_par_runtime_after_find_exp( 36 | par_runtime, all_exposures 37 | ) 38 | 39 | if ( 40 | not os.path.exists(exp_IDs_path) 41 | or not job_exclusive 42 | or int(job_exclusive) & 1 43 | ): 44 | # Run job 1 if exposure ID list file does not exist or 45 | # job_exclusive is 1 or not set 46 | key = "1" 47 | jobs[key].print_intro() 48 | jobs[key].check_numbers(par_runtime=par_runtime, indices=[0, 1]) 49 | 50 | all_exposures = get_all_exposures( 51 | jobs[key]._paths_in_dir[1], verbose=True 52 | ) 53 | par_runtime = update_par_runtime_after_find_exp( 54 | par_runtime, all_exposures 55 | ) 56 | 57 | jobs[key].write_IDs_to_file("exp_numbers.txt", all_exposures) 58 | 59 | jobs[key].check_numbers(par_runtime, indices=[2]) 60 | 61 | print_par_runtime(par_runtime, verbose=verbose) 62 | 63 | # Get all keys after "1" 64 | keys = sorted(jobs.keys(), key=int) 65 | _ = keys.pop(0) 66 | 67 | for key in keys: 68 | if job_exclusive and not int(key) & int(job_exclusive): 69 | continue 70 | jobs[key].print_intro() 71 | jobs[key].check_numbers(par_runtime=par_runtime) 72 | 73 | return 0 74 | 75 | 76 | if __name__ == "__main__": 77 | sys.exit(main(sys.argv)) 78 | -------------------------------------------------------------------------------- /scripts/python/update_runs_log_file.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import re 4 | import sys 5 | import os 6 | 7 | 8 | # TODO: move to cs_util 9 | def matching_subdirs(base_dir, pattern, tail=False): 10 | 11 | # Find all matching subdirectories 12 | subdirs = [] 13 | for entry in os.listdir(base_dir): 14 | full_path = os.path.join(base_dir, entry) 15 | if os.path.isdir(full_path): 16 | found = False 17 | 18 | # Look for pattern at start or end 19 | if pattern in entry: 20 | 21 | # Get full path or last part ("tail") 22 | if not tail: 23 | path = full_path 24 | else: 25 | head, tail = os.path.split(full_path) 26 | path = tail 27 | 28 | # Remove postfix in case of multiple runs of same module 29 | path = re.sub("_run_.\d?", "", path) 30 | 31 | # Append to result 32 | subdirs.append(path) 33 | 34 | # Sort according to creation date 35 | if not tail: 36 | subdirs.sort(key=os.path.getctime) 37 | 38 | return subdirs 39 | 40 | 41 | def get_module_runs(subdirs): 42 | 43 | all_runs = {} 44 | for subdir in subdirs: 45 | runs = matching_subdirs(subdir, "_runner", tail=True) 46 | if len(runs) > 0: 47 | all_runs[subdir] = runs 48 | 49 | return all_runs 50 | 51 | 52 | def update_log_file(module_runs, log_name): 53 | 54 | with open(log_name, "w") as f_out: 55 | for key in module_runs: 56 | print(key, file=f_out, end=" ") 57 | print(",".join(module_runs[key]), file=f_out) 58 | 59 | 60 | def main(argv=None): 61 | 62 | # Set default parameters 63 | # p_def = params_default() 64 | 65 | # Command line options 66 | # options, args = parse_options(p_def) 67 | 68 | # if check_options(options) is False: 69 | # return 1 70 | 71 | # param = update_param(p_def, options) 72 | 73 | base_dir = "./output" 74 | pattern = "run_sp_" 75 | log_name = f"{base_dir}/log_run_sp.txt" 76 | 77 | subdirs = matching_subdirs(base_dir, pattern) 78 | module_runs = get_module_runs(subdirs) 79 | # save_prev(log_name) 80 | update_log_file(module_runs, log_name) 81 | 82 | return 0 83 | 84 | 85 | if __name__ == "__main__": 86 | sys.exit(main(sys.argv)) 87 | -------------------------------------------------------------------------------- /scripts/python/vignets2image.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from shapepipe.pipeline import file_io as io 4 | 5 | from astropy.io import fits 6 | 7 | import re 8 | import os 9 | import sys 10 | import glob 11 | import numpy as np 12 | 13 | 14 | def map_vignet(img_arr, dtype): 15 | """Map vignet 16 | Map vignet on one single image. 17 | 18 | Parameters 19 | ---------- 20 | img_arr : numpy.ndarray 21 | Array of vignets to map 22 | dtype : str 23 | dtype of the data 24 | 25 | Returns 26 | ------- 27 | img_map : numpy.ndarray 28 | Array containing all the vignets mapped on one single image 29 | nx : int 30 | Number of objects along one side (assumed square image) 31 | """ 32 | 33 | n_obj = img_arr.shape[0] 34 | xs = img_arr[0].shape[0] 35 | ys = img_arr[0].shape[1] 36 | 37 | nx = int(np.sqrt(n_obj)) 38 | if nx * nx != n_obj: 39 | nx += 1 40 | ny = nx 41 | 42 | img_map = np.ones((xs * nx, ys * ny), dtype=dtype) 43 | 44 | ii = 0 45 | jj = 0 46 | for i in range(n_obj): 47 | if jj > nx - 1: 48 | jj = 0 49 | ii += 1 50 | img_map[ii * xs : (ii + 1) * xs, jj * ys : (jj + 1) * ys] = img_arr[i] 51 | jj += 1 52 | 53 | return img_map, nx 54 | 55 | 56 | def main(argv=None): 57 | 58 | path = "." 59 | pattern = "UNIONS_" 60 | prefix_out = "img_" 61 | 62 | files = glob.glob("{}/{}*.fits".format(path, pattern)) 63 | 64 | for input_path in files: 65 | output_path = "{}{}".format(prefix_out, os.path.basename(input_path)) 66 | 67 | hdu = fits.open(input_path) 68 | 69 | image, nx = map_vignet(hdu[2].data["VIGNET"], "float32") 70 | print("file = {}, nx = {}".format(input_path, nx)) 71 | 72 | fout = io.FITSCatalog( 73 | output_path, open_mode=io.BaseCatalog.OpenMode.ReadWrite 74 | ) 75 | fout.save_as_fits(image, image=True) 76 | 77 | return 0 78 | 79 | 80 | if __name__ == "__main__": 81 | sys.exit(main(sys.argv)) 82 | -------------------------------------------------------------------------------- /scripts/sh/canfar_selection.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Name: canfar_selection.sh 4 | # Description: Create selection of canfar-run tiles for 5 | # post-processing. 6 | # Author: Martin Kilbinger 7 | # Date: 06/2020 8 | 9 | 10 | # Command line arguments 11 | if [ $# != 2 ] && [ $# != 3 ]; then 12 | echo "Usage: $(basename "$0") IN_DIR LIST_PATH [OUT_DIR]" 13 | echo " IN_DIR: input directory with .tgz files" 14 | echo " LIST_PATH: ASCII file with tile IDs" 15 | echo " OUT_DIR: output directory name, default=tile ID file base name" 16 | exit 1 17 | fi 18 | 19 | IN_DIR=$1 20 | LIST_PATH=$2 21 | 22 | if [ $# == 3 ]; then 23 | OUT_DIR=$3 24 | else 25 | OUT_DIR=`echo "${LIST_PATH%.*}"` 26 | fi 27 | 28 | echo "IN_DIR=$IN_DIR" 29 | echo "LIST_PATH=$LIST_PATH" 30 | echo "OUT_DIR=$OUT_DIR" 31 | 32 | 33 | ## Functions 34 | function link_s () { 35 | target=$1 36 | link_name=$2 37 | 38 | if [ -e "$link_name" ]; then 39 | echo "link with name $link_name already exists, skipping..." 40 | else 41 | echo "create link $target <- $link_name" 42 | ln -s $target $link_name 43 | fi 44 | } 45 | 46 | 47 | # Variables 48 | 49 | pwd=`pwd` 50 | 51 | # Tar archive base names 52 | NAMES=( 53 | "psfex" 54 | "psfex_interp_exp" 55 | "setools_mask" 56 | "setools_stat" 57 | "setools_plot" 58 | "final_cat" 59 | "pipeline_flag" 60 | "logs" 61 | ) 62 | 63 | ## Start 64 | if [[ -d $OUT_DIR ]]; then 65 | echo "Error: Output directory $OUT_DIR already exists, exiting..." 66 | exit 2 67 | fi 68 | 69 | mkdir $OUT_DIR 70 | for i in `cat $LIST_PATH`; do 71 | for out in ${NAMES[@]}; do 72 | target=$pwd/$IN_DIR/${out}_$i.tgz 73 | link_name=$OUT_DIR/${out}_$i.tgz 74 | link_s $target $link_name 75 | done 76 | done 77 | -------------------------------------------------------------------------------- /scripts/sh/curl_canfar_monitor_local.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # -H "Accept-Encoding: gzip" faster? 4 | 5 | SSL=~/.ssl/cadcproxy.pem 6 | SESSION=https://ws-uv.canfar.net/skaha/v0/session 7 | 8 | type=$1 9 | 10 | echo "type=$type" 11 | 12 | for session_ID in `cat ids.txt`; do 13 | cmd="curl -E $SSL $SESSION/$session_ID?view=$type" 14 | echo $cmd 15 | $cmd 16 | done 17 | 18 | exit 0 19 | 20 | while [ 1 ]; do 21 | session_ID=`tail -n 1 ids.txt` 22 | cmd="curl -E $SSL $SESSION/$session_ID?view=$type" 23 | echo $cmd 24 | $cmd 25 | done 26 | -------------------------------------------------------------------------------- /scripts/sh/missing_unique.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | FILES=("summary/missing_job_128_ngmix_runner_*.txt") 4 | temp="temp_temp.tmp" 5 | temp2="temp_temp2.tmp" 6 | out="missing_job_128_ngmix_runner_cut.txt" 7 | 8 | i=0 9 | for file in ${FILES[@]}; do 10 | 11 | echo $file $i 12 | 13 | if [ "$i" == "0" ]; then 14 | cp $file $temp 15 | else 16 | comm -12 <(sort $file) <(sort $temp) > $temp2 17 | cp $temp2 $temp 18 | fi 19 | 20 | wc $file $temp 21 | 22 | ((i=i+1)) 23 | 24 | done 25 | 26 | mv $temp $out 27 | rm $temp2 28 | -------------------------------------------------------------------------------- /scripts/sh/post_proc_sp.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Name: post_proc_sp.bash 4 | # Description: Post-process downloaded canfar results and 5 | # creates final merge catalog. 6 | # Author: Martin Kilbinger 7 | # Date: 06/2020 8 | # Package: shapepipe 9 | 10 | # The shapepipe python virtual environment needs to 11 | # be active to run this script. 12 | 13 | 14 | # Command line arguments 15 | 16 | ## Default values 17 | psf='mccd' 18 | 19 | ## Help string 20 | usage="Usage: $(basename "$0") [OPTIONS] 21 | \n\nOptions:\n 22 | -h\tthis message\n 23 | -p, --psf MODEL\n 24 | \tPSF model, one in ['psfex'|'mccd'], default='$psf'\n 25 | " 26 | 27 | ## Parse command line 28 | while [ $# -gt 0 ]; do 29 | case "$1" in 30 | -h) 31 | echo -ne $usage 32 | exit 0 33 | ;; 34 | -p|--psf) 35 | psf="$2" 36 | shift 37 | ;; 38 | *) 39 | echo -ne $usage 40 | exit 1 41 | ;; 42 | esac 43 | shift 44 | done 45 | 46 | ## Check options 47 | if [ "$psf" != "psfex" ] && [ "$psf" != "mccd" ]; then 48 | echo "PSF (option -p) needs to be 'psfex' or 'mccd'" 49 | exit 2 50 | fi 51 | 52 | # Paths 53 | export SP_RUN=. 54 | SP_BASE=$HOME/astro/repositories/github/shapepipe 55 | SP_CONFIG=$SP_BASE/example/cfis 56 | 57 | 58 | # To download results from canfar, use 59 | # 60 | # canfar_download_results.sh 61 | # 62 | # On candide this needs to be done on 63 | # the login node. 64 | 65 | # To Un-tar all .tgz results files, use 66 | # 67 | # $SP_BASE/scripts/sh/untar_results.sh 68 | 69 | 70 | # PSF 71 | 72 | ## Collect all psfinterp results 73 | combine_runs -p $psf -t psf 74 | 75 | ## Merge all psfinterp results and compute PSF residuals 76 | shapepipe_run -c $SP_CONFIG/config_MsPl_$psf.ini 77 | 78 | 79 | # Galaxies 80 | 81 | ## Prepare output directory with links to all 'final_cat' result files 82 | combine_runs 83 | 84 | ## Merge final output files to single mother catalog 85 | input_final=output/run_sp_combined/make_catalog_runner/output 86 | merge_final_cat -i $input_final -p $SP_CONFIG/final_cat.param -v 87 | -------------------------------------------------------------------------------- /scripts/sh/remove_duplicates.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | count=0 4 | for dir in exp_runs/*; do 5 | if compgen -G "$dir/output/run_sp_exp_Sx*" > /dev/null; then 6 | n=`ls -rdtl $dir/output/run_sp_exp_Sx* | wc -l` 7 | #echo $dir $n $count 8 | if [ "$n" == "2" ]; then 9 | n_remove=$((n-1)) 10 | rm -rf `ls -rdt1 $dir/output/run_sp_exp_Sx* | head -n $n_remove` 11 | echo rm -rf `ls -rdt1 $dir/output/run_sp_exp_Sx* | head -n $n_remove` 12 | count=$((count+1)) 13 | fi 14 | fi 15 | done 16 | 17 | echo $count 18 | -------------------------------------------------------------------------------- /scripts/sh/remove_duplicates_tiles.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | count=0 4 | for dir in tile_runs/*; do 5 | if compgen -G "$dir/output//run_sp_tile_PsViSmVi*" > /dev/null; then 6 | n=`ls -rdtl $dir/output/run_sp_tile_PsViSmVi* | wc -l` 7 | if [ "$n" != "2" ]; then 8 | ((n_remove=n-1)) 9 | echo $dir $n $n_remove 10 | rm -rf `ls -rdt1 $dir/output/run_sp_tile_PsViSmVi* | head -n $n_remove` 11 | (count=(count+1)) 12 | fi 13 | fi 14 | done 15 | 16 | echo $count 17 | -------------------------------------------------------------------------------- /scripts/sh/untar_results.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Name: untar_results.bash 4 | # Description: Untar .tgz files = results of ShapePipe runs 5 | # Author: Martin Kilbinger 6 | # Date: 05/2020 7 | # Package: shapepipe 8 | 9 | # Command line arguments 10 | 11 | ## Default values 12 | psf="mccd" 13 | output="" 14 | 15 | ## Help string 16 | usage="Usage: $(basename "$0") [OPTIONS] 17 | \n\nOptions:\n 18 | -h\tthis message\n 19 | -p, --psf MODEL\n 20 | \tPSF model, one in ['psfex'|'mccd'], default='$psf'\n 21 | -o, --output FILE\n 22 | \tOutput file to list untared archives, default None\n 23 | " 24 | 25 | ## Parse command line 26 | while [ $# -gt 0 ]; do 27 | case "$1" in 28 | -h) 29 | echo -ne $usage 30 | exit 0 31 | ;; 32 | -p|--psf) 33 | psf="$2" 34 | shift 35 | ;; 36 | -o|--output) 37 | output="$2" 38 | shift 39 | ;; 40 | *) 41 | echo -ne usage 42 | exit 1 43 | ;; 44 | esac 45 | shift 46 | done 47 | 48 | 49 | NAMES=( 50 | "final_cat" 51 | "setools_mask" 52 | "setools_stat" 53 | "setools_plot" 54 | "pipeline_flag" 55 | ) 56 | 57 | if [ "$psf" == "psfex" ]; then 58 | NAMES+=( 59 | "psfex_interp_exp" 60 | ) 61 | elif [ "$psf" == "mccd" ]; then 62 | NAMES+=( 63 | "mccd_fit_val_runner" 64 | ) 65 | fi 66 | 67 | if [[ "$output" != "" ]]; then 68 | echo "Resetting output file $output" 69 | rm -f $output 70 | fi 71 | 72 | # Check number of files 73 | for out in ${NAMES[@]}; do 74 | echo "$out" 75 | FILES=${out}_*.tgz 76 | n_files=${#FILES[@]} 77 | n_ok=0 78 | n_fail=0 79 | for file in $FILES; do 80 | tar xf $file 81 | res=$? 82 | if [ $res == 0 ]; then 83 | ((n_ok=n_ok+1)) 84 | if [[ "$output" != "" ]]; then 85 | echo $file >> $output 86 | fi 87 | else 88 | ((n_fail=n_fail+1)) 89 | fi 90 | done 91 | echo " $n_files tgz files, $n_ok successful untar commands, $n_fail failures" 92 | done 93 | -------------------------------------------------------------------------------- /src/shapepipe/__init__.py: -------------------------------------------------------------------------------- 1 | """SHAPEPIPE PACKAGE. 2 | 3 | ShapePipe is a galaxy shape measurement pipeline. 4 | 5 | :Author: Samuel Farrens 6 | 7 | """ 8 | 9 | __all__ = ["modules", "pipeline", "utilities"] 10 | 11 | from importlib.metadata import metadata, version 12 | from . import * 13 | 14 | __version__ = version("shapepipe") 15 | __about__ = metadata("shapepipe").get("Summary") 16 | -------------------------------------------------------------------------------- /src/shapepipe/modules/__init__.py: -------------------------------------------------------------------------------- 1 | """SHAPEPIPE MODULES. 2 | 3 | This module contains sub-modules that can be run with ShapePipe. 4 | 5 | :Author: Samuel Farrens 6 | 7 | """ 8 | 9 | import os 10 | 11 | # Get a list of all files and directories in the modules directory 12 | modules_dir = os.listdir(os.path.dirname(os.path.abspath(__file__))) 13 | # List all ShapePipe module packages 14 | __all__ = [dir for dir in modules_dir if dir.endswith("_package")] 15 | # List all ShapePipe module runners 16 | __module_list__ = [file for file in modules_dir if file.endswith("_runner.py")] 17 | -------------------------------------------------------------------------------- /src/shapepipe/modules/execute_example_runner.py: -------------------------------------------------------------------------------- 1 | """EXECUTE MODULE EXAMPLE. 2 | 3 | This module defines methods for an example command line execution module. 4 | 5 | :Author: Samuel Farrens 6 | 7 | """ 8 | 9 | from shapepipe.modules.module_decorator import module_runner 10 | from shapepipe.pipeline.execute import execute 11 | 12 | 13 | @module_runner( 14 | input_module="python_example_runner", 15 | version="1.0", 16 | file_pattern="pyex_output", 17 | file_ext=".cat", 18 | executes="head", 19 | run_method="parallel", 20 | ) 21 | def execute_example_runner( 22 | input_file_list, 23 | run_dirs, 24 | file_number_string, 25 | config, 26 | module_config_sec, 27 | w_log, 28 | ): 29 | """Define The Execute Example Runner.""" 30 | command_line = f"head {input_file_list[0]}" 31 | output_file_name = ( 32 | f'{run_dirs["output"]}/head_output{file_number_string}.txt' 33 | ) 34 | 35 | stdout, stderr = execute(command_line) 36 | 37 | text_file = open(output_file_name, "w") 38 | text_file.write(stdout) 39 | 40 | return stdout, stderr 41 | -------------------------------------------------------------------------------- /src/shapepipe/modules/find_exposures_package/__init__.py: -------------------------------------------------------------------------------- 1 | """FIND EXPOSURES PACKAGE. 2 | 3 | This package contains the module for ``find_exposures``. 4 | 5 | :Author: Martin Kilbinger 6 | 7 | :Parent module: ``get_images_runner`` 8 | 9 | :Input: Tile image 10 | 11 | :Output: Single-exposure ID list 12 | 13 | Description 14 | =========== 15 | 16 | Identify the exposure images that were co-added to produce the tiles 17 | (stacked image). The image names are listed in the tile FITS header, 18 | which is read by this module to extract the names. 19 | 20 | The output ASCII file contains the image base names (without file extension). 21 | 22 | Note that this module is specific for CFIS, in particular the FITS keyword for 23 | identification (``HISTORY``), and the exposure file patterns. 24 | 25 | Module-specific config file entries 26 | =================================== 27 | 28 | COLNUM : int 29 | Column number to find exposure in fits header of tile image for the HISTORY 30 | string 31 | EXP_PREFIX: str 32 | Prefix of exposures 33 | """ 34 | 35 | __all__ = ["find_exposures.py"] 36 | -------------------------------------------------------------------------------- /src/shapepipe/modules/find_exposures_runner.py: -------------------------------------------------------------------------------- 1 | """FIND_EXPOSURES RUNNER. 2 | 3 | Module runner for ``find_exposures``. 4 | 5 | :Author: Martin Kilbinger 6 | 7 | """ 8 | 9 | from shapepipe.modules.find_exposures_package import find_exposures 10 | from shapepipe.modules.module_decorator import module_runner 11 | 12 | 13 | @module_runner( 14 | version="1.1", 15 | file_pattern=["image"], 16 | file_ext=".fits", 17 | depends=["numpy", "astropy", "sip_tpv"], 18 | numbering_scheme="_0", 19 | ) 20 | def find_exposures_runner( 21 | input_file_list, 22 | run_dirs, 23 | file_number_string, 24 | config, 25 | module_config_sec, 26 | w_log, 27 | ): 28 | """Define The Find Exposures Runner.""" 29 | # Get input file name of tile 30 | input_file_name = input_file_list[0] 31 | 32 | # Create output ascii file name 33 | output_path = f'{run_dirs["output"]}/exp_numbers{file_number_string}.txt' 34 | 35 | # Give column number for exposure name in fits header 36 | colnum = config.getint(module_config_sec, "COLNUM") 37 | 38 | # Give the prefix of exposures 39 | exp_prefix = config.get(module_config_sec, "EXP_PREFIX") 40 | # Create find exposures class instance 41 | find_exp_inst = find_exposures.FindExposures( 42 | input_file_name, 43 | output_path, 44 | w_log, 45 | colnum, 46 | exp_prefix, 47 | ) 48 | 49 | # Run processing 50 | find_exp_inst.process() 51 | 52 | # No return objects 53 | return None, None 54 | -------------------------------------------------------------------------------- /src/shapepipe/modules/get_images_package/__init__.py: -------------------------------------------------------------------------------- 1 | """GET IMAGES PACKAGE. 2 | 3 | This package contains the module for ``get_images``. 4 | 5 | :Author: Martin Kilbinger 6 | 7 | :Parent module: None or ``find_exposure_runner`` 8 | 9 | :Input: Image ID list 10 | 11 | :Output: Images 12 | 13 | Description 14 | =========== 15 | 16 | Retrieve images. Download using the ``vos`` python library from the 17 | `VOSpace `_ software, or by creating 18 | symbolic links to exising images. 19 | 20 | Note that the input for this module are (1) an ASCII file specifying the parent 21 | image IDs, and (2) the images to retrieve. 22 | For (1) the standard config entries ``FILE_PATTERN``, ``FILE_EXT``, 23 | ``NUMBERING_SCHEME`` are used. For (2), additional config entries are to be 24 | specified, see below. 25 | 26 | Module-specific config file entries 27 | =================================== 28 | 29 | RETRIEVE : str 30 | Retrieval method, ``vos`` or ``symlink`` 31 | RETRIEVE_OPTIONS : str, optional 32 | Options to pass to retrieval method 33 | N_TRY : int, optional 34 | Ff ``RETRIEVE = vos``, this option specifies the number of attempts to 35 | download an image in case of vos failure; the default value is ``3`` 36 | INPUT_PATH : list 37 | Input path(s) of images 38 | INPUT_FILE_PATTERN : list 39 | Input file pattern(s) with image ID as dummy template 40 | INPUT_FILE_EXT : list 41 | Input file extension(s) 42 | OUTPUT_FILE_PATTERN : list 43 | Output file pattern(s) 44 | INPUT_NUMBERING : str 45 | Input numbering scheme, python regexp 46 | CHECK_EXISTING_DIR : str, optional 47 | If given, search this directory (recursively) for existing images, which 48 | will then not be downloaded 49 | N_EXPECTED : int, optional 50 | If ``CHECK_EXISTING_DIR`` is given, this option specifies the number of 51 | expected images; the default value is ``1`` 52 | 53 | """ 54 | 55 | __all__ = ["get_images.py"] 56 | -------------------------------------------------------------------------------- /src/shapepipe/modules/make_cat_package/__init__.py: -------------------------------------------------------------------------------- 1 | r"""MAKE CATALOGUE PACKAGE. 2 | 3 | This package contains the module for ``make_cat``. 4 | 5 | :Author: Axel Guinot 6 | 7 | :Parent modules: 8 | 9 | - ``sextractor_runner`` 10 | - ``spread_model_runner`` 11 | - ``psfex_interp_runner`` or ``mccd_interp_runner`` 12 | - ``ngmix_runner`` 13 | 14 | :Input: SExtractor catalogues, sqlite catalogue 15 | 16 | :Output: SExtractor catalogue 17 | 18 | Description 19 | =========== 20 | 21 | This module creates a *final* catalogue combining the output of various 22 | previous module runs. This gathers all relevant information on the measured 23 | galaxies for weak-lensing post-processing. This includes galaxy detection and 24 | basic measurement parameters, the PSF model at galaxy positions, the 25 | spread-model classification, and the shape measurement. 26 | 27 | Module-specific config file entries 28 | =================================== 29 | 30 | SM_DO_CLASSIFICATION : bool, optional 31 | Adds spread-model star/galaxy classification flag as the column 32 | ``SPREAD_CLASS`` to the output if ``True`` 33 | SM_STAR_THRESH : float, optional 34 | Threshold :math:`s_{\rm star, thresh}` for star selection; object is 35 | classified as a star if 36 | :math:`|x s + 2 \sigma_s | < s_{\textrm{star, thresh}}` 37 | where :math:`s` is the spread model and :math:`\sigma_s` is the spread 38 | model error; default value is ``0.003`` 39 | SM_GAL_THRESH : float, optional 40 | Threshold :math:`s_{\rm gal, thresh}` for galaxy selection; object is 41 | classified as a galaxy if 42 | :math:`s + 2 \sigma_s > s_{\textrm{gal, thresh}}` where :math:`s` is the 43 | spread model and :math:`\sigma_s` is the spread model error; default value 44 | is ``0.01`` 45 | SHAPE_MEASUREMENT_TYPE : list 46 | Shape measurement method, valid options are ``ngmix`` and/or ``galsim`` 47 | SAVE_PSF_DATA : bool, optional 48 | Save PSF information if ``True``; default value is ``False`` 49 | TILE_LIST : str, optional 50 | Path to list of all tile IDs, used to flag objects in areas of overlap 51 | between tiles 52 | 53 | """ 54 | 55 | __all__ = ["make_cat"] 56 | -------------------------------------------------------------------------------- /src/shapepipe/modules/match_external_package/__init__.py: -------------------------------------------------------------------------------- 1 | """MATCH EXTERNAL PACKAGE. 2 | 3 | This package contains the module for ``match_external``. 4 | 5 | :Authors: Martin Kilbinger, Xavier Jimenez 6 | 7 | :Parent module: ``sextractor_runner`` 8 | 9 | :Input: SExtractor catalogue 10 | 11 | :Output: Matched catalogue 12 | 13 | Description 14 | =========== 15 | 16 | This module matches an external catalogue to a ShapePipe (SExtractor) 17 | catalogue. Matching is performed by checking if the 2D angular separation 18 | between objects is less than a specified tolerance. 19 | 20 | Module-specific config file entries 21 | =================================== 22 | 23 | TOLERANCE : float 24 | Tolerance of matching distance in arcseconds 25 | COL_MATCH : list 26 | Internal SExtractor column names to be matched 27 | HDU : int, optional 28 | Internal SExtractor catalogue HDU number; default is ``2`` 29 | MODE : str 30 | Run mode for module, options are ``CLASSIC`` or ``MULTI-EPOCH`` 31 | EXTERNAL_CAT_PATH : str 32 | Path to the external catalogue 33 | EXTERNAL_COL_MATCH : list 34 | External SExtractor column names to be matched 35 | EXTERNAL_COL_COPY : list 36 | External SExtractor column names to be copied to the matched catalogue 37 | EXTERNAL_HDU : int, optional 38 | External SExtractor catalogue HDU number; default is ``1`` 39 | PREFIX : str, optional 40 | Output file name prefix; default is ``cat_matched`` 41 | MARK_NON_MATCHED : float, optional 42 | If set all objects will be included in the output catalogue and unmatched 43 | objects will be marked with the value specified 44 | OUTPUT_DISTANCE : bool, optional 45 | Option to output the matching distance; default is ``False`` 46 | 47 | """ 48 | 49 | __all__ = ["match_external"] 50 | -------------------------------------------------------------------------------- /src/shapepipe/modules/mccd_fit_runner.py: -------------------------------------------------------------------------------- 1 | """MCCD FIT RUNNER. 2 | 3 | This file is the pipeline fit runner for the MCCD package. 4 | 5 | :Author: Tobias Liaudat 6 | 7 | """ 8 | 9 | import mccd 10 | 11 | from shapepipe.modules.mccd_package import shapepipe_auxiliary_mccd as aux_mccd 12 | from shapepipe.modules.module_decorator import module_runner 13 | 14 | 15 | @module_runner( 16 | version="1.1", 17 | input_module=["mccd_preprocessing_runner"], 18 | file_pattern=["train_star_selection"], 19 | file_ext=[".fits"], 20 | numbering_scheme="-0000000", 21 | depends=["numpy", "mccd", "galsim"], 22 | run_method="parallel", 23 | ) 24 | def mccd_fit_runner( 25 | input_file_list, 26 | run_dirs, 27 | file_number_string, 28 | config, 29 | module_config_sec, 30 | w_log, 31 | ): 32 | """Define The MCCD Fit Runner.""" 33 | # Recover the MCCD config file and its params 34 | config_file_path = config.getexpanded(module_config_sec, "CONFIG_PATH") 35 | mccd_mode = config.get(module_config_sec, "MODE") 36 | verbose = config.getboolean(module_config_sec, "VERBOSE") 37 | 38 | # Parse MCCD config file 39 | mccd_parser = mccd.auxiliary_fun.MCCDParamsParser(config_file_path) 40 | mccd_parser.parse_document() 41 | 42 | # Prepare inputs to run the main fit function 43 | trainstar_path = input_file_list[0] 44 | output_dir = run_dirs["output"] + "/" 45 | saving_name = "fitted_model" 46 | 47 | if mccd_mode == "FIT": 48 | aux_mccd.mccd_fit_pipeline( 49 | trainstar_path=trainstar_path, 50 | file_number_string=file_number_string, 51 | mccd_parser=mccd_parser, 52 | output_dir=output_dir, 53 | verbose=verbose, 54 | saving_name=saving_name, 55 | w_log=w_log, 56 | ) 57 | 58 | else: 59 | raise ValueError('mccd_fit_runner should be called when the MODE is "FIT".') 60 | 61 | # No return objects 62 | return None, None 63 | -------------------------------------------------------------------------------- /src/shapepipe/modules/mccd_val_runner.py: -------------------------------------------------------------------------------- 1 | """MCCD VAL RUNNER. 2 | 3 | This file is the pipeline validation runner for the MCCD package. 4 | 5 | :Author: Tobias Liaudat 6 | 7 | """ 8 | 9 | import mccd 10 | 11 | from shapepipe.modules.mccd_package import shapepipe_auxiliary_mccd as aux_mccd 12 | from shapepipe.modules.module_decorator import module_runner 13 | 14 | 15 | @module_runner( 16 | version="1.1", 17 | input_module=["mccd_preprocessing_runner"], 18 | file_pattern=["fitted_model", "test_star_selection"], 19 | file_ext=[".npy", ".fits"], 20 | numbering_scheme="-0000000", 21 | depends=["numpy", "mccd", "galsim"], 22 | run_method="parallel", 23 | ) 24 | def mccd_val_runner( 25 | input_file_list, 26 | run_dirs, 27 | file_number_string, 28 | config, 29 | module_config_sec, 30 | w_log, 31 | ): 32 | """Define The MCCD Validation Runner.""" 33 | # Recover the MCCD config file and its params 34 | config_file_path = config.getexpanded(module_config_sec, "CONFIG_PATH") 35 | mccd_mode = config.get(module_config_sec, "MODE") 36 | verbose = config.getboolean(module_config_sec, "VERBOSE") 37 | 38 | # Parse MCCD config file 39 | mccd_parser = mccd.auxiliary_fun.MCCDParamsParser(config_file_path) 40 | mccd_parser.parse_document() 41 | 42 | # Prepare inputs to run the main fit function 43 | output_dir = run_dirs["output"] + "/" 44 | val_saving_name = "validation_psf" 45 | 46 | # Extract the MCCD model path 47 | mccd_model_path = input_file_list[0] 48 | # Validation stars are in the second position of the list 49 | teststar_path = input_file_list[1] 50 | 51 | if mccd_mode == "VALIDATION": 52 | 53 | aux_mccd.mccd_validation_pipeline( 54 | teststar_path=teststar_path, 55 | mccd_model_path=mccd_model_path, 56 | mccd_parser=mccd_parser, 57 | output_dir=output_dir, 58 | file_number_string=file_number_string, 59 | w_log=w_log, 60 | val_saving_name=val_saving_name, 61 | ) 62 | 63 | else: 64 | raise ValueError( 65 | "The mccd_val_runner should only be called when the MODE is " 66 | + '"VALIDATION".' 67 | ) 68 | 69 | # No return objects 70 | return None, None 71 | -------------------------------------------------------------------------------- /src/shapepipe/modules/merge_headers_package/__init__.py: -------------------------------------------------------------------------------- 1 | """MERGE HEADERS PACKAGE. 2 | 3 | This package contains the module for ``merge_headers``. 4 | 5 | :Author: Axel Guinot 6 | 7 | :Parent module: ``split_exp_runner`` 8 | 9 | :Input: Numpy binary files (``.npy``) with single-exposure header information 10 | 11 | :Output: Single SQL file with combined header information 12 | 13 | Description 14 | =========== 15 | 16 | This pipeline module merges the WCS information (image transformation and 17 | distortions, computed during astrometrical calibration) for each CCD of 18 | single-exposure images. The merged information is saved as a single SQL file. 19 | 20 | Module-specific config file entries 21 | =================================== 22 | 23 | OUTPUT_PATH : str, optional 24 | Overrides the default module output directory under ``[FILE]:OUTPUT_DIR`` 25 | 26 | """ 27 | 28 | __all__ = ["merge_headers"] 29 | -------------------------------------------------------------------------------- /src/shapepipe/modules/merge_headers_package/merge_headers.py: -------------------------------------------------------------------------------- 1 | """MERGE HEADERS. 2 | 3 | This module merges the output *header* files of the ``split_exp`` 4 | module. It creates a binnary file that contains the WCS of each CCD for each 5 | exposure. 6 | 7 | :Author: Axel Guinot 8 | 9 | """ 10 | 11 | import os 12 | import re 13 | 14 | import numpy as np 15 | from sqlitedict import SqliteDict 16 | 17 | 18 | def merge_headers(input_file_list, output_dir): 19 | """Merge Headers. 20 | 21 | This function opens the files in the input file list and merges them into 22 | a SqliteDict file provided they match the appropriate pattern. 23 | 24 | Parameters 25 | ---------- 26 | input_file_list : list 27 | List of input files 28 | output_dir : str 29 | Output path 30 | 31 | Raises 32 | ------ 33 | TypeError 34 | For invalid ``output_dir`` type 35 | 36 | """ 37 | if not isinstance(output_dir, str): 38 | raise TypeError( 39 | "Output directory for merge headers must be a string " 40 | + f"not {type(output_dir)}." 41 | ) 42 | 43 | # Open SqliteDict file 44 | final_file = SqliteDict(f"{output_dir}/log_exp_headers.sqlite") 45 | # Set matching pattern 46 | pattern = "headers-" 47 | 48 | for file_path in input_file_list: 49 | # Extract file path 50 | file_path_scalar = file_path[0] 51 | # Check file name against pattern 52 | file_name = os.path.split(file_path_scalar)[1] 53 | file_base_name = os.path.splitext(file_name)[0] 54 | pattern_split = re.split(pattern, file_base_name) 55 | if len(pattern_split) < 2: 56 | raise IndexError( 57 | f'Regex "{pattern}" not found in base name "{file_base_name}".' 58 | ) 59 | key = pattern_split[1] 60 | # Load Numpy binary file 61 | final_file[key] = np.load(file_path_scalar, allow_pickle=True) 62 | 63 | # Close file 64 | final_file.commit() 65 | final_file.close() 66 | -------------------------------------------------------------------------------- /src/shapepipe/modules/merge_headers_runner.py: -------------------------------------------------------------------------------- 1 | """MERGE HEADERS RUNNER. 2 | 3 | Module runner for ``merge_headers``. 4 | 5 | :Author: Axel Guinot 6 | 7 | """ 8 | 9 | from shapepipe.modules.merge_headers_package.merge_headers import merge_headers 10 | from shapepipe.modules.module_decorator import module_runner 11 | 12 | 13 | @module_runner( 14 | input_module="split_exp_runner", 15 | version="1.1", 16 | file_pattern=["headers"], 17 | file_ext=[".npy"], 18 | depends=["numpy", "sqlitedict"], 19 | run_method="serial", 20 | ) 21 | def merge_headers_runner( 22 | input_file_list, 23 | run_dirs, 24 | file_number_string, 25 | config, 26 | module_config_sec, 27 | w_log, 28 | ): 29 | """Define The Merge Headers Runner.""" 30 | # Set output directory 31 | output_dir = run_dirs["output"] 32 | if config.has_option(module_config_sec, "OUTPUT_PATH"): 33 | output_dir = config.getexpanded(module_config_sec, "OUTPUT_PATH") 34 | 35 | # Log output directory 36 | w_log.info(f"output_dir = {output_dir}") 37 | 38 | # Merge header files 39 | merge_headers(input_file_list, output_dir) 40 | 41 | w_log.info(f"Merged {len(input_file_list)} input file headers") 42 | 43 | # No return objects 44 | return None, None 45 | -------------------------------------------------------------------------------- /src/shapepipe/modules/merge_sep_cats_package/__init__.py: -------------------------------------------------------------------------------- 1 | r"""MERGE SEP CATS PACKAGE. 2 | 3 | This package contains the module for ``merge_sep_cat``. 4 | 5 | :Author: Martin Kilbinger 6 | 7 | :Parent module: ``ngmix_runner`` 8 | 9 | :Input: SExtractor catalogue 10 | 11 | :Output: SExtractor catalogue 12 | 13 | Description 14 | =========== 15 | 16 | This module merges separate input catalogues. This applies to 17 | a run of the previous module that was split into several (parallel) 18 | sub-processes to save wall-clock time. This is useful for time-consuming 19 | modules, such as galaxy shape measurement e.g. via ``ngmix_runner``. This 20 | module merges the outputs of those sub-processes back into a complete output 21 | catalogue. 22 | 23 | Note that the input catalogues should not contain identical objects. 24 | 25 | Module-specific config file entries 26 | =================================== 27 | 28 | N_SPLIT_MAX : int 29 | Number of input sub-catalogues 30 | WARNING : str, optional 31 | Warning action if one or more input catalogue is missing; default is 32 | ``error`` (one of the actions defined 33 | `here `_) 34 | 35 | """ 36 | 37 | __all__ = ["merge_sep_cats.py"] 38 | -------------------------------------------------------------------------------- /src/shapepipe/modules/merge_sep_cats_runner.py: -------------------------------------------------------------------------------- 1 | """MERGE SEP CATS RUNNER. 2 | 3 | Module runner for ``merge_sep_cats`` 4 | 5 | :Authors: Morgan Schmitz, Axel Guinot, 6 | Martin Kilbinger 7 | 8 | """ 9 | 10 | from shapepipe.modules.merge_sep_cats_package.merge_sep_cats import MergeSep 11 | from shapepipe.modules.module_decorator import module_runner 12 | 13 | 14 | @module_runner( 15 | version="1.1", 16 | input_module="ngmix_runner", 17 | file_pattern=["ngmix"], 18 | file_ext=[".fits"], 19 | depends=["numpy"], 20 | ) 21 | def merge_sep_cats_runner( 22 | input_file_list, 23 | run_dirs, 24 | file_number_string, 25 | config, 26 | module_config_sec, 27 | w_log, 28 | ): 29 | """Define The Merge SEP Catalogues Runner.""" 30 | # Get config entries 31 | n_split_max = config.getint(module_config_sec, "N_SPLIT_MAX") 32 | 33 | file_pattern = config.getlist(module_config_sec, "FILE_PATTERN") 34 | file_ext = config.getlist(module_config_sec, "FILE_EXT") 35 | 36 | if config.has_option(module_config_sec, "WARNING"): 37 | warning = config.get(module_config_sec, "WARNING") 38 | else: 39 | warning = "error" 40 | 41 | # Create merge sep cat class instance 42 | merge_sep_inst = MergeSep( 43 | input_file_list, 44 | file_number_string, 45 | file_pattern, 46 | file_ext, 47 | run_dirs["output"], 48 | n_split_max, 49 | warning, 50 | w_log, 51 | ) 52 | 53 | # Run processing 54 | merge_sep_inst.process() 55 | 56 | # No return objects 57 | return None, None 58 | -------------------------------------------------------------------------------- /src/shapepipe/modules/merge_starcat_package/__init__.py: -------------------------------------------------------------------------------- 1 | """MERGE STAR CATALOGUES PACKAGE. 2 | 3 | This package contains the module for ``merge_starcat``. 4 | 5 | :Author: Tobias Liaudat, Martin Kilbinger 6 | 7 | :Parent module: ``mccd_fit_val_runner``, ``mccd_interp_runner``, 8 | ``mccd_val_runner`` or ``psfex_interp_runner`` 9 | 10 | :Input: PSFEx or MCCD star catalogues 11 | 12 | :Output: Merged star catalogue 13 | 14 | Description 15 | =========== 16 | 17 | This module merges the star catalogues used for building either PSFEx or MCCD 18 | PSF models. 19 | 20 | Module-specific config file entries 21 | =================================== 22 | 23 | PSF_MODEL : str 24 | PSF model used; options are ``psfex`` or ``mccd``, ``setools'' 25 | HDU : int, optional 26 | HDU number of input catalogue table, default is ``1`` for ``mccd`` 27 | and ``2`` for ``psfex`` and ``setools`` 28 | 29 | """ 30 | 31 | __all__ = ["merge_starcat"] 32 | -------------------------------------------------------------------------------- /src/shapepipe/modules/merge_starcat_runner.py: -------------------------------------------------------------------------------- 1 | """MERGE STARCAT RUNNER. 2 | 3 | Module runner for ``merge_starcat``. 4 | 5 | :Author: Tobias Liaudat, Morgan Schmitz, Axel Guinot, Martin Kilbinger 6 | 7 | """ 8 | 9 | from shapepipe.modules.merge_starcat_package import merge_starcat 10 | from shapepipe.modules.module_decorator import module_runner 11 | 12 | 13 | @module_runner( 14 | version="1.1", 15 | input_module=["mccd_fit_val_runner"], 16 | file_pattern=["validation_psf"], 17 | file_ext=[".fits"], 18 | numbering_scheme="-0000000", 19 | depends=["numpy", "astropy"], 20 | run_method="serial", 21 | ) 22 | def merge_starcat_runner( 23 | input_file_list, 24 | run_dirs, 25 | file_number_string, 26 | config, 27 | module_config_sec, 28 | w_log, 29 | ): 30 | """Define The Merge Star Catalogues Runner.""" 31 | # Read config file options 32 | psf_model = config.get(module_config_sec, "PSF_MODEL") 33 | allowed_psf_models = ("psfex", "mccd", "setools") 34 | if psf_model not in allowed_psf_models: 35 | raise ValueError( 36 | f"Invalid config entry PSF_MODEL={psf_model} found, " 37 | + f"needs to be one of {allowed_psf_models}" 38 | ) 39 | 40 | # Get input catalogue HDU number 41 | if config.has_option(module_config_sec, "HDU"): 42 | hdu = config.getint(module_config_sec, "HDU") 43 | else: 44 | hdu = 1 45 | 46 | # Input star catalogue type, to be added to output header for information 47 | if config.has_option(module_config_sec, "INPUT_CAT_TYPE"): 48 | input_cat_type = config.get(module_config_sec, "INPUT_CAT_TYPE") 49 | else: 50 | input_cat_type = None 51 | 52 | # Set output directory 53 | output_dir = run_dirs["output"] 54 | 55 | # Set merge class to use 56 | if psf_model == "mccd": 57 | MSC = merge_starcat.MergeStarCatMCCD 58 | elif psf_model == "psfex": 59 | MSC = merge_starcat.MergeStarCatPSFEX 60 | elif psf_model == "setools": 61 | MSC = merge_starcat.MergeStarCatSetools 62 | 63 | # Create instance of merge class 64 | merge_inst = MSC(input_file_list, output_dir, w_log, hdu_table=hdu, input_cat_type=input_cat_type) 65 | 66 | # Run processing 67 | merge_inst.process() 68 | 69 | # No return objects 70 | return None, None 71 | -------------------------------------------------------------------------------- /src/shapepipe/modules/module_runners.py: -------------------------------------------------------------------------------- 1 | """MODULE RUNNERS. 2 | 3 | This module defines methods for running the pipeline modules. 4 | 5 | :Author: Samuel Farrens 6 | 7 | """ 8 | 9 | from importlib import import_module 10 | 11 | 12 | def get_module_runners(modules): 13 | """Get Module Runners. 14 | 15 | Import the specified module runners. 16 | 17 | Parameters 18 | ---------- 19 | modules : list 20 | List of module names 21 | 22 | Returns 23 | ------- 24 | dict 25 | Dictionary of module runners 26 | 27 | """ 28 | package = "shapepipe.modules" 29 | 30 | module_runners = dict( 31 | [ 32 | ( 33 | module, 34 | getattr( 35 | import_module(f".{module}", package=package), 36 | module, 37 | ), 38 | ) 39 | for module in modules 40 | ] 41 | ) 42 | 43 | return module_runners 44 | -------------------------------------------------------------------------------- /src/shapepipe/modules/ngmix_package/__init__.py: -------------------------------------------------------------------------------- 1 | """NGMIX PACKAGE. 2 | 3 | This package contains the module for ``ngmix``. 4 | 5 | :Author: Axel Guinot 6 | 7 | :Parent modules: 8 | 9 | - ``sextractor_runner`` 10 | - ``psfex_interp_runner`` or ``mccd_interp_runner`` 11 | - ``vignetmaker_runner`` 12 | 13 | :Input: Galaxy image vignets 14 | 15 | :Output: Shape catalogue 16 | 17 | Description 18 | =========== 19 | 20 | This module calls the weak-lensing shape measurment and metacalibration 21 | software NGMIX :cite:`sheldon:15`. In particular the GalSim :cite:`rowe:15` 22 | fitting method of NGXMIX is called to obtain galaxy shape measurements. 23 | The metacalibration routines of NGMIX are also called to provide all of the 24 | measurements required to calibrate the shear values. 25 | 26 | Module-specific config file entries 27 | =================================== 28 | 29 | MAG_ZP : float 30 | Photometric zero point 31 | PIXEL_SCALE : float 32 | Pixel scale in arcseconds 33 | LOG_WCS : str 34 | Path to world coordinate system log file (``*sqlite``) 35 | ID_OBJ_MIN : int 36 | ID of first galaxy object to be processed; not used if set to ``-1`` 37 | (default) 38 | ID_OBJ_MAX : int 39 | ID of last galaxy object to be processed; not used if set to ``-1`` 40 | (default) 41 | 42 | """ 43 | 44 | __all__ = ["ngmix"] 45 | -------------------------------------------------------------------------------- /src/shapepipe/modules/pastecat_package/__init__.py: -------------------------------------------------------------------------------- 1 | """PASTE CATALOGUES PACKAGE. 2 | 3 | This package contains the module for ``pastecat``. 4 | 5 | :Authors: Martin Kilbinger, Axel Guinot 6 | 7 | :Parent module: ``sextractor_runner`` 8 | 9 | :Input: SExtractor catalogues 10 | 11 | :Output: Pasted catalogue 12 | 13 | Description 14 | =========== 15 | 16 | This module pastes (or concatenates) a series of SExtractor catalogues into 17 | as single output catalogue. 18 | 19 | Module-specific config file entries 20 | =================================== 21 | 22 | CHECK_COL_NAME : str, optional 23 | SExtractor column name to use to ensure consistency between the number of 24 | rows for the catalogues to be pasted 25 | HDU : list, optional 26 | HDU numbers of the input catalogues; default is set to ``2`` for all 27 | catalogues 28 | PREFIX : str, optional 29 | Ouput file name prefix; default is ``cat_pasted`` 30 | EXT_NAME : list, optional 31 | List of HDU extension names; default is to use the input file names 32 | 33 | """ 34 | 35 | __all__ = ["pastecat"] 36 | -------------------------------------------------------------------------------- /src/shapepipe/modules/psfex_interp_package/__init__.py: -------------------------------------------------------------------------------- 1 | r"""PSFEX INTERPOLATION PACKAGE. 2 | 3 | This package contains the module for ``psfex_interp``. 4 | 5 | :Author: Axel Guinot 6 | 7 | :Parent modules: 8 | 9 | - ``psfex_runner`` 10 | - ``setools_runner`` 11 | 12 | :Input: PSFEx PSF model 13 | 14 | :Output: PSFEx PSF estimates at interpolated positions 15 | 16 | Description 17 | =========== 18 | 19 | This module interpolates the PSFEx PSF model. 20 | 21 | Module-specific config file entries 22 | =================================== 23 | 24 | MODE : str 25 | Run mode for module, options are ``CLASSIC``, ``MULTI-EPOCH`` 26 | or ``VALIDATION`` 27 | POSITION_PARAMS : list 28 | List of position parameter value names in the SExtractor output catalogue 29 | GET_SHAPES : bool 30 | Option to compute shapes for the PSF model 31 | STAR_THRESH : int 32 | Threshold of stars under which the PSF is not interpolated 33 | CHI2_THRESH : int 34 | Threshold for chi squared (:math:`\chi^2`) 35 | ME_DOT_PSF_DIR : str 36 | Module name of last run producing PSFEx PSF model files, for multi-epoch 37 | processing. The specifier "last:" is not required 38 | ME_DOT_PSF_PATTERN : str 39 | Input file name pattern for PSFEx PSF model files, for multi-epoch 40 | processing 41 | ME_LOG_WCS : str 42 | Path to world coordinate system log file (``*sqlite``) 43 | 44 | """ 45 | 46 | __all__ = ["psfex_interp"] 47 | -------------------------------------------------------------------------------- /src/shapepipe/modules/psfex_package/__init__.py: -------------------------------------------------------------------------------- 1 | """PSFEX PACKAGE. 2 | 3 | This package contains the module for ``psfex``. 4 | 5 | :Authors: Axel Guinot, Tobias Liaudat 6 | 7 | :Parent modules: 8 | 9 | - ``sextractor_runner`` 10 | - ``setools_runner`` 11 | 12 | :Input: SExtractor catalogue 13 | 14 | :Output: PSFEx PSF model 15 | 16 | Description 17 | =========== 18 | 19 | This module calls the community standard point spread function (PSF) modelling 20 | software PSFEx :cite:`bertin:11` to build a PSF model. 21 | 22 | This module requires a previous run of SExtractor. 23 | 24 | Module-specific config file entries 25 | =================================== 26 | 27 | EXEC_PATH : str, optional 28 | Full path to the PSFEx executable (``psfex``) on the system; if not set the 29 | version controlled PSFEx installation in the ShapePipe environment 30 | will be used 31 | DOT_PSFEX_FILE : str 32 | Full path to the ``.psfex`` configuration file for PSFEx 33 | CHECKIMAGE : list, optional 34 | Check-image types, i.e. diagnostic FITS images that are created on output 35 | 36 | """ 37 | 38 | __all__ = ["psfex_script"] 39 | -------------------------------------------------------------------------------- /src/shapepipe/modules/psfex_runner.py: -------------------------------------------------------------------------------- 1 | """PSFEX RUNNER. 2 | 3 | Module runner for ``psfex``. 4 | 5 | :Author: Axel Guinot 6 | 7 | """ 8 | 9 | from shapepipe.modules.module_decorator import module_runner 10 | from shapepipe.modules.psfex_package.psfex_script import PSFExCaller 11 | from shapepipe.pipeline.execute import execute 12 | 13 | 14 | @module_runner( 15 | input_module="setools_runner", 16 | version="1.0", 17 | file_pattern=["star_selection"], 18 | file_ext=[".fits"], 19 | executes="psfex", 20 | ) 21 | def psfex_runner( 22 | input_file_list, 23 | run_dirs, 24 | file_number_string, 25 | config, 26 | module_config_sec, 27 | w_log, 28 | ): 29 | """Define The PSFEx Runner.""" 30 | # Extract psfex run configurations 31 | if config.has_option(module_config_sec, "EXEC_PATH"): 32 | psfex_executable_path = config.getexpanded( 33 | module_config_sec, "EXEC_PATH" 34 | ) 35 | else: 36 | psfex_executable_path = "psfex" 37 | output_dir = run_dirs["output"] 38 | 39 | outcatalog_name = f"{output_dir}/psfex_cat{file_number_string}.cat" 40 | 41 | psfex_config_file = config.getexpanded(module_config_sec, "DOT_PSFEX_FILE") 42 | 43 | input_file_path = input_file_list[0] 44 | 45 | # Check image options 46 | if config.has_option(module_config_sec, "CHECKIMAGE"): 47 | check_image_list = config.getlist(module_config_sec, "CHECKIMAGE") 48 | else: 49 | check_image_list = [""] 50 | 51 | # Create psfex caller class instance 52 | psfex_inst = PSFExCaller( 53 | psfex_executable_path, 54 | input_file_path, 55 | psfex_config_file, 56 | output_dir, 57 | outcatalog_name, 58 | check_image_list, 59 | ) 60 | 61 | # Generate psfex command line 62 | command_line = psfex_inst.generate_command() 63 | w_log.info(f"Running command '{command_line}'") 64 | 65 | # Execute command line 66 | stderr, stdout = execute(command_line) 67 | 68 | # Parse psfex errors 69 | stdout, stderr = PSFExCaller.parse_errors(stderr, stdout) 70 | 71 | # Return stdout and stderr 72 | return stdout, stderr 73 | -------------------------------------------------------------------------------- /src/shapepipe/modules/python_example_package/__init__.py: -------------------------------------------------------------------------------- 1 | """PYTHON EXAMPLE PACKAGE. 2 | 3 | This package contains the module for ``python_example``. 4 | 5 | :Author: Samuel Farrens 6 | 7 | :Parent module: None 8 | 9 | :Input: Dummy ``.txt`` files 10 | 11 | :Output: Dummy ``.cat`` files 12 | 13 | .. warning:: 14 | This is a example module. It should only be used for running tests or as a 15 | reference for developing new modules. 16 | 17 | Description 18 | =========== 19 | 20 | Example package structure for a Python module. 21 | 22 | Module-specific config file entries 23 | =================================== 24 | 25 | MESSAGE : str 26 | Message to be included in module logs 27 | 28 | """ 29 | 30 | __all__ = ["python_example"] 31 | -------------------------------------------------------------------------------- /src/shapepipe/modules/python_example_package/python_example.py: -------------------------------------------------------------------------------- 1 | """PYTHON EXAMPLE. 2 | 3 | This module contains an example Python class. 4 | 5 | :Author: Samuel Farrens 6 | 7 | .. warning:: 8 | This is a example module. It should only be used for running tests or as a 9 | reference for developing new modules. 10 | 11 | """ 12 | 13 | import time 14 | 15 | from numpy.random import randint 16 | 17 | 18 | class PythonExample: 19 | """Python Example. 20 | 21 | An example Python class. 22 | 23 | Parameters 24 | ---------- 25 | sleep_time : int 26 | Sleep time in seconds 27 | 28 | """ 29 | 30 | def __init__(self, sleep_time=None): 31 | 32 | if not isinstance(sleep_time, type(None)): 33 | self.sleep_time = sleep_time 34 | 35 | else: 36 | self.sleep_time = randint(1, 10) 37 | 38 | def _wait(self): 39 | """Wait. 40 | 41 | Wait for :math:`n` seconds. 42 | 43 | """ 44 | time.sleep(self.sleep_time) 45 | 46 | def _read_file(self, file_name): 47 | """Read File. 48 | 49 | Read input file content. 50 | 51 | Parameters 52 | ---------- 53 | file_name : str 54 | Name of file to read 55 | 56 | Returns 57 | ------- 58 | str 59 | Content of file 60 | 61 | """ 62 | with open(file_name) as data_file: 63 | content = data_file.read().replace("\n", "") 64 | 65 | return content 66 | 67 | def read_files(self, file_name1, file_name2): 68 | """Read Files. 69 | 70 | Read two input files. 71 | 72 | Parameters 73 | ---------- 74 | file_name1 : str 75 | Name of first file 76 | file_name2 : str 77 | Name of second file 78 | 79 | """ 80 | self._wait() 81 | content1 = self._read_file(file_name1) 82 | content2 = self._read_file(file_name2) 83 | 84 | self.content = f"{content1} and {content2}" 85 | 86 | def write_file(self, file_name, message): 87 | """Write File. 88 | 89 | Write content to file. 90 | 91 | Parameters 92 | ---------- 93 | file_name : str 94 | Name of output file 95 | Message : str 96 | Content to write to file 97 | 98 | """ 99 | new_content = message + str(self.content) 100 | 101 | text_file = open(file_name, "w") 102 | text_file.write(new_content) 103 | text_file.close() 104 | -------------------------------------------------------------------------------- /src/shapepipe/modules/python_example_runner.py: -------------------------------------------------------------------------------- 1 | """PYTHON MODULE EXAMPLE. 2 | 3 | This module defines methods for an example Python module. 4 | 5 | :Author: Samuel Farrens 6 | 7 | """ 8 | 9 | from shapepipe.modules.module_decorator import module_runner 10 | from shapepipe.modules.python_example_package import python_example 11 | 12 | 13 | @module_runner( 14 | version="1.1", 15 | file_pattern=["numbers", "letters"], 16 | file_ext=".txt", 17 | depends=[ 18 | "numpy", 19 | "astropy", 20 | "galsim", 21 | "joblib", 22 | "mccd", 23 | "ngmix", 24 | "pandas", 25 | "pysap", 26 | "scipy", 27 | "sf_tools", 28 | "sip_tpv", 29 | "sqlitedict", 30 | "treecorr", 31 | ], 32 | run_method="parallel", 33 | ) 34 | def python_example_runner( 35 | input_file_list, 36 | run_dirs, 37 | file_number_string, 38 | config, 39 | module_config_sec, 40 | w_log, 41 | ): 42 | """Define The Python Example Runner.""" 43 | # Set output file name 44 | output_file_name = f'{run_dirs["output"]}/pyex_output{file_number_string}.cat' 45 | 46 | # Retrieve log message from config file 47 | message = config.get(module_config_sec, "MESSAGE") 48 | 49 | # Create an instance of the Python example class 50 | py_ex_inst = python_example.PythonExample(0) 51 | 52 | # Read input files 53 | py_ex_inst.read_files(*input_file_list) 54 | 55 | # Write output files 56 | py_ex_inst.write_file(output_file_name, message) 57 | 58 | # Return file content and no stderr 59 | return py_ex_inst.content, None 60 | -------------------------------------------------------------------------------- /src/shapepipe/modules/random_cat_package/__init__.py: -------------------------------------------------------------------------------- 1 | """RANDOM CATALOGUE PACKAGE. 2 | 3 | This package contains the module for ``random_cat``. 4 | 5 | :Author: Martin Kilbinger 6 | 7 | :Parent module: None 8 | 9 | :Input: Images and masks 10 | 11 | :Output: Random catalogue FITS file 12 | 13 | Description 14 | =========== 15 | 16 | This module creates a random catalogue, and computes the tile area accounting 17 | for overlapping and masked regions. 18 | 19 | Module-specific config file entries 20 | =================================== 21 | 22 | N_RANDOM : float 23 | The number of random objects requested on output 24 | DENSITY : bool, optional 25 | Option to interpret the number of random objects per square degree; the 26 | default is ``False`` 27 | SAVE_MASK_AS_HEALPIX : bool 28 | Output healpix mask if ``True`` 29 | HEALPIX_OUT_FILE_BASE : str, optional 30 | Output halpix mask file base name; used only if SAVE_MASK_AS_HEALPIX is 31 | ``True`` 32 | HEALPIX_OUT_NSIDE : int, optional 33 | Output healpix mask nside; used only if SAVE_MASK_AS_HEALPIX is ``True`` 34 | 35 | """ 36 | 37 | __all__ = ["random_cat.py"] 38 | -------------------------------------------------------------------------------- /src/shapepipe/modules/random_cat_runner.py: -------------------------------------------------------------------------------- 1 | """RANDOM CAT RUNNER. 2 | 3 | Module runner for ``random_cat``. 4 | 5 | :Author: Martin Kilbinger 6 | 7 | """ 8 | 9 | from shapepipe.modules.module_decorator import module_runner 10 | from shapepipe.modules.random_cat_package.random_cat import RandomCat 11 | 12 | 13 | @module_runner( 14 | version="1.1", 15 | file_pattern=["image", "pipeline_flag"], 16 | file_ext=[".fits", "fits"], 17 | depends=["astropy"], 18 | numbering_scheme="_0", 19 | ) 20 | def random_cat_runner( 21 | input_file_list, 22 | run_dirs, 23 | file_number_string, 24 | config, 25 | module_config_sec, 26 | w_log, 27 | ): 28 | """Define The Random Catalogue Runner.""" 29 | # Get input file names of image and mask 30 | input_image_name = input_file_list[0] 31 | input_mask_name = input_file_list[1] 32 | 33 | # Set output file name 34 | if config.has_option(module_config_sec, "OUTPUT_FILE_PATTERN"): 35 | output_file_pattern = config.get( 36 | module_config_sec, "OUTPUT_FILE_PATTERN" 37 | ) 38 | else: 39 | output_file_pattern = "random_cat" 40 | 41 | # Get number of random objects requested on output 42 | n_rand = config.getfloat(module_config_sec, "N_RANDOM") 43 | 44 | # Flag whether n_rand is total (DENSITY=False, default) 45 | # or per square degree (DENSITY=True) 46 | if config.has_option(module_config_sec, "DENSITY"): 47 | density = config.getboolean(module_config_sec, "DENSITY") 48 | else: 49 | density = False 50 | 51 | # Get healpix output options 52 | save_mask_as_healpix = config.getboolean( 53 | module_config_sec, "SAVE_MASK_AS_HEALPIX" 54 | ) 55 | if save_mask_as_healpix: 56 | healpix_options = {} 57 | for option_trunc in ['FILE_BASE', 'OUT_NSIDE']: 58 | option = f'HEALPIX_OUT_{option_trunc}' 59 | healpix_options[option_trunc] = config.get( 60 | module_config_sec, option 61 | ) 62 | # Create rand cat class instance 63 | rand_cat_inst = RandomCat( 64 | input_image_name, 65 | input_mask_name, 66 | run_dirs["output"], 67 | file_number_string, 68 | output_file_pattern, 69 | n_rand, 70 | density, 71 | w_log, 72 | healpix_options, 73 | ) 74 | 75 | # Run processing 76 | rand_cat_inst.process() 77 | 78 | # No return objects 79 | return None, None 80 | -------------------------------------------------------------------------------- /src/shapepipe/modules/serial_example_runner.py: -------------------------------------------------------------------------------- 1 | """SERIAL MODULE EXAMPLE. 2 | 3 | This module defines methods for an example serial module. 4 | 5 | :Author: Samuel Farrens 6 | 7 | """ 8 | 9 | from shapepipe.modules.module_decorator import module_runner 10 | 11 | 12 | class Dummy(object): 13 | """Dummy Class.""" 14 | 15 | def __init__(self): 16 | 17 | pass 18 | 19 | def _read_file(self, file_name): 20 | """Read File.""" 21 | with open(file_name) as data_file: 22 | content = data_file.read().replace("\n", "") 23 | 24 | return content 25 | 26 | def read_files(self, input_file_list): 27 | """Read Files.""" 28 | self.content = "" 29 | 30 | for file_list in input_file_list: 31 | for file_name in file_list: 32 | self.content += self._read_file(file_name) 33 | 34 | def write_file(self, file_name): 35 | """Write Files.""" 36 | text_file = open(file_name, "w") 37 | text_file.write(self.content) 38 | text_file.close() 39 | 40 | 41 | @module_runner( 42 | version="1.1", 43 | input_module="python_example_runner", 44 | file_pattern=["numbers", "letters", "pyex_output"], 45 | file_ext=[".txt", ".txt", ".cat"], 46 | depends="numpy", 47 | run_method="serial", 48 | ) 49 | def serial_example_runner( 50 | input_file_list, 51 | run_dirs, 52 | file_number_string, 53 | config, 54 | module_config_sec, 55 | w_log, 56 | ): 57 | """Define The Serial Example Runner.""" 58 | output_file_name = f'{run_dirs["output"]}/serial_outputfile_number_string.cat' 59 | 60 | inst = Dummy() 61 | inst.read_files(input_file_list) 62 | inst.write_file(output_file_name) 63 | 64 | return inst.content, None 65 | -------------------------------------------------------------------------------- /src/shapepipe/modules/setools_runner.py: -------------------------------------------------------------------------------- 1 | """SETOOLS RUNNER. 2 | 3 | Module runner for ``setools``. 4 | 5 | :Author: Axel Guinot 6 | 7 | """ 8 | 9 | from shapepipe.modules.module_decorator import module_runner 10 | from shapepipe.modules.setools_package.setools import SETools 11 | 12 | 13 | @module_runner( 14 | input_module="sextractor_runner", 15 | version="1.1", 16 | file_pattern=["sexcat"], 17 | file_ext=[".fits"], 18 | depends=["numpy", "matplotlib"], 19 | ) 20 | def setools_runner( 21 | input_file_list, 22 | run_dirs, 23 | file_number_string, 24 | config, 25 | module_config_sec, 26 | w_log, 27 | ): 28 | """Define The SETools Runner.""" 29 | # Get path to setools configuration file 30 | config_file = config.getexpanded(module_config_sec, "SETOOLS_CONFIG_PATH") 31 | 32 | # Create instance of SETools 33 | se_inst = SETools( 34 | input_file_list[0], 35 | run_dirs["output"], 36 | file_number_string, 37 | config_file, 38 | ) 39 | 40 | # Process inputs 41 | se_inst.process(w_log) 42 | 43 | # No return objects 44 | return None, None 45 | -------------------------------------------------------------------------------- /src/shapepipe/modules/split_exp_package/__init__.py: -------------------------------------------------------------------------------- 1 | """SPLIT EXPOSURE PACKAGE. 2 | 3 | This package contains the module for ``split_exp``. 4 | 5 | :Author: Axel Guinot 6 | 7 | :Parent module: ``get_images_runner`` 8 | 9 | :Input: Single-exposure image, weight, or flag file 10 | 11 | :Output: Single-exposure single-HDU file; header numpy binary file (``.npy``) 12 | if input is an image 13 | 14 | Description 15 | =========== 16 | 17 | This module splits up single-exposure FITS files. 18 | Each CCD of a single-exposure mosaic image is stored in a HDU. This module 19 | saves each HDU in a separate single-exposure single-HDU FITS file, such 20 | that each resulting file only contains data from one CCD. 21 | 22 | Module-specific config file entries 23 | =================================== 24 | 25 | OUTPUT_SUFFIX : list 26 | Output file name prefixes. Special strings are: 27 | 28 | ``image``: 29 | 30 | 1. Header is saved to numpy binary file (``.npy``); 31 | 2. Header WCS is saved in output FITS file header; 32 | 3. Geader WCS coordinates are transformed from pv to sip using the 33 | ``sip_tpv`` package 34 | 35 | ``flag``: data is save as ``int16`` 36 | 37 | N_HDU : int 38 | Number of HDUs (CCDs) of the input mosaic FITS file 39 | 40 | """ 41 | 42 | __all__ = ["split_exp.py"] 43 | -------------------------------------------------------------------------------- /src/shapepipe/modules/split_exp_runner.py: -------------------------------------------------------------------------------- 1 | """SPLIT EXP RUNNER. 2 | 3 | Module runner for ``split_exp``. 4 | 5 | :Author: Axel Guinot 6 | 7 | """ 8 | 9 | from shapepipe.modules.module_decorator import module_runner 10 | from shapepipe.modules.split_exp_package.split_exp import SplitExposures 11 | 12 | 13 | @module_runner( 14 | version="1.1", 15 | input_module="get_images_runner", 16 | file_pattern=["image", "weight", "flag"], 17 | file_ext=[".fz", ".fz", ".fz"], 18 | depends=["numpy", "astropy", "sip_tpv"], 19 | run_method="parallel", 20 | ) 21 | def split_exp_runner( 22 | input_file_list, 23 | run_dirs, 24 | file_number_string, 25 | config, 26 | module_config_sec, 27 | w_log, 28 | ): 29 | """Define The Split Exposures Runner.""" 30 | # Get file suffix 31 | output_suffix = config.getlist(module_config_sec, "OUTPUT_SUFFIX") 32 | # Get the number of HDUs 33 | n_hdu = config.getint(module_config_sec, "N_HDU") 34 | 35 | # Create split exposures class instance 36 | split_inst = SplitExposures( 37 | input_file_list, 38 | run_dirs["output"], 39 | file_number_string, 40 | output_suffix, 41 | n_hdu, 42 | ) 43 | 44 | # Process splitting 45 | split_inst.process() 46 | 47 | # No return objects 48 | return None, None 49 | -------------------------------------------------------------------------------- /src/shapepipe/modules/spread_model_package/__init__.py: -------------------------------------------------------------------------------- 1 | """SPREAD MODEL PACKAGE. 2 | 3 | This package contains the module for ``spread_model``. 4 | 5 | :Author: Axel Guinot 6 | 7 | :Parent modules: 8 | 9 | - ``sextractor_runner`` 10 | - ``psfex_interp_runner`` or ``mccd_interp_runner`` 11 | and ``vignetmaker_runner`` 12 | 13 | :Input: SExtractor output catalogue, PSFEx output catalogue and vignet files 14 | 15 | :Output: Updated SExtractor or new FITS file 16 | 17 | Description 18 | =========== 19 | 20 | This module refines the galaxy sample that will be used for shape measurement 21 | using the spread model method of :cite:`desai:12` and :cite:`mohr:12`. 22 | 23 | Module-specific config file entries 24 | =================================== 25 | 26 | PREFIX : str, optional 27 | Ouput file prefix 28 | PIXEL_SCALE : float 29 | Pixel scale in arcseconds 30 | OUTPUT_MODE : str 31 | Output mode, options are ``add`` to add the outputs to the input 32 | SExtractor catalogue or ``new`` to generte a new output catalogue 33 | 34 | """ 35 | 36 | __all__ = ["spread_model"] 37 | -------------------------------------------------------------------------------- /src/shapepipe/modules/spread_model_runner.py: -------------------------------------------------------------------------------- 1 | """SPREAD MODEL RUNNER. 2 | 3 | Module runner for ``spread_model``. 4 | 5 | :Author: Axel Guinot 6 | 7 | """ 8 | 9 | from shapepipe.modules.module_decorator import module_runner 10 | from shapepipe.modules.spread_model_package.spread_model import SpreadModel 11 | 12 | 13 | @module_runner( 14 | version="1.1", 15 | input_module=[ 16 | "sextractor_runner", 17 | "psfex_interp_runner", 18 | "vignetmaker_runner", 19 | ], 20 | file_pattern=["sexcat", "galaxy_psf", "weight_vign"], 21 | file_ext=[".fits", ".sqlite", ".fits"], 22 | depends=["numpy", "galsim"], 23 | run_method="parallel", 24 | ) 25 | def spread_model_runner( 26 | input_file_list, 27 | run_dirs, 28 | file_number_string, 29 | config, 30 | module_config_sec, 31 | w_log, 32 | ): 33 | """Define The Spread Model Runner.""" 34 | # Get input files 35 | sex_cat_path, psf_cat_path, weight_cat_path = input_file_list 36 | 37 | # Get file prefix (optional) 38 | if config.has_option(module_config_sec, "PREFIX"): 39 | prefix = config.get(module_config_sec, "PREFIX") 40 | if (prefix.lower() != "none") & (prefix != ""): 41 | prefix = prefix + "_" 42 | else: 43 | prefix = "" 44 | else: 45 | prefix = "" 46 | 47 | # Get pixel scale and output mode 48 | pixel_scale = config.getfloat(module_config_sec, "PIXEL_SCALE") 49 | output_mode = config.get(module_config_sec, "OUTPUT_MODE") 50 | 51 | # Set output file path 52 | file_name = f"{prefix}sexcat_sm{file_number_string}.fits" 53 | output_path = f'{run_dirs["output"]}/{file_name}' 54 | 55 | # Create spread model class instance 56 | sm_inst = SpreadModel( 57 | sex_cat_path, 58 | psf_cat_path, 59 | weight_cat_path, 60 | output_path, 61 | pixel_scale, 62 | output_mode, 63 | ) 64 | 65 | # Process spread model computation 66 | sm_inst.process() 67 | 68 | # No return objects 69 | return None, None 70 | -------------------------------------------------------------------------------- /src/shapepipe/modules/uncompress_fits_package/__init__.py: -------------------------------------------------------------------------------- 1 | """UNCOMPRESS FITS PACKAGE. 2 | 3 | This package contains the module for ``uncompress_fits``. 4 | 5 | :Author: Axel Guinot, Martin Kilbinger 6 | 7 | :Parent module: ``get_images_runner`` 8 | 9 | :Input: Compressed FITS file 10 | 11 | :Ouput: Uncompressed FITS file 12 | 13 | Description 14 | =========== 15 | 16 | This module uncompresses FITS files (e.g. images, weights, flags), and ignores 17 | dummy HDUs with no image data. The compressed FITS weight images contain 18 | a primary empty HDU #0 that will interfere with further processing with 19 | ShapePipe. The uncompressed image is saved as FITS file with a single image 20 | HDU. 21 | 22 | Module-specific config file entries 23 | =================================== 24 | 25 | HDU_DATA : int, optional 26 | HDU number of input data; the default value is ``0`` 27 | OUTPUT_PATTERN : str 28 | Output file pattern 29 | 30 | """ 31 | 32 | __all__ = ["uncompress_fits"] 33 | -------------------------------------------------------------------------------- /src/shapepipe/modules/uncompress_fits_package/uncompress_fits.py: -------------------------------------------------------------------------------- 1 | """UNCOMPRESS FITS. 2 | 3 | This module uncompresses FITS images and saves them as a single-HDU FITS file. 4 | 5 | :Author: Axel Guinot, Martin Kilbinger 6 | 7 | """ 8 | 9 | from astropy.io import fits 10 | 11 | 12 | class Uncompress(object): 13 | """Uncompress. 14 | 15 | This class handles the uncompress process of compressed FITS files. 16 | 17 | Parameters 18 | ---------- 19 | input_file_list : numpy.ndarray 20 | Input files 21 | output_pattern_list : numpy.ndarray 22 | Output file pattern 23 | output_dir : str 24 | Output directory 25 | file_number_string : str 26 | Image numbering (pipeline ID string) 27 | data_hdu : int 28 | Input HDU number 29 | 30 | """ 31 | 32 | def __init__( 33 | self, 34 | input_file_list, 35 | output_pattern_list, 36 | output_dir, 37 | file_number_string, 38 | data_hdu, 39 | ): 40 | 41 | self._input_file_list = input_file_list 42 | self._output_pattern_list = output_pattern_list 43 | self._output_dir = output_dir 44 | self._file_number_string = file_number_string 45 | self._data_hdu = data_hdu 46 | 47 | def process(self): 48 | """Process. 49 | 50 | Main function to process uncompressing. 51 | 52 | """ 53 | # Go through input list 54 | for idx in range(len(self._input_file_list)): 55 | 56 | # Get data and header 57 | data = fits.getdata(self._input_file_list[idx], self._data_hdu) 58 | header = fits.getheader(self._input_file_list[idx], self._data_hdu) 59 | 60 | # Create and write new FITS file with that HDU only 61 | hdu = fits.PrimaryHDU(data, header) 62 | hdul = fits.HDUList([hdu]) 63 | hdul.writeto( 64 | f"{self._output_dir}/" 65 | + f"{self._output_pattern_list[idx]}" 66 | + f"{self._file_number_string}.fits" 67 | ) 68 | -------------------------------------------------------------------------------- /src/shapepipe/modules/uncompress_fits_runner.py: -------------------------------------------------------------------------------- 1 | """UNCOMPRESS FITS RUNNER. 2 | 3 | Module runner for ``uncompress_fits``. 4 | 5 | :Author: Axel Guinot, Martin Kilbinger 6 | 7 | """ 8 | 9 | from shapepipe.modules.module_decorator import module_runner 10 | from shapepipe.modules.uncompress_fits_package import uncompress_fits 11 | 12 | 13 | @module_runner( 14 | version="1.1", 15 | file_pattern=["image"], 16 | file_ext=[".fits"], 17 | numbering_scheme="_0", 18 | ) 19 | def uncompress_fits_runner( 20 | input_file_list, 21 | run_dirs, 22 | file_number_string, 23 | config, 24 | module_config_sec, 25 | w_log, 26 | ): 27 | """Define The Uncompress Fits Runner.""" 28 | # Get HDU number of input image data 29 | if config.has_option(module_config_sec, "HDU_DATA"): 30 | data_hdu = config.getint(module_config_sec, "HDU_DATA") 31 | else: 32 | data_hdu = 0 33 | 34 | # Get output patterns 35 | output_pattern_list = config.getlist(module_config_sec, "OUTPUT_PATTERN") 36 | 37 | # Check consistency of input and output list lengths 38 | if len(input_file_list) != len(output_pattern_list): 39 | raise ValueError( 40 | f"Lists INPUT_PATH ({len(input_file_list)})" 41 | + f" and OUTPUT_PATTERN ({len(output_pattern_list)})" 42 | + "need to be of equal length." 43 | ) 44 | 45 | # Create instance of uncompress 46 | uncompress_inst = uncompress_fits.Uncompress( 47 | input_file_list, 48 | output_pattern_list, 49 | run_dirs["output"], 50 | file_number_string, 51 | data_hdu, 52 | ) 53 | 54 | # Process uncompressing 55 | uncompress_inst.process() 56 | 57 | # No return objects 58 | return None, None 59 | -------------------------------------------------------------------------------- /src/shapepipe/modules/vignetmaker_package/__init__.py: -------------------------------------------------------------------------------- 1 | """VIGNET MAKER PACKAGE. 2 | 3 | This package contains the module(s) for ``vignetmaker``. 4 | 5 | :Author: Axel Guinot 6 | 7 | :Parent module: ``sextractor_runner`` 8 | 9 | :Input: SExtractor output files 10 | 11 | :Output: Vignet FITS files 12 | 13 | Description 14 | =========== 15 | 16 | This module generates vignets (or postage stamps) around galaxy positions from 17 | SExtractor output files and saves the results to FITS files. 18 | 19 | Module-specific config file entries 20 | =================================== 21 | 22 | MASKING : bool, optional 23 | Option to modify the SExtractor masking value (``-1e29``) in the vignets; 24 | the default value is ``False`` 25 | MASK_VALUE : float 26 | Value to use for masking in the vignets 27 | STAMP_SIZE : int 28 | Size of the vignet in pixels, note that this must be an odd integer 29 | COORD : str 30 | Coordinate convention, options are ``PIX`` for pixel coordinates 31 | or ``SPHE`` for world coordinates 32 | POSITION_PARAMS : list 33 | List of position parameter value names in the SExtractor output catalogue 34 | MODE : str 35 | Run mode for module, options are ``CLASSIC`` or ``MULTI-EPOCH`` 36 | PREFIX : str or list 37 | Output file name prefix(es) 38 | ME_IMAGE_DIR : list 39 | Module names of last run producing single-exposure flags, images, weights, 40 | and SExtractor background images, for multi-epoch processing. The specifier 41 | "last:" is not required 42 | ME_IMAGE_PATTERN : list 43 | Input file name patterns for flag, image, weight, and SExtractor background 44 | files, for multi-epoch processing 45 | ME_LOG_WCS : str 46 | Path to world coordinate system log file (``*sqlite``) 47 | 48 | """ 49 | 50 | __all__ = ["vignetmaker"] 51 | -------------------------------------------------------------------------------- /src/shapepipe/pipeline/__init__.py: -------------------------------------------------------------------------------- 1 | """SHAPEPIPE PIPELINE. 2 | 3 | This module contains sub-modules for handling the pipeline job management. 4 | 5 | :Author: Samuel Farrens 6 | 7 | """ 8 | 9 | __all__ = [ 10 | "args", 11 | "config", 12 | "dependency_handler", 13 | "execute", 14 | "file_handler", 15 | "file_io", 16 | "job_handler", 17 | "mpi_run", 18 | "run_log", 19 | "shared", 20 | "str_handler", 21 | "timeout", 22 | "worker_handler", 23 | ] 24 | -------------------------------------------------------------------------------- /src/shapepipe/pipeline/execute.py: -------------------------------------------------------------------------------- 1 | """EXECUTE. 2 | 3 | This module defines methods for running the pipeline modules. 4 | 5 | :Author: Samuel Farrens 6 | 7 | """ 8 | 9 | import os 10 | import subprocess as sp 11 | 12 | 13 | def execute(command_line): 14 | """Execute. 15 | 16 | This method executes a given command line. 17 | 18 | Parameters 19 | ---------- 20 | command_line : str 21 | The command line to be executed 22 | 23 | Returns 24 | ------- 25 | tuple 26 | Stdout and stderr (both type str) 27 | 28 | Raises 29 | ------ 30 | TypeError 31 | For invalid input type 32 | 33 | """ 34 | if not isinstance(command_line, str): 35 | raise TypeError("Command line must be a string.") 36 | 37 | command = command_line.split() 38 | check_executable(command[0]) 39 | 40 | process = sp.Popen(command, stdout=sp.PIPE, stderr=sp.PIPE) 41 | stdout, stderr = process.communicate() 42 | 43 | return stdout.decode("utf-8"), stderr.decode("utf-8") 44 | 45 | 46 | def check_executable(exe_name): 47 | """Check if Input is Executable. 48 | 49 | This methid checks if the input executable exists. 50 | 51 | Parameters 52 | ---------- 53 | exe_name : str 54 | Executable name 55 | 56 | Raises 57 | ------ 58 | TypeError 59 | For invalid input type 60 | OSError 61 | For non-existent executable 62 | 63 | """ 64 | if not isinstance(exe_name, str): 65 | raise TypeError("Executable name must be a string.") 66 | 67 | def is_exe(fpath): 68 | return os.path.isfile(fpath) and os.access(fpath, os.X_OK) 69 | 70 | fpath, fname = os.path.split(exe_name) 71 | 72 | if not fpath: 73 | res = any( 74 | [ 75 | is_exe(os.path.join(path, exe_name)) 76 | for path in os.environ["PATH"].split(os.pathsep) 77 | ] 78 | ) 79 | 80 | else: 81 | res = is_exe(exe_name) 82 | 83 | if not res: 84 | raise OSError( 85 | f"{exe_name} does not appear to be a valid executable on this " 86 | + "system." 87 | ) 88 | -------------------------------------------------------------------------------- /src/shapepipe/pipeline/mpi_run.py: -------------------------------------------------------------------------------- 1 | """MPI RUN. 2 | 3 | This module includes functions for handling the distribution of MPI jobs. 4 | 5 | :Author: Samuel Farrens 6 | 7 | """ 8 | 9 | from shapepipe.pipeline.worker_handler import WorkerHandler 10 | 11 | 12 | def split_mpi_jobs(jobs, batch_size): 13 | """Split MPI Jobs. 14 | 15 | Split the number of MPI jobs over the number of processes. 16 | 17 | Parameters 18 | ---------- 19 | jobs : list 20 | List of MPI jobs 21 | batch_size : int 22 | Batch size 23 | 24 | Returns 25 | ------- 26 | list 27 | Split list of jobs 28 | 29 | """ 30 | return [jobs[_i::batch_size] for _i in range(batch_size)] 31 | 32 | 33 | def submit_mpi_jobs( 34 | jobs, 35 | config, 36 | timeout, 37 | run_dirs, 38 | module_runner, 39 | worker_log, 40 | verbose, 41 | ): 42 | """Submit MPI Jobs. 43 | 44 | This method distributes the jobs to the workers using MPI. 45 | 46 | """ 47 | result = [] 48 | 49 | for process in jobs: 50 | 51 | w_log_name = worker_log(module_runner.__name__, process[0]) 52 | 53 | wh = WorkerHandler(verbose=verbose) 54 | result.append( 55 | wh.worker( 56 | process[1:], 57 | process[0], 58 | w_log_name, 59 | run_dirs, 60 | config, 61 | timeout, 62 | module_runner, 63 | ) 64 | ) 65 | 66 | return result 67 | -------------------------------------------------------------------------------- /src/shapepipe/pipeline/timeout.py: -------------------------------------------------------------------------------- 1 | """TIMEOUT. 2 | 3 | This module defines a function for handling job timeout limits. 4 | 5 | :Author: Samuel Farrens 6 | 7 | """ 8 | 9 | import signal 10 | from functools import wraps 11 | 12 | 13 | def with_timeout(timeout, log_file): 14 | """Timeout Limit Decorator. 15 | 16 | This method provides a timeout decorator for a given input function. 17 | 18 | Parameters 19 | ---------- 20 | timeout : int 21 | Timeout limit in seconds 22 | log_file : logging.Logger 23 | Logging instance 24 | 25 | Raises 26 | ------ 27 | TimeoutError 28 | For process exceeding timeout limit 29 | 30 | """ 31 | 32 | def handler(signum, frame): 33 | raise TimeoutError( 34 | f"The process time exceeded {timeout}s in {log_file}" 35 | ) 36 | 37 | def decorator(decorated): 38 | 39 | @wraps(decorated) 40 | def inner(*args, **kwargs): 41 | signal.signal(signal.SIGALRM, handler) 42 | signal.alarm(timeout) 43 | return decorated(*args, **kwargs) 44 | 45 | return inner 46 | 47 | return decorator 48 | -------------------------------------------------------------------------------- /src/shapepipe/shapepipe_run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """SHAPEPIPE RUN SCRIPT 4 | 5 | This script runs the shape measurement pipeline. 6 | 7 | :Author: Samuel Farrens 8 | 9 | """ 10 | 11 | from sys import exit 12 | 13 | from shapepipe.run import run 14 | 15 | 16 | def main(args=None): 17 | 18 | run(args) 19 | 20 | 21 | if __name__ == "__main__": 22 | exit(main()) 23 | -------------------------------------------------------------------------------- /src/shapepipe/tests/test_get_images.py: -------------------------------------------------------------------------------- 1 | """UNIT TESTS FOR MODULE PACKAGE: GET_IMAGES. 2 | 3 | This module contains unit tests for the module package 4 | shapepipe.modules.get_images_package.get_images 5 | 6 | :Author: Martin Kilbinger 7 | 8 | """ 9 | 10 | from unittest import TestCase 11 | 12 | import numpy as np 13 | import numpy.testing as npt 14 | 15 | from shapepipe.modules.get_images_package import get_images 16 | 17 | 18 | class GetImagesTestCase(TestCase): 19 | 20 | def setUp(self): 21 | 22 | self.number_tile = "123.456" 23 | self.number_exp = "2490092p" 24 | self.number_int = 123456 25 | 26 | def tearDown(self): 27 | 28 | self.number_tile = None 29 | self.number_exp = None 30 | self.number_int = None 31 | 32 | def test_in2out_pattern(self): 33 | 34 | npt.assert_string_equal( 35 | get_images.in2out_pattern(self.number_tile), "123-456" 36 | ) 37 | 38 | npt.assert_string_equal( 39 | get_images.in2out_pattern(self.number_exp), "2490092" 40 | ) 41 | 42 | npt.assert_raises( 43 | TypeError, 44 | get_images.in2out_pattern, 45 | self.number_int, 46 | ) 47 | -------------------------------------------------------------------------------- /src/shapepipe/tests/test_pipeline.py: -------------------------------------------------------------------------------- 1 | """UNIT TESTS FOR PIPELINE. 2 | 3 | This module contains unit tests for the shapepipe.pipeline module. 4 | 5 | :Author: Samuel Farrens 6 | 7 | """ 8 | 9 | from unittest import TestCase 10 | 11 | import numpy as np 12 | import numpy.testing as npt 13 | 14 | from shapepipe.pipeline import * 15 | 16 | 17 | class ExecuteTestCase(TestCase): 18 | 19 | def setUp(self): 20 | 21 | self.command_line = "echo 1" 22 | self.output_tuple = ("1\n", "") 23 | 24 | def tearDown(self): 25 | 26 | self.command_line = None 27 | self.output_tuple = None 28 | 29 | def test_execute(self): 30 | 31 | npt.assert_raises(TypeError, execute.execute, 1) 32 | npt.assert_equal(execute.execute(self.command_line), self.output_tuple) 33 | 34 | def test_check_executable(self): 35 | 36 | npt.assert_raises(TypeError, execute.check_executable, 1) 37 | npt.assert_raises(OSError, execute.check_executable, "") 38 | self.assertIsNone(execute.check_executable("/bin/ls")) 39 | -------------------------------------------------------------------------------- /src/shapepipe/utilities/__init__.py: -------------------------------------------------------------------------------- 1 | """SHAPEPIPE UTILITIES. 2 | 3 | This module contains tools that can be used by pipeline modules or external 4 | scripts. 5 | 6 | :Author: Samuel Farrens 7 | 8 | """ 9 | 10 | __all__ = ["file_system", "cfis", "galaxy", "summary"] 11 | -------------------------------------------------------------------------------- /src/shapepipe/utilities/file_system.py: -------------------------------------------------------------------------------- 1 | """FILE SYSTEM TOOLS. 2 | 3 | This module defines methods for managing actions on the file system. 4 | 5 | :Author: Samuel Farrens 6 | 7 | """ 8 | 9 | import os 10 | 11 | 12 | class FileSystemError(Exception): 13 | """File System Error. 14 | 15 | Generic error that is raised by the file system. 16 | 17 | """ 18 | 19 | pass 20 | 21 | 22 | def check_dir(dir_name): 23 | """Check Directory. 24 | 25 | Check if directory exists. 26 | 27 | Parameters 28 | ---------- 29 | dir_name : str 30 | Directory name 31 | 32 | Raises 33 | ------ 34 | TypeError 35 | If directory name is not a string 36 | 37 | """ 38 | if not isinstance(dir_name, str): 39 | raise TypeError( 40 | f"Directory name must be of type string, not {type(dir_name)}" 41 | ) 42 | 43 | return os.path.isdir(dir_name) 44 | 45 | 46 | def mkdir(dir_name, check_created=True, exist_ok=True): 47 | """Make Directory. 48 | 49 | This method creates a directory in the specified path. 50 | 51 | Parameters 52 | ---------- 53 | dir_name : str 54 | Directory name with full path 55 | check_created : bool 56 | Check if directory is properly created or already exists, raise error 57 | if not found (default is True) 58 | exist_ok : bool 59 | If False raise an error if the directory alredy exists (default is 60 | True) 61 | 62 | Raises 63 | ------ 64 | FileSystemError 65 | If directory already exists 66 | FileSystemError 67 | If directory not properly created 68 | 69 | """ 70 | os.makedirs(dir_name, exist_ok=exist_ok) 71 | 72 | if check_created and not check_dir(dir_name): 73 | raise FileSystemError( 74 | f'Directory "{dir_name}" not found after mkdir command.' 75 | ) 76 | -------------------------------------------------------------------------------- /xterm-start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | xterm 3 | --------------------------------------------------------------------------------