├── .DS_Store ├── .devcontainer ├── Dockerfile ├── devontainer.json └── docker-compose.yml ├── .github └── workflows │ ├── autoblack.yml │ ├── docker.yml │ └── run_tests.yml ├── .gitignore ├── LICENSE ├── README.md ├── data ├── .DS_Store └── THRAWS │ └── l1c │ └── l1c_cropped_tif │ └── Greece_5_2.tif ├── dockerfile ├── docs └── GLOSSARY.md ├── environment.yml ├── pyraws ├── __init__.py ├── database │ ├── L1C_files.json │ ├── OFFSETS.xlsx │ ├── coregistration_study_db.csv │ ├── shift_lut.csv │ └── thraws_db.csv ├── l1 │ ├── l1_event.py │ └── l1_tile.py ├── raw │ ├── raw_event.py │ └── raw_granule.py └── utils │ ├── band_shape_utils.py │ ├── constants.py │ ├── database_utils.py │ ├── date_utils.py │ ├── l1_utils.py │ ├── parse_csv_utils.py │ ├── raw_utils.py │ ├── shape_utils.py │ └── visualization_utils.py ├── pyraws_install.sh ├── quickstart ├── API_demonstration │ ├── etna_00_granules.png │ └── pyraws_Raw_API_demonstration.ipynb ├── BBOX_L1C2RAW │ ├── lightglue_homography.ipynb │ ├── results │ │ ├── Australia_1_original.pdf │ │ ├── Australia_1_transformed.pdf │ │ ├── Barren_Island_00_original.pdf │ │ ├── Barren_Island_00_transformed.pdf │ │ ├── Greece_5_original.pdf │ │ └── Greece_5_transformed.pdf │ └── tests │ │ ├── .DS_Store │ │ ├── l1c_Greece_5.jpg │ │ └── raw_Greece_5.jpg ├── DB.md ├── DB_creation │ └── pyraws_Raw_L1_database_creation.ipynb ├── INSTALLATION.md ├── README.md └── geographical_distribution │ └── pyraws_dataset_geographical_distribution.ipynb ├── resources ├── images │ ├── PyRawS_logo.png │ ├── coregistration.png │ ├── database_structure.png │ ├── etna_00_granules.png │ ├── granule_superimposition.png │ └── quicklook_yt.png ├── img_geo.ai └── img_geo.png ├── scripts_and_studies ├── coregistration_study │ ├── coregistration_profiling.py │ ├── coregistration_profiling_results │ │ └── desktop │ │ │ ├── profiling_n_images_16_coreg_coarse_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_16_coreg_coarse_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_16_coreg_coarse_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_16_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_16_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_16_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_16_coreg_super_glue_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_16_coreg_super_glue_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_16_coreg_super_glue_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_16_coreg_super_glue_device_cpu_equalize_True_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_16_coreg_super_glue_device_cpu_equalize_True_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_16_coreg_super_glue_device_cpu_equalize_True_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_16_coreg_super_glue_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_16_coreg_super_glue_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_16_coreg_super_glue_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_1_coreg_coarse_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_1_coreg_coarse_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_1_coreg_coarse_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_1_coreg_coarse_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_warm_up_1.txt │ │ │ ├── profiling_n_images_1_coreg_coarse_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_warm_up_2.txt │ │ │ ├── profiling_n_images_1_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_1_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_1_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_1_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_warm_up_1.txt │ │ │ ├── profiling_n_images_1_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_warm_up_2.txt │ │ │ ├── profiling_n_images_1_coreg_super_glue_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_1_coreg_super_glue_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_1_coreg_super_glue_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_1_coreg_super_glue_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_warm_up_1.txt │ │ │ ├── profiling_n_images_1_coreg_super_glue_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_warm_up_2.txt │ │ │ ├── profiling_n_images_1_coreg_super_glue_device_cpu_equalize_True_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_1_coreg_super_glue_device_cpu_equalize_True_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_1_coreg_super_glue_device_cpu_equalize_True_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_1_coreg_super_glue_device_cpu_equalize_True_bands_B8A_B11_B12_test_iteration_warm_up_1.txt │ │ │ ├── profiling_n_images_1_coreg_super_glue_device_cpu_equalize_True_bands_B8A_B11_B12_test_iteration_warm_up_2.txt │ │ │ ├── profiling_n_images_1_coreg_super_glue_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_1_coreg_super_glue_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_1_coreg_super_glue_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_1_coreg_super_glue_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_warm_up_1.txt │ │ │ ├── profiling_n_images_1_coreg_super_glue_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_warm_up_2.txt │ │ │ ├── profiling_n_images_2_coreg_coarse_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_2_coreg_coarse_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_2_coreg_coarse_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_2_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_2_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_2_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_2_coreg_super_glue_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_2_coreg_super_glue_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_2_coreg_super_glue_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_2_coreg_super_glue_device_cpu_equalize_True_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_2_coreg_super_glue_device_cpu_equalize_True_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_2_coreg_super_glue_device_cpu_equalize_True_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_2_coreg_super_glue_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_2_coreg_super_glue_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_2_coreg_super_glue_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_4_coreg_coarse_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_4_coreg_coarse_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_4_coreg_coarse_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_4_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_4_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_4_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_4_coreg_super_glue_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_4_coreg_super_glue_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_4_coreg_super_glue_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_4_coreg_super_glue_device_cpu_equalize_True_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_4_coreg_super_glue_device_cpu_equalize_True_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_4_coreg_super_glue_device_cpu_equalize_True_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_4_coreg_super_glue_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_4_coreg_super_glue_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_4_coreg_super_glue_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_8_coreg_coarse_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_8_coreg_coarse_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_8_coreg_coarse_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_8_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_8_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_8_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_8_coreg_super_glue_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_8_coreg_super_glue_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_8_coreg_super_glue_device_cpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_8_coreg_super_glue_device_cpu_equalize_True_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_8_coreg_super_glue_device_cpu_equalize_True_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ ├── profiling_n_images_8_coreg_super_glue_device_cpu_equalize_True_bands_B8A_B11_B12_test_iteration_3.txt │ │ │ ├── profiling_n_images_8_coreg_super_glue_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt │ │ │ ├── profiling_n_images_8_coreg_super_glue_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt │ │ │ └── profiling_n_images_8_coreg_super_glue_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt │ ├── coregistration_study.py │ ├── coregistration_study_notebooks_utils.py │ ├── coregistration_study_results │ │ ├── coregistration_study_results_THRAWS │ │ │ └── coregistration_study_B8A_B11_2023_2_1_20_36_8.csv │ │ ├── coregistration_study_results_d6 │ │ │ ├── coregistration_study_B01_B09_2023_1_26_11_10_46.csv │ │ │ ├── coregistration_study_B01_B12_2023_1_26_9_50_54.csv │ │ │ ├── coregistration_study_B02_B08_2023_1_26_11_12_46.csv │ │ │ ├── coregistration_study_B03_B05_2023_1_26_12_15_33.csv │ │ │ ├── coregistration_study_B04_B05_2023_1_26_12_22_44.csv │ │ │ ├── coregistration_study_B05_B11_2023_1_26_12_30_33.csv │ │ │ ├── coregistration_study_B06_B07_2023_1_26_12_36_12.csv │ │ │ ├── coregistration_study_B07_B8A_2023_1_26_12_41_46.csv │ │ │ ├── coregistration_study_B08_B03_2023_1_26_12_47_9.csv │ │ │ ├── coregistration_study_B09_B10_2023_1_26_13_11_16.csv │ │ │ ├── coregistration_study_B11_B06_2023_1_26_13_25_46.csv │ │ │ ├── coregistration_study_B12_B01_2023_1_26_13_13_56.csv │ │ │ ├── coregistration_study_B8A_B11_2023_1_26_13_16_33.csv │ │ │ └── coregistration_study_B8A_B12_2023_1_26_13_20_58.csv │ │ └── coregistration_study_results_dataset_v1 │ │ │ ├── coregistration_study_B01_B09_2022_10_2_0_39_4.csv │ │ │ ├── coregistration_study_B02_B08_2022_9_30_20_44_44.csv │ │ │ ├── coregistration_study_B03_B05_2022_10_10_15_33_17.csv │ │ │ ├── coregistration_study_B03_B10_2022_10_1_9_34_49.csv │ │ │ ├── coregistration_study_B04_B05_2022_10_1_14_36_2.csv │ │ │ ├── coregistration_study_B05_B11_2022_10_1_16_52_7.csv │ │ │ ├── coregistration_study_B06_B07_2022_10_1_18_54_57.csv │ │ │ ├── coregistration_study_B07_B8A_2022_10_1_19_56_56.csv │ │ │ ├── coregistration_study_B08_B03_2022_10_1_8_33_6.csv │ │ │ ├── coregistration_study_B09_B10_2022_10_5_21_45_22.csv │ │ │ ├── coregistration_study_B10_B04_2022_10_1_13_14_15.csv │ │ │ ├── coregistration_study_B10_B11_2022_10_6_9_47_33.csv │ │ │ ├── coregistration_study_B11_B06_2022_10_1_17_52_57.csv │ │ │ ├── coregistration_study_B12_B01_2022_10_2_8_57_26.csv │ │ │ ├── coregistration_study_B8A_B11_2022_9_27_17_52_42.csv │ │ │ └── coregistration_study_B8A_B12_2022_10_2_9_40_25.csv │ ├── coregistration_study_utils.py │ ├── find_best_coregistration_values_digital_image_correlation.py │ ├── pyraws_coregistration_profiling.ipynb │ ├── pyraws_coregistration_study.ipynb │ ├── run_coregistration_study.bat │ ├── run_test_coarse.bat │ ├── run_test_coarse.sh │ ├── run_test_superglue.bat │ └── run_test_superglue.sh ├── dataset_preparation │ └── THRAWS │ │ ├── fire │ │ └── fires_db.csv │ │ └── volcano_eruptions │ │ ├── create_json_polygons.py │ │ ├── create_volcano_csv.py │ │ ├── eruption_list.xlsx │ │ ├── json_polygon_from_point.py │ │ ├── my_json_polygon.json │ │ └── volcanoes_db_final.csv ├── granules_filtering │ └── granules_filtering.py ├── hta_detection_algorithms │ ├── hta_detection.py │ └── s2pix_detector.py ├── patch_study │ └── patchify.py └── runscripts │ ├── create_quicklook.py │ ├── create_tif.py │ ├── crop_L1_tiles.py │ ├── download_thraws.py │ └── generate_coregistration_plot.py └── setup.py /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESA-PhiLab/PyRawS/95d291388192d1c71ae302775c3bcf636717dc00/.DS_Store -------------------------------------------------------------------------------- /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | ############################################# INSTRUCTIONS ############################################# 2 | # build image: docker build -t pyraws:latest --build-arg CACHEBUST=$(date +%s) -f dockerfile . 3 | # run image: docker run -it --rm -p 8888:8888 pyraws:latest 4 | ############################################# END ############################################# 5 | 6 | # Use the official PyTorch base image 7 | FROM pytorch/pytorch:1.11.0-cuda11.3-cudnn8-runtime 8 | 9 | # Set the working directory inside the container 10 | WORKDIR /workspace 11 | 12 | # Install Python dependencies for pyraws: 13 | RUN apt-get update && apt-get install -y git ffmpeg libsm6 libxext6 14 | # install pyraws 15 | ARG CACHEBUST=1 16 | RUN git clone -b main https://github.com/ESA-PhiLab/PyRawS.git && cd PyRawS \ 17 | && /bin/bash -c "source ./pyraws_install.sh" 18 | 19 | RUN echo "source activate pyraws" > ~/.bashrc 20 | ENV PATH /opt/conda/envs/pyraws/bin:$PATH 21 | -------------------------------------------------------------------------------- /.devcontainer/devontainer.json: -------------------------------------------------------------------------------- 1 | // For format details, see https://aka.ms/devcontainer.json. For config options, see the 2 | // README at: https://github.com/devcontainers/templates/tree/main/src/anaconda 3 | 4 | { 5 | "name": "pyraws", 6 | "dockerComposeFile": "docker-compose.yml", 7 | "service": "pyraws", 8 | "workspaceFolder": "/workspace", 9 | "settings": { 10 | "terminal.integrated.shell.linux": "/bin/bash" 11 | }, 12 | "extensions": [ 13 | "ms-toolsai.jupyter", 14 | "ms-toolsai.vscode-jupyter-cell-tags", 15 | "ms-toolsai.jupyter-renderers", 16 | "ms-toolsai.vscode-jupyter-powertoys", 17 | "ms-toolsai.vscode-jupyter-slideshow", 18 | "donjayamanne.python-extension-pack" 19 | ], 20 | // Use 'postCreateCommand' to run commands after the container is created. 21 | //"postCreateCommand": "conda install -n simulator ipykernel --update-deps --force-reinstall" 22 | } -------------------------------------------------------------------------------- /.devcontainer/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.8" 2 | 3 | services: 4 | pyraws: 5 | image: pyraws:latest 6 | deploy: 7 | resources: 8 | reservations: 9 | devices: 10 | - driver: nvidia 11 | count: 1 12 | capabilities: [gpu] 13 | build: 14 | context: . 15 | dockerfile: Dockerfile 16 | volumes: 17 | - ../:/workspace/ 18 | platform: linux/amd64 19 | 20 | command: sleep infinity -------------------------------------------------------------------------------- /.github/workflows/autoblack.yml: -------------------------------------------------------------------------------- 1 | name: autoblack 2 | on: [pull_request] 3 | jobs: 4 | build: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - uses: actions/checkout@v1 8 | - name: Set up Python 3.9 9 | uses: actions/setup-python@v1 10 | with: 11 | python-version: 3.9 12 | - name: Install Black 13 | run: pip install black 14 | - name: Run black --check . 15 | run: black --check . -------------------------------------------------------------------------------- /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | # This workflow uses actions that are not certified by GitHub. 2 | # They are provided by a third-party and are governed by 3 | # separate terms of service, privacy policy, and support 4 | # documentation. 5 | 6 | # GitHub recommends pinning actions to a commit SHA. 7 | # To get a newer version, you will need to update the SHA. 8 | # You can also reference a tag or branch, but the action may change without warning. 9 | 10 | name: Build and Push Docker Image for PYRAWS 11 | 12 | on: 13 | push: 14 | 15 | branches: [ "main" ] 16 | 17 | jobs: 18 | build: 19 | runs-on: ubuntu-latest 20 | steps: 21 | - 22 | name: Checkout 23 | uses: actions/checkout@v3 24 | - 25 | name: Login to Docker Hub 26 | uses: docker/login-action@v2 27 | with: 28 | username: ${{ secrets.DOCKER_USERNAME }} 29 | password: ${{ secrets.DOCKER_PASSWORD }} 30 | - 31 | name: Set up Docker Buildx 32 | uses: docker/setup-buildx-action@v2 33 | - 34 | name: Build and push 35 | uses: docker/build-push-action@v4 36 | with: 37 | context: . 38 | file: ./Dockerfile 39 | push: true 40 | tags: ${{ secrets.DOCKER_USERNAME }}/pyraws:latest 41 | -------------------------------------------------------------------------------- /.github/workflows/run_tests.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | name: Running tests 4 | 5 | on: 6 | push: 7 | branches: 8 | - main 9 | pull_request: 10 | workflow_dispatch: 11 | jobs: 12 | lint_flake8: 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: Set up Python 3.9 18 | uses: actions/setup-python@v2 19 | with: 20 | python-version: 3.9 21 | - name: Lint with flake8 22 | run: | 23 | pip install flake8 24 | flake8 . --count --show-source --statistics --max-line-length=127 --ignore=E402,W503,E203 25 | 26 | build: 27 | runs-on: ubuntu-latest 28 | 29 | steps: 30 | - uses: actions/checkout@v2 31 | - name: Set up Python 3.9 32 | uses: actions/setup-python@v2 33 | with: 34 | python-version: 3.9 35 | - name: Search for severe code errors with flake8 36 | run: | 37 | # stop the build if there are Python syntax errors or undefined names 38 | pip install flake8 39 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --max-line-length=127 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *__pycache__ 2 | */.ipynb_checkpoints/* 3 | pyraws.egg-info/* 4 | pyraws/sys_cfg.py 5 | pyraws/data/* 6 | scripts_and_studies/Test/ 7 | *.pkl 8 | scripts_and_studies/coregistration_study/superglue_models 9 | pyraws/sys_config.py 10 | *data/ 11 | 12 | scripts_and_studies/coregistration_study/* -------------------------------------------------------------------------------- /data/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESA-PhiLab/PyRawS/95d291388192d1c71ae302775c3bcf636717dc00/data/.DS_Store -------------------------------------------------------------------------------- /data/THRAWS/l1c/l1c_cropped_tif/Greece_5_2.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESA-PhiLab/PyRawS/95d291388192d1c71ae302775c3bcf636717dc00/data/THRAWS/l1c/l1c_cropped_tif/Greece_5_2.tif -------------------------------------------------------------------------------- /dockerfile: -------------------------------------------------------------------------------- 1 | ############################################# INSTRUCTIONS ############################################# 2 | # build image: docker build -t pyraws:latest --build-arg CACHEBUST=$(date +%s) -f dockerfile . 3 | # run image: docker run -it --rm -p 8888:8888 pyraws:latest 4 | ############################################# END ############################################# 5 | 6 | 7 | # Use the official PyTorch base image 8 | FROM pytorch/pytorch:1.11.0-cuda11.3-cudnn8-runtime 9 | 10 | # Set the working directory inside the container 11 | WORKDIR /workdir/PyProjects/ 12 | 13 | # Install Python dependencies for pyraws: 14 | RUN apt-get update && apt-get install -y git ffmpeg libsm6 libxext6 15 | # install pyraws 16 | ARG CACHEBUST=1 17 | RUN git clone -b main https://github.com/ESA-PhiLab/PyRawS.git 18 | RUN cd PyRawS && /bin/bash -c "source ./pyraws_install.sh" 19 | RUN echo "source activate pyraws" > ~/.bashrc 20 | ENV PATH /opt/conda/envs/pyraws/bin:$PATH 21 | -------------------------------------------------------------------------------- /docs/GLOSSARY.md: -------------------------------------------------------------------------------- 1 | 2 | ## Glossary 3 | * ### Coarse coregistration 4 | Lightweight spatial coregistration method optimized for onboard-satellite applications. It simply shifts the various bands of a fixed factor that depends only on the bands, the satellite and detector number. 5 | 6 | * ### Sentinel-2 L0 data 7 | Sentinel-2 data at `level-0` (`L0`) are data that are transmitted to Ground from Sentinel-2 satellites. The `L0` format is compressed to diminish downlink bandwidth requirements. For more information, refer to the [Sentinel-2 Products Specification Document](https://sentinel.esa.int/documents/247904/685211/sentinel-2-products-specification-document) 8 | 9 | * ### Sentinel-2 Raw data 10 | In the frame of this project, the [Sentinel-2 Raw](#sentinel-2-raw-data) represents a particular product in the Sentinel-2 processing chain that matches a decompressed version of [Sentinel-2 L0 data](sentinel-2-l0-data) with additional metadata that are produced on ground. Once decompressed, `Sentinel-2 Raw data` are the data available on Ground that better emulate the one produced by Sentinel-2 detectors with the exception of the effects due to compression and onboard equalization, which are not compensated at this stage. Therefore, `Sentinel-2 raw data` are those exploited in this project. For more information, refer to the [Sentinel-2 Products Specification Document](https://sentinel.esa.int/documents/247904/685211/sentinel-2-products-specification-document).
11 | **N.B**: the nomenclature ```raw data``` and its location in the Sentinel-2 processing chain is specific for this project only. 12 | 13 | * ### Sentinel-2 Raw granule 14 | A `granule` is the image acquired by a Sentinel-2 detector during a single acquisition lasting 3.6 s. Granules are defined at [L0](#sentinel-2-raw-data) level. However, since the processing perfomed on the ground between L0 and raw data does not alter the image content (with the exception of the decompression process) but just provide additional metadata, granules are defined also at [Sentinel-2 Raw](#sentinel-2-raw-data) level. 15 | Given the pushbroom nature of the Sentinel-2 sensor, bands do not look at the same area at [Raw](#sentinel-2-raw-data) level. For more information, refer to the [Sentinel-2 Products Specification Document](https://sentinel.esa.int/documents/247904/685211/sentinel-2-products-specification-document) 16 | 17 | * ### Sentinel-2 Raw event 18 | [Sentinel-2 Raw data](#sentinel-2-raw-data) are produced by decompressing [Sentinel-2 L0 data](sentinel-2-raw-data). To download L0 data, it is necessary to specify one polygon that surrounds a particular area-of-interest. This leads to download all those [Sentinel-2 Raw granules](#sentinel-2-raw-granule) whose reference band intersects the specified polygon. Such collection is a `Raw-event`. Each `Raw-event` matches one of the `ID_event` entry of the database.
19 | For each `Raw-event`, we do not provide all the collection of [Sentinel-2 Raw granules](#sentinel-2-Raw-granule), but only the set of [Raw data useful granules](#raw-data-useful-granule) and [Raw data complementary granules](#Raw-data-complementary-granule). For an intuitive example, please, check [Raw events and granules](#raw-events-and-raw-granules). 20 | 21 | * ### Sentinel-2 L1C data 22 | The `Level 1-C` (`L1C`) is one format for `Sentinel-2` data. To convert [Sentinel-2 Raw data](#sentinel-2-raw-data) to `L1C` data, numerous processing steps are applied to correct defects, including bands coregistration, ortho-rectification, decompression, noise-suppression and other. For more information, refer to the [Sentinel-2 Products Specification Document](https://sentinel.esa.int/documents/247904/685211/sentinel-2-products-specification-document). 23 | 24 | * ### Sentinel-2 L1C event 25 | Same concept for [Sentinel-2 Raw events](#sentinel-2-raw-event) but applied on [Sentinel-2 L1C data](#sentinel-2-l1c-data). 26 | 27 | * ### Sentinel-2 L1C tile 28 | The `Sentinel-2 L1C tile` is the minimum `L1C` product that can be downloaded. 29 | 30 | * ### Raw complementary granule 31 | Given a certain set of bands of interest `[Bx,By,...,Bz]`, `Raw complementarey granules` are the granules adjacents at [Raw-useful-granules](#raw-useful-granule) that that can be used to fill missing pixels of `[By,...,Bz]` bands due to their coregistration with respecto the band `Bx`. For an intuitive example, please, check [Raw events and granules](#raw-events-and-raw-granules). 32 | 33 | * ### Raw useful granule 34 | Given a certain set of bands of interest `[Bx,By,...,Bz]`, where `Bx` is the first band in the set, an `Raw useful granule` is one of the collection of [Sentinel-2 Raw granules](#sentinel-2-raw-granule) that compose a [Sentinel-2 Raw event](#sentinel-2-raw-event) whose band `Bx` include (or intersects) a certain area of interest (e.g., an eruption or an area covered by a fire). For an intuitive example, please, check [Raw data events and granules](#raw-events-and-raw-granules). -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: pyraws 2 | channels: 3 | - conda-forge 4 | - pytorch 5 | dependencies: 6 | - python=3.9 7 | - geopy 8 | - numpy 9 | - geopandas 10 | - geopy 11 | - scipy 12 | - matplotlib 13 | - termcolor 14 | - tqdm 15 | - glymur 16 | - cudatoolkit=10.2 17 | - pytorch==1.11.0 18 | - torchvision==0.12.0 19 | - scikit-learn 20 | - scikit-image 21 | - rasterio 22 | - pip 23 | - pip: 24 | - tifffile 25 | - opencv-python 26 | -------------------------------------------------------------------------------- /pyraws/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from .raw.raw_granule import Raw_granule 4 | from .raw.raw_event import Raw_event 5 | from .l1.l1_tile import L1C_tile 6 | from .l1.l1_event import L1C_event 7 | from .utils.constants import ( 8 | BAND_NAMES, 9 | BANDS_RAW_SHAPE_DICT, 10 | BAND_NAMES_REAL_ORDER, 11 | BAND_SPATIAL_RESOLUTION_DICT, 12 | BAND_SPATIAL_RESOLUTION_DICT_ACROSS, 13 | DATABASE_FILE_DICTIONARY, 14 | ) 15 | from .utils.date_utils import get_timestamp, parse_string_date 16 | from .utils.database_utils import ( 17 | get_event_info, 18 | get_events_list, 19 | get_event_granule_bb_dict, 20 | ) 21 | from .utils.shape_utils import get_granules_polys, create_rectangular_polygon 22 | from .utils.visualization_utils import ( 23 | equalize_tensor, 24 | plot_event, 25 | plot_img1_vs_img2_bands, 26 | ) 27 | from .utils.parse_csv_utils import parse_csv 28 | 29 | __all__ = [ 30 | "BAND_NAMES", 31 | "BAND_NAMES_REAL_ORDER", 32 | "BANDS_RAW_SHAPE_DICT", 33 | "BAND_SPATIAL_RESOLUTION_DICT", 34 | "BAND_SPATIAL_RESOLUTION_DICT_ACROSS", 35 | "create_rectangular_polygon", 36 | "DATABASE_FILE_DICTIONARY", 37 | "equalize_tensor", 38 | "get_event_bounding_box", 39 | "get_event_granule_bb_dict", 40 | "get_events_list", 41 | "get_event_info", 42 | "get_l1C_image_default_path", 43 | "get_granules_polys", 44 | "get_timestamp", 45 | "L1C_event", 46 | "L1C_tile", 47 | "parse_csv", 48 | "parse_string_date", 49 | "plot_event", 50 | "plot_img1_vs_img2_bands", 51 | "parse_string_date", 52 | "Raw_granule", 53 | "Raw_event", 54 | "read_L1C_event", 55 | "read_L1C_image_from_tif", 56 | ] 57 | 58 | # Initialize logger 59 | logger = logging.getLogger(__name__) 60 | 61 | 62 | def set_log_level(level=logging.WARN): 63 | """Allows setting global log level for the application. 64 | 65 | Args: 66 | level (logging.level, optional): Level to set, available are 67 | (logging.DEBUG,logging.INFO,logging.WARN,logging.ERROR). 68 | Defaults to logging.WARN. 69 | """ 70 | logger.setLevel(level) 71 | if level == 10: 72 | logger.info("Log level set to debug") 73 | elif level == 20: 74 | logger.info("Log level set to info") 75 | # we still store it in case we might write some logfile or sth later 76 | elif level == 30: 77 | logger.info("Log level set to warn") 78 | elif level == 40: 79 | logger.info("Log level set to error") 80 | -------------------------------------------------------------------------------- /pyraws/database/OFFSETS.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESA-PhiLab/PyRawS/95d291388192d1c71ae302775c3bcf636717dc00/pyraws/database/OFFSETS.xlsx -------------------------------------------------------------------------------- /pyraws/database/coregistration_study_db.csv: -------------------------------------------------------------------------------- 1 | ID_event,Sat,class 2 | Barren_Island_00,S2A,eruption 3 | Barren_Island_01,S2A,eruption 4 | Barren_Island_02,S2B,eruption 5 | Barren_Island_10,S2A,eruption 6 | Barren_Island_11,S2A,eruption 7 | Chillan_Nevados_de_00,S2B,eruption 8 | Chillan_Nevados_de_01,S2A,eruption 9 | Chillan_Nevados_de_02,S2B,eruption 10 | Copahue_00,S2A,eruption 11 | Copahue_01,S2B,eruption 12 | Copahue_10,S2B,eruption 13 | Copahue_20,S2A,eruption 14 | Copahue_21,S2A,eruption 15 | Copahue_30,S2A,eruption 16 | Etna_00,S2B,eruption 17 | Etna_01,S2A,eruption 18 | Etna_02,S2B,eruption 19 | Etna_03,S2B,eruption 20 | Fuego_00,S2B,eruption 21 | Fuego_01,S2A,eruption 22 | Fuego_02,S2A,eruption 23 | Karangetang_00,S2A,eruption 24 | Karangetang_01,S2B,eruption 25 | Karangetang_02,S2A,eruption 26 | Krakatau_00,S2B,eruption 27 | Krakatau_01,S2A,eruption 28 | Krakatau_10,S2B,eruption 29 | Krysuvik-Trolladyngja_00,S2B,eruption 30 | Krysuvik-Trolladyngja_01,S2B,eruption 31 | Krysuvik-Trolladyngja_02,S2B,eruption 32 | Krysuvik-Trolladyngja_03,S2B,eruption 33 | La_Palma_00,S2A,eruption 34 | La_Palma_01,S2A,eruption 35 | La_Palma_02,S2A,eruption 36 | Mayon_00,S2B,eruption 37 | Mayon_01,S2B,eruption 38 | Mayon_02,S2B,eruption 39 | Nyamulagira_00,S2B,eruption 40 | Nyamulagira_01,S2A,eruption 41 | Nyamulagira_02,S2A,eruption 42 | Piton_de_la_Fournaise_00,S2B,eruption 43 | Piton_de_la_Fournaise_01,S2A,eruption 44 | Piton_de_la_Fournaise_02,S2A,eruption 45 | Piton_de_la_Fournaise_10,S2B,eruption 46 | Piton_de_la_Fournaise_11,S2B,eruption 47 | Piton_de_la_Fournaise_12,S2A,eruption 48 | Piton_de_la_Fournaise_30,S2B,eruption 49 | Piton_de_la_Fournaise_31,S2A,eruption 50 | Piton_de_la_Fournaise_32,S2B,eruption 51 | Piton_de_la_Fournaise_40,S2B,eruption 52 | Poas_10,S2B,eruption 53 | Poas_11,S2B,eruption 54 | Poas_20,S2A,eruption 55 | Poas_21,S2A,eruption 56 | Raung_10,S2A,eruption 57 | Raung_11,S2B,eruption 58 | Raung_12,S2A,eruption 59 | San_Miguel_10,S2A,eruption 60 | San_Miguel_11,S2A,eruption 61 | San_Miguel_12,S2A,eruption 62 | Sangay_00,S2A,eruption 63 | Sangay_01,S2A,eruption 64 | Sangay_10,S2B,eruption 65 | Sangay_11,S2A,eruption 66 | Sangay_12,S2B,eruption 67 | Sangay_31,S2B,eruption 68 | Santa_Maria_00,S2A,eruption 69 | Soputan_10,S2B,eruption 70 | Stromboli_01,S2A,eruption 71 | Stromboli_02,S2A,eruption 72 | Telica_00,S2A,eruption 73 | Telica_01,S2A,eruption 74 | Telica_02,S2B,eruption 75 | Telica_10,S2A,eruption 76 | Tinakula_00,S2A,eruption 77 | Tinakula_01,S2A,eruption 78 | Tinakula_02,S2B,eruption 79 | Ulawun_10,S2A,eruption 80 | Ulawun_20,S2B,eruption 81 | Ulawun_21,S2B,eruption 82 | -------------------------------------------------------------------------------- /pyraws/database/shift_lut.csv: -------------------------------------------------------------------------------- 1 | ,satellite,registration_mode,detector,S09_01,S08_02,S05_03,S10_03,S05_04,S11_05,S07_06,S8A_07,S03_08,S10_09,S04_10,S06_11,S01_12,S11_8A,S12_8A 2 | 0,S2A,upsampling,1,"[28, 0]","[169, -1]","[239, -2]","[37, -14]","[84, -1]","[74, -14]","[84, 0]","[83, 0]","[168, -2]","[-183, -10]","[84, 24]","[10, 14]","[22, 13]","[-177, -13]","[22, -14]" 3 | 1,S2A,downsampling,1,"[28, 0]","[169, -1]","[239, -2]","[37, -14]","[84, -1]","[74, -14]","[84, 0]","[83, 0]","[168, -2]","[-183, -10]","[84, 24]","[10, 14]","[22, 13]","[-177, -13]","[22, -14]" 4 | 2,S2B,upsampling,1,"[29, 0]","[169, -1]","[239, -2]","[39, -4]","[84, -1]","[77, -3]","[84, 0]","[83, 0]","[169, -2]","[-183, 0]","[78, 6]","[7, 2]","[21, 2]","[-175, -2]","[25, -3]" 5 | 3,S2B,downsampling,1,"[29, 0]","[169, -1]","[239, -2]","[39, -4]","[84, -1]","[77, -3]","[84, 0]","[83, 0]","[169, -2]","[-183, 0]","[78, 6]","[7, 2]","[21, 2]","[-175, -2]","[25, -3]" 6 | 4,S2A,upsampling,2,"[-29, 1]","[-168, 2]","[-240, 3]","[-36, -14]","[-84, 1]","[-69, -13]","[-84, 1]","[-85, 1]","[-168, 2]","[185, -23]","[-96, 34]","[-16, 14]","[-22, 15]","[186, -16]","[-16, -14]" 7 | 5,S2A,downsampling,2,"[-29, 1]","[-168, 2]","[-240, 3]","[-36, -14]","[-84, 1]","[-69, -13]","[-84, 1]","[-85, 1]","[-168, 2]","[185, -23]","[-96, 34]","[-16, 14]","[-22, 15]","[186, -16]","[-16, -14]" 8 | 6,S2B,upsampling,2,"[-29, 1]","[-167, 2]","[-239, 3]","[-36, -2]","[-84, 1]","[-67, -2]","[-84, 1]","[-85, 1]","[-168, 2]","[184, -10]","[-88, 8]","[-18, 3]","[-23, 4]","[188, -6]","[-14, -3]" 9 | 7,S2B,downsampling,2,"[-29, 1]","[-167, 2]","[-239, 3]","[-36, -2]","[-84, 1]","[-67, -2]","[-84, 1]","[-85, 1]","[-168, 2]","[184, -10]","[-88, 8]","[-18, 3]","[-23, 4]","[188, -6]","[-14, -3]" 10 | 8,S2A,upsampling,3,"[28, 0]","[166, -1]","[236, -1]","[37, -14]","[83, -1]","[73, -14]","[83, 0]","[82, 0]","[167, -1]","[-180, -11]","[82, 28]","[9, 13]","[21, 13]","[-175, -13]","[22, -14]" 11 | 9,S2A,downsampling,3,"[28, 0]","[166, -1]","[236, -1]","[37, -14]","[83, -1]","[73, -14]","[83, 0]","[82, 0]","[167, -1]","[-180, -11]","[82, 28]","[9, 13]","[21, 13]","[-175, -13]","[22, -14]" 12 | 10,S2B,upsampling,3,"[28, 0]","[166, 0]","[236, -1]","[38, -3]","[83, -1]","[75, -3]","[83, 0]","[82, 0]","[166, -1]","[-180, -1]","[78, 6]","[7, 3]","[21, 2]","[-173, -2]","[24, -3]" 13 | 11,S2B,downsampling,3,"[28, 0]","[166, 0]","[236, -1]","[38, -3]","[83, -1]","[75, -3]","[83, 0]","[82, 0]","[166, -1]","[-180, -1]","[78, 6]","[7, 3]","[21, 2]","[-173, -2]","[24, -3]" 14 | 12,S2A,upsampling,4,"[-29, 1]","[-165, 1]","[-236, 1]","[-35, -12]","[-83, 0]","[-69, -12]","[-83, 1]","[-84, 1]","[-165, 1]","[183, -17]","[-92, 26]","[-15, 13]","[-22, 13]","[183, -14]","[-16, -13]" 15 | 13,S2A,downsampling,4,"[-29, 1]","[-165, 1]","[-236, 1]","[-35, -12]","[-83, 0]","[-69, -12]","[-83, 1]","[-84, 1]","[-165, 1]","[183, -17]","[-92, 26]","[-15, 13]","[-22, 13]","[183, -14]","[-16, -13]" 16 | 14,S2B,upsampling,4,"[-29, 1]","[-166, 1]","[-236, 1]","[-34, 0]","[-83, 0]","[-67, -1]","[-83, 1]","[-84, 1]","[-166, 1]","[185, -5]","[-102, 2]","[-17, 2]","[-23, 2]","[185, -3]","[-14, -2]" 17 | 15,S2B,downsampling,4,"[-29, 1]","[-166, 1]","[-236, 1]","[-34, 0]","[-83, 0]","[-67, -1]","[-83, 1]","[-84, 1]","[-166, 1]","[185, -5]","[-102, 2]","[-17, 2]","[-23, 2]","[185, -3]","[-14, -2]" 18 | 16,S2A,upsampling,5,"[28, 0]","[164, 0]","[233, -1]","[38, -13]","[83, 0]","[73, -13]","[82, 0]","[81, 0]","[164, 0]","[-178, -11]","[72, 22]","[9, 13]","[21, 13]","[-172, -13]","[23, -13]" 19 | 17,S2A,downsampling,5,"[28, 0]","[164, 0]","[233, -1]","[38, -13]","[83, 0]","[73, -13]","[82, 0]","[81, 0]","[164, 0]","[-178, -11]","[72, 22]","[9, 13]","[21, 13]","[-172, -13]","[23, -13]" 20 | 18,S2B,upsampling,5,"[28, 0]","[164, 0]","[233, -1]","[38, -3]","[82, 0]","[75, -2]","[82, 0]","[82, 0]","[164, 0]","[-177, -3]","[70, 6]","[7, 2]","[20, 2]","[-170, -2]","[24, -2]" 21 | 19,S2B,downsampling,5,"[28, 0]","[164, 0]","[233, -1]","[38, -3]","[82, 0]","[75, -2]","[82, 0]","[82, 0]","[164, 0]","[-177, -3]","[70, 6]","[7, 2]","[20, 2]","[-170, -2]","[24, -2]" 22 | 20,S2A,upsampling,6,"[-29,0]","[-164,0]","[-236,0]","[-20, -17]","[-83,0]","[-67,-12]","[-83,0]","[-84,0]","[-165,0]","[164,-17]","[-186, 34]","[-16,12]","[-22,12]","[183,-13]","[-15,-12]" 23 | 21,S2A,downsampling,6,"[-29,0]","[-164,0]","[-236,0]","[-20, -17]","[-83,0]","[-67,-12]","[-83,0]","[-84,0]","[-165,0]","[164,-17]","[-186, 34]","[-16,12]","[-22,12]","[183,-13]","[-15,-12]" 24 | 22,S2B,upsampling,6,"[-29,0]","[-165,0]","[-236,0]","[-1, -1]","[-83,0]","[-66,-2]","[-83,0]","[-84,0]","[-165,0]","[183,-1]","[-300, 2]","[-18,2]","[-23,2]","[185,-2]","[-13,-2]" 25 | 23,S2B,downsampling,6,"[-29,0]","[-165,0]","[-236,0]","[-1, -1]","[-83,0]","[-66,-2]","[-83,0]","[-84,0]","[-165,0]","[183,-1]","[-300, 2]","[-18,2]","[-23,2]","[185,-2]","[-13,-2]" 26 | 24,S2A,upsampling,7,"[28, 0]","[164, 0]","[234, 0]","[38, -12]","[83, 0]","[73, -12]","[83, 0]","[81, 0]","[164, 1]","[-178, -11]","[72, 22]","[9, 12]","[21, 12]","[-173, -12]","[22, -12]" 27 | 25,S2A,downsampling,7,"[28, 0]","[164, 0]","[234, 0]","[38, -12]","[83, 0]","[73, -12]","[83, 0]","[81, 0]","[164, 1]","[-178, -11]","[72, 22]","[9, 12]","[21, 12]","[-173, -12]","[22, -12]" 28 | 26,S2B,upsampling,7,"[28, 0]","[164, 0]","[232, 0]","[38, 0]","[82, 0]","[75, -1]","[82, 0]","[81, 0]","[164, 0]","[-176, -1]","[66, 2]","[7, 1]","[20, 1]","[-170, -1]","[24, -1]" 29 | 27,S2B,downsampling,7,"[28, 0]","[164, 0]","[232, 0]","[38, 0]","[82, 0]","[75, -1]","[82, 0]","[81, 0]","[164, 0]","[-176, -1]","[66, 2]","[7, 1]","[20, 1]","[-170, -1]","[24, -1]" 30 | 28,S2A,upsampling,8,"[-29, 0]","[-165, -1]","[-236, -1]","[-35, -12]","[-83, -1]","[-68, -12]","[-83, 0]","[-84, 0]","[-166, -1]","[183, -9]","[-94, 22]","[-16, 12]","[-22, 11]","[184, -11]","[-15, -12]" 31 | 29,S2A,downsampling,8,"[-29, 0]","[-165, -1]","[-236, -1]","[-35, -12]","[-83, -1]","[-68, -12]","[-83, 0]","[-84, 0]","[-166, -1]","[183, -9]","[-94, 22]","[-16, 12]","[-22, 11]","[184, -11]","[-15, -12]" 32 | 30,S2B,upsampling,8,"[-29, 0]","[-165, -1]","[-236, -1]","[-34, -2]","[-83, -1]","[-66, -2]","[-83, 0]","[-84, 0]","[-166, -1]","[184, 1]","[-98, 2]","[-18, 1]","[-23, 1]","[186, -1]","[-13, -1]" 33 | 31,S2B,downsampling,8,"[-29, 0]","[-165, -1]","[-236, -1]","[-34, -2]","[-83, -1]","[-66, -2]","[-83, 0]","[-84, 0]","[-166, -1]","[184, 1]","[-98, 2]","[-18, 1]","[-23, 1]","[186, -1]","[-13, -1]" 34 | 32,S2A,upsampling,9,"[28, 0]","[166, 1]","[235, 0]","[38, -10]","[83, 0]","[74, -10]","[83, 0]","[81, 0]","[166, 1]","[-179, -11]","[74, 20]","[8, 11]","[21, 11]","[-173, -11]","[24, -11]" 35 | 33,S2A,downsampling,9,"[28, 0]","[166, 1]","[235, 0]","[38, -10]","[83, 0]","[74, -10]","[83, 0]","[81, 0]","[166, 1]","[-179, -11]","[74, 20]","[8, 11]","[21, 11]","[-173, -11]","[24, -11]" 36 | 34,S2B,upsampling,9,"[28, 0]","[165, 1]","[234, 0]","[38, -2]","[83, 0]","[76, -1]","[83, 0]","[82, 0]","[165, 1]","[-179, -2]","[76, 4]","[6, 1]","[20, 1]","[-171, -2]","[25, -1]" 37 | 35,S2B,downsampling,9,"[28, 0]","[165, 1]","[234, 0]","[38, -2]","[83, 0]","[76, -1]","[83, 0]","[82, 0]","[165, 1]","[-179, -2]","[76, 4]","[6, 1]","[20, 1]","[-171, -2]","[25, -1]" 38 | 36,S2A,upsampling,10,"[-29, -1]","[-165, -1]","[-237, -2]","[-36, -12]","[-83, -1]","[-68, -12]","[-84, -1]","[-85, -1]","[-166, -1]","[183, -5]","[-88, 22]","[-17, 11]","[-22, 11]","[185, -9]","[-15, -12]" 39 | 37,S2A,downsampling,10,"[-29, -1]","[-165, -1]","[-237, -2]","[-36, -12]","[-83, -1]","[-68, -12]","[-84, -1]","[-85, -1]","[-166, -1]","[183, -5]","[-88, 22]","[-17, 11]","[-22, 11]","[185, -9]","[-15, -12]" 40 | 38,S2B,upsampling,10,"[-29, -1]","[-165, -2]","[-236, -3]","[-34, -3]","[-83, -1]","[-65, -1]","[-84, -1]","[-84, -1]","[-166, -1]","[184, 4]","[-98, 4]","[-19, 0]","[-23, 0]","[187, 2]","[-12, -1]" 41 | 39,S2B,downsampling,10,"[-29, -1]","[-165, -2]","[-236, -3]","[-34, -3]","[-83, -1]","[-65, -1]","[-84, -1]","[-84, -1]","[-166, -1]","[184, 4]","[-98, 4]","[-19, 0]","[-23, 0]","[187, 2]","[-12, -1]" 42 | 40,S2A,upsampling,11,"[28, 0]","[167, 1]","[238, 1]","[38, -11]","[84, 0]","[75, -10]","[84, 0]","[83, 0]","[169, 1]","[-183, -14]","[82, 26]","[8, 11]","[22, 11]","[-176, -12]","[24, -11]" 43 | 41,S2A,downsampling,11,"[28, 0]","[167, 1]","[238, 1]","[38, -11]","[84, 0]","[75, -10]","[84, 0]","[83, 0]","[169, 1]","[-183, -14]","[82, 26]","[8, 11]","[22, 11]","[-176, -12]","[24, -11]" 44 | 42,S2B,upsampling,11,"[-29, -1]","[-165, -2]","[-236, -3]","[-34, -3]","[-83, -1]","[-65, -1]","[-84, -1]","[-84, -1]","[-166, -1]","[184, 4]","[-98, 4]","[-19, 0]","[-23, 0]","[187, 2]","[-12, -1]" 45 | 43,S2B,downsampling,11,"[-29, -1]","[-165, -2]","[-236, -3]","[-34, -3]","[-83, -1]","[-65, -1]","[-84, -1]","[-84, -1]","[-166, -1]","[184, 4]","[-98, 4]","[-19, 0]","[-23, 0]","[187, 2]","[-12, -1]" 46 | 44,S2A,upsampling,12,"[-30, -2]","[-172, -3]","[-244, -4]","[-36, -14]","[-86, -2]","[-70, -12]","[-86, -2]","[-87, -2]","[-172, -3]","[191, -1]","[-104, 26]","[-18, 10]","[-23, 9]","[192, -7]","[-15, -11]" 47 | 45,S2A,downsampling,12,"[-30, -2]","[-172, -3]","[-244, -4]","[-36, -14]","[-86, -2]","[-70, -12]","[-86, -2]","[-87, -2]","[-172, -3]","[191, -1]","[-104, 26]","[-18, 10]","[-23, 9]","[192, -7]","[-15, -11]" 48 | 46,S2B,upsampling,12,"[-29, -1]","[-165, -2]","[-236, -3]","[-34, -3]","[-83, -1]","[-65, -1]","[-84, -1]","[-84, -1]","[-166, -1]","[184, 4]","[-98, 4]","[-19, 0]","[-23, 0]","[187, 2]","[-12, -1]" 49 | 47,S2B,downsampling,12,"[-29, -1]","[-165, -2]","[-236, -3]","[-34, -3]","[-83, -1]","[-65, -1]","[-84, -1]","[-84, -1]","[-166, -1]","[184, 4]","[-98, 4]","[-19, 0]","[-23, 0]","[187, 2]","[-12, -1]" 50 | -------------------------------------------------------------------------------- /pyraws/utils/band_shape_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from termcolor import colored 3 | from torch.nn import Upsample 4 | from torch.nn.functional import interpolate 5 | from .constants import BAND_NAMES, BAND_SPATIAL_RESOLUTION_DICT 6 | 7 | 8 | def image_band_upsample(img_band, band_name, target_spatial_resolution, upsample_mode): 9 | """Upsample an image band to a target spatial resolution through an upsample mode. 10 | 11 | Args: 12 | img_band (torch.tensor): image band. 13 | band_name (string): band name. 14 | target_spatial_resolution (float): target resolution (m). 15 | upsample_mode (string): band name. 16 | 17 | Raises: 18 | ValueError: unsupported band name. 19 | ValueError: unsupported upsample mode. 20 | 21 | Returns: 22 | torch.tensor: upsampled band. 23 | """ 24 | 25 | if not (band_name in BAND_NAMES): 26 | raise ValueError("Unsupported band name: " + colored(band_name, "red") + ".") 27 | 28 | # print("Upsampling band: "+colored(band_name, "blue")+".") 29 | upsample_factor = ( 30 | BAND_SPATIAL_RESOLUTION_DICT[band_name] / target_spatial_resolution 31 | ) 32 | if not (upsample_mode in ["nearest", "bilinear", "bicubic"]): 33 | raise ValueError( 34 | "Upsample mode " 35 | + colored(upsample_mode, "blue") 36 | + " not supported. Please, choose among: " 37 | "nearest" 38 | ", " 39 | "bilinear" 40 | ", " 41 | "bicubic" 42 | "." 43 | ) 44 | 45 | if upsample_factor <= 1.0: 46 | print( 47 | colored("Warnings", "red") 48 | + ". The requested target resolution (" 49 | + colored(target_spatial_resolution, "blue") 50 | + ") is lower or equal to the orginal band resolution (" 51 | + colored(band_name, "red") 52 | + "," 53 | + colored(BAND_SPATIAL_RESOLUTION_DICT[band_name], "green") 54 | + ")." 55 | ) 56 | return img_band 57 | 58 | if upsample_factor != int(upsample_factor): 59 | print( 60 | colored("Warnings", "red") 61 | + ". Upsample factor truncanted from " 62 | + upsample_factor 63 | + " to " 64 | + int(upsample_factor) 65 | + "." 66 | ) 67 | # else: 68 | # print("Upsample factor: "+colored(upsample_factor, "blue")+".") 69 | upsample_factor = int(upsample_factor) 70 | upsample_method = Upsample( 71 | scale_factor=upsample_factor, mode=upsample_mode, align_corners=True 72 | ) 73 | with torch.no_grad(): 74 | return upsample_method(img_band.unsqueeze(0).unsqueeze(0)).squeeze(0).squeeze(0) 75 | 76 | 77 | def image_band_resize( 78 | img_upsample_band, band_name, upsampled_img_spatial_resolution, interpolate_mode 79 | ): 80 | """Resize an upsampled image band to a the orginal spatial resolution through an interpolate mode. 81 | 82 | Args: 83 | img_upsample_band (torch.tensor): upsampled image to resize 84 | band_name (string): band name 85 | upsampled_img_spatial_resolution (float): spatial resolution of the input upsampled image. 86 | interpolate_mode (string): interpolated mode. 87 | 88 | Raises: 89 | ValueError: unsupported band name. 90 | ValueError: unsupported interpolated mode. 91 | 92 | Returns: 93 | torch.tensor: resized image. 94 | """ 95 | if not (band_name in BAND_NAMES): 96 | raise ValueError("Unsupported band name: " + colored(band_name, "red") + ".") 97 | 98 | # print("Downsampling band: "+colored(band_name, "blue")+".") 99 | downsample_factor = ( 100 | BAND_SPATIAL_RESOLUTION_DICT[band_name] / upsampled_img_spatial_resolution 101 | ) 102 | 103 | if not (interpolate_mode in ["nearest", "bilinear", "bicubic"]): 104 | raise ValueError( 105 | "Interpolate mode " 106 | + colored(interpolate_mode, "blue") 107 | + " not supported. Please, choose among: " 108 | "nearest" 109 | ", " 110 | "bilinear" 111 | ", " 112 | "bicubic" 113 | "." 114 | ) 115 | 116 | if downsample_factor <= 1.0: 117 | print( 118 | colored("Warnings", "red") 119 | + ". The upsampled image resolution (" 120 | + colored(upsampled_img_spatial_resolution, "blue") 121 | + ") is lower or equal to the orginal band resolution (" 122 | + colored(band_name, "red") 123 | + "," 124 | + colored(BAND_SPATIAL_RESOLUTION_DICT[band_name], "green") 125 | + ")." 126 | ) 127 | return img_upsample_band 128 | 129 | if downsample_factor != int(downsample_factor): 130 | print( 131 | colored("Warnings", "red") 132 | + ". Upsample factor truncanted from " 133 | + downsample_factor 134 | + " to " 135 | + int(downsample_factor) 136 | + "." 137 | ) 138 | # else: 139 | # print("Downsample factor: "+colored(downsample_factor, "blue")+".") 140 | downsample_factor = int(downsample_factor) 141 | 142 | size = ( 143 | int(img_upsample_band.shape[0] / downsample_factor), 144 | int(img_upsample_band.shape[1] / downsample_factor), 145 | ) 146 | with torch.no_grad(): 147 | return ( 148 | interpolate( 149 | img_upsample_band.unsqueeze(0).unsqueeze(0), 150 | size=size, 151 | mode=interpolate_mode, 152 | align_corners=True, 153 | ) 154 | .squeeze(0) 155 | .squeeze(0) 156 | ) 157 | -------------------------------------------------------------------------------- /pyraws/utils/constants.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | # ------------------------------------------------------------------------------------------------- 4 | # Database constants 5 | # ------------------------------------------------------------------------------------------------- 6 | # Database file names 7 | DATABASE_FILE_DICTIONARY = { 8 | "THRAWS": "thraws_db.csv", 9 | "vessel_detection": "vessel_detection_db.csv", 10 | } 11 | 12 | # ------------------------------------------------------------------------------------------------- 13 | # General constants 14 | # ------------------------------------------------------------------------------------------------- 15 | # Band names 16 | BAND_NAMES = [ 17 | "B01", 18 | "B02", 19 | "B03", 20 | "B04", 21 | "B05", 22 | "B06", 23 | "B07", 24 | "B08", 25 | "B8A", 26 | "B09", 27 | "B10", 28 | "B11", 29 | "B12", 30 | ] 31 | # Band resolution along-track dict 32 | BAND_SPATIAL_RESOLUTION_DICT = { 33 | "B01": 60.0, 34 | "B02": 10.0, 35 | "B03": 10.0, 36 | "B04": 10.0, 37 | "B05": 20.0, 38 | "B06": 20.0, 39 | "B07": 20.0, 40 | "B08": 10.0, 41 | "B8A": 20.0, 42 | "B09": 60.0, 43 | "B10": 60.0, 44 | "B11": 20.0, 45 | "B12": 20.0, 46 | } 47 | # Band names physical order 48 | BAND_NAMES_REAL_ORDER = [ 49 | "B02", 50 | "B08", 51 | "B03", 52 | "B10", 53 | "B04", 54 | "B05", 55 | "B11", 56 | "B06", 57 | "B07", 58 | "B8A", 59 | "B12", 60 | "B01", 61 | "B09", 62 | ] # Band names in the order they compare in the detectors. 63 | # SWIR bands 64 | SWIR_BANDS = ["B10", "B11", "B12"] 65 | 66 | # ------------------------------------------------------------------------------------------------- 67 | # Raw constants 68 | # ------------------------------------------------------------------------------------------------- 69 | # Bands shape dictionary 70 | BANDS_RAW_SHAPE_DICT = { 71 | "B02": torch.Size([2304, 2592]), 72 | "B08": torch.Size([2304, 2592]), 73 | "B03": torch.Size([2304, 2592]), 74 | "B10": torch.Size([384, 1296]), 75 | "B04": torch.Size([2304, 2592]), 76 | "B05": torch.Size([1152, 1296]), 77 | "B11": torch.Size([1152, 1296]), 78 | "B06": torch.Size([1152, 1296]), 79 | "B07": torch.Size([1152, 1296]), 80 | "B8A": torch.Size([1152, 1296]), 81 | "B12": torch.Size([1152, 1296]), 82 | "B01": torch.Size([384, 1296]), 83 | "B09": torch.Size([384, 1296]), 84 | } 85 | # Band resolution across-track dict 86 | BAND_SPATIAL_RESOLUTION_DICT_ACROSS = { 87 | "B01": 20.0, 88 | "B02": 10.0, 89 | "B03": 10.0, 90 | "B04": 10.0, 91 | "B05": 20.0, 92 | "B06": 20.0, 93 | "B07": 20.0, 94 | "B08": 10.0, 95 | "B8A": 20.0, 96 | "B09": 20.0, 97 | "B10": 20.0, 98 | "B11": 20.0, 99 | "B12": 20.0, 100 | } 101 | 102 | # ------------------------------------------------------------------------------------------------- 103 | # L1-c constants 104 | # ------------------------------------------------------------------------------------------------- 105 | # Default quantification value 106 | S2_DEFAULT_QUANTIFICATION_VALUE = ( 107 | 10000 # Default value used in case of missing metadata. 108 | ) 109 | -------------------------------------------------------------------------------- /pyraws/utils/date_utils.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | 4 | def get_timestamp(): 5 | """Return current timestamp. 6 | 7 | Returns: 8 | str: timestamp 9 | """ 10 | timestamp = datetime.now() 11 | return ( 12 | str(timestamp.year) 13 | + "_" 14 | + str(timestamp.month) 15 | + "_" 16 | + str(timestamp.day) 17 | + "_" 18 | + str(timestamp.hour) 19 | + "_" 20 | + str(timestamp.minute) 21 | + "_" 22 | + str(timestamp.second) 23 | ) 24 | 25 | 26 | def parse_string_date(date_string): 27 | """Parse a date in string format and returns a datetime. 28 | 29 | Args: 30 | date_string (string): date string 31 | 32 | Returns: 33 | datetime: correspondent datetime. 34 | """ 35 | year = date_string[:4] 36 | month = date_string[4:6] 37 | day = date_string[6:8] 38 | hour = date_string[9:11] 39 | minute = date_string[11:13] 40 | second = date_string[13:] 41 | return datetime( 42 | int(year), int(month), int(day), int(hour), int(minute), int(second) 43 | ) 44 | -------------------------------------------------------------------------------- /pyraws/utils/parse_csv_utils.py: -------------------------------------------------------------------------------- 1 | import csv 2 | 3 | 4 | def parse_csv(csv_name): 5 | """Parse a CSV file and return a list of rows. 6 | 7 | 8 | :csv_name: CSV name. 9 | :return: (list of rows. Each row is a dictionary.) 10 | """ 11 | row_list = [] 12 | try: 13 | with open(csv_name) as csv_file: 14 | csv_reader = csv.DictReader(csv_file) 15 | for row in csv_reader: 16 | row_list.append(row) 17 | except: # noqa: E722 18 | return ValueError("Impossible to parse CSV file: ", csv_name) 19 | 20 | return row_list 21 | -------------------------------------------------------------------------------- /pyraws/utils/shape_utils.py: -------------------------------------------------------------------------------- 1 | import geopandas as gpd 2 | import geopy 3 | from ..raw.raw_event import Raw_event 4 | from shapely.geometry import Polygon 5 | import numpy as np 6 | 7 | 8 | def get_granules_polys(requested_bands, event_name, device): 9 | """Returns the polygons over bands coordinates and correspondent indices. 10 | 11 | Args: 12 | requested_bands (list): requested bands. 13 | event_name (st): event ID. 14 | device (torch.device): device. 15 | 16 | Returns: 17 | dict: {granule_name : polygon} 18 | dict: {granule_name (e.g., granule_0_STACKED_T_granule_1) : index of granules stacked along track} 19 | """ 20 | raw_event = Raw_event(device=device) 21 | raw_event.from_database(event_name, requested_bands, verbose=False) 22 | stackable_granules, stackable_couples = raw_event.stack_granules_couples() 23 | names = [x.get_granule_info()[0] for x in stackable_granules] 24 | coords = [ 25 | x.get_bands_coordinates(latlon_format=False)[requested_bands[0]] 26 | for x in stackable_granules 27 | ] 28 | polys = [Polygon(x) for x in coords] 29 | poly_dict = dict(zip(names, polys)) 30 | polys_index = dict(zip(names, stackable_couples)) 31 | return poly_dict, polys_index 32 | 33 | 34 | def create_polygon_from_coordinates( 35 | lat, lon, center_distance_x=5000, center_distance_y=14000, json_name=None 36 | ): 37 | """Creates a rectangular polygon from coordinates (lat, lon). If json_name is not None, the polygon is saved as JSON file. 38 | 39 | Args: 40 | lat (float): center latitude. 41 | lon (float): center longitude. 42 | center_distance_x (int, optional): Horizontal distance from the center in m. Defaults to 5000. 43 | center_distance_y (int, optional): Vertical distance from the center in m. Defaults to 14000. 44 | json_name (str, optional): output JSON file name. If None, no file is created. Defaults to None. 45 | 46 | Returns: 47 | dict: polygon dict {"SW" : (lat, lon), "NW" : (lat, lon), "NE" : (lat, lon), "SE" : (lat, lon)} 48 | """ 49 | center_coordinates = {"lat": lat, "lon": lon} 50 | polygon_dict = create_rectangular_polygon( 51 | center_coordinates, center_distance_x, center_distance_y 52 | ) 53 | polygon = Polygon( 54 | [polygon_dict["SW"], polygon_dict["NW"], polygon_dict["NE"], polygon_dict["SE"]] 55 | ) 56 | json_content = gpd.GeoSeries([polygon]).to_json() 57 | 58 | if json_name is not None: 59 | with open(json_name, "w") as f: 60 | f.write(json_content) 61 | f.write("\n\n") 62 | f.write(str(polygon_dict["SW"])) 63 | f.write("\n\n") 64 | f.write(str(polygon_dict["NW"])) 65 | f.write("\n\n") 66 | f.write(str(polygon_dict["NE"])) 67 | f.write("\n\n") 68 | f.write(str(polygon_dict["SE"])) 69 | 70 | return polygon 71 | 72 | 73 | def create_rectangular_polygon( 74 | center_coordinates, center_distance_x, center_distance_y 75 | ): 76 | """Creates a rectangular polygon from coordinates (lat, lon). 77 | Args: 78 | center_coordinates (dict): center coordinates {"lat" : lat, "lon" : lon} 79 | center_distance_x (int, optional): Horizontal distance from the center in m. Defaults to 5000. 80 | center_distance_y (int, optional): Vertical distance from the center in m. Defaults to 14000. 81 | 82 | Returns: 83 | dict: polygon dict {"SW" : (lat, lon), "NW" : (lat, lon), "NE" : (lat, lon), "SE" : (lat, lon)} 84 | """ 85 | 86 | center_distance = np.sqrt( 87 | center_distance_x * center_distance_x + center_distance_y * center_distance_y 88 | ) 89 | bearing_angle_0 = 90 - np.arcsin(center_distance_y / center_distance) * 180 / np.pi 90 | # print(center_distance) 91 | 92 | bearing_angles = { 93 | "SW": 180 + bearing_angle_0, 94 | "NW": 360 - bearing_angle_0, 95 | "NE": bearing_angle_0, 96 | "SE": 180 - bearing_angle_0, 97 | } 98 | # print(bearing_angles) 99 | polygon_dict = {"SW": 0, "NW": 0, "NE": 0, "SE": 0} # South-West, North-West, ... 100 | for bearing_keys in list(bearing_angles.keys()): 101 | polygon_vertex = geopy.distance.distance(meters=center_distance).destination( 102 | (center_coordinates["lat"], center_coordinates["lon"]), 103 | bearing=bearing_angles[bearing_keys], 104 | ) 105 | polygon_dict[bearing_keys] = [polygon_vertex[1], polygon_vertex[0]] 106 | return polygon_dict 107 | -------------------------------------------------------------------------------- /pyraws/utils/visualization_utils.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import torch 3 | import numpy as np 4 | import matplotlib.patches as patches 5 | 6 | 7 | def image_histogram_equalization(image, number_bins=255): 8 | # from http://www.janeriksolem.net/histogram-equalization-with-python-and.html 9 | 10 | # get image histogram 11 | image_histogram, bins = np.histogram(image.flatten(), number_bins, density=True) 12 | cdf = image_histogram.cumsum() # cumulative distribution function 13 | cdf = (number_bins - 1) * cdf / cdf[-1] # normalize 14 | 15 | # use linear interpolation of cdf to find new pixel values 16 | image_equalized = np.interp(image.flatten(), bins[:-1], cdf) 17 | 18 | return image_equalized.reshape(image.shape), cdf 19 | 20 | 21 | def equalize_tensor(raw_granule_tensor, n_std=2): 22 | """Equalizes a tensor for a better visualization by clipping outliers of a histogram higher and lower than 23 | pixels value mean *- n_std times the standarda deviation. 24 | 25 | Args: 26 | raw_granule_tensor (torch.tensor): tensor to equalize. 27 | n_std (int, optional): Number of times the standard deviation. Defaults to 2. 28 | 29 | Returns: 30 | torch.tensor: equalized tensor. 31 | """ 32 | raw_granule_tensor_equalized = raw_granule_tensor.clone() 33 | for n in range(raw_granule_tensor.shape[2]): 34 | band = raw_granule_tensor_equalized[:, :, n] 35 | band_mean, band_std = band.mean(), band.std() 36 | # Histogram clipping: 37 | band[band < band_mean - n_std * band_std] = band_mean - n_std * band_std 38 | band[band > band_mean + n_std * band_std] = band_mean + n_std * band_std 39 | 40 | band, cdf = image_histogram_equalization(band.numpy(), number_bins=2 ** 16) 41 | band = torch.from_numpy(band) 42 | # band_clahe = clahe.apply((band.numpy() * CONVERSION ).astype(np.uint8)) 43 | # raw_granule_tensor_equalized[:,:,n]= torch.from_numpy(band_clahe/CONVERSION) 44 | raw_granule_tensor_equalized[:, :, n] = band 45 | 46 | return raw_granule_tensor_equalized 47 | 48 | 49 | def plot_img1_vs_img2_bands( 50 | img1_band, 51 | img2_band, 52 | img_name_list, 53 | alert_matrix=None, 54 | alert_matrix_unregistered=None, 55 | save_path=None, 56 | ): 57 | """Util function to visualize and compare the bands of two different images. It also allows adding an alert matrix. 58 | 59 | Args: 60 | img1_band (torch.tensor): first image band. 61 | img2_band (torch.tensor): second image band. 62 | img_name_list (list): list of names of different images. 63 | alert_matrix (torch.tensor, opional): if not None, the hotmap of normal band is shown. Defaults to None. 64 | alert_matrix_unregistered (torch.tensor, opional): if not None, the hotmap of unregstered band is shown. 65 | Defaults to None. 66 | save_path (string, optional): if not None, the image is saved at save_path. Defaults to None. 67 | """ 68 | cmap = "bone" 69 | fig, (ax1, ax2) = plt.subplots(1, 2) 70 | ax1.imshow(img1_band.detach().cpu().numpy(), cmap=cmap) 71 | if alert_matrix is not None: 72 | ax1.contour(alert_matrix.detach().cpu().numpy(), colors="r") 73 | ax1.grid(False) 74 | ax1.axis("off") 75 | ax1.title.set_text(img_name_list[0]) 76 | ax2.imshow(img2_band.detach().cpu().numpy(), cmap=cmap) 77 | if alert_matrix_unregistered is not None: 78 | ax2.contour(alert_matrix_unregistered.detach().cpu().numpy(), colors="r") 79 | ax2.grid(False) 80 | ax2.axis("off") 81 | ax2.title.set_text(img_name_list[1]) 82 | fig.tight_layout() 83 | plt.show() 84 | if save_path is not None: 85 | plt.savefig(save_path) 86 | 87 | 88 | def plot_event(img, img_name, bbox_list, alert_matrix=None, save_path=None): 89 | """Util function to visualize and compare the bands of two different images. It also allows adding an alert matrix. 90 | 91 | Args: 92 | img (torch.tensor): img. 93 | img_name (string): image_name. 94 | bbox_list (skimage properties): bbox list. 95 | alert_matrix (torch.tensor, opional): if not None, the hotmap of normal band is shown. Defaults to None. 96 | save_path (string, optional): if not None, the image is saved at save_path. Defaults to None. 97 | """ 98 | cmap = "bone" 99 | fig, ax = plt.subplots() 100 | ax.imshow(img.detach().cpu().numpy(), cmap=cmap) 101 | if alert_matrix is not None: 102 | ax.contour(alert_matrix.detach().cpu().numpy(), colors="r") 103 | ax.grid(False) 104 | ax.axis("off") 105 | ax.title.set_text(img_name) 106 | for prop in bbox_list: 107 | bbox = prop.bbox # x, y, width, height 108 | rect = patches.Rectangle( 109 | (bbox[1], bbox[0]), 110 | abs(bbox[1] - bbox[3]), 111 | abs(bbox[0] - bbox[2]), 112 | linewidth=2, 113 | edgecolor="y", 114 | facecolor="none", 115 | ) 116 | ax.add_patch(rect) 117 | 118 | fig.tight_layout() 119 | if save_path is not None: 120 | plt.savefig(save_path) 121 | -------------------------------------------------------------------------------- /pyraws_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script installs the pyraws package and its dependencies 4 | # Check if Conda is installed 5 | if ! command -v conda &> /dev/null 6 | then 7 | echo "Conda is not installed. Please install Conda and try again." 8 | echo "See https://docs.anaconda.com/anaconda/install/ for more information." 9 | exit 1 10 | fi 11 | 12 | # Create a new Conda environment called pyraws 13 | conda create --name pyraws python=3.9 -y 14 | 15 | # Activate the environment 16 | source $(conda info --base)/etc/profile.d/conda.sh 17 | if conda activate pyraws; then 18 | echo "pyraws environment activated" 19 | else 20 | echo "pyraws environment not found" 21 | exit 1 22 | fi 23 | 24 | 25 | # Define PyTorch version and components 26 | pytorch_version="1.11.0" 27 | torchvision_version="0.12.0" 28 | torchaudio_version="0.11.0" 29 | 30 | # Function to install PyTorch 31 | install_pytorch() { 32 | conda install -y pytorch==$pytorch_version torchvision==$torchvision_version torchaudio==$torchaudio_version $1 -c pytorch 33 | } 34 | 35 | # Install via pip: 36 | # pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cu113 37 | 38 | # Detect operating system 39 | case "$OSTYPE" in 40 | "darwin"*) 41 | # macOS 42 | echo "Detected macOS" 43 | install_pytorch "cpuonly" 44 | ;; 45 | "linux-gnu"|"linux-gnueabihf") 46 | # Linux 47 | echo "Detected Linux" 48 | # Install PyTorch with CUDA 11.3 support 49 | install_pytorch "cudatoolkit=11.3" 50 | ;; 51 | "msys"|"win32") 52 | # Windows 53 | echo "Detected Windows" 54 | # Install PyTorch with CUDA 11.3 support 55 | install_pytorch "cudatoolkit=11.3" 56 | ;; 57 | *) 58 | echo "Unsupported operating system" 59 | exit 1 60 | ;; 61 | esac 62 | 63 | 64 | # get absolute path of current working directory 65 | # and setup the sys_cfg.py file 66 | echo "PYRAWS_HOME_PATH = '$(pwd)'" > pyraws/sys_cfg.py 67 | echo "DATA_PATH = '$(pwd)/data'" >> pyraws/sys_cfg.py 68 | 69 | # install pyraws 70 | pip install -e . 71 | -------------------------------------------------------------------------------- /quickstart/API_demonstration/etna_00_granules.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESA-PhiLab/PyRawS/95d291388192d1c71ae302775c3bcf636717dc00/quickstart/API_demonstration/etna_00_granules.png -------------------------------------------------------------------------------- /quickstart/BBOX_L1C2RAW/results/Australia_1_original.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESA-PhiLab/PyRawS/95d291388192d1c71ae302775c3bcf636717dc00/quickstart/BBOX_L1C2RAW/results/Australia_1_original.pdf -------------------------------------------------------------------------------- /quickstart/BBOX_L1C2RAW/results/Australia_1_transformed.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESA-PhiLab/PyRawS/95d291388192d1c71ae302775c3bcf636717dc00/quickstart/BBOX_L1C2RAW/results/Australia_1_transformed.pdf -------------------------------------------------------------------------------- /quickstart/BBOX_L1C2RAW/results/Barren_Island_00_original.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESA-PhiLab/PyRawS/95d291388192d1c71ae302775c3bcf636717dc00/quickstart/BBOX_L1C2RAW/results/Barren_Island_00_original.pdf -------------------------------------------------------------------------------- /quickstart/BBOX_L1C2RAW/results/Barren_Island_00_transformed.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESA-PhiLab/PyRawS/95d291388192d1c71ae302775c3bcf636717dc00/quickstart/BBOX_L1C2RAW/results/Barren_Island_00_transformed.pdf -------------------------------------------------------------------------------- /quickstart/BBOX_L1C2RAW/results/Greece_5_original.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESA-PhiLab/PyRawS/95d291388192d1c71ae302775c3bcf636717dc00/quickstart/BBOX_L1C2RAW/results/Greece_5_original.pdf -------------------------------------------------------------------------------- /quickstart/BBOX_L1C2RAW/results/Greece_5_transformed.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESA-PhiLab/PyRawS/95d291388192d1c71ae302775c3bcf636717dc00/quickstart/BBOX_L1C2RAW/results/Greece_5_transformed.pdf -------------------------------------------------------------------------------- /quickstart/BBOX_L1C2RAW/tests/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESA-PhiLab/PyRawS/95d291388192d1c71ae302775c3bcf636717dc00/quickstart/BBOX_L1C2RAW/tests/.DS_Store -------------------------------------------------------------------------------- /quickstart/BBOX_L1C2RAW/tests/l1c_Greece_5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESA-PhiLab/PyRawS/95d291388192d1c71ae302775c3bcf636717dc00/quickstart/BBOX_L1C2RAW/tests/l1c_Greece_5.jpg -------------------------------------------------------------------------------- /quickstart/BBOX_L1C2RAW/tests/raw_Greece_5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESA-PhiLab/PyRawS/95d291388192d1c71ae302775c3bcf636717dc00/quickstart/BBOX_L1C2RAW/tests/raw_Greece_5.jpg -------------------------------------------------------------------------------- /quickstart/DB.md: -------------------------------------------------------------------------------- 1 | ## Working with Database 2 | 3 | This section of the respository contains an example of database (`THRAWS`) that is used by PyRawS to read and process [Sentinel-2 Raw data](#sentinel-2-raw-data) and [Sentinel-2 L1 data](#sentinel-2-l1-data) correctly. The minimal fields of the database include: 4 | 5 | > [!WARNING] 6 | > Additionally, make sure that all required dependencies and packages are installed before running the code in the notebook. Please note that it is important to carefully follow the instructions in the notebook to ensure that a database is created correctly and without errors. 7 | 8 | 9 | 10 | 11 | 12 | | ID_event | Start | End | Sat | Coords (Lat, Lon) | class | Raw_useful_granules | Raw_complementary_granules | Polygon_square | Raw_files | l1c_files | bbox_list | bbox_list_merged | Polygon | 13 | |-----------------------|-------------|-------------------|-----|----------------------|----------|---------------------|----------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------|--------------------------------------------|-------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 14 | | Barren_Island_00 | 28/09/2018 | 28/09/2018 23:59 | S2A | (12.28474241, 93.86212046) | eruption | [2] | [4] | POLYGON ((93.82076157486179 12.244061101857305, 93.8207488665122 12.325417348895849, 93.9034920534878 12.325417348895849, 93.90347934513821 12.244061101857305, 93.82076157486179 12.244061101857305)) | ['S2A_OPER_MSI_L0__GR_EPAE_20180929T132957_S20180928T042450_D10_N02.06', 'S2A_OPER_MSI_L0__GR_EPAE_20180929T132957_S20180928T042453_D09_N02.06', 'S2A_OPER_MSI_L0__GR_EPAE_20180929T132957_S20180928T042453_D10_N02.06', 'S2A_OPER_MSI_L0__GR_EPAE_20180929T132957_S20180928T042457_D09_N02.06', 'S2A_OPER_MSI_L0__GR_EPAE_20180929T132957_S20180928T042457_D10_N02.06'] | ['S2A_MSIL1C_20180928T040541_N0206_R047_T46PEU_20180929T135308'] | {2: [[[403, 290], [427, 284], [435, 316], [410, 322]]]} | {2: [[[403, 290], [427, 284], [435, 316], [410, 322], [403, 290]]]} | POLYGON((93.81618097217948 12.158183811523221, 93.81613704194822 12.411292126632908, 93.90810387805179 12.41129212663291, 93.90805994782052 12.158183811523221, 93.81618097217948 12.158183811523221)) | 15 | | Barren_Island_01 | 28/10/2018 | 28/10/2018 23:59 | S2A | (12.28474241, 93.86212046) | eruption | [2] | [4] | POLYGON ((93.82076157486179 12.244061101857305, 93.8207488665122 12.325417348895849, 93.9034920534878 12.325417348895849, 93.90347934513821 12.244061101857305, 93.82076157486179 12.244061101857305)) | ['S2A_OPER_MSI_L0__GR_EPAE_20181028T064658_S20181028T042451_D10_N02.06', 'S2A_OPER_MSI_L0__GR_EPAE_20181028T064658_S20181028T042455_D09_N02.06', 'S2A_OPER_MSI_L0__GR_EPAE_20181028T064658_S20181028T042455_D10_N02.06', 'S2A_OPER_MSI_L0__GR_EPAE_20181028T064658_S20181028T042458_D09_N02.06', 'S2A_OPER_MSI_L0__GR_EPAE_20181028T064658_S20181028T042458_D10_N02.06'] | ['S2A_MSIL1C_20181028T040851_N0206_R047_T46PEU_20181028T070450'] | {2: [[[391, 59], [412, 54], [418, 16 | 17 | 18 | 19 | 20 | 21 | * **ID_event**: ID of the event (e.g., volcanic-eruption, wildfire, not-event). All the other fields of the row are referred to that `Sentinel-2` acquisition. 22 | * **class**: class of the event (e.g., eruption, fire, not-event). Leave it **empty** 23 | * **Raw_useful_granules**: list of [Raw useful granules](#raw-useful-granule). Set to `None` or leave it empty if you do not know what are the [Raw useful granules](#raw-useful-granule). 24 | * **Raw_complementary_granules**: list of [Raw complementry granules](#raw-complementary-granule). Set to `None` or leave it empty if you do not know what are the [Raw complementry granules](#raw-complementary-granule). 25 | * **Raw_files**: list of [Raw granules](#sentinel-2-raw-granule) (**mandatory**). 26 | * **l1c_files**: list of [L1 tiles](#sentinel-2-l1c-tile) (mandatory if you need L1C data). 27 | * **bbox_list**: dictionary {[Raw useful granules](#raw-useful-granule) : [bounding box list for that granule]}. Set to `None` or leave it **empty** if you do not know the bounding box location. 28 | 29 | To create a new database (e.g., `my_database_name`), please, proceed as follows: 30 | 31 | 1. Create a ".csv" file with the structure shown above and place it into the `database`subfloders (e.g., `my_db.csv`). You can use start from the [thraws_db.csv](https://github.com/ESA-PhiLab/PyRawS/-/blob/main/PyRawS/database/thraws_db.csv) database and edit it accordingly to your specification. 32 | 2. Create subdirectory `my_database_name` in the `data` subdirectory and populate it with the corresponding [Sentinel-2 Raw data](#sentinel-2-raw-data) and [Sentinel-2 L1 data](#sentinel-2-l1-data) as described in [Data directory](#data-directory). 33 | 3. Update the `DATABASE_FILE_DICTIONARY` in `PyRawS/utils/constants.py` as follows: 34 | 35 | ```DATABASE_FILE_DICTIONARY={"THRAWS" : "thraws_db.csv", "my_database_name" : "my_db.csv"}``` 36 | 37 | > [!IMPORTANT] 38 | > In case you want to create your own database of event for your target aplication, the user should refer to the notebook called "database_creation.ipynb". This notebook contains the necessary code and instructions for creating the database. Simply follow the steps outlined in the notebook to successfully create something similar to the "THRAWS" database. 39 | 40 | > [!TIP] 41 | > **N.B** The creation of a database is not mandatory. However, it is strongly advisable. Indeed, without creating a database you can still open `Raw data` as described in [Open a Raw event from path](#open-a-raw-event-from-path). However, some pieces of information such as the [Raw useful granules](#raw-useful-granule) associated to a specific event, the event bounding boxes or the image class can be retrieved only when the database is set-up. -------------------------------------------------------------------------------- /quickstart/DB_creation/pyraws_Raw_L1_database_creation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "attachments": {}, 5 | "cell_type": "markdown", 6 | "metadata": {}, 7 | "source": [ 8 | "# PYRAWS - DB API demonstration." 9 | ] 10 | }, 11 | { 12 | "attachments": {}, 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "This notebook is to show and demonstrate the use of Application Program Interface (API) to create a Raw-L1 dataset for API integration with the `Sentinel-2 Raw data`, corresponding to a decompressed version of [Sentinel-2 L0 data](https://sentinel.esa.int/documents/247904/685211/sentinel-2-products-specification-document) with additional metada. The notebook has been developed in the framework of the `PYRAWS` project.
The DB API are demonstrated on the `Temperature Hotspots RAW Sentinel-2 (THRAWS)` dataset." 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "metadata": {}, 22 | "source": [ 23 | "# 1) Imports, paths and variables" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": null, 29 | "metadata": { 30 | "notebookRunGroups": { 31 | "groupValue": "1" 32 | } 33 | }, 34 | "outputs": [], 35 | "source": [ 36 | "import os\n", 37 | "os.environ['CUDA_VISIBLE_DEVICES']='5'" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "Autoreload." 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "metadata": { 51 | "notebookRunGroups": { 52 | "groupValue": "1" 53 | } 54 | }, 55 | "outputs": [], 56 | "source": [ 57 | "%load_ext autoreload\n", 58 | "%autoreload 2" 59 | ] 60 | }, 61 | { 62 | "cell_type": "markdown", 63 | "metadata": {}, 64 | "source": [ 65 | "Imports." 66 | ] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "execution_count": null, 71 | "metadata": {}, 72 | "outputs": [], 73 | "source": [ 74 | "from pyraws.utils.database_utils import DatabaseHandler\n", 75 | "from pathlib import Path\n", 76 | "import pandas as pd\n", 77 | "\n", 78 | "try:\n", 79 | " from pyraws.sys_cfg import PYRAWS_HOME_PATH, DATA_PATH\n", 80 | "except:\n", 81 | " raise ValueError(\"sys_cfg.py not found. Please, refer to README.md for instructions on how to generate it.\")" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "metadata": {}, 87 | "source": [ 88 | "This import is to remove odd errors on `libiomp5md.dll`. If you do not have them, you can skip it" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": null, 94 | "metadata": { 95 | "notebookRunGroups": { 96 | "groupValue": "1" 97 | } 98 | }, 99 | "outputs": [], 100 | "source": [ 101 | "os.environ['KMP_DUPLICATE_LIB_OK']='True'" 102 | ] 103 | }, 104 | { 105 | "attachments": {}, 106 | "cell_type": "markdown", 107 | "metadata": {}, 108 | "source": [ 109 | "# 2) - DB Creation" 110 | ] 111 | }, 112 | { 113 | "attachments": {}, 114 | "cell_type": "markdown", 115 | "metadata": {}, 116 | "source": [ 117 | "The DatabaseHandler class is used for managing and parsing information from a database consisting of folders and files. The class takes two arguments: db_name is the name of the database and datapath is the path of the data. The class initializes with a fetcher function which is used to fetch the database and the L1 and Raw products." 118 | ] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": null, 123 | "metadata": {}, 124 | "outputs": [], 125 | "source": [ 126 | "db_name = 'THRAWS'\n", 127 | "DB = DatabaseHandler(db_name)" 128 | ] 129 | }, 130 | { 131 | "attachments": {}, 132 | "cell_type": "markdown", 133 | "metadata": {}, 134 | "source": [ 135 | "The class has a single_parser method, which takes in the path of an event folder and extracts metadata and other information from the files within the folder. The meta_extract function within the method is used to extract the xmlns, Satellite_Code and the polygon of the event from the Inventory_Metadata.xml file. The gain_xml function within the method is used to locate the Inventory_Metadata.xml file. Instead, the parser method is used to parse the entire L0 folder." 136 | ] 137 | }, 138 | { 139 | "attachments": {}, 140 | "cell_type": "markdown", 141 | "metadata": {}, 142 | "source": [ 143 | "The class also has a parser method which iterates over the L0_products and calls the single_parser method on each of them. The returned information is saved in a pandas dataframe called db." 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": null, 149 | "metadata": {}, 150 | "outputs": [], 151 | "source": [ 152 | "DB.parser()" 153 | ] 154 | }, 155 | { 156 | "cell_type": "code", 157 | "execution_count": null, 158 | "metadata": {}, 159 | "outputs": [], 160 | "source": [ 161 | "dataframe = DB.db\n", 162 | "dataframe" 163 | ] 164 | }, 165 | { 166 | "attachments": {}, 167 | "cell_type": "markdown", 168 | "metadata": {}, 169 | "source": [ 170 | "# 3) - DB Saving" 171 | ] 172 | }, 173 | { 174 | "attachments": {}, 175 | "cell_type": "markdown", 176 | "metadata": {}, 177 | "source": [ 178 | "Save the current database to a csv file." 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": null, 184 | "metadata": {}, 185 | "outputs": [], 186 | "source": [ 187 | "db_path = Path(PYRAWS_HOME_PATH) / 'pyraws' / 'database' / 'test_db_toremove.csv'\n", 188 | "\n", 189 | "dataframe.to_csv(db_path, index=False)\n" 190 | ] 191 | }, 192 | { 193 | "attachments": {}, 194 | "cell_type": "markdown", 195 | "metadata": {}, 196 | "source": [ 197 | "Please note that this class depends on the existence of raw and l1c folders, if they don't exist it will return an error or a message as per the class fetcher function. Also this class uses pathlib, pandas and xml.etree.ElementTree." 198 | ] 199 | } 200 | ], 201 | "metadata": { 202 | "kernelspec": { 203 | "display_name": "pyraws", 204 | "language": "python", 205 | "name": "python3" 206 | }, 207 | "language_info": { 208 | "codemirror_mode": { 209 | "name": "ipython", 210 | "version": 3 211 | }, 212 | "file_extension": ".py", 213 | "mimetype": "text/x-python", 214 | "name": "python", 215 | "nbconvert_exporter": "python", 216 | "pygments_lexer": "ipython3", 217 | "version": "3.9.15" 218 | }, 219 | "orig_nbformat": 4, 220 | "vscode": { 221 | "interpreter": { 222 | "hash": "e44427996730295f72801af1daa8325e72b53a1ddad2a1fad7e798c530cb13b3" 223 | } 224 | } 225 | }, 226 | "nbformat": 4, 227 | "nbformat_minor": 2 228 | } 229 | -------------------------------------------------------------------------------- /quickstart/INSTALLATION.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | We provide several methods to install pyraws: 4 | 5 | ## 1. Build from source 6 | For Linux, the installation is straightforward. 7 | You just need to run the following command from the main directory: 8 | ```bash 9 | source pyraws_install.sh 10 | ``` 11 | > [!WARNING] 12 | > Your datates should be placed in the /data directory in the main. 13 | 14 | > [!TIP] 15 | > You can also install by the environment.yml. 16 | 17 | ## 2. PyPi 18 | 19 | From CLI: 20 | 21 | ```bash 22 | pip3 install pyraws 23 | ``` 24 | 25 | > [!WARNING] 26 | > Create a file `sys_cfg.py` in the directory `/pyraws/pyraws/`, and add two variables as follows: 27 | > ```cmd 28 | > PYRAWS_HOME_PATH="Absolute path to the main pyraws directory." 29 | > DATA_PATH="Absolute path to the data directory." 30 | > ``` 31 | > By default the data directory is located in PyRawS main directory. 32 | 33 | 34 | ## 3. Docker Compose 35 | 36 | PyRawS comes also delivered in a [docker image](sirbastiano94/pyraws:latest ). To use PyRawS with docker, you can pull the image or build it trough the Dockerfile. 37 | 38 | We already set the docker-compose.yml file integrated with the capabilities of NVIDIA docker. You can run the container trough the compose API: 39 | 40 | >>```bash 41 | >> .devcontainer/ > docker-compose up --build 42 | >>``` 43 | 44 | You can also run the container trough the devcontainer extension of VSCode, a devcontainer.json has been already provided. 45 | 46 | 47 | ## *Set-up for coregistration study* 48 | 49 | If you want to perform the coregistration study, you need to: 50 | 51 | 1. clone the repository [SuperGlue Inference and Evaluation Demo Script](https://github.com/magicleap/SuperGluePretrainedNetwork); 52 | 2. rename the subdirectory `models` to `superglue_models`; 53 | 3. move `superglue_models` into `coregistration_study`. 54 | 55 |
56 | 57 | Coregistration study results can be generated by using the `pyraws_coregistration_study.ipynb` notebook. The database `coregistration_study_db.csv` containing info on the discarded eruption events is used to generate results in the notebook. -------------------------------------------------------------------------------- /resources/images/PyRawS_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESA-PhiLab/PyRawS/95d291388192d1c71ae302775c3bcf636717dc00/resources/images/PyRawS_logo.png -------------------------------------------------------------------------------- /resources/images/coregistration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESA-PhiLab/PyRawS/95d291388192d1c71ae302775c3bcf636717dc00/resources/images/coregistration.png -------------------------------------------------------------------------------- /resources/images/database_structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESA-PhiLab/PyRawS/95d291388192d1c71ae302775c3bcf636717dc00/resources/images/database_structure.png -------------------------------------------------------------------------------- /resources/images/etna_00_granules.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESA-PhiLab/PyRawS/95d291388192d1c71ae302775c3bcf636717dc00/resources/images/etna_00_granules.png -------------------------------------------------------------------------------- /resources/images/granule_superimposition.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESA-PhiLab/PyRawS/95d291388192d1c71ae302775c3bcf636717dc00/resources/images/granule_superimposition.png -------------------------------------------------------------------------------- /resources/images/quicklook_yt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESA-PhiLab/PyRawS/95d291388192d1c71ae302775c3bcf636717dc00/resources/images/quicklook_yt.png -------------------------------------------------------------------------------- /resources/img_geo.ai: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESA-PhiLab/PyRawS/95d291388192d1c71ae302775c3bcf636717dc00/resources/img_geo.ai -------------------------------------------------------------------------------- /resources/img_geo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESA-PhiLab/PyRawS/95d291388192d1c71ae302775c3bcf636717dc00/resources/img_geo.png -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_profiling_results/desktop/profiling_n_images_16_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 2 | Name Self CPU % Self CPU CPU total % CPU total CPU time avg Self CUDA Self CUDA % CUDA total CUDA time avg # of Calls 3 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 4 | void at::native::(anonymous namespace)::grid_sampler... 0.00% 0.000us 0.00% 0.000us 0.000us 12.519ms 88.88% 12.519ms 391.219us 32 5 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 817.000us 5.80% 817.000us 8.510us 96 6 | void at::native::vectorized_elementwise_kernel<4, at... 0.00% 0.000us 0.00% 0.000us 0.000us 376.000us 2.67% 376.000us 5.875us 64 7 | volta_sgemm_128x32_tn 0.00% 0.000us 0.00% 0.000us 0.000us 146.000us 1.04% 146.000us 4.562us 32 8 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 95.000us 0.67% 95.000us 2.969us 32 9 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 61.000us 0.43% 61.000us 1.906us 32 10 | Memcpy HtoD (Pageable -> Device) 0.00% 0.000us 0.00% 0.000us 0.000us 51.000us 0.36% 51.000us 0.785us 65 11 | void (anonymous namespace)::elementwise_kernel_with_... 0.00% 0.000us 0.00% 0.000us 0.000us 20.000us 0.14% 20.000us 0.625us 32 12 | cudaMalloc 0.08% 1.388ms 0.08% 1.388ms 198.286us 0.000us 0.00% 0.000us 0.000us 7 13 | cudaMemcpyAsync 0.05% 802.000us 0.05% 802.000us 12.531us 0.000us 0.00% 0.000us 0.000us 64 14 | cudaStreamSynchronize 0.40% 6.889ms 0.40% 6.889ms 107.641us 0.000us 0.00% 0.000us 0.000us 64 15 | cudaLaunchKernel 67.48% 1.165s 67.48% 1.165s 3.640ms 0.000us 0.00% 0.000us 0.000us 320 16 | cudaFree 31.98% 551.937ms 31.98% 551.937ms 183.979ms 0.000us 0.00% 0.000us 0.000us 3 17 | cudaDeviceGetAttribute 0.00% 1.000us 0.00% 1.000us 0.091us 0.000us 0.00% 0.000us 0.000us 11 18 | cudaMemcpy 0.00% 22.000us 0.00% 22.000us 22.000us 0.000us 0.00% 0.000us 0.000us 1 19 | cudaFuncSetAttribute 0.00% 3.000us 0.00% 3.000us 0.018us 0.000us 0.00% 0.000us 0.000us 169 20 | cudaEventCreateWithFlags 0.00% 3.000us 0.00% 3.000us 0.167us 0.000us 0.00% 0.000us 0.000us 18 21 | cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFla... 0.00% 32.000us 0.00% 32.000us 1.000us 0.000us 0.00% 0.000us 0.000us 32 22 | cudaDeviceSynchronize 0.01% 196.000us 0.01% 196.000us 196.000us 0.000us 0.00% 0.000us 0.000us 1 23 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 24 | Self CPU time total: 1.726s 25 | Self CUDA time total: 14.085ms 26 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_profiling_results/desktop/profiling_n_images_16_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 2 | Name Self CPU % Self CPU CPU total % CPU total CPU time avg Self CUDA Self CUDA % CUDA total CUDA time avg # of Calls 3 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 4 | void at::native::(anonymous namespace)::grid_sampler... 0.00% 0.000us 0.00% 0.000us 0.000us 12.529ms 89.02% 12.529ms 391.531us 32 5 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 807.000us 5.73% 807.000us 8.406us 96 6 | void at::native::vectorized_elementwise_kernel<4, at... 0.00% 0.000us 0.00% 0.000us 0.000us 362.000us 2.57% 362.000us 5.656us 64 7 | volta_sgemm_128x32_tn 0.00% 0.000us 0.00% 0.000us 0.000us 147.000us 1.04% 147.000us 4.594us 32 8 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 96.000us 0.68% 96.000us 3.000us 32 9 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 62.000us 0.44% 62.000us 1.938us 32 10 | Memcpy HtoD (Pageable -> Device) 0.00% 0.000us 0.00% 0.000us 0.000us 43.000us 0.31% 43.000us 0.662us 65 11 | void (anonymous namespace)::elementwise_kernel_with_... 0.00% 0.000us 0.00% 0.000us 0.000us 28.000us 0.20% 28.000us 0.875us 32 12 | cudaMalloc 0.08% 1.308ms 0.08% 1.308ms 186.857us 0.000us 0.00% 0.000us 0.000us 7 13 | cudaMemcpyAsync 0.04% 771.000us 0.04% 771.000us 12.047us 0.000us 0.00% 0.000us 0.000us 64 14 | cudaStreamSynchronize 0.40% 6.885ms 0.40% 6.885ms 107.578us 0.000us 0.00% 0.000us 0.000us 64 15 | cudaLaunchKernel 67.63% 1.162s 67.63% 1.162s 3.630ms 0.000us 0.00% 0.000us 0.000us 320 16 | cudaFree 31.84% 546.824ms 31.84% 546.824ms 182.275ms 0.000us 0.00% 0.000us 0.000us 3 17 | cudaDeviceGetAttribute 0.00% 1.000us 0.00% 1.000us 0.091us 0.000us 0.00% 0.000us 0.000us 11 18 | cudaMemcpy 0.00% 24.000us 0.00% 24.000us 24.000us 0.000us 0.00% 0.000us 0.000us 1 19 | cudaFuncSetAttribute 0.00% 3.000us 0.00% 3.000us 0.018us 0.000us 0.00% 0.000us 0.000us 169 20 | cudaEventCreateWithFlags 0.00% 2.000us 0.00% 2.000us 0.111us 0.000us 0.00% 0.000us 0.000us 18 21 | cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFla... 0.00% 32.000us 0.00% 32.000us 1.000us 0.000us 0.00% 0.000us 0.000us 32 22 | cudaDeviceSynchronize 0.01% 138.000us 0.01% 138.000us 138.000us 0.000us 0.00% 0.000us 0.000us 1 23 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 24 | Self CPU time total: 1.718s 25 | Self CUDA time total: 14.074ms 26 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_profiling_results/desktop/profiling_n_images_16_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 2 | Name Self CPU % Self CPU CPU total % CPU total CPU time avg Self CUDA Self CUDA % CUDA total CUDA time avg # of Calls 3 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 4 | void at::native::(anonymous namespace)::grid_sampler... 0.00% 0.000us 0.00% 0.000us 0.000us 12.521ms 89.13% 12.521ms 391.281us 32 5 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 798.000us 5.68% 798.000us 8.312us 96 6 | void at::native::vectorized_elementwise_kernel<4, at... 0.00% 0.000us 0.00% 0.000us 0.000us 365.000us 2.60% 365.000us 5.703us 64 7 | volta_sgemm_128x32_tn 0.00% 0.000us 0.00% 0.000us 0.000us 155.000us 1.10% 155.000us 4.844us 32 8 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 82.000us 0.58% 82.000us 2.562us 32 9 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 62.000us 0.44% 62.000us 1.938us 32 10 | Memcpy HtoD (Pageable -> Device) 0.00% 0.000us 0.00% 0.000us 0.000us 43.000us 0.31% 43.000us 0.662us 65 11 | void (anonymous namespace)::elementwise_kernel_with_... 0.00% 0.000us 0.00% 0.000us 0.000us 22.000us 0.16% 22.000us 0.688us 32 12 | cudaMalloc 0.08% 1.335ms 0.08% 1.335ms 190.714us 0.000us 0.00% 0.000us 0.000us 7 13 | cudaMemcpyAsync 0.04% 776.000us 0.04% 776.000us 12.125us 0.000us 0.00% 0.000us 0.000us 64 14 | cudaStreamSynchronize 0.40% 6.841ms 0.40% 6.841ms 106.891us 0.000us 0.00% 0.000us 0.000us 64 15 | cudaLaunchKernel 67.45% 1.164s 67.45% 1.164s 3.638ms 0.000us 0.00% 0.000us 0.000us 320 16 | cudaFree 32.01% 552.470ms 32.01% 552.470ms 184.157ms 0.000us 0.00% 0.000us 0.000us 3 17 | cudaDeviceGetAttribute 0.00% 1.000us 0.00% 1.000us 0.091us 0.000us 0.00% 0.000us 0.000us 11 18 | cudaMemcpy 0.00% 22.000us 0.00% 22.000us 22.000us 0.000us 0.00% 0.000us 0.000us 1 19 | cudaFuncSetAttribute 0.00% 3.000us 0.00% 3.000us 0.018us 0.000us 0.00% 0.000us 0.000us 169 20 | cudaEventCreateWithFlags 0.00% 2.000us 0.00% 2.000us 0.111us 0.000us 0.00% 0.000us 0.000us 18 21 | cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFla... 0.00% 32.000us 0.00% 32.000us 1.000us 0.000us 0.00% 0.000us 0.000us 32 22 | cudaDeviceSynchronize 0.01% 193.000us 0.01% 193.000us 193.000us 0.000us 0.00% 0.000us 0.000us 1 23 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 24 | Self CPU time total: 1.726s 25 | Self CUDA time total: 14.048ms 26 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_profiling_results/desktop/profiling_n_images_1_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 2 | Name Self CPU % Self CPU CPU total % CPU total CPU time avg Self CUDA Self CUDA % CUDA total CUDA time avg # of Calls 3 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 4 | void at::native::(anonymous namespace)::grid_sampler... 0.00% 0.000us 0.00% 0.000us 0.000us 783.000us 88.67% 783.000us 391.500us 2 5 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 52.000us 5.89% 52.000us 8.667us 6 6 | void at::native::vectorized_elementwise_kernel<4, at... 0.00% 0.000us 0.00% 0.000us 0.000us 25.000us 2.83% 25.000us 6.250us 4 7 | volta_sgemm_128x32_tn 0.00% 0.000us 0.00% 0.000us 0.000us 11.000us 1.25% 11.000us 5.500us 2 8 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 6.000us 0.68% 6.000us 3.000us 2 9 | Memcpy HtoD (Pageable -> Device) 0.00% 0.000us 0.00% 0.000us 0.000us 3.000us 0.34% 3.000us 0.600us 5 10 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 3.000us 0.34% 3.000us 1.500us 2 11 | cudaMalloc 0.06% 1.050ms 0.06% 1.050ms 175.000us 0.000us 0.00% 0.000us 0.000us 6 12 | cudaMemcpyAsync 0.00% 67.000us 0.00% 67.000us 16.750us 0.000us 0.00% 0.000us 0.000us 4 13 | cudaStreamSynchronize 0.00% 24.000us 0.00% 24.000us 6.000us 0.000us 0.00% 0.000us 0.000us 4 14 | cudaLaunchKernel 67.49% 1.149s 67.49% 1.149s 57.430ms 0.000us 0.00% 0.000us 0.000us 20 15 | void (anonymous namespace)::elementwise_kernel_with_... 0.00% 0.000us 0.00% 0.000us 0.000us 0.000us 0.00% 0.000us 0.000us 2 16 | cudaFree 32.43% 551.823ms 32.43% 551.823ms 183.941ms 0.000us 0.00% 0.000us 0.000us 3 17 | cudaDeviceGetAttribute 0.00% 1.000us 0.00% 1.000us 0.091us 0.000us 0.00% 0.000us 0.000us 11 18 | cudaMemcpy 0.00% 21.000us 0.00% 21.000us 21.000us 0.000us 0.00% 0.000us 0.000us 1 19 | cudaFuncSetAttribute 0.00% 4.000us 0.00% 4.000us 0.024us 0.000us 0.00% 0.000us 0.000us 169 20 | cudaEventCreateWithFlags 0.00% 10.000us 0.00% 10.000us 0.556us 0.000us 0.00% 0.000us 0.000us 18 21 | cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFla... 0.00% 2.000us 0.00% 2.000us 1.000us 0.000us 0.00% 0.000us 0.000us 2 22 | cudaDeviceSynchronize 0.01% 184.000us 0.01% 184.000us 184.000us 0.000us 0.00% 0.000us 0.000us 1 23 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 24 | Self CPU time total: 1.702s 25 | Self CUDA time total: 883.000us 26 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_profiling_results/desktop/profiling_n_images_1_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 2 | Name Self CPU % Self CPU CPU total % CPU total CPU time avg Self CUDA Self CUDA % CUDA total CUDA time avg # of Calls 3 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 4 | void at::native::(anonymous namespace)::grid_sampler... 0.00% 0.000us 0.00% 0.000us 0.000us 783.000us 87.78% 783.000us 391.500us 2 5 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 52.000us 5.83% 52.000us 8.667us 6 6 | void at::native::vectorized_elementwise_kernel<4, at... 0.00% 0.000us 0.00% 0.000us 0.000us 26.000us 2.91% 26.000us 6.500us 4 7 | volta_sgemm_128x32_tn 0.00% 0.000us 0.00% 0.000us 0.000us 11.000us 1.23% 11.000us 5.500us 2 8 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 8.000us 0.90% 8.000us 4.000us 2 9 | Memcpy HtoD (Pageable -> Device) 0.00% 0.000us 0.00% 0.000us 0.000us 6.000us 0.67% 6.000us 1.200us 5 10 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 4.000us 0.45% 4.000us 2.000us 2 11 | void (anonymous namespace)::elementwise_kernel_with_... 0.00% 0.000us 0.00% 0.000us 0.000us 2.000us 0.22% 2.000us 1.000us 2 12 | cudaMalloc 0.06% 1.064ms 0.06% 1.064ms 177.333us 0.000us 0.00% 0.000us 0.000us 6 13 | cudaMemcpyAsync 0.00% 63.000us 0.00% 63.000us 15.750us 0.000us 0.00% 0.000us 0.000us 4 14 | cudaStreamSynchronize 0.00% 24.000us 0.00% 24.000us 6.000us 0.000us 0.00% 0.000us 0.000us 4 15 | cudaLaunchKernel 67.88% 1.156s 67.88% 1.156s 57.794ms 0.000us 0.00% 0.000us 0.000us 20 16 | cudaFree 32.04% 545.617ms 32.04% 545.617ms 181.872ms 0.000us 0.00% 0.000us 0.000us 3 17 | cudaDeviceGetAttribute 0.00% 1.000us 0.00% 1.000us 0.091us 0.000us 0.00% 0.000us 0.000us 11 18 | cudaMemcpy 0.00% 19.000us 0.00% 19.000us 19.000us 0.000us 0.00% 0.000us 0.000us 1 19 | cudaFuncSetAttribute 0.00% 9.000us 0.00% 9.000us 0.053us 0.000us 0.00% 0.000us 0.000us 169 20 | cudaEventCreateWithFlags 0.00% 2.000us 0.00% 2.000us 0.111us 0.000us 0.00% 0.000us 0.000us 18 21 | cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFla... 0.00% 2.000us 0.00% 2.000us 1.000us 0.000us 0.00% 0.000us 0.000us 2 22 | cudaDeviceSynchronize 0.01% 191.000us 0.01% 191.000us 191.000us 0.000us 0.00% 0.000us 0.000us 1 23 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 24 | Self CPU time total: 1.703s 25 | Self CUDA time total: 892.000us 26 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_profiling_results/desktop/profiling_n_images_1_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 2 | Name Self CPU % Self CPU CPU total % CPU total CPU time avg Self CUDA Self CUDA % CUDA total CUDA time avg # of Calls 3 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 4 | void at::native::(anonymous namespace)::grid_sampler... 0.00% 0.000us 0.00% 0.000us 0.000us 781.000us 88.85% 781.000us 390.500us 2 5 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 50.000us 5.69% 50.000us 8.333us 6 6 | void at::native::vectorized_elementwise_kernel<4, at... 0.00% 0.000us 0.00% 0.000us 0.000us 23.000us 2.62% 23.000us 5.750us 4 7 | volta_sgemm_128x32_tn 0.00% 0.000us 0.00% 0.000us 0.000us 10.000us 1.14% 10.000us 5.000us 2 8 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 6.000us 0.68% 6.000us 3.000us 2 9 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 4.000us 0.46% 4.000us 2.000us 2 10 | Memcpy HtoD (Pageable -> Device) 0.00% 0.000us 0.00% 0.000us 0.000us 3.000us 0.34% 3.000us 0.600us 5 11 | void (anonymous namespace)::elementwise_kernel_with_... 0.00% 0.000us 0.00% 0.000us 0.000us 2.000us 0.23% 2.000us 1.000us 2 12 | cudaMalloc 0.06% 1.023ms 0.06% 1.023ms 170.500us 0.000us 0.00% 0.000us 0.000us 6 13 | cudaMemcpyAsync 0.00% 69.000us 0.00% 69.000us 17.250us 0.000us 0.00% 0.000us 0.000us 4 14 | cudaStreamSynchronize 0.00% 23.000us 0.00% 23.000us 5.750us 0.000us 0.00% 0.000us 0.000us 4 15 | cudaLaunchKernel 67.86% 1.163s 67.86% 1.163s 58.151ms 0.000us 0.00% 0.000us 0.000us 20 16 | cudaFree 32.06% 549.462ms 32.06% 549.462ms 183.154ms 0.000us 0.00% 0.000us 0.000us 3 17 | cudaDeviceGetAttribute 0.00% 1.000us 0.00% 1.000us 0.091us 0.000us 0.00% 0.000us 0.000us 11 18 | cudaMemcpy 0.00% 22.000us 0.00% 22.000us 22.000us 0.000us 0.00% 0.000us 0.000us 1 19 | cudaFuncSetAttribute 0.00% 11.000us 0.00% 11.000us 0.065us 0.000us 0.00% 0.000us 0.000us 169 20 | cudaEventCreateWithFlags 0.00% 2.000us 0.00% 2.000us 0.111us 0.000us 0.00% 0.000us 0.000us 18 21 | cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFla... 0.00% 7.000us 0.00% 7.000us 3.500us 0.000us 0.00% 0.000us 0.000us 2 22 | cudaDeviceSynchronize 0.01% 148.000us 0.01% 148.000us 148.000us 0.000us 0.00% 0.000us 0.000us 1 23 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 24 | Self CPU time total: 1.714s 25 | Self CUDA time total: 879.000us 26 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_profiling_results/desktop/profiling_n_images_1_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_warm_up_1.txt: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 2 | Name Self CPU % Self CPU CPU total % CPU total CPU time avg Self CUDA Self CUDA % CUDA total CUDA time avg # of Calls 3 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 4 | void at::native::(anonymous namespace)::grid_sampler... 0.00% 0.000us 0.00% 0.000us 0.000us 785.000us 88.50% 785.000us 392.500us 2 5 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 52.000us 5.86% 52.000us 8.667us 6 6 | void at::native::vectorized_elementwise_kernel<4, at... 0.00% 0.000us 0.00% 0.000us 0.000us 25.000us 2.82% 25.000us 6.250us 4 7 | volta_sgemm_128x32_tn 0.00% 0.000us 0.00% 0.000us 0.000us 11.000us 1.24% 11.000us 5.500us 2 8 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 7.000us 0.79% 7.000us 3.500us 2 9 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 4.000us 0.45% 4.000us 2.000us 2 10 | void (anonymous namespace)::elementwise_kernel_with_... 0.00% 0.000us 0.00% 0.000us 0.000us 2.000us 0.23% 2.000us 1.000us 2 11 | Memcpy HtoD (Pageable -> Device) 0.00% 0.000us 0.00% 0.000us 0.000us 1.000us 0.11% 1.000us 0.200us 5 12 | cudaMalloc 0.05% 1.221ms 0.05% 1.221ms 203.500us 0.000us 0.00% 0.000us 0.000us 6 13 | cudaMemcpyAsync 0.00% 82.000us 0.00% 82.000us 20.500us 0.000us 0.00% 0.000us 0.000us 4 14 | cudaStreamSynchronize 0.00% 23.000us 0.00% 23.000us 5.750us 0.000us 0.00% 0.000us 0.000us 4 15 | cudaLaunchKernel 41.82% 1.081s 41.82% 1.081s 54.026ms 0.000us 0.00% 0.000us 0.000us 20 16 | cudaFree 58.12% 1.502s 58.12% 1.502s 500.547ms 0.000us 0.00% 0.000us 0.000us 3 17 | cudaDeviceGetAttribute 0.00% 1.000us 0.00% 1.000us 0.091us 0.000us 0.00% 0.000us 0.000us 11 18 | cudaMemcpy 0.00% 29.000us 0.00% 29.000us 29.000us 0.000us 0.00% 0.000us 0.000us 1 19 | cudaFuncSetAttribute 0.00% 6.000us 0.00% 6.000us 0.036us 0.000us 0.00% 0.000us 0.000us 169 20 | cudaEventCreateWithFlags 0.00% 12.000us 0.00% 12.000us 0.667us 0.000us 0.00% 0.000us 0.000us 18 21 | cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFla... 0.00% 6.000us 0.00% 6.000us 3.000us 0.000us 0.00% 0.000us 0.000us 2 22 | cudaDeviceSynchronize 0.00% 128.000us 0.00% 128.000us 128.000us 0.000us 0.00% 0.000us 0.000us 1 23 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 24 | Self CPU time total: 2.584s 25 | Self CUDA time total: 887.000us 26 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_profiling_results/desktop/profiling_n_images_1_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_warm_up_2.txt: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 2 | Name Self CPU % Self CPU CPU total % CPU total CPU time avg Self CUDA Self CUDA % CUDA total CUDA time avg # of Calls 3 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 4 | void at::native::(anonymous namespace)::grid_sampler... 0.00% 0.000us 0.00% 0.000us 0.000us 785.000us 88.00% 785.000us 392.500us 2 5 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 52.000us 5.83% 52.000us 8.667us 6 6 | void at::native::vectorized_elementwise_kernel<4, at... 0.00% 0.000us 0.00% 0.000us 0.000us 26.000us 2.91% 26.000us 6.500us 4 7 | volta_sgemm_128x32_tn 0.00% 0.000us 0.00% 0.000us 0.000us 11.000us 1.23% 11.000us 5.500us 2 8 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 6.000us 0.67% 6.000us 3.000us 2 9 | Memcpy HtoD (Pageable -> Device) 0.00% 0.000us 0.00% 0.000us 0.000us 5.000us 0.56% 5.000us 1.000us 5 10 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 4.000us 0.45% 4.000us 2.000us 2 11 | void (anonymous namespace)::elementwise_kernel_with_... 0.00% 0.000us 0.00% 0.000us 0.000us 3.000us 0.34% 3.000us 1.500us 2 12 | cudaMalloc 0.06% 998.000us 0.06% 998.000us 166.333us 0.000us 0.00% 0.000us 0.000us 6 13 | cudaMemcpyAsync 0.00% 71.000us 0.00% 71.000us 17.750us 0.000us 0.00% 0.000us 0.000us 4 14 | cudaStreamSynchronize 0.00% 24.000us 0.00% 24.000us 6.000us 0.000us 0.00% 0.000us 0.000us 4 15 | cudaLaunchKernel 67.30% 1.078s 67.30% 1.078s 53.886ms 0.000us 0.00% 0.000us 0.000us 20 16 | cudaFree 32.62% 522.348ms 32.62% 522.348ms 174.116ms 0.000us 0.00% 0.000us 0.000us 3 17 | cudaDeviceGetAttribute 0.00% 1.000us 0.00% 1.000us 0.091us 0.000us 0.00% 0.000us 0.000us 11 18 | cudaMemcpy 0.00% 24.000us 0.00% 24.000us 24.000us 0.000us 0.00% 0.000us 0.000us 1 19 | cudaFuncSetAttribute 0.00% 9.000us 0.00% 9.000us 0.053us 0.000us 0.00% 0.000us 0.000us 169 20 | cudaEventCreateWithFlags 0.00% 2.000us 0.00% 2.000us 0.111us 0.000us 0.00% 0.000us 0.000us 18 21 | cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFla... 0.00% 3.000us 0.00% 3.000us 1.500us 0.000us 0.00% 0.000us 0.000us 2 22 | cudaDeviceSynchronize 0.01% 173.000us 0.01% 173.000us 173.000us 0.000us 0.00% 0.000us 0.000us 1 23 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 24 | Self CPU time total: 1.601s 25 | Self CUDA time total: 892.000us 26 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_profiling_results/desktop/profiling_n_images_2_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 2 | Name Self CPU % Self CPU CPU total % CPU total CPU time avg Self CUDA Self CUDA % CUDA total CUDA time avg # of Calls 3 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 4 | void at::native::(anonymous namespace)::grid_sampler... 0.00% 0.000us 0.00% 0.000us 0.000us 1.569ms 88.90% 1.569ms 392.250us 4 5 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 103.000us 5.84% 103.000us 8.583us 12 6 | void at::native::vectorized_elementwise_kernel<4, at... 0.00% 0.000us 0.00% 0.000us 0.000us 47.000us 2.66% 47.000us 5.875us 8 7 | volta_sgemm_128x32_tn 0.00% 0.000us 0.00% 0.000us 0.000us 21.000us 1.19% 21.000us 5.250us 4 8 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 12.000us 0.68% 12.000us 3.000us 4 9 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 8.000us 0.45% 8.000us 2.000us 4 10 | Memcpy HtoD (Pageable -> Device) 0.00% 0.000us 0.00% 0.000us 0.000us 3.000us 0.17% 3.000us 0.333us 9 11 | void (anonymous namespace)::elementwise_kernel_with_... 0.00% 0.000us 0.00% 0.000us 0.000us 2.000us 0.11% 2.000us 0.500us 4 12 | cudaMalloc 0.07% 1.275ms 0.07% 1.275ms 182.143us 0.000us 0.00% 0.000us 0.000us 7 13 | cudaMemcpyAsync 0.01% 112.000us 0.01% 112.000us 14.000us 0.000us 0.00% 0.000us 0.000us 8 14 | cudaStreamSynchronize 0.01% 228.000us 0.01% 228.000us 28.500us 0.000us 0.00% 0.000us 0.000us 8 15 | cudaLaunchKernel 67.78% 1.170s 67.78% 1.170s 29.260ms 0.000us 0.00% 0.000us 0.000us 40 16 | cudaFree 32.12% 554.606ms 32.12% 554.606ms 184.869ms 0.000us 0.00% 0.000us 0.000us 3 17 | cudaDeviceGetAttribute 0.00% 1.000us 0.00% 1.000us 0.091us 0.000us 0.00% 0.000us 0.000us 11 18 | cudaMemcpy 0.00% 23.000us 0.00% 23.000us 23.000us 0.000us 0.00% 0.000us 0.000us 1 19 | cudaFuncSetAttribute 0.00% 11.000us 0.00% 11.000us 0.065us 0.000us 0.00% 0.000us 0.000us 169 20 | cudaEventCreateWithFlags 0.00% 2.000us 0.00% 2.000us 0.111us 0.000us 0.00% 0.000us 0.000us 18 21 | cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFla... 0.00% 4.000us 0.00% 4.000us 1.000us 0.000us 0.00% 0.000us 0.000us 4 22 | cudaDeviceSynchronize 0.01% 134.000us 0.01% 134.000us 134.000us 0.000us 0.00% 0.000us 0.000us 1 23 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 24 | Self CPU time total: 1.727s 25 | Self CUDA time total: 1.765ms 26 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_profiling_results/desktop/profiling_n_images_2_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 2 | Name Self CPU % Self CPU CPU total % CPU total CPU time avg Self CUDA Self CUDA % CUDA total CUDA time avg # of Calls 3 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 4 | void at::native::(anonymous namespace)::grid_sampler... 0.00% 0.000us 0.00% 0.000us 0.000us 1.565ms 88.47% 1.565ms 391.250us 4 5 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 103.000us 5.82% 103.000us 8.583us 12 6 | void at::native::vectorized_elementwise_kernel<4, at... 0.00% 0.000us 0.00% 0.000us 0.000us 47.000us 2.66% 47.000us 5.875us 8 7 | volta_sgemm_128x32_tn 0.00% 0.000us 0.00% 0.000us 0.000us 22.000us 1.24% 22.000us 5.500us 4 8 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 10.000us 0.57% 10.000us 2.500us 4 9 | Memcpy HtoD (Pageable -> Device) 0.00% 0.000us 0.00% 0.000us 0.000us 8.000us 0.45% 8.000us 0.889us 9 10 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 8.000us 0.45% 8.000us 2.000us 4 11 | void (anonymous namespace)::elementwise_kernel_with_... 0.00% 0.000us 0.00% 0.000us 0.000us 6.000us 0.34% 6.000us 1.500us 4 12 | cudaMalloc 0.07% 1.275ms 0.07% 1.275ms 182.143us 0.000us 0.00% 0.000us 0.000us 7 13 | cudaMemcpyAsync 0.01% 129.000us 0.01% 129.000us 16.125us 0.000us 0.00% 0.000us 0.000us 8 14 | cudaStreamSynchronize 0.01% 169.000us 0.01% 169.000us 21.125us 0.000us 0.00% 0.000us 0.000us 8 15 | cudaLaunchKernel 67.99% 1.174s 67.99% 1.174s 29.338ms 0.000us 0.00% 0.000us 0.000us 40 16 | cudaFree 31.91% 550.742ms 31.91% 550.742ms 183.581ms 0.000us 0.00% 0.000us 0.000us 3 17 | cudaDeviceGetAttribute 0.00% 1.000us 0.00% 1.000us 0.091us 0.000us 0.00% 0.000us 0.000us 11 18 | cudaMemcpy 0.00% 27.000us 0.00% 27.000us 27.000us 0.000us 0.00% 0.000us 0.000us 1 19 | cudaFuncSetAttribute 0.00% 6.000us 0.00% 6.000us 0.036us 0.000us 0.00% 0.000us 0.000us 169 20 | cudaEventCreateWithFlags 0.00% 2.000us 0.00% 2.000us 0.111us 0.000us 0.00% 0.000us 0.000us 18 21 | cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFla... 0.00% 4.000us 0.00% 4.000us 1.000us 0.000us 0.00% 0.000us 0.000us 4 22 | cudaDeviceSynchronize 0.01% 145.000us 0.01% 145.000us 145.000us 0.000us 0.00% 0.000us 0.000us 1 23 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 24 | Self CPU time total: 1.726s 25 | Self CUDA time total: 1.769ms 26 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_profiling_results/desktop/profiling_n_images_2_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 2 | Name Self CPU % Self CPU CPU total % CPU total CPU time avg Self CUDA Self CUDA % CUDA total CUDA time avg # of Calls 3 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 4 | void at::native::(anonymous namespace)::grid_sampler... 0.00% 0.000us 0.00% 0.000us 0.000us 1.568ms 88.44% 1.568ms 392.000us 4 5 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 106.000us 5.98% 106.000us 8.833us 12 6 | void at::native::vectorized_elementwise_kernel<4, at... 0.00% 0.000us 0.00% 0.000us 0.000us 46.000us 2.59% 46.000us 5.750us 8 7 | volta_sgemm_128x32_tn 0.00% 0.000us 0.00% 0.000us 0.000us 22.000us 1.24% 22.000us 5.500us 4 8 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 13.000us 0.73% 13.000us 3.250us 4 9 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 8.000us 0.45% 8.000us 2.000us 4 10 | Memcpy HtoD (Pageable -> Device) 0.00% 0.000us 0.00% 0.000us 0.000us 6.000us 0.34% 6.000us 0.667us 9 11 | void (anonymous namespace)::elementwise_kernel_with_... 0.00% 0.000us 0.00% 0.000us 0.000us 4.000us 0.23% 4.000us 1.000us 4 12 | cudaMalloc 0.07% 1.355ms 0.07% 1.355ms 193.571us 0.000us 0.00% 0.000us 0.000us 7 13 | cudaMemcpyAsync 0.01% 170.000us 0.01% 170.000us 21.250us 0.000us 0.00% 0.000us 0.000us 8 14 | cudaStreamSynchronize 0.01% 191.000us 0.01% 191.000us 23.875us 0.000us 0.00% 0.000us 0.000us 8 15 | cudaLaunchKernel 70.11% 1.282s 70.11% 1.282s 32.054ms 0.000us 0.00% 0.000us 0.000us 40 16 | cudaFree 29.78% 544.705ms 29.78% 544.705ms 181.568ms 0.000us 0.00% 0.000us 0.000us 3 17 | cudaDeviceGetAttribute 0.00% 1.000us 0.00% 1.000us 0.091us 0.000us 0.00% 0.000us 0.000us 11 18 | cudaMemcpy 0.00% 40.000us 0.00% 40.000us 40.000us 0.000us 0.00% 0.000us 0.000us 1 19 | cudaFuncSetAttribute 0.00% 40.000us 0.00% 40.000us 0.238us 0.000us 0.00% 0.000us 0.000us 168 20 | cudaEventCreateWithFlags 0.00% 9.000us 0.00% 9.000us 0.500us 0.000us 0.00% 0.000us 0.000us 18 21 | cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFla... 0.00% 4.000us 0.00% 4.000us 1.000us 0.000us 0.00% 0.000us 0.000us 4 22 | cudaDeviceSynchronize 0.01% 152.000us 0.01% 152.000us 152.000us 0.000us 0.00% 0.000us 0.000us 1 23 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 24 | Self CPU time total: 1.829s 25 | Self CUDA time total: 1.773ms 26 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_profiling_results/desktop/profiling_n_images_4_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 2 | Name Self CPU % Self CPU CPU total % CPU total CPU time avg Self CUDA Self CUDA % CUDA total CUDA time avg # of Calls 3 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 4 | void at::native::(anonymous namespace)::grid_sampler... 0.00% 0.000us 0.00% 0.000us 0.000us 3.133ms 89.03% 3.133ms 391.625us 8 5 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 206.000us 5.85% 206.000us 8.583us 24 6 | void at::native::vectorized_elementwise_kernel<4, at... 0.00% 0.000us 0.00% 0.000us 0.000us 91.000us 2.59% 91.000us 5.688us 16 7 | volta_sgemm_128x32_tn 0.00% 0.000us 0.00% 0.000us 0.000us 39.000us 1.11% 39.000us 4.875us 8 8 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 24.000us 0.68% 24.000us 3.000us 8 9 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 15.000us 0.43% 15.000us 1.875us 8 10 | Memcpy HtoD (Pageable -> Device) 0.00% 0.000us 0.00% 0.000us 0.000us 7.000us 0.20% 7.000us 0.412us 17 11 | void (anonymous namespace)::elementwise_kernel_with_... 0.00% 0.000us 0.00% 0.000us 0.000us 4.000us 0.11% 4.000us 0.500us 8 12 | cudaMalloc 0.07% 1.263ms 0.07% 1.263ms 180.429us 0.000us 0.00% 0.000us 0.000us 7 13 | cudaMemcpyAsync 0.01% 209.000us 0.01% 209.000us 13.062us 0.000us 0.00% 0.000us 0.000us 16 14 | cudaStreamSynchronize 0.07% 1.135ms 0.07% 1.135ms 70.938us 0.000us 0.00% 0.000us 0.000us 16 15 | cudaLaunchKernel 68.04% 1.187s 68.04% 1.187s 14.837ms 0.000us 0.00% 0.000us 0.000us 80 16 | cudaFree 31.79% 554.631ms 31.79% 554.631ms 184.877ms 0.000us 0.00% 0.000us 0.000us 3 17 | cudaDeviceGetAttribute 0.00% 1.000us 0.00% 1.000us 0.091us 0.000us 0.00% 0.000us 0.000us 11 18 | cudaMemcpy 0.00% 23.000us 0.00% 23.000us 23.000us 0.000us 0.00% 0.000us 0.000us 1 19 | cudaFuncSetAttribute 0.00% 9.000us 0.00% 9.000us 0.053us 0.000us 0.00% 0.000us 0.000us 169 20 | cudaEventCreateWithFlags 0.00% 3.000us 0.00% 3.000us 0.167us 0.000us 0.00% 0.000us 0.000us 18 21 | cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFla... 0.00% 8.000us 0.00% 8.000us 1.000us 0.000us 0.00% 0.000us 0.000us 8 22 | cudaDeviceSynchronize 0.01% 172.000us 0.01% 172.000us 172.000us 0.000us 0.00% 0.000us 0.000us 1 23 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 24 | Self CPU time total: 1.744s 25 | Self CUDA time total: 3.519ms 26 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_profiling_results/desktop/profiling_n_images_4_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 2 | Name Self CPU % Self CPU CPU total % CPU total CPU time avg Self CUDA Self CUDA % CUDA total CUDA time avg # of Calls 3 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 4 | void at::native::(anonymous namespace)::grid_sampler... 0.00% 0.000us 0.00% 0.000us 0.000us 3.135ms 89.01% 3.135ms 391.875us 8 5 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 203.000us 5.76% 203.000us 8.458us 24 6 | void at::native::vectorized_elementwise_kernel<4, at... 0.00% 0.000us 0.00% 0.000us 0.000us 90.000us 2.56% 90.000us 5.625us 16 7 | volta_sgemm_128x32_tn 0.00% 0.000us 0.00% 0.000us 0.000us 37.000us 1.05% 37.000us 4.625us 8 8 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 24.000us 0.68% 24.000us 3.000us 8 9 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 15.000us 0.43% 15.000us 1.875us 8 10 | Memcpy HtoD (Pageable -> Device) 0.00% 0.000us 0.00% 0.000us 0.000us 13.000us 0.37% 13.000us 0.765us 17 11 | void (anonymous namespace)::elementwise_kernel_with_... 0.00% 0.000us 0.00% 0.000us 0.000us 5.000us 0.14% 5.000us 0.625us 8 12 | cudaMalloc 0.07% 1.273ms 0.07% 1.273ms 181.857us 0.000us 0.00% 0.000us 0.000us 7 13 | cudaMemcpyAsync 0.01% 205.000us 0.01% 205.000us 12.812us 0.000us 0.00% 0.000us 0.000us 16 14 | cudaStreamSynchronize 0.07% 1.180ms 0.07% 1.180ms 73.750us 0.000us 0.00% 0.000us 0.000us 16 15 | cudaLaunchKernel 67.93% 1.167s 67.93% 1.167s 14.593ms 0.000us 0.00% 0.000us 0.000us 80 16 | cudaFree 31.90% 548.191ms 31.90% 548.191ms 182.730ms 0.000us 0.00% 0.000us 0.000us 3 17 | cudaDeviceGetAttribute 0.00% 1.000us 0.00% 1.000us 0.091us 0.000us 0.00% 0.000us 0.000us 11 18 | cudaMemcpy 0.00% 25.000us 0.00% 25.000us 25.000us 0.000us 0.00% 0.000us 0.000us 1 19 | cudaFuncSetAttribute 0.00% 11.000us 0.00% 11.000us 0.065us 0.000us 0.00% 0.000us 0.000us 169 20 | cudaEventCreateWithFlags 0.00% 2.000us 0.00% 2.000us 0.111us 0.000us 0.00% 0.000us 0.000us 18 21 | cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFla... 0.00% 8.000us 0.00% 8.000us 1.000us 0.000us 0.00% 0.000us 0.000us 8 22 | cudaDeviceSynchronize 0.01% 182.000us 0.01% 182.000us 182.000us 0.000us 0.00% 0.000us 0.000us 1 23 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 24 | Self CPU time total: 1.719s 25 | Self CUDA time total: 3.522ms 26 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_profiling_results/desktop/profiling_n_images_4_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 2 | Name Self CPU % Self CPU CPU total % CPU total CPU time avg Self CUDA Self CUDA % CUDA total CUDA time avg # of Calls 3 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 4 | void at::native::(anonymous namespace)::grid_sampler... 0.00% 0.000us 0.00% 0.000us 0.000us 3.140ms 88.83% 3.140ms 392.500us 8 5 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 207.000us 5.86% 207.000us 8.625us 24 6 | void at::native::vectorized_elementwise_kernel<4, at... 0.00% 0.000us 0.00% 0.000us 0.000us 98.000us 2.77% 98.000us 6.125us 16 7 | volta_sgemm_128x32_tn 0.00% 0.000us 0.00% 0.000us 0.000us 37.000us 1.05% 37.000us 4.625us 8 8 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 25.000us 0.71% 25.000us 3.125us 8 9 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 16.000us 0.45% 16.000us 2.000us 8 10 | Memcpy HtoD (Pageable -> Device) 0.00% 0.000us 0.00% 0.000us 0.000us 7.000us 0.20% 7.000us 0.412us 17 11 | void (anonymous namespace)::elementwise_kernel_with_... 0.00% 0.000us 0.00% 0.000us 0.000us 5.000us 0.14% 5.000us 0.625us 8 12 | cudaMalloc 0.07% 1.290ms 0.07% 1.290ms 184.286us 0.000us 0.00% 0.000us 0.000us 7 13 | cudaMemcpyAsync 0.01% 213.000us 0.01% 213.000us 13.312us 0.000us 0.00% 0.000us 0.000us 16 14 | cudaStreamSynchronize 0.07% 1.147ms 0.07% 1.147ms 71.688us 0.000us 0.00% 0.000us 0.000us 16 15 | cudaLaunchKernel 67.78% 1.166s 67.78% 1.166s 14.572ms 0.000us 0.00% 0.000us 0.000us 80 16 | cudaFree 32.06% 551.393ms 32.06% 551.393ms 183.798ms 0.000us 0.00% 0.000us 0.000us 3 17 | cudaDeviceGetAttribute 0.00% 1.000us 0.00% 1.000us 0.091us 0.000us 0.00% 0.000us 0.000us 11 18 | cudaMemcpy 0.00% 25.000us 0.00% 25.000us 25.000us 0.000us 0.00% 0.000us 0.000us 1 19 | cudaFuncSetAttribute 0.00% 10.000us 0.00% 10.000us 0.059us 0.000us 0.00% 0.000us 0.000us 169 20 | cudaEventCreateWithFlags 0.00% 2.000us 0.00% 2.000us 0.111us 0.000us 0.00% 0.000us 0.000us 18 21 | cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFla... 0.00% 8.000us 0.00% 8.000us 1.000us 0.000us 0.00% 0.000us 0.000us 8 22 | cudaDeviceSynchronize 0.01% 180.000us 0.01% 180.000us 180.000us 0.000us 0.00% 0.000us 0.000us 1 23 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 24 | Self CPU time total: 1.720s 25 | Self CUDA time total: 3.535ms 26 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_profiling_results/desktop/profiling_n_images_8_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_1.txt: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 2 | Name Self CPU % Self CPU CPU total % CPU total CPU time avg Self CUDA Self CUDA % CUDA total CUDA time avg # of Calls 3 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 4 | void at::native::(anonymous namespace)::grid_sampler... 0.00% 0.000us 0.00% 0.000us 0.000us 6.270ms 88.92% 6.270ms 391.875us 16 5 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 407.000us 5.77% 407.000us 8.479us 48 6 | void at::native::vectorized_elementwise_kernel<4, at... 0.00% 0.000us 0.00% 0.000us 0.000us 191.000us 2.71% 191.000us 5.969us 32 7 | volta_sgemm_128x32_tn 0.00% 0.000us 0.00% 0.000us 0.000us 73.000us 1.04% 73.000us 4.562us 16 8 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 45.000us 0.64% 45.000us 2.812us 16 9 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 32.000us 0.45% 32.000us 2.000us 16 10 | Memcpy HtoD (Pageable -> Device) 0.00% 0.000us 0.00% 0.000us 0.000us 21.000us 0.30% 21.000us 0.636us 33 11 | void (anonymous namespace)::elementwise_kernel_with_... 0.00% 0.000us 0.00% 0.000us 0.000us 12.000us 0.17% 12.000us 0.750us 16 12 | cudaMalloc 0.07% 1.282ms 0.07% 1.282ms 183.143us 0.000us 0.00% 0.000us 0.000us 7 13 | cudaMemcpyAsync 0.02% 394.000us 0.02% 394.000us 12.312us 0.000us 0.00% 0.000us 0.000us 32 14 | cudaStreamSynchronize 0.17% 3.006ms 0.17% 3.006ms 93.938us 0.000us 0.00% 0.000us 0.000us 32 15 | cudaLaunchKernel 67.69% 1.177s 67.69% 1.177s 7.356ms 0.000us 0.00% 0.000us 0.000us 160 16 | cudaFree 32.03% 556.971ms 32.03% 556.971ms 185.657ms 0.000us 0.00% 0.000us 0.000us 3 17 | cudaDeviceGetAttribute 0.00% 1.000us 0.00% 1.000us 0.091us 0.000us 0.00% 0.000us 0.000us 11 18 | cudaMemcpy 0.00% 24.000us 0.00% 24.000us 24.000us 0.000us 0.00% 0.000us 0.000us 1 19 | cudaFuncSetAttribute 0.00% 11.000us 0.00% 11.000us 0.065us 0.000us 0.00% 0.000us 0.000us 169 20 | cudaEventCreateWithFlags 0.00% 7.000us 0.00% 7.000us 0.412us 0.000us 0.00% 0.000us 0.000us 17 21 | cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFla... 0.00% 22.000us 0.00% 22.000us 1.375us 0.000us 0.00% 0.000us 0.000us 16 22 | cudaDeviceSynchronize 0.01% 180.000us 0.01% 180.000us 180.000us 0.000us 0.00% 0.000us 0.000us 1 23 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 24 | Self CPU time total: 1.739s 25 | Self CUDA time total: 7.051ms 26 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_profiling_results/desktop/profiling_n_images_8_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_2.txt: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 2 | Name Self CPU % Self CPU CPU total % CPU total CPU time avg Self CUDA Self CUDA % CUDA total CUDA time avg # of Calls 3 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 4 | void at::native::(anonymous namespace)::grid_sampler... 0.00% 0.000us 0.00% 0.000us 0.000us 6.270ms 88.90% 6.270ms 391.875us 16 5 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 410.000us 5.81% 410.000us 8.542us 48 6 | void at::native::vectorized_elementwise_kernel<4, at... 0.00% 0.000us 0.00% 0.000us 0.000us 185.000us 2.62% 185.000us 5.781us 32 7 | volta_sgemm_128x32_tn 0.00% 0.000us 0.00% 0.000us 0.000us 81.000us 1.15% 81.000us 5.062us 16 8 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 46.000us 0.65% 46.000us 2.875us 16 9 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 32.000us 0.45% 32.000us 2.000us 16 10 | Memcpy HtoD (Pageable -> Device) 0.00% 0.000us 0.00% 0.000us 0.000us 15.000us 0.21% 15.000us 0.455us 33 11 | void (anonymous namespace)::elementwise_kernel_with_... 0.00% 0.000us 0.00% 0.000us 0.000us 14.000us 0.20% 14.000us 0.875us 16 12 | cudaMalloc 0.08% 1.321ms 0.08% 1.321ms 188.714us 0.000us 0.00% 0.000us 0.000us 7 13 | cudaMemcpyAsync 0.03% 494.000us 0.03% 494.000us 15.438us 0.000us 0.00% 0.000us 0.000us 32 14 | cudaStreamSynchronize 0.15% 2.584ms 0.15% 2.584ms 80.750us 0.000us 0.00% 0.000us 0.000us 32 15 | cudaLaunchKernel 67.89% 1.176s 67.89% 1.176s 7.351ms 0.000us 0.00% 0.000us 0.000us 160 16 | cudaFree 31.85% 551.765ms 31.85% 551.765ms 183.922ms 0.000us 0.00% 0.000us 0.000us 3 17 | cudaDeviceGetAttribute 0.00% 1.000us 0.00% 1.000us 0.091us 0.000us 0.00% 0.000us 0.000us 11 18 | cudaMemcpy 0.00% 26.000us 0.00% 26.000us 26.000us 0.000us 0.00% 0.000us 0.000us 1 19 | cudaFuncSetAttribute 0.00% 14.000us 0.00% 14.000us 0.083us 0.000us 0.00% 0.000us 0.000us 169 20 | cudaEventCreateWithFlags 0.00% 2.000us 0.00% 2.000us 0.111us 0.000us 0.00% 0.000us 0.000us 18 21 | cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFla... 0.00% 17.000us 0.00% 17.000us 1.062us 0.000us 0.00% 0.000us 0.000us 16 22 | cudaDeviceSynchronize 0.01% 152.000us 0.01% 152.000us 152.000us 0.000us 0.00% 0.000us 0.000us 1 23 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 24 | Self CPU time total: 1.733s 25 | Self CUDA time total: 7.053ms 26 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_profiling_results/desktop/profiling_n_images_8_coreg_coarse_device_gpu_equalize_False_bands_B8A_B11_B12_test_iteration_3.txt: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 2 | Name Self CPU % Self CPU CPU total % CPU total CPU time avg Self CUDA Self CUDA % CUDA total CUDA time avg # of Calls 3 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 4 | void at::native::(anonymous namespace)::grid_sampler... 0.00% 0.000us 0.00% 0.000us 0.000us 6.273ms 88.81% 6.273ms 392.062us 16 5 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 404.000us 5.72% 404.000us 8.417us 48 6 | void at::native::vectorized_elementwise_kernel<4, at... 0.00% 0.000us 0.00% 0.000us 0.000us 185.000us 2.62% 185.000us 5.781us 32 7 | volta_sgemm_128x32_tn 0.00% 0.000us 0.00% 0.000us 0.000us 79.000us 1.12% 79.000us 4.938us 16 8 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 49.000us 0.69% 49.000us 3.062us 16 9 | Memcpy HtoD (Pageable -> Device) 0.00% 0.000us 0.00% 0.000us 0.000us 30.000us 0.42% 30.000us 0.909us 33 10 | void at::native::elementwise_kernel<128, 2, at::nati... 0.00% 0.000us 0.00% 0.000us 0.000us 30.000us 0.42% 30.000us 1.875us 16 11 | void (anonymous namespace)::elementwise_kernel_with_... 0.00% 0.000us 0.00% 0.000us 0.000us 13.000us 0.18% 13.000us 0.812us 16 12 | cudaMalloc 0.07% 1.289ms 0.07% 1.289ms 184.143us 0.000us 0.00% 0.000us 0.000us 7 13 | cudaMemcpyAsync 0.02% 403.000us 0.02% 403.000us 12.594us 0.000us 0.00% 0.000us 0.000us 32 14 | cudaStreamSynchronize 0.18% 3.098ms 0.18% 3.098ms 96.812us 0.000us 0.00% 0.000us 0.000us 32 15 | cudaLaunchKernel 67.80% 1.170s 67.80% 1.170s 7.309ms 0.000us 0.00% 0.000us 0.000us 160 16 | cudaFree 31.91% 550.362ms 31.91% 550.362ms 183.454ms 0.000us 0.00% 0.000us 0.000us 3 17 | cudaDeviceGetAttribute 0.00% 1.000us 0.00% 1.000us 0.091us 0.000us 0.00% 0.000us 0.000us 11 18 | cudaMemcpy 0.00% 22.000us 0.00% 22.000us 22.000us 0.000us 0.00% 0.000us 0.000us 1 19 | cudaFuncSetAttribute 0.00% 5.000us 0.00% 5.000us 0.030us 0.000us 0.00% 0.000us 0.000us 169 20 | cudaEventCreateWithFlags 0.00% 10.000us 0.00% 10.000us 0.556us 0.000us 0.00% 0.000us 0.000us 18 21 | cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFla... 0.00% 16.000us 0.00% 16.000us 1.000us 0.000us 0.00% 0.000us 0.000us 16 22 | cudaDeviceSynchronize 0.01% 179.000us 0.01% 179.000us 179.000us 0.000us 0.00% 0.000us 0.000us 1 23 | ------------------------------------------------------- ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ ------------ 24 | Self CPU time total: 1.725s 25 | Self CUDA time total: 7.063ms 26 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_study_notebooks_utils.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | sys.path.insert(1, os.path.join("..", "..")) 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | 8 | 9 | def extract_test_time(test_name): 10 | """Extracts test results. 11 | 12 | Args: 13 | test_name (str): test path. 14 | 15 | Returns: 16 | dict: results dictionary. 17 | """ 18 | f = open(test_name, "r") 19 | results_txt = f.read() 20 | f.close() 21 | results = results_txt.split("\n")[3].split(" ") 22 | 23 | results_clean = [] 24 | for x in results: 25 | if x != "": 26 | results_clean.append(x) 27 | 28 | n_images = int(test_name.split("_n_images_")[1].split("_")[0]) 29 | coreg_type = test_name.split("_coreg_")[1].split("_")[0] 30 | device = test_name.split("_device_")[1].split("_")[0] 31 | equalize = test_name.split("_equalize_")[1].split("_")[0] 32 | test_iteration = test_name.split("test_iteration_")[1][:-4] 33 | 34 | if results_clean[-3][-2:] == "ms": 35 | results = float(results_clean[-3][:-2]) / 1000 36 | else: 37 | results = float(results_clean[-3][:-2]) 38 | 39 | result_dict = { 40 | "n_images": n_images, 41 | "coreg_type": coreg_type, 42 | "device": device, 43 | "equalize": equalize, 44 | "result [s]": results, 45 | } 46 | 47 | return test_iteration, result_dict 48 | 49 | 50 | def clean_coregistration_histogram(values): 51 | """Clean coregistration histogram by removing values outside -std and std. 52 | 53 | Args: 54 | values (numpy array): values to clean. 55 | 56 | Returns: 57 | numpy array: cleaned histogram. 58 | """ 59 | values_mean = values.mean() 60 | vaues_std = values.std() 61 | values = values[values <= values_mean + 0.1 * vaues_std] 62 | values = values[values >= values_mean - 0.1 * vaues_std] 63 | return values 64 | 65 | 66 | def generate_histograms(results_df, satellite, detector, path="histograms", bins=25): 67 | """Generate histogram for coregistration study results. 68 | 69 | Args: 70 | results_df (pandas df): coregistration study results as pandas dataframe. 71 | satellite (str): satellite. 72 | detector (int): detector number. 73 | path (str, optional): output path to save histogram. Defaults to "histograms". 74 | bins (int, optional): Number of histogram beams. Defaults to 25. 75 | """ 76 | os.makedirs(path, exist_ok=True) 77 | results_df_satellite = results_df[results_df["Sat"] == satellite] 78 | results_satellite_detector = results_df_satellite[ 79 | results_df_satellite["detector_number"] == detector 80 | ] 81 | results_satellite_detector_N_v = np.array(results_satellite_detector.N_v.to_list()) 82 | results_satellite_detector_N_h = np.array(results_satellite_detector.N_h.to_list()) 83 | 84 | plt.figure() 85 | plt.hist(results_satellite_detector_N_v, bins=bins) 86 | plt.xlabel("N_v") 87 | plt.ylabel("Number of occurrences") 88 | plt.title("Hist_" + str(satellite) + "_" + str(detector) + "_Nv") 89 | plt.savefig( 90 | os.path.join(path, "hist_" + str(satellite) + "_" + str(detector) + "_Nv.png") 91 | ) 92 | plt.figure() 93 | plt.hist(results_satellite_detector_N_h, bins=bins) 94 | plt.xlabel("N_h") 95 | plt.ylabel("Number of occurrences") 96 | plt.title("Hist_" + str(satellite) + "_" + str(detector) + "_Nh") 97 | plt.savefig( 98 | os.path.join(path, "hist_" + str(satellite) + "_" + str(detector) + "_Nh.png") 99 | ) 100 | return 101 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_study_results/coregistration_study_results_d6/coregistration_study_B01_B09_2023_1_26_11_10_46.csv: -------------------------------------------------------------------------------- 1 | ID_event,granule_couple,coordinates,sensing_time,detector_number,N_v,N_h,N_eff 2 | Mexico_0,"[0, 2]",[ 25.43221955 -100.2269484 ],2021-03-25 17:26:43,6,-28.0,0.0,28.0 3 | Mexico_0,"[2, 4]",[ 25.21910749 -100.28070636],2021-03-25 17:26:47,6,-29.0,0.0,29.0 4 | Greece_5,"[0, 2]",[38.54562942 24.2867681 ],2019-07-05 09:19:45,6,-28.0,1.0,28.0178514522438 5 | Greece_5,"[2, 4]",[38.33379261 24.22004876],2019-07-05 09:19:49,6,-29.0,0.0,29.0 6 | Kenya_0,"[0, 2]",[-0.18613292 37.37748282],2019-02-28 07:59:53,6,-29.0,0.0,29.0 7 | Kenya_0,"[2, 4]",[-0.40036289 37.3305413 ],2019-02-28 07:59:56,6,-29.0,1.0,29.017236257093817 8 | Hungary_0,"[0, 1]",[47.36679297 19.84754081],2017-08-03 09:47:23,6,-29.0,0.0,29.0 9 | France_1,"[0, 2]",[43.60980383 5.83979465],2017-07-24 10:38:47,6,-29.0,-0.0,29.0 10 | Italy_2,"[1, 2]",[41.76594104 12.75711796],2021-08-13 10:09:06,6,-29.0,-1.0,29.017236257093817 11 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_study_results/coregistration_study_results_d6/coregistration_study_B01_B12_2023_1_26_9_50_54.csv: -------------------------------------------------------------------------------- 1 | ID_event,granule_couple,coordinates,sensing_time,detector_number,N_v,N_h,N_eff 2 | Mexico_0,"[0, 2]",[ 25.43221955 -100.2269484 ],2021-03-25 17:26:43,6,23.0,-2.0,23.08679276123039 3 | Mexico_0,"[2, 4]",[ 25.21910749 -100.28070636],2021-03-25 17:26:47,6,23.0,-2.0,23.08679276123039 4 | Greece_5,"[0, 2]",[38.54562942 24.2867681 ],2019-07-05 09:19:45,6,22.0,-12.0,25.059928172283335 5 | Greece_5,"[2, 4]",[38.33379261 24.22004876],2019-07-05 09:19:49,6,22.0,-12.0,25.059928172283335 6 | Kenya_0,"[0, 2]",[-0.18613292 37.37748282],2019-02-28 07:59:53,6,22.0,-12.0,25.059928172283335 7 | Kenya_0,"[2, 4]",[-0.40036289 37.3305413 ],2019-02-28 07:59:56,6,22.0,-12.0,25.059928172283335 8 | Hungary_0,"[0, 1]",[47.36679297 19.84754081],2017-08-03 09:47:23,6,22.0,-12.0,25.059928172283335 9 | Hungary_0,"[1, 2]",[47.15672067 19.76401333],2017-08-03 09:47:27,6,22.0,-12.0,25.059928172283335 10 | France_1,"[0, 2]",[43.60980383 5.83979465],2017-07-24 10:38:47,6,23.0,-2.0,23.08679276123039 11 | Italy_2,"[0, 1]",[41.977177 12.82933963],2021-08-13 10:09:03,6,23.0,-2.0,23.08679276123039 12 | Italy_2,"[1, 2]",[41.76594104 12.75711796],2021-08-13 10:09:06,6,23.0,-2.0,23.08679276123039 13 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_study_results/coregistration_study_results_d6/coregistration_study_B02_B08_2023_1_26_11_12_46.csv: -------------------------------------------------------------------------------- 1 | ID_event,granule_couple,coordinates,sensing_time,detector_number,N_v,N_h,N_eff 2 | Mexico_0,"[0, 2]",[ 25.43221955 -100.2269484 ],2021-03-25 17:26:43,6,-164.0,0.0,164.0 3 | Mexico_0,"[2, 4]",[ 25.21910749 -100.28070636],2021-03-25 17:26:47,6,-164.0,0.0,164.0 4 | Greece_5,"[2, 4]",[38.33379261 24.22004876],2019-07-05 09:19:49,6,-165.0,1.0,165.00303027520434 5 | Kenya_0,"[0, 2]",[-0.18613292 37.37748282],2019-02-28 07:59:53,6,-164.0,0.0,164.0 6 | Kenya_0,"[2, 4]",[-0.40036289 37.3305413 ],2019-02-28 07:59:56,6,-164.0,0.0,164.0 7 | Hungary_0,"[0, 1]",[47.36679297 19.84754081],2017-08-03 09:47:23,6,-165.0,0.0,165.0 8 | France_1,"[0, 2]",[43.60980383 5.83979465],2017-07-24 10:38:47,6,-166.0,0.0,166.0 9 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_study_results/coregistration_study_results_d6/coregistration_study_B03_B05_2023_1_26_12_15_33.csv: -------------------------------------------------------------------------------- 1 | ID_event,granule_couple,coordinates,sensing_time,detector_number,N_v,N_h,N_eff 2 | Mexico_0,"[0, 2]",[ 25.43221955 -100.2269484 ],2021-03-25 17:26:43,6,-235.0,-0.0,235.0 3 | Mexico_0,"[2, 4]",[ 25.21910749 -100.28070636],2021-03-25 17:26:47,6,-235.0,-0.0,235.0 4 | Greece_5,"[0, 2]",[38.54562942 24.2867681 ],2019-07-05 09:19:45,6,-236.0,0.0,236.0 5 | Greece_5,"[2, 4]",[38.33379261 24.22004876],2019-07-05 09:19:49,6,-236.0,0.0,236.0 6 | Kenya_0,"[0, 2]",[-0.18613292 37.37748282],2019-02-28 07:59:53,6,-234.0,0.0,234.0 7 | Kenya_0,"[2, 4]",[-0.40036289 37.3305413 ],2019-02-28 07:59:56,6,-235.0,-0.0,235.0 8 | Hungary_0,"[0, 1]",[47.36679297 19.84754081],2017-08-03 09:47:23,6,-237.0,0.0,237.0 9 | Hungary_0,"[1, 2]",[47.15672067 19.76401333],2017-08-03 09:47:27,6,-237.0,0.0,237.0 10 | France_1,"[0, 2]",[43.60980383 5.83979465],2017-07-24 10:38:47,6,-236.0,-0.0,236.0 11 | Italy_2,"[0, 1]",[41.977177 12.82933963],2021-08-13 10:09:03,6,-236.0,-0.0,236.0 12 | Italy_2,"[1, 2]",[41.76594104 12.75711796],2021-08-13 10:09:06,6,-236.0,0.0,236.0 13 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_study_results/coregistration_study_results_d6/coregistration_study_B04_B05_2023_1_26_12_22_44.csv: -------------------------------------------------------------------------------- 1 | ID_event,granule_couple,coordinates,sensing_time,detector_number,N_v,N_h,N_eff 2 | Mexico_0,"[0, 2]",[ 25.43221955 -100.2269484 ],2021-03-25 17:26:43,6,-83.0,-0.0,83.0 3 | Mexico_0,"[2, 4]",[ 25.21910749 -100.28070636],2021-03-25 17:26:47,6,-83.0,0.0,83.0 4 | Greece_5,"[0, 2]",[38.54562942 24.2867681 ],2019-07-05 09:19:45,6,-83.0,-0.0,83.0 5 | Greece_5,"[2, 4]",[38.33379261 24.22004876],2019-07-05 09:19:49,6,-83.0,-0.0,83.0 6 | Kenya_0,"[0, 2]",[-0.18613292 37.37748282],2019-02-28 07:59:53,6,-82.0,-0.0,82.0 7 | Kenya_0,"[2, 4]",[-0.40036289 37.3305413 ],2019-02-28 07:59:56,6,-82.0,-0.0,82.0 8 | Hungary_0,"[0, 1]",[47.36679297 19.84754081],2017-08-03 09:47:23,6,-83.0,-0.0,83.0 9 | Hungary_0,"[1, 2]",[47.15672067 19.76401333],2017-08-03 09:47:27,6,-83.0,-0.0,83.0 10 | France_1,"[0, 2]",[43.60980383 5.83979465],2017-07-24 10:38:47,6,-83.0,-0.0,83.0 11 | Italy_2,"[0, 1]",[41.977177 12.82933963],2021-08-13 10:09:03,6,-83.0,-0.0,83.0 12 | Italy_2,"[1, 2]",[41.76594104 12.75711796],2021-08-13 10:09:06,6,-83.0,-0.0,83.0 13 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_study_results/coregistration_study_results_d6/coregistration_study_B05_B11_2023_1_26_12_30_33.csv: -------------------------------------------------------------------------------- 1 | ID_event,granule_couple,coordinates,sensing_time,detector_number,N_v,N_h,N_eff 2 | Mexico_0,"[0, 2]",[ 25.43221955 -100.2269484 ],2021-03-25 17:26:43,6,-65.0,-2.0,65.03076195155643 3 | Mexico_0,"[2, 4]",[ 25.21910749 -100.28070636],2021-03-25 17:26:47,6,-65.0,-2.0,65.03076195155643 4 | Greece_5,"[0, 2]",[38.54562942 24.2867681 ],2019-07-05 09:19:45,6,-67.0,-12.0,68.06614430096654 5 | Greece_5,"[2, 4]",[38.33379261 24.22004876],2019-07-05 09:19:49,6,-67.0,-12.0,68.06614430096654 6 | Kenya_0,"[0, 2]",[-0.18613292 37.37748282],2019-02-28 07:59:53,6,-67.0,-12.0,68.06614430096654 7 | Kenya_0,"[2, 4]",[-0.40036289 37.3305413 ],2019-02-28 07:59:56,6,-67.0,-12.0,68.06614430096654 8 | Hungary_0,"[0, 1]",[47.36679297 19.84754081],2017-08-03 09:47:23,6,-68.0,-12.0,69.05070600652827 9 | Hungary_0,"[1, 2]",[47.15672067 19.76401333],2017-08-03 09:47:27,6,-68.0,-12.0,69.05070600652827 10 | France_1,"[0, 2]",[43.60980383 5.83979465],2017-07-24 10:38:47,6,-66.0,-2.0,66.03029607687671 11 | Italy_2,"[0, 1]",[41.977177 12.82933963],2021-08-13 10:09:03,6,-66.0,-2.0,66.03029607687671 12 | Italy_2,"[1, 2]",[41.76594104 12.75711796],2021-08-13 10:09:06,6,-66.0,-2.0,66.03029607687671 13 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_study_results/coregistration_study_results_d6/coregistration_study_B06_B07_2023_1_26_12_36_12.csv: -------------------------------------------------------------------------------- 1 | ID_event,granule_couple,coordinates,sensing_time,detector_number,N_v,N_h,N_eff 2 | Mexico_0,"[0, 2]",[ 25.43221955 -100.2269484 ],2021-03-25 17:26:43,6,-83.0,0.0,83.0 3 | Mexico_0,"[2, 4]",[ 25.21910749 -100.28070636],2021-03-25 17:26:47,6,-83.0,0.0,83.0 4 | Greece_5,"[0, 2]",[38.54562942 24.2867681 ],2019-07-05 09:19:45,6,-83.0,0.0,83.0 5 | Greece_5,"[2, 4]",[38.33379261 24.22004876],2019-07-05 09:19:49,6,-83.0,0.0,83.0 6 | Kenya_0,"[0, 2]",[-0.18613292 37.37748282],2019-02-28 07:59:53,6,-83.0,0.0,83.0 7 | Kenya_0,"[2, 4]",[-0.40036289 37.3305413 ],2019-02-28 07:59:56,6,-83.0,0.0,83.0 8 | Hungary_0,"[0, 1]",[47.36679297 19.84754081],2017-08-03 09:47:23,6,-84.0,0.0,84.0 9 | Hungary_0,"[1, 2]",[47.15672067 19.76401333],2017-08-03 09:47:27,6,-84.0,0.0,84.0 10 | France_1,"[0, 2]",[43.60980383 5.83979465],2017-07-24 10:38:47,6,-83.0,0.0,83.0 11 | Italy_2,"[0, 1]",[41.977177 12.82933963],2021-08-13 10:09:03,6,-83.0,0.0,83.0 12 | Italy_2,"[1, 2]",[41.76594104 12.75711796],2021-08-13 10:09:06,6,-83.0,0.0,83.0 13 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_study_results/coregistration_study_results_d6/coregistration_study_B07_B8A_2023_1_26_12_41_46.csv: -------------------------------------------------------------------------------- 1 | ID_event,granule_couple,coordinates,sensing_time,detector_number,N_v,N_h,N_eff 2 | Mexico_0,"[0, 2]",[ 25.43221955 -100.2269484 ],2021-03-25 17:26:43,6,-84.0,0.0,84.0 3 | Mexico_0,"[2, 4]",[ 25.21910749 -100.28070636],2021-03-25 17:26:47,6,-84.0,0.0,84.0 4 | Greece_5,"[0, 2]",[38.54562942 24.2867681 ],2019-07-05 09:19:45,6,-84.0,0.0,84.0 5 | Greece_5,"[2, 4]",[38.33379261 24.22004876],2019-07-05 09:19:49,6,-84.0,0.0,84.0 6 | Kenya_0,"[0, 2]",[-0.18613292 37.37748282],2019-02-28 07:59:53,6,-83.0,0.0,83.0 7 | Kenya_0,"[2, 4]",[-0.40036289 37.3305413 ],2019-02-28 07:59:56,6,-84.0,0.0,84.0 8 | Hungary_0,"[0, 1]",[47.36679297 19.84754081],2017-08-03 09:47:23,6,-84.0,0.0,84.0 9 | Hungary_0,"[1, 2]",[47.15672067 19.76401333],2017-08-03 09:47:27,6,-84.0,0.0,84.0 10 | France_1,"[0, 2]",[43.60980383 5.83979465],2017-07-24 10:38:47,6,-84.0,0.0,84.0 11 | Italy_2,"[0, 1]",[41.977177 12.82933963],2021-08-13 10:09:03,6,-84.0,0.0,84.0 12 | Italy_2,"[1, 2]",[41.76594104 12.75711796],2021-08-13 10:09:06,6,-84.0,0.0,84.0 13 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_study_results/coregistration_study_results_d6/coregistration_study_B08_B03_2023_1_26_12_47_9.csv: -------------------------------------------------------------------------------- 1 | ID_event,granule_couple,coordinates,sensing_time,detector_number,N_v,N_h,N_eff 2 | Mexico_0,"[0, 2]",[ 25.43221955 -100.2269484 ],2021-03-25 17:26:43,6,-165.0,0.0,165.0 3 | Mexico_0,"[2, 4]",[ 25.21910749 -100.28070636],2021-03-25 17:26:47,6,-165.0,0.0,165.0 4 | Greece_5,"[2, 4]",[38.33379261 24.22004876],2019-07-05 09:19:49,6,-166.0,0.0,166.0 5 | Kenya_0,"[0, 2]",[-0.18613292 37.37748282],2019-02-28 07:59:53,6,-164.0,0.0,164.0 6 | Kenya_0,"[2, 4]",[-0.40036289 37.3305413 ],2019-02-28 07:59:56,6,-164.0,0.0,164.0 7 | France_1,"[0, 2]",[43.60980383 5.83979465],2017-07-24 10:38:47,6,-166.0,0.0,166.0 8 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_study_results/coregistration_study_results_d6/coregistration_study_B09_B10_2023_1_26_13_11_16.csv: -------------------------------------------------------------------------------- 1 | ID_event,granule_couple,coordinates,sensing_time,detector_number,N_v,N_h,N_eff 2 | Mexico_0,"[0, 2]",[ 25.43221955 -100.2269484 ],2021-03-25 17:26:43,6,183.0,-2.0,183.0109286354233 3 | Mexico_0,"[2, 4]",[ 25.21910749 -100.28070636],2021-03-25 17:26:47,6,183.0,-2.0,183.0109286354233 4 | Greece_5,"[0, 2]",[38.54562942 24.2867681 ],2019-07-05 09:19:45,6,112.0,-31.0,116.21101496846157 5 | Kenya_0,"[0, 2]",[-0.18613292 37.37748282],2019-02-28 07:59:53,6,182.0,-13.0,182.4636950190366 6 | Hungary_0,"[0, 1]",[47.36679297 19.84754081],2017-08-03 09:47:23,6,181.0,-12.0,181.39735389470266 7 | Hungary_0,"[1, 2]",[47.15672067 19.76401333],2017-08-03 09:47:27,6,182.0,-13.0,182.4636950190366 8 | France_1,"[0, 2]",[43.60980383 5.83979465],2017-07-24 10:38:47,6,184.0,-0.0,184.0 9 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_study_results/coregistration_study_results_d6/coregistration_study_B11_B06_2023_1_26_13_25_46.csv: -------------------------------------------------------------------------------- 1 | ID_event,granule_couple,coordinates,sensing_time,detector_number,N_v,N_h,N_eff 2 | Mexico_0,"[0, 2]",[ 25.43221955 -100.2269484 ],2021-03-25 17:26:43,6,-18.0,2.0,18.110770276274835 3 | Mexico_0,"[2, 4]",[ 25.21910749 -100.28070636],2021-03-25 17:26:47,6,-18.0,2.0,18.110770276274835 4 | Greece_5,"[0, 2]",[38.54562942 24.2867681 ],2019-07-05 09:19:45,6,-16.0,12.0,20.0 5 | Greece_5,"[2, 4]",[38.33379261 24.22004876],2019-07-05 09:19:49,6,-16.0,12.0,20.0 6 | Kenya_0,"[0, 2]",[-0.18613292 37.37748282],2019-02-28 07:59:53,6,-16.0,12.0,20.0 7 | Kenya_0,"[2, 4]",[-0.40036289 37.3305413 ],2019-02-28 07:59:56,6,-16.0,12.0,20.0 8 | France_1,"[0, 2]",[43.60980383 5.83979465],2017-07-24 10:38:47,6,-18.0,2.0,18.110770276274835 9 | Italy_2,"[0, 1]",[41.977177 12.82933963],2021-08-13 10:09:03,6,-18.0,2.0,18.110770276274835 10 | Italy_2,"[1, 2]",[41.76594104 12.75711796],2021-08-13 10:09:06,6,-18.0,2.0,18.110770276274835 11 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_study_results/coregistration_study_results_d6/coregistration_study_B12_B01_2023_1_26_13_13_56.csv: -------------------------------------------------------------------------------- 1 | ID_event,granule_couple,coordinates,sensing_time,detector_number,N_v,N_h,N_eff 2 | Mexico_0,"[0, 2]",[ 25.43221955 -100.2269484 ],2021-03-25 17:26:43,6,-23.0,2.0,23.08679276123039 3 | Mexico_0,"[2, 4]",[ 25.21910749 -100.28070636],2021-03-25 17:26:47,6,-23.0,2.0,23.08679276123039 4 | Greece_5,"[0, 2]",[38.54562942 24.2867681 ],2019-07-05 09:19:45,6,-22.0,12.0,25.059928172283335 5 | Greece_5,"[2, 4]",[38.33379261 24.22004876],2019-07-05 09:19:49,6,-22.0,12.0,25.059928172283335 6 | Kenya_0,"[0, 2]",[-0.18613292 37.37748282],2019-02-28 07:59:53,6,-22.0,12.0,25.059928172283335 7 | Kenya_0,"[2, 4]",[-0.40036289 37.3305413 ],2019-02-28 07:59:56,6,-22.0,12.0,25.059928172283335 8 | Hungary_0,"[0, 1]",[47.36679297 19.84754081],2017-08-03 09:47:23,6,-22.0,12.0,25.059928172283335 9 | Hungary_0,"[1, 2]",[47.15672067 19.76401333],2017-08-03 09:47:27,6,-22.0,12.0,25.059928172283335 10 | France_1,"[0, 2]",[43.60980383 5.83979465],2017-07-24 10:38:47,6,-23.0,2.0,23.08679276123039 11 | Italy_2,"[0, 1]",[41.977177 12.82933963],2021-08-13 10:09:03,6,-23.0,2.0,23.08679276123039 12 | Italy_2,"[1, 2]",[41.76594104 12.75711796],2021-08-13 10:09:06,6,-23.0,2.0,23.08679276123039 13 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_study_results/coregistration_study_results_d6/coregistration_study_B8A_B11_2023_1_26_13_16_33.csv: -------------------------------------------------------------------------------- 1 | ID_event,granule_couple,coordinates,sensing_time,detector_number,N_v,N_h,N_eff 2 | Mexico_0,"[0, 2]",[ 25.43221955 -100.2269484 ],2021-03-25 17:26:43,6,184.0,-2.0,184.0108692441835 3 | Mexico_0,"[2, 4]",[ 25.21910749 -100.28070636],2021-03-25 17:26:47,6,184.0,-2.0,184.0108692441835 4 | Greece_5,"[0, 2]",[38.54562942 24.2867681 ],2019-07-05 09:19:45,6,183.0,-13.0,183.46116755324545 5 | Greece_5,"[2, 4]",[38.33379261 24.22004876],2019-07-05 09:19:49,6,184.0,-12.0,184.39088914585776 6 | Kenya_0,"[0, 2]",[-0.18613292 37.37748282],2019-02-28 07:59:53,6,182.0,-13.0,182.4636950190366 7 | Kenya_0,"[2, 4]",[-0.40036289 37.3305413 ],2019-02-28 07:59:56,6,183.0,-13.0,183.46116755324545 8 | France_1,"[0, 2]",[43.60980383 5.83979465],2017-07-24 10:38:47,6,186.0,-2.0,186.01075237738274 9 | Italy_2,"[0, 1]",[41.977177 12.82933963],2021-08-13 10:09:03,6,186.0,-2.0,186.01075237738274 10 | Italy_2,"[1, 2]",[41.76594104 12.75711796],2021-08-13 10:09:06,6,186.0,-1.0,186.00268815261785 11 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/coregistration_study_results/coregistration_study_results_d6/coregistration_study_B8A_B12_2023_1_26_13_20_58.csv: -------------------------------------------------------------------------------- 1 | ID_event,granule_couple,coordinates,sensing_time,detector_number,N_v,N_h,N_eff 2 | Mexico_0,"[0, 2]",[ 25.43221955 -100.2269484 ],2021-03-25 17:26:43,6,-14.0,-2.0,14.142135623730951 3 | Mexico_0,"[2, 4]",[ 25.21910749 -100.28070636],2021-03-25 17:26:47,6,-13.0,-2.0,13.152946437965905 4 | Greece_5,"[0, 2]",[38.54562942 24.2867681 ],2019-07-05 09:19:45,6,-15.0,-12.0,19.209372712298546 5 | Greece_5,"[2, 4]",[38.33379261 24.22004876],2019-07-05 09:19:49,6,-15.0,-12.0,19.209372712298546 6 | Kenya_0,"[0, 2]",[-0.18613292 37.37748282],2019-02-28 07:59:53,6,-15.0,-12.0,19.209372712298546 7 | Kenya_0,"[2, 4]",[-0.40036289 37.3305413 ],2019-02-28 07:59:56,6,-15.0,-12.0,19.209372712298546 8 | France_1,"[0, 2]",[43.60980383 5.83979465],2017-07-24 10:38:47,6,-14.0,-1.0,14.035668847618199 9 | Italy_2,"[0, 1]",[41.977177 12.82933963],2021-08-13 10:09:03,6,-14.0,-2.0,14.142135623730951 10 | Italy_2,"[1, 2]",[41.76594104 12.75711796],2021-08-13 10:09:06,6,-12.0,-3.0,12.36931687685298 11 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/run_coregistration_study.bat: -------------------------------------------------------------------------------- 1 | python coregistration_study.py --event_class fire --detector_n 6 --bands "[B01, B09]" --equalize --output_dir_path "C:\\Users\\Gabriele Meoni\\Documents\\projects\\end2end\\pyraws\\scripts_and_studies\\coregistration_study\\coregistration_study_results_d6" 2 | python coregistration_study.py --event_class fire --detector_n 6 --bands "[B02, B08]" --equalize --output_dir_path "C:\\Users\\Gabriele Meoni\\Documents\\projects\\end2end\\pyraws\\scripts_and_studies\\coregistration_study\\coregistration_study_results_d6" 3 | python coregistration_study.py --event_class fire --detector_n 6 --bands "[B03, B05]" --equalize --output_dir_path "C:\\Users\\Gabriele Meoni\\Documents\\projects\\end2end\\pyraws\\scripts_and_studies\\coregistration_study\\coregistration_study_results_d6" 4 | python coregistration_study.py --event_class fire --detector_n 6 --bands "[B04, B05]" --equalize --output_dir_path "C:\\Users\\Gabriele Meoni\\Documents\\projects\\end2end\\pyraws\\scripts_and_studies\\coregistration_study\\coregistration_study_results_d6" 5 | python coregistration_study.py --event_class fire --detector_n 6 --bands "[B05, B11]" --equalize --output_dir_path "C:\\Users\\Gabriele Meoni\\Documents\\projects\\end2end\\pyraws\\scripts_and_studies\\coregistration_study\\coregistration_study_results_d6" 6 | python coregistration_study.py --event_class fire --detector_n 6 --bands "[B06, B07]" --equalize --output_dir_path "C:\\Users\\Gabriele Meoni\\Documents\\projects\\end2end\\pyraws\\scripts_and_studies\\coregistration_study\\coregistration_study_results_d6" 7 | python coregistration_study.py --event_class fire --detector_n 6 --bands "[B07, B8A]" --equalize --output_dir_path "C:\\Users\\Gabriele Meoni\\Documents\\projects\\end2end\\pyraws\\scripts_and_studies\\coregistration_study\\coregistration_study_results_d6" 8 | python coregistration_study.py --event_class fire --detector_n 6 --bands "[B08, B03]" --equalize --output_dir_path "C:\\Users\\Gabriele Meoni\\Documents\\projects\\end2end\\pyraws\\scripts_and_studies\\coregistration_study\\coregistration_study_results_d6" 9 | python coregistration_study.py --event_class fire --detector_n 6 --bands "[B09, B10]" --equalize --output_dir_path "C:\\Users\\Gabriele Meoni\\Documents\\projects\\end2end\\pyraws\\scripts_and_studies\\coregistration_study\\coregistration_study_results_d6" 10 | python coregistration_study.py --event_class fire --detector_n 6 --bands "[B12, B01]" --equalize --output_dir_path "C:\\Users\\Gabriele Meoni\\Documents\\projects\\end2end\\pyraws\\scripts_and_studies\\coregistration_study\\coregistration_study_results_d6" 11 | python coregistration_study.py --event_class fire --detector_n 6 --bands "[B8A, B11]" --equalize --output_dir_path "C:\\Users\\Gabriele Meoni\\Documents\\projects\\end2end\\pyraws\\scripts_and_studies\\coregistration_study\\coregistration_study_results_d6" 12 | python coregistration_study.py --event_class fire --detector_n 6 --bands "[B8A, B12]" --equalize --output_dir_path "C:\\Users\\Gabriele Meoni\\Documents\\projects\\end2end\\pyraws\\scripts_and_studies\\coregistration_study\\coregistration_study_results_d6" 13 | python coregistration_study.py --event_class fire --detector_n 6 --bands "[B11, B06]" --equalize --output_dir_path "C:\\Users\\Gabriele Meoni\\Documents\\projects\\end2end\\pyraws\\scripts_and_studies\\coregistration_study\\coregistration_study_results_d6" 14 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/run_test_coarse.bat: -------------------------------------------------------------------------------- 1 | python coregistration_profiling.py --coreg_type coarse --n_event 1 --test_iteration warm_up_1 2 | python coregistration_profiling.py --coreg_type coarse --n_event 1 --test_iteration warm_up_2 3 | python coregistration_profiling.py --coreg_type coarse --n_event 1 --test_iteration 1 4 | python coregistration_profiling.py --coreg_type coarse --n_event 1 --test_iteration 2 5 | python coregistration_profiling.py --coreg_type coarse --n_event 1 --test_iteration 3 6 | python coregistration_profiling.py --coreg_type coarse --n_event 2 --test_iteration 1 7 | python coregistration_profiling.py --coreg_type coarse --n_event 2 --test_iteration 2 8 | python coregistration_profiling.py --coreg_type coarse --n_event 2 --test_iteration 3 9 | python coregistration_profiling.py --coreg_type coarse --n_event 4 --test_iteration 1 10 | python coregistration_profiling.py --coreg_type coarse --n_event 4 --test_iteration 2 11 | python coregistration_profiling.py --coreg_type coarse --n_event 4 --test_iteration 3 12 | python coregistration_profiling.py --coreg_type coarse --n_event 8 --test_iteration 1 13 | python coregistration_profiling.py --coreg_type coarse --n_event 8 --test_iteration 2 14 | python coregistration_profiling.py --coreg_type coarse --n_event 8 --test_iteration 3 15 | python coregistration_profiling.py --coreg_type coarse --n_event 16 --test_iteration 1 16 | python coregistration_profiling.py --coreg_type coarse --n_event 16 --test_iteration 2 17 | python coregistration_profiling.py --coreg_type coarse --n_event 16 --test_iteration 3 18 | python coregistration_profiling.py --coreg_type coarse --n_event 1 --device gpu --test_iteration warm_up_1 19 | python coregistration_profiling.py --coreg_type coarse --n_event 1 --device gpu --test_iteration warm_up_2 20 | python coregistration_profiling.py --coreg_type coarse --n_event 1 --device gpu --test_iteration 1 21 | python coregistration_profiling.py --coreg_type coarse --n_event 1 --device gpu --test_iteration 2 22 | python coregistration_profiling.py --coreg_type coarse --n_event 1 --device gpu --test_iteration 3 23 | python coregistration_profiling.py --coreg_type coarse --n_event 2 --device gpu --test_iteration 1 24 | python coregistration_profiling.py --coreg_type coarse --n_event 2 --device gpu --test_iteration 2 25 | python coregistration_profiling.py --coreg_type coarse --n_event 2 --device gpu --test_iteration 3 26 | python coregistration_profiling.py --coreg_type coarse --n_event 4 --device gpu --test_iteration 1 27 | python coregistration_profiling.py --coreg_type coarse --n_event 4 --device gpu --test_iteration 2 28 | python coregistration_profiling.py --coreg_type coarse --n_event 4 --device gpu --test_iteration 3 29 | python coregistration_profiling.py --coreg_type coarse --n_event 8 --device gpu --test_iteration 1 30 | python coregistration_profiling.py --coreg_type coarse --n_event 8 --device gpu --test_iteration 2 31 | python coregistration_profiling.py --coreg_type coarse --n_event 8 --device gpu --test_iteration 3 32 | python coregistration_profiling.py --coreg_type coarse --n_event 16 --device gpu --test_iteration 1 33 | python coregistration_profiling.py --coreg_type coarse --n_event 16 --device gpu --test_iteration 2 34 | python coregistration_profiling.py --coreg_type coarse --n_event 16 --device gpu --test_iteration 3 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/run_test_coarse.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | python coregistration_profiling.py --coreg_type coarse --n_event 1 --test_iteration warm_up_1 4 | python coregistration_profiling.py --coreg_type coarse --n_event 1 --test_iteration warm_up_2 5 | python coregistration_profiling.py --coreg_type coarse --n_event 1 --test_iteration 1 6 | python coregistration_profiling.py --coreg_type coarse --n_event 1 --test_iteration 2 7 | python coregistration_profiling.py --coreg_type coarse --n_event 1 --test_iteration 3 8 | python coregistration_profiling.py --coreg_type coarse --n_event 2 --test_iteration 1 9 | python coregistration_profiling.py --coreg_type coarse --n_event 2 --test_iteration 2 10 | python coregistration_profiling.py --coreg_type coarse --n_event 2 --test_iteration 3 11 | python coregistration_profiling.py --coreg_type coarse --n_event 4 --test_iteration 1 12 | python coregistration_profiling.py --coreg_type coarse --n_event 4 --test_iteration 2 13 | python coregistration_profiling.py --coreg_type coarse --n_event 4 --test_iteration 3 14 | python coregistration_profiling.py --coreg_type coarse --n_event 8 --test_iteration 1 15 | python coregistration_profiling.py --coreg_type coarse --n_event 8 --test_iteration 2 16 | python coregistration_profiling.py --coreg_type coarse --n_event 8 --test_iteration 3 17 | python coregistration_profiling.py --coreg_type coarse --n_event 16 --test_iteration 1 18 | python coregistration_profiling.py --coreg_type coarse --n_event 16 --test_iteration 2 19 | python coregistration_profiling.py --coreg_type coarse --n_event 16 --test_iteration 3 20 | python coregistration_profiling.py --coreg_type coarse --n_event 1 --device gpu --test_iteration warm_up_1 21 | python coregistration_profiling.py --coreg_type coarse --n_event 1 --device gpu --test_iteration warm_up_2 22 | python coregistration_profiling.py --coreg_type coarse --n_event 1 --device gpu --test_iteration 1 23 | python coregistration_profiling.py --coreg_type coarse --n_event 1 --device gpu --test_iteration 2 24 | python coregistration_profiling.py --coreg_type coarse --n_event 1 --device gpu --test_iteration 3 25 | python coregistration_profiling.py --coreg_type coarse --n_event 2 --device gpu --test_iteration 1 26 | python coregistration_profiling.py --coreg_type coarse --n_event 2 --device gpu --test_iteration 2 27 | python coregistration_profiling.py --coreg_type coarse --n_event 2 --device gpu --test_iteration 3 28 | python coregistration_profiling.py --coreg_type coarse --n_event 4 --device gpu --test_iteration 1 29 | python coregistration_profiling.py --coreg_type coarse --n_event 4 --device gpu --test_iteration 2 30 | python coregistration_profiling.py --coreg_type coarse --n_event 4 --device gpu --test_iteration 3 31 | python coregistration_profiling.py --coreg_type coarse --n_event 8 --device gpu --test_iteration 1 32 | python coregistration_profiling.py --coreg_type coarse --n_event 8 --device gpu --test_iteration 2 33 | python coregistration_profiling.py --coreg_type coarse --n_event 8 --device gpu --test_iteration 3 34 | python coregistration_profiling.py --coreg_type coarse --n_event 16 --device gpu --test_iteration 1 35 | python coregistration_profiling.py --coreg_type coarse --n_event 16 --device gpu --test_iteration 2 36 | python coregistration_profiling.py --coreg_type coarse --n_event 16 --device gpu --test_iteration 3 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/run_test_superglue.bat: -------------------------------------------------------------------------------- 1 | python coregistration_profiling.py --coreg_type super_glue --n_event 1 --test_iteration warm_up_1 2 | python coregistration_profiling.py --coreg_type super_glue --n_event 1 --test_iteration warm_up_2 3 | python coregistration_profiling.py --coreg_type super_glue --n_event 1 --test_iteration 1 4 | python coregistration_profiling.py --coreg_type super_glue --n_event 1 --test_iteration 2 5 | python coregistration_profiling.py --coreg_type super_glue --n_event 1 --test_iteration 3 6 | python coregistration_profiling.py --coreg_type super_glue --n_event 2 --test_iteration 1 7 | python coregistration_profiling.py --coreg_type super_glue --n_event 2 --test_iteration 2 8 | python coregistration_profiling.py --coreg_type super_glue --n_event 2 --test_iteration 3 9 | python coregistration_profiling.py --coreg_type super_glue --n_event 4 --test_iteration 1 10 | python coregistration_profiling.py --coreg_type super_glue --n_event 4 --test_iteration 2 11 | python coregistration_profiling.py --coreg_type super_glue --n_event 4 --test_iteration 3 12 | python coregistration_profiling.py --coreg_type super_glue --n_event 8 --test_iteration 1 13 | python coregistration_profiling.py --coreg_type super_glue --n_event 8 --test_iteration 2 14 | python coregistration_profiling.py --coreg_type super_glue --n_event 8 --test_iteration 3 15 | python coregistration_profiling.py --coreg_type super_glue --n_event 16 --test_iteration 1 16 | python coregistration_profiling.py --coreg_type super_glue --n_event 16 --test_iteration 2 17 | python coregistration_profiling.py --coreg_type super_glue --n_event 16 --test_iteration 3 18 | python coregistration_profiling.py --coreg_type super_glue --n_event 1 --device gpu --test_iteration warm_up_1 19 | python coregistration_profiling.py --coreg_type super_glue --n_event 1 --device gpu --test_iteration warm_up_2 20 | python coregistration_profiling.py --coreg_type super_glue --n_event 1 --device gpu --test_iteration 1 21 | python coregistration_profiling.py --coreg_type super_glue --n_event 1 --device gpu --test_iteration 2 22 | python coregistration_profiling.py --coreg_type super_glue --n_event 1 --device gpu --test_iteration 3 23 | python coregistration_profiling.py --coreg_type super_glue --n_event 2 --device gpu --test_iteration 1 24 | python coregistration_profiling.py --coreg_type super_glue --n_event 2 --device gpu --test_iteration 2 25 | python coregistration_profiling.py --coreg_type super_glue --n_event 2 --device gpu --test_iteration 3 26 | python coregistration_profiling.py --coreg_type super_glue --n_event 4 --device gpu --test_iteration 1 27 | python coregistration_profiling.py --coreg_type super_glue --n_event 4 --device gpu --test_iteration 2 28 | python coregistration_profiling.py --coreg_type super_glue --n_event 4 --device gpu --test_iteration 3 29 | python coregistration_profiling.py --coreg_type super_glue --n_event 8 --device gpu --test_iteration 1 30 | python coregistration_profiling.py --coreg_type super_glue --n_event 8 --device gpu --test_iteration 2 31 | python coregistration_profiling.py --coreg_type super_glue --n_event 8 --device gpu --test_iteration 3 32 | python coregistration_profiling.py --coreg_type super_glue --n_event 16 --device gpu --test_iteration 1 33 | python coregistration_profiling.py --coreg_type super_glue --n_event 16 --device gpu --test_iteration 2 34 | python coregistration_profiling.py --coreg_type super_glue --n_event 16 --device gpu --test_iteration 3 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /scripts_and_studies/coregistration_study/run_test_superglue.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | python coregistration_profiling.py --coreg_type super_glue --n_event 1 --test_iteration warm_up_1 4 | python coregistration_profiling.py --coreg_type super_glue --n_event 1 --test_iteration warm_up_2 5 | python coregistration_profiling.py --coreg_type super_glue --n_event 1 --test_iteration 1 6 | python coregistration_profiling.py --coreg_type super_glue --n_event 1 --test_iteration 2 7 | python coregistration_profiling.py --coreg_type super_glue --n_event 1 --test_iteration 3 8 | python coregistration_profiling.py --coreg_type super_glue --n_event 2 --test_iteration 1 9 | python coregistration_profiling.py --coreg_type super_glue --n_event 2 --test_iteration 2 10 | python coregistration_profiling.py --coreg_type super_glue --n_event 2 --test_iteration 3 11 | python coregistration_profiling.py --coreg_type super_glue --n_event 4 --test_iteration 1 12 | python coregistration_profiling.py --coreg_type super_glue --n_event 4 --test_iteration 2 13 | python coregistration_profiling.py --coreg_type super_glue --n_event 4 --test_iteration 3 14 | python coregistration_profiling.py --coreg_type super_glue --n_event 8 --test_iteration 1 15 | python coregistration_profiling.py --coreg_type super_glue --n_event 8 --test_iteration 2 16 | python coregistration_profiling.py --coreg_type super_glue --n_event 8 --test_iteration 3 17 | python coregistration_profiling.py --coreg_type super_glue --n_event 16 --test_iteration 1 18 | python coregistration_profiling.py --coreg_type super_glue --n_event 16 --test_iteration 2 19 | python coregistration_profiling.py --coreg_type super_glue --n_event 16 --test_iteration 3 20 | python coregistration_profiling.py --coreg_type super_glue --n_event 1 --device gpu --test_iteration warm_up_1 21 | python coregistration_profiling.py --coreg_type super_glue --n_event 1 --device gpu --test_iteration warm_up_2 22 | python coregistration_profiling.py --coreg_type super_glue --n_event 1 --device gpu --test_iteration 1 23 | python coregistration_profiling.py --coreg_type super_glue --n_event 1 --device gpu --test_iteration 2 24 | python coregistration_profiling.py --coreg_type super_glue --n_event 1 --device gpu --test_iteration 3 25 | python coregistration_profiling.py --coreg_type super_glue --n_event 2 --device gpu --test_iteration 1 26 | python coregistration_profiling.py --coreg_type super_glue --n_event 2 --device gpu --test_iteration 2 27 | python coregistration_profiling.py --coreg_type super_glue --n_event 2 --device gpu --test_iteration 3 28 | python coregistration_profiling.py --coreg_type super_glue --n_event 4 --device gpu --test_iteration 1 29 | python coregistration_profiling.py --coreg_type super_glue --n_event 4 --device gpu --test_iteration 2 30 | python coregistration_profiling.py --coreg_type super_glue --n_event 4 --device gpu --test_iteration 3 31 | python coregistration_profiling.py --coreg_type super_glue --n_event 8 --device gpu --test_iteration 1 32 | python coregistration_profiling.py --coreg_type super_glue --n_event 8 --device gpu --test_iteration 2 33 | python coregistration_profiling.py --coreg_type super_glue --n_event 8 --device gpu --test_iteration 3 34 | python coregistration_profiling.py --coreg_type super_glue --n_event 16 --device gpu --test_iteration 1 35 | python coregistration_profiling.py --coreg_type super_glue --n_event 16 --device gpu --test_iteration 2 36 | python coregistration_profiling.py --coreg_type super_glue --n_event 16 --device gpu --test_iteration 3 37 | 38 | 39 | -------------------------------------------------------------------------------- /scripts_and_studies/dataset_preparation/THRAWS/fire/fires_db.csv: -------------------------------------------------------------------------------- 1 | event,l1c_tile,satellite,lat,lon 2 | Spain_0,S2B_MSIL1C_20210910T105619_N0301_R094_T30SUF_20210910T114452,S2B,36.5377,5.1721 3 | France_0,S2A_MSIL1C_20210817T103021_N0301_R108_T31TGJ_20210817T123758,S2A,43.3328,6.4114 4 | Greece_0,S2B_MSIL1C_20210818T090559_N0301_R050_T34SFH_20210818T100108,S2B,38.1095,23.3288 5 | Portugal_0,S2B_MSIL1C_20210814T110619_N0301_R137_T29SPB_20210814T120505,S2B,37.5834,-6.7966 6 | Spain_1,S2B_MSIL1C_20210814T110619_N0301_R137_T30TUK_20210814T120505,S2B,39.9546,-4.3299 7 | Greece_1,S2A_MSIL1C_20210806T092031_N0301_R093_T34SFF_20210806T103112,S2A,36.8062,22.4375 8 | Greece_2,S2B_MSIL1C_20210802T084559_N0301_R107_T35SNA_20210802T093857,S2B,37.0356,27.8118 9 | Latvia_0,S2A_MSIL1C_20210726T095031_N0301_R079_T35VLD_20210726T103639,S2A,57.3541,24.8478 10 | Mexico_0,S2B_MSIL1C_20210325T170849_N0209_R112_T14RLP_20210325T205012,S2B,25.3829,-100.4084 11 | Australia_0,S2A_MSIL1C_20210205T021341_N0209_R060_T50HMJ_20210205T055002,S2A,-32.6424,116.704 12 | Spain_2,S2B_MSIL1C_20200829T110629_N0209_R137_T30TTK_20200829T122327,S2B,40.1657,-5.6997 13 | Greece_3,S2B_MSIL1C_20200823T090559_N0209_R050_T34SFF_20200823T101613,S2B,36.7116,22.4537 14 | Ukraine_0,S2B_MSIL1C_20200412T085549_N0209_R007_T35UQT_20200412T111506,S2B,51.3637,30.0296 15 | Australia_1,S2A_MSIL1C_20191107T235251_N0208_R130_T56JMM_20191108T011214,S2A,-29.9192,152.3357 16 | Italy_0,S2B_MSIL1C_20190920T095029_N0208_R079_T33TVF_20190920T115647,S2B,41.168,14.4398 17 | Bolivia_0,S2A_MSIL1C_20190821T142041_N0208_R010_T20LNH_20190821T155923,S2A,-15.8904,-62.3047 18 | Bolivia_1,S2A_MSIL1C_20190801T142041_N0208_R010_T20LQH_20190801T174138,S2A,-15.6869,-60.488 19 | Greece_4,S2A_MSIL1C_20190814T090601_N0208_R050_T35SKC_20190814T104032,S2A,38.6479,23.68 20 | Greenland_0,S2A_MSIL1C_20190812T150921_N0208_R025_T21WXQ_20190812T201743,S2A,66.9996,-53.1842 21 | Spain_3,S2B_MSIL1C_20190723T105629_N0208_R094_T30TXM_20190723T130323,S2B,42.152,-1.6434 22 | Greece_5,S2A_MSIL1C_20190705T090601_N0207_R050_T35SKC_20190705T113504,S2A,38.4886,24.0979 23 | Kenya_0,S2A_MSIL1C_20190228T073841_N0207_R092_T37MBV_20190228T093948,S2A,-0.2038,37.2755 24 | Latvia_1,S2A_MSIL1C_20180801T095031_N0206_R079_T35VLE_20180801T134035,S2A,57.9515,24.8847 25 | Sweden_0,S2A_MSIL1C_20180717T104021_N0206_R008_T33VVJ_20180717T160515,S2A,61.815,14.4926 26 | Spain_4,S2B_MSIL1C_20170729T112119_N0205_R037_T29TQF_20170729T112341,S2B,41.1379,-6.4064 27 | Hungary_0,S2A_MSIL1C_20170803T094031_N0205_R036_T34TDT_20170803T094046,S2A,47.3183,19.8541 28 | Spain_5,S2A_MSIL1C_20170728T105621_N0205_R094_T30SWH_20170728T110406,S2A,38.4359,-2.3611 29 | France_1,S2B_MSIL1C_20170724T103019_N0205_R108_T31TGJ_20170724T103022,S2B,43.6791,5.6015 30 | Italy_1,S2B_MSIL1C_20170712T095029_N0205_R079_T33TVF_20170712T095623,S2B,40.8263,14.4229 31 | Italy_2,S2B_MSIL1C_20210813T100029_N0301_R122_T33TUG_20210813T120753,S2B,41.96563,12.81618 32 | Korea_0,S2A_MSIL1C_20220305T020701_N0400_R103_T52SEG_20220305T035602,S2A,37.56,129.0413 33 | -------------------------------------------------------------------------------- /scripts_and_studies/dataset_preparation/THRAWS/volcano_eruptions/create_json_polygons.py: -------------------------------------------------------------------------------- 1 | import geopandas 2 | from shapely.geometry import Polygon 3 | from create_volcano_csv import parse_csv 4 | import argparse 5 | import os 6 | from glob import glob 7 | 8 | 9 | def create_output_dir(out_dir): 10 | print("Creating the directory: ", out_dir) 11 | if os.path.isdir(out_dir): 12 | print("Cleaning the ", out_dir, "directory.") 13 | for file_to_remove in sorted(glob(os.path.join(out_dir, "*"))): 14 | os.remove(out_dir) 15 | else: 16 | os.mkdir(out_dir) 17 | 18 | 19 | def parse_coordinate(coordinate_str): 20 | lon = float( 21 | coordinate_str[coordinate_str.find("[") + 1 : coordinate_str.find(",") - 1] 22 | ) 23 | lat = float( 24 | coordinate_str[coordinate_str.find(",") + 1 : coordinate_str.find("]") - 1] 25 | ) 26 | return (lon, lat) 27 | 28 | 29 | def get_polygon(eruption): 30 | coord_sw = parse_coordinate(eruption["Pol-SW"]) 31 | coord_nw = parse_coordinate(eruption["Pol-NW"]) 32 | coord_ne = parse_coordinate(eruption["Pol-NE"]) 33 | coord_se = parse_coordinate(eruption["Pol-SE"]) 34 | return Polygon([coord_sw, coord_nw, coord_ne, coord_se]) 35 | 36 | 37 | def main(): 38 | parser = argparse.ArgumentParser() 39 | parser.add_argument( 40 | "--eruption_cvs", 41 | type=str, 42 | help="Eruption list CSV.", 43 | default="eruption_selected.csv", 44 | ) 45 | parser.add_argument( 46 | "--json_dir_path", 47 | type=str, 48 | help="Path to the directory where JSON files are stored.", 49 | default="./json", 50 | ) 51 | pargs = parser.parse_args() 52 | csv_name = pargs.eruption_cvs 53 | csv_content = parse_csv(csv_name) 54 | json_dir = pargs.json_dir_path 55 | 56 | create_output_dir(json_dir) 57 | 58 | for eruption in csv_content: 59 | polygon = get_polygon(eruption) 60 | json_content = geopandas.GeoSeries([polygon]).to_json() 61 | 62 | with open( 63 | os.path.join( 64 | json_dir, eruption["Name"].replace(" ", "_").replace("/", "_") + ".json" 65 | ), 66 | "w", 67 | ) as f: 68 | f.write(json_content) 69 | 70 | 71 | if __name__ == "__main__": 72 | main() 73 | -------------------------------------------------------------------------------- /scripts_and_studies/dataset_preparation/THRAWS/volcano_eruptions/eruption_list.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESA-PhiLab/PyRawS/95d291388192d1c71ae302775c3bcf636717dc00/scripts_and_studies/dataset_preparation/THRAWS/volcano_eruptions/eruption_list.xlsx -------------------------------------------------------------------------------- /scripts_and_studies/dataset_preparation/THRAWS/volcano_eruptions/json_polygon_from_point.py: -------------------------------------------------------------------------------- 1 | import geopandas 2 | from shapely.geometry import Polygon 3 | import argparse 4 | 5 | # from l0_dataset_preparation.volcano_eruptions.create_volcano_csv import create_rectangular_polygon, parse_csv 6 | from .create_volcano_csv import create_rectangular_polygon 7 | 8 | 9 | def create_polygon_from_coordinates( 10 | lat, lon, center_distance_x, center_distance_y, json_name=None 11 | ): 12 | center_coordinates = {"lat": lat, "lon": lon} 13 | polygon_dict = create_rectangular_polygon( 14 | center_coordinates, center_distance_x, center_distance_y 15 | ) 16 | polygon = Polygon( 17 | [polygon_dict["SW"], polygon_dict["NW"], polygon_dict["NE"], polygon_dict["SE"]] 18 | ) 19 | json_content = geopandas.GeoSeries([polygon]).to_json() 20 | 21 | if json_name is not None: 22 | with open(json_name, "w") as f: 23 | f.write(json_content) 24 | f.write("\n\n") 25 | f.write(str(polygon_dict["SW"])) 26 | f.write("\n\n") 27 | f.write(str(polygon_dict["NW"])) 28 | f.write("\n\n") 29 | f.write(str(polygon_dict["NE"])) 30 | f.write("\n\n") 31 | f.write(str(polygon_dict["SE"])) 32 | 33 | return polygon 34 | 35 | 36 | def main(): 37 | parser = argparse.ArgumentParser() 38 | parser.add_argument("--lat", type=float, help="Point latitude.") 39 | parser.add_argument("--lon", type=float, help="Point latitude.") 40 | parser.add_argument( 41 | "--dx_km", 42 | type=float, 43 | help="Horizontal distance from the polygon center in km.", 44 | default=5.0, 45 | ) 46 | parser.add_argument( 47 | "--dy_km", 48 | type=float, 49 | help="Vertical distance from the polygon center in km.", 50 | default=14.0, 51 | ) 52 | parser.add_argument( 53 | "--json_name", type=str, help="json file name.", default="my_json_polygon.json" 54 | ) 55 | pargs = parser.parse_args() 56 | create_polygon_from_coordinates( 57 | pargs.lat, pargs.lon, pargs.dx_km * 1000, pargs.dy_km * 1000, pargs.json_name 58 | ) 59 | 60 | 61 | if __name__ == "__main__": 62 | main() 63 | -------------------------------------------------------------------------------- /scripts_and_studies/dataset_preparation/THRAWS/volcano_eruptions/my_json_polygon.json: -------------------------------------------------------------------------------- 1 | {"type": "FeatureCollection", "features": [{"id": "0", "type": "Feature", "properties": {}, "geometry": {"type": "Polygon", "coordinates": [[[-22.37621324140913, 63.765616876827806], [-22.37712257266201, 64.01679244683255], [-22.172633599113848, 64.01679244683255], [-22.173542930366732, 63.765616876827806], [-22.37621324140913, 63.765616876827806]]]}, "bbox": [-22.37712257266201, 63.765616876827806, -22.172633599113848, 64.01679244683255]}], "bbox": [-22.37712257266201, 63.765616876827806, -22.172633599113848, 64.01679244683255]} 2 | 3 | [-22.37621324140913, 63.765616876827806] 4 | 5 | [-22.37712257266201, 64.01679244683255] 6 | 7 | [-22.172633599113848, 64.01679244683255] 8 | 9 | [-22.173542930366732, 63.765616876827806] -------------------------------------------------------------------------------- /scripts_and_studies/hta_detection_algorithms/hta_detection.py: -------------------------------------------------------------------------------- 1 | # [1] - Liu, Yongxue, et al. "Detecting high-temperature anomalies from Sentinel-2 MSI images." 2 | # ISPRS Journal of Photogrammetry and Remote Sensing 177 (2021): 174-193. 3 | # The next functions implement a simplified version of the algorithm described in [1]. 4 | 5 | import torch 6 | import torchvision 7 | 8 | 9 | def get_TAI(s2_l1c_img): 10 | """Returns the th Thermal Anomaly Index (TAI) as defined in [1]. 11 | 12 | Args: 13 | s2_l1c_img (torch.tensor): Sentinel2-L1C image. 14 | 15 | Returns: 16 | torch.tensor: TAI (with negative values saturated a 0) 17 | """ 18 | 19 | tai = torch.div(s2_l1c_img[:, :, 2] - s2_l1c_img[:, :, 1], s2_l1c_img[:, :, 0]) 20 | 21 | return torch.where(tai >= torch.zeros_like(tai), tai, torch.zeros_like(tai)) 22 | 23 | 24 | def get_TAI_mean(tai): 25 | """It returns the TAI mean as defined in [1]. TAI_mean is calculated by using a 15x15 convolutional filter. 26 | The input image is padded with the values at the boundaries to avoid boundaries false positives. 27 | 28 | Args: 29 | tai (torch.tensor): tai index 30 | 31 | Returns: 32 | torch.tensor: convoluted alert-map 33 | """ 34 | conv = torch.nn.Conv2d(1, 1, 31) 35 | weight = torch.nn.Parameter( 36 | 1 / 31 * torch.ones([31, 31]).unsqueeze(0).unsqueeze(0), requires_grad=False 37 | ) 38 | 39 | if tai.device.type == "cuda": 40 | conv = conv.cuda() 41 | conv.load_state_dict({"weight": weight, "bias": torch.zeros([1])}, strict=False) 42 | with torch.no_grad(): 43 | padding = torchvision.transforms.Pad((15, 15, 15, 15), padding_mode="edge") 44 | img_pad = padding(tai) 45 | tai_mean = ( 46 | conv(torch.tensor(img_pad, dtype=torch.float32).unsqueeze(0).unsqueeze(0)) 47 | .squeeze(0) 48 | .squeeze(0) 49 | ) 50 | del weight 51 | del conv 52 | del img_pad 53 | torch.cuda.empty_cache() 54 | return tai_mean 55 | 56 | 57 | def get_TAI_buffer(tai, tai_mean): 58 | """Return the TAI buffer as in [1] for each active pixel. 59 | 60 | Args: 61 | tai (torch.tensor): TAI index. 62 | tai_mean (torch.tensor): TAI mean (over 15 pixel ray) buffer. 63 | 64 | Returns: 65 | torch.tensor: TAI buffer. 66 | """ 67 | with torch.no_grad(): 68 | s_index = torch.where(tai - tai_mean >= 0.45, 1, 0).nonzero() 69 | padding = torchvision.transforms.Pad( 70 | (15, 15, 15, 15), padding_mode="constant", fill=0 71 | ) 72 | s_pad = padding(tai) 73 | b = torch.zeros_like(s_pad) 74 | for r, c in s_index: 75 | b[r - 15 : r + 15, c - 15 : c + 15] = 1 76 | 77 | b = b[15:-15, 15:-15] # Removing edge pixels having no information. 78 | 79 | return b 80 | 81 | 82 | def cluster_9px(img): 83 | """It performs the convolution to detect clusters of 9 activate pixels (current pixel and 8 surrounding pixels) are at 1. 84 | 85 | Args: 86 | img (torch.tensor): input alert-matrix 87 | 88 | Returns: 89 | torch.tensor: convoluted alert-map 90 | """ 91 | 92 | conv = torch.nn.Conv2d(1, 1, 3) 93 | weight = torch.nn.Parameter( 94 | torch.tensor([[[[1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]]]]), 95 | requires_grad=False, 96 | ) 97 | img_pad = torch.nn.functional.pad(img, (1, 1, 1, 1), mode="constant", value=1) 98 | if img.device.type == "cuda": 99 | conv = conv.cuda() 100 | conv.load_state_dict({"weight": weight, "bias": torch.zeros([1])}, strict=False) 101 | with torch.no_grad(): 102 | surrounded = ( 103 | conv(torch.tensor(img_pad, dtype=torch.float32).unsqueeze(0).unsqueeze(0)) 104 | .squeeze(0) 105 | .squeeze(0) 106 | ) 107 | # Adding clipping to the mean 108 | surrounded = torch.where( 109 | surrounded >= 0.45, 0.45 * torch.ones_like(surrounded), surrounded 110 | ) 111 | del weight 112 | del conv 113 | del img_pad 114 | torch.cuda.empty_cache() 115 | return surrounded 116 | 117 | 118 | def manage_saturated_thermal_anomalies(s2_l1c_img): 119 | """It checks thermal anomalies as described in [1] by returning a mask to control not-8-connected thermal anomalies. 120 | 121 | Args: 122 | s2_l1c_img (torch.tensor): Sentinel2 L1C image. 123 | 124 | Returns: 125 | torch.tensor: saturated thermal anomalies mask 126 | """ 127 | saturated_mask = torch.logical_and( 128 | torch.where(s2_l1c_img[:, :, 2] >= 1, 1, 0), 129 | torch.where(s2_l1c_img[:, :, 2] >= 1, 1, 0), 130 | ) 131 | saturated_mask_8_connected = cluster_9px(saturated_mask) 132 | return torch.logical_or( 133 | saturated_mask_8_connected, torch.logical_not(saturated_mask) 134 | ) 135 | 136 | 137 | def get_image_lost_source_mask(s2_l1c_img): 138 | """It removes pixels with lost packages as in [1] by returning a mask. 139 | 140 | Args: 141 | s2_l1c_img (torch.tensor): Sentinel2 L1C image. 142 | 143 | Returns: 144 | torch.tensor: nask to exclude image lost. 145 | """ 146 | source_lost_mask_negated = torch.logical_or( 147 | torch.where(s2_l1c_img[:, :, 0] < 0.01, 1, 0), 148 | torch.where(s2_l1c_img[:, :, 1] < 0.05, 1, 0), 149 | ) 150 | return torch.logical_not(source_lost_mask_negated) 151 | 152 | 153 | def remove_false_positives(s2_l1c_img, thermal_anomalies_map): 154 | """Removing false positives due to saturated surfaces, image packet loss or edge anomalies. 155 | 156 | Args: 157 | s2_l1c_img (torch.tensor): Sentinel2 L1C image 158 | thermal_anomalies_map (torch.tensor): thermal anomalies binary map to clean 159 | 160 | Returns: 161 | torch.tensor: clean thermal_anomalies_map 162 | """ 163 | thermal_saturated_mask = manage_saturated_thermal_anomalies(s2_l1c_img) 164 | thermal_saturated_mask_no_saturation = torch.logical_and( 165 | thermal_anomalies_map, thermal_saturated_mask 166 | ) 167 | source_lost_mask = get_image_lost_source_mask(s2_l1c_img) 168 | thermal_anomalies_no_lost_no_saturation = torch.logical_and( 169 | thermal_saturated_mask_no_saturation, source_lost_mask 170 | ) 171 | # Removing pixel affected by NaN 172 | nan_mask = torch.logical_not( 173 | torch.logical_or( 174 | torch.logical_or( 175 | torch.isnan(s2_l1c_img[:, :, 0]), torch.isnan(s2_l1c_img[:, :, 1]) 176 | ), 177 | torch.isnan(s2_l1c_img[:, :, 2]), 178 | ) 179 | ) 180 | thermal_anomalies_clean_map = torch.logical_and( 181 | thermal_anomalies_no_lost_no_saturation, nan_mask 182 | ) 183 | return thermal_anomalies_clean_map 184 | 185 | 186 | def extract_high_tempearature_anomalies(s2_l1c_img): 187 | """Extracts the hight temperature anomaly map as in [1] without time-series check. 188 | 189 | Args: 190 | s2_l1c_img (torch.tensor): Sentinel2 L1C image. 191 | 192 | Returns: 193 | torch.tensor: high-temperature anomalis map 194 | """ 195 | tai = get_TAI(s2_l1c_img) 196 | tai_mean = get_TAI_mean(tai) 197 | b = get_TAI_buffer(tai, tai_mean) 198 | thr_1 = torch.logical_and( 199 | torch.where( 200 | tai >= 0.45 * torch.ones_like(tai), 201 | torch.ones_like(tai), 202 | torch.zeros_like(tai), 203 | ), 204 | b, 205 | ) # Selecting pixels in the buffer meeting TAI >= 0.45. 206 | thr_2 = torch.where( 207 | s2_l1c_img[:, :, 2] > 2 * s2_l1c_img[:, :, 1] - s2_l1c_img[:, :, 0], 208 | torch.ones_like(tai), 209 | torch.zeros_like(tai), 210 | ) # Condition p_far_SWIR - p_near_SWIR > p_near_SWIR - p_NIR. 211 | thr_3 = torch.where( 212 | s2_l1c_img[:, :, 2] > 0.15 * torch.ones_like(tai), 213 | torch.ones_like(tai), 214 | torch.zeros_like(tai), 215 | ) # Removing low reflectivity surfaces in FAR SWIR. 216 | thermal_anomaly_map = torch.logical_and(torch.logical_and(thr_1, thr_2), thr_3) 217 | thermal_anomaly_clean_map = remove_false_positives(s2_l1c_img, thermal_anomaly_map) 218 | return thermal_anomaly_clean_map, thermal_anomaly_map 219 | -------------------------------------------------------------------------------- /scripts_and_studies/patch_study/patchify.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | sys.path.insert(1, os.path.join("..", "..")) 5 | sys.path.insert( 6 | 1, os.path.join("..", "..", "scripts_and_studies", "hta_detection_algorithms") 7 | ) 8 | from pyraws.raw.raw_event import Raw_event 9 | from shapely.geometry import Polygon 10 | import torch 11 | import matplotlib.pyplot as plt 12 | 13 | # CONFIG_ 14 | os.environ["KMP_DUPLICATE_LIB_OK"] = "True" 15 | if torch.cuda.is_available(): 16 | device = torch.device("cuda") 17 | else: 18 | device = torch.device("cpu") 19 | 20 | plt.rcParams["figure.figsize"] = [10, 10] 21 | 22 | 23 | def mulitBox(bbox_list): 24 | if any(isinstance(i, list) for i in bbox_list): 25 | return True 26 | 27 | return False 28 | 29 | 30 | def sliding_window(image, window_size, overlap): 31 | """ 32 | Generate sub-images of the input image using a sliding window. 33 | Parameters: 34 | - image (2D numpy array): The input image. 35 | - window_size (int): The size (in pixels) of the sliding window. 36 | - overlap (float): The degree of overlap between two consecutive window movements. 37 | 38 | Returns: 39 | - sub_images (dict): A dictionary with the top-left and bottom-right corner positions (x1, y1, x2, y2) 40 | as keys and the sub-images as values. 41 | """ 42 | # Get the dimensions of the input image 43 | h, w = image.shape[:2] 44 | 45 | # Calculate the step size (in pixels) for the sliding window 46 | step_size = int(window_size * (1 - overlap)) 47 | 48 | # Initialize the dictionary of sub-images 49 | sub_images = {} 50 | 51 | # Slide the window across the image 52 | for y in range(0, h, step_size): 53 | for x in range(0, w, step_size): 54 | # Extract the sub-image 55 | sub_image = image[y : y + window_size, x : x + window_size] 56 | klam = window_size - sub_image.shape[1] 57 | teta = window_size - sub_image.shape[0] 58 | sub_image = image[ 59 | y - teta : y - teta + window_size, x - klam : x - klam + window_size 60 | ] 61 | 62 | # Add the sub-image to the dictionary with the positions as the key 63 | sub_images[ 64 | (x - klam, y - teta, x - klam + window_size, y - teta + window_size) 65 | ] = sub_image 66 | return sub_images 67 | 68 | 69 | # Event config: 70 | def find_granule_dim(event_name: str, key_granule): 71 | requested_bands = ["B04", "B8A", "B11", "B12"] 72 | raw_event = Raw_event(device=device) 73 | raw_event.from_database(event_name, requested_bands) 74 | 75 | raw_granule_registered = raw_event.coarse_coregistration( 76 | [key_granule], 77 | use_complementary_granules=True, 78 | crop_empty_pixels=True, 79 | verbose=False, 80 | ) 81 | raw_granule_tensor = raw_granule_registered.as_tensor()[:, :, :] 82 | return [raw_granule_tensor.size(0), raw_granule_tensor.size(1)] 83 | 84 | 85 | def pos2Shap(pos, offY): 86 | x1, y1, x2, y2 = pos 87 | # Only for calculating the polygon and the intersection. 88 | ShapP1 = (x1, offY - y1) 89 | ShapP2 = (x1, offY - y2) 90 | ShapP3 = (x2, offY - y2) 91 | ShapP4 = (x2, offY - y1) 92 | ShapP5 = ShapP1 93 | 94 | Shapatch = Polygon([ShapP1, ShapP2, ShapP3, ShapP4, ShapP5]).convex_hull 95 | return Shapatch 96 | 97 | 98 | def main(): 99 | pass 100 | 101 | 102 | if __name__ == "__main__": 103 | main() 104 | -------------------------------------------------------------------------------- /scripts_and_studies/runscripts/create_tif.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pyraws.l0.l0_event import L0_event 3 | from pyraws.utils.database_utils import get_events_list 4 | 5 | import argparse 6 | import torch 7 | from termcolor import colored 8 | from tqdm import tqdm 9 | import matplotlib.pyplot as plt 10 | 11 | 12 | def main(): 13 | parser = argparse.ArgumentParser() 14 | 15 | parser.add_argument( 16 | "--bands", 17 | type=str, 18 | help='bands to coregister list in format ""[Bxx,Byy,...,Bzz]""', 19 | default="[B02,B08,B03,B10,B04,B05,B11,B06,B07,B8A,B12,B01,B09]", 20 | ) 21 | parser.add_argument( 22 | "--output_tif_dir", 23 | type=str, 24 | help="output TIF dir.", 25 | default="my_tif_dir_complementary", 26 | ) 27 | 28 | pargs = parser.parse_args() 29 | requested_bands_str = pargs.bands 30 | requested_bands_str = requested_bands_str.replace(" ", "")[1:-1] 31 | bands = [x for x in requested_bands_str.split(",")] 32 | output_tif_dir = pargs.output_tif_dir 33 | 34 | os.makedirs(output_tif_dir, exist_ok=True) 35 | 36 | if torch.cuda.is_available(): 37 | device = torch.device("cuda") 38 | else: 39 | device = torch.device("cpu") 40 | 41 | events_list = get_events_list("THRAWS") 42 | 43 | for event in tqdm(events_list, "Accessing event..."): 44 | print("Processing event: ", colored(event, "blue") + ".") 45 | try: 46 | l0_event = L0_event(device=device) 47 | l0_event.from_database(event, bands, verbose=False) 48 | except: # noqa: E722 49 | print("Skipping event: ", colored(event, "red") + ".") 50 | 51 | if l0_event.is_void(): 52 | print("Skipping event: ", colored(event, "red") + ".") 53 | continue 54 | 55 | l0_event_swir = L0_event() 56 | l0_event_swir.from_database(event, ["B8A", "B11", "B12"], verbose=False) 57 | l0_event_rgb = L0_event() 58 | l0_event_rgb.from_database(event, ["B02", "B03", "B04"], verbose=False) 59 | granules_list = list(range(len(l0_event.get_granules_info().keys()))) 60 | 61 | for granule in granules_list: 62 | l0_granule_n = l0_event.get_granule(granule) 63 | granule_info = l0_granule_n.get_granule_info() 64 | save_path_n = os.path.join( 65 | pargs.output_tif_dir, 66 | event 67 | + "_" 68 | + str(granule) 69 | + "_" 70 | + granule_info[0][:3] 71 | + "_" 72 | + granule_info[4], 73 | ) 74 | os.makedirs(save_path_n, exist_ok=True) 75 | print( 76 | "Exporting to tif file: " + colored(event + "_" + str(granule), "green") 77 | ) 78 | l0_granule_n.export_to_tif(save_path_n) 79 | l0_granule_rgb_n = l0_event_rgb.coarse_coregistration([granule]) 80 | l0_granule_swir_n = l0_event_swir.coarse_coregistration([granule]) 81 | l0_granule_rgb_n.show_bands_superimposition() 82 | plt.savefig( 83 | os.path.join(save_path_n, event + "_" + str(granule) + "_rgb.png") 84 | ) 85 | plt.close() 86 | l0_granule_swir_n.show_bands_superimposition() 87 | plt.savefig( 88 | os.path.join(save_path_n, event + "_" + str(granule) + "_swir.png") 89 | ) 90 | 91 | print("processing " + colored("finished", "green") + ".") 92 | 93 | 94 | if __name__ == "__main__": 95 | main() 96 | -------------------------------------------------------------------------------- /scripts_and_studies/runscripts/crop_L1_tiles.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | sys.path.insert(1, "..") 5 | from pyraws.raw.raw_event import Raw_event 6 | from pyraws.l1.l1_event import L1C_event 7 | from pyraws.utils.l1_utils import read_L1C_image_from_tif 8 | from pyraws.utils.database_utils import get_events_list 9 | import torch 10 | from termcolor import colored 11 | import matplotlib.pyplot as plt 12 | from tqdm import tqdm 13 | import argparse 14 | 15 | 16 | def main(): 17 | parser = argparse.ArgumentParser() 18 | parser.add_argument( 19 | "--bands", 20 | type=str, 21 | help='bands to coregister list in format ""[Bxx,Byy,...,Bzz]""', 22 | default="[B02,B08,B03,B10,B04,B05,B11,B06,B07,B8A,B12,B01,B09]", 23 | ) 24 | parser.add_argument( 25 | "--output_tif_dir", 26 | type=str, 27 | help="Path to the output directory containing cropped l1c tif files.", 28 | default=r"C:\Users\Gabriele Meoni\Documents\end2end\end2end\data\l1c\l1c_cropped_tif", 29 | ) 30 | parser.add_argument( 31 | "--output_quicklooks_dir", 32 | type=str, 33 | help="Path to the output directory containing quicklooks for raw and l1 images.", 34 | default=r"C:\Users\Gabriele Meoni\Documents\end2end\end2end\data\l1c\l1c_cropped_tif\qls", 35 | ) 36 | parser.add_argument("--database", type=str, help="Database name.", default="THRAWS") 37 | 38 | pargs = parser.parse_args() 39 | requested_bands_str = pargs.bands 40 | requested_bands_str = requested_bands_str.replace(" ", "")[1:-1] 41 | requested_bands = [x for x in requested_bands_str.split(",")] 42 | events_list = get_events_list(database=pargs.database) 43 | output_tif_dir = pargs.output_tif_dir 44 | output_quicklooks_di = pargs.output_quicklooks_dir 45 | 46 | if torch.cuda.is_available(): 47 | device = torch.device("cuda") 48 | else: 49 | device = torch.device("cpu") 50 | plt.rcParams["figure.figsize"] = [10, 10] 51 | for event in tqdm( 52 | events_list[1:], desc="Processing events... " 53 | ): # Skipping first event, being ":" 54 | print("Processing event: " + colored(event, "blue")) 55 | # -------------Create quicklook------------- 56 | plt.close() 57 | try: 58 | raw_event = Raw_event(device=device) 59 | raw_event.from_database( 60 | event, requested_bands, verbose=False, database=pargs.database 61 | ) 62 | l1c_event = L1C_event(device=device) 63 | l1c_event.from_database(event, requested_bands, database=pargs.database) 64 | except: # noqa: E722 65 | print("Skipping event: ", colored(event, "red")) 66 | continue 67 | 68 | if raw_event.is_void() or l1c_event.is_void(): 69 | print("Skipping event: ", colored(event, "red")) 70 | continue 71 | 72 | figure, ax = plt.subplots(1, 2) 73 | 74 | # Extracting useful_granules 75 | useful_granules = raw_event.get_useful_granules_idx() 76 | 77 | if ( 78 | not (len(useful_granules)) 79 | or (useful_granules is None) 80 | or any( 81 | [ 82 | True if (x[0] is None) or (x[1] is None) else False 83 | for x in useful_granules 84 | ] 85 | ) 86 | ): 87 | print("Skipping event: ", colored(event, "red")) 88 | continue 89 | 90 | for useful_granule in tqdm(useful_granules, desc="Parsing useful granules..."): 91 | if useful_granule is [None, None]: 92 | print("Skipping event: ", colored(event, "red")) 93 | break 94 | print("Processing granule: " + colored(useful_granule, "cyan")) 95 | raw_granule = raw_event.coarse_coregistration( 96 | sorted(useful_granule), crop_empty_pixels=False 97 | ) 98 | band_shifted_dict = raw_granule.get_bands_coordinates() 99 | raw_granule_coordinates = band_shifted_dict[requested_bands[0]] 100 | _ = l1c_event.crop_tile( 101 | raw_granule_coordinates, 102 | output_tif_dir, 103 | out_name_ending=str(useful_granule[0]) + "_" + str(useful_granule[1]), 104 | lat_lon_format=True, 105 | ) 106 | l1c_tif, _, _ = read_L1C_image_from_tif( 107 | event, 108 | out_name_ending=str(useful_granule[0]) + "_" + str(useful_granule[1]), 109 | device=device, 110 | ) 111 | plt.close() 112 | bands_superimposed = raw_granule.as_tensor() 113 | bands_superimposed = bands_superimposed / bands_superimposed.max() 114 | fig, ax = plt.subplots(1, 2) 115 | if device == torch.device("cuda"): 116 | ax[0].imshow(bands_superimposed.detach().cpu().numpy()) 117 | else: 118 | ax[0].imshow(bands_superimposed) 119 | 120 | ax[1].imshow(l1c_tif) 121 | plt.savefig( 122 | os.path.join( 123 | output_quicklooks_di, 124 | event 125 | + "_" 126 | + str(useful_granule[0]) 127 | + "_" 128 | + str(useful_granule[1]) 129 | + "_qls.png", 130 | ) 131 | ) 132 | 133 | 134 | if __name__ == "__main__": 135 | main() 136 | -------------------------------------------------------------------------------- /scripts_and_studies/runscripts/download_thraws.py: -------------------------------------------------------------------------------- 1 | import os 2 | import requests 3 | import zipfile 4 | from typing import List 5 | from tqdm import tqdm 6 | 7 | 8 | def download_files(links: List[str], output_dir: str) -> None: 9 | """Download files to a directory. 10 | 11 | Args: 12 | links: A list of URLs to download. 13 | output_dir: A directory to download the files to. 14 | """ 15 | # Create the output directory if it does not exist 16 | os.makedirs(output_dir, exist_ok=True) 17 | # Download each link 18 | for link in tqdm(links, desc="Downloading files", mininterval=5): 19 | # Create a filename from the link 20 | base_name = link.split("?")[0] 21 | filename = os.path.join(output_dir, os.path.basename(base_name)) 22 | # Download the file 23 | with requests.get(link, stream=True) as response: 24 | # Raise an exception if the status code is not 200 25 | response.raise_for_status() 26 | # Write the file to disk 27 | with open(filename, "wb") as file: 28 | for chunk in response.iter_content(chunk_size=8192): 29 | file.write(chunk) 30 | # If the file is a zip file, extract it 31 | if filename.endswith(".zip"): 32 | with zipfile.ZipFile(filename, "r") as zip_file: 33 | zip_file.extractall(output_dir) 34 | # Delete the zip file 35 | os.remove(filename) 36 | 37 | 38 | if __name__ == "__main__": 39 | links = [ 40 | f"https://zenodo.org/record/7908728/files/{x}.zip?download=1" 41 | for x in range(1, 2) 42 | ] + ["https://zenodo.org/record/7908728/files/L1C_files.json?download=1"] 43 | 44 | download_files(links, "./data") 45 | -------------------------------------------------------------------------------- /scripts_and_studies/runscripts/generate_coregistration_plot.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | sys.path.insert(1, os.path.join("..", "..")) 5 | from pyraws.raw.raw_event import Raw_event 6 | import matplotlib.pyplot as plt 7 | import argparse 8 | 9 | 10 | def main(): 11 | parser = argparse.ArgumentParser() 12 | parser.add_argument( 13 | "--bands", 14 | type=str, 15 | help='bands in format ""[Bxx,Byy,...,Bzz]"" ', 16 | default="[B04, B8A, B11]", 17 | ) 18 | parser.add_argument( 19 | "--granule_index", type=int, help="Event name to start.", default=2 20 | ) 21 | parser.add_argument( 22 | "--event_name", 23 | type=str, 24 | help="If specified, only the events of that type will be used.", 25 | default="Etna_00", 26 | ) 27 | 28 | # Bands to open. 29 | pargs = parser.parse_args() 30 | requested_bands_str = pargs.bands 31 | requested_bands_str = requested_bands_str.replace(" ", "")[1:-1] 32 | bands_list = [x for x in requested_bands_str.split(",")] 33 | 34 | # Instantiate an empty Raw_event 35 | raw_event = Raw_event() 36 | 37 | # Read "Etna_00" from THRAWS 38 | raw_event.from_database( # Database ID_EVENT 39 | id_event=pargs.event_name, 40 | # Bands to open. Leave to None to use all the bands. 41 | bands_list=bands_list, 42 | # If True, verbose mode is on. 43 | verbose=False, 44 | # Database name 45 | database="THRAWS", 46 | ) 47 | 48 | raw_coregistered_granule_1 = raw_event.coarse_coregistration( # granule index to coregister. 49 | granules_idx=[pargs.granule_index] 50 | ) 51 | 52 | # Perform the corase coregistration of the "Etna_00" event. 53 | # Missing pixels will be cropped. 54 | raw_coregistered_granule_1_with_crop = raw_event.coarse_coregistration( # granule index to coregister. 55 | granules_idx=[pargs.granule_index], 56 | # Cropping missing pixels. 57 | crop_empty_pixels=True, 58 | ) 59 | 60 | # Perform the corase coregistration of the "Etna_00" event. 61 | # Missing pixels will be cropped. 62 | raw_coregistered_granule_1_with_fill = raw_event.coarse_coregistration( # granule index to coregister. 63 | granules_idx=[pargs.granule_index], 64 | # Search for filling elements among adjacent L0 granules 65 | use_complementary_granules=True, 66 | # Cropping missing pixels when compatible L0 granules are not available 67 | crop_empty_pixels=True, 68 | ) 69 | 70 | raw_granule_1 = raw_event.get_granule(pargs.granule_index) 71 | raw_granule_1_t = raw_granule_1.as_tensor() 72 | raw_coregistered_granule_1_t = raw_coregistered_granule_1.as_tensor() 73 | raw_coregistered_granule_1_with_crop_t = ( 74 | raw_coregistered_granule_1_with_crop.as_tensor() 75 | ) 76 | raw_coregistered_granule_1_with_fill_t = ( 77 | raw_coregistered_granule_1_with_fill.as_tensor() 78 | ) 79 | 80 | raw_granule_1_t = raw_granule_1_t / raw_granule_1_t.max() 81 | raw_coregistered_granule_1_t = ( 82 | raw_coregistered_granule_1_t / raw_coregistered_granule_1_t.max() 83 | ) 84 | raw_coregistered_granule_1_with_crop_t = ( 85 | raw_coregistered_granule_1_with_crop_t 86 | / raw_coregistered_granule_1_with_crop_t.max() 87 | ) 88 | raw_coregistered_granule_1_with_fill_t = ( 89 | raw_coregistered_granule_1_with_fill_t 90 | / raw_coregistered_granule_1_with_fill_t.max() 91 | ) 92 | 93 | plt.rcParams["figure.figsize"] = [10, 10] 94 | fig, ax = plt.subplots(2, 2) 95 | 96 | ax[0, 0].imshow(raw_granule_1_t) 97 | ax[0, 0].set_title("Original") 98 | ax[0, 1].imshow(raw_coregistered_granule_1_t) 99 | ax[0, 1].set_title("Coregistration") 100 | ax[1, 0].imshow(raw_coregistered_granule_1_with_crop_t) 101 | ax[1, 0].set_title("Coregistration with crop") 102 | ax[1, 1].imshow(raw_coregistered_granule_1_with_fill_t) 103 | ax[1, 1].set_title("Coregistration with fill") 104 | 105 | plt.setp(plt.gcf().get_axes(), xticks=[], yticks=[]) 106 | plt.show() 107 | 108 | 109 | if __name__ == "__main__": 110 | main() 111 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """A setuptools based setup module. 2 | See: 3 | https://packaging.python.org/guides/distributing-packages-using-setuptools/ 4 | https://github.com/pypa/sampleproject 5 | """ 6 | 7 | # Always prefer setuptools over distutils 8 | from setuptools import setup 9 | 10 | setup( 11 | name="pyraws", 12 | version="1.0.0", 13 | description="Python for RAW Sentinel2 data (PyRawS) is a powerful open-source Python package" 14 | + " that provides a comprehensive set of tools for working with Sentinel-2 Raw data. It provides utilities for" 15 | + " coarse spatial bands coregistration, geo-referencing, data visualization, and image processing.", 16 | long_description=open("README.md", encoding="cp437").read(), 17 | long_description_content_type="text/markdown", 18 | url="https://github.com/ESA-PhiLab/PyRawS", 19 | author="ESA Philab", 20 | author_email="gabriele.meoni@esa.int", 21 | install_requires=[ 22 | "geopy", 23 | "geopandas", 24 | "glymur", 25 | "numpy", 26 | "opencv-python", 27 | "pandas", 28 | "scipy", 29 | "matplotlib", 30 | "termcolor", 31 | "torch", 32 | "tqdm", 33 | "torchvision", 34 | "scikit-image", 35 | "scikit-learn", 36 | "rasterio", 37 | "tifffile", 38 | ], 39 | classifiers=[ 40 | "Development Status :: 3 - Alpha", 41 | "Intended Audience :: Developers", 42 | "Intended Audience :: Science/Research", 43 | "Topic :: Scientific/Engineering :: Computer Vision", 44 | "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", 45 | "Programming Language :: Python :: 3.8", 46 | ], 47 | packages=["pyraws", "pyraws.database", "pyraws.raw", "pyraws.l1", "pyraws.utils"], 48 | python_requires=">=3.8, <4", 49 | project_urls={"Source": "https://github.com/ESA-PhiLab/PyRawS"}, 50 | ) 51 | --------------------------------------------------------------------------------