├── .gitignore ├── .readthedocs.yaml ├── Dockerfile ├── LICENSE ├── README.md ├── conda ├── conda_grs.sh └── grs_conda_3.6.yml ├── docs ├── Makefile ├── build │ └── .jupyter_cache │ │ ├── __version__.txt │ │ ├── executed │ │ ├── 1bc4a56eaa0307e71beb5cadd6ca8760 │ │ │ └── base.ipynb │ │ ├── 345bd43dc797dabc6234c8a7e0bc9979 │ │ │ └── base.ipynb │ │ ├── 64bc28fb066db4daaf9996b2b7e3938c │ │ │ └── base.ipynb │ │ └── e552464529466d09195451aac980ec9f │ │ │ └── base.ipynb │ │ └── global.db ├── environment.yml ├── make.bat ├── requirements.txt └── source │ ├── GRS API │ ├── grs.acutils.Aerosol.rst │ ├── grs.acutils.CamsParams.rst │ ├── grs.acutils.GaseousTransmittance.rst │ ├── grs.acutils.Gases.rst │ ├── grs.acutils.Misc.rst │ ├── grs.acutils.Rasterization.rst │ ├── grs.acutils.rst │ ├── grs.auxdata.AuxData.rst │ ├── grs.auxdata.SensorData.rst │ ├── grs.auxdata.rst │ ├── grs.cams.CamsProduct.rst │ ├── grs.cams.rst │ ├── grs.data.aux.rst │ ├── grs.data.lut.gases.rst │ ├── grs.data.lut.rst │ ├── grs.data.rst │ ├── grs.grs_process.Process.rst │ ├── grs.grs_process.rst │ ├── grs.mask.Masking.rst │ ├── grs.mask.Settings.rst │ ├── grs.mask.rst │ ├── grs.output.L2aProduct.rst │ ├── grs.output.rst │ ├── grs.product.Product.rst │ ├── grs.product.rst │ ├── grs.run.main.rst │ └── grs.run.rst │ ├── _static │ ├── GRS_TBX.png │ ├── GRS_TBX.svg │ ├── GRS_driver.png │ ├── GRS_driver.svg │ ├── GRS_l2bgen.png │ ├── GRS_l2bgen.svg │ ├── INRAE_logo.png │ ├── OFB_logo.png │ ├── cnes_logo.png │ ├── custom.css │ ├── example_image_level.png │ ├── grs_logo_light.png │ ├── grs_logo_v0.png │ ├── grs_logo_v0.svg │ ├── grstbx_visual_tool.gif │ ├── magellium_logo.png │ └── magellium_logo_eo.png │ ├── _templates │ ├── custom-class-template.rst │ └── custom-module-template.rst │ ├── api.rst │ ├── conf.py │ ├── examples │ ├── advanced.rst │ ├── auxdata.rst │ ├── basics.rst │ ├── download_cams.ipynb │ ├── grs_v2_landsat_dev.ipynb │ ├── grs_v2_simple.ipynb │ ├── grs_visu_simple.ipynb │ ├── roi_grs_visu_simple.json │ └── visualization.rst │ ├── history.rst │ ├── index.rst │ ├── support.rst │ └── usage.rst ├── ecmwf ├── download_cams_all.py ├── download_cams_cds.py ├── download_cams_cds_day.py ├── pbs_cds.pbs └── pbs_ecmwf.pbs ├── environment.yml ├── exe ├── .~lock.list_grs_cnes_obs2mod.csv# ├── __init__.py ├── acix_grs_proc.py ├── aeronet-oc_sites.txt ├── amazone_sites.txt ├── call_amalthee.py ├── call_subprocess.py ├── call_subprocess_multiproc.py ├── download_list.py ├── global_config.yml ├── grs_call.py ├── grs_from_list.py ├── grs_proc_sites.py ├── grs_tiles_cnes.py ├── grs_tiles_cnes_v2.py ├── grs_validation.py ├── hymotep_config.yml ├── launcher.py ├── list │ ├── List_images_grs_template.csv │ ├── construct_list.py │ ├── gernez_list_grs_dec_2021.csv │ ├── list_grs_cnes_gernez.csv │ ├── list_grs_cnes_gernez_tmp.csv │ ├── list_grs_cnes_guimaraes.csv │ ├── list_grs_cnes_guimaraes2016-2017.csv │ ├── list_grs_cnes_hafeez.csv │ ├── list_grs_cnes_mauricio.csv │ ├── list_grs_cnes_obs2mod.csv │ ├── list_grs_cnes_seine.csv │ ├── list_grs_cnes_srikanta.csv │ ├── list_grs_cnes_template.csv │ ├── list_grs_cordoba.csv │ ├── list_grs_gernez_feb2022.csv │ ├── list_grs_gernez_juillet_2021.csv │ ├── list_grs_gernez_sep2021.csv │ ├── list_grs_jegou.csv │ ├── list_grs_jegou_may2022.csv │ ├── list_grs_jirau.csv │ ├── list_grs_plata_estuary.csv │ ├── list_grs_redshade_1.csv │ ├── list_grs_redshade_2.csv │ ├── list_grs_redshade_2022.csv │ ├── list_grs_redshade_3.csv │ ├── list_grs_redshade_4.csv │ ├── list_grs_santarem.csv │ ├── list_grs_ukraine.csv │ ├── list_grs_unesco_chad_all.csv │ ├── list_grs_unesco_plata_2015.csv │ ├── list_grs_unesco_plata_2018.csv │ ├── list_grs_unesco_plata_2019.csv │ ├── list_grs_unesco_plata_2020.csv │ ├── list_grs_unesco_plata_2021.csv │ ├── list_grs_validation_get.csv │ ├── list_jegou_jallais.csv │ ├── list_jegou_jallais2.csv │ └── list_landsat_jegou.csv ├── list_validation │ ├── info_aeronet_oc_v15.csv │ ├── list_grs_aeronet_oc.csv │ └── list_grs_aeronet_oc_test.csv ├── procutils.py ├── sync_rclone_dropbox.sh ├── test_mp.py └── upload_L2_dropbox.sh ├── grs ├── __init__.py ├── acutils.py ├── auxdata.py ├── cams.py ├── config.yml ├── config_template.yml ├── config_template_cnes.yml ├── data │ ├── __init__.py │ ├── aux │ │ ├── __init__.py │ │ ├── mean_rglint_small_angles_vza_le_12_sza_le_60.txt │ │ └── rayleigh_bodhaine.txt │ └── lut │ │ └── gases │ │ ├── __init__.py │ │ ├── lut_abs_opt_thickness_normalized.nc │ │ └── water_vapor_transmittance.nc ├── grs_process.py ├── landsat_angles │ ├── OLI │ │ ├── Makefile │ │ ├── ias_lib │ │ │ ├── Makefile │ │ │ ├── ias_angle_gen_calculate_angles_rpc.o │ │ │ ├── ias_angle_gen_find_scas.o │ │ │ ├── ias_angle_gen_initialize.o │ │ │ ├── ias_angle_gen_read_ang.o │ │ │ ├── ias_angle_gen_utilities.o │ │ │ ├── ias_angle_gen_write_image.o │ │ │ ├── ias_geo_convert_dms2deg.o │ │ │ ├── ias_logging.o │ │ │ ├── ias_math_compute_unit_vector.o │ │ │ ├── ias_math_compute_vector_length.o │ │ │ ├── ias_math_find_line_segment_intersection.o │ │ │ ├── ias_misc_convert_to_uppercase.o │ │ │ ├── ias_misc_create_output_image_trim_lut.o │ │ │ ├── ias_misc_write_envi_header.o │ │ │ ├── ias_odl_free_tree.o │ │ │ ├── ias_odl_get_field.o │ │ │ ├── ias_odl_read_tree.o │ │ │ ├── ias_parm_check_ranges.o │ │ │ ├── ias_parm_map_odl_type.o │ │ │ ├── ias_parm_provide_help.o │ │ │ ├── ias_parm_read.o │ │ │ ├── ias_satellite_attributes.o │ │ │ ├── lablib3.o │ │ │ ├── landsat8.o │ │ │ └── libl8ang.a │ │ └── l8_angles │ └── TM │ │ ├── IAS-Log │ │ ├── Makefile │ │ ├── ias_lib │ │ ├── Makefile │ │ ├── gxx_angle_gen_calculate_angles_rpc.o │ │ ├── gxx_angle_gen_calculate_vector.o │ │ ├── gxx_angle_gen_find_dir.o │ │ ├── gxx_angle_gen_geo_utilities.o │ │ ├── gxx_angle_gen_initialize.o │ │ ├── gxx_angle_gen_initialize_transformation.o │ │ ├── gxx_angle_gen_interpolate_ephemeris.o │ │ ├── gxx_angle_gen_read_ang.o │ │ ├── gxx_angle_gen_write_image.o │ │ ├── gxx_dmsdeg.o │ │ ├── gxx_dot.o │ │ ├── gxx_geod2cart.o │ │ ├── gxx_get_units.o │ │ ├── gxx_unit.o │ │ ├── lablib3.o │ │ ├── liblandsatang.a │ │ ├── xxx_Band.o │ │ ├── xxx_CloseODL.o │ │ ├── xxx_CloseUnmap.o │ │ ├── xxx_ConvertString.o │ │ ├── xxx_Errno.o │ │ ├── xxx_FileLock.o │ │ ├── xxx_GetDirFiles.o │ │ ├── xxx_GetODLField.o │ │ ├── xxx_GetTempName.o │ │ ├── xxx_GetTime.o │ │ ├── xxx_LogError.o │ │ ├── xxx_LogStatus.o │ │ ├── xxx_OpenMap.o │ │ ├── xxx_OpenODL.o │ │ ├── xxx_RecursiveDeletion.o │ │ ├── xxx_Sensor.o │ │ └── xxx_strtoupper.o │ │ └── landsat_angles ├── mask.py ├── output.py ├── product.py └── run.py ├── grs_launcher.pbs ├── grs_mpi_launcher.pbs ├── grs_validation_launcher.pbs ├── grsdata └── gases │ ├── lut_abs_opt_thickness_normalized.nc │ └── water_vapor_transmittance.nc ├── images ├── Fig_valid_qualit_sea_scale.png ├── S2B_MSIL1C_20220701T103629_N0400_R008_T31TFJ_20220701T112552_RGB.png ├── S2B_MSIL2Agrs_20220701T103629_N0400_R008_T31TFJ_20220701T112552_BRDFg.png ├── S2B_MSIL2Agrs_20220701T103629_N0400_R008_T31TFJ_20220701T112552_RGB.png ├── S2B_MSIL2Agrs_20220701T103629_N0400_R008_T31TFJ_20220701T112552_RGB_sunglint.png ├── example_snap_grs_image.png └── flowchart_sunglint_S2.png ├── issues ├── cams │ ├── test_cams.py │ └── test_cams_cds.py ├── dem │ ├── __init__.py │ ├── check_dem_availability.py │ ├── dem_slope_aspect.py │ ├── download_srtm.sh │ ├── readme.txt │ └── zipping_srtm.sh ├── flags │ └── test_flags.py ├── format │ ├── formats_and_extensions.py │ ├── test │ ├── test.bin │ ├── test.csv │ ├── test.data │ │ └── vector_data │ │ │ ├── ground_control_points.csv │ │ │ └── pins.csv │ ├── test.dim │ ├── test.h5 │ ├── test.nc │ ├── test.tif │ ├── test.xml │ └── test_snappy_writer.py ├── grs_test │ ├── grs_test.py │ └── grs_test_ v2.py ├── lut_interpol │ ├── lut_interpol_test.py │ ├── lut_read_f.f90 │ ├── read_lut.cpython-36m-x86_64-linux-gnu.so │ └── read_lut.cpython-38-x86_64-linux-gnu.so ├── multiproc │ └── multiproc_snappy.py ├── resampling │ ├── test2_snappy_dispose.py │ ├── test2_snappy_resampling.py │ ├── test_snappy_landsat_op.py │ └── test_snappy_resampling.py ├── s2sampler │ ├── README.md │ ├── test2_snappy_resampling.py │ └── test_pythonS2resample.py └── timelag │ ├── S2_timelag_MSI.csv │ ├── S2_timelag_MSI_rel.csv │ ├── mega_glinted_pixels.csv │ └── plot_toa_refl_timelag.py ├── launch_grs_exemple.pbs ├── notebook ├── grs_v2_demo.ipynb ├── grs_v2_l2bgen.ipynb ├── grs_v2_opac_formation_v20240222.ipynb ├── grs_v2_process.ipynb ├── grs_v2_prototyping-cnes.ipynb └── grs_v2_prototyping.ipynb ├── pbs_ecmwf.pbs ├── pypi_README.rst ├── pyproject.toml ├── requirements.txt ├── run_docker.sh └── test ├── grs_test_ v2.py └── grs_test_ v2_cnes.py /.gitignore: -------------------------------------------------------------------------------- 1 | */grs/config.py 2 | 3 | # Byte-compiled / optimized / DLL files 4 | */*.o 5 | *.o 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | 11 | # Distribution / packaging 12 | .Python 13 | #build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | db.sqlite3 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # Jupyter Notebook 75 | .ipynb_checkpoints 76 | 77 | # IPython 78 | profile_default/ 79 | ipython_config.py 80 | 81 | # pyenv 82 | .python-version 83 | 84 | # celery beat schedule file 85 | celerybeat-schedule 86 | 87 | # SageMath parsed files 88 | *.sage.py 89 | 90 | # Environments 91 | .env 92 | .venv 93 | env/ 94 | venv/ 95 | ENV/ 96 | env.bak/ 97 | venv.bak/ 98 | 99 | # Spyder project settings 100 | .spyderproject 101 | .spyproject 102 | 103 | # Rope project settings 104 | .ropeproject 105 | 106 | # mkdocs documentation 107 | /site 108 | 109 | # mypy 110 | .mypy_cache/ 111 | .dmypy.json 112 | dmypy.json 113 | 114 | # Pyre type checker 115 | .pyre/ 116 | 117 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 118 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 119 | 120 | # User-specific stuff 121 | .idea 122 | .idea/**/workspace.xml 123 | .idea/**/tasks.xml 124 | .idea/**/usage.statistics.xml 125 | .idea/**/dictionaries 126 | .idea/**/shelf 127 | 128 | # Generated files 129 | .idea/**/contentModel.xml 130 | 131 | # Sensitive or high-churn files 132 | .idea/**/dataSources/ 133 | .idea/**/dataSources.ids 134 | .idea/**/dataSources.local.xml 135 | .idea/**/sqlDataSources.xml 136 | .idea/**/dynamic.xml 137 | .idea/**/uiDesigner.xml 138 | .idea/**/dbnavigator.xml 139 | 140 | # Gradle 141 | .idea/**/gradle.xml 142 | .idea/**/libraries 143 | 144 | # Gradle and Maven with auto-import 145 | # When using Gradle or Maven with auto-import, you should exclude module files, 146 | # since they will be recreated, and may cause churn. Uncomment if using 147 | # auto-import. 148 | # .idea/modules.xml 149 | # .idea/*.iml 150 | # .idea/modules 151 | 152 | # CMake 153 | cmake-build-*/ 154 | 155 | # Mongo Explorer plugin 156 | .idea/**/mongoSettings.xml 157 | 158 | # File-based project format 159 | *.iws 160 | 161 | # IntelliJ 162 | out/ 163 | 164 | # mpeltonen/sbt-idea plugin 165 | .idea_modules/ 166 | 167 | # JIRA plugin 168 | atlassian-ide-plugin.xml 169 | 170 | # Cursive Clojure plugin 171 | .idea/replstate.xml 172 | 173 | # Crashlytics plugin (for Android Studio and IntelliJ) 174 | com_crashlytics_export_strings.xml 175 | crashlytics.properties 176 | crashlytics-build.properties 177 | fabric.properties 178 | 179 | # Editor-based Rest Client 180 | .idea/httpRequests 181 | 182 | # Android studio 3.1+ serialized cache file 183 | .idea/caches/build_file_checksums.ser 184 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | build: 4 | os: ubuntu-22.04 5 | tools: 6 | python: "mambaforge-22.9" 7 | 8 | sphinx: 9 | configuration: docs/source/conf.py 10 | fail_on_warning: false 11 | 12 | formats: all 13 | 14 | conda: 15 | environment: docs/environment.yml 16 | 17 | python: 18 | install: 19 | - requirements: requirements.txt 20 | - method: pip 21 | path: . 22 | extra_requirements: 23 | - docs -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG IMAGE_SOURCE 2 | FROM ${IMAGE_SOURCE}ubuntu:22.04 AS systemdependencies 3 | # FROM ubuntu:kinetic AS systemdependencies 4 | LABEL maintainer: "robin.buratti@magellium.fr" 5 | 6 | ENV LANG C.UTF-8 7 | ENV LC_ C.UTF-8 8 | 9 | RUN ulimit -s unlimited 10 | 11 | # Proxy from secret volumes 12 | RUN if [ -f "/kaniko/run/secrets/http_proxy" ]; then export http_proxy=$(cat /kaniko/run/secrets/http_proxy); export https_proxy=$(cat /kaniko/run/secrets/https_proxy); fi && \ 13 | apt-get update -y && \ 14 | apt-get install -y ca-certificates 15 | 16 | # Ajout des certificats 17 | COPY cert[s]/* /usr/local/share/ca-certificates/ 18 | RUN update-ca-certificates 19 | 20 | # Install libraries 21 | RUN if [ -f "/kaniko/run/secrets/http_proxy" ]; then export http_proxy=$(cat /kaniko/run/secrets/http_proxy); export https_proxy=$(cat /kaniko/run/secrets/https_proxy); fi \ 22 | && apt-get -qq update \ 23 | && DEBIAN_FRONTEND=noninteractive apt-get -qq install -y --no-install-recommends \ 24 | software-properties-common \ 25 | gcc \ 26 | python3.10 \ 27 | python3-dev \ 28 | build-essential \ 29 | gdal-bin \ 30 | libgdal-dev \ 31 | && rm -rf /var/lib/apt/lists/* 32 | 33 | ENV CPLUS_INCLUDE_PATH=/usr/include/gdal 34 | ENV C_INCLUDE_PATH=/usr/include/gdal 35 | 36 | # GRS INSTALL 37 | WORKDIR /home/ 38 | COPY ecmwf ./grs2/ecmwf 39 | COPY exe ./grs2/exe 40 | COPY grs ./grs2/grs 41 | COPY grsdata ./grs2/grsdata 42 | COPY setup.py ./grs2/ 43 | WORKDIR /home/grs2 44 | 45 | 46 | 47 | ######################### 48 | FROM ${IMAGE_SOURCE}ubuntu:22.04 49 | LABEL maintainer: "robin.buratti@magellium.fr" 50 | 51 | ENV LANG C.UTF-8 52 | ENV LC_ C.UTF-8 53 | 54 | RUN ulimit -s unlimited 55 | 56 | RUN if [ -f "/kaniko/run/secrets/http_proxy" ]; then export http_proxy=$(cat /kaniko/run/secrets/http_proxy); export https_proxy=$(cat /kaniko/run/secrets/https_proxy); fi \ 57 | && apt-get -qq update \ 58 | && DEBIAN_FRONTEND=noninteractive apt-get -qq install -y --no-install-recommends \ 59 | python-is-python3 \ 60 | python3.10 \ 61 | python3-dev \ 62 | python3-pip \ 63 | python3-affine \ 64 | python3-gdal \ 65 | python3-lxml \ 66 | python3-xmltodict \ 67 | gdal-bin \ 68 | && rm -rf /var/lib/apt/lists/* 69 | 70 | # get GRS from systemdependencies 71 | COPY --from=systemdependencies /home/grs2 /home/grs2 72 | 73 | WORKDIR /home/grs2 74 | 75 | # Add additionnal dependencies + GRS 76 | RUN if [ -f "/kaniko/run/secrets/http_proxy" ]; then export http_proxy=$(cat /kaniko/run/secrets/http_proxy); export https_proxy=$(cat /kaniko/run/secrets/https_proxy); fi \ 77 | && pip3 install \ 78 | --trusted-host pypi.org --trusted-host pypi.python.org --trusted-host files.pythonhosted.org \ 79 | --no-cache-dir \ 80 | # requirements 81 | cdsapi \ 82 | dask \ 83 | dask[array] \ 84 | docopt \ 85 | geopandas \ 86 | matplotlib \ 87 | netCDF4 \ 88 | numpy \ 89 | pandas \ 90 | pyproj \ 91 | python-dateutil \ 92 | scipy \ 93 | "xarray==2023.8.0" \ 94 | # other dependencies 95 | "eoreader==0.20.3" \ 96 | numba \ 97 | && pip3 install --trusted-host pypi.org --trusted-host pypi.python.org --trusted-host files.pythonhosted.org . 98 | 99 | RUN mkdir -p /datalake/watcal/GRS \ 100 | && cp -r grsdata /datalake/watcal/GRS/ 101 | 102 | WORKDIR /home/ 103 | #ENTRYPOINT ["tail", "-f", "/dev/null"] 104 | -------------------------------------------------------------------------------- /conda/conda_grs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Après installation de snap, faire un lien symbolique de esasnappy vers snappy (qui se trouve dans le .snap du home 4 | 5 | #desactivate conda s'il est activé 6 | conda deactivate 2>/dev/null 7 | 8 | #On récupère la connexion au proxy 9 | #[ "$_" = "$0" ] && echo "Ce script doit être sourcé!" && exit 127 10 | 11 | #read -s -p "Please enter your proxy password ? " _passwd || exit 127 12 | #echo 13 | 14 | #_user=${_user:-$USER} 15 | 16 | #if [ -z ${_passwd} ] 17 | #then 18 | # echo "No password" 19 | #else 20 | # 21 | # export http_proxy="http://${_user}:${_passwd}@proxy-surf.loc.cnes.fr:8050" 22 | # export https_proxy="http://${_user}:${_passwd}@proxy-surf.loc.cnes.fr:8050" 23 | 24 | #echo 'http_proxy variable has to be set : '$http_proxy 25 | 26 | # export ftp_proxy="${http_proxy}" 27 | # export no_proxy=cnes.fr,sis.cnes.fr,gitlab.cnes.fr 28 | 29 | # unset _passwd 30 | # unset _user 31 | #fi 32 | 33 | #load avail module for conda, snap and jdk 34 | module load snap/8.0 conda jdk/1.8.0_112 35 | 36 | #switch to gcc/4.8.5 37 | #module unload gcc 38 | 39 | #set the variable environment for je jpy and snap use 40 | export JDK_HOME=$JDKHOME 41 | export JAVAHOME=$JAVA_HOME 42 | export SNAP_HOME=$SNAPHOME 43 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:. 44 | 45 | 46 | #activate conda environement python 3.8 47 | conda activate /work/ALT/swot/aval/OBS2CO/conda_env/grs_py3.8 48 | echo $USER 49 | 50 | #configuration for snap 51 | # go to the $SNAPHOME environment and configure snappy 52 | pythonpath=$(which python) 53 | echo $pythonpath 54 | 55 | pippath=$(which pip) 56 | echo $pippath 57 | 58 | #Install snappy 59 | #$SNAPHOME/bin/snappy-conf $pythonpath 60 | 61 | if [ ! -d $CONDA_PREFIX/lib/python3.6/site-packages/esasnappy ] 62 | then 63 | ln -s /work/ALT/swot/aval/OBS2CO/snap/snappy_3.8 $CONDA_PREFIX/lib/python3.8/site-packages/esasnappy 64 | fi 65 | 66 | 67 | echo $'snap.versionCheck.interval=NEVER\nsnap.jai.tileCacheSize=1024' > $HOME/.snap/etc/snap.properties 68 | 69 | #Install the jpy package used by snappy 70 | #pip install /work/ALT/swot/aval/OBS2CO/snap/snappy/lib/jpy-0.9.0-cp36-cp36m-linux_x86_64.whl nco 71 | 72 | #Modifier la config jpy pour java 73 | #cat $CONDA_PREFIX/lib/python3.6/site-packages/jpyconfig.py | grep '#java_home' 74 | 75 | #if [ $? == 1 ] 76 | #then 77 | # sed -i 's/java_home/#java_home/g' $CONDA_PREFIX/lib/python3.6/site-packages/jpyconfig.py 78 | # sed -i 's/jvm_dll/#jvm_dll/g' $CONDA_PREFIX/lib/python3.6/site-packages/jpyconfig.py 79 | #fi 80 | 81 | #if it is not already installed 82 | #conda install GDAL 83 | #compilation 84 | 85 | cd /work/ALT/swot/aval/OBS2CO/git/grs2 86 | 87 | #catch egm96 data 88 | #mkdir -p $HOME/.snap/auxdata/dem/egm96 89 | #cp /work/ALT/swot/aval/OBS2CO/git/grsdata/dem/ww15mgh_b.zip $HOME/.snap/auxdata/dem/egm96/ww15mgh_b.zip 90 | 91 | 92 | if [ $# != 0 ] 93 | then 94 | if [ $1 = "-c" ] 95 | then 96 | echo "Compilation" 97 | make 98 | else 99 | if [ $1 = "-i" ] 100 | then 101 | echo "install" 102 | python setup.py install 103 | else 104 | if [ $1 = "-ci" ] 105 | then 106 | echo 'compilation and installation' 107 | make 108 | python setup.py install 109 | fi 110 | fi 111 | fi 112 | fi 113 | 114 | 115 | echo "Tester GRS sur une tuile sentinel 2 : grs /work/ALT/swot/aval/OBS2CO/git/grsdata/INPUT_DATA/S2B_MSIL1C_20180927T103019_N0206_R108_T31TGK_20180927T143835.SAFE --shape test/data/shape/SPO04.shp --odir test/results/ --aerosol cams_forecast --dem --resolution 20 \nPreciser le répertoire de sortie dans --odir" 116 | -------------------------------------------------------------------------------- /conda/grs_conda_3.6.yml: -------------------------------------------------------------------------------- 1 | name: grs_py3.6 2 | channels: 3 | - anaconda 4 | - conda-forge 5 | - orfeotoolbox 6 | - defaults 7 | dependencies: 8 | - ca-certificates=2020.10.14=0 9 | - certifi=2020.6.20=py36_0 10 | - openssl=1.1.1h=h7b6447c_0 11 | - pip=20.2.4=py36_0 12 | - _libgcc_mutex=0.1=main 13 | - blas=1.0=mkl 14 | - bzip2=1.0.8=h7b6447c_0 15 | - cairo=1.14.12=h8948797_3 16 | - cfitsio=3.470=hb7c8383_2 17 | - curl=7.67.0=hbc83047_0 18 | - expat=2.2.10=he6710b0_2 19 | - fontconfig=2.13.0=h9420a91_0 20 | - freetype=2.10.4=h5ab3b9f_0 21 | - freexl=1.0.5=h14c3975_0 22 | - gdal=3.0.2=py36hbb2a789_0 23 | - geos=3.8.0=he6710b0_0 24 | - geotiff=1.5.1=h21e8280_1 25 | - giflib=5.1.4=h14c3975_1 26 | - glib=2.63.1=h5a9c865_0 27 | - hdf4=4.2.13=h3ca952b_2 28 | - hdf5=1.10.4=hb1b8bf9_0 29 | - icu=58.2=he6710b0_3 30 | - intel-openmp=2020.2=254 31 | - jpeg=9b=h024ee3a_2 32 | - json-c=0.13.1=h1bed415_0 33 | - kealib=1.4.7=hd0c454d_6 34 | - krb5=1.16.4=h173b8e3_0 35 | - ld_impl_linux-64=2.33.1=h53a641e_7 36 | - libboost=1.67.0=h46d08c1_4 37 | - libcurl=7.67.0=h20c2e04_0 38 | - libdap4=3.19.1=h6ec2957_0 39 | - libedit=3.1.20191231=h14c3975_1 40 | - libffi=3.2.1=hf484d3e_1007 41 | - libgcc-ng=9.1.0=hdf63c60_0 42 | - libgdal=3.0.2=h27ab9cc_0 43 | - libgfortran-ng=7.3.0=hdf63c60_0 44 | - libkml=1.3.0=h590aaf7_4 45 | - libnetcdf=4.6.1=h11d0813_2 46 | - libpng=1.6.37=hbc83047_0 47 | - libpq=11.2=h20c2e04_0 48 | - libspatialite=4.3.0a=h793db0d_0 49 | - libssh2=1.9.0=h1ba5d50_1 50 | - libstdcxx-ng=9.1.0=hdf63c60_0 51 | - libtiff=4.1.0=h2733197_0 52 | - libuuid=1.0.3=h1bed415_2 53 | - libxcb=1.14=h7b6447c_0 54 | - libxml2=2.9.10=hb55368b_3 55 | - lz4-c=1.8.1.2=h14c3975_0 56 | - mkl=2020.2=256 57 | - mkl-service=2.3.0=py36he904b0f_0 58 | - mkl_fft=1.2.0=py36h23d657b_0 59 | - mkl_random=1.1.1=py36h0573a6f_0 60 | - ncurses=6.2=he6710b0_1 61 | - numpy=1.19.1=py36hbc911f0_0 62 | - numpy-base=1.19.1=py36hfa32c7d_0 63 | - openjpeg=2.3.0=h05c96fa_1 64 | - pcre=8.44=he6710b0_0 65 | - pixman=0.40.0=h7b6447c_0 66 | - poppler=0.65.0=h581218d_1 67 | - poppler-data=0.4.9=0 68 | - postgresql=11.2=h20c2e04_0 69 | - proj=6.2.1=haa6030c_0 70 | - python=3.6.10=h191fe78_1 71 | - readline=7.0=h7b6447c_5 72 | - setuptools=50.3.0=py36hb0f4dca_1 73 | - six=1.15.0=py_0 74 | - sqlite=3.33.0=h62c20be_0 75 | - tbb=2018.0.5=h6bb024c_0 76 | - tiledb=1.6.3=h1fb8f14_0 77 | - tk=8.6.10=hbc83047_0 78 | - wheel=0.35.1=py_0 79 | - xerces-c=3.2.3=h780794e_0 80 | - xz=5.2.5=h7b6447c_0 81 | - zlib=1.2.11=h7b6447c_3 82 | - zstd=1.3.7=h0b5b093_0 83 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/build/.jupyter_cache/__version__.txt: -------------------------------------------------------------------------------- 1 | 1.0.0 -------------------------------------------------------------------------------- /docs/build/.jupyter_cache/executed/1bc4a56eaa0307e71beb5cadd6ca8760/base.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "191fff75-50ea-4467-bca4-8db0e7961e3a", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import os, sys\n", 11 | "\n", 12 | "import numpy as np\n", 13 | "import pandas\n", 14 | "from scipy.interpolate import interp1d\n", 15 | "import xarray as xr\n", 16 | "import calendar\n", 17 | "import matplotlib.pyplot as plt\n", 18 | "\n", 19 | "import logging\n", 20 | "import calendar, datetime\n", 21 | "import cdsapi\n", 22 | "\n", 23 | "opj = os.path.join" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": 2, 29 | "id": "4b5d0ed7-6696-4159-a1a7-a8a24332e321", 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "odir = '/data/cams/france'\n", 34 | "latmax,lonmin,latmin,lonmax = 52.,-6.,41,10\n", 35 | "\n", 36 | "odir = '/data/cams/world'\n", 37 | "latmax,lonmin,latmin,lonmax = 90,-180,-90,180\n", 38 | "year=2023\n", 39 | "\n", 40 | "\n", 41 | "for month in range(10,13):\n", 42 | " \n", 43 | " date_start='{:4d}-{:02d}-{:02d}'.format(year,month,1)\n", 44 | " date_end='{:4d}-{:02d}-{:02d}'.format(year,month,calendar.monthrange(year, month)[1])\n", 45 | " name = '{:4d}-{:02d}'.format(year,month)\n", 46 | " filepath = opj(odir,'cams_forecast_'+name+'.nc')\n", 47 | " if os.path.exists(filepath):\n", 48 | " continue\n", 49 | " print(filepath)\n", 50 | "\n", 51 | " type='forecast'\n", 52 | " variable = [\n", 53 | " '10m_u_component_of_wind', '10m_v_component_of_wind',\n", 54 | " '2m_temperature',\n", 55 | " 'mean_sea_level_pressure', 'surface_pressure',\n", 56 | " 'ammonium_aerosol_optical_depth_550nm', 'black_carbon_aerosol_optical_depth_550nm',\n", 57 | " 'dust_aerosol_optical_depth_550nm',\n", 58 | " 'nitrate_aerosol_optical_depth_550nm', 'organic_matter_aerosol_optical_depth_550nm',\n", 59 | " 'sea_salt_aerosol_optical_depth_550nm',\n", 60 | " 'secondary_organic_aerosol_optical_depth_550nm', 'sulphate_aerosol_optical_depth_550nm',\n", 61 | " 'total_aerosol_optical_depth_1240nm',\n", 62 | " 'total_aerosol_optical_depth_469nm',\n", 63 | " 'total_aerosol_optical_depth_550nm',\n", 64 | " 'total_aerosol_optical_depth_670nm',\n", 65 | " 'total_aerosol_optical_depth_865nm',\n", 66 | " 'total_column_carbon_monoxide',\n", 67 | " 'total_column_methane',\n", 68 | " 'total_column_nitrogen_dioxide',\n", 69 | " 'total_column_ozone', 'total_column_water_vapour']\n", 70 | " area = [latmax,lonmin,latmin,lonmax]\n", 71 | " c = cdsapi.Client()\n", 72 | " \n", 73 | " c.retrieve(\n", 74 | " 'cams-global-atmospheric-composition-forecasts',\n", 75 | " {\n", 76 | " 'date': date_start + '/' + date_end,\n", 77 | " 'type': type,\n", 78 | " 'format': 'netcdf'\n", 79 | " ,\n", 80 | " 'variable': variable,\n", 81 | " 'time': ['00:00', '12:00'],\n", 82 | " 'leadtime_hour': ['0', '3', '6', '9'],\n", 83 | " 'area': area,\n", 84 | " },\n", 85 | " filepath)\n", 86 | "\n" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": null, 92 | "id": "205ff221-9717-4464-9336-ee581077cee5", 93 | "metadata": {}, 94 | "outputs": [], 95 | "source": [] 96 | } 97 | ], 98 | "metadata": { 99 | "kernelspec": { 100 | "display_name": "Python 3 (ipykernel)", 101 | "language": "python", 102 | "name": "python3" 103 | }, 104 | "language_info": { 105 | "codemirror_mode": { 106 | "name": "ipython", 107 | "version": 3 108 | }, 109 | "file_extension": ".py", 110 | "mimetype": "text/x-python", 111 | "name": "python", 112 | "nbconvert_exporter": "python", 113 | "pygments_lexer": "ipython3", 114 | "version": "3.9.15" 115 | } 116 | }, 117 | "nbformat": 4, 118 | "nbformat_minor": 5 119 | } -------------------------------------------------------------------------------- /docs/build/.jupyter_cache/global.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/docs/build/.jupyter_cache/global.db -------------------------------------------------------------------------------- /docs/environment.yml: -------------------------------------------------------------------------------- 1 | name: grs 2 | channels: 3 | - conda-forge 4 | - default 5 | dependencies: 6 | - gdal 7 | - xarray 8 | - rioxarray 9 | - pyproj 10 | - pip>=20.1 # pip is needed as dependency 11 | - pip: 12 | - geopandas 13 | - jupyter 14 | - cartopy 15 | - importlib_resources 16 | - cdsapi>=0.6.1 17 | - GRSdriver >= 1.0.3 18 | - xmltodict 19 | - docopt 20 | - netCDF4 21 | - numba 22 | - numpy 23 | - pandas>=2.0.3 24 | - s2cloudless>=1.7.1 25 | - eoreader 26 | - pystac 27 | - holoviews 28 | - datashader 29 | - bokeh 30 | - -r requirements.txt -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.https://www.sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | -r ../requirements.txt 2 | 3 | # Doc 4 | ipython 5 | sphinx <=7.2.6 6 | sphinx-book-theme<=1.1.2 7 | sphinx-copybutton 8 | myst-nb 9 | myst-parser 10 | linkify-it-py 11 | 12 | geopandas 13 | jupyter 14 | matplotlib 15 | cartopy 16 | hvplot 17 | 18 | -------------------------------------------------------------------------------- /docs/source/GRS API/grs.acutils.Aerosol.rst: -------------------------------------------------------------------------------- 1 | Aerosol 2 | ======= 3 | 4 | .. currentmodule:: grs.acutils 5 | 6 | .. autoclass:: Aerosol 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: -------------------------------------------------------------------------------- /docs/source/GRS API/grs.acutils.CamsParams.rst: -------------------------------------------------------------------------------- 1 | CamsParams 2 | ========== 3 | 4 | .. currentmodule:: grs.acutils 5 | 6 | .. autoclass:: CamsParams 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: -------------------------------------------------------------------------------- /docs/source/GRS API/grs.acutils.GaseousTransmittance.rst: -------------------------------------------------------------------------------- 1 | GaseousTransmittance 2 | ==================== 3 | 4 | .. currentmodule:: grs.acutils 5 | 6 | .. autoclass:: GaseousTransmittance 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: -------------------------------------------------------------------------------- /docs/source/GRS API/grs.acutils.Gases.rst: -------------------------------------------------------------------------------- 1 | Gases 2 | ===== 3 | 4 | .. currentmodule:: grs.acutils 5 | 6 | .. autoclass:: Gases 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: -------------------------------------------------------------------------------- /docs/source/GRS API/grs.acutils.Misc.rst: -------------------------------------------------------------------------------- 1 | Misc 2 | ==== 3 | 4 | .. currentmodule:: grs.acutils 5 | 6 | .. autoclass:: Misc 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: -------------------------------------------------------------------------------- /docs/source/GRS API/grs.acutils.Rasterization.rst: -------------------------------------------------------------------------------- 1 | Rasterization 2 | ============= 3 | 4 | .. currentmodule:: grs.acutils 5 | 6 | .. autoclass:: Rasterization 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: -------------------------------------------------------------------------------- /docs/source/GRS API/grs.acutils.rst: -------------------------------------------------------------------------------- 1 | acutils 2 | ======= 3 | 4 | .. automodule:: grs.acutils 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | .. rubric:: Classes 17 | 18 | .. autosummary:: 19 | :toctree: 20 | :template: custom-class-template.rst 21 | 22 | Aerosol 23 | CamsParams 24 | GaseousTransmittance 25 | Gases 26 | Misc 27 | Rasterization 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /docs/source/GRS API/grs.auxdata.AuxData.rst: -------------------------------------------------------------------------------- 1 | AuxData 2 | ======= 3 | 4 | .. currentmodule:: grs.auxdata 5 | 6 | .. autoclass:: AuxData 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: -------------------------------------------------------------------------------- /docs/source/GRS API/grs.auxdata.SensorData.rst: -------------------------------------------------------------------------------- 1 | SensorData 2 | ========== 3 | 4 | .. currentmodule:: grs.auxdata 5 | 6 | .. autoclass:: SensorData 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: -------------------------------------------------------------------------------- /docs/source/GRS API/grs.auxdata.rst: -------------------------------------------------------------------------------- 1 | auxdata 2 | ======= 3 | 4 | .. automodule:: grs.auxdata 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | .. rubric:: Classes 17 | 18 | .. autosummary:: 19 | :toctree: 20 | :template: custom-class-template.rst 21 | 22 | AuxData 23 | SensorData 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /docs/source/GRS API/grs.cams.CamsProduct.rst: -------------------------------------------------------------------------------- 1 | CamsProduct 2 | =========== 3 | 4 | .. currentmodule:: grs.cams 5 | 6 | .. autoclass:: CamsProduct 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: -------------------------------------------------------------------------------- /docs/source/GRS API/grs.cams.rst: -------------------------------------------------------------------------------- 1 | cams 2 | ==== 3 | 4 | .. automodule:: grs.cams 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | .. rubric:: Classes 17 | 18 | .. autosummary:: 19 | :toctree: 20 | :template: custom-class-template.rst 21 | 22 | CamsProduct 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /docs/source/GRS API/grs.data.aux.rst: -------------------------------------------------------------------------------- 1 | aux 2 | === 3 | 4 | .. automodule:: grs.data.aux 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /docs/source/GRS API/grs.data.lut.gases.rst: -------------------------------------------------------------------------------- 1 | gases 2 | ===== 3 | 4 | .. automodule:: grs.data.lut.gases 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /docs/source/GRS API/grs.data.lut.rst: -------------------------------------------------------------------------------- 1 | lut 2 | === 3 | 4 | .. automodule:: grs.data.lut 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | .. rubric:: Modules 25 | 26 | .. autosummary:: 27 | :toctree: 28 | :template: custom-module-template.rst 29 | :recursive: 30 | 31 | grs.data.lut.gases 32 | 33 | -------------------------------------------------------------------------------- /docs/source/GRS API/grs.data.rst: -------------------------------------------------------------------------------- 1 | data 2 | ==== 3 | 4 | .. automodule:: grs.data 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | .. rubric:: Modules 25 | 26 | .. autosummary:: 27 | :toctree: 28 | :template: custom-module-template.rst 29 | :recursive: 30 | 31 | grs.data.aux 32 | grs.data.lut 33 | 34 | -------------------------------------------------------------------------------- /docs/source/GRS API/grs.grs_process.Process.rst: -------------------------------------------------------------------------------- 1 | Process 2 | ======= 3 | 4 | .. currentmodule:: grs.grs_process 5 | 6 | .. autoclass:: Process 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: -------------------------------------------------------------------------------- /docs/source/GRS API/grs.grs_process.rst: -------------------------------------------------------------------------------- 1 | grs\_process 2 | ============ 3 | 4 | .. automodule:: grs.grs_process 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | .. rubric:: Classes 17 | 18 | .. autosummary:: 19 | :toctree: 20 | :template: custom-class-template.rst 21 | 22 | Process 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /docs/source/GRS API/grs.mask.Masking.rst: -------------------------------------------------------------------------------- 1 | Masking 2 | ======= 3 | 4 | .. currentmodule:: grs.mask 5 | 6 | .. autoclass:: Masking 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: -------------------------------------------------------------------------------- /docs/source/GRS API/grs.mask.Settings.rst: -------------------------------------------------------------------------------- 1 | Settings 2 | ======== 3 | 4 | .. currentmodule:: grs.mask 5 | 6 | .. autoclass:: Settings 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: -------------------------------------------------------------------------------- /docs/source/GRS API/grs.mask.rst: -------------------------------------------------------------------------------- 1 | mask 2 | ==== 3 | 4 | .. automodule:: grs.mask 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | .. rubric:: Classes 17 | 18 | .. autosummary:: 19 | :toctree: 20 | :template: custom-class-template.rst 21 | 22 | Masking 23 | Settings 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /docs/source/GRS API/grs.output.L2aProduct.rst: -------------------------------------------------------------------------------- 1 | L2aProduct 2 | ========== 3 | 4 | .. currentmodule:: grs.output 5 | 6 | .. autoclass:: L2aProduct 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: -------------------------------------------------------------------------------- /docs/source/GRS API/grs.output.rst: -------------------------------------------------------------------------------- 1 | output 2 | ====== 3 | 4 | .. automodule:: grs.output 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | .. rubric:: Classes 17 | 18 | .. autosummary:: 19 | :toctree: 20 | :template: custom-class-template.rst 21 | 22 | L2aProduct 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /docs/source/GRS API/grs.product.Product.rst: -------------------------------------------------------------------------------- 1 | Product 2 | ======= 3 | 4 | .. currentmodule:: grs.product 5 | 6 | .. autoclass:: Product 7 | :members: 8 | :show-inheritance: 9 | :inherited-members: -------------------------------------------------------------------------------- /docs/source/GRS API/grs.product.rst: -------------------------------------------------------------------------------- 1 | product 2 | ======= 3 | 4 | .. automodule:: grs.product 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | .. rubric:: Classes 17 | 18 | .. autosummary:: 19 | :toctree: 20 | :template: custom-class-template.rst 21 | 22 | Product 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /docs/source/GRS API/grs.run.main.rst: -------------------------------------------------------------------------------- 1 | grs.run.main 2 | ============ 3 | 4 | .. currentmodule:: grs.run 5 | 6 | .. autofunction:: main -------------------------------------------------------------------------------- /docs/source/GRS API/grs.run.rst: -------------------------------------------------------------------------------- 1 | run 2 | === 3 | 4 | .. automodule:: grs.run 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | .. rubric:: Functions 13 | 14 | .. autosummary:: 15 | :toctree: 16 | :template: custom-base-template.rst 17 | 18 | main 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /docs/source/_static/GRS_TBX.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/docs/source/_static/GRS_TBX.png -------------------------------------------------------------------------------- /docs/source/_static/GRS_driver.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/docs/source/_static/GRS_driver.png -------------------------------------------------------------------------------- /docs/source/_static/GRS_l2bgen.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/docs/source/_static/GRS_l2bgen.png -------------------------------------------------------------------------------- /docs/source/_static/INRAE_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/docs/source/_static/INRAE_logo.png -------------------------------------------------------------------------------- /docs/source/_static/OFB_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/docs/source/_static/OFB_logo.png -------------------------------------------------------------------------------- /docs/source/_static/cnes_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/docs/source/_static/cnes_logo.png -------------------------------------------------------------------------------- /docs/source/_static/custom.css: -------------------------------------------------------------------------------- 1 | :root { 2 | --pst-color-inline-code: 0, 0, 0 !important; 3 | } 4 | 5 | 6 | table { 7 | display: block; 8 | overflow-x: auto; 9 | word-break: keep-all; 10 | border-spacing: 0; 11 | } 12 | 13 | th { 14 | background-color: #c4d5e780; 15 | } 16 | 17 | td, th { 18 | margin: 0 !important; 19 | border: 1px solid #dee2e6 !important; 20 | padding: 0.2rem !important; 21 | vertical-align: middle !important; 22 | } 23 | 24 | td { 25 | white-space: nowrap; 26 | } 27 | 28 | td:empty { 29 | background-color: #ececec !important; 30 | } 31 | 32 | th:first-child, td:first-child { 33 | background-color: #c4d5e7; 34 | position: sticky; 35 | left: 0; 36 | } 37 | 38 | /* 1 row and one or two columns */ 39 | tr:first-child:nth-last-child(1) th:first-child:nth-last-child(2), 40 | tr:first-child:nth-last-child(1) th:first-child:nth-last-child(2) ~ th, 41 | tr:first-child:nth-last-child(1) th:first-child:nth-last-child(1) 42 | { 43 | background-color: #c4d5e780 !important; 44 | } 45 | 46 | tr:first-child:nth-last-child(1) td:first-child:nth-last-child(2), 47 | tr:first-child:nth-last-child(1) td:first-child:nth-last-child(2) ~ td, 48 | tr:first-child:nth-last-child(1) td:first-child:nth-last-child(1){ 49 | background-color: white !important; 50 | position: inherit; 51 | left: inherit; 52 | } 53 | 54 | td p { 55 | margin-bottom: 0 !important; 56 | vertical-align: middle !important; 57 | padding: 0 !important; 58 | } 59 | 60 | /*.bd-sidebar {*/ 61 | /* max-width: 400px !important;*/ 62 | /*}*/ 63 | 64 | /*@media (min-width: 1200px) {*/ 65 | /* .container,*/ 66 | /* .container-lg,*/ 67 | /* .container-md,*/ 68 | /* .container-sm,*/ 69 | /* .container-xl {*/ 70 | /* max-width: 1700px !important;*/ 71 | /* }*/ 72 | /*}*/ 73 | 74 | /* Newlines (\a) and spaces (\20) before each parameter */ 75 | .sig-param::before { 76 | content: "\a\20\20\20\20"; 77 | white-space: pre; 78 | } 79 | 80 | /* Newline after the last parameter (so the closing bracket is on a new line) */ 81 | dt em.sig-param:last-of-type::after { 82 | content: "\a"; 83 | white-space: pre; 84 | } 85 | 86 | 87 | /* To have blue background of width of the block (instead of width of content) */ 88 | dl.class > dt:first-of-type { 89 | display: block !important; 90 | } 91 | 92 | .bd-sidebar .nav ul { 93 | padding: 0 0 0 0.5rem; 94 | } 95 | 96 | code { 97 | background: #f2f2f1; 98 | font-size: 100%; 99 | } 100 | 101 | td:first-child code, 102 | td:first-child code ~ td, 103 | th:first-child code, 104 | th:first-child code ~ th { 105 | background: none; 106 | } 107 | 108 | tr:first-child:nth-last-child(1) td:first-child:nth-last-child(2) code, 109 | tr:first-child:nth-last-child(1) td:first-child:nth-last-child(2) code ~ td, 110 | tr:first-child:nth-last-child(1) td:first-child:nth-last-child(1) code { 111 | background: #f2f2f1; 112 | } 113 | 114 | dl.class dt, 115 | dl.exception dt, 116 | dl.data dt, 117 | dl.function dt { 118 | margin: 6px 0; 119 | font-size: 90%; 120 | line-height: normal; 121 | background: #e7f2fa; 122 | color: #2980B9; 123 | border-top: solid 3px #6ab0de; 124 | padding: 6px; 125 | position: relative; 126 | } 127 | 128 | dl.class dl dt, 129 | dl.exception dl dt, 130 | dl.data dl dt, 131 | dl.function dl dt, 132 | dl.py.method dd dl dt{ 133 | margin-bottom: 6px; 134 | border: none; 135 | border-left: solid 3px #ccc; 136 | background: #f0f0f0; 137 | color: #555; 138 | } 139 | 140 | dl.py.method dt{ 141 | margin-bottom: 6px; 142 | border: none; 143 | border-left: solid 3px rgba(27, 93, 163, 0.5); 144 | background: #f0f0f0; 145 | color: #555; 146 | } 147 | 148 | dl.class .viewcode-link, 149 | dl.exception .viewcode-link, 150 | dl.data .viewcode-link, 151 | dl.function .viewcode-link { 152 | color: #27AE60; 153 | font-size: 80%; 154 | } 155 | 156 | 157 | dl.py.attribute dt, 158 | dl.py.property dt{ 159 | margin-bottom: 6px; 160 | border: none; 161 | border-left: solid 3px #c4d5e7; 162 | background: #f0f0f0; 163 | color: #555; 164 | } -------------------------------------------------------------------------------- /docs/source/_static/example_image_level.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/docs/source/_static/example_image_level.png -------------------------------------------------------------------------------- /docs/source/_static/grs_logo_light.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/docs/source/_static/grs_logo_light.png -------------------------------------------------------------------------------- /docs/source/_static/grs_logo_v0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/docs/source/_static/grs_logo_v0.png -------------------------------------------------------------------------------- /docs/source/_static/grs_logo_v0.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 7 | 8 | 10 | 34 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /docs/source/_static/grstbx_visual_tool.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/docs/source/_static/grstbx_visual_tool.gif -------------------------------------------------------------------------------- /docs/source/_static/magellium_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/docs/source/_static/magellium_logo.png -------------------------------------------------------------------------------- /docs/source/_static/magellium_logo_eo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/docs/source/_static/magellium_logo_eo.png -------------------------------------------------------------------------------- /docs/source/_templates/custom-class-template.rst: -------------------------------------------------------------------------------- 1 | {{ name | escape | underline}} 2 | 3 | .. currentmodule:: {{ module }} 4 | 5 | .. autoclass:: {{ objname }} 6 | :members: 7 | :show-inheritance: 8 | :inherited-members: -------------------------------------------------------------------------------- /docs/source/_templates/custom-module-template.rst: -------------------------------------------------------------------------------- 1 | {{ name | escape | underline}} 2 | 3 | .. automodule:: {{ fullname }} 4 | 5 | {% block attributes %} 6 | {% if attributes %} 7 | .. rubric:: Module Attributes 8 | 9 | .. autosummary:: 10 | :toctree: 11 | :template: custom-base-template.rst 12 | {% for item in attributes %} 13 | {{ item }} 14 | {%- endfor %} 15 | {% endif %} 16 | {% endblock %} 17 | 18 | {% block functions %} 19 | {% if functions %} 20 | .. rubric:: {{ _('Functions') }} 21 | 22 | .. autosummary:: 23 | :toctree: 24 | :template: custom-base-template.rst 25 | {% for item in functions %} 26 | {{ item }} 27 | {%- endfor %} 28 | {% endif %} 29 | {% endblock %} 30 | 31 | {% block classes %} 32 | {% if classes %} 33 | .. rubric:: {{ _('Classes') }} 34 | 35 | .. autosummary:: 36 | :toctree: 37 | :template: custom-class-template.rst 38 | {% for item in classes %} 39 | {{ item }} 40 | {%- endfor %} 41 | {% endif %} 42 | {% endblock %} 43 | 44 | {% block exceptions %} 45 | {% if exceptions %} 46 | .. rubric:: {{ _('Exceptions') }} 47 | 48 | .. autosummary:: 49 | :toctree: 50 | :template: custom-base-template.rst 51 | {% for item in exceptions %} 52 | {{ item }} 53 | {%- endfor %} 54 | {% endif %} 55 | {% endblock %} 56 | 57 | {% block modules %} 58 | {% if modules %} 59 | .. rubric:: Modules 60 | 61 | .. autosummary:: 62 | :toctree: 63 | :template: custom-module-template.rst 64 | :recursive: 65 | {% for item in modules %} 66 | {{ item }} 67 | {%- endfor %} 68 | {% endif %} 69 | {% endblock %} -------------------------------------------------------------------------------- /docs/source/api.rst: -------------------------------------------------------------------------------- 1 | .. _api: 2 | 3 | 4 | 5 | This page provides an auto-generated summary of GRS's API. 6 | 7 | 8 | .. autosummary:: 9 | :toctree: GRS API 10 | :template: custom-module-template.rst 11 | :recursive: 12 | 13 | grs.acutils 14 | grs.auxdata 15 | grs.cams 16 | grs.data 17 | grs.grs_process 18 | grs.mask 19 | grs.output 20 | grs.product 21 | grs.run -------------------------------------------------------------------------------- /docs/source/examples/advanced.rst: -------------------------------------------------------------------------------- 1 | Advanced 2 | ============== 3 | .. toctree:: 4 | :maxdepth: 1 5 | :caption: Experimental Landsat 6 | 7 | grs_v2_landsat_dev 8 | -------------------------------------------------------------------------------- /docs/source/examples/auxdata.rst: -------------------------------------------------------------------------------- 1 | Auxiliary Data 2 | ============== 3 | .. toctree:: 4 | :maxdepth: 1 5 | :caption: CAMS 6 | 7 | download_cams 8 | -------------------------------------------------------------------------------- /docs/source/examples/basics.rst: -------------------------------------------------------------------------------- 1 | Basics 2 | ======== 3 | .. toctree:: 4 | :maxdepth: 1 5 | :caption: Basics 6 | 7 | grs_v2_simple 8 | -------------------------------------------------------------------------------- /docs/source/examples/roi_grs_visu_simple.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "name": "roi_grs_visu_simple", 4 | "crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::32631" } }, 5 | "features": [ 6 | { "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 653314.901260369690135, 4825477.949759116396308 ], [ 679632.77098701777868, 4831405.748235942795873 ], [ 683542.455221180571243, 4805095.140593723393977 ], [ 656416.355584109551273, 4801948.410354549065232 ], [ 656791.586651421617717, 4805516.185986168682575 ], [ 653314.901260369690135, 4825477.949759116396308 ] ] ] } } 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /docs/source/examples/visualization.rst: -------------------------------------------------------------------------------- 1 | Visualization 2 | ============= 3 | .. toctree:: 4 | :maxdepth: 1 5 | :caption: Visualization 6 | 7 | grs_visu_simple 8 | -------------------------------------------------------------------------------- /docs/source/history.rst: -------------------------------------------------------------------------------- 1 | 2 | Version history 3 | ================== 4 | 5 | 1.0.0: 6 | - grs for Sentinel2 7 | 8 | 1.1.0: 9 | - adaptation to Landsat 4, 5, 7, 8 10 | 11 | 1.1.1: 12 | - small changes 13 | 14 | 1.1.2: 15 | - output in Rrs unit 16 | 17 | 1.1.3 18 | - small changes 19 | 20 | 1.1.4: 21 | - set cloud mask; (compressed) netcdf4 output format 22 | 23 | 1.2.0: 24 | - load full data matrix from image instead of line by line pixel extraction (preparation for multipixel retrieval algorithm 25 | 26 | 1.2.1: 27 | - enable high latitude processing (change of DEM), option to process all pixels before masking "non-water" pixel 28 | 29 | 1.2.2: 30 | - Interpolation based on nearest-neighbor to keep tile-edge pixels. 31 | - Implementation of product.dispose to minimize memory usage in the jvm of snap 32 | 33 | 1.2.3: 34 | - compliant with version 8 of SNAP: 35 | - change output writing (now directly in NETCDF4, i.e., compressed, at the end of the process) 36 | - new utils get_subset 37 | 38 | 1.2.4: 39 | - option to load MAJA and WaterDetect mask and export masks in output file 40 | - option to process WaterDetect Water pixels only 41 | 42 | 1.3.0: 43 | - update for CAMS data (cds version): 44 | - new aod wavelengths and spectral ssa from 2018 onwards 45 | - adjustment for absorbing aerosol through ssa 46 | 47 | 1.3.1: 48 | - update CAMS data extraction from xarray and fix for longitude conventions 49 | 50 | 1.3.2: 51 | - Important improvement in LUT interpolation and access 52 | 53 | 1.3.3: 54 | - fix small bug for sunglint BRDF output 55 | 56 | 1.3.4: 57 | - add slope and shade from DEM 58 | 59 | 1.4.0: 60 | - process image by rectangular chunks 61 | 62 | 1.5.0: 63 | - change output parameter with addition of ndwi_nir and ndwi_swir 64 | 65 | 2.0.0: 66 | - remove snappy skeleton, simplification of the previous option, no more handling for Landsat 67 | 68 | 2.0.1: 69 | - fix for I/O 70 | 71 | 2.0.2: 72 | - memory optimization 73 | 74 | 2.0.3: 75 | - add surfwater 76 | 77 | 2.0.4: 78 | - adapt format for QGIS 79 | 80 | 2.0.5: 81 | - new output format compliant with SNAP "beam format" 82 | 83 | 2.1.0: 84 | - Major update on radiative transfer look-up tables and aerosol models 85 | 86 | 2.1.1: 87 | - package data 88 | 89 | 2.1.2: 90 | - fix for tiles straddling Greenwich meridian 91 | 92 | 2.1.3: 93 | - new cams automated loading and revised parameters 94 | 95 | 2.1.4: 96 | - change input/output feature for grs process 97 | (i.e., can be called as simple function within a script) 98 | - Change of output variable with addition of bitmask flags 99 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. GRS documentation master file, created by 2 | sphinx-quickstart on Fri Mar 8 11:27:53 2024. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | GRS documentation 7 | =============================== 8 | 9 | GRS (Glint Removal for Sentinel-2-like sensors) 10 | ------------------------------------------------- 11 | 12 | The GRS algorithm `Harmel et al., 13 | (2018) `__ 14 | was specifically developed to handle and correct for the direct sunlight reflected by the water surface and potentially reaching the sensor (i.e., 15 | sunglint signal) of Sentinel-2-like mission, that is nadir or near-nadir viewing sensor with SWIR bands. The GRS 16 | processor consists of three main modules to correct for: 17 | 18 | - gaseous absorption, 19 | - diffuse light from sky and its reflection by the air-water interface and 20 | - the sunglint signal in order to retrieve the water-leaving signal at the water surface level. 21 | 22 | 23 | .. figure:: _static/example_image_level.png 24 | :alt: image_output 25 | 26 | .. toctree:: 27 | :maxdepth: 3 28 | :caption: Getting started: 29 | 30 | usage 31 | 32 | .. toctree:: 33 | :maxdepth: 2 34 | :caption: Tutorials 35 | 36 | examples/basics 37 | examples/auxdata 38 | examples/advanced 39 | examples/visualization 40 | 41 | .. toctree:: 42 | :maxdepth: 3 43 | :caption: GRS API 44 | :hidden: 45 | 46 | api 47 | 48 | API reference 49 | ------------- 50 | 51 | .. autosummary:: 52 | :template: custom-module-template.rst 53 | 54 | grs.acutils 55 | grs.auxdata 56 | grs.cams 57 | grs.data 58 | grs.grs_process 59 | grs.mask 60 | grs.output 61 | grs.product 62 | grs.run 63 | 64 | .. toctree:: 65 | :maxdepth: 1 66 | :caption: For Contributors 67 | :hidden: 68 | 69 | history 70 | 71 | Interactive data manipulation 72 | ============================= 73 | 74 | Please see the GRS Toolbox package `grstbx `__ 75 | 76 | .. figure:: _static/grstbx_visual_tool.gif 77 | :alt: animated_dashboard 78 | 79 | Indices and tables 80 | ================== 81 | 82 | * :ref:`genindex` 83 | * :ref:`modindex` 84 | * :ref:`search` 85 | 86 | 87 | 88 | .. toctree:: 89 | :maxdepth: 1 90 | :caption: Funding 91 | 92 | support 93 | -------------------------------------------------------------------------------- /docs/source/support.rst: -------------------------------------------------------------------------------- 1 | Supporting institutions 2 | ============================= 3 | 4 | .. image:: _static/cnes_logo.png 5 | :width: 25% 6 | 7 | .. image:: _static/INRAE_logo.png 8 | :width: 25% 9 | 10 | .. image:: _static/OFB_logo.png 11 | :width: 15% 12 | 13 | --- 14 | 15 | .. image:: _static/magellium_logo.png 16 | :width: 25% 17 | 18 | .. image:: _static/magellium_logo_eo.png 19 | :width: 50% 20 | :target: https://earthobservation.magellium.com/ 21 | -------------------------------------------------------------------------------- /docs/source/usage.rst: -------------------------------------------------------------------------------- 1 | Usage 2 | ===== 3 | 4 | Installation 5 | ------------ 6 | 7 | These instructions will get you a copy of the project up and running on 8 | your local machine for development and testing purposes. See deployment 9 | for notes on how to deploy the project on a live system. 10 | 11 | Download the LUT files 12 | ~~~~~~~~~~~~~~~~~~~~~~~ 13 | 14 | click on 15 | `grsdata `__ 16 | to download and save in your desired path (your_GRSDATA_PATH) 17 | 18 | Installation with conda environment 19 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 20 | 21 | :: 22 | 23 | conda activate "name of your conda env" 24 | 25 | Python >= 3.9 is recommended, example: 26 | 27 | :: 28 | 29 | conda create python=3.10 -n grs_v2 30 | conda activate grs_v2 31 | 32 | Then, install python dependencies: 33 | 34 | :: 35 | 36 | conda install -c conda-forge eoreader cdsapi netCDF4 docopt xmltodict numba 37 | 38 | Set the ``config.yml`` file: 39 | 40 | :: 41 | 42 | path: 43 | grsdata: your_GRSDATA_PATH 44 | 45 | Finally, install grs with: 46 | 47 | .. code:: 48 | 49 | pip install . 50 | 51 | Testing 52 | ------- 53 | 54 | After installation, you can type: 55 | 56 | .. code:: 57 | 58 | grs -h 59 | 60 | You should see something like: 61 | 62 | .. code:: 63 | 64 | Executable to process Sentinel-2 L1C images for aquatic environment 65 | 66 | Usage: 67 | grs [--cams_file file] [-o ] [--odir ] [--resolution res] [--scale_aot factor] [--levname ] [--no_clobber] [--allpixels] [--surfwater file] [--dem_file file] [--snap_compliant] 68 | grs -h | --help 69 | grs -v | --version 70 | 71 | Options: 72 | -h --help Show this screen. 73 | -v --version Show version. 74 | 75 | Input file to be processed 76 | 77 | --cams_file file Absolute path of the CAMS file to be used (mandatory) 78 | 79 | -o ofile Full (absolute or relative) path to output L2 image. 80 | --odir odir Ouput directory [default: ./] 81 | --levname lev Level naming used for output product [default: L2Agrs] 82 | --no_clobber Do not process if already exists. 83 | --resolution=res spatial resolution of the scene pixels 84 | --allpixels force to process all pixels whatever they are masked (cloud, vegetation...) or not 85 | --surfwater file Absolute path of the surfwater geotiff file to be used 86 | --dem_file file Absolute path of the DEM geotiff file (already subset for the S2 tile) 87 | --scale_aot factor scaling factor applied to CAMS aod550 raster 88 | [default: 1] 89 | --opac_model name Force the aerosol model (OPAC) to be 'name' 90 | (choice: ['ARCT_rh70', 'COAV_rh70', 'DESE_rh70', 91 | 'MACL_rh70', 'URBA_rh70']) 92 | --snap_compliant Export output to netcdf aligned with "beam" for ESA SNAP software 93 | 94 | Example: 95 | grs /data/satellite/S2/L1C/S2B_MSIL1C_20220731T103629_N0400_R008_T31TFJ_20220731T124834.SAFE --cams_file /data/satellite/S2/cnes/CAMS/2022-07-31-cams-global-atmospheric-composition-forecasts.nc --resolution 60 96 | For CNES datalake: 97 | grs /work/datalake/S2-L1C/31TFJ/2023/06/16/S2B_MSIL1C_20230616T103629_N0509_R008_T31TFJ_20230616T111826.SAFE --cams_file /work/datalake/watcal/ECMWF/CAMS/2023/06/16/2023-06-16-cams-global-atmospheric-composition-forecasts.nc --odir /work/datalake/watcal/test --resolution 20 --dem_file /work/datalake/static_aux/MNT/COP-DEM_GLO-30-DGED_S2_tiles/COP-DEM_GLO-30-DGED_31TFJ.tif 98 | -------------------------------------------------------------------------------- /ecmwf/download_cams_all.py: -------------------------------------------------------------------------------- 1 | import os, sys 2 | import logging 3 | 4 | import argparse 5 | from datetime import date 6 | import calendar 7 | from ecmwfapi import ECMWFDataServer 8 | 9 | """ 10 | This function download the cams data used in grs 11 | 12 | The main program takes one argument : the mode cams_reanalysis or cams_forecast needed 13 | 14 | """ 15 | 16 | 17 | def main(dic): 18 | data_type = dic['mode'] 19 | month = dic['month'] 20 | year_start = int(dic['year_start']) 21 | year_end = int(dic['year_end']) 22 | area = "90/-180/-90/180" 23 | 24 | if month != 'all': 25 | today = date.today() 26 | d1 = today.strftime("%Y%m%d") 27 | 28 | server = ECMWFDataServer() 29 | 30 | # dataset will be download by default from 2017 to 2020 31 | step = '0' 32 | if data_type == 'cams_forecast': 33 | class_ = 'mc' 34 | dataset = 'cams_nrealtime' 35 | time = '00:00:00' 36 | step = '0/6/12/18' 37 | type = 'fc' 38 | 39 | # data will be download by default from 2000 to 2017 40 | elif data_type == 'cams_reanalysis': 41 | class_ = 'mc' 42 | dataset = 'cams_reanalysis' 43 | # date = '20150301/TO/20170101' 44 | time = '00:00:00/06:00:00/12:00:00/18:00:00' 45 | type = 'an' 46 | 47 | elif data_type == 'interim': 48 | class_ = 'ei' 49 | dataset = 'interim' 50 | type = 'an' 51 | else: 52 | logging.info('Error: not appropriate dataset for ecmwf/cams download') 53 | sys.exit() 54 | 55 | # specify the period to catch data 56 | # try: 57 | for year in range(year_start, year_end + 1): 58 | odir = "/datalake/watcal/ECMWF/CAMS/" + str(year) + "/" 59 | if not os.path.exists(odir): 60 | os.makedirs(odir) 61 | for month in range(1, 13): 62 | target = odir + str(year) + "-" + str(month).zfill( 63 | 2) + "_month_" + data_type + ".nc" 64 | 65 | if os.path.exists(target): 66 | continue 67 | 68 | numberOfDays = calendar.monthrange(year, month)[1] 69 | date = str(year) + str(month).zfill(2) + "01/TO/" + str(year) + str(month).zfill(2) + str(numberOfDays) 70 | logging.info(date) 71 | server.retrieve({ 72 | 'class': class_, 73 | 'dataset': dataset, 74 | 'date': date, 75 | 'grid': "0.125/0.125", 76 | 'levtype': 'sfc', 77 | 'param': "125.210/137.128/151.128/165.128/166.128/167.128/206.128/207.210/213.210/214.210/215.210/216.210", 78 | 'stream': 'oper', 79 | 'step': step, 80 | 'time': time, 81 | 'type': type, 82 | 'format': 'netcdf', 83 | 'area': "90/-180/-90/180", 84 | 'target': target 85 | }) 86 | return 87 | # except: 88 | # logging.info('Error: not appropriate cams settings for download. Refers to ecmwf.') 89 | # sys.exit() 90 | 91 | 92 | if __name__ == '__main__': 93 | parser = argparse.ArgumentParser(description='Download Cams datasets.') 94 | parser.add_argument('mode', 95 | help='the cams_forecast or cams_reanalysis mode') 96 | parser.add_argument('month', 97 | help='the month from which data will be downloaded. Set all for downloading data from 2000 to today. Set a date in %Y%m%d format if you want to download data from this date until now.') 98 | parser.add_argument('year_start', 99 | help='the start year of period to be downloaded in case of month = all ') 100 | parser.add_argument('year_end', 101 | help='the end year of period to be downloaded in case of month = all ') 102 | args = parser.parse_args() 103 | 104 | main(vars(args)) 105 | -------------------------------------------------------------------------------- /ecmwf/pbs_cds.pbs: -------------------------------------------------------------------------------- 1 | #Recuperer les donnees ecmwf cams à travers un job PBS : 2 | #To be launched with -V option 3 | #!/bin/bash 4 | #PBS -V 5 | #PBS -N ecmwf_cams 6 | #PBS -l select=2:ncpus=4:mem=2000mb 7 | #PBS -l walltime=48:00:00 8 | 9 | module load conda 10 | 11 | conda activate /work/scratch/harmelt/grs_py3.6 12 | # using gitlab package 13 | api_call=/work/ALT/swot/aval/OBS2CO/git/grs2/ecmwf/download_cams_cds.py 14 | # or personal package: 15 | api_call=/work/scratch/harmelt/dev/grs2/ecmwf/download_cams_cds.py 16 | 17 | # ln -s /etc/pki/tls $CONDA_PREFIX/ssl 18 | 19 | #$CONDA_PREFIX/bin/python $api_call 2021 2021 forecast & 20 | $CONDA_PREFIX/bin/python $api_call 2022 2022 forecast & 21 | #$CONDA_PREFIX/bin/python $api_call 2019 2020 forecast & 22 | 23 | wait 24 | 25 | 26 | -------------------------------------------------------------------------------- /ecmwf/pbs_ecmwf.pbs: -------------------------------------------------------------------------------- 1 | #Recuperer les donnees ecmwf cams à travers un job PBS : 2 | #To be launched with -V option 3 | #!/bin/bash 4 | 5 | #PBS -V 6 | #PBS -N ecmwf_cams 7 | #PBS -l select=6:ncpus=2:mem=2000mb 8 | #PBS -l walltime=48:00:00 9 | 10 | module load conda 11 | 12 | conda activate /work/scratch/harmelt/grs_py3.6 13 | 14 | # ln -s /etc/pki/tls $CONDA_PREFIX/ssl 15 | for year in 2021 2015; do 16 | $CONDA_PREFIX/bin/python /work/ALT/swot/aval/OBS2CO/git/grs2/ecmwf/download_cams_all.py cams_forecast all $year $year & 17 | done 18 | #$CONDA_PREFIX/bin/python /work/ALT/swot/aval/OBS2CO/git/grs2/ecmwf/download_cams_all.py cams-reanalysis all 2003 2006 & 19 | #$CONDA_PREFIX/bin/python /work/ALT/swot/aval/OBS2CO/git/grs2/ecmwf/download_cams_all.py cams-reanalysis all 2007 2009 & 20 | #$CONDA_PREFIX/bin/python /work/ALT/swot/aval/OBS2CO/git/grs2/ecmwf/download_cams_all.py cams-reanalysis all 2010 2012 & 21 | #$CONDA_PREFIX/bin/python /work/ALT/swot/aval/OBS2CO/git/grs2/ecmwf/download_cams_all.py cams-reanalysis all 2013 2015 & 22 | #$CONDA_PREFIX/bin/python /work/ALT/swot/aval/OBS2CO/git/grs2/ecmwf/download_cams_all.py cams-reanalysis all 2016 2018 23 | 24 | wait 25 | 26 | 27 | -------------------------------------------------------------------------------- /exe/.~lock.list_grs_cnes_obs2mod.csv#: -------------------------------------------------------------------------------- 1 | ,harmelt,visu01.sis.cnes.fr,16.04.2021 11:08,file:///home/eh/harmelt/.config/libreoffice/4; -------------------------------------------------------------------------------- /exe/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/exe/__init__.py -------------------------------------------------------------------------------- /exe/aeronet-oc_sites.txt: -------------------------------------------------------------------------------- 1 | site lon lat alt tile 2 | COVE_SEAPRISM -75.710000 36.900000 .000000 18SVF 3 | Galata_Platform 28.193190 43.044624 .000000 35TNH 4 | Gloria 29.359670 44.599970 30.000000 35TPK 5 | GOT_Seaprism 101.412300 9.286200 .000000 47PQL 6 | Gustav_Dalen_Tower 17.466830 58.594170 .000000 33VXF 7 | Helsinki_Lighthouse 24.926360 59.948970 .000000 35VLG 8 | Ieodo_Station 125.182447 32.122953 .000000 51SYR 9 | Lake_Erie -83.193600 41.825600 173.00000 17TLG 10 | LISCO -73.341767 40.954517 0.000000 18TXL 11 | Lucinda 146.386100 -18.519800 .000000 55KDV 12 | MVCO -70.550000 41.300000 0.000000 19TCF 13 | Palgrunden 13.151500 58.755333 44.000000 33VUF 14 | Socheongcho 124.738039 37.423133 .000000 51SXB 15 | Thornton_C-power 2.955278 51.532500 .000000 31UDT 16 | USC_SEAPRISM -118.117820 33.563710 .000000 11SLT 17 | Venise 12.508300 45.313900 .000000 32TQR 18 | WaveCIS_Site_CSI_6 -90.483333 28.866667 .0000 15RYN 19 | WaveCIS_Site_CSI_6 -90.483333 28.866667 .0000 15RYM 20 | Zeebrugge-MOW1 3.120000 51.362000 .000000 31UES -------------------------------------------------------------------------------- /exe/amazone_sites.txt: -------------------------------------------------------------------------------- 1 | site lon lat alt tile 2 | 22KFB -49.612 -21.378 380 22KFB -------------------------------------------------------------------------------- /exe/call_amalthee.py: -------------------------------------------------------------------------------- 1 | ''' 2 | command to load L1C and L2A images on datalake from the given list 3 | 4 | example: 5 | python3 call_amalthee.py list_grs_cnes_template.csv 6 | ''' 7 | 8 | import os, sys 9 | import numpy as np 10 | import pandas as pd 11 | import glob 12 | from datetime import datetime, timedelta 13 | import time 14 | 15 | # CNES lib for datalake managment 16 | # import lxml 17 | 18 | from libamalthee import Amalthee 19 | 20 | 21 | # start_date, end_date = '2021-03-01', '2021-03-30' 22 | # tile, lon, lat = '31TGK', '14.6', '14' 23 | 24 | sitefile = 'exe/list/list_grs_cnes_seine.csv' # 'exe/list_grs_cnes_seine.csv' # 25 | sitefile = sys.argv[1] 26 | sites = pd.read_csv(sitefile) 27 | 28 | # -------------------------------------------------------------------------------- 29 | 30 | for idx, site in sites.iterrows(): 31 | # load row of list file 32 | if site.iloc[0] == 0: 33 | continue 34 | 35 | L2A = Amalthee('theia') 36 | L1C = Amalthee('peps') 37 | name, start_date, end_date, sat, tile, resolution, flag = site.iloc[1:8] 38 | if start_date == end_date: 39 | end_date = (datetime.strptime(end_date, '%Y-%m-%d').date() + timedelta(days=1)).__str__() 40 | 41 | parameters = {"productType": "S2MSI1C", "tileid": tile} 42 | L1C.search("S2ST", start_date, end_date, parameters) 43 | idL1C = L1C.fill_datalake() 44 | L1C.check_datalake() 45 | parameters = {'processingLevel': 'LEVEL2A', 'location':'T'+tile} #'lon': str(lon), 'lat': str(lat)} 46 | L2A.search("SENTINEL2", start_date, end_date, parameters) 47 | idL2A = L2A.fill_datalake() 48 | 49 | # if idx == 1: 50 | # break 51 | finished = False 52 | finished_L1C = False 53 | finished_L2A = False 54 | iwait = 0 55 | while not finished: 56 | print('tile',tile) 57 | res = L1C.check_datalake() 58 | print('L1C',res) 59 | try: 60 | if (res['status'] == 'no_request_made') or (res['status'] == 'done'): 61 | finished_L1C = True 62 | print('L1C job finished or canceled', res) 63 | L1C.delete_request(idL1C) 64 | except: 65 | pass 66 | res = L2A.check_datalake() 67 | print('L2A',res) 68 | 69 | try: 70 | if (res['status'] == 'no_request_made') or (res['status'] == 'done'): 71 | finished_L2A = True 72 | print('L2A job finished or canceled', res) 73 | L2A.delete_request(idL2A) 74 | except: 75 | pass 76 | # L2A.products.loc[L2A.products.state == 'failed', 'available'] = True 77 | #finished = (all(L2A.products.available) or finished_L2A) and (all(L1C.products.available) or finished_L1C) 78 | finished = ( finished_L2A) and ( finished_L1C) 79 | 80 | if finished: 81 | break 82 | 83 | time.sleep(123) 84 | 85 | iwait += 1 86 | if iwait > 20: 87 | print('time limit exceeded') 88 | finished = True 89 | 90 | print('Finished!!') 91 | -------------------------------------------------------------------------------- /exe/call_subprocess.py: -------------------------------------------------------------------------------- 1 | import os, sys 2 | import subprocess 3 | 4 | 5 | 6 | dir = os.path.dirname(os.path.abspath(__file__)) 7 | exe = os.path.join(dir, 'grs_proc_sites.py') 8 | site_file = os.path.join(dir, 'aeronet-oc_sites.txt') 9 | odir = os.path.abspath('/nfs/DP/S2/L2/GRS/aeronet-oc') 10 | site_file = os.path.join(dir, 'amazone_sites.txt') 11 | odir = os.path.abspath('/nfs/DP/S2/L2/GRS/amazone') 12 | site_file = os.path.join(dir, 'gernez_sites.txt') 13 | odir = os.path.abspath('/nfs/DP/S2/L2/GRS/gernez') 14 | 15 | 16 | while True: 17 | try: 18 | pipeline_out = subprocess.call(['python3', exe, site_file, odir], stderr=subprocess.STDOUT) 19 | except: 20 | sys.exit() 21 | -------------------------------------------------------------------------------- /exe/call_subprocess_multiproc.py: -------------------------------------------------------------------------------- 1 | import os, sys 2 | import pandas as pd 3 | import subprocess 4 | from multiprocessing import Pool 5 | 6 | 7 | def call(command): 8 | print(command) 9 | pipeline_out = subprocess.call(command, stderr=subprocess.STDOUT, shell=True) 10 | return 11 | 12 | 13 | ncore=8 14 | ifile = 'tmp_grslist_220725_103950' 15 | command = pd.read_csv(ifile).values 16 | 17 | with Pool(processes=ncore) as pool: 18 | pool.map(call, command) 19 | pool.close 20 | -------------------------------------------------------------------------------- /exe/download_list.py: -------------------------------------------------------------------------------- 1 | ''' 2 | command to download images from a datasheet list 3 | ''' 4 | 5 | import os, sys 6 | import re 7 | import pandas as pd 8 | import glob 9 | import datetime 10 | import multiprocessing 11 | 12 | # sys.path.extend([os.path.abspath(__file__)]) 13 | sys.path.extend([os.path.abspath('exe')]) 14 | from procutils import misc 15 | 16 | misc = misc() 17 | from sid import download_image 18 | # to get image provider info under variable 'dic' 19 | from sid.config import * 20 | 21 | 22 | list_file = '/local/AIX/tristan.harmel/project/acix/AERONETOC_Matchups_List_harmel.xlsx' 23 | list_file = '/local/AIX/tristan.harmel/project/acix/ACIXII_Aqua_PhaseII_scene_tile_IDs_harmel.xlsx' 24 | list_file = '/local/AIX/tristan.harmel/project/acix/ACIX_scene_tile_IDs_L1C_Updated_5_28_2019.xlsx' 25 | sites = pd.read_excel(list_file) # , sep=' ') 26 | 27 | missions = ['all', 'S2', 'Landsat'] 28 | mission = missions[1] 29 | 30 | # number of processors to be used 31 | ncore = 12 32 | if mission == 'S2': 33 | ncore = 2 34 | 35 | command = [] 36 | list = [] 37 | _file='' 38 | for idx, site in sites.iterrows(): 39 | if site.iloc[0] != site.iloc[0]: 40 | continue 41 | name, lat, lon, date_raw, time, basename, time_diff = site.iloc[0:7] 42 | time_diff = re.sub(':.*','',str(time_diff)) 43 | print(name, lat, lon, date_raw, time, basename, time_diff) 44 | # get date in pratical format 45 | try: 46 | date = datetime.datetime.strptime(date_raw, '%d-%m-%Y') + datetime.timedelta(hours=int(time_diff)) 47 | except: 48 | date = date_raw + datetime.timedelta(hours=int(time_diff)) 49 | sensor = misc.get_sensor(basename) 50 | if sensor == None: 51 | print('non standard image, not processed: ', basename) 52 | continue 53 | productimage = sensor[1] 54 | sat = sensor[2] 55 | 56 | # skip S2/Landsat if mission == Landsat/S2 57 | if (('Landsat' in productimage) & (mission == 'S2')) | (('S2' in productimage) & (mission == 'Landsat')): 58 | continue 59 | 60 | idir = dic[productimage]['path'] 61 | file = os.path.join(idir, basename) 62 | # ------------------------ 63 | # input file naming 64 | # Warning: only .zip or .tgz are permitted 65 | file = file.replace('SAFE', 'zip') 66 | if productimage != 'S2_ESA': 67 | file = file.replace('.tgz', '') 68 | file = file + '.tgz' 69 | 70 | # check if image is already downloaded 71 | if (not os.path.exists(file)) & (file != _file): 72 | _file=file 73 | print(sensor, name, file) 74 | list.append(file) 75 | fromdate = date.strftime('%Y-%m-%d') 76 | todate = datetime.datetime.strftime(date + datetime.timedelta(days=2), '%Y-%m-%d') 77 | cloudmax = str(100) 78 | 79 | script = dic[productimage]['script'] 80 | write = dic[productimage]['path'] 81 | auth = dic[productimage]['auth'] 82 | if productimage == 'S2_ESA': 83 | auth = os.path.abspath('/local/AIX/tristan.harmel/git/sat/sid/auxdata/apihub_th.txt') 84 | tile = misc.get_tile(basename) 85 | command.append([script, lat, lon, write, auth, tile, sat, cloudmax, fromdate, todate, productimage]) 86 | print(fromdate,' to ',todate, basename) 87 | print(site) 88 | # download image 89 | # download_image.mp_worker(command) 90 | 91 | p = multiprocessing.Pool(ncore) 92 | p.map(download_image.mp_worker, command) 93 | p.close() 94 | -------------------------------------------------------------------------------- /exe/global_config.yml: -------------------------------------------------------------------------------- 1 | { 2 | #"input_file" : "/datalake/watcal/INPUT_TEST/S2B_MSIL1C_20180927T103019_N0206_R108_T31TGK_20180927T143835.SAFE", 3 | #"input_file": "/datalake/S2-L1C/33PVQ/2022/01/23/S2A_MSIL1C_20220123T092301_N0301_R093_T33PVQ_20220123T130718.SAFE", 4 | #"input_file": "/datalake/watcal/INPUT_TEST/S2A_MSIL1C_20220411T101601_N0400_R065_T31TGH_20220411T124038.SAFE", 5 | "input_file": "/datalake/S2-L1C/33TVF/2022/04/10/S2B_MSIL1C_20220410T100029_N0400_R122_T33TVF_20220410T121931.SAFE", 6 | "maja_xml" : "", 7 | "dem_path" : "/datalake/watcal/auxdata/dem", 8 | "data_root" : '/datalake/watcal/GRS/grsdata', 9 | "tmp_dir" : '/tmp', 10 | "aerosol": "cds_forecast", 11 | "cams_folder" : '/datalake/watcal/ECMWF/CAMS', 12 | "chain_version" : 1.0, 13 | "product_counter" : 003, 14 | "level" : "INFO", # niveau de log : info, debug, warning, error 15 | "output_dir": "/datalake/watcal/OUTPUT_TEST", # répertoire où seront ecrits les produits générés 16 | "outfile": "", # by default, +-=input_file 17 | "hymotep_config":"/app/grs/exe/hymotep_config.yml", 18 | #"hymotep_config":"./exe/hymotep_config.yml", 19 | "logfile": "/tmp/log.txt", 20 | "shapefile": "", 21 | #"shapefile": "../test/data/shape/SPO04.shp", 22 | "waterdetect_file" : "", 23 | "waterdetect_only": "", 24 | "noclobber":True, 25 | "xblock":512, 26 | "yblock":512, 27 | "activate_dem":True 28 | } 29 | -------------------------------------------------------------------------------- /exe/grs_call.py: -------------------------------------------------------------------------------- 1 | 2 | import sys 3 | 4 | def grs_call(self,p): 5 | args,fjunk = p 6 | for arg in args: 7 | file_tbp, outfile, wkt, altitude, aerosol, aeronet_file, resolution, \ 8 | aot550, angstrom, unzip, untar, startrow, angleonly = arg 9 | print('yop',file_tbp) 10 | #return 11 | try: 12 | from grs import grs_process 13 | grs_process.Process().execute(file_tbp, outfile, wkt, altitude=altitude, aerosol=aerosol, 14 | dem=None, aeronet_file=aeronet_file, resolution=resolution, 15 | aot550=aot550, angstrom=angstrom, unzip=unzip, untar=untar, 16 | startrow=startrow, angleonly=angleonly) 17 | except: 18 | print('-------------------------------') 19 | print('error for file ', file_tbp, ' skip') 20 | print('-------------------------------') 21 | with open(fjunk, "a") as myfile: 22 | myfile.write(file_tbp + ' error during grs \n') 23 | continue 24 | # here sys.exit instead of "return" to terminate and close snappy and free memory 25 | #sys.exit() 26 | return 27 | 28 | if __name__ == '__main__': 29 | p = sys.argv[1] 30 | grs_call(p) -------------------------------------------------------------------------------- /exe/hymotep_config.yml: -------------------------------------------------------------------------------- 1 | { 2 | "altitude": 0, 3 | "aeronet_file": "no", 4 | "resolution": 20, 5 | "aot550": 0.1, 6 | "angstrom": 0.5, 7 | "startrow": 0, 8 | "dem": True, 9 | "memory_safe" : False, 10 | "angleonly": False, 11 | "grs_a": False, 12 | "output": "Rrs", 13 | } 14 | -------------------------------------------------------------------------------- /exe/list/List_images_grs_template.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,Latitude,Longitude,extent_NS (km),extent_EW (km),altitude (m),resolution (m) 2 | 0,petrol,2019-03-11,2019-03-11,S2,29TPM,46.53,-7.02,5,5,0,10 3 | 0,townsville,,,S2,,-19.02,146.8,5,5,0,10 4 | 0,,2015-03-01,2019-01-31,S2,31TFL,45.52,4.84,100,10,175,10 5 | 0,,2015-03-01,2019-01-31,S2,31TFJ,43.76,4.91,100,100,20,20 6 | 0,,2015-03-01,2019-01-31,S2,31TGL,,,,,, 7 | 0,,2015-03-01,2019-01-31,S2,31UDQ,,,,,, 8 | 0,,2017-07-01,2017-07-31,S2,20MPD,-1.43,-61.62,100,100,15,20 9 | 0,,2015-03-01,2019-01-31,S2,20MQB,-3.2,-60.7,100,100,15,20 10 | 0,,2015-03-01,2019-01-31,S2,18LXP,,,,,, 11 | 0,,2015-03-01,2019-01-31,S2,20LLR,-8.907,-64.023,31,33,82,20 12 | 0,,2015-03-01,2019-01-31,S2,20MQC,-2.4,-60.9,55,40,15,20 13 | 0,,2015-03-01,2019-01-31,S2,21MXT,,,,,, 14 | 0,,2015-03-01,2019-01-31,S2,47QRC,,,,,, 15 | 0,,2015-03-01,2019-01-31,S2,48QTH,,,,,, 16 | 0,,2015-03-01,2019-01-31,S2,31PCR,,,,,, 17 | 0,,2015-03-01,2019-01-31,S2,31PDQ,,,,,, 18 | 0,,2015-03-01,2019-01-31,S2,37PGN,11.5,41.64,20,15,342,20 19 | 0,,2019-01-01,2019-07-31,S2,23KNV,-18.6,-44.5,100,100,680,20 20 | 0,,2019-01-01,2019-07-31,S2,23KNU,-19.5,-44.5,100,100,720,20 21 | 0,,2019-01-01,2019-07-31,S2,23KNT,-20.5,-44.5,100,100,750,20 22 | 0,fragozo,2015-03-01,2019-04-30,S2,24LZQ,-9.66,-35.85,15,11,2,10 23 | 0,fortaleza,2019-06-30,2019-12-17,S2,24MWA,-4.16,-38.58,60,50,32,20 24 | 0,castanhao,2019-06-30,2019-12-17,S2,24MWU,-5.58,-38.53,30,40,80,20 25 | 0,castanhao,2019-06-30,2019-12-17,S2,24MVU,-6.24,-39.11,30,50,193,20 26 | 1,jirau,2019-06-30,2020-03-31,S2,20LKQ,-9.6,-65.07,70,85,100,20 27 | 1,jirau,2019-06-30,2020-03-31,S2,20LLQ,-9.22,-64.58,40,58,80,20 28 | 0,chad,2015-03-01,2019-07-31,S2,33PVQ,13.08,14.5,60,80,280,20 29 | 0,manaus,2019-08-30,2019-09-01,S2,20MRB,-3.16,-59.8,40,100,26,20 30 | 0,oros,2018-07-30,2018-08-17,S2,24MVU,-6.25,-39.05,36,60,193,20 31 | -------------------------------------------------------------------------------- /exe/list/construct_list.py: -------------------------------------------------------------------------------- 1 | import os 2 | import glob 3 | import pandas as pd 4 | opj = os.path.join 5 | idir="/datalake/watcal/L8-L1-C2" 6 | start_date='2013-01-01' 7 | end_date='2022-09-01' 8 | 9 | files = glob.glob(opj(idir,'*','*','*','*','*.tar')) 10 | files = pd.DataFrame({"file":files}) 11 | files = files.file.str.split('/',expand=True).iloc[:,1:] 12 | files.columns=['rep1','rep2','sat','tile','year','mm','dd','file'] 13 | 14 | tiles = files.tile.unique() 15 | list = pd.DataFrame({'process (yes if 1)':1, 16 | 'Site Name':'_v15','start_date':start_date,'end_date':end_date,'satellite':'landsat','tile':tiles,'resolution (m)':30,'flag':1 17 | 18 | }) 19 | list.to_csv('./exe/list/list_landsat_jegou.csv',index=False) -------------------------------------------------------------------------------- /exe/list/gernez_list_grs_dec_2021.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 1,Aphanizomenon_Sweden,2015-08-12,2015-08-14,S2,33VXF,20,0 3 | 1,Aphanizomenon_Sweden,2015-08-12,2015-08-14,S2,33VXE,20,0 4 | 1,Aphanizomenon_Sweden,2016-07-24,2016-07-26,S2,33VXF,20,0 5 | 1,Aphanizomenon_Sweden,2016-07-24,2016-07-26,S2,33VXE,20,0 6 | 1,Aphanizomenon_Sweden,2016-07-24,2016-07-26,S2,34VEL,20,0 7 | 1,Aphanizomenon_Sweden,2019-07-25,2019-07-27,S2,33VXF,20,0 8 | 1,Aphanizomenon_Sweden,2019-07-25,2019-07-27,S2,33VXE,20,0 9 | 1,Asterionellopsis_France,2017-06-17,2017-07-18,S2,31UCR,20,0 10 | 1,Asterionellopsis_France,2018-06-02,2018-07-04,S2,31UCR,20,0 11 | 1,Aureococcus_NYC,2016-07-16,2016-07-28,S2,18TWL,20,1 12 | 1,Aureococcus_NYC,2016-07-16,2016-07-28,S2,18TXL,20,1 13 | 1,Chaetoceros_Seine,2016-07-05,2016-07-17,S2,30UYA,20,1 14 | 1,Chaetoceros_Seine,2016-07-05,2016-07-17,S2,30UYV,20,1 15 | 1,Chaetoceros_Somme,2016-08-31,2016-09-02,S2,31UCR,20,1 16 | 1,Ceratium_Puget,2017-07-29,2017-07-31,S2,10TET,20,1 17 | 1,Ceratium_Puget,2018-07-14,2018-07-16,S2,10TET,20,1 18 | 1,Cylindrotheca_Thau,2018-07-31,2018-09-01,S2,31TEJ,20,1 19 | 1,Gonyaulax_Vancouver,2018-07-14,2018-07-26,S2,10UEV,20,1 20 | 1,Gonyaulax_Vancouver,2018-07-14,2018-07-26,S2,10UEU,20,1 21 | 1,Leptocylindrus_Seine,2016-08-14,2016-09-15,S2,30UYA,20,1 22 | 1,Leptocylindrus_Seine,2016-08-14,2016-09-15,S2,30UYV,20,1 23 | 1,Leptocylindrus_Seine,2016-08-11,2016-08-23,S2,30UXV,20,1 24 | 1,Leptocylindrus_Seine,2020-06-21,2020-06-23,S2,30UYA,20,1 25 | 1,Leptocylindrus_Seine,2020-06-21,2020-06-23,S2,30UYV,20,1 26 | 1,Leptocylindrus_Seine,2020-06-21,2020-06-23,S2,30UXV,20,1 27 | 1,Leptocylindrus_Loire,2017-10-03,2017-10-05,S2,30TWT,20,1 28 | 1,Lingulodinium_Loire,2021-07-01,2021-09-31,S2,30TWT,20,1 29 | 1,Lingulodinium_Lorient,2021-07-01,2021-09-31,S2,30TVT,20,1 30 | 1,Margalefidinium_Korea,2019-09-18,2019-09-20,S2,52SCD,20,1 31 | 1,Margalefidinium_China,2015-09-02,2015-09-04,S2,50SNH,20,1 32 | 1,Margalefidinium_China,2015-09-02,2015-09-04,S2,50SNJ,20,1 33 | 1,Mesodinium_Aberdeen,2019-05-30,2019-06-01,S2,10TDT,20,1 34 | 1,Noctiluca_Vigo,2021-09-03,2021-09-05,S2,29TNG,20,1 35 | 1,Noctiluca_Canaria,2016-04-12,2016-04-14,S2,28RDR,20,1 36 | 1,Noctiluca_Oman,2019-03-10,2019-03-15,S2,41QKF,20,1 37 | 1,Noctiluca_Oman,2019-03-10,2019-03-15,S2,41QKG,20,1 38 | 1,Noctiluca_Oman,2019-03-10,2019-03-15,S2,41QLF,20,1 39 | 1,Noctiluca_China,2019-06-07,2019-06-09,S2,51STU,20,0 40 | 1,Noctiluca_China,2019-06-07,2019-06-09,S2,51SUU,20,0 41 | 1,Noctiluca_China,2019-06-07,2019-06-09,S2,51SUT,20,0 42 | 1,Noctiluca_China,2019-06-07,2019-06-09,S2,51STT,20,0 43 | 1,Noctiluca_Puget,2019-05-10,2019-05-12,S2,10TET,20,1 44 | 1,Nodularia_Baltic,2019-07-19,2019-07-21,S2,34VCJ,20,0 45 | 1,Nodularia_Baltic,2019-07-19,2019-07-21,S2,34VDH,20,0 46 | 1,Phaeocystis_Somme,2016-04-13,2016-04-15,S2,31UCR,20,1 47 | 1,Phaeocystis_Somme,2018-05-30,2018-06-04,S2,31UCR,20,1 48 | 1,Phaeocystis_Somme,2019-05-13,2019-05-17,S2,31UCR,20,1 49 | 1,Protoceratium_Puget,2020-07-25,2020-08-06,S2,10TET,20,1 50 | 1,Pseudochattonella_Peru,2016-03-04,2016-03-06,S2,18GXU,20,1 51 | 1,Pseudonitzschia_Vilaine,2020-07-29,2020-08-05,S2,30TWT,20,1 52 | 1,Pseudonitzschia_Vilaine,2019-05-21,2019-05-23,S2,30UUU,20,1 53 | 1,Pseudonitzschia_Berre,2020-10-23,2020-10-25,S2,31TFJ,20,1 54 | 1,Trichodesmium_Guadeloupe,2018-03-17,2018-03-19,S2,20PPC,20,0 55 | 1,Trichodesmium_Australia,2018-06-02,2018-06-04,S2,55KDU,20,0 56 | 1,Trichodesmium_Australia,2018-06-02,2018-06-04,S2,55KDV,20,0 57 | 1,Trichodesmium_Australia,2018-03-10,2018-03-12,S2,56HLJ,20,0 58 | 1,Trichodesmium_Australia,2018-03-10,2018-03-12,S2,56HLH,20,0 59 | -------------------------------------------------------------------------------- /exe/list/list_grs_cnes_gernez.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,Latitude,Longitude,resolution (m) 2 | 0,Malaga,2018-07-12,2018-07-29,S2,30SUF,36.33,-4.6,20 3 | 1,Vilaine,2019-07-20,2019-07-22,S2,30TWT,47.3,-2.55,20 4 | 1,Oleron,2020-04-10,2020-04-19,S2,30TXR,46.86,-1.34,20 5 | 1,Olonne,2020-04-02,2020-04-04,S2,30TWS,46.44,-2.1,20 6 | 1,Olonne,2020-04-10,2020-04-12,S2,30TWS,46.44,-2.1,20 7 | 1,Vie,2021-03-21,2021-04-02,S2,30TWS,46.69,-2.1,20 8 | 1,Loire,2021-03-21,2021-04-02,S2,30TWT,47.15,-2.5,20 9 | 1,Prorocentrum_record,2018-11-03,2018-12-29,S2,18SVH,38.39,-75.13,20 10 | 1,Phaeocystis_Scheldt,2019-05-15,2019-05-17,S2,31UET,51.54,3.4,20 11 | 1,Lingulodinium_Tavira,2019-07-15,2019-07-17,S2,29SPB,37.1,-7.35,20 12 | 1,Lingulodinium_Capetown,2019-02-24,2019-02-26,S2,34HBH,-34.25,18.66,20 13 | 1,Karenia_Venice,2018-07-27,2018-07-29,S2,17RLK,26.9,-82.5,20 14 | 1,Seine,2018-08-19,2018-08-21,S2,30UYV,49.38,-0.1,20 15 | 1,Seine,2019-08-29,2019-08-31,S2,30UYV,49.45,-0.12,20 16 | 1,Berre,2018-08-15,2018-08-17,S2,31TFJ,43.45,5.11,20 17 | 1,Berre,2018-09-19,2018-09-21,S2,31TFJ,43.45,5.11,20 18 | 1,Baltic_Cyano,2015-08-06,2015-08-08,S2,34VDK,57.9,20.8,20 19 | 1,Baltic_Cyano,2019-07-19,2019-07-21,S2,34VCJ,57.25,18.43,20 20 | 1,Trichodesmium_Australia,2018-06-02,2018-06-04,S2,55KDV,-18.85,146.64,20 21 | 1,Trichodesmium_Corinthe,2019-11-24,2019-11-26,S2,34SFH,37.88,23.15,20 22 | 1,Chahabar,2016-11-18,2016-11-20,S2,41RKJ,25.25,60.55,20 23 | 1,Chahabar,2016-12-08,2016-12-10,S2,41RKJ,25.25,60.55,20 24 | 1,Chahabar,2017-10-24,2017-10-26,S2,41RKJ,25.25,60.55,20 25 | 1,Chahabar,2017-11-18,2017-11-30,S2,41RKJ,25.25,60.55,20 26 | 1,Chahabar,2017-12-23,2018-01-14,S2,41RKJ,25.25,60.55,20 27 | 1,Chahabar,2018-03-28,2018-03-30,S2,41RKJ,25.25,60.55,20 28 | 1,Chahabar,2018-12-03,2016-12-15,S2,41RKJ,25.25,60.55,20 29 | 1,Chahabar,2021-03-27,2021-04-08,S2,41RKJ,25.25,60.55,20 30 | -------------------------------------------------------------------------------- /exe/list/list_grs_cnes_gernez_tmp.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,Latitude,Longitude,resolution (m) 2 | 1,Chahabar,2017-10-24,2017-10-26,S2,41RKJ,25.25,60.55,20 3 | 1,Chahabar,2017-11-18,2017-11-30,S2,41RKJ,25.25,60.55,20 4 | 1,Chahabar,2017-12-23,2018-01-14,S2,41RKJ,25.25,60.55,20 5 | 1,Chahabar,2018-03-28,2018-03-30,S2,41RKJ,25.25,60.55,20 6 | 1,Chahabar,2018-12-03,2016-12-15,S2,41RKJ,25.25,60.55,20 7 | 1,Chahabar,2021-03-27,2021-04-08,S2,41RKJ,25.25,60.55,20 -------------------------------------------------------------------------------- /exe/list/list_grs_cnes_guimaraes.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,Latitude,Longitude,resolution (m) 2 | 1,JIRAU_T20LKQ,2018-01-01,2021-04-30,S2,20LKQ,-9.535,-65.233,20 3 | 1,JIRAU_T20LLQ,2018-01-01,2021-04-30,S2,20LLQ,-9.54,-64.322,20 4 | -------------------------------------------------------------------------------- /exe/list/list_grs_cnes_guimaraes2016-2017.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,Latitude,Longitude,resolution (m) 2 | 1,JIRAU_T20LKQ,2016-01-01,2017-12-31,S2,20LKQ,-9.535,-65.233,20 3 | 1,JIRAU_T20LLQ,2016-01-01,2017-12-31,S2,20LLQ,-9.54,-64.322,20 4 | -------------------------------------------------------------------------------- /exe/list/list_grs_cnes_hafeez.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,Latitude,Longitude,resolution (m) 2 | 1,hongkong,2021-03-01,2021-03-31,S2,49QHE,,,20 3 | 1,hongkong,2021-03-01,2021-03-31,S2,49QHF,,,20 4 | 0,hongkong,2021-05-01,2021-05-31,S2,49QHE,,,20 -------------------------------------------------------------------------------- /exe/list/list_grs_cnes_mauricio.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,Latitude,Longitude,resolution (m) 2 | 1,,2020-01-01,2020-12-31,S2,22KGV,,,20 3 | 0,,2021-01-01,2021-09-15,S2,22KGV,,,20 -------------------------------------------------------------------------------- /exe/list/list_grs_cnes_obs2mod.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 1,_v20,2021-12-01,2021-12-31,S2,31TFJ,20,1 3 | 0,_v20,2023-01-01,2023-06-21,S2,31TEJ,20,1 4 | 0,_v20,2020-01-01,2020-12-31,S2,31NDH,20,1 5 | 0,_v20,2021-01-01,2021-12-31,S2,31NDH,20,1 6 | 0,_v20,2022-01-01,2022-12-31,S2,31NDH,20,1 -------------------------------------------------------------------------------- /exe/list/list_grs_cnes_seine.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 0,_v13,2019-01-01,2019-12-31,S2,31UCQ,20,1 3 | 0,_v13,2019-01-01,2019-12-31,S2,31UDQ,20,1 4 | 0,_v13,2019-01-01,2019-12-31,S2,31UFP,20,1 5 | 1,_v13,2018-01-01,2018-12-31,S2,31UEP,20,1 6 | 1,_v13,2019-01-01,2019-12-31,S2,31UEP,20,1 7 | 1,_v13,2020-01-01,2020-12-31,S2,31UEP,20,1 8 | 0,_v13,2019-01-01,2019-12-31,S2,31UDP,20,1 -------------------------------------------------------------------------------- /exe/list/list_grs_cnes_srikanta.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,Latitude,Longitude,resolution (m) 2 | 1,Greece,2018-06-06,2018-06-09,S2,35SMD,39.2,26.4,20 3 | 1,Greece,2019-04-18,2019-06-07,S2,35SMD,39.2,26.4,20 4 | 0,Greece,2021-06-11,2021-06-11,S2,35SMD,39.2,26.4,20 5 | 1,Cyprus,2018-12-15,2018-12-15,S2,36SWD,34.7,33.6,20 6 | -------------------------------------------------------------------------------- /exe/list/list_grs_cnes_template.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,Latitude,Longitude,resolution (m) 2 | 0,chad,2020-01-01,2020-12-31,S2,33PUR,13.9,13.6,20 3 | 0,chad,2020-01-01,2020-12-31,S2,33PUQ,13,13.6,20 4 | 0,chad,2020-01-01,2020-12-31,S2,33PVQ,13.08,14.6,20 5 | 0,chad,2020-01-01,2020-12-31,S2,33PVP,12.1,14.6,20 6 | 0,chad,2020-01-01,2020-12-31,S2,33PWQ,13,15.5,20 7 | 0,chad,2020-01-01,2020-12-31,S2,33PWP,12.17,15.5,20 8 | 1,chad,2020-01-01,2020-12-31,S2,33PVR,14,14.6,20 9 | 0,chad,2020-01-01,2020-12-31,S2,33PXM,10.35,16.4,20 10 | 0,chad,2020-01-01,2020-12-31,S2,33PZL,9.44,18.23,20 11 | 0,chad,2020-01-01,2020-12-31,S2,33NWJ,7.64,15.5,20 12 | 0,chad,2020-01-01,2020-12-31,S2,33PXK,8.54,16.4,20 13 | 0,chad,2020-01-01,2020-12-31,S2,33PWM,10.35,15.5,20 14 | 0,chad,2020-01-01,2020-12-31,S2,33PXK,8.55,16.4,20 15 | 0,chad,2020-01-01,2020-12-31,S2,33PVQ,13.08,14.6,20 16 | -------------------------------------------------------------------------------- /exe/list/list_grs_cordoba.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 1,_v15,2021-01-01,2022-07-05,S2,20JLL,20,1 3 | 1,_v15,2020-01-01,2020-12-31,S2,20JLL,20,1 4 | 1,_v15,2019-01-01,2019-12-31,S2,20JLL,20,1 5 | 6 | -------------------------------------------------------------------------------- /exe/list/list_grs_gernez_feb2022.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 0,Asterionellopsis_France,2018-06-02,2018-07-04,S2,31UCR,20,0 3 | 0,Gonyaulax_Vancouver,2018-07-14,2018-07-26,S2,10UEV,20,1 4 | 0,Gonyaulax_Vancouver,2018-07-14,2018-07-26,S2,10UEU,20,1 5 | 0,Karenia_Saga,2019-07-30,2019-08-01,S2,52SFB,20,1 6 | 0,Margalefidinium_Korea,2019-09-16,2019-09-18,S2,52SCD,20,1 7 | 0,Cyano_Philippines,2020-09-03,2020-09-20,S2,51PUR,20,0 8 | 0,Noctiluca_Oman,2019-03-10,2019-03-15,S2,41QKF,20,0 9 | 0,Noctiluca_Oman,2019-03-10,2019-03-15,S2,41QKG,20,0 10 | 0,Noctiluca_Oman,2019-03-10,2019-03-15,S2,41QLF,20,0 11 | 0,Protoceratium_Puget,2020-07-25,2020-08-06,S2,10TET,20,1 12 | 0,Protoceratium_Puget,2020-07-25,2020-08-06,S2,10UEU,20,1 13 | 0,Aphanizomenon_Salvador,2019-02-15,2019-02-17,S2,16PCV,20,0 14 | 0,Nodularia_Poland,2020-06-25,2020-06-27,S2,34UCF,20,0 15 | 0,Nodularia_Poland,2018-07-24,2018-07-27,S2,34UCF,20,0 16 | 0,Nodularia_Sweden,2018-07-30,2018-08-01,S2,33VXF,20,0 17 | 0,Nodularia_Sweden,2018-07-30,2018-08-01,S2,33VXE,20,0 18 | 0,Trichodesmium_Brazil,2019-01-29,2019-02-02,S2,22JGR,20,0 19 | 0,Trichodesmium_Brazil,2019-01-29,2019-02-02,S2,22JGS,20,0 20 | 0,Trichodesmium_Syndey,2018-03-16,2018-03-18,S2,56HLJ,20,0 21 | 0,Trichodesmium_Fidji,2018-11-24,2018-11-29,S2,60KXF,20,0 22 | 0,Microcystis_Brazil,2019-02-26,2019-03-02,S2,22JDL,20,0 23 | 0,Microcystis_Lake_Taihu,2019-05-03,2019-05-30,S2,51RTQ,20,0 24 | 0,Heterosigma_Chili,2021-03-25,2021-04-04,S2,18GYU,20,1 25 | 1,Planktothryx_Italy,2022-04-01,2022-04-30,S2,33TVF,20,1 26 | -------------------------------------------------------------------------------- /exe/list/list_grs_gernez_juillet_2021.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,Latitude,Longitude,resolution (m) 2 | 1,Heterosigma_Chili,2021-03-27,2021-04-04,S2,18GYU,-42.45,-72.43,20 3 | 1,Cyano_Philippines,2020-09-03,2020-09-30,S2,51PUR,14.37,121.3,20 4 | 1,Cyano_Philippines,2020-09-03,2020-09-30,S2,51PUS,14.37,121.3,20 5 | 1,Margalefidinium_Korea,2018-07-28,2018-07-30,S2,52SCD,34.5,127.5,20 6 | 1,Margalefidinium_Korea,2019-09-18,2019-09-20,S2,52SCD,34.5,127.5,20 7 | 1,Pseudochattonella_Peru,2016-03-04,2016-03-06,S2,18GXU,-42,-73.25,20 8 | 1,Margalefidinium_China,2015-09-20,2015-09-04,S2,50SNH,38.9,117.9,20 9 | 1,Margalefidinium_China,2015-09-20,2015-09-04,S2,50SNJ,38.9,117.9,20 10 | 1,Anabaenopsis_Menorca,2018-06-18,2018-07-30,S2,31TFE,39.95,4.25,20 11 | 1,Aphanizomenon_Salvador,2019-02-15,2019-02-17,S2,16PCV,13.1,-88.6,20 12 | 1,Phaeocystis_Salvador_Nicaragua,2018-01-11,2018-01-23,S2,16PDV,13.35,-87.8,20 13 | 1,Noctiluca_Greece,2019-03-24,2019-03-26,S2,34TFK,40.6,22.89,20 14 | 1,Margalefidinium_Salvador,2017-07-05,2017-07-07,S2,16PBV,13.4,-89.2,20 15 | 1,Karenia_Seto,2018-08-01,2018-08-13,S2,53SLT,34.36,133.4,20 16 | 1,Karenia_Seto,2018-08-01,2018-08-13,S2,53SLU,34.36,133.4,20 17 | 1,Karenia_Saga,2019-07-30,2019-08-01,S2,52SFB,33,130.4,20 18 | 1,Mesodinium_Baja,2017-03-12,2017-03-14,S2,12RWP,25.99,-110.16,20 19 | 1,Mesodinium_Baja,2017-03-12,2017-03-14,S2,12RXP,25.99,-110.16,20 20 | 1,Gymnodinium_Baja,2018-04-17,2018-04-19,S2,11RQQ,31.15,-114.22,20 21 | 1,Nodularia_Poland,2018-07-24,2018-08-01,S2,34UCF,54.5,18.8,20 22 | 1,Nodularia_Poland,2020-06-25,2020-06-27,S2,34UCF,54.5,18.8,20 23 | 1,Nodularia_Poland,2020-08-14,2020-08-16,S2,34UCF,54.5,18.8,20 24 | 1,Nodularia_Finland,2018-07-26,2018-07-28,S2,34VEL,59.5,21.6,20 25 | 1,Gonyaulax_South_Africa,2017-01-30,2017-02-01,S2,34HBH,18.65,-34.25,20 26 | 1,Aphanizomenon_Sweden,2015-08-12,2015-08-14,S2,33VXF,58.7,17.7,20 27 | 1,Aphanizomenon_Sweden,2015-08-12,2015-08-14,S2,33VXE,57.9,17.5,20 28 | 1,Aphanizomenon_Sweden,2016-07-24,2016-07-26,S2,33VXF,58.7,17.7,20 29 | 1,Aphanizomenon_Sweden,2016-07-24,2016-07-26,S2,33VXE,57.9,17.5,20 30 | 1,Aphanizomenon_Sweden,2016-07-24,2016-07-26,S2,34VEL,59.3,22.,20 31 | 1,Anabaena_Sweden,2017-07-17,2017-07-19,S2,33VXF,58.7,17.7,20 32 | 1,Anabaena_Sweden,2017-07-17,2017-07-19,S2,33VXE,57.9,17.5,20 33 | 1,Nodularia_Sweden,2018-07-30,2018-08-01,S2,33VXF,58.7,17.7,20 34 | 1,Nodularia_Sweden,2018-07-30,2018-08-01,S2,33VXE,57.9,17.5,20 35 | 1,Aphanizomenon_Sweden,2019-07-25,2019-07-27,S2,33VXF,58.7,17.7,20 36 | 1,Aphanizomenon_Sweden,2019-07-25,2019-07-27,S2,33VXE,57.9,17.5,20 37 | 1,Margalefidinium_Chesapeake,2017-08-03,2017-08-05,S2,18SUG,37.28,-76.55,20 38 | 1,Margalefidinium_Chesapeake,2020-08-07,2020-08-09,S2,18SUG,37.45,-76.65,20 39 | 1,Margalefidinium_Chesapeake,2020-09-06,2020-09-08,S2,18SUG,37.45,-76.65,20 40 | 1,Aureococcus_NYC,2016-07-16,2017-07-28,S2,18TWL,40.82,-73.77,20 41 | 1,Aureococcus_NYC,2016-07-26,2017-07-28,S2,18TXL,40.89,-73.7,20 42 | 1,Trichodesmium_Guadeloupe,2018-03-27,2018-03-29,S2,20PPC,15.95,-61.82,20 43 | 1,Heterosigma_Peru,2017-03-01,2017-03-13,S2,18LTM,-12.4,-77.08,20 44 | 1,Margalefidinium_Chili,2020-03-25,2020-04-09,S2,18GXS,-43.84,-73.48,20 45 | -------------------------------------------------------------------------------- /exe/list/list_grs_gernez_sep2021.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 1,Heterosigma_Chili,2021-03-27,2021-04-04,S2,18GYU,20,1 3 | 1,Cyano_Philippines,2020-09-03,2020-09-30,S2,51PUR,20,0 4 | 1,Cyano_Philippines,2020-09-03,2020-09-30,S2,51PUS,20,0 5 | 1,Margalefidinium_Korea,2018-07-28,2018-07-30,S2,52SCD,20,1 6 | 1,Margalefidinium_Korea,2019-09-18,2019-09-20,S2,52SCD,20,1 7 | 1,Pseudochattonella_Peru,2016-03-04,2016-03-06,S2,18GXU,20,1 8 | 1,Margalefidinium_China,2015-09-02,2015-09-04,S2,50SNH,20,1 9 | 1,Margalefidinium_China,2015-09-02,2015-09-04,S2,50SNJ,20,1 10 | 1,Anabaenopsis_Menorca,2018-06-18,2018-07-30,S2,31TFE,20,1 11 | 1,Aphanizomenon_Salvador,2019-02-15,2019-02-17,S2,16PCV,20,0 12 | 1,Phaeocystis_Salvador_Nicaragua,2018-01-11,2018-01-23,S2,16PDV,20,1 13 | 1,Noctiluca_Greece,2019-03-24,2019-03-26,S2,34TFK,20,1 14 | 1,Margalefidinium_Salvador,2017-07-05,2017-07-07,S2,16PBV,20,1 15 | 1,Karenia_Seto,2018-08-01,2018-08-13,S2,53SLT,20,1 16 | 1,Karenia_Seto,2018-08-01,2018-08-13,S2,53SLU,20,1 17 | 1,Karenia_Saga,2019-07-30,2019-08-01,S2,52SFB,20,1 18 | 1,Mesodinium_Baja,2017-03-12,2017-03-14,S2,12RWP,20,1 19 | 1,Mesodinium_Baja,2017-03-12,2017-03-14,S2,12RXP,20,1 20 | 1,Gymnodinium_Baja,2018-04-17,2018-04-19,S2,11RQQ,20,1 21 | 1,Nodularia_Poland,2018-07-24,2018-08-01,S2,34UCF,20,0 22 | 1,Nodularia_Poland,2020-06-25,2020-06-27,S2,34UCF,20,0 23 | 1,Nodularia_Poland,2020-08-14,2020-08-16,S2,34UCF,20,0 24 | 1,Nodularia_Finland,2018-07-26,2018-07-28,S2,34VEL,20,0 25 | 1,Gonyaulax_South_Africa,2017-01-30,2017-02-01,S2,34HBH,20,1 26 | 1,Aphanizomenon_Sweden,2015-08-12,2015-08-14,S2,33VXF,20,0 27 | 1,Aphanizomenon_Sweden,2015-08-12,2015-08-14,S2,33VXE,20,0 28 | 1,Aphanizomenon_Sweden,2016-07-24,2016-07-26,S2,33VXF,20,0 29 | 1,Aphanizomenon_Sweden,2016-07-24,2016-07-26,S2,33VXE,20,0 30 | 1,Aphanizomenon_Sweden,2016-07-24,2016-07-26,S2,34VEL,20,0 31 | 1,Anabaena_Sweden,2017-07-17,2017-07-19,S2,33VXF,20,0 32 | 1,Anabaena_Sweden,2017-07-17,2017-07-19,S2,33VXE,20,0 33 | 1,Nodularia_Sweden,2018-07-30,2018-08-01,S2,33VXF,20,0 34 | 1,Nodularia_Sweden,2018-07-30,2018-08-01,S2,33VXE,20,0 35 | 1,Aphanizomenon_Sweden,2019-07-25,2019-07-27,S2,33VXF,20,0 36 | 1,Aphanizomenon_Sweden,2019-07-25,2019-07-27,S2,33VXE,20,0 37 | 1,Margalefidinium_Chesapeake,2017-08-03,2017-08-05,S2,18SUG,20,1 38 | 1,Margalefidinium_Chesapeake,2020-08-07,2020-08-09,S2,18SUG,20,1 39 | 1,Margalefidinium_Chesapeake,2020-09-06,2020-09-08,S2,18SUG,20,1 40 | 1,Aureococcus_NYC,2016-07-16,2016-07-28,S2,18TWL,20,1 41 | 1,Aureococcus_NYC,2016-07-16,2016-07-28,S2,18TXL,20,1 42 | 1,Margalefidinium_Chili,2020-03-25,2020-04-09,S2,18GXS,20,1 43 | 1,Trichodesmium_Australia,2018-03-17,2018-03-19,S2,20PPC,20,0 44 | 1,Trichodesmium_Guadeloupe,2018-06-02,2018-06-04,S2,55KDU,20,0 45 | 1,Trichodesmium_Guadeloupe,2018-06-02,2018-06-04,S2,55KDV,20,0 46 | 1,Cyano_Baltic,2019-07-19,2019-07-21,S2,34VCJ,20,0 47 | 1,Cyano_Baltic,2019-07-19,2019-07-21,S2,34VDH,20,0 48 | 1,Red_tide_Chesapeake,2021-09-01,2021-09-08,S2,18SUJ,20,1 49 | 1,Lingulodinium_Loire,2021-08-13,2021-09-14,S2,30TWT,20,1 50 | 1,Lingulodinium_Lorient,2021-08-13,2021-09-12,S2,30TVT,20,1 -------------------------------------------------------------------------------- /exe/list/list_grs_jegou.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 1,_v13,2018-01-01,2018-12-31,S2,44WPC,20,1 3 | 1,_v13,2019-01-01,2019-12-31,S2,44WPC,20,1 4 | 1,_v13,2018-01-01,2018-12-31,S2,45WVT,20,1 5 | 1,_v13,2019-01-01,2019-12-31,S2,45WVT,20,1 6 | -------------------------------------------------------------------------------- /exe/list/list_grs_jegou_may2022.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 1,_v14,2015-06-01,2017-01-01,S2,45WVQ,20,1 3 | 1,_v14,2018-01-01,2019-12-31,S2,51XXA,20,1 4 | 1,_v14,2021-05-01,2021-12-31,S2,45WVQ,20,1 5 | 1,_v14,2021-05-01,2021-12-31,S2,51XXA,20,1 6 | 1,_v14,2021-05-01,2021-12-31,S2,57WWS,20,1 7 | 1,_v14,2021-05-01,2021-12-31,S2,51WWQ,20,1 8 | 1,_v14,2021-05-01,2021-12-31,S2,08WNV,20,1 9 | 1,_v14,2021-05-01,2021-12-31,S2,41WPP,20,1 10 | 1,_v14,2021-05-01,2021-12-31,S2,45WVT,20,1 11 | 1,_v14,2021-05-01,2021-12-31,S2,03VXJ,20,1 -------------------------------------------------------------------------------- /exe/list/list_grs_jirau.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 1,v14,2022-01-01,2022-04-31,S2,20LLQ,20,1 3 | 1,v14,2022-01-01,2022-04-31,S2,20LKQ,20,1 4 | 1,v14,2022-01-01,2022-04-31,S2,19LHJ,20,1 5 | 6 | 7 | -------------------------------------------------------------------------------- /exe/list/list_grs_plata_estuary.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 1,_v15,2021-10-01,2022-07-20,S2,21HUC,20,1 3 | 1,_v15,2021-10-01,2022-07-20,S2,21HUB,20,1 4 | 1,_v15,2021-10-01,2022-07-20,S2,21HVC,20,1 5 | 1,_v15,2021-10-01,2022-07-20,S2,21HVB,20,1 6 | 1,_v15,2021-10-01,2022-07-20,S2,21HVA,20,1 7 | 8 | 9 | -------------------------------------------------------------------------------- /exe/list/list_grs_redshade_1.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 1,Aphanizomenon_Sweden,2015-08-12,2015-08-14,S2,33VXF,20,0 3 | 1,Aphanizomenon_Sweden,2015-08-12,2015-08-14,S2,33VXE,20,0 4 | 1,Aphanizomenon_Sweden,2016-07-24,2016-07-26,S2,33VXF,20,0 5 | 1,Aphanizomenon_Sweden,2016-07-24,2016-07-26,S2,33VXE,20,0 6 | 1,Aphanizomenon_Sweden,2016-07-24,2016-07-26,S2,34VEL,20,0 7 | 1,Aphanizomenon_Sweden,2019-07-25,2019-07-27,S2,33VXF,20,0 8 | 1,Aphanizomenon_Sweden,2019-07-25,2019-07-27,S2,33VXE,20,0 9 | 1,Asterionellopsis_France,2017-06-17,2017-07-18,S2,31UCR,20,0 10 | 1,Asterionellopsis_France,2018-06-02,2018-07-04,S2,31UCR,20,0 -------------------------------------------------------------------------------- /exe/list/list_grs_redshade_2.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 0,Aureococcus_NYC,2016-07-16,2016-07-28,S2,18TWL,20,1 3 | 0,Aureococcus_NYC,2016-07-16,2016-07-28,S2,18TXL,20,1 4 | 0,Chaetoceros_Seine,2016-07-05,2016-07-17,S2,30UYA,20,1 5 | 0,Chaetoceros_Seine,2016-07-05,2016-07-17,S2,30UYV,20,1 6 | 0,Chaetoceros_Somme,2016-08-31,2016-09-02,S2,31UCR,20,1 7 | 1,Ceratium_Puget,2017-07-29,2017-07-31,S2,10TET,20,1 8 | 1,Ceratium_Puget,2018-07-14,2018-07-16,S2,10TET,20,1 9 | 1,Cylindrotheca_Thau,2018-07-31,2018-09-01,S2,31TEJ,20,1 10 | 1,Gonyaulax_Vancouver,2018-07-14,2018-07-26,S2,10UEV,20,1 11 | 1,Gonyaulax_Vancouver,2018-07-14,2018-07-26,S2,10UEU,20,1 -------------------------------------------------------------------------------- /exe/list/list_grs_redshade_2022.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 1,Trichodesmium_Guadeloupe,2018-03-27,2018-03-29,S2,20PPC,20,0 3 | 1,Lingulodinium_Loire,2021-07-01,2021-07-16,S2,30TWT,20,1 4 | 1,Ceratium_Puget,2017-07-29,2017-07-31,S2,10TET,20,1 5 | 1,Ceratium_Puget,2018-07-14,2018-07-16,S2,10TET,20,1 6 | 1,Cylindrotheca_Thau,2018-07-31,2018-09-01,S2,31TEJ,20,1 7 | 1,Gonyaulax_Vancouver,2018-07-14,2018-07-26,S2,10UEV,20,1 8 | 1,Gonyaulax_Vancouver,2018-07-14,2018-07-26,S2,10UEU,20,1 9 | 1,Margalefidinium_Korea,2019-09-18,2019-09-20,S2,52SCD,20,1 10 | 1,Margalefidinium_China,2015-09-02,2015-09-04,S2,50SNH,20,1 11 | 1,Margalefidinium_China,2015-09-02,2015-09-04,S2,50SNJ,20,1 12 | 1,Mesodinium_Aberdeen,2019-05-30,2019-06-01,S2,10TDT,20,1 13 | 1,Noctiluca_Vigo,2021-09-03,2021-09-05,S2,29TNG,20,1 14 | 1,Noctiluca_Canaria,2016-04-12,2016-04-14,S2,28RDR,20,1 15 | 1,Noctiluca_Oman,2019-03-10,2019-03-15,S2,41QKF,20,1 16 | 1,Noctiluca_Oman,2019-03-10,2019-03-15,S2,41QKG,20,1 17 | 1,Noctiluca_Oman,2019-03-10,2019-03-15,S2,41QLF,20,1 18 | 1,Noctiluca_China,2019-06-07,2019-06-09,S2,51STT,20,0 19 | 1,Noctiluca_China,2019-06-07,2019-06-09,S2,51SUT,20,0 20 | 1,Noctiluca_Puget,2019-05-10,2019-05-12,S2,10TET,20,1 21 | 1,Phaeocystis_Somme,2016-04-13,2016-04-15,S2,31UCR,20,1 22 | 1,Phaeocystis_Somme,2018-05-30,2018-06-01,S2,31UCR,20,1 23 | 1,Phaeocystis_Somme,2019-05-13,2019-05-17,S2,31UCR,20,1 24 | 1,Protoceratium_Puget,2020-07-25,2020-08-06,S2,10TET,20,1 25 | 1,Pseudonitzschia_Vilaine,2020-07-29,2020-08-05,S2,30TWT,20,1 26 | 1,Pseudonitzschia_Vilaine,2019-05-21,2019-05-23,S2,30UUU,20,1 27 | 1,Pseudonitzschia_Berre,2020-10-23,2020-10-25,S2,31TFJ,20,1 28 | 29 | -------------------------------------------------------------------------------- /exe/list/list_grs_redshade_3.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 1,Leptocylindrus_Seine,2016-08-14,2016-09-15,S2,30UYA,20,1 3 | 1,Leptocylindrus_Seine,2016-08-14,2016-09-15,S2,30UYV,20,1 4 | 1,Leptocylindrus_Seine,2016-08-11,2016-08-23,S2,30UXV,20,1 5 | 1,Leptocylindrus_Seine,2020-06-21,2020-06-23,S2,30UYA,20,1 6 | 1,Leptocylindrus_Seine,2020-06-21,2020-06-23,S2,30UYV,20,1 7 | 1,Leptocylindrus_Seine,2020-06-21,2020-06-23,S2,30UXV,20,1 8 | 1,Leptocylindrus_Loire,2017-10-03,2017-10-05,S2,30TWT,20,1 9 | 1,Lingulodinium_Loire,2021-07-01,2021-09-30,S2,30TWT,20,1 10 | 1,Lingulodinium_Lorient,2021-07-01,2021-09-30,S2,30TVT,20,1 11 | 1,Margalefidinium_Korea,2019-09-18,2019-09-20,S2,52SCD,20,1 12 | 1,Margalefidinium_China,2015-09-02,2015-09-04,S2,50SNH,20,1 13 | 1,Margalefidinium_China,2015-09-02,2015-09-04,S2,50SNJ,20,1 14 | 1,Mesodinium_Aberdeen,2019-05-30,2019-06-01,S2,10TDT,20,1 15 | 1,Noctiluca_Vigo,2021-09-03,2021-09-05,S2,29TNG,20,1 16 | 1,Noctiluca_Canaria,2016-04-12,2016-04-14,S2,28RDR,20,1 17 | 1,Noctiluca_Oman,2019-03-10,2019-03-15,S2,41QKF,20,1 18 | 1,Noctiluca_Oman,2019-03-10,2019-03-15,S2,41QKG,20,1 19 | 1,Noctiluca_Oman,2019-03-10,2019-03-15,S2,41QLF,20,1 -------------------------------------------------------------------------------- /exe/list/list_grs_redshade_4.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 0,Noctiluca_China,2019-06-07,2019-06-09,S2,51STU,20,0 3 | 0,Noctiluca_China,2019-06-07,2019-06-09,S2,51SUU,20,0 4 | 0,Noctiluca_China,2019-06-07,2019-06-09,S2,51SUT,20,0 5 | 0,Noctiluca_China,2019-06-07,2019-06-09,S2,51STT,20,0 6 | 0,Noctiluca_Puget,2019-05-10,2019-05-12,S2,10TET,20,1 7 | 0,Nodularia_Baltic,2019-07-19,2019-07-21,S2,34VCJ,20,0 8 | 0,Nodularia_Baltic,2019-07-19,2019-07-21,S2,34VDH,20,0 9 | 0,Phaeocystis_Somme,2016-04-13,2016-04-15,S2,31UCR,20,1 10 | 0,Phaeocystis_Somme,2018-05-30,2018-06-04,S2,31UCR,20,1 11 | 0,Phaeocystis_Somme,2019-05-13,2019-05-17,S2,31UCR,20,1 12 | 0,Protoceratium_Puget,2020-07-25,2020-08-06,S2,10TET,20,1 13 | 0,Pseudochattonella_Peru,2016-03-04,2016-03-06,S2,18GXU,20,1 14 | 0,Pseudonitzschia_Vilaine,2020-07-29,2020-08-05,S2,30TWT,20,1 15 | 0,Pseudonitzschia_Vilaine,2019-05-21,2019-05-23,S2,30UUU,20,1 16 | 0,Pseudonitzschia_Berre,2020-10-23,2020-10-25,S2,31TFJ,20,1 17 | 0,Trichodesmium_Guadeloupe,2018-03-17,2018-03-19,S2,20PPC,20,0 18 | 0,Trichodesmium_Australia,2018-06-02,2018-06-04,S2,55KDU,20,0 19 | 1,Trichodesmium_Australia,2018-06-02,2018-06-04,S2,55KDV,20,0 20 | 1,Trichodesmium_Australia,2018-03-10,2018-03-12,S2,56HLJ,20,0 21 | 1,Trichodesmium_Australia,2018-03-10,2018-03-12,S2,56HLH,20,0 22 | -------------------------------------------------------------------------------- /exe/list/list_grs_santarem.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 0,_v13,2020-01-01,2020-12-31,S2,21MYT,20,1 3 | 1,_v13,2021-01-01,2021-12-31,S2,21MYT,20,1 4 | 0,_v13,2022-01-01,2022-01-31,S2,21MYT,20,1 5 | 0,_v13,2020-06-20,2022-06-24,S2,17RNK,20,1 6 | 7 | -------------------------------------------------------------------------------- /exe/list/list_grs_ukraine.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 1,_v15,2022-01-01,2022-09-01,S2,36TXT,20,1 3 | 1,_v15,2022-01-01,2022-09-01,S2,36TWT,20,1 4 | 1,_v15,2022-01-01,2022-09-01,S2,36TWS,20,1 5 | 1,_v15,2022-01-01,2022-09-01,S2,36TVS,20,1 6 | 7 | -------------------------------------------------------------------------------- /exe/list/list_grs_unesco_chad_all.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 1,_v15,2016-01-01,2023-05-15,S2,33PUR,20,1 3 | 1,_v15,2016-01-01,2023-05-15,S2,33PUQ,20,1 4 | 1,_v15,2016-01-01,2023-05-15,S2,33PVQ,20,1 5 | 1,_v15,2016-01-01,2023-05-15,S2,33PVP,20,1 6 | 1,_v15,2016-01-01,2023-05-15,S2,33PWQ,20,1 7 | 1,_v15,2016-01-01,2023-05-15,S2,33PWP,20,1 8 | 1,_v15,2016-01-01,2023-05-15,S2,33PVR,20,1 9 | 1,_v15,2016-01-01,2023-05-15,S2,33PUP,20,1 10 | 1,_v15,2016-01-01,2023-05-15,S2,33PTR,20,1 11 | -------------------------------------------------------------------------------- /exe/list/list_grs_unesco_plata_2015.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 1,unesco,2015-01-01,2017-12-31,S2,22KDV,20,1 3 | 1,unesco,2015-01-01,2017-12-31,S2,22KEV,20,1 4 | 1,unesco,2015-01-01,2017-12-31,S2,22KFV,20,1 5 | 1,unesco,2015-01-01,2017-12-31,S2,22KGV,20,1 6 | 7 | -------------------------------------------------------------------------------- /exe/list/list_grs_unesco_plata_2018.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 1,unesco,2018-01-01,2018-12-31,S2,22KDV,20,1 3 | 1,unesco,2018-01-01,2018-12-31,S2,22KEV,20,1 4 | 1,unesco,2018-01-01,2018-12-31,S2,22KFV,20,1 5 | 1,unesco,2018-01-01,2018-12-31,S2,22KGV,20,1 6 | 7 | -------------------------------------------------------------------------------- /exe/list/list_grs_unesco_plata_2019.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 0,unesco,2019-01-01,2019-12-31,S2,22KDV,20,1 3 | 0,unesco,2019-01-01,2019-12-31,S2,22KEV,20,1 4 | 0,unesco,2019-01-01,2019-12-31,S2,22KFV,20,1 5 | 1,unesco,2019-01-01,2019-12-31,S2,22KGV,20,1 6 | 7 | -------------------------------------------------------------------------------- /exe/list/list_grs_unesco_plata_2020.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 1,unesco,2020-01-01,2020-12-31,S2,22KDV,20,1 3 | 1,unesco,2020-01-01,2020-12-31,S2,22KEV,20,1 4 | 1,unesco,2020-01-01,2020-12-31,S2,22KFV,20,1 5 | 1,unesco,2020-01-01,2020-12-31,S2,22KGV,20,1 6 | 7 | -------------------------------------------------------------------------------- /exe/list/list_grs_unesco_plata_2021.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 1,unesco,2021-10-01,2021-12-31,S2,22KDV,20,1 3 | 1,unesco,2021-10-01,2021-12-31,S2,22KEV,20,1 4 | 1,unesco,2021-10-01,2021-12-31,S2,22KFV,20,1 5 | 1,unesco,2021-10-01,2021-12-31,S2,22KGV,20,1 6 | 7 | -------------------------------------------------------------------------------- /exe/list/list_grs_validation_get.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 0,_v15,2021-05-01,2022-08-31,S2,22KGA,20,1 3 | 0,_v15,2021-05-01,2022-08-31,S2,22KHA,20,1 4 | 0,_v15,2021-05-01,2022-08-31,S2,23KKQ,20,1 5 | 0,_v15,2021-05-01,2022-08-31,S2,23KLQ,20,1 6 | 0,_v15,2021-05-01,2022-08-31,S2,23KMQ,20,1 7 | 1,_v15,2021-01-01,2022-07-31,S2,48QTE,20,1 8 | 1,_v15,2021-01-01,2022-07-31,S2,31TFL,20,1 9 | 1,_v15,2021-01-01,2022-07-31,S2,20LLQ,20,1 10 | 1,_v15,2021-01-01,2022-07-31,S2,20LKQ,20,1 11 | 1,_v15,2021-01-01,2022-07-31,S2,20LKP,20,1 12 | 1,_v15,2022-03-01,2022-08-31,S2,19QCB,20,1 -------------------------------------------------------------------------------- /exe/list/list_jegou_jallais.csv: -------------------------------------------------------------------------------- 1 | process,Site.Name,start_date,end_date,satellite,tile,resolution..m.,flag 2 | 0,_v13,2018-03-13,2018-10-27,S2,57WWS,20,1 3 | 0,_v13,2019-02-17,2019-10-27,S2,57WWS,20,1 4 | 0,_v13,2019-03-06,2019-10-27,S2,51WWQ,20,1 5 | 0,_v13,2018-02-09,2018-10-27,S2,08WNV,20,1 6 | 0,_v13,2019-02-12,2019-11-01,S2,08WNV,20,1 7 | 0,_v13,2019-02-19,2019-10-24,S2,45WVT,20,1 8 | 0,_v13,2018-02-24,2018-11-01,S2,41WPP,20,1 9 | 0,_v13,2019-02-09,2019-11-02,S2,41WPP,20,1 10 | 0,_v13,2017-05-02,2017-10-27,S2,57WWS,20,1 11 | 0,_v13,2020-05-11,2020-10-24,S2,57WWS,20,1 12 | 0,_v13,2017-05-06,2017-09-18,S2,08WNV,20,1 13 | 0,_v13,2020-02-12,2020-10-31,S2,08WNV,20,1 14 | 0,_v13,2017-04-14,2017-10-24,S2,51WWQ,20,1 15 | 0,_v13,2018-02-08,2018-11-02,S2,51WWQ,20,1 16 | 0,_v13,2020-05-14,2020-11-02,S2,51WWQ,20,1 17 | 0,_v13,2017-02-19,2017-11-02,S2,41WPP,20,1 18 | 1,_v13,2020-02-05,2020-11-05,S2,41WPP,20,1 19 | 1,_v13,2017-02-27,2017-11-01,S2,45WVT,20,1 20 | 1,_v13,2018-02-24,2018-11-01,S2,45WVT,20,1 21 | 1,_v13,2017-02-09,2017-02-25,S2,41WPQ,20,1 22 | 1,_v13,2017-01-26,2017-11-07,S2,03VXJ,20,1 23 | 1,_v13,2018-01-31,2018-11-17,S2,03VXJ,20,1 24 | 1,_v13,2019-01-23,2019-12-01,S2,03VXJ,20,1 25 | 1,_v13,2020-01-01,2020-11-16,S2,03VXJ,20,1 26 | 1,_v13,2019-10-28,2020-02-16,S2,57WWS,20,1 27 | 1,_v13,2019-10-28,2020-04-28,S2,51WWQ,20,1 28 | 1,_v13,2019-10-25,2020-10-23,S2,45WVT,20,1 29 | -------------------------------------------------------------------------------- /exe/list/list_jegou_jallais2.csv: -------------------------------------------------------------------------------- 1 | process,Site.Name,start_date,end_date,satellite,tile,resolution..m.,flag 2 | 1,_v14,2015-01-01,2016-12-31,S2,57WWS,20,1 3 | 1,_v14,2015-01-01,2016-12-31,S2,51WWQ,20,1 4 | 1,_v14,2015-01-01,2016-12-31,S2,08WNV,20,1 5 | 1,_v14,2015-01-01,2016-12-31,S2,45WVT,20,1 6 | 1,_v14,2015-01-01,2016-12-31,S2,41WPP,20,1 7 | 1,_v14,2015-01-01,2016-12-31,S2,41WPQ,20,1 8 | 1,_v14,2015-01-01,2016-12-31,S2,03VXJ,20,1 9 | -------------------------------------------------------------------------------- /exe/list/list_landsat_jegou.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m),flag 2 | 0,_v15,2013-01-01,2022-08-31,landsat,161227,30,1 3 | 0,_v15,2013-01-01,2022-09-01,landsat,104012,30,1 4 | 0,_v15,2013-01-01,2022-09-01,landsat,133009,30,1 5 | 0,_v15,2013-01-01,2022-09-01,landsat,230231,30,1 6 | 0,_v15,2013-01-01,2022-09-01,landsat,061013,30,1 7 | 0,_v15,2013-01-01,2022-09-01,landsat,178233,30,1 8 | 0,_v15,2013-01-01,2022-09-01,landsat,204231,30,1 9 | 0,_v15,2013-01-01,2022-09-01,landsat,077017,30,1 10 | 0,_v15,2013-01-01,2022-09-01,landsat,063012,30,1 11 | 0,_v15,2013-01-01,2022-09-01,landsat,063013,30,1 12 | 0,_v15,2013-01-01,2022-09-01,landsat,008231,30,1 13 | 0,_v15,2013-01-01,2022-09-01,landsat,130009,30,1 14 | 0,_v15,2013-01-01,2022-09-01,landsat,139231,30,1 15 | 0,_v15,2013-01-01,2022-09-01,landsat,179232,30,1 16 | 0,_v15,2013-01-01,2022-09-01,landsat,206231,30,1 17 | 0,_v15,2013-01-01,2022-09-01,landsat,159227,30,1 18 | 0,_v15,2013-01-01,2022-09-01,landsat,133008,30,1 19 | 0,_v15,2013-01-01,2022-09-01,landsat,153011,30,1 20 | 0,_v15,2013-01-01,2022-09-01,landsat,152013,30,1 21 | 0,_v15,2013-01-01,2022-09-01,landsat,062012,30,1 22 | 0,_v15,2013-01-01,2022-09-01,landsat,129009,30,1 23 | 0,_v15,2013-01-01,2022-09-01,landsat,164013,30,1 24 | 0,_v15,2013-01-01,2022-09-01,landsat,160227,30,1 25 | 0,_v15,2013-01-01,2022-09-01,landsat,154012,30,1 26 | 0,_v15,2013-01-01,2022-09-01,landsat,154011,30,1 27 | 0,_v15,2013-01-01,2022-09-01,landsat,198235,30,1 28 | 0,_v15,2013-01-01,2022-09-01,landsat,226233,30,1 29 | 0,_v15,2013-01-01,2022-09-01,landsat,227233,30,1 30 | 0,_v15,2013-01-01,2022-09-01,landsat,228233,30,1 31 | 0,_v15,2013-01-01,2022-09-01,landsat,106011,30,1 32 | 0,_v15,2013-01-01,2022-09-01,landsat,127013,30,1 33 | 0,_v15,2013-01-01,2022-09-01,landsat,166013,30,1 34 | 0,_v15,2013-01-01,2022-09-01,landsat,132009,30,1 35 | 0,_v15,2013-01-01,2022-09-01,landsat,010231,30,1 36 | 0,_v15,2013-01-01,2022-09-01,landsat,155011,30,1 37 | 0,_v15,2013-01-01,2022-09-01,landsat,129013,30,1 38 | 0,_v15,2013-01-01,2022-09-01,landsat,156011,30,1 39 | 0,_v15,2013-01-01,2022-09-01,landsat,078017,30,1 40 | 0,_v15,2013-01-01,2022-09-01,landsat,153012,30,1 41 | 0,_v15,2013-01-01,2022-09-01,landsat,137232,30,1 42 | 0,_v15,2013-01-01,2022-09-01,landsat,062013,30,1 43 | 0,_v15,2013-01-01,2022-09-01,landsat,138231,30,1 44 | 0,_v15,2013-01-01,2022-09-01,landsat,205231,30,1 45 | 0,_v15,2013-01-01,2022-09-01,landsat,106012,30,1 46 | 0,_v15,2013-01-01,2022-09-01,landsat,159228,30,1 47 | 0,_v15,2013-01-01,2022-09-01,landsat,180232,30,1 48 | 0,_v15,2013-01-01,2022-09-01,landsat,078016,30,1 49 | 0,_v15,2013-01-01,2022-09-01,landsat,009231,30,1 50 | 0,_v15,2013-01-01,2022-09-01,landsat,076017,30,1 51 | 0,_v15,2013-01-01,2022-09-01,landsat,229233,30,1 52 | 0,_v15,2013-01-01,2022-09-01,landsat,151013,30,1 53 | 1,_v15,2013-01-01,2022-09-01,landsat,131009,30,1 54 | 1,_v15,2013-01-01,2022-09-01,landsat,199235,30,1 55 | 1,_v15,2013-01-01,2022-09-01,landsat,153013,30,1 56 | 1,_v15,2013-01-01,2022-09-01,landsat,200235,30,1 57 | 1,_v15,2013-01-01,2022-09-01,landsat,105012,30,1 58 | 1,_v15,2013-01-01,2022-09-01,landsat,138232,30,1 59 | 1,_v15,2013-01-01,2022-09-01,landsat,165013,30,1 60 | 1,_v15,2013-01-01,2022-09-01,landsat,178232,30,1 61 | 1,_v15,2013-01-01,2022-09-01,landsat,197235,30,1 62 | 1,_v15,2013-01-01,2022-09-01,landsat,128013,30,1 63 | -------------------------------------------------------------------------------- /exe/list_validation/info_aeronet_oc_v15.csv: -------------------------------------------------------------------------------- 1 | site,start_date,end_date,number 2 | Abu_Al_Bukhoosh,2015-07-01,2015-07-01,0 3 | ARIAKE_TOWER,2018-04-19,2022-12-13,1830 4 | Bahia_Blanca,2020-01-23,2022-12-13,2362 5 | Banana_River,2022-06-16,2022-12-03,573 6 | Bari_Waterfront,2015-07-01,2015-07-01,0 7 | Blyth_NOAH,2016-04-22,2017-09-19,32 8 | Casablanca_Platform,2019-04-02,2022-12-09,13955 9 | Chesapeake_Bay,2021-10-16,2022-12-13,3137 10 | COVE_SEAPRISM,2015-11-16,2016-01-09,162 11 | Gageocho_Station,2015-07-01,2015-07-01,0 12 | Galata_Platform,2015-02-02,2022-11-22,13174 13 | Gloria,2015-01-01,2019-08-07,7606 14 | Gotland,2015-07-01,2015-07-01,0 15 | GOT_Seaprism,2015-01-01,2016-01-31,99 16 | Grizzly_Bay,2019-03-11,2019-12-09,1220 17 | Gustav_Dalen_Tower,2015-05-27,2022-09-06,6034 18 | HBOI,2015-07-01,2015-07-01,0 19 | Helsinki_Lighthouse,2015-05-16,2019-09-06,1520 20 | Ieodo_Station,2016-10-06,2019-08-16,59 21 | Irbe_Lighthouse,2018-07-24,2022-09-22,3538 22 | KAUST_Campus,2015-07-01,2015-07-01,0 23 | Kemigawa_Offshore,2019-09-02,2022-12-12,1317 24 | Lake_Erie,2016-07-19,2022-10-24,1136 25 | Lake_Okeechobee,2018-08-09,2020-10-26,1408 26 | Lake_Okeechobee_N,2021-01-19,2022-01-05,546 27 | LISCO,2015-01-01,2022-12-13,2553 28 | Lucinda,2015-01-14,2022-09-29,2860 29 | MVCO,2015-01-01,2022-12-13,2000 30 | Palgrunden,2015-05-18,2022-08-22,2748 31 | PLOCAN_Tower,2022-09-04,2022-12-13,935 32 | Sacramento_River,2021-05-18,2022-09-03,115 33 | San_Marco_Platform,2020-10-06,2022-12-13,5802 34 | Section-7_Platform,2019-08-27,2022-11-09,13719 35 | Socheongcho,2015-10-12,2022-12-11,1800 36 | South_Greenbay,2018-07-06,2022-10-09,419 37 | Thornton_C-power,2015-04-09,2018-11-08,1379 38 | USC_SEAPRISM,2015-01-14,2022-12-10,4836 39 | USC_SEAPRISM_2,2015-08-28,2019-09-23,1354 40 | Venise,2015-01-01,2022-11-27,6326 41 | WaveCIS_Site_CSI_6,2015-01-06,2022-04-30,3424 42 | Zeebrugge-MOW1,2015-01-04,2022-05-09,734 43 | -------------------------------------------------------------------------------- /exe/list_validation/list_grs_aeronet_oc.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Name,start_date,end_date,satellite,tile,resolution (m),Flag (1:applied),Latitude,Longitude 2 | 1,Abu_Al_Bukhoosh,2015-07-01,2015-07-01,S2,39RYJ,20,1,25.495,53.145833 3 | 1,ARIAKE_TOWER,2018-04-19,2022-12-13,S2,52SFB,20,1,33.103617,130.27195 4 | 1,Bahia_Blanca,2020-01-23,2022-12-13,S2,20HPB,20,1,-39.148,-61.722 5 | 1,Banana_River,2022-06-16,2022-12-03,S2,17RNM,20,1,28.367,-80.633 6 | 1,Bari_Waterfront,2015-07-01,2015-07-01,S2,33TXF,20,1,41.1168,16.8934 7 | 1,Blyth_NOAH,2016-04-22,2017-09-19,S2,30UWG,20,1,55.146399,-1.420859 8 | 1,Casablanca_Platform,2019-04-02,2022-12-09,S2,31TCF,20,1,40.717,1.358 9 | 1,Chesapeake_Bay,2021-10-16,2022-12-13,S2,18SUJ,20,1,39.124,-76.359 10 | 1,COVE_SEAPRISM,2015-11-16,2016-01-09,S2,18SVF,20,1,36.9,-75.71 11 | 1,Gageocho_Station,2015-07-01,2015-07-01,S2,51SXT,20,1,33.941991,124.592912 12 | 1,Galata_Platform,2015-02-02,2022-11-22,S2,35TNH,20,1,43.044624,28.19319 13 | 1,Gloria,2015-01-01,2019-08-07,S2,35TPK,20,1,44.59997,29.35967 14 | 1,Gotland,2015-07-01,2015-07-01,S2,34VCK,20,1,57.917,18.95 15 | 1,GOT_Seaprism,2015-01-01,2016-01-31,S2,47PQL,20,1,9.2862,101.4123 16 | 1,Grizzly_Bay,2019-03-11,2019-12-09,S2,10SEH,20,1,38.108,-122.056 17 | 1,Gustav_Dalen_Tower,2015-05-27,2022-09-06,S2,33VXF,20,1,58.59417,17.46683 18 | 1,HBOI,2015-07-01,2015-07-01,S2,17RNL,20,1,27.534,-80.357 19 | 1,Helsinki_Lighthouse,2015-05-16,2019-09-06,S2,35VLG,20,1,59.94897,24.92636 20 | 1,Ieodo_Station,2016-10-06,2019-08-16,S2,51SYR,20,1,32.122953,125.182447 21 | 1,Irbe_Lighthouse,2018-07-24,2022-09-22,S2,34VEK,20,1,57.75092,21.72297 22 | 1,KAUST_Campus,2015-07-01,2015-07-01,S2,37QEE,20,1,22.304833,39.102833 23 | 1,Kemigawa_Offshore,2019-09-02,2022-12-12,S2,54SVE,20,1,35.611,140.023 24 | 1,Lake_Erie,2016-07-19,2022-10-24,S2,17TLG,20,1,41.8256,-83.1936 25 | 1,Lake_Okeechobee,2018-08-09,2020-10-26,S2,17RNK,20,1,26.9017,-80.7892 26 | 1,Lake_Okeechobee_N,2021-01-19,2022-01-05,S2,17RNL,20,1,27.139,-80.789 27 | 1,LISCO,2015-01-01,2022-12-13,S2,18TXL,20,1,40.954517,-73.341767 28 | 1,Lucinda,2015-01-14,2022-09-29,S2,55KDV,20,1,-18.5198,146.3861 29 | 1,MVCO,2015-01-01,2022-12-13,S2,19TCF,20,1,41.325,-70.5667 30 | 1,Palgrunden,2015-05-18,2022-08-22,S2,33VUF,20,1,58.755333,13.1515 31 | 1,PLOCAN_Tower,2022-09-04,2022-12-13,S2,28RDS,20,1,28.041,-15.385 32 | 1,Sacramento_River,2021-05-18,2022-09-03,S2,10SEH,20,1,38.05,-121.888 33 | 1,San_Marco_Platform,2020-10-06,2022-12-13,S2,37MFS,20,1,-2.942,40.215 34 | 1,Section-7_Platform,2019-08-27,2022-11-09,S2,35TPK,20,1,44.546,29.447 35 | 1,Socheongcho,2015-10-12,2022-12-11,S2,51SXB,20,1,37.423133,124.738039 36 | 1,South_Greenbay,2018-07-06,2022-10-09,S2,16TDQ,20,1,44.59633,-87.9512 37 | 1,Thornton_C-power,2015-04-09,2018-11-08,S2,31UDT,20,1,51.5325,2.955278 38 | 1,USC_SEAPRISM,2015-01-14,2022-12-10,S2,11SLT,20,1,33.56371,-118.11782 39 | 1,USC_SEAPRISM_2,2015-08-28,2019-09-23,S2,11SLT,20,1,33.56371,-118.11782 40 | 1,Venise,2015-01-01,2022-11-27,S2,32TQR,20,1,45.3139,12.5083 41 | 1,WaveCIS_Site_CSI_6,2015-01-06,2022-04-30,S2,15RYN,20,1,28.866667,-90.483333 42 | 1,Zeebrugge-MOW1,2015-01-04,2022-05-09,S2,31UES,20,1,51.362,3.12 43 | -------------------------------------------------------------------------------- /exe/list_validation/list_grs_aeronet_oc_test.csv: -------------------------------------------------------------------------------- 1 | process (yes if 1),Name,start_date,end_date,satellite,tile,resolution (m),Flag (1:applied),Latitude,Longitude 2 | 1,Bahia_Blanca,2020-01-23,2022-12-13,S2,20HPB,20,1,-39.148,-61.722 3 | 1,Banana_River,2022-06-16,2022-12-03,S2,17RNM,20,1,28.367,-80.633 -------------------------------------------------------------------------------- /exe/sync_rclone_dropbox.sh: -------------------------------------------------------------------------------- 1 | # source ~/set_proxy.sh 2 | # 3 | 4 | module load rclone 5 | 6 | 7 | list_file="exe/list/list_grs_gernez_feb2022.csv" 8 | list_file="exe/list/list_grs_cnes_obs2mod.csv" 9 | 10 | year=2022 11 | for tiledate in `awk -F ',' 'NR > 1 && $1 == 1 {print $6$3}' $list_file`; do 12 | tile=${tiledate:0:5} 13 | year=${tiledate:5:4} 14 | 15 | year=2022 16 | echo rclone sync -P --include='*.nc' /datalake/watcal/S2-L2GRS/$tile/$year dropbox_harmel:/satellite/S2/cnes/$tile/$year 17 | #rclone sync -P --include='*.nc' /datalake/watcal/S2-L2GRS/$tile dropbox_harmel:/satellite/S2/cnes/$tile 18 | rclone sync -P --include='*v15.nc' /datalake/watcal/S2-L2GRS/$tile/$year dropbox_harmel:/satellite/S2/cnes/$tile/$year 19 | done 20 | 21 | #year=2015 22 | #tile=33VXF 23 | #rclone sync /datalake/watcal/S2-L2GRS/$tile/$year dropbox_harmel:/satellite/S2/cnes/$tile/$year 24 | -------------------------------------------------------------------------------- /exe/test_mp.py: -------------------------------------------------------------------------------- 1 | import multiprocessing as mp 2 | 3 | def test(img): 4 | from grs import grs_process 5 | 6 | print(img) 7 | 8 | imgs =['1','2','3','4','5','6','7','8'] 9 | p = mp.Pool(3) 10 | p.map(test,imgs) -------------------------------------------------------------------------------- /exe/upload_L2_dropbox.sh: -------------------------------------------------------------------------------- 1 | 2 | for tile in 30SUF 30TWT 30TXR 30TWS 30TWT 18SVH 31UET 29SPB 34HBH 17RLK 30UYV 34VDK 34VCJ 55KDV 34SFH 41RKJ 31TFJ 18GYU 17RLK; do 3 | dropbox_uploader.sh -s upload /datalake/watcal/S2-L2GRS/$tile /satellite/S2/cnes/ 4 | done 5 | 6 | 7 | list_file="exe/list_grs_cnes_gernez.csv" 8 | 9 | list_file="exe/list_grs_gernez_juillet_2021.csv" 10 | list_file="exe/list_grs_cnes_hafeez.csv" 11 | 12 | for tile in `awk -F ',' '{print $6}' $list_file`; do 13 | dropbox_uploader.sh -s upload /datalake/watcal/S2-L2GRS/$tile /satellite/S2/cnes/ 14 | done 15 | 16 | year=2021 17 | tile=31TFJ 18 | dropbox_uploader.sh -s -x .incomplete upload /datalake/watcal/S2-L2GRS/$tile/$year /satellite/S2/cnes/$tile 19 | -------------------------------------------------------------------------------- /grs/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | 3 | Version history 4 | ================== 5 | 6 | 1.0.0: 7 | - grs for Sentinel2 8 | 9 | 1.1.0: 10 | - adaptation to Landsat 4, 5, 7, 8 11 | 12 | 1.1.1: 13 | - small changes 14 | 15 | 1.1.2: 16 | - output in Rrs unit 17 | 18 | 1.1.3 19 | - small changes 20 | 21 | 1.1.4: 22 | - set cloud mask; (compressed) netcdf4 output format 23 | 24 | 1.2.0: 25 | - load full data matrix from image instead of line by line pixel extraction (preparation for multipixel retrieval algorithm 26 | 27 | 1.2.1: 28 | - enable high latitude processing (change of DEM), option to process all pixels before masking "non-water" pixel 29 | 30 | 1.2.2: 31 | - Interpolation based on nearest-neighbor to keep tile-edge pixels. 32 | - Implementation of product.dispose to minimize memory usage in the jvm of snap 33 | 34 | 1.2.3: 35 | - compliant with version 8 of SNAP: 36 | - change output writing (now directly in NETCDF4, i.e., compressed, at the end of the process) 37 | - new utils get_subset 38 | 39 | 1.2.4: 40 | - option to load MAJA and WaterDetect mask and export masks in output file 41 | - option to process WaterDetect Water pixels only 42 | 43 | 1.3.0: 44 | - update for CAMS data (cds version): 45 | - new aod wavelengths and spectral ssa from 2018 onwards 46 | - adjustment for absorbing aerosol through ssa 47 | 48 | 1.3.1: 49 | - update CAMS data extraction from xarray and fix for longitude conventions 50 | 51 | 1.3.2: 52 | - Important improvement in LUT interpolation and access 53 | 54 | 1.3.3: 55 | - fix small bug for sunglint BRDF output 56 | 57 | 1.3.4: 58 | - add slope and shade from DEM 59 | 60 | 1.4.0: 61 | - process image by rectangular chunks 62 | 63 | 1.5.0: 64 | - change output parameter with addition of ndwi_nir and ndwi_swir 65 | 66 | 2.0.0: 67 | - remove snappy skeleton, simplification of the previous option, no more handling for Landsat 68 | 69 | 2.0.1: 70 | - fix for I/O 71 | 72 | 2.0.2: 73 | - memory optimization 74 | 75 | 2.0.3: 76 | - add surfwater 77 | 78 | 2.0.4: 79 | - adapt format for QGIS 80 | 81 | 2.0.5: 82 | - new output format compliant with SNAP "beam format" 83 | 84 | 2.1.0: 85 | - Major update on radiative transfer look-up tables and aerosol models 86 | 87 | 2.1.1: 88 | - package data 89 | 90 | 2.1.2: 91 | - fix for tiles straddling Greenwich meridian 92 | 93 | 2.1.3: 94 | - new cams automated loading and revised parameters 95 | 96 | 2.1.4: 97 | - change input/output feature for grs process 98 | (i.e., can be called as simple function within a script) 99 | - Change of output variable with addition of bitmask flags 100 | 101 | 2.1.5: 102 | - repackaging of some classes 103 | - include Landsat 8 and 9 imageries 104 | 105 | 2.1.6: 106 | - use GRSdriver instead s2driver 107 | 108 | 2.1.7: 109 | - accounts for absorption/scattering interaction for transmittance of absorbing gases 110 | ''' 111 | 112 | __package__ = 'grs' 113 | __version__ = '2.1.7' 114 | 115 | 116 | from .acutils import Aerosol, Misc, Rasterization 117 | from .cams import CamsProduct 118 | from .auxdata import SensorData, AuxData 119 | from .product import Product 120 | from .output import L2aProduct 121 | from .mask import Masking 122 | from .grs_process import Process 123 | 124 | import logging 125 | 126 | #init logger 127 | logger = logging.getLogger() 128 | 129 | level = logging.getLevelName("INFO") 130 | logger.setLevel(level) 131 | -------------------------------------------------------------------------------- /grs/config.yml: -------------------------------------------------------------------------------- 1 | path: 2 | grsdata: '/data/grs/grsdata' 3 | toa_lut: 'toa_lut_opac_wind_light_v2.nc' 4 | trans_lut: 'transmittance_lut_opac_wind_light_v2.nc' 5 | cams: '/data/cams' 6 | 7 | processor: 8 | ncpu: 8 9 | chunk: 512 10 | netcdf_engine: 'h5netcdf' -------------------------------------------------------------------------------- /grs/config_template.yml: -------------------------------------------------------------------------------- 1 | path: 2 | grsdata: '/data/grs/grsdata' 3 | toa_lut: 'toa_lut_opac_wind_light_v2.nc' 4 | trans_lut: 'transmittance_lut_opac_wind_light_v2.nc' 5 | cams: '/data/cams' 6 | 7 | processor: 8 | ncpu: 8 9 | chunk: 512 10 | netcdf_engine: 'h5netcdf' -------------------------------------------------------------------------------- /grs/config_template_cnes.yml: -------------------------------------------------------------------------------- 1 | path: 2 | grsdata: '/work/datalake//watcal/GRS/grsdata_v21' 3 | toa_lut: 'toa_lut_opac_wind_light_v2.nc' 4 | trans_lut: 'transmittance_lut_opac_wind_light_v2.nc' 5 | cams: '/work/datalake/watcal/cams' 6 | 7 | processor: 8 | ncpu: 8 9 | chunk: 512 10 | netcdf_engine: 'h5netcdf' -------------------------------------------------------------------------------- /grs/data/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Where the LUTs are. 3 | ''' -------------------------------------------------------------------------------- /grs/data/aux/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/data/aux/__init__.py -------------------------------------------------------------------------------- /grs/data/lut/gases/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/data/lut/gases/__init__.py -------------------------------------------------------------------------------- /grs/data/lut/gases/lut_abs_opt_thickness_normalized.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/data/lut/gases/lut_abs_opt_thickness_normalized.nc -------------------------------------------------------------------------------- /grs/data/lut/gases/water_vapor_transmittance.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/data/lut/gases/water_vapor_transmittance.nc -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/Makefile: -------------------------------------------------------------------------------- 1 | #---------------------------------------------------------------------------- 2 | # Set up the compiler options. 3 | #---------------------------------------------------------------------------- 4 | OPTS = -g -Wall -O2 -march=nocona -mfpmath=sse -msse2 5 | CC = gcc 6 | #------------------------------------------------------- 7 | # Define the executable(s), the pdf(s), and the html(s). 8 | #------------------------------------------------------- 9 | LIBS = -L./ias_lib/ -ll8ang -lm 10 | 11 | INCS = \ 12 | -I./ias_lib/ \ 13 | -I./ 14 | 15 | #--------------------------------------------------------- 16 | # Define the source code object files for each executable. 17 | #--------------------------------------------------------- 18 | SRC = \ 19 | l8_angles.c \ 20 | angles_api.c 21 | 22 | OBJ = $(SRC:.c=.o) 23 | 24 | #------------------------------------------------------------------------------ 25 | # Targets for each executable. 26 | #------------------------------------------------------------------------------ 27 | all: l8_angles 28 | 29 | l8_angles: ias_lib/libl8ang.a $(OBJ) 30 | $(CC) $(OPTS) $(INCS) -o $@ $^ $(LIBS) 31 | 32 | ias_lib/libl8ang.a: 33 | cd ias_lib && make 34 | 35 | #--------------------------------------- 36 | # Target to distribute the executable(s) 37 | #--------------------------------------- 38 | .c.o: $(INCS) $(LIBS) 39 | $(CC) $(OPTS) $(INCS) -c -o $@ $< 40 | clean: 41 | /bin/rm -f *.o 42 | -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/Makefile: -------------------------------------------------------------------------------- 1 | #---------------------------------------------------------------------------- 2 | # Set up the compiler options. 3 | #---------------------------------------------------------------------------- 4 | OPTS = -g -Wall -O2 -march=nocona -mfpmath=sse -msse2 5 | CC = gcc 6 | #------------------------------------------------------- 7 | # Define the executable(s), the pdf(s), and the html(s). 8 | #------------------------------------------------------- 9 | LIBS = libl8ang.a 10 | 11 | INCS = \ 12 | ias_angle_gen_distro.h \ 13 | ias_angle_gen_private.h \ 14 | ias_angle_gen_includes.h \ 15 | ias_math.h \ 16 | ias_const.h \ 17 | ias_types.h \ 18 | ias_structures.h \ 19 | ias_logging.h \ 20 | ias_miscellaneous.h \ 21 | ias_odl.h \ 22 | ias_satellite_attributes.h 23 | 24 | #--------------------------------------------------------- 25 | # Define the source code object files for each executable. 26 | #--------------------------------------------------------- 27 | SRC = \ 28 | ias_angle_gen_calculate_angles_rpc.c \ 29 | ias_angle_gen_read_ang.c \ 30 | ias_angle_gen_utilities.c \ 31 | ias_angle_gen_initialize.c \ 32 | ias_angle_gen_write_image.c \ 33 | ias_angle_gen_find_scas.c \ 34 | ias_geo_convert_dms2deg.c \ 35 | ias_math_compute_unit_vector.c \ 36 | ias_math_compute_vector_length.c \ 37 | ias_math_find_line_segment_intersection.c \ 38 | ias_logging.c \ 39 | ias_misc_create_output_image_trim_lut.c \ 40 | ias_misc_convert_to_uppercase.c \ 41 | ias_misc_write_envi_header.c \ 42 | ias_odl_free_tree.c \ 43 | ias_odl_get_field.c \ 44 | ias_odl_read_tree.c \ 45 | ias_parm_provide_help.c \ 46 | ias_parm_read.c \ 47 | ias_parm_map_odl_type.c \ 48 | ias_parm_check_ranges.c \ 49 | ias_satellite_attributes.c \ 50 | landsat8.c \ 51 | lablib3.c 52 | 53 | OBJ = $(SRC:.c=.o) 54 | 55 | #------------------------------------------------------------------------------ 56 | # Targets for each executable. 57 | #------------------------------------------------------------------------------ 58 | all: $(LIBS) 59 | 60 | libl8ang.a: $(OBJ) 61 | ar -r $@ $(OBJ) 62 | 63 | #--------------------------------------- 64 | # Target to distribute the executable(s) 65 | #--------------------------------------- 66 | .c.o: 67 | $(CC) $(OPTS) -c $< -o $@ 68 | 69 | clean: 70 | /bin/rm -f *.o *.a 71 | -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/ias_angle_gen_calculate_angles_rpc.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/ias_angle_gen_calculate_angles_rpc.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/ias_angle_gen_find_scas.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/ias_angle_gen_find_scas.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/ias_angle_gen_initialize.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/ias_angle_gen_initialize.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/ias_angle_gen_read_ang.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/ias_angle_gen_read_ang.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/ias_angle_gen_utilities.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/ias_angle_gen_utilities.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/ias_angle_gen_write_image.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/ias_angle_gen_write_image.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/ias_geo_convert_dms2deg.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/ias_geo_convert_dms2deg.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/ias_logging.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/ias_logging.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/ias_math_compute_unit_vector.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/ias_math_compute_unit_vector.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/ias_math_compute_vector_length.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/ias_math_compute_vector_length.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/ias_math_find_line_segment_intersection.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/ias_math_find_line_segment_intersection.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/ias_misc_convert_to_uppercase.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/ias_misc_convert_to_uppercase.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/ias_misc_create_output_image_trim_lut.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/ias_misc_create_output_image_trim_lut.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/ias_misc_write_envi_header.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/ias_misc_write_envi_header.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/ias_odl_free_tree.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/ias_odl_free_tree.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/ias_odl_get_field.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/ias_odl_get_field.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/ias_odl_read_tree.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/ias_odl_read_tree.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/ias_parm_check_ranges.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/ias_parm_check_ranges.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/ias_parm_map_odl_type.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/ias_parm_map_odl_type.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/ias_parm_provide_help.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/ias_parm_provide_help.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/ias_parm_read.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/ias_parm_read.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/ias_satellite_attributes.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/ias_satellite_attributes.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/lablib3.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/lablib3.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/landsat8.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/landsat8.o -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/ias_lib/libl8ang.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/ias_lib/libl8ang.a -------------------------------------------------------------------------------- /grs/landsat_angles/OLI/l8_angles: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/OLI/l8_angles -------------------------------------------------------------------------------- /grs/landsat_angles/TM/IAS-Log: -------------------------------------------------------------------------------- 1 | IAS local software log 2 | 3 | Task Process id Unit Name Line Number Error Message 4 | ---- ---------- -------------------------------------------------- ----------- ------------- 5 | 2018 012 15:28:10 23808 xxx_Sensor.c 109 Setting sensor to UNKNOWN 6 | 2018 012 15:28:49 23812 xxx_Sensor.c 109 Setting sensor to UNKNOWN 7 | 2018 012 15:32:42 23951 xxx_Sensor.c 109 Setting sensor to UNKNOWN 8 | 2018 012 15:38:16 24235 xxx_Sensor.c 109 Setting sensor to UNKNOWN 9 | 2018 012 15:40:40 24242 xxx_Sensor.c 109 Setting sensor to UNKNOWN 10 | 2018 012 15:41:34 24252 xxx_Sensor.c 109 Setting sensor to UNKNOWN 11 | 2018 012 15:44:27 24287 xxx_Sensor.c 109 Setting sensor to UNKNOWN 12 | 2018 012 15:46:07 24323 xxx_Sensor.c 109 Setting sensor to UNKNOWN 13 | 2018 012 15:46:27 24328 xxx_Sensor.c 109 Setting sensor to UNKNOWN 14 | 2018 012 16:01:56 24381 xxx_Sensor.c 109 Setting sensor to UNKNOWN 15 | -------------------------------------------------------------------------------- /grs/landsat_angles/TM/Makefile: -------------------------------------------------------------------------------- 1 | #---------------------------------------------------------------------------- 2 | # Set up the compiler options. 3 | #---------------------------------------------------------------------------- 4 | OPTS = -g -Wall -O2 -march=nocona -mfpmath=sse -msse2 \ 5 | -DIAS_NO_SENSOR_META_SUPPORT 6 | CC = gcc 7 | #------------------------------------------------------- 8 | # Define the executable(s), the pdf(s), and the html(s). 9 | #------------------------------------------------------- 10 | LIBS = -L./ias_lib/ -llandsatang -lm 11 | 12 | INCS = -I./ias_lib/ -I./ 13 | 14 | #--------------------------------------------------------- 15 | # Define the source code object files for each executable. 16 | #--------------------------------------------------------- 17 | SRC = landsat_angles.c 18 | 19 | OBJ = $(SRC:.c=.o) 20 | 21 | #------------------------------------------------------------------------------ 22 | # Targets for each executable. 23 | #------------------------------------------------------------------------------ 24 | all: landsat_angles 25 | 26 | landsat_angles: ias_lib/liblandsatang.a $(OBJ) 27 | $(CC) $(OPTS) $(INCS) -o $@ $^ $(LIBS) 28 | 29 | ias_lib/liblandsatang.a: 30 | cd ias_lib && make 31 | 32 | #--------------------------------------- 33 | # Target to distribute the executable(s) 34 | #--------------------------------------- 35 | .c.o: $(INCS) $(LIBS) 36 | $(CC) $(OPTS) $(INCS) -c -o $@ $< 37 | clean: 38 | /bin/rm -f *.o 39 | -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/Makefile: -------------------------------------------------------------------------------- 1 | #---------------------------------------------------------------------------- 2 | # Set up the compiler options. 3 | #---------------------------------------------------------------------------- 4 | OPTS = -g -Wall -O2 -march=nocona -mfpmath=sse -msse2 \ 5 | -DIAS_NO_SENSOR_META_SUPPORT -DIAS_NOVAS_NOT_AVAILABLE 6 | CC = gcc 7 | #------------------------------------------------------- 8 | # Define the executable(s), the pdf(s), and the html(s). 9 | #------------------------------------------------------- 10 | LIBS = liblandsatang.a 11 | 12 | INCS = -I ./ 13 | 14 | #--------------------------------------------------------- 15 | # Define the source code object files for each executable. 16 | #--------------------------------------------------------- 17 | SRC = \ 18 | gxx_angle_gen_calculate_angles_rpc.c \ 19 | gxx_angle_gen_calculate_vector.c \ 20 | gxx_angle_gen_find_dir.c \ 21 | gxx_angle_gen_geo_utilities.c \ 22 | gxx_angle_gen_initialize.c \ 23 | gxx_angle_gen_initialize_transformation.c \ 24 | gxx_angle_gen_interpolate_ephemeris.c \ 25 | gxx_angle_gen_read_ang.c \ 26 | gxx_angle_gen_write_image.c \ 27 | gxx_dmsdeg.c \ 28 | gxx_dot.c \ 29 | gxx_geod2cart.c \ 30 | gxx_get_units.c \ 31 | gxx_unit.c \ 32 | xxx_Band.c \ 33 | xxx_CloseODL.c \ 34 | xxx_CloseUnmap.c \ 35 | xxx_ConvertString.c \ 36 | xxx_Errno.c \ 37 | xxx_FileLock.c \ 38 | xxx_GetDirFiles.c \ 39 | xxx_GetODLField.c \ 40 | xxx_GetTempName.c \ 41 | xxx_GetTime.c \ 42 | xxx_LogError.c \ 43 | xxx_LogStatus.c \ 44 | xxx_OpenMap.c \ 45 | xxx_OpenODL.c \ 46 | xxx_RecursiveDeletion.c \ 47 | xxx_Sensor.c \ 48 | xxx_strtoupper.c \ 49 | lablib3.c 50 | 51 | OBJ = $(SRC:.c=.o) 52 | 53 | #------------------------------------------------------------------------------ 54 | # Targets for each executable. 55 | #------------------------------------------------------------------------------ 56 | all: $(LIBS) 57 | 58 | liblandsatang.a: $(OBJ) 59 | ar -r $@ $(OBJ) 60 | 61 | #--------------------------------------- 62 | # Target to distribute the executable(s) 63 | #--------------------------------------- 64 | .c.o: 65 | $(CC) $(OPTS) $(INCS) -c $< -o $@ 66 | 67 | clean: 68 | /bin/rm -f *.o *.a 69 | -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/gxx_angle_gen_calculate_angles_rpc.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/gxx_angle_gen_calculate_angles_rpc.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/gxx_angle_gen_calculate_vector.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/gxx_angle_gen_calculate_vector.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/gxx_angle_gen_find_dir.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/gxx_angle_gen_find_dir.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/gxx_angle_gen_geo_utilities.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/gxx_angle_gen_geo_utilities.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/gxx_angle_gen_initialize.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/gxx_angle_gen_initialize.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/gxx_angle_gen_initialize_transformation.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/gxx_angle_gen_initialize_transformation.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/gxx_angle_gen_interpolate_ephemeris.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/gxx_angle_gen_interpolate_ephemeris.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/gxx_angle_gen_read_ang.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/gxx_angle_gen_read_ang.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/gxx_angle_gen_write_image.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/gxx_angle_gen_write_image.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/gxx_dmsdeg.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/gxx_dmsdeg.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/gxx_dot.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/gxx_dot.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/gxx_geod2cart.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/gxx_geod2cart.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/gxx_get_units.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/gxx_get_units.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/gxx_unit.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/gxx_unit.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/lablib3.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/lablib3.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/liblandsatang.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/liblandsatang.a -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/xxx_Band.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/xxx_Band.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/xxx_CloseODL.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/xxx_CloseODL.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/xxx_CloseUnmap.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/xxx_CloseUnmap.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/xxx_ConvertString.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/xxx_ConvertString.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/xxx_Errno.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/xxx_Errno.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/xxx_FileLock.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/xxx_FileLock.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/xxx_GetDirFiles.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/xxx_GetDirFiles.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/xxx_GetODLField.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/xxx_GetODLField.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/xxx_GetTempName.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/xxx_GetTempName.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/xxx_GetTime.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/xxx_GetTime.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/xxx_LogError.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/xxx_LogError.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/xxx_LogStatus.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/xxx_LogStatus.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/xxx_OpenMap.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/xxx_OpenMap.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/xxx_OpenODL.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/xxx_OpenODL.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/xxx_RecursiveDeletion.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/xxx_RecursiveDeletion.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/xxx_Sensor.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/xxx_Sensor.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/ias_lib/xxx_strtoupper.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/ias_lib/xxx_strtoupper.o -------------------------------------------------------------------------------- /grs/landsat_angles/TM/landsat_angles: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grs/landsat_angles/TM/landsat_angles -------------------------------------------------------------------------------- /grs_launcher.pbs: -------------------------------------------------------------------------------- 1 | #Lancer grs à travers un job PBS : 2 | 3 | #!/bin/bash 4 | 5 | #PBS -N grs 6 | #PBS -l select=1:ncpus=16:mem=32gb 7 | #PBS -l place=pack 8 | #PBS -l walltime=24:00:00 9 | 10 | #load avail module for conda, snap and jdk 11 | module load netcdf snap/8.0 conda jdk/1.8.0_112 12 | 13 | #otb-depends/7.0-python3.6.5 14 | 15 | #set the variable environment for je jpy and snap use 16 | export JDK_HOME=$JDKHOME 17 | export JAVAHOME=$JAVA_HOME 18 | export SNAP_HOME=$SNAPHOME 19 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:. 20 | 21 | ##activate conda environement python 3.6 22 | conda activate /work/scratch/harmelt/grs_py3.6 23 | cd /work/ALT/swot/aval/OBS2CO/git/grs2 24 | 25 | 26 | #python exe/grs_tiles_cnes.py 27 | 28 | grs /datalake/S2-L1C//33PVQ/2020/11/14/S2B_MSIL1C_20201114T092239_N0209_R093_T33PVQ_20201114T113340.SAFE --odir /datalake/watcal/S2-L2GRS/test \ 29 | --aerosol cams_forecast --maja /datalake/S2-L2A-THEIA/33PVQ/2020/11/14/SENTINEL2B_20201114-093704-775_L2A_T33PVQ_C_V2-2/SENTINEL2B_20201114-093704-775_L2A_T33PVQ_C_V2-2_MTD_ALL.xml \ 30 | --dem -------------------------------------------------------------------------------- /grs_mpi_launcher.pbs: -------------------------------------------------------------------------------- 1 | # launch with qsub (PBS job) 2 | #!/bin/bash 3 | 4 | 5 | #PBS -N grs2_obs2 6 | #PBS -l select=1:ncpus=40:mpiprocs=8:mem=180gb:generation=g2019 7 | #PBS -l walltime=48:00:00 8 | #mpiprocs=4: 9 | # echo /home/eh/harmelt/proxyfile.txt | source /home/eh/harmelt/set_proxy.sh 10 | 11 | # 12 | # set file path with list of images to process 13 | # format/header of the file in csv: 14 | # process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m), flag 15 | 16 | 17 | list_file=list_grs_cnes_obs2mod.csv 18 | #list_file=list_grs_ukraine.csv 19 | #list_file=list_grs_cordoba.csv 20 | #list_file=list_landsat_jegou.csv 21 | 22 | GRSHOME=/work/scratch/harmelt/dev/grs2 23 | list_filepath=$GRSHOME/exe/list/$list_file 24 | 25 | module load conda 26 | # ------------------------ 27 | # first section: load L1C and L2A images with Amalthee 28 | # ------------------------ 29 | path_of_env=/softs/rh7/conda-envs/amalthee_prod_client 30 | conda activate $path_of_env 31 | $path_of_env/bin/python $GRSHOME/exe/call_amalthee.py $list_filepath 32 | 33 | # ------------------------ 34 | # second section: process available couples of L1C/L2A images with GRS 35 | # ------------------------ 36 | ##activate conda environement 37 | path_of_env=grs_v2 38 | 39 | conda activate $path_of_env 40 | 41 | export OMP_NUM_THREADS=6 42 | 43 | #open monitoring tool 44 | module load monitoring/1.0 45 | start_monitoring.sh --name GRS_$PBS_JOBID 46 | python $GRSHOME/exe/grs_tiles_cnes_v2.py $list_filepath $OMP_NUM_THREADS 47 | stop_monitoring.sh --name GRS_$PBS_JOBID 48 | 49 | # test example: 50 | # grs /datalake/S2-L1C/33PVR/2020/12/19/S2A_MSIL1C_20201219T092411_N0209_R093_T33PVR_20201219T103844.SAFE -o /datalake/watcal/S2-L2GRS/33PVR/2020/12/19/33PVR/S2A_MSIl2grs_20201219T092411_N0209_R093_T33PVR_20201219T103844 --aerosol cams_forecast --maja /datalake/S2-L2A-THEIA/33PVR/2020/12/19/SENTINEL2A_20201219-093647-800_L2A_T33PVR_C_V2-2/SENTINEL2A_20201219-093647-800_L2A_T33PVR_C_V2-2_MTD_ALL.xml --dem 51 | # grs test/data/S2B_MSIL1C_20180927T103019_N0206_R108_T31TGK_20180927T143835.SAFE --shape test/data/shape/SPO04.shp --odir test/results/ --aerosol cams_forecast --dem --resolution 20 52 | # grs /datalake/watcal/L8_L1_C2/008231/2022/05/31/LC08_L1TP_008231_20220531_20220609_02_T1.tar --odir /datalake/watcal/L8_L2GRS_C2/008231/2022/05/31/ --aerosol cams_forecast --dem --resolution 20 53 | -------------------------------------------------------------------------------- /grs_validation_launcher.pbs: -------------------------------------------------------------------------------- 1 | # launch with qsub (PBS job) 2 | #!/bin/bash 3 | 4 | 5 | #PBS -N grs_obs2 6 | #PBS -l select=1:ncpus=40:mpiprocs=12:mem=180gb:generation=g2019 7 | #PBS -l walltime=48:00:00 8 | #mpiprocs=4: 9 | # echo /home/eh/harmelt/proxyfile.txt | source /home/eh/harmelt/set_proxy.sh 10 | 11 | # 12 | # set file path with list of images to process 13 | # format/header of the file in csv: 14 | # process (yes if 1),Site Name,start_date,end_date,satellite,tile,resolution (m), flag 15 | 16 | 17 | list_file=list_grs_aeronet_oc_test.csv 18 | 19 | GRSHOME=/work/scratch/harmelt/dev/grs2 20 | list_filepath=$GRSHOME/exe/list_validation/$list_file 21 | 22 | module load conda 23 | # ------------------------ 24 | # first section: load L1C and L2A images with Amalthee 25 | # ------------------------ 26 | path_of_env=/softs/rh7/conda-envs/amalthee_prod_client 27 | conda activate $path_of_env 28 | $path_of_env/bin/python $GRSHOME/exe/call_amalthee.py $list_filepath 29 | 30 | # ------------------------ 31 | # second section: process available couples of L1C/L2A images with GRS 32 | # ------------------------ 33 | 34 | #load modules snap and jdk 35 | module load snap/8.0 jdk/1.8.0_112 36 | 37 | ##activate conda environement 38 | path_of_env=grs_env 39 | conda activate $path_of_env 40 | 41 | #set the variable environment for je jpy and snap use 42 | export JDK_HOME=$JDKHOME 43 | export JAVAHOME=$JAVA_HOME 44 | export SNAP_HOME=$SNAPHOME 45 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:. 46 | export OMP_NUM_THREADS=12 47 | 48 | #open monitoring tool 49 | module load monitoring/1.0 50 | 51 | python $GRSHOME/exe/grs_validation.py $list_filepath $OMP_NUM_THREADS 52 | 53 | 54 | # test example: 55 | # grs /datalake/S2-L1C/33PVR/2020/12/19/S2A_MSIL1C_20201219T092411_N0209_R093_T33PVR_20201219T103844.SAFE -o /datalake/watcal/S2-L2GRS/33PVR/2020/12/19/33PVR/S2A_MSIl2grs_20201219T092411_N0209_R093_T33PVR_20201219T103844 --aerosol cams_forecast --maja /datalake/S2-L2A-THEIA/33PVR/2020/12/19/SENTINEL2A_20201219-093647-800_L2A_T33PVR_C_V2-2/SENTINEL2A_20201219-093647-800_L2A_T33PVR_C_V2-2_MTD_ALL.xml --dem 56 | # grs test/data/S2B_MSIL1C_20180927T103019_N0206_R108_T31TGK_20180927T143835.SAFE --shape test/data/shape/SPO04.shp --odir test/results/ --aerosol cams_forecast --dem --resolution 20 57 | # grs /datalake/watcal/L8_L1_C2/008231/2022/05/31/LC08_L1TP_008231_20220531_20220609_02_T1.tar --odir /datalake/watcal/L8_L2GRS_C2/008231/2022/05/31/ --aerosol cams_forecast --dem --resolution 20 58 | -------------------------------------------------------------------------------- /grsdata/gases/lut_abs_opt_thickness_normalized.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grsdata/gases/lut_abs_opt_thickness_normalized.nc -------------------------------------------------------------------------------- /grsdata/gases/water_vapor_transmittance.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/grsdata/gases/water_vapor_transmittance.nc -------------------------------------------------------------------------------- /images/Fig_valid_qualit_sea_scale.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/images/Fig_valid_qualit_sea_scale.png -------------------------------------------------------------------------------- /images/S2B_MSIL1C_20220701T103629_N0400_R008_T31TFJ_20220701T112552_RGB.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/images/S2B_MSIL1C_20220701T103629_N0400_R008_T31TFJ_20220701T112552_RGB.png -------------------------------------------------------------------------------- /images/S2B_MSIL2Agrs_20220701T103629_N0400_R008_T31TFJ_20220701T112552_BRDFg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/images/S2B_MSIL2Agrs_20220701T103629_N0400_R008_T31TFJ_20220701T112552_BRDFg.png -------------------------------------------------------------------------------- /images/S2B_MSIL2Agrs_20220701T103629_N0400_R008_T31TFJ_20220701T112552_RGB.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/images/S2B_MSIL2Agrs_20220701T103629_N0400_R008_T31TFJ_20220701T112552_RGB.png -------------------------------------------------------------------------------- /images/S2B_MSIL2Agrs_20220701T103629_N0400_R008_T31TFJ_20220701T112552_RGB_sunglint.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/images/S2B_MSIL2Agrs_20220701T103629_N0400_R008_T31TFJ_20220701T112552_RGB_sunglint.png -------------------------------------------------------------------------------- /images/example_snap_grs_image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/images/example_snap_grs_image.png -------------------------------------------------------------------------------- /images/flowchart_sunglint_S2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/images/flowchart_sunglint_S2.png -------------------------------------------------------------------------------- /issues/cams/test_cams.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datetime as dt 3 | from dateutil.relativedelta import relativedelta 4 | import matplotlib 5 | 6 | from grs import AuxData 7 | 8 | lat, lon = 47, -1.5 9 | 10 | cams_folder = os.path.abspath('/nfs/DP/ECMWF/CAMS') 11 | type='cams_forecast' 12 | #type='cams_reanalysis' 13 | 14 | def datespan(startDate, endDate, delta=relativedelta(months=+1)): 15 | currentDate = startDate 16 | while currentDate < endDate: 17 | yield currentDate 18 | currentDate += delta 19 | 20 | for date in datespan(dt.datetime(2019,7,17,12),dt.datetime.now()+relativedelta(months=-1)):#.datetime(2019,8,17,12)): 21 | print(date) 22 | cams_file=os.path.join(cams_folder, date.strftime('%Y-%m') + '_month_'+type+'.nc') 23 | 24 | cams = AuxData.cams() 25 | cams.load_cams_data(cams_file, date, data_type=type) 26 | 27 | def wktbox(center_lon, center_lat, width=1, height=1): 28 | ''' 29 | 30 | :param center_lon: decimal longitude 31 | :param center_laat: decimal latitude 32 | :param width: width of the box in km 33 | :param height: haight of the box in km 34 | :return: wkt of the box centered on provided coordinates 35 | ''' 36 | from math import sqrt, atan, pi 37 | import pyproj 38 | geod = pyproj.Geod(ellps='WGS84') 39 | width, height = width * 1000, height * 1000 40 | rect_diag = sqrt(width ** 2 + height ** 2) 41 | 42 | azimuth1 = atan(width / height) 43 | azimuth2 = atan(-width / height) 44 | azimuth3 = atan(width / height) + pi # first point + 180 degrees 45 | azimuth4 = atan(-width / height) + pi # second point + 180 degrees 46 | 47 | pt1_lon, pt1_lat, _ = geod.fwd(center_lon, center_lat, azimuth1 * 180 / pi, rect_diag) 48 | pt2_lon, pt2_lat, _ = geod.fwd(center_lon, center_lat, azimuth2 * 180 / pi, rect_diag) 49 | pt3_lon, pt3_lat, _ = geod.fwd(center_lon, center_lat, azimuth3 * 180 / pi, rect_diag) 50 | pt4_lon, pt4_lat, _ = geod.fwd(center_lon, center_lat, azimuth4 * 180 / pi, rect_diag) 51 | 52 | wkt_point = 'POINT (%.6f %.6f)' % (center_lon, center_lat) 53 | wkt_poly = 'POLYGON (( %.6f %.6f, %.6f %.6f, %.6f %.6f, %.6f %.6f, %.6f %.6f ))' % ( 54 | pt1_lon, pt1_lat, pt2_lon, pt2_lat, pt3_lon, pt3_lat, pt4_lon, pt4_lat, pt1_lon, pt1_lat) 55 | return wkt_poly 56 | 57 | wkt = wktbox(lon, lat,50,50) 58 | 59 | 60 | cams.get_cams_ancillary(cams_file, date, wkt) 61 | for i in range(4): 62 | date=date + datetime.timedelta(days=1) 63 | cams.get_cams_aerosol(cams_file, date, wkt) 64 | print(date.ctime(),cams.aot_rast[1,...].min(),cams.aot_rast[1,...].mean(),cams.aot_rast[1,...].max()) 65 | 66 | 67 | -------------------------------------------------------------------------------- /issues/dem/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/issues/dem/__init__.py -------------------------------------------------------------------------------- /issues/dem/check_dem_availability.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Main program 3 | ''' 4 | 5 | from pathlib import Path 6 | 7 | 8 | import os, shutil 9 | import zipfile 10 | import tarfile 11 | import glob 12 | import numpy as np 13 | import xarray as xr 14 | 15 | import matplotlib.pyplot as plt 16 | import matplotlib 17 | #matplotlib.use('TkAgg') 18 | 19 | from esasnappy import ProductData, ProductIO 20 | from grs import info,utils, acutils, AuxData 21 | 22 | 23 | #file='/sat_data/satellite/sentinel2/L1C/31TFJ/S2A_MSIL1C_20201004T104031_N0209_R008_T31TFJ_20201004T125253.SAFE' 24 | file='/datalake/S2-L1C/21MYT/2022/01/14/S2A_MSIL1C_20220114T141051_N0301_R110_T21MYT_20220114T155152.SAFE' 25 | file='/datalake/S2-L1C/57WWS/2016/10/10/S2A_MSIL1C_20161010T012652_N0204_R074_T57WWS_20161010T012650.SAFE' 26 | 27 | 28 | 29 | 30 | ################################## 31 | # Get sensor auxiliary data 32 | ################################## 33 | 34 | print('Get sensor auxiliary data') 35 | _utils = utils() 36 | 37 | sensor = _utils.get_sensor(file) 38 | sensordata = AuxData.sensordata(sensor) 39 | resolution = sensordata.resolution 40 | indband = sensordata.indband 41 | 42 | 43 | print("Reading...") 44 | print(file) 45 | product = ProductIO.readProduct(file) 46 | 47 | ################################## 48 | # Generate l2h object 49 | ################################## 50 | 51 | l2h = info(product, sensordata) 52 | l2h.headerfile = file 53 | #l2h.product = _utils.s2_resampler(l2h.product, resolution=resolution) 54 | l2h.Product = _utils.resampler(l2h.Product, resolution=resolution) 55 | 56 | 57 | ################################## 58 | # GET IMAGE AND RASTER PROPERTIES 59 | ################################## 60 | l2h.get_bands(l2h.band_names) 61 | l2h.print_info() 62 | l2h.aux = AuxData.cams() 63 | l2h.get_product_info() 64 | l2h.wkt, lonmin, lonmax, latmin, latmax = _utils.get_extent(l2h.Product) 65 | 66 | ################################## 67 | ## ADD ELEVATION BAND 68 | ################################## 69 | high_latitude = (latmax >= 60) | (latmin <= -60) 70 | l2h.get_elevation(high_latitude) 71 | 72 | plt.imshow(l2h.elevation) -------------------------------------------------------------------------------- /issues/dem/dem_slope_aspect.py: -------------------------------------------------------------------------------- 1 | from osgeo import gdal 2 | import numpy as np 3 | import rasterio 4 | import xarray as xr 5 | import richdem as rd 6 | 7 | import matplotlib.pyplot as plt 8 | import matplotlib 9 | 10 | file='./test/results/S2B_MSIL2grs_20180927T103019_N0206_R108_T31TGK_20180927T143835.nc' 11 | img = xr.open_dataset(file) 12 | 13 | sza=np.radians(65) 14 | sun_azi=np.radians(120) 15 | 16 | dem = np.array(img.elevation) 17 | dem= rd.rdarray(dem,no_data=np.nan) 18 | slope= np.radians(rd.TerrainAttribute(dem, attrib="slope_degrees")) 19 | aspect = rd.TerrainAttribute(dem, attrib="aspect") 20 | aspect_rd = np.radians(aspect) 21 | shaded = np.cos(sza) * np.cos(slope) + np.sin(sza)\ 22 | * np.sin(slope) * np.cos(sun_azi - aspect_rd) 23 | 24 | fig,axs=plt.subplots(2,2,figsize=(10,8)) 25 | axs=axs.ravel() 26 | axs[0].imshow(dem,cmap=plt.cm.gist_earth) 27 | axs[0].imshow(shaded,cmap=plt.cm.gray,alpha=0.5) 28 | axs[1].imshow(slope,cmap=plt.cm.gray) 29 | axs[2].imshow(aspect,cmap=plt.cm.Spectral_r) 30 | axs[3].imshow(255*(shaded+1)/2,cmap=plt.cm.gray) 31 | 32 | -------------------------------------------------------------------------------- /issues/dem/download_srtm.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | odir="/home/eh/harmelt/.snap/auxdata/dem/SRTM\ 3Sec" 4 | eval cd $odir 5 | 6 | for i in $(seq -f "%02g" 1 99);do #((i=1;i<100;i++));do 7 | for j in $(seq -f "%02g" 1 99);do # ((j=1;j<100;j++));do 8 | echo wget https://download.esa.int/step/auxdata/dem/SRTM90/tiff/srtm_"$i"_"$j".zip 9 | wget https://download.esa.int/step/auxdata/dem/SRTM90/tiff/srtm_"$i"_"$j".zip 10 | done 11 | done -------------------------------------------------------------------------------- /issues/dem/readme.txt: -------------------------------------------------------------------------------- 1 | PROCESSED SRTM DATA VERSION 4.1 2 | 3 | The data distributed here are in ARC GRID, ARC ASCII and Geotiff format, in 4 | decimal degrees and datum WGS84. They are derived from the USGS/NASA SRTM data. 5 | CIAT have processed this data to provide seamless continuous topography 6 | surfaces. Areas with regions of no data in the original SRTM data have been 7 | filled using interpolation methods described by Reuter et al. (2007). 8 | 9 | Version 4.1 has the following enhancements over V4.0: 10 | - Improved ocean mask used, which includes some small islands previously being 11 | lost in the cut data. 12 | - Single no-data line of pixels along meridians fixed. 13 | - All GeoTiffs with 6000 x 6000 pixels. 14 | - For ASCII format files the projection definition is included in .prj files. 15 | - For GeoTiff format files the projection definition is in the .tfw (ESRI TIFF 16 | World) and a .hdr file that reports PROJ.4 equivelent projection definitions. 17 | 18 | DISTRIBUTION 19 | 20 | Users are prohibited from any commercial, non-free resale, or redistribution 21 | without explicit written permission from CIAT. Users should acknowledge CIAT as 22 | the source used in the creation of any reports, publications, new data sets, 23 | derived products, or services resulting from the use of this data set. CIAT also 24 | request reprints of any publications and notification of any redistributing 25 | efforts. For commercial access to the data, send requests to Andy Jarvis 26 | (a.jarvis@cgiar.org). 27 | 28 | NO WARRANTY OR LIABILITY 29 | 30 | CIAT provides these data without any warranty of any kind whatsoever, either 31 | express or implied, including warranties of merchantability and fitness for a 32 | particular purpose. CIAT shall not be liable for incidental, consequential, or 33 | special damages arising out of the use of any data. 34 | 35 | ACKNOWLEDGMENT AND CITATION 36 | 37 | We kindly ask any users to cite this data in any published material produced 38 | using this data, and if possible link web pages to the CIAT-CSI SRTM website 39 | (http://srtm.csi.cgiar.org). 40 | 41 | Citations should be made as follows: 42 | 43 | Jarvis A., H.I. Reuter, A. Nelson, E. Guevara, 2008, Hole-filled seamless SRTM 44 | data V4, International Centre for Tropical Agriculture (CIAT), available from 45 | http://srtm.csi.cgiar.org. 46 | 47 | REFERENCES 48 | 49 | Reuter H.I, A. Nelson, A. Jarvis, 2007, An evaluation of void filling 50 | interpolation methods for SRTM data, International Journal of Geographic 51 | Information Science, 21:9, 983-1008. 52 | -------------------------------------------------------------------------------- /issues/dem/zipping_srtm.sh: -------------------------------------------------------------------------------- 1 | 2 | dir=/datalake/static_aux/MNT/SRTM_90m 3 | odir="/home/eh/harmelt/.snap/auxdata/dem/SRTM\ 3Sec" 4 | for f in $dir/*hdr; do 5 | b=${f/.hdr/} 6 | b_=${b##*/} 7 | echo zip $odir/$b_.zip $b.hdr $b.tfw $b.tif 8 | eval zip $odir/$b_.zip $b.hdr $b.tfw $b.tif 9 | done 10 | -------------------------------------------------------------------------------- /issues/format/formats_and_extensions.py: -------------------------------------------------------------------------------- 1 | from esasnappy import jpy 2 | 3 | ProductIOPlugInManager = jpy.get_type('org.esa.snap.core.dataio.ProductIOPlugInManager') 4 | ProductReaderPlugIn = jpy.get_type('org.esa.snap.core.dataio.ProductReaderPlugIn') 5 | ProductWriterPlugIn = jpy.get_type('org.esa.snap.core.dataio.ProductWriterPlugIn') 6 | 7 | read_plugins = ProductIOPlugInManager.getInstance().getAllReaderPlugIns() 8 | write_plugins = ProductIOPlugInManager.getInstance().getAllWriterPlugIns() 9 | 10 | print('Writer formats') 11 | while write_plugins.hasNext(): 12 | plugin = write_plugins.next() 13 | print('["'+ plugin.getDefaultFileExtensions()[0]+'","'+ plugin.getFormatNames()[0]+'"],') 14 | 15 | print('Reader formats') 16 | while read_plugins.hasNext(): 17 | plugin = read_plugins.next() 18 | print(' ', plugin.getDefaultFileExtensions()[0], plugin.getFormatNames()[0]) -------------------------------------------------------------------------------- /issues/format/test: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/issues/format/test -------------------------------------------------------------------------------- /issues/format/test.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/issues/format/test.bin -------------------------------------------------------------------------------- /issues/format/test.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/issues/format/test.csv -------------------------------------------------------------------------------- /issues/format/test.data/vector_data/ground_control_points.csv: -------------------------------------------------------------------------------- 1 | #defaultCSS=symbol:plus; stroke:#ff8800; stroke-opacity:0.8; stroke-width:1.0 2 | org.esa.snap.GroundControlPoint geometry:Point style_css:String label:String text:String pixelPos:Point geoPos:Point dateTime:Date 3 | -------------------------------------------------------------------------------- /issues/format/test.data/vector_data/pins.csv: -------------------------------------------------------------------------------- 1 | #defaultCSS=symbol:pin; fill:#0000ff; fill-opacity:0.7; stroke:#ffffff; stroke-opacity:1.0; stroke-width:0.5 2 | org.esa.snap.Pin geometry:Point style_css:String label:String text:String pixelPos:Point geoPos:Point dateTime:Date 3 | -------------------------------------------------------------------------------- /issues/format/test.dim: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | DIMAP 5 | BEAM-DATAMODEL-V1 6 | 7 | 8 | BEAM-PRODUCT 9 | test 10 | 11 | 12 | 13 | test 14 | 15 | 16 | 100 17 | 100 18 | 1 19 | 20 | 21 | ENVI 22 | ENVI File Format 23 | BAND_SEPARATE 24 | 25 | 26 | 0 27 | 28 | 29 | 30 | 31 | 0 32 | Solar zenith angle in deg. 33 | SZA 34 | 100 35 | 100 36 | float32 37 | 0.0 38 | 0.0 39 | 0.0 40 | 1.0 41 | 0.0 42 | false 43 | true 44 | NaN 45 | 46 | 47 | -------------------------------------------------------------------------------- /issues/format/test.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/issues/format/test.h5 -------------------------------------------------------------------------------- /issues/format/test.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/issues/format/test.tif -------------------------------------------------------------------------------- /issues/format/test_snappy_writer.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from esasnappy import Product, ProductIO, ProductData 3 | 4 | formats = ([".dim", "BEAM-DIMAP"], 5 | [".h5", "HDF5"], 6 | [".img", "GDAL-HFA-WRITER"], 7 | [".jp2", "JPEG2000"], 8 | [".bin", "Generic Binary BSQ"], 9 | [".tif", "GeoTIFF+XML"], 10 | [".grd", "GDAL-GSBG-WRITER"], 11 | [".nc", "NetCDF-CF"], 12 | [".nc", "NetCDF-BEAM"], 13 | [".nc", "NetCDF4-BEAM"], 14 | [".tif", "GeoTIFF"], 15 | ["", "JP2"], 16 | [".tif", "GDAL-GTiff-WRITER"], 17 | [".tif", "GeoTIFF-BigTIFF"], 18 | [".csv", "CSV"], 19 | [".bmp", "GDAL-BMP-WRITER"], 20 | [".rst", "GDAL-RST-WRITER"], 21 | [".nc", "NetCDF4-CF"], 22 | [".hdr", "ENVI"]) 23 | 24 | for format in formats: 25 | print() 26 | print('---------------') 27 | print(format) 28 | file = 'issues/test' + format[0] 29 | 30 | product = Product('test', 'test', 100, 100) 31 | band = product.addBand('SZA', ProductData.TYPE_FLOAT32) 32 | band.setModified(True) 33 | band.setNoDataValue(np.nan) 34 | band.setNoDataValueUsed(True) 35 | product.getBand('SZA').setDescription('Solar zenith angle in deg.') 36 | 37 | writer = ProductIO.getProductWriter(format[1]) 38 | 39 | product.setProductWriter(writer) 40 | try: 41 | #product.writeHeader(file) 42 | ProductIO.writeProduct(product, file, format[1]) 43 | p = ProductIO.readProduct(file) 44 | try: 45 | 46 | print(p.getBand('SZA').getDescription()) 47 | print('SUCCESS!!!!!!!') 48 | except: 49 | print('error for format ', format[1]) 50 | except: 51 | print('no header allowed for', format[1]) 52 | print() 53 | -------------------------------------------------------------------------------- /issues/lut_interpol/lut_interpol_test.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Main program 3 | ''' 4 | 5 | from pathlib import Path 6 | 7 | 8 | import os, shutil 9 | import zipfile 10 | import tarfile 11 | import glob 12 | import numpy as np 13 | import xarray as xr 14 | 15 | import matplotlib.pyplot as plt 16 | import matplotlib 17 | #matplotlib.use('TkAgg') 18 | 19 | from esasnappy import ProductData, ProductIO 20 | from grs import info,utils, acutils, AuxData 21 | import issues.lut_interpol.read_lut as read_lut 22 | 23 | #file='/sat_data/satellite/sentinel2/L1C/31TFJ/S2A_MSIL1C_20201004T104031_N0209_R008_T31TFJ_20201004T125253.SAFE' 24 | file='/datalake/S2-L1C/21MYT/2022/01/14/S2A_MSIL1C_20220114T141051_N0301_R110_T21MYT_20220114T155152.SAFE' 25 | 26 | 27 | 28 | ################################## 29 | # Get sensor auxiliary data 30 | ################################## 31 | 32 | print('Get sensor auxiliary data') 33 | _utils = utils() 34 | 35 | sensor = _utils.get_sensor(file) 36 | sensordata = AuxData.sensordata(sensor) 37 | resolution = sensordata.resolution 38 | indband = sensordata.indband 39 | 40 | 41 | print("Reading...") 42 | print(file) 43 | product = ProductIO.readProduct(file) 44 | 45 | ################################## 46 | # Generate l2h object 47 | ################################## 48 | 49 | l2h = info(product, sensordata) 50 | l2h.headerfile = file 51 | #l2h.product = _utils.s2_resampler(l2h.product, resolution=resolution) 52 | l2h.Product = _utils.resampler(l2h.Product, resolution=resolution) 53 | 54 | 55 | ################################## 56 | # GET IMAGE AND RASTER PROPERTIES 57 | ################################## 58 | l2h.get_bands(l2h.band_names) 59 | l2h.print_info() 60 | l2h.aux = AuxData.cams() 61 | l2h.get_product_info() 62 | 63 | ##################################### 64 | # LOAD LUT FOR ATMOSPHERIC CORRECTION 65 | ##################################### 66 | # load lut 67 | #l2h.lutfine='/home/harmel/DATA/projet/grs/grsdata/LUT/S2A/lut_osoaa_band_integrated_aot0.01_aero_rg0.10_sig0.46_ws2_pressure1015.2.nc' 68 | #l2h.lutcoarse='/home/harmel/DATA/projet/grs/grsdata/LUT/S2A/lut_osoaa_band_integrated_aot0.01_aero_rg0.80_sig0.60_ws2_pressure1015.2.nc' 69 | 70 | print('loading lut...', l2h.lutfine) 71 | lutf = acutils.lut(l2h.band_names) 72 | lutc = acutils.lut(l2h.band_names) 73 | lutf.load_lut(l2h.lutfine, indband) 74 | lutc.load_lut(l2h.lutcoarse, indband) 75 | 76 | 77 | 78 | l2h.rot = l2h.SensorData.rot 79 | # normalization of Cext to get spectral dependence of fine and coarse modes 80 | nCext_f = lutf.Cext / lutf.Cext550 81 | nCext_c = lutc.Cext / lutc.Cext550 82 | l2h.set_outfile('test.nc') 83 | l2h.wkt, lonmin, lonmax, latmin, latmax = _utils.get_extent(l2h.Product) 84 | l2h.crs = str(l2h.Product.getBand(l2h.band_names[0]).getGeoCoding().getImageCRS()) 85 | 86 | ###################################### 87 | # Create output l2 product 88 | # 'l2_product' 89 | ###################################### 90 | 91 | #l2h.create_product() 92 | # l2h.load_data() 93 | # w, h = l2h.width, l2h.height 94 | # 95 | # 96 | # def remove_na(arr): 97 | # return arr[~np.isnan(arr)] 98 | # 99 | # sza_= remove_na(np.unique(l2h.sza.round(1))) 100 | # vza_= remove_na(np.unique(l2h.vza.round(1))) 101 | # azi_= remove_na(np.unique(l2h.razi.round(0))) 102 | 103 | sza_ = np.array([30,40,55,56,60]) 104 | vza_= np.array([1,2,5,6.6,13]) 105 | azi_= np.array([90,100,180]) 106 | 107 | 108 | 109 | plt.figure() 110 | lutc.refl.isel(sza=-1,azi=1,vza=1).plot(x='wl',hue='aot',marker='o') 111 | plt.figure() 112 | lutf.refl.isel(sza=-1,azi=1,vza=1).plot(x='aot',hue='wl',marker='o') 113 | 114 | f_ = lutf.refl.interp(azi=azi_).interp(vza=vza_).interp(sza=sza_) 115 | plt.figure() 116 | f_.isel(sza=-1,azi=-1,vza=1).plot(x='wl',hue='aot') 117 | 118 | rlut = f_ 119 | f_.dims 120 | aot_ = np.array(f_.aot, dtype=float)#,order='F') 121 | read_lut.read_lut(*f_.shape,aot_, sza_, azi_, vza_, rlut[::-1]) 122 | read_lut.read_lut(6, 11, 5, 3, 5,aot_, sza_, azi_, vza_, rlut) 123 | 124 | ################################## 125 | # 126 | ######## END 127 | # 128 | ################################## 129 | 130 | 131 | 132 | 133 | # szalut=range(10,70) 134 | # sza=18.5 135 | # r_ = 100 136 | # for i in range(len(szalut)): 137 | # r__ = abs(szalut[i] - sza) 138 | # if (r__ > r_) : 139 | # isza = i - 1 140 | # break 141 | # else: 142 | # isza=i 143 | # r_ = r__ 144 | # 145 | # print(isza,sza,szalut[isza]) 146 | -------------------------------------------------------------------------------- /issues/lut_interpol/lut_read_f.f90: -------------------------------------------------------------------------------- 1 | subroutine read_lut(naot, nband, nsza, nazi, nvza, & 2 | & aotlut, szalut, azilut, vzalut, & 3 | & rlut) 4 | ! f2py -c -m read_lut lut_read_f.f90 5 | 6 | 7 | !-------------------------------------------------------------------- 8 | ! nband: number of satellite bands to be processed 9 | ! naot, nsza, nazi, nvza : dimensions of lut data arrays 10 | ! vza, sza, azi: satellite angle data 11 | ! rtoa: satellite L1C reflectance (corected from gas absorption) 12 | ! rg_ratio: spectral variation (referred to as epsilon in Harmel et al., RSE; 201?) 13 | ! TODO 14 | !-------------------------------------------------------------------- 15 | 16 | implicit none 17 | 18 | integer, parameter :: dp = kind(0.d0) 19 | integer, parameter :: sp = kind(0.) 20 | integer, parameter :: rtype = sp 21 | 22 | integer, intent(in) :: nband, naot, nsza, nazi, nvza 23 | real(rtype), dimension(naot), intent(in) :: aotlut 24 | real(rtype), dimension(nsza), intent(in) :: szalut 25 | real(rtype), dimension(nazi), intent(in) :: azilut 26 | real(rtype), dimension(nvza), intent(in) :: vzalut 27 | 28 | real(rtype), dimension(naot, nband, nsza, nazi, nvza), intent(in) :: rlut 29 | 30 | !f2py intent(in) nband,naot,nsza,nazi,nvza,aotlut,szalut, razilut, vzalut 31 | !f2py intent(in) rlut 32 | !f2py depend(naot) aotlut 33 | !f2py depend(nsza) szalut 34 | !f2py depend(nazi) azilut 35 | !f2py depend(nvza) vzalut 36 | !f2py depend(naot,nband,nsza,nazi,nvza) rlut 37 | 38 | integer :: i, iband 39 | 40 | !---------------------------------------------- 41 | real(rtype), parameter :: pi = 4 * atan (1.0_rtype) 42 | real(rtype), parameter :: degrad = pi / 180._rtype 43 | 44 | 45 | iband=1 46 | do i =1,naot 47 | print*,aotlut(i),rlut(i,iband,1,1,1) 48 | end do 49 | 50 | return 51 | 52 | end subroutine read_lut -------------------------------------------------------------------------------- /issues/lut_interpol/read_lut.cpython-36m-x86_64-linux-gnu.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/issues/lut_interpol/read_lut.cpython-36m-x86_64-linux-gnu.so -------------------------------------------------------------------------------- /issues/lut_interpol/read_lut.cpython-38-x86_64-linux-gnu.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Tristanovsk/grs/77c92d3b850b98615ae8645b84ebb0a907bdd24f/issues/lut_interpol/read_lut.cpython-38-x86_64-linux-gnu.so -------------------------------------------------------------------------------- /issues/multiproc/multiproc_snappy.py: -------------------------------------------------------------------------------- 1 | 2 | import os, sys 3 | from multiprocessing import Pool 4 | 5 | def exit(): 6 | os._exit(0) 7 | 8 | def call_snappy(i): 9 | 10 | 11 | print(i,' call of snappy') 12 | sys.exit() 13 | import esasnappy 14 | exit() 15 | #sys.exit() 16 | 17 | if __name__ == '__main__': 18 | with Pool(processes=3) as pool: 19 | processes = pool.map(call_snappy, range(30),1) 20 | print('processes ',processes) 21 | pool.close() 22 | # pool.join() 23 | print('end') -------------------------------------------------------------------------------- /issues/resampling/test2_snappy_dispose.py: -------------------------------------------------------------------------------- 1 | import time,sys 2 | import numpy as np 3 | from esasnappy import ProductIO, GPF, jpy 4 | 5 | 6 | sampler=sys.argv[1] 7 | file=sys.argv[2] 8 | 9 | def generic_resampler(product, resolution=20, method='Nearest'): 10 | '''method: Nearest, Bilinear''' 11 | 12 | GPF.getDefaultInstance().getOperatorSpiRegistry().loadOperatorSpis() 13 | 14 | HashMap = jpy.get_type('java.util.HashMap') 15 | 16 | parameters = HashMap() 17 | parameters.put('targetResolution', resolution) 18 | parameters.put('upsampling', method) 19 | parameters.put('downsampling', 'Mean') 20 | parameters.put('flagDownsampling', 'FlagMedianAnd') 21 | parameters.put('resampleOnPyramidLevels', False) 22 | 23 | return GPF.createProduct('Resample', parameters, product) 24 | 25 | def s2_resampler(product, resolution=20, upmethod='Nearest', downmethod='Mean', 26 | flag='FlagMedianAnd', opt=False): 27 | 28 | '''Resampler operator dedicated to Sentinel2-msi characteristics (e.g., viewing angles) 29 | :param product: S2-msi product as provided by esasnappy ProductIO.readProduct() 30 | :param resolution: target resolution in meters (10, 20, 60) 31 | :param upmethod: interpolation method ('Nearest', 'Bilinear', 'Bicubic') 32 | :param downmethod: aggregation method ('First', 'Min', 'Max', 'Mean', 'Median') 33 | :param flag: method for flags aggregation ('First', 'FlagAnd', 'FlagOr', 'FlagMedianAnd', 'FlagMedianOr') 34 | :param opt: resample on pyramid levels (True/False) 35 | :return: interpolated target product''' 36 | 37 | res = str(resolution) 38 | 39 | resampler = jpy.get_type('org.esa.s2tbx.s2msi.resampler.S2ResamplingOp') 40 | 41 | op = resampler() 42 | op.setParameter('targetResolution', res) 43 | op.setParameter('upsampling', upmethod) 44 | op.setParameter('downsampling', downmethod) 45 | op.setParameter('flagDownsampling', flag) 46 | op.setParameter('resampleOnPyramidLevels', opt) 47 | op.setSourceProduct(product) 48 | 49 | return op.getTargetProduct() 50 | 51 | 52 | file='/DATA/Satellite/SENTINEL2/SPO04/L1C/S2B_MSIL1C_20180927T103019_N0206_R108_T31TGK_20180927T143835.SAFE' 53 | product = ProductIO.readProduct(file) 54 | bands = product.getBandNames() 55 | p_core = generic_resampler(product) 56 | p_s2tbx = s2_resampler(product) 57 | w, h = product.getSceneRasterWidth(),product.getSceneRasterHeight() 58 | 59 | #h=512 60 | def time_load(p_,angles=True): 61 | 62 | tt=time.time 63 | tdiff=[] 64 | for band in list(bands)[::-1]: 65 | print(band) 66 | arr = np.empty((w,h)) 67 | t=[] 68 | t.append(tt()) 69 | if 'mean' in band: 70 | continue 71 | # if 'view' in band: 72 | # p_ = product 73 | # else: 74 | # p_ = product 75 | # p_ = p_core 76 | p_.getBand(band).readPixels(0, 0, w, h, arr) 77 | p_.getBand(band).dispose() 78 | #p_.getBand(band).readPixels(0,0,w,h,arr) 79 | t.append(tt()) 80 | tdiff.append(np.diff(t)) 81 | 82 | return 'mean time to load a row: %.4f'%np.mean(tdiff)+'s' 83 | 84 | 85 | if sampler == 'core': 86 | print('1st call with angles from core sampler: ',time_load(p_core,angles=False)) 87 | 88 | else: 89 | print('1st call without angles from s2tbx sampler: ',time_load(p_s2tbx,angles=False)) 90 | 91 | -------------------------------------------------------------------------------- /issues/resampling/test2_snappy_resampling.py: -------------------------------------------------------------------------------- 1 | import time,sys 2 | import numpy as np 3 | from esasnappy import ProductIO, GPF, jpy 4 | 5 | 6 | sampler=sys.argv[1] 7 | file=sys.argv[2] 8 | 9 | def generic_resampler(product, resolution=20, method='Nearest'): 10 | '''method: Nearest, Bilinear''' 11 | 12 | GPF.getDefaultInstance().getOperatorSpiRegistry().loadOperatorSpis() 13 | 14 | HashMap = jpy.get_type('java.util.HashMap') 15 | 16 | parameters = HashMap() 17 | parameters.put('targetResolution', resolution) 18 | parameters.put('upsampling', method) 19 | parameters.put('downsampling', 'Mean') 20 | parameters.put('flagDownsampling', 'FlagMedianAnd') 21 | parameters.put('resampleOnPyramidLevels', False) 22 | 23 | return GPF.createProduct('Resample', parameters, product) 24 | 25 | def s2_resampler(product, resolution=20, upmethod='Nearest', downmethod='Mean', 26 | flag='FlagMedianAnd', opt=False): 27 | 28 | '''Resampler operator dedicated to Sentinel2-msi characteristics (e.g., viewing angles) 29 | :param product: S2-msi product as provided by esasnappy ProductIO.readProduct() 30 | :param resolution: target resolution in meters (10, 20, 60) 31 | :param upmethod: interpolation method ('Nearest', 'Bilinear', 'Bicubic') 32 | :param downmethod: aggregation method ('First', 'Min', 'Max', 'Mean', 'Median') 33 | :param flag: method for flags aggregation ('First', 'FlagAnd', 'FlagOr', 'FlagMedianAnd', 'FlagMedianOr') 34 | :param opt: resample on pyramid levels (True/False) 35 | :return: interpolated target product''' 36 | 37 | res = str(resolution) 38 | 39 | resampler = jpy.get_type('org.esa.s2tbx.s2msi.resampler.S2ResamplingOp') 40 | 41 | op = resampler() 42 | op.setParameter('targetResolution', res) 43 | op.setParameter('upsampling', upmethod) 44 | op.setParameter('downsampling', downmethod) 45 | op.setParameter('flagDownsampling', flag) 46 | op.setParameter('resampleOnPyramidLevels', opt) 47 | op.setSourceProduct(product) 48 | 49 | return op.getTargetProduct() 50 | 51 | 52 | file='/DATA/Satellite/SENTINEL2/SPO04/L1C/S2B_MSIL1C_20180927T103019_N0206_R108_T31TGK_20180927T143835.SAFE' 53 | product = ProductIO.readProduct(file) 54 | bands = product.getBandNames() 55 | p_core = generic_resampler(product) 56 | p_s2tbx = s2_resampler(product) 57 | w, h = product.getSceneRasterWidth(),product.getSceneRasterHeight() 58 | 59 | h=512 60 | def time_load(p_,angles=True): 61 | 62 | tt=time.time 63 | tdiff=[] 64 | for band in list(bands)[::-1]: 65 | arr = np.empty((w,h)) 66 | t=[] 67 | t.append(tt()) 68 | p_.getBand(band).readPixels(0,0,w,h,arr) 69 | p_.getBand(band).dispose() 70 | t.append(tt()) 71 | tdiff.append(np.diff(t)) 72 | 73 | return 'mean time to load a row: %.4f'%np.mean(tdiff)+'s' 74 | 75 | 76 | if sampler == 'core': 77 | print('1st call with angles from core sampler: ',time_load(p_core,angles=False)) 78 | print('2nd call with angles from core sampler: ',time_load(p_core,angles=False)) 79 | else: 80 | print('1st call without angles from s2tbx sampler: ',time_load(p_s2tbx,angles=False)) 81 | print('2nd call without angles from s2tbx sampler: ',time_load(p_s2tbx,angles=False)) 82 | -------------------------------------------------------------------------------- /issues/resampling/test_snappy_landsat_op.py: -------------------------------------------------------------------------------- 1 | import time,sys 2 | import numpy as np 3 | from esasnappy import ProductIO, GPF, jpy 4 | import geopandas as gpd 5 | 6 | #file=sys.argv[1] 7 | file='/home/harmel/Dropbox/rus/S2/gernez/L1C/S2A_MSIL1C_20170210T082051_N0204_R121_T33HYD_20170210T083752.SAFE/' 8 | file='/DATA/Satellite/LANDSAT/L8/L1TP/LC08_L1TP_196029_20180914_20180928_01_T1/LC08_L1TP_196029_20180914_20180928_01_T1_MTL.txt' 9 | shapefile = '/DATA/OBS2CO/vrac/shape/SPO04/SPO04.shp' 10 | 11 | 12 | def generic_resampler(product, resolution=20, method='Bilinear'): 13 | '''method: Nearest, Bilinear''' 14 | 15 | GPF.getDefaultInstance().getOperatorSpiRegistry().loadOperatorSpis() 16 | 17 | HashMap = jpy.get_type('java.util.HashMap') 18 | 19 | parameters = HashMap() 20 | parameters.put('targetResolution', resolution) 21 | parameters.put('upsampling', method) 22 | parameters.put('downsampling', 'Mean') 23 | parameters.put('flagDownsampling', 'First') 24 | parameters.put('resampleOnPyramidLevels', False) 25 | 26 | return GPF.createProduct('Resample', parameters, product) 27 | 28 | def get_subset(product, wkt): 29 | '''subset from wkt POLYGON ''' 30 | SubsetOp = jpy.get_type('org.esa.snap.core.gpf.common.SubsetOp') 31 | WKTReader = jpy.get_type('com.vividsolutions.jts.io.WKTReader') 32 | 33 | grid = WKTReader().read(wkt) 34 | op = SubsetOp() 35 | op.setSourceProduct(product) 36 | op.setGeoRegion(grid) 37 | op.setCopyMetadata(True) 38 | return op.getTargetProduct() 39 | 40 | 41 | 42 | 43 | def shp2wkt(shapefile): 44 | tmp = gpd.GeoDataFrame.from_file(shapefile) 45 | tmp.to_crs(epsg=4326, inplace=True) 46 | return tmp.geometry.values[0].to_wkt() 47 | 48 | 49 | product = ProductIO.readProduct(file) 50 | 51 | p_core = generic_resampler(l2h.product_) 52 | 53 | wkt = shp2wkt(shapefile) 54 | p_ = get_subset(p_core,wkt) 55 | w, h = p_.getSceneRasterWidth(),p_.getSceneRasterHeight() 56 | arr__=p_.getBand('green').readPixels(0,0,w,h,np.zeros((w,h))) 57 | 58 | # 59 | # def time_load(p_,angles=True): 60 | # 61 | # tt=time.time 62 | # tdiff=[] 63 | # for band in p_.getBandNames(): #('B1', 'B2', 'B3', 'B4', 'B5', 'B6', 'B7', 'B8', 'B8A', 'B11', 'B12'): 64 | # print(band) 65 | # arr = np.empty(w) 66 | # t=[] 67 | # t.append(tt()) 68 | # p_.getBand(band).readPixels(0,0,w,1,arr) 69 | # #t.append(tt()) 70 | # if angles: 71 | # angband='view_zenith_'+band 72 | # p_.getBand(angband).readPixels(0,0,w,1,arr) 73 | # t.append(tt()) 74 | # tdiff.append(np.diff(t)) 75 | # 76 | # return 'mean time to load a row: %.4f'%np.mean(tdiff)+'s' 77 | # 78 | # print('1st call with angles from core sampler: ',time_load(p_core)) 79 | # print('2nd call with angles from core sampler: ',time_load(p_core)) 80 | # print('1st call with angles from s2tbx sampler: ',time_load(p_s2tbx)) 81 | # print('2nd call with angles from s2tbx sampler: ',time_load(p_s2tbx)) 82 | # print('1st call without angles from s2tbx sampler: ',time_load(p_s2tbx,angles=False)) 83 | # print('2nd call without angles from s2tbx sampler: ',time_load(p_s2tbx,angles=False)) 84 | # 85 | # 86 | -------------------------------------------------------------------------------- /issues/resampling/test_snappy_resampling.py: -------------------------------------------------------------------------------- 1 | import time,sys 2 | import tracemalloc 3 | import numpy as np 4 | from esasnappy import ProductIO, GPF, jpy 5 | 6 | 7 | #file=sys.argv[1] 8 | file='/home/harmel/Dropbox/rus/S2/gernez/L1C/S2A_MSIL1C_20170210T082051_N0204_R121_T33HYD_20170210T083752.SAFE/' 9 | file='/DATA/Satellite/LANDSAT/L8/L1TP/LC08_L1TP_196029_20180914_20180928_01_T1/LC08_L1TP_196029_20180914_20180928_01_T1_MTL.txt' 10 | 11 | 12 | def generic_resampler(product, resolution=20, method='Bilinear'): 13 | '''method: Nearest, Bilinear''' 14 | 15 | GPF.getDefaultInstance().getOperatorSpiRegistry().loadOperatorSpis() 16 | 17 | HashMap = jpy.get_type('java.util.HashMap') 18 | 19 | parameters = HashMap() 20 | parameters.put('targetResolution', resolution) 21 | parameters.put('upsampling', method) 22 | parameters.put('downsampling', 'Mean') 23 | parameters.put('flagDownsampling', 'FlagMedianAnd') 24 | parameters.put('resampleOnPyramidLevels', False) 25 | 26 | return GPF.createProduct('Resample', parameters, product) 27 | 28 | def s2_resampler(product, resolution=20, upmethod='Bilinear', downmethod='Mean', 29 | flag='FlagMedianAnd', opt=False): 30 | 31 | '''Resampler operator dedicated to Sentinel2-msi characteristics (e.g., viewing angles) 32 | :param product: S2-msi product as provided by esasnappy ProductIO.readProduct() 33 | :param resolution: target resolution in meters (10, 20, 60) 34 | :param upmethod: interpolation method ('Nearest', 'Bilinear', 'Bicubic') 35 | :param downmethod: aggregation method ('First', 'Min', 'Max', 'Mean', 'Median') 36 | :param flag: method for flags aggregation ('First', 'FlagAnd', 'FlagOr', 'FlagMedianAnd', 'FlagMedianOr') 37 | :param opt: resample on pyramid levels (True/False) 38 | :return: interpolated target product''' 39 | 40 | res = str(resolution) 41 | 42 | resampler = jpy.get_type('org.esa.s2tbx.s2msi.resampler.S2ResamplingOp') 43 | 44 | op = resampler() 45 | op.setParameter('targetResolution', res) 46 | op.setParameter('upsampling', upmethod) 47 | op.setParameter('downsampling', downmethod) 48 | op.setParameter('flagDownsampling', flag) 49 | op.setParameter('resampleOnPyramidLevels', opt) 50 | op.setSourceProduct(product) 51 | 52 | return op.getTargetProduct() 53 | 54 | 55 | 56 | product = ProductIO.readProduct(file) 57 | p_core = generic_resampler(product) 58 | p_s2tbx = s2_resampler(product) 59 | w, h = product.getSceneRasterWidth(),product.getSceneRasterHeight() 60 | 61 | h=512 62 | def time_load(p_,angles=True): 63 | 64 | tt=time.time 65 | tdiff=[] 66 | for band in p_.getBandNames(): #('B1', 'B2', 'B3', 'B4', 'B5', 'B6', 'B7', 'B8', 'B8A', 'B11', 'B12'): 67 | #print(band) 68 | arr = np.empty((w,h)) 69 | t=[] 70 | t.append(tt()) 71 | p_.getBand(band).readPixels(0,0,w,h,arr) 72 | #t.append(tt()) 73 | if angles: 74 | angband='view_zenith_'+band 75 | p_.getBand(angband).readPixels(0,0,w,h,arr) 76 | t.append(tt()) 77 | tdiff.append(np.diff(t)) 78 | 79 | return 'mean time to load a row: %.4f'%np.mean(tdiff)+'s' 80 | 81 | arg=sys.argv[1] 82 | if arg == 'core': 83 | print('1st call with angles from core sampler: ',time_load(p_core,angles=False)) 84 | print('2nd call with angles from core sampler: ',time_load(p_core,angles=False)) 85 | else: 86 | print('1st call without angles from s2tbx sampler: ',time_load(p_s2tbx,angles=False)) 87 | print('2nd call without angles from s2tbx sampler: ',time_load(p_s2tbx,angles=False)) 88 | 89 | # snapshot = tracemalloc.take_snapshot() 90 | # top_stats = snapshot.statistics('lineno') 91 | # 92 | # print("[ Top 10 ]") 93 | # for stat in top_stats[:10]: 94 | # print('s2tbx',stat) 95 | # print('1st call with angles from core sampler: ',time_load(p_core)) 96 | # print('2nd call with angles from core sampler: ',time_load(p_core)) 97 | # print('1st call with angles from s2tbx sampler: ',time_load(p_s2tbx)) 98 | # print('2nd call with angles from s2tbx sampler: ',time_load(p_s2tbx)) 99 | -------------------------------------------------------------------------------- /issues/s2sampler/README.md: -------------------------------------------------------------------------------- 1 | see https://forum.step.esa.int/t/large-time-difference-when-using-core-or-s2tbx-resamplers-for-s2-data-with-view-angles-from/11791/3 2 | 3 | run example: 4 | ```bash 5 | /usr/bin/time -f "RSS=%M elapsed=%E cpu.sys=%S .user=%U" python3 test2_snappy_resampling.py 'core' S2A_MSIL1C_20170210T082051_N0204_R121_T33HYD_20170210T083752.SAFE 6 | ``` -------------------------------------------------------------------------------- /issues/s2sampler/test2_snappy_resampling.py: -------------------------------------------------------------------------------- 1 | import time,sys 2 | import numpy as np 3 | from esasnappy import ProductIO, GPF, jpy 4 | 5 | from grs import s2_angle 6 | 7 | file='/DATA/Satellite/SENTINEL2/venice/L1C/S2A_OPER_PRD_MSIL1C_PDMC_20170121T221023_R022_V20150902T101026_20150902T101026.SAFE' 8 | 9 | sampler=sys.argv[1] 10 | file=sys.argv[2] 11 | 12 | 13 | def generic_resampler(product, resolution=20, method='Bilinear'): 14 | '''method: Nearest, Bilinear''' 15 | 16 | GPF.getDefaultInstance().getOperatorSpiRegistry().loadOperatorSpis() 17 | 18 | HashMap = jpy.get_type('java.util.HashMap') 19 | 20 | parameters = HashMap() 21 | parameters.put('targetResolution', resolution) 22 | parameters.put('upsampling', method) 23 | parameters.put('downsampling', 'Mean') 24 | parameters.put('flagDownsampling', 'FlagMedianAnd') 25 | parameters.put('resampleOnPyramidLevels', False) 26 | 27 | return GPF.createProduct('Resample', parameters, product) 28 | 29 | def s2_resampler(product, resolution=20, upmethod='Bilinear', downmethod='Mean', 30 | flag='FlagMedianAnd', opt=False): 31 | 32 | '''Resampler operator dedicated to Sentinel2-msi characteristics (e.g., viewing angles) 33 | :param product: S2-msi product as provided by esasnappy ProductIO.readProduct() 34 | :param resolution: target resolution in meters (10, 20, 60) 35 | :param upmethod: interpolation method ('Nearest', 'Bilinear', 'Bicubic') 36 | :param downmethod: aggregation method ('First', 'Min', 'Max', 'Mean', 'Median') 37 | :param flag: method for flags aggregation ('First', 'FlagAnd', 'FlagOr', 'FlagMedianAnd', 'FlagMedianOr') 38 | :param opt: resample on pyramid levels (True/False) 39 | :return: interpolated target product''' 40 | 41 | res = str(resolution) 42 | 43 | resampler = jpy.get_type('org.esa.s2tbx.s2msi.resampler.S2ResamplingOp') 44 | 45 | op = resampler() 46 | op.setParameter('targetResolution', res) 47 | op.setParameter('upsampling', upmethod) 48 | op.setParameter('downsampling', downmethod) 49 | op.setParameter('flagDownsampling', flag) 50 | op.setParameter('resampleOnPyramidLevels', opt) 51 | op.setSourceProduct(product) 52 | 53 | return op.getTargetProduct() 54 | 55 | #------------ 56 | # calculation 57 | import os 58 | xml=os.path.join(file,'GRANULE/S2A_OPER_MSI_L1C_TL_EPA__20161008T200750_A001020_T32TQR_N02.04/S2A_OPER_MTD_L1C_TL_EPA__20161008T200750_A001020_T32TQR.xml') 59 | 60 | 61 | 62 | s2_angle.s2angle_v2().angle_computation(xml) 63 | 64 | product = ProductIO.readProduct(file) 65 | product.getMetadataRoot().getElement('Granules').getElement('Level-1C_Tile_12QWM').getElement('Geometric_Info') 66 | 67 | 68 | p_core = generic_resampler(product) 69 | p_s2tbx = s2_resampler(product) 70 | w, h = product.getSceneRasterWidth(),product.getSceneRasterHeight() 71 | 72 | h=512 73 | def time_load(p_,angles=True): 74 | 75 | tt=time.time 76 | tdiff=[] 77 | for band in p_.getBandNames(): 78 | arr = np.empty((w,h)) 79 | t=[] 80 | t.append(tt()) 81 | arr = p_.getBand(band).readPixels(0,0,w,h,arr) 82 | t.append(tt()) 83 | tdiff.append(np.diff(t)) 84 | 85 | return 'mean time to load a row: %.4f'%np.mean(tdiff)+'s' 86 | 87 | arg=sys.argv[1] 88 | if arg == 'core': 89 | print('1st call with angles from core sampler: ',time_load(p_core,angles=False)) 90 | print('2nd call with angles from core sampler: ',time_load(p_core,angles=False)) 91 | else: 92 | print('1st call without angles from s2tbx sampler: ',time_load(p_s2tbx,angles=False)) 93 | print('2nd call without angles from s2tbx sampler: ',time_load(p_s2tbx,angles=False)) 94 | -------------------------------------------------------------------------------- /issues/s2sampler/test_pythonS2resample.py: -------------------------------------------------------------------------------- 1 | 2 | import os, glob 3 | import numpy as np 4 | import xarray as xr 5 | import xml.etree.ElementTree as ET 6 | 7 | 8 | from collections import namedtuple 9 | 10 | 11 | opj =os.path.join 12 | imageSAFE = "test/data/S2B_MSIL1C_20180927T103019_N0206_R108_T31TGK_20180927T143835.SAFE" 13 | xml_file=glob.glob(opj(imageSAFE,'GRANULE','*','MTD_TL.xml'))[0] 14 | with open(xml_file) as xml_: 15 | tree = ET.parse(xml_) 16 | root = tree.getroot() 17 | 18 | 19 | # Internal parsing function for angular grids 20 | def parse_angular_grid_node(node): 21 | values = [] 22 | for c in node.find('Values_List'): 23 | values.append(np.array([float(t) for t in c.text.split()])) 24 | values_array = np.stack(values) 25 | return values_array 26 | 27 | # Parse sun angles 28 | Angles = namedtuple('Angles', 'zenith azimuth') 29 | 30 | sun_angles = Angles( 31 | parse_angular_grid_node( 32 | root.find('.//Angles_Grids_List/Sun_Angles_Grids/Zenith')), 33 | parse_angular_grid_node( 34 | root.find( 35 | './/Angles_Grids_List/Sun_Angles_Grids/Azimuth'))) 36 | 37 | # Parse incidence angles 38 | self.incidence_angles = {} 39 | for b in root.find( 40 | './/Angles_Grids_List/Viewing_Incidence_Angles_Grids_List' 41 | ): 42 | if b.attrib['band_id'] != 'B1': 43 | band_key = self.Band(b.attrib['band_id']) 44 | band_dict = {} 45 | for d in b.findall('Viewing_Incidence_Angles_Grids'): 46 | det_key = self.Detector(int(d.attrib['detector_id'])) 47 | zen = parse_angular_grid_node(d.find('Zenith')) 48 | az = parse_angular_grid_node(d.find('Azimuth')) 49 | band_dict[det_key] = Angles(zen, az) 50 | self.incidence_angles[band_key] = band_dict 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /issues/timelag/S2_timelag_MSI.csv: -------------------------------------------------------------------------------- 1 | Inter-band Pair Temporal offset between bands (seconds) 2 | data from https://earth.esa.int/web/sentinel/technical-guides/sentinel-2-msi/msi-instrument 3 | date: 2020-06-25 4 | band1,band2,timelag 5 | B08,B02,0.264 6 | B03,B08,0.264 7 | B03,B02,0.527 8 | B10,B03,0.324 9 | B10,B02,0.851 10 | B04,B10,0.154 11 | B04,B02,1.005 12 | B05,B04,0.264 13 | B05,B02,1.269 14 | B11,B05,0.199 15 | B11,B02,1.468 16 | B06,B11,0.057 17 | B06,B02,1.525 18 | B07,B06,0.265 19 | B07,B02,1.79 20 | B8a,B07,0.265 21 | B8a,B02,2.055 22 | B12,B8a,0.03 23 | B12,B02,2.085 24 | B01,B12,0.229 25 | B01,B02,2.314 26 | B09,B01,0.271 27 | B09,B02,2.586 28 | -------------------------------------------------------------------------------- /issues/timelag/S2_timelag_MSI_rel.csv: -------------------------------------------------------------------------------- 1 | "time lag in second between MSI bands" 2 | band,B02,B08,B03,B10,B04,B05,B11,B06,B07,B08a,B12,B01,B09 3 | timelag,0.,0.264,0.527,0.851,1.005,1.2690000000000001,1.4680000000000002,1.525,1.79,2.055,2.085,2.314,2.5860000000000003 4 | -------------------------------------------------------------------------------- /issues/timelag/mega_glinted_pixels.csv: -------------------------------------------------------------------------------- 1 | Wavelength Pin 1 Pin 2 Pin 3 Pin 4 Pin 5 Pin 6 Pin 7 Pin 8 Pin 9 2 | 443.0 0.321 0.5112 0.44 0.1164 0.11570000000000001 0.11330000000000001 0.2179 0.44580000000000003 0.7843 3 | 490.0 0.3473 0.6591 0.512 0.0903 0.08650000000000001 0.08560000000000001 0.1322 1.4623000000000002 1.584 4 | 560.0 0.3487 0.68 0.5196000000000001 0.07930000000000001 0.07050000000000001 0.06670000000000001 0.1164 1.4675 1.5451000000000001 5 | 665.0 0.37670000000000003 0.7299 0.5646 0.0529 0.0509 0.0492 0.1587 1.627 1.7147000000000001 6 | 705.0 0.3527 0.6876 0.5576 0.0683 0.0563 0.049300000000000004 0.10790000000000001 1.387 1.5571000000000002 7 | 740.0 0.3516 0.6901 0.5537000000000001 0.058600000000000006 0.046700000000000005 0.037000000000000005 0.1013 1.3215000000000001 1.5257 8 | 783.0 0.37370000000000003 0.7141000000000001 0.5785 0.0611 0.048100000000000004 0.037700000000000004 0.1409 1.2994 1.496 9 | 842.0 0.3467 0.7291000000000001 0.5543 0.0519 0.037000000000000005 0.0303 0.0637 1.7274 1.8407 10 | 865.0 0.3811 0.7243 0.5880000000000001 0.049600000000000005 0.0381 0.027 0.12380000000000001 1.2302 1.4133 11 | 945.0 0.0489 0.08600000000000001 0.0724 0.0086 0.007200000000000001 0.0054 0.027700000000000002 0.0631 0.126 12 | 1375.0 0.0013000000000000002 0.002 0.0019 0.0015 0.0012000000000000001 5.0E-4 0.0014 0.0026000000000000003 0.0033 13 | 1610.0 0.3356 0.677 0.5444 0.009600000000000001 0.0081 0.0058000000000000005 0.0847 1.206 1.2032 14 | 2190.0 0.26 0.4994 0.40390000000000004 0.0029000000000000002 0.005200000000000001 0.0034000000000000002 0.0884 0.8015 0.9743 15 | 16 | -------------------------------------------------------------------------------- /issues/timelag/plot_toa_refl_timelag.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import pandas as pd 4 | 5 | import matplotlib as mpl 6 | import matplotlib.pyplot as plt 7 | from mpl_toolkits.axes_grid1 import make_axes_locatable 8 | 9 | plt.rcParams.update({'font.size': 16}) 10 | 11 | file='issues/timelag/S2_timelag_MSI_rel.csv' 12 | lags = pd.read_csv(file,header=None,skiprows=1,index_col=0) 13 | lags = lags.T.sort_values('band') 14 | reflfile = 'issues/timelag/mega_glinted_pixels.csv' 15 | refl = pd.read_csv(reflfile,sep='\t') 16 | refl['lags']= lags.timelag.values.astype('float') 17 | refl= refl.set_index(['Wavelength','lags']) 18 | 19 | fig, axs = plt.subplots(figsize=(8, 5)) 20 | #axs = axs.ravel() 21 | for name, refl_ in refl.iteritems(): 22 | print(refl_.index) 23 | wl =refl_.index._get_level_values(0) 24 | timelag= refl_.index._get_level_values(1) 25 | axs.plot(wl, refl_, label=name , alpha=0.75) 26 | c =axs.scatter(wl, refl_, cmap="Spectral_r",c=timelag, lw=0.5, alpha=1) 27 | plt.legend() 28 | 29 | divider = make_axes_locatable(axs) 30 | cax = divider.append_axes('right', size='5%', pad=0.05) 31 | cbar = fig.colorbar(c,cax=cax, format=mpl.ticker.ScalarFormatter(), 32 | shrink=1.0, fraction=0.1, pad=0) 33 | 34 | axs.set_ylabel('TOA reflectance (L1C)') 35 | axs.set_xlabel('Wavelength (nm)') 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /launch_grs_exemple.pbs: -------------------------------------------------------------------------------- 1 | #Lancer grs à travers un job PBS : 2 | 3 | #!/bin/bash 4 | 5 | #PBS -N grs 6 | #PBS -l select=1:ncpus=12:mem=60gb 7 | #PBS -l walltime=24:00:00 8 | 9 | #load avail module for conda, snap and jdk 10 | module load snap/8.0 conda jdk/1.8.0_112 11 | 12 | #otb-depends/7.0-python3.6.5 13 | 14 | #set the variable environment for je jpy and snap use 15 | export JDK_HOME=$JDKHOME 16 | export JAVAHOME=$JAVA_HOME 17 | export SNAP_HOME=$SNAPHOME 18 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:. 19 | 20 | ##activate conda environement python 3.6 21 | conda activate grs_py3.6 22 | cd /work/ALT/swot/aval/OBS2CO/git/grs2 23 | 24 | grs /datalake/S2-L1C/48QWE/2020/04/28/S2A_MSIL1C_20200428T032541_N0209_R018_T48QWE_20200428T062927.SAFE --aerosol cams_forecast --odir /work/ALT/swot/aval/OBS2CO/Test-GRS-sisppeo/results/ --dem --resolution 20 --waterdetect /work/ALT/swot/aval/OBS2CO/Output_WaterDetect/Mekong/2020-04/T48QWE/SENTINEL2A_20200428-034316-659_L2A_T48QWE_C_V2-2/AC_NB_mndwindwi/AC_NB_mndwindwi_water_mask.tif 25 | 26 | #grs /work/ALT/swot/aval/OBS2CO/git/grsdata/INPUT_DATA/S2B_MSIL1C_20180927T103019_N0206_R108_T31TGK_20180927T143835.SAFE --shape test/data/shape/SPO04.shp --odir test/results/ --aerosol cams_forecast --dem --resolution 20 27 | 28 | #grs /work/ALT/swot/aval/OBS2CO/git/grsdata/INPUT_DATA/IMAGES/S2B_MSIL1C_20201214T092309_N0209_R093_T33PVP_20201214T113135.SAFE --odir test_cluster/results/ --aerosol cams_forecast --dem --resolution 20 29 | 30 | #grs /work/ALT/swot/aval/OBS2CO/git/grsdata/INPUT_DATA/IMAGES/S2A_MSIL1C_20190602T104031_N0207_R008_T31TFJ_20190602T125323.SAFE --odir testi_cluster/results/ --aerosol cams_forecast --dem --resolution 20 31 | 32 | #grs /work/ALT/swot/aval/OBS2CO/git/grsdata/INPUT_DATA/IMAGES/S2A_MSIL1C_20201219T092411_N0209_R093_T33PVQ_20201219T103844.SAFE --odir test/results/ --aerosol cams_forecast --dem --resolution 20 33 | 34 | #grs /work/ALT/swot/aval/OBS2CO/git/grsdata/INPUT_DATA/IMAGES/S2B_MSIL1C_20190607T104029_N0207_R008_T31TFJ_20190607T125247.SAFE --odir test/results/ --aerosol cams_forecast --dem --resolution 20 35 | -------------------------------------------------------------------------------- /pbs_ecmwf.pbs: -------------------------------------------------------------------------------- 1 | #Recuperer les donnees ecmwf cams à travers un job PBS : 2 | 3 | #!/bin/bash 4 | 5 | #PBS -N ecmwf_cams 6 | #PBS -l select=1:ncpus=4:mem=20000mb 7 | #PBS -l walltime=72:00:00 8 | 9 | module load python 10 | 11 | python /work/ALT/swot/aval/OBS2CO/git/grs2/ecmwf/download_cams_all.py cams_forecast all 12 | -------------------------------------------------------------------------------- /pypi_README.rst: -------------------------------------------------------------------------------- 1 | Usage 2 | ===== 3 | 4 | WARNING: installation from `GitHub repository `__ is recommended 5 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 6 | 7 | 8 | Installation 9 | ~~~~~~~~~~~~~~~~~~~~~~~ 10 | 11 | These instructions will get you a copy of the project up and running on 12 | your local machine for development and testing purposes. See deployment 13 | for notes on how to deploy the project on a live system. 14 | 15 | Download the LUT files 16 | ----------------------- 17 | 18 | click on 19 | `grsdata `__ 20 | to download and save in your desired path (your_GRSDATA_PATH) 21 | 22 | Installation with conda environment 23 | ------------------------------------ 24 | 25 | :: 26 | 27 | conda activate "name of your conda env" 28 | 29 | Python >= 3.9 is recommended, example: 30 | 31 | :: 32 | 33 | conda create python=3.10 -n grs_v2 34 | conda activate grs_v2 35 | 36 | Then, install python dependencies: 37 | 38 | :: 39 | 40 | conda install -c conda-forge eoreader cdsapi netCDF4 docopt xmltodict numba 41 | 42 | Set the ``config.yml`` file: 43 | 44 | :: 45 | 46 | path: 47 | grsdata: your_GRSDATA_PATH 48 | 49 | Finally, install grs with: 50 | 51 | .. code:: 52 | 53 | pip install . 54 | 55 | Testing 56 | ~~~~~~~~ 57 | 58 | After installation, you can type: 59 | 60 | .. code:: 61 | 62 | grs -h 63 | 64 | You should see something like: 65 | 66 | .. code:: 67 | 68 | Executable to process Sentinel-2 L1C images for aquatic environment 69 | 70 | Usage: 71 | grs [--cams_file file] [-o ] [--odir ] [--resolution res] [--scale_aot factor] [--levname ] [--no_clobber] [--allpixels] [--surfwater file] [--dem_file file] [--snap_compliant] 72 | grs -h | --help 73 | grs -v | --version 74 | 75 | Options: 76 | -h --help Show this screen. 77 | -v --version Show version. 78 | 79 | Input file to be processed 80 | 81 | --cams_file file Absolute path of the CAMS file to be used (mandatory) 82 | 83 | -o ofile Full (absolute or relative) path to output L2 image. 84 | --odir odir Ouput directory [default: ./] 85 | --levname lev Level naming used for output product [default: L2Agrs] 86 | --no_clobber Do not process if already exists. 87 | --resolution=res spatial resolution of the scene pixels 88 | --allpixels force to process all pixels whatever they are masked (cloud, vegetation...) or not 89 | --surfwater file Absolute path of the surfwater geotiff file to be used 90 | --dem_file file Absolute path of the DEM geotiff file (already subset for the S2 tile) 91 | --scale_aot factor scaling factor applied to CAMS aod550 raster 92 | [default: 1] 93 | --opac_model name Force the aerosol model (OPAC) to be 'name' 94 | (choice: ['ARCT_rh70', 'COAV_rh70', 'DESE_rh70', 95 | 'MACL_rh70', 'URBA_rh70']) 96 | --snap_compliant Export output to netcdf aligned with "beam" for ESA SNAP software 97 | 98 | Example: 99 | grs /data/satellite/S2/L1C/S2B_MSIL1C_20220731T103629_N0400_R008_T31TFJ_20220731T124834.SAFE --cams_file /data/satellite/S2/cnes/CAMS/2022-07-31-cams-global-atmospheric-composition-forecasts.nc --resolution 60 100 | For CNES datalake: 101 | grs /work/datalake/S2-L1C/31TFJ/2023/06/16/S2B_MSIL1C_20230616T103629_N0509_R008_T31TFJ_20230616T111826.SAFE --cams_file /work/datalake/watcal/ECMWF/CAMS/2023/06/16/2023-06-16-cams-global-atmospheric-composition-forecasts.nc --odir /work/datalake/watcal/test --resolution 20 --dem_file /work/datalake/static_aux/MNT/COP-DEM_GLO-30-DGED_S2_tiles/COP-DEM_GLO-30-DGED_31TFJ.tif 102 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | # pyproject.toml 2 | [build-system] 3 | requires = ["setuptools>=61.0.0", "wheel"] 4 | build-backend = "setuptools.build_meta" 5 | 6 | [project] 7 | name = "GRSprocessor" 8 | version = "2.1.7" 9 | description = "GRS processor for atmospheric correction of high-spatial resolution and multispectral satellite images" 10 | readme = "pypi_README.rst" 11 | authors = [{ name = "Tristan Harmel", email = "tristan.harmel@gmail.com" }] 12 | license = { file = "LICENSE" } 13 | classifiers = [ 14 | "License :: OSI Approved :: Apache Software License", 15 | "Programming Language :: Python", 16 | "Programming Language :: Python :: 3", 17 | ] 18 | keywords = ["Sentinel-2","Landsat-8","Landsat-9","atmospheric correction", "EO satellite"] 19 | requires-python = ">=3.9" 20 | dynamic = ["dependencies"] 21 | 22 | [tool.setuptools.packages.find] 23 | where = ["."] 24 | 25 | [tool.setuptools.package-data] 26 | #"grs"= [] 27 | "grs"= ['*.yml','data/lut/gases/*.nc', 'data/aux/*.txt'] 28 | 29 | [tool.setuptools.exclude-package-data] 30 | "*" = [".gitattributes"] 31 | 32 | [project.scripts] 33 | grs = "grs.run:main" 34 | 35 | #dynamic = ["dependencies"] 36 | [tool.setuptools.dynamic] 37 | dependencies = { file = ["requirements.txt"] } 38 | 39 | [project.optional-dependencies] 40 | dev = ["black", "bumpver", "isort", "pip-tools", "pytest"] 41 | 42 | [project.urls] 43 | Homepage = "https://github.com/Tristanovsk/grs" 44 | Documentation = "https://grs.readthedocs.io" -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | 2 | importlib_resources 3 | 4 | cdsapi>=0.6.1 5 | GRSdriver >= 1.0.3 6 | xmltodict 7 | docopt 8 | rasterio 9 | netCDF4 10 | numba 11 | numpy 12 | pandas>=2.0.3 13 | pyproj 14 | xarray 15 | rioxarray>=0.15.0 16 | s2cloudless>=1.7.1 17 | matplotlib 18 | pyyaml -------------------------------------------------------------------------------- /run_docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo The ID of the Docker image is........"$1" 3 | echo The image to be processed is........."$2" 4 | echo The CAMS files are located at........"$3" 5 | echo The output file will be named........"$4" 6 | echo The output file will located at......"$5" 7 | echo The requested resolution is.........."$6" 8 | echo The SURFWATER files are located at..."$7" 9 | 10 | # Create and parametrize container 11 | sudo docker run -d --name grs2 $1 12 | sudo docker cp $2 grs2:/home/L1C 13 | sudo docker cp $3 grs2:/home/CAMS 14 | 15 | # Launch processing and retreive result 16 | if [ -z "$7" ] 17 | then 18 | sudo docker exec grs2 grs /home/L1C --cams_file /home/CAMS -o $4 --resolution $6 19 | else 20 | sudo docker cp $7 grs2:/home/SW 21 | sudo docker exec grs2 grs /home/L1C --surfwater /home/SW --cams_file /home/CAMS -o $4 --resolution $6 22 | fi 23 | sudo docker cp grs2:/home/$4 $5/$4 24 | echo Result written at location $5/$4 25 | 26 | # Delete container 27 | sudo docker kill grs2 28 | sudo docker rm grs2 29 | --------------------------------------------------------------------------------