├── projects ├── AquaINFRA │ ├── README.MD │ ├── requirements_AquaINFRA.txt │ ├── doc │ │ └── pyOWT_description.docx │ ├── pygeoapi_processes │ │ ├── config.TEMPLATE.json │ │ ├── hereon_pyowt.json │ │ ├── README.md │ │ ├── owt_classification.json │ │ ├── hereon_pyowt.py │ │ └── owt_classification.py │ ├── cmems │ │ ├── extract_spec.csv │ │ └── product_wavebands.txt │ ├── data │ │ ├── Rrs_demo_AquaINFRA_msi.csv │ │ ├── Rrs_demo_AquaINFRA_olci.csv │ │ └── Rrs_demo_AquaINFRA_hyper.csv │ └── run_AquaINFRA.py ├── zenodo │ ├── requirements_zenodo.txt │ ├── README.md │ ├── data_reshape_to_netcdf_prepare.R │ ├── data_reshape_to_netcdf_olci.py │ └── data_reshape_to_netcdf_hyper.py └── covm_shrinkage │ ├── figs │ ├── m_Rrs.png │ ├── avw_abc.png │ ├── avw_ndi.png │ └── m_nRrs.png │ ├── calculate_OWT_centroids_v03.py │ ├── calculate_OWT_centroids_v99.py │ ├── calculate_OWT_centroids_v01.py │ └── calculate_OWT_centroids_v02.py ├── pyowt ├── satellite_handlers │ ├── __init__.py │ ├── cmems_products.py │ ├── srf_convolution.py │ ├── envi_liu_products.py │ ├── lakecci_products.py │ └── eumetsat_olci_level2.py ├── data │ ├── OWT_centroids.nc │ ├── v01 │ │ └── OWT_centroids.nc │ ├── v02 │ │ └── OWT_centroids.nc │ ├── AVW_all_regression_800.txt │ └── sensor_band_library.yaml ├── __init__.py ├── OpticalVariables.py ├── OWT.py └── PlotOWT.py ├── .gitattributes ├── figs └── owt_spec.png ├── requirements.txt ├── data ├── generate_AquaINFRA_demo_data.py └── generate_sensor_lib_yaml.py ├── setup.py ├── run_examples.py ├── .gitignore └── README.md /projects/AquaINFRA/README.MD: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pyowt/satellite_handlers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /projects/zenodo/requirements_zenodo.txt: -------------------------------------------------------------------------------- 1 | pandas 2 | numpy 3 | xarray 4 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /figs/owt_spec.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bishun945/pyOWT/HEAD/figs/owt_spec.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | pandas 3 | scipy 4 | PyYAML 5 | netCDF4 6 | xarray 7 | matplotlib 8 | -------------------------------------------------------------------------------- /pyowt/data/OWT_centroids.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bishun945/pyOWT/HEAD/pyowt/data/OWT_centroids.nc -------------------------------------------------------------------------------- /projects/AquaINFRA/requirements_AquaINFRA.txt: -------------------------------------------------------------------------------- 1 | pygeoapi 2 | requests 3 | xarray 4 | lxml 5 | numpy 6 | pandas 7 | -------------------------------------------------------------------------------- /pyowt/data/v01/OWT_centroids.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bishun945/pyOWT/HEAD/pyowt/data/v01/OWT_centroids.nc -------------------------------------------------------------------------------- /pyowt/data/v02/OWT_centroids.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bishun945/pyOWT/HEAD/pyowt/data/v02/OWT_centroids.nc -------------------------------------------------------------------------------- /projects/covm_shrinkage/figs/m_Rrs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bishun945/pyOWT/HEAD/projects/covm_shrinkage/figs/m_Rrs.png -------------------------------------------------------------------------------- /projects/covm_shrinkage/figs/avw_abc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bishun945/pyOWT/HEAD/projects/covm_shrinkage/figs/avw_abc.png -------------------------------------------------------------------------------- /projects/covm_shrinkage/figs/avw_ndi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bishun945/pyOWT/HEAD/projects/covm_shrinkage/figs/avw_ndi.png -------------------------------------------------------------------------------- /projects/covm_shrinkage/figs/m_nRrs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bishun945/pyOWT/HEAD/projects/covm_shrinkage/figs/m_nRrs.png -------------------------------------------------------------------------------- /projects/AquaINFRA/doc/pyOWT_description.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bishun945/pyOWT/HEAD/projects/AquaINFRA/doc/pyOWT_description.docx -------------------------------------------------------------------------------- /projects/AquaINFRA/pygeoapi_processes/config.TEMPLATE.json: -------------------------------------------------------------------------------- 1 | { 2 | "download_dir": "/var/www/nginx/download/", 3 | "download_url": "https://ourserver/download/", 4 | "pyowt": { 5 | "example_input_data_dir": ".../pygeoapi/process/pyOWT/data/", 6 | "input_data_dir": "/.../inputs_temp/", 7 | "path_sensor_band_library": ".../pygeoapi/process/pyOWT/data/sensor_band_library.yaml" 8 | } 9 | } -------------------------------------------------------------------------------- /projects/AquaINFRA/cmems/extract_spec.csv: -------------------------------------------------------------------------------- 1 | Wavelength,Pin 1,Pin 2,Pin 3 2 | 400,0.009545312,0.01002326,0.008956626 3 | 412,0.009061946,0.009702374,0.008768859 4 | 442,0.008761284,0.009075798,0.008305777 5 | 490,0.006077132,0.006217873,0.005930974 6 | 510,0.003323314,0.003664486,0.003133767 7 | 560,0.001480336,0.001757222,0.001393472 8 | 620,3.23E-04,4.36E-04,2.52E-04 9 | 665,1.83E-04,2.75E-04,1.12E-04 10 | 673,8.55E-05,3.19E-04,1.73E-04 11 | 681,3.07E-04,2.72E-04,1.03E-04 12 | 708,9.71E-06,2.66E-04,6.22E-05 13 | 778,9.13E-05,1.15E-04,1.38E-04 14 | 865,5.05E-05,2.72E-05,9.71E-06 -------------------------------------------------------------------------------- /projects/zenodo/README.md: -------------------------------------------------------------------------------- 1 | # project-zenodo 2 | 3 | Shun Bi 4 | 5 | 10.10.2024 6 | 7 | This project folders contains scripts that reformat the training data set of the proposed optical water type classification framework (Bi and Hieronymi 2024). The data set is now accessible in https://zenodo.org/records/12803329. 8 | 9 | The path in the scripts might be different from the current setup as the structure reconstruction is done on another working environment. 10 | 11 | Note that due to some GitHub policy, some large files are ignored for restoring in this repo, please find them in zenodo link or aks me directly via shun.bi@outlook.com 12 | -------------------------------------------------------------------------------- /data/generate_AquaINFRA_demo_data.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | d0 = pd.read_csv('data/Rrs_demo.csv') 4 | d = d0.pivot_table(index='SampleID', columns='wavelen', values='Rrs').round(7) 5 | 6 | # Hyperspectral version 7 | d.to_csv('data/Rrs_demo_AquaINFRA_hyper.csv', index=False) 8 | 9 | # OLCI version 10 | wavelen_olci_nominal = [400, 412, 444, 490, 510, 560, 620, 666, 674, 682, 710, 754, 780, 866] 11 | d[wavelen_olci_nominal].to_csv('data/Rrs_demo_AquaINFRA_olci.csv', index=False) 12 | 13 | # MSI version 14 | wavelen_msi_nomial = [444, 492, 560, 666, 704, 740, 784, 836] 15 | d[wavelen_msi_nomial].to_csv('data/Rrs_demo_AquaINFRA_msi.csv', index=False) 16 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | import pyowt 3 | 4 | setup( 5 | name=pyowt.__package__, 6 | version=pyowt.__version__, 7 | author='Shun Bi', 8 | packages=find_packages(), 9 | install_requires=[ 10 | 'numpy', 11 | 'pandas', 12 | 'scipy', 13 | 'PyYAML', 14 | 'netCDF4', 15 | 'xarray', 16 | 'matplotlib' 17 | ], 18 | extras_require={ 19 | 'geeowt': [ 20 | 'geemap', 21 | 'earthengine-api' 22 | ], 23 | 'satellite_handlers': [ 24 | 'osgeo', 25 | 'lxml' 26 | ] 27 | }, 28 | include_package_data=True, 29 | package_data={ 30 | 'pyowt': ['data/*'] 31 | }, 32 | license_files=('LICENSE') 33 | ) 34 | 35 | # Please use the command to install the package in terminal 36 | # pip install -e . 37 | -------------------------------------------------------------------------------- /projects/AquaINFRA/data/Rrs_demo_AquaINFRA_msi.csv: -------------------------------------------------------------------------------- 1 | 444,492,560,666,704,740,784,836 2 | 0.0044726,0.0032257,0.0007454,6.52e-05,3.45e-05,7.9e-06,8.7e-06,4.4e-06 3 | 0.0022858,0.0028376,0.0021504,0.0004164,0.0002796,7.34e-05,8.52e-05,4.93e-05 4 | 0.0050867,0.0073629,0.0109993,0.0031077,0.0022934,0.0005786,0.0007363,0.0004077 5 | 6.29e-05,0.0001418,0.0003543,0.0009131,0.0013127,0.0005679,0.0007269,0.0004483 6 | 0.0130191,0.0166833,0.0103156,0.0014616,0.0008975,0.0002221,0.0002692,0.0001497 7 | 0.0028387,0.0051732,0.0158768,0.008511,0.0212157,0.0112792,0.0130719,0.0081564 8 | 0.0069813,0.0045773,0.0012137,0.0001194,6.69e-05,1.48e-05,1.88e-05,9e-06 9 | 0.0095694,0.0156169,0.026465,0.0112396,0.0105354,0.002719,0.0032287,0.0018311 10 | 0.0087944,0.0138136,0.0239576,0.0261465,0.0279304,0.0132366,0.0157176,0.0102149 11 | 0.0020272,0.0031943,0.0081196,0.0030949,0.0048022,0.0012743,0.0017194,0.0008577 12 | -------------------------------------------------------------------------------- /projects/zenodo/data_reshape_to_netcdf_prepare.R: -------------------------------------------------------------------------------- 1 | rm(list = ls()) 2 | 3 | library(data.table) 4 | library(magrittr) 5 | 6 | library(ggplot2) 7 | 8 | data.table::setDTthreads(percent = 95) 9 | 10 | d <- fread("/Users/Bi/Documents/GitHub/OWT/Paper/Generate_training_data_set/data/OWT_simulated_data.gz") 11 | d[, ID := paste(type, source, SampleID)] 12 | 13 | d_sing <- d[wavelen == 440, .(ID, source, type, Chl, ISM, ag440, A_d, G_d, Sal, Temp, a_frac, cocco_frac)][order(ID)] 14 | 15 | d_spec <- d[,.(ID, wavelen, Rrs, agp, bp, bbp, aw, bw, ad, aph)] 16 | 17 | var_spec <- c("Rrs", "agp", "bp", "bbp", "aw", "bw", "ad", "aph") 18 | 19 | d_var_spec <- lapply(var_spec, \(x) dcast(d_spec, ID ~ wavelen, value.var = x)[order(ID)]) 20 | names(d_var_spec) <- var_spec 21 | 22 | fwrite(d_sing, "/Users/Bi/Documents/GitHub/pyOWT/test_running/reshape_netcdf_temp_data/single_variable.csv") 23 | 24 | for(var in var_spec){ 25 | fwrite(d_var_spec[[var]], sprintf("/Users/Bi/Documents/GitHub/pyOWT/test_running/reshape_netcdf_temp_data/spec_%s.csv", var)) 26 | } 27 | -------------------------------------------------------------------------------- /projects/covm_shrinkage/calculate_OWT_centroids_v03.py: -------------------------------------------------------------------------------- 1 | # 07.11.2024 Shun 2 | # In some of applications, the input Rrs spectrum is significantly affected by 3 | # various noise such as skyligth residuals, sun glint, or atmospheric correction 4 | # I have realized such disturbance will eventually change the water classification 5 | # results even though, for example, it is a just slight offset for blue ocean 6 | # spectrum (with red shifted AVW and lifting Area values). 7 | # To enhance the resistance of the OWT framework on such dataset, one should add 8 | # artificial noise into the training data and re-calcualte the centroids. 9 | # However, this might contrary the original intention of the classification 10 | # that is able to detect the unclassifiable spectrum with low data quality... 11 | 12 | # Some testings are played around here... 13 | 14 | import xarray as xr 15 | import numpy as np 16 | import pandas as pd 17 | import matplotlib.pyplot as plt 18 | 19 | fn = "projects/zenodo/owt_BH2024_training_data_hyper.nc" 20 | ds = xr.open_dataset(fn) 21 | wavelen_org = ds['wavelen'].values 22 | Rrs_org = ds['Rrs'].values 23 | ind = np.where((wavelen_org >= 400) & (wavelen_org <= 800))[0] 24 | wavelen = wavelen_org[ind] 25 | Rrs_mat = Rrs_org[:,ind] 26 | Rrs = Rrs_mat[10,:] 27 | 28 | plt.plot(wavelen, Rrs, label = 'original') 29 | plt.plot(wavelen, Rrs + 0.001, label = '+0.001') 30 | plt.legend() 31 | plt.show() 32 | -------------------------------------------------------------------------------- /projects/AquaINFRA/data/Rrs_demo_AquaINFRA_olci.csv: -------------------------------------------------------------------------------- 1 | 400,412,444,490,510,560,620,666,674,682,710,754,780,866 2 | 0.0041708,0.0043365,0.0044726,0.0033592,0.0016906,0.0007454,0.0001239,6.52e-05,6.1e-05,5.64e-05,2.69e-05,7.4e-06,8.4e-06,2.8e-06 3 | 0.0026823,0.0024629,0.0022858,0.0028334,0.0028867,0.0021504,0.0007171,0.0004164,0.0003884,0.0003724,0.0002254,7.02e-05,8.17e-05,3.27e-05 4 | 0.004288,0.0044431,0.0050867,0.0072377,0.008918,0.0109993,0.004967,0.0031077,0.0029189,0.002868,0.0018364,0.0005693,0.0007007,0.0002819 5 | 3.04e-05,3.63e-05,6.29e-05,0.0001378,0.0001857,0.0003543,0.0007116,0.0009131,0.0008869,0.0009371,0.0012263,0.000566,0.0006928,0.0003091 6 | 0.0119063,0.0120304,0.0130191,0.0167585,0.0143639,0.0103156,0.0024813,0.0014616,0.0013809,0.0013189,0.0007096,0.000216,0.0002574,0.0001012 7 | 0.0025174,0.0025228,0.0028387,0.0050606,0.0075614,0.0158768,0.0118261,0.008511,0.0069232,0.0074286,0.0220888,0.0108041,0.0125482,0.0053101 8 | 0.0106798,0.0096109,0.0069813,0.0047628,0.0024989,0.0012137,0.0002129,0.0001194,0.0001128,0.000106,5.16e-05,1.47e-05,1.8e-05,6.3e-06 9 | 0.0086336,0.0086861,0.0095694,0.0153518,0.0200474,0.026465,0.0190993,0.0112396,0.0099426,0.0100967,0.0086692,0.0025982,0.0030822,0.001196 10 | 0.0061017,0.0067437,0.0087944,0.0135679,0.0165487,0.0239576,0.0287407,0.0261465,0.0246193,0.0250746,0.0259659,0.013,0.0151499,0.0071332 11 | 0.0021119,0.0020311,0.0020272,0.0031355,0.0043933,0.0081196,0.0056503,0.0030949,0.002577,0.0027895,0.0040503,0.0012835,0.0016308,0.000607 12 | -------------------------------------------------------------------------------- /projects/AquaINFRA/cmems/product_wavebands.txt: -------------------------------------------------------------------------------- 1 | Email from Martin (17.10.2024) contains brief description of some CMEMS data sets that can be used for pyOWT testing... 2 | 3 | Copernicus Marine Service 4 | 5 | CMEMS_BAL_HROC 6 | 20230708_P1D_CMEMS_HROC_L3-optics_BAL_34V_100m-v01.nc 7 | (Baltic, from Sentinel-2, 100 m, C2RCC-Acolite, Brockmann) 8 | RRS443, RRS492, RRS560, RRS665, RRS704, RRS740, RRS783, RRS865 9 | 10 | CMEMS_ATL_MYINT 11 | 20230708_cmems_obs-oc_atl_bgc-reflectance_myint_l3-olci-300m_P1D.nc 12 | (North Atlantic Ocean, from Sentinel-3, 300 m, GlobColour, Acri) 13 | RRS400, RRS412, RRS443, RRS490, RRS510, RRS560, RRS620, RRS665, RRS674, RRS681, RRS709 14 | 15 | CMEMS_BAL_NRT 16 | 20241010_cmems_obs-oc_bal_bgc-reflectance_nrt_l3-olci-300m_P1D.nc 17 | (Baltic, from Sentinel-3, 300 m, Polymer, CNR) 18 | RRS400, RRS412_5, RRS442_5, RRS490, RRS510, RRS560, RRS620, RRS665, RRS673_75, RRS681_25, RRS708_75, RRS778_75, RRS865 19 | 20 | CMEMS_BAL_MYINT 21 | (Baltic, from diverse sensors, CNR) 22 | 20230708_cmems_obs-oc_bal_bgc-reflectance_myint_l3-multi-1km_P1D.nc 23 | RRS412, RRS443, RRS490, RRS510, RRS560, RRS665 24 | 25 | CMEMS_MED_MYINT 26 | 20230708_cmems_obs-oc_med_bgc-reflectance_myint_l3-olci-300m_P1D.nc 27 | (Mediterranean Sea, from Sentinel-3, 300 m, Polymer, CNR) 28 | RRS400, RRS412_5, RRS442_5, RRS490, RRS510, RRS560, RRS620, RRS665, RRS673_75, RRS681_25, RRS708_75, RRS778_75, RRS865 29 | 30 | 31 | Copernicus Land Service 32 | 33 | (Germany, from Sentinel-2, Polymer) 34 | Rw443_rep, Rw490_rep, Rw560_rep, Rw665_rep, Rw705_rep, Rw740_rep, Rw783_rep, Rw842_rep, Rw865_rep, Rw945_rep, Rw1375_rep, Rw1610_rep, Rw2190_rep à all not Rrs, but Rw!!! 35 | (Fully normalized water leaving reflectances at the waveband indicated in the layer name (443nm, 490nm, etc.)., using most representative spectrum within the aggregation period) 36 | 37 | https://land.copernicus.eu/en/products/water-bodies/lake-water-quality-near-real-time-v2-0-300m -------------------------------------------------------------------------------- /pyowt/satellite_handlers/cmems_products.py: -------------------------------------------------------------------------------- 1 | import xarray as xr 2 | import numpy as np 3 | import re 4 | from pyowt.OpticalVariables import OpticalVariables 5 | from pyowt.OWT import OWT 6 | 7 | class cmems_products(): 8 | 9 | def __init__(self, filename, sensor): 10 | """Reader of CMEMS products 11 | 12 | Args: 13 | filename (str): Filename of the CMEMS products. 14 | sensor (str): The sensor name of the product waveband configurations. 15 | Only accepts CMEMS_BAL_HROC, CMEMS_BAL_NRT, and CMEMS_MED_MYINT 16 | 17 | Returns: 18 | A class of `reader_cmems_products` that includes 19 | - filename 20 | - filename_output: for generating output netcdf file 21 | - sensor 22 | - Rrs: np.array used to feed into `OpticalVariables` and `OWT` 23 | - ds_Rrs: xr.dataset that used to append OWT results and with wavelength attrs for SNAP usage 24 | """ 25 | 26 | self.filename = filename 27 | self.filename_output = filename[:-3] + '_result.nc' 28 | self.sensor = sensor 29 | 30 | ds = xr.open_dataset(filename) 31 | 32 | if sensor == 'CMEMS_BAL_HROC': 33 | wavelen_sel =['RRS443', 'RRS492', 'RRS560', 'RRS665', 'RRS704', 'RRS740', 'RRS783', 'RRS865'] 34 | wavelengths = [int(re.search(r'\d+', wl).group()) for wl in wavelen_sel] 35 | elif sensor == 'CMEMS_BAL_NRT' or sensor == 'CMEMS_MED_MYINT': 36 | wavelen_sel = ['RRS400', 'RRS412_5', 'RRS442_5', 'RRS490', 'RRS510', 'RRS560', 'RRS620', 'RRS665', 'RRS673_75', 'RRS681_25', 'RRS708_75', 'RRS778_75', 'RRS865'] 37 | wavelengths = [int(re.search(r'\d+', wl).group()) for wl in wavelen_sel] 38 | else: 39 | raise ValueError('Please select CMEMS_BAL_HROC, CMEMS_BAL_NRT, or CMEMS_MED_MYINT for `sensor`.') 40 | 41 | if 'time' in ds.dims: 42 | ds_Rrs = ds[wavelen_sel].isel(time=0) 43 | else: 44 | ds_Rrs = ds[wavelen_sel] 45 | 46 | # do the correction on negative values 47 | Rrs = ds_Rrs.to_array().transpose('lat', 'lon', 'variable').values 48 | Rrs[Rrs < 0] = np.nan 49 | 50 | self.Rrs = Rrs 51 | self.wavelen = wavelengths 52 | 53 | for i, wavelength in enumerate(wavelengths): 54 | variable_name = list(ds_Rrs.data_vars)[i] 55 | ds_Rrs[variable_name].attrs['radiation_wavelength'] = wavelength 56 | ds_Rrs[variable_name].attrs['radiation_wavelength_unit'] = 'nm' 57 | 58 | self.ds_Rrs = ds_Rrs 59 | 60 | self.classification() 61 | 62 | def classification(self): 63 | ov = OpticalVariables(Rrs=self.Rrs, band=self.wavelen, sensor=self.sensor) 64 | owt = OWT(ov.AVW, ov.Area, ov.NDI) 65 | self.ds_Rrs['type_idx'] = (('lat', 'lon'), owt.type_idx.astype(np.int32)) 66 | self.ds_Rrs['type_idx'].attrs['description'] = 'Type index classification' 67 | self.ds_Rrs.to_netcdf(self.filename_output) 68 | 69 | if __name__ == "__main__": 70 | 71 | fn = '/Users/apple/Satellite_data/CMEMS/20241011_cmems_obs-oc_blk_bgc-reflectance_nrt_l3-olci-300m_P1D.nc' 72 | cmems = cmems_products(fn, sensor='CMEMS_BAL_NRT') 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /projects/AquaINFRA/pygeoapi_processes/hereon_pyowt.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.0.1", 3 | "id": "hereon-pyowt", 4 | "use_case": "HEREON", 5 | "title": {"en": "Hello pyOWT"}, 6 | "description": { 7 | "en": "An example process that ..." 8 | }, 9 | "jobControlOptions": ["sync-execute", "async-execute"], 10 | "keywords": ["hello world", "example", "echo"], 11 | "links": [{ 12 | "type": "text/html", 13 | "rel": "about", 14 | "title": "information", 15 | "href": "https://github.com/bishun945/pyOWT", 16 | "hreflang": "en-US" 17 | }], 18 | "inputs": { 19 | "input_data_url": { 20 | "title": "Input data", 21 | "description": "URL to your input file. (TODO: Not quite sure what it needs to contain.) Or try the existing examples: \"Rrs_demo_AquaINFRA_hyper.csv\", \"Rrs_demo_AquaINFRA_msi.csv\", \"Rrs_demo_AquaINFRA_olci.csv\"", 22 | "schema": { 23 | "type": "string" 24 | }, 25 | "minOccurs": 1, 26 | "maxOccurs": 1, 27 | "keywords": ["hereon", "pyOWT"] 28 | }, 29 | "input_option": { 30 | "title": "Type of input", 31 | "description": "Type of input: \"csv\" for text data input; \"sat\" for satellite products input (e.g., Sentinel-3 OLCI Level-2)", 32 | "schema": { 33 | "type": "string" 34 | }, 35 | "minOccurs": 1, 36 | "maxOccurs": 1, 37 | "keywords": ["hereon", "pyOWT"] 38 | }, 39 | "sensor": { 40 | "title": "Sensor name", 41 | "description": "Name of you sensor. Check pyOWT documentation for supported sensors. Examples: \"HYPER\", \"AVW700\", \"MODIS_Aqua\", \"MODIS_Terra\", \"OLCI_S3A\", \"OLCI_S3B\", \"MERIS\", \"SeaWiFS\", \"HawkEye\", \"OCTS\", \"GOCI\", \"VIIRS_NPP\", \"VIIRS_JPSS1\", \"VIIRS_JPSS2\", \"CZCS\", \"MSI_S2A\", \"MSI_S2B\", \"OLI\", \"ENMAP_HSI\", \"CMEMS_HROC_L3_optics\", \"cmems_P1D400\", \"AERONET_OC_1\", \"AERONET_OC_2\".", 42 | "schema": { 43 | "type": "string" 44 | }, 45 | "minOccurs": 1, 46 | "maxOccurs": 1, 47 | "keywords": ["hereon", "pyOWT"] 48 | }, 49 | "output_option": { 50 | "title": "Output option", 51 | "description": "Output option. 1: for standard output; 2: for extensive output with memberships of all types", 52 | "schema": { 53 | "type": "string" 54 | }, 55 | "minOccurs": 1, 56 | "maxOccurs": 1, 57 | "keywords": ["hereon", "pyOWT"] 58 | } 59 | }, 60 | "outputs": { 61 | "some_output": { 62 | "title": "Standard output / what is this? TODO", 63 | "description": "Standard output or extensive output. Please ask Martin Hieronimy or Shun Bi / HEREON for details. TODO,", 64 | "schema": { 65 | "type": "object", 66 | "contentMediaType": "application/json" 67 | } 68 | } 69 | 70 | }, 71 | "example": { 72 | "inputs": { 73 | "input_data_url": "Rrs_demo_AquaINFRA_hyper.csv", 74 | "input_option": "csv", 75 | "sensor": "MSI_S2A", 76 | "output_option": "1" 77 | } 78 | } 79 | } -------------------------------------------------------------------------------- /projects/AquaINFRA/pygeoapi_processes/README.md: -------------------------------------------------------------------------------- 1 | # pyOWT as OGC API processes (AquaINFRA project) 2 | 3 | ## What is the AquaINFRA project? 4 | 5 | Read here: https://aquainfra.eu/ 6 | 7 | ## What are OGC processes? 8 | 9 | ... TODO Write or find a quick introduction ... 10 | 11 | Read here: https://ogcapi.ogc.org/ 12 | 13 | ## What is pygeoapi? 14 | 15 | ...TODO... 16 | 17 | Read here: https://pygeoapi.io/ 18 | 19 | ## Steps to deploy this as OGC processes 20 | 21 | * Deploy an instance of pygeoapi (https://pygeoapi.io/). We will assume it is running on `localhost:5000`. 22 | * Go to the `process` directory of your installation, i.e. `cd /.../pygeoapi/pygeoapi/process`. 23 | * Clone this repo and checkout this branch 24 | * Open the `plugin.py` file (`vi /.../pygeoapi/pygeoapi/plugin.py`) and add this line to the `'process'` section: 25 | 26 | ``` 27 | 'process': { 28 | 'HelloWorld': 'pygeoapi.process.hello_world.HelloWorldProcessor', 29 | ... 30 | 'HEREON_PyOWT_Processor': 'pygeoapi.process.pyOWT.pygeoapi_processes.hereon_pyowt.HEREON_PyOWT_Processor', 31 | 32 | ... 33 | ``` 34 | 35 | * Open the `pygeoapi-config.yaml` file (`vi /.../pygeoapi/pygeoapi-config.yaml`) and add these lines to the `resources` section: 36 | 37 | ``` 38 | resources: 39 | 40 | ... 41 | 42 | hereon-pyowt: 43 | type: process 44 | processor: 45 | name: HEREON_PyOWT_Processor 46 | 47 | ``` 48 | 49 | * Config file: Make sure you have a `config.json` sitting either in pygeoapi's current working dir (`...TODO...`) or in an arbitrary path that pygeoapi can find through the environment variable `PYOWT_CONFIG_FILE`. 50 | * When running with flask or starlette, you can add that env var by adding the line `os.environ['PYOWT_CONFIG_FILE'] = '/.../config.json'` to `/.../pygeoapi/pygeoapi/starlette_app.py` 51 | * Make sure this config file contains: 52 | 53 | ``` 54 | { 55 | ... 56 | "pyowt": { 57 | "example_input_data_dir": "/.../pygeoapi/process/pyOWT/data/", 58 | "input_data_dir": "/.../inputs/", 59 | "path_sensor_band_library": "/.../pygeoapi/process/pyOWT/data/sensor_band_library.yaml" 60 | }, 61 | 62 | ... 63 | } 64 | ``` 65 | 66 | * Downloading of results: 67 | ** If you don't need this right now, just put any writeable path into `download_dir`, where you want the results to be written. Put some dummy value into `download_url`. 68 | ** If you want users to be able to download results from remote, have some webserver running (e.g. `nginx` or `apache2`) that you can use to serve static files. The directory for the static results and the URL where that is reachable have to be written into `download_dir` and `download_url`. 69 | * Make sure to create a directory for inputs, and write it into `input_data_dir` of the config file. The inputs that the users provide will be downloaded and stored there. 70 | 71 | 72 | * Install the following python packages: See https://github.com/merretbuurman/pyOWT/blob/main/requirements.txt 73 | * Start pygeoapi following their documentation 74 | * Now you should be able to call the processes using any HTTP client, for example curl. Example requests can be found on top of the `hereon_pyowt.py` file. For the first examepl, call `curl -X POST "http://localhost:5000/processes/hereon-pyowt/execution" -H "Content-Type: application/json" -d "{\"inputs\":{\"input_data_url\": \"https://raw.githubusercontent.com/bishun945/pyOWT/refs/heads/AquaINFRA/data/Rrs_demo_AquaINFRA_hyper.csv\", \"input_option\":\"csv\", \"sensor\":\"HYPER\", \"output_option\": 1}}" 75 | `. 76 | 77 | 78 | -------------------------------------------------------------------------------- /pyowt/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | 3 | Version history 4 | =============== 5 | 6 | 0.32: 7 | - Use absolute file path of OWT_centroids.nc 8 | - Clean the root folder 9 | 10 | 0.40: 11 | - Deprecated run and run_classification functions in OpticalVariables and OWT, respectively 12 | - Add manipulation of input Rrs for OpticalVariables which now supports any input shapes 13 | - Add new classes `PlotOV` and `PlotSpec` for plotting 14 | - Add new csv file (data/OWT_mean_spec.csv) for mean spectra of OWTs 15 | - run_examples.py revised accordingly 16 | 17 | 0.41: 18 | - Fig bugs in satellite sensor waveband settings and add new setups 19 | - Add new subcalss in `OpticalVariables`, `ArrayWithAttributes`, to allow attributes for np.array 20 | - A new demo of Rrs updated in the folder `data` 21 | - `run_examples.py` add Google Drive link for demo satellite data 22 | 23 | 0.50: 24 | - Reorganize the structure for tidiness 25 | - Add `projects` folder to contain scripts for different application. 26 | - `projects` now has two folders, AquaINFRA for scripts related to the project, and zenodo for its data preparation 27 | - Merge Merret's PR for AquaINFRA project which employs pygeoapi module for cloud processing 28 | - Collect pyowt-related scripts to `pyowt` for better management. 29 | - Have a `setup.py` for development install mode, `pip install -e .` 30 | 31 | 0.60: 32 | - Add a new version of classification centroids which has been shrinked with smaller covariance 33 | - Since different centroid versions were added, the data folder was revised correspondingly 34 | - Add satellite_handlers for processing cmems and eumetsat olci level-2 products 35 | 36 | 0.61: 37 | - add ENVI format reader in ./satellite_handlers which is supposed to read Liu's ENVI results but can be used (modified) for other ENVI results 38 | - Add warning for eumetsat_olci_level2 if uninstalled packages are imported 39 | - OpticalVariable now supports to find nearest waveband to calculate AVW (just like I did for RGB bands). However, it still needs more strict input checking... 40 | 41 | 0.62: 42 | - Fixed the bug when sensor is None for AVW calculation (issue from Merret) 43 | 44 | 0.63: 45 | - Rewrite OWT centroids generation codes (now for v01 and v02) 46 | - pyowt.OpticalVariables now checks the input hyperspectral Rrs is ranging from 400 to 800 nm 47 | - pyowt.OpticalVariables now interpolates hyperspectral Rrs into 1 nm interval if they are not 48 | - satellite_handlers: detect projection for Liu's image files 49 | - modified README.md in main 50 | 51 | 0.64: 52 | - Reprocessed the spectral convolution based on SRF files downloaded from the NASA Ocean Color website 53 | - The convoluted Rrs were then calculated multispectral AVW and used to perform regression analysis with hyperspectral AVW 54 | - Add new functions pyowt/satellite_handlers/srf_convolution.py for the above work 55 | - The pyowt/data/AVW_all_regression_800.txt file was updated 56 | - The sensor names in pyowt/data/AVW_all_regression_800.txt has been updated according to the NASA netcdf files 57 | - Demos in projects/AquaINFRA/run_AquaINFRA.py and projects/AquaINFRA/run_AquaINFRA2.py were revised due to sensor name changing 58 | 59 | 0.65: 60 | - Bugs fixed for Sentinel-2 band setups, regression coefficients of which are updated correspondingly 61 | - Tested a new version of centroid (shrunk) but probably will be deprecated in the future... 62 | - Added a new satellite handler for Dr. Liu's ENVI format data (basically converting them to netcdf file) 63 | 64 | 0.66: 65 | - Add new option for OpticalVariables if given Rrs is a 4-d array, say (wavelen, time, lat, lon) 66 | - Add support for HSI-PRISMA hyperspectral setups as requested by Alice Fabbretto 67 | 68 | ''' 69 | 70 | __package__ = "pyOWT" 71 | __version__ = "0.66" 72 | 73 | -------------------------------------------------------------------------------- /pyowt/data/AVW_all_regression_800.txt: -------------------------------------------------------------------------------- 1 | sensor,0,1,2,3,4,5,rsq 2 | HSI-EnMAP,-5123.3925915330865,45.81759209671841,-0.15812061848622233,0.00027871792569230964,-2.4381941858441533e-07,8.437266404438027e-11,0.9995701052047068 3 | HSI-PRISMA,-3503.3308592123626,33.972106742706636,-0.12375853633302919,0.0002302826314945178,-2.1154950194588934e-07,7.662469937509436e-11,0.9997985869495292 4 | OCI-PACE,12420.927340247854,-116.70174018492953,0.4416919584965311,-0.0008192008483812781,7.498574002346378e-07,-2.7084954090330456e-10,0.9996721138391486 5 | OCTS-adeos,31991.844312651403,-264.6492264429165,0.8625582829833672,-0.0013674914848182888,1.0585401445925585e-06,-3.2006009349031937e-10,0.9930156386158084 6 | modis-aqua,-10635.495792274109,114.50058180543087,-0.471801034039196,0.0009564136568449472,-9.46804524445356e-07,3.6686468657761406e-10,0.9941221459983309 7 | GOCI-coms,1896.967752208679,-21.769706525597655,0.09773386396105538,-0.00019473478538870907,1.8445175476777178e-07,-6.757770459595705e-11,0.9967416946728425 8 | MERIS-envisat,17428.91439413519,-158.1710651025021,0.574045871152209,-0.0010214755511502608,8.971016920117559e-07,-3.114381478502365e-10,0.9990686267235483 9 | viirs-jpss-1,20014.85274710591,-180.15292207265693,0.6433881073538233,-0.0011212042289725263,9.6065383823323e-07,-3.24667048669621e-10,0.9970326368708627 10 | viirs-jpss-2,22290.075137763524,-200.83951469481727,0.7176112259035379,-0.0012525555332896178,1.0753496683765124e-06,-3.6423182733282526e-10,0.9969910385242937 11 | OLI-landsat-8,-36065.57223060982,311.279290822571,-1.0681159620310456,0.00183551674043896,-1.570657532681669e-06,5.340744759383116e-10,0.9956260422912874 12 | SeaWiFS-orbview-2,62196.82152381222,-536.8970810283811,1.8349660396389935,-0.0030882730915826537,2.567726455437257e-06,-8.45067974840417e-10,0.9921925464656853 13 | olci-s3a,5742.934141892621,-56.527896361185896,0.2253995880309725,-0.0004304717877905865,4.011091751944576e-07,-1.4632380315093962e-10,0.9981502465667923 14 | olci-s3b,5801.340835196076,-57.1079132081412,0.2276760468887858,-0.0004348892892816114,4.0534750637280585e-07,-1.4793117946106901e-10,0.99816544433645 15 | HAWKEYE-seahawk1,64362.10447966226,-566.5590730624227,1.9771946472972577,-0.0034029756075924047,2.896748773824024e-06,-9.76946200223289e-10,0.997010439844551 16 | msi-sentinel-2a,7511.535686529454,-75.83137081777728,0.29624709648459485,-0.0005460345617704848,4.852391422147407e-07,-1.674897152644667e-10,0.9964641028438193 17 | msi-sentinel-2b,9186.173460417429,-89.91717160074819,0.34345828472242435,-0.0006247868904198355,5.505771579111363e-07,-1.8904432466789992e-10,0.9965665278363708 18 | viirs-suomi-npp,26266.585513214828,-237.72227946185114,0.8527891299747234,-0.0014972184938676712,1.2940376440383716e-06,-4.414543112332327e-10,0.9971036856992522 19 | modis-terra,-9816.633507179204,107.47250674859048,-0.44812668235657865,0.0009173828039015771,-9.153883821450262e-07,3.57021429756954e-10,0.9941078451633664 20 | CMEMS_BAL_HROC,7511.535686529454,-75.83137081777728,0.29624709648459485,-0.0005460345617704848,4.852391422147407e-07,-1.674897152644667e-10,0.9964641028438193 21 | CMEMS_BAL_NRT,3938.4183007820143,-40.93684906985742,0.1728073840616755,-0.0003443136715752401,3.3296521404857035e-07,-1.256263818252623e-10,0.997184491309797 22 | CMEMS_MED_MYINT,3938.4183007820143,-40.93684906985742,0.1728073840616755,-0.0003443136715752401,3.3296521404857035e-07,-1.256263818252623e-10,0.997184491309797 23 | AERONET_OC_1,3736.07589498351,-28.736684792299066,0.08930876486952419,-0.00012311730882765614,7.483666220156514e-08,-1.447090299956597e-11,0.9964725118155443 24 | AERONET_OC_2,36270.02999972947,-292.2340333913869,0.920911828495326,-0.0013981685204011573,1.020625153165826e-06,-2.8394883324487584e-10,0.9900100420572393 25 | modis-gee,15475.788966408856,-127.25794528811628,0.4150161474492979,-0.0006538390599446767,5.006842457102536e-07,-1.4879196402740318e-10,0.9950285582160813 26 | LakeCCI-MERIS,17471.225556457193,-158.65849062000802,0.5762414380811818,-0.001026324890008469,9.023589500017456e-07,-3.1367349692892736e-10,0.9989739629313836 27 | -------------------------------------------------------------------------------- /data/generate_sensor_lib_yaml.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | 3 | sensor_AVW_bands_library = { 4 | "MODIS_Aqua": [412, 443, 469, 488, 531, 547, 555, 645, 667, 678, 748, 859], 5 | "MODIS_Terra": [412, 443, 469, 488, 531, 547, 555, 645, 667, 678, 748, 859], 6 | "OLCI_S3A": [400, 412, 443, 490, 510, 560, 620, 665, 674, 682, 709, 754, 779, 866], 7 | "OLCI_S3B": [400, 412, 443, 490, 510, 560, 620, 665, 674, 681, 709, 754, 779, 866], 8 | "MERIS_ENVISAT": [413, 443, 490, 510, 560, 620, 665, 681, 709, 754, 779, 865], 9 | "SEAWIFS_ORBVIEW2": [412, 443, 490, 510, 555, 670, 865], 10 | "HAWKEYE_SEAHAWK1": [412, 447, 488, 510, 556, 670, 752, 867], 11 | "OCTS_ADEOS": [412, 443, 490, 516, 565, 667, 862], 12 | "GOCI_COMS": [412, 443, 490, 555, 660, 680, 745, 865], 13 | "VIIRS_SNPP": [410, 443, 486, 551, 671, 745, 862], 14 | "VIIRS_JPSS1": [411, 445, 489, 556, 667, 746, 868], 15 | "VIIRS_JPSS2": [411, 445, 488, 555, 671, 747, 868], 16 | "CZCS_NIMBUS7": [443, 520, 550, 670], 17 | "MSI_S2A": [443, 492, 560, 665, 704, 740, 783, 835], 18 | "MSI_S2B": [442, 492, 559, 665, 704, 739, 780, 835], 19 | "OLI_LC8": [443, 482, 561, 655, 865], 20 | "CMEMS_BAL_HROC": [443, 492, 560, 665, 704, 740, 783, 865], 21 | "CMEMS_BAL_NRT": [400, 412, 443, 490, 510, 560, 620, 665, 674, 682, 709, 779, 866], 22 | "CMEMS_MED_MYINT": [400, 412, 443, 490, 510, 560, 620, 665, 674, 682, 709, 779, 866], 23 | "AERONET_OC_1": [400, 412, 443, 490, 510, 560, 620, 665, 779, 866], 24 | "AERONET_OC_2": [412, 443, 490, 532, 551, 667, 870], 25 | } 26 | 27 | sensor_RGB_bands_library = { 28 | "MODIS_Aqua": [443, 555, 667], 29 | "MODIS_Terra": [443, 555, 667], 30 | "OLCI_S3A": [443, 560, 665], 31 | "OLCI_S3B": [443, 560, 665], 32 | "MERIS_ENVISAT": [443, 560, 665], 33 | "SEAWIFS_ORBVIEW2": [443, 555, 670], 34 | "HAWKEYE_SEAHAWK1": [447, 556, 670], 35 | "OCTS_ADEOS": [443, 565, 667], 36 | "GOCI_COMS": [443, 555, 660], 37 | "VIIRS_SNPP": [443, 551, 671], 38 | "VIIRS_JPSS1": [445, 556, 667], 39 | "VIIRS_JPSS2": [445, 555, 671], 40 | "CZCS_NIMBUS7": [443, 550, 670], 41 | "MSI_S2A": [443, 560, 665], 42 | "MSI_S2B": [442, 559, 665], 43 | "OLI_LC8": [443, 561, 655], 44 | "CMEMS_BAL_HROC": [443, 560, 665], 45 | "CMEMS_BAL_NRT": [443, 560, 665], 46 | "CMEMS_MED_MYINT": [443, 560, 665], 47 | "AERONET_OC_1": [443, 560, 665], 48 | "AERONET_OC_2": [443, 551, 667], 49 | } 50 | 51 | sensor_RGB_min_max = { 52 | "MODIS_Aqua": {"min": 412, "max": 667}, 53 | "MODIS_Terra": {"min": 412, "max": 667}, 54 | "OLCI_S3A": {"min": 400, "max": 866}, 55 | "OLCI_S3B": {"min": 400, "max": 866}, 56 | "MERIS_ENVISAT": {"min": 413, "max": 865}, 57 | "SEAWIFS_ORBVIEW2": {"min": 412, "max": 865}, 58 | "HAWKEYE_SEAHAWK1": {"min": 412, "max": 867}, 59 | "OCTS_ADEOS": {"min": 412, "max": 862}, 60 | "GOCI_COMS": {"min": 412, "max": 865}, 61 | "VIIRS_SNPP": {"min": 410, "max": 862}, 62 | "VIIRS_JPSS1": {"min": 411, "max": 868}, 63 | "VIIRS_JPSS2": {"min": 411, "max": 868}, 64 | "CZCS_NIMBUS7": {"min": 443, "max": 670}, 65 | "MSI_S2A": {"min": 443, "max": 835}, 66 | "MSI_S2B": {"min": 442, "max": 835}, 67 | "OLI_LC8": {"min": 443, "max": 865}, 68 | "CMEMS_BAL_HROC": {"min": 443, "max": 865}, 69 | "CMEMS_BAL_NRT": {"min": 400, "max": 866}, 70 | "CMEMS_MED_MYINT": {"min": 400, "max": 866}, 71 | "AERONET_OC_1": {"min": 400, "max": 866}, 72 | "AERONET_OC_2": {"min": 412, "max": 870}, 73 | } 74 | 75 | AVW_regression_coef = "data/AVW_all_regression_800.txt" 76 | 77 | 78 | lib_800 = { 79 | 'sensor_AVW_bands_library': sensor_AVW_bands_library, 80 | 'sensor_RGB_bands_library': sensor_RGB_bands_library, 81 | 'sensor_RGB_min_max': sensor_RGB_min_max, 82 | 'AVW_regression_coef': AVW_regression_coef 83 | } 84 | 85 | 86 | data = { 87 | 'lib_800': lib_800 88 | } 89 | 90 | # Save to a YAML file 91 | with open('pyowt/data/sensor_band_library.yaml', 'w') as file: 92 | yaml.dump(data, file) 93 | 94 | print("Data written finished!") 95 | 96 | # with open('data/sensor_band_library.yaml', 'r') as file: 97 | # data = yaml.load(file, Loader=yaml.FullLoader) -------------------------------------------------------------------------------- /run_examples.py: -------------------------------------------------------------------------------- 1 | """ 2 | (1) Test for simulated Rrs data (Bi et al. 2023 model) 3 | """ 4 | 5 | if True: 6 | 7 | import pandas as pd 8 | from pyowt.OpticalVariables import OpticalVariables 9 | from pyowt.OWT import OWT 10 | 11 | d0 = pd.read_csv("./data/Rrs_demo.csv") 12 | d = d0.pivot_table(index='SampleID', columns='wavelen', values='Rrs') 13 | owt_train = d0[d0['wavelen']==350].sort_values(by="SampleID").type.values 14 | 15 | # data preparation for `ov` and `owt` classes 16 | Rrs = d.values.reshape(d.shape[0], 1, d.shape[1]) 17 | band = d.columns.tolist() 18 | 19 | # create `ov` class to calculate three optical variables 20 | ov = OpticalVariables(Rrs=Rrs, band=band) 21 | 22 | # create `owt` class to run optical classification 23 | owt = OWT(ov.AVW, ov.Area, ov.NDI) 24 | 25 | owt_result = owt.type_str.flatten() 26 | 27 | # print(owt_train) 28 | print('Result OWT:', owt_result) 29 | 30 | # OWT result visualization 31 | from pyowt.PlotOWT import PlotOV, PlotSpec 32 | PlotOV(owt) 33 | # PlotSpec(owt, ov) 34 | 35 | 36 | 37 | """ 38 | (2) Test for satellite data (A4O results) 39 | """ 40 | 41 | if True: 42 | 43 | # OWT 44 | from pyowt.OpticalVariables import OpticalVariables 45 | from pyowt.OWT import OWT 46 | 47 | # data processing 48 | from netCDF4 import Dataset as ds 49 | import re 50 | import numpy as np 51 | import os 52 | 53 | # plot 54 | import matplotlib.pyplot as plt 55 | import matplotlib.colors as mcolors 56 | 57 | def str_match(input, pattern, ind=False): 58 | """Find out the matched strings 59 | 60 | Args: 61 | input (list): list of strings 62 | pattern (character): the string to be matched 63 | ind (bool, optional): True will return the index in the input list; 64 | otherwise, the matched strings. Defaults to False. 65 | 66 | Returns: 67 | list: characters or indices of matched variables 68 | """ 69 | pattern = re.compile(pattern) 70 | if ind: 71 | indices = [i for i in range(len(input)) if pattern.match(input[i])] 72 | else: 73 | indices = [v for v in input if pattern.match(v)] 74 | return indices 75 | 76 | # data preparation for `ov` and `owt` classes 77 | # this nc file is about 327MB 78 | # and can be downloaded from https://drive.google.com/file/d/14ZBq2bUw-dzrXGz2wnCxR-cN09iCA_6v/view?usp=sharing 79 | fn = "data/GermanBight_20160720T093421_subset.nc" 80 | nc_link = "https://drive.google.com/file/d/14ZBq2bUw-dzrXGz2wnCxR-cN09iCA_6v/view?usp=sharing" 81 | 82 | if not os.path.exists(fn): 83 | print(f"File {fn} doesn't exist.") 84 | print(f"Please download it via {nc_link}") 85 | exit() 86 | 87 | else: 88 | print(f"Found nc file in {fn}!") 89 | 90 | d = ds(fn, mode="r") 91 | name_var = list(d.variables.keys()) 92 | var_A4O_Rrs = str_match(name_var, r"A4O_Rrs_\d+$") 93 | wavelen_A4O_Rrs = np.array([float(i.split("_")[-1]) for i in var_A4O_Rrs]) 94 | Rrs_A4O = np.array([d.variables[x][:] for x in var_A4O_Rrs]).transpose(1, 2, 0) 95 | 96 | ## the required variables are: 97 | # >>> Rrs_A4O.shape 98 | # (1006, 1509, 16) # a 3d np.array where 1006 and 1509 for lon-lat coord and 16 is for sixteen bands 99 | # >>> wavelen_A4O_Rrs 100 | # array([ 400., 412., 442., 490., 510., 560., 620., 665., 674., 101 | # 681., 709., 754., 779., 865., 885., 1020.]) 102 | 103 | # create `ov` class to calculate three optical variables 104 | ov = OpticalVariables(Rrs=Rrs_A4O, band=wavelen_A4O_Rrs, sensor="OLCI_S3A") 105 | 106 | # create `owt` class to run optical classification 107 | owt = OWT(ov.AVW, ov.Area, ov.NDI) 108 | 109 | # setup colarbar 110 | cmap = mcolors.ListedColormap(owt.dict_idx_color.values()) 111 | bounds = np.arange(-1.5, 10.5) 112 | norm = plt.cm.colors.BoundaryNorm(bounds, cmap.N) 113 | 114 | plt.figure(figsize=(8, 6)) 115 | 116 | plt.imshow(owt.type_idx, cmap=cmap, norm=norm) 117 | cbar = plt.colorbar(ticks = np.arange(-1, 10), orientation="horizontal") 118 | cbar.ax.set_xticklabels(owt.dict_idx_name.values()) 119 | 120 | # plt.show() 121 | plt.savefig("type_owt.png", dpi=300, bbox_inches="tight") 122 | 123 | plt.close() 124 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 105 | __pypackages__/ 106 | 107 | # Celery stuff 108 | celerybeat-schedule 109 | celerybeat.pid 110 | 111 | # SageMath parsed files 112 | *.sage.py 113 | 114 | # Environments 115 | .env 116 | .venv 117 | env/ 118 | venv/ 119 | ENV/ 120 | env.bak/ 121 | venv.bak/ 122 | 123 | # Spyder project settings 124 | .spyderproject 125 | .spyproject 126 | 127 | # Rope project settings 128 | .ropeproject 129 | 130 | # mkdocs documentation 131 | /site 132 | 133 | # mypy 134 | .mypy_cache/ 135 | .dmypy.json 136 | dmypy.json 137 | 138 | # Pyre type checker 139 | .pyre/ 140 | 141 | # pytype static type analyzer 142 | .pytype/ 143 | 144 | # Cython debug symbols 145 | cython_debug/ 146 | 147 | # PyCharm 148 | # JetBrains specific template is maintainted in a separate JetBrains.gitignore that can 149 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 150 | # and can be added to the global gitignore or merged into this file. For a more nuclear 151 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 152 | #.idea/ 153 | 154 | .DS_Store 155 | archive/ 156 | test_running/ 157 | data/GermanBight_20160720T093421_subset.nc 158 | type_owt.png 159 | test_martin_issued_bug.py 160 | projects/zenodo/owt_BH2024_training_data_olci.nc 161 | projects/zenodo/owt_BH2024_training_data_hyper.nc 162 | projects/zenodo/reshape_netcdf_temp_data/single_variable.csv 163 | projects/zenodo/reshape_netcdf_temp_data/spec_ad.csv 164 | projects/zenodo/reshape_netcdf_temp_data/spec_agp.csv 165 | projects/zenodo/reshape_netcdf_temp_data/spec_aph.csv 166 | projects/zenodo/reshape_netcdf_temp_data/spec_aw.csv 167 | projects/zenodo/reshape_netcdf_temp_data/spec_bbp.csv 168 | projects/zenodo/reshape_netcdf_temp_data/spec_bp.csv 169 | projects/zenodo/reshape_netcdf_temp_data/spec_bw.csv 170 | projects/zenodo/reshape_netcdf_temp_data/spec_Rrs.csv 171 | 172 | # liu's mission on hold 173 | projects/Liu/* 174 | 175 | projects/geeowt_notebook/* 176 | projects/AVW_regression/* 177 | projects/EXT_DATASET/* 178 | projects/TCA/* 179 | projects/OLCI_adapter/* 180 | projects/lake_cci/* 181 | 182 | pyowt/geeowt/* 183 | 184 | pyowt/data/v99/* 185 | 186 | *test.ipynb 187 | 188 | -------------------------------------------------------------------------------- /projects/covm_shrinkage/calculate_OWT_centroids_v99.py: -------------------------------------------------------------------------------- 1 | import xarray as xr 2 | import numpy as np 3 | import pandas as pd 4 | import os 5 | from datetime import datetime 6 | 7 | from pyowt.OpticalVariables import OpticalVariables 8 | from pyowt.OWT import OWT 9 | 10 | # some setups 11 | version_tag = 'v99' 12 | date_tag = datetime.today().strftime("%Y-%m-%d") 13 | # date_tag = '2024-10-21' 14 | VersionLog = ( 15 | "v99" 16 | ) 17 | 18 | 19 | # load training data set 20 | fn = "projects/zenodo/owt_BH2024_training_data_hyper.nc" 21 | ds = xr.open_dataset(fn) 22 | 23 | wavelen = ds['wavelen'].values # from 400 to 900 nm 24 | Rrs = ds['Rrs'].values 25 | type_define = ds['type'].values # original OWT definition (BH2024) 26 | 27 | lamBC = 0.226754 # Box-Cox transform coefficient, from BH2024 28 | N = 10 # number of OWT, from BH2024 29 | 30 | # calculate optcal variables: avw, area, and ndi 31 | ov = OpticalVariables(Rrs=Rrs, band=wavelen, version='v99') 32 | AVW, Area, NDI = ov.AVW, ov.Area, ov.NDI 33 | ABC = OWT.trans_boxcox(Area, lamBC) 34 | ov_mat = np.hstack((AVW, ABC, NDI)) 35 | 36 | # select the used index 37 | index_used = range(Rrs.shape[0]) 38 | 39 | # filter selected data 40 | type_used = type_define[index_used] 41 | ov_mat = ov_mat[index_used, :] 42 | unique_types = np.sort(np.unique(type_used)) 43 | 44 | 45 | # keep the same data structure 46 | # - mean: (N, 3) 47 | # - covm: (3, 3, N) 48 | mean_new = np.zeros((N, 3)) 49 | covm_new = np.zeros((3, 3, N)) 50 | 51 | for i, t in enumerate(unique_types): 52 | indices = np.where(type_used == t)[0] 53 | samples = ov_mat[indices, :] 54 | 55 | mean_type = np.mean(samples, axis=0) 56 | covm_type = np.cov(samples, rowvar=False) 57 | 58 | mean_new[i, :] = mean_type 59 | covm_new[:, :, i] = covm_type 60 | 61 | 62 | 63 | # save to centroid netcdf files 64 | 65 | dc_new = xr.Dataset( 66 | { 67 | "mean": (("type", "var1"), mean_new), 68 | "covm": (("var1", "var2", "type"), covm_new), 69 | }, 70 | coords={ 71 | "type": ["1", "2", "3a", "3b", "4a", "4b", "5a", "5b", "6", "7"], 72 | "var1": ["AVW", "ABC", "NDI"], 73 | "var2": ["AVW", "ABC", "NDI"] 74 | }, 75 | ) 76 | 77 | dc_new["mean"].attrs["long_name"] = "Mean matrix of three optical variables for ten types. Shape (10, 3)" 78 | dc_new["mean"].attrs["units"] = "unitless" 79 | 80 | dc_new["covm"].attrs["long_name"] = "Covariance matrix of three optical variables for ten types. Shape (3, 3, 10)" 81 | dc_new["covm"].attrs["units"] = "unitless" 82 | 83 | dc_new.attrs['Description'] = 'Centroids and covariance matrix data for the optical water type classification by Bi and Hieronymi (2024)' 84 | dc_new.attrs['Version'] = version_tag 85 | dc_new.attrs['VersionLog'] = VersionLog 86 | dc_new.attrs['Author'] = 'Shun Bi' 87 | dc_new.attrs['Email'] = 'Shun.Bi@outlook.com' 88 | dc_new.attrs['CreateDate'] = date_tag 89 | dc_new.attrs['lamBC'] = lamBC 90 | dc_new.attrs['lamBC_description'] = 'Lambda coefficient for Box-Cox transformation for Area' 91 | dc_new.attrs['TypeName'] = '1, 2, 3a, 3b, 4a, 4b, 5a, 5b, 6, 7' 92 | dc_new.attrs['TypeColorName'] = 'blue, yellow, orange, cyan, green, purple, darkblue, red, chocolate, darkcyan' 93 | dc_new.attrs['TypeColorHex'] = '#0000FF, #FFFF00, #FFA500, #00FFFF, #00FF00, #A020F0, #00008B, #FF0000, #D2691E, #008B8B' 94 | dc_new.attrs['Reference'] = 'Bi, S., and Hieronymi, M. (2024). Holistic optical water type classification for ocean, coastal, and inland waters. Limnology & Oceanography, lno.12606. doi: 10.1002/lno.12606' 95 | 96 | output_file = f"pyowt/data/{version_tag}/OWT_centroids.nc" 97 | output_folder = os.path.dirname(output_file) 98 | if not os.path.exists(output_folder): 99 | os.makedirs(output_folder) 100 | 101 | dc_new.to_netcdf(output_file) 102 | 103 | 104 | 105 | # Update OWT_mean_spec.csv for plotting 106 | 107 | results = [] 108 | 109 | for t in unique_types: 110 | 111 | indices = np.where(type_used == t)[0] 112 | Rrs_samples = Rrs[indices, :] 113 | Area_samples = Area[indices, 0] 114 | 115 | m_Rrs_t = np.mean(Rrs_samples, axis=0) 116 | sd_Rrs_t = np.std(Rrs_samples, axis=0) 117 | lo_Rrs_t = np.quantile(Rrs_samples, 0.05, axis=0) 118 | up_Rrs_t = np.quantile(Rrs_samples, 0.95, axis=0) 119 | 120 | nRrs_samples = Rrs_samples / Area_samples[:, np.newaxis] 121 | 122 | m_nRrs_t = np.mean(nRrs_samples, axis=0) 123 | sd_nRrs_t = np.std(nRrs_samples, axis=0) 124 | lo_nRrs_t = np.quantile(nRrs_samples, 0.05, axis=0) 125 | up_nRrs_t = np.quantile(nRrs_samples, 0.95, axis=0) 126 | 127 | for i in range(len(wavelen)): 128 | results.append([ 129 | t, 130 | wavelen[i], 131 | 132 | m_Rrs_t[i], 133 | sd_Rrs_t[i], 134 | lo_Rrs_t[i], 135 | up_Rrs_t[i], 136 | 137 | m_nRrs_t[i], 138 | sd_nRrs_t[i], 139 | lo_nRrs_t[i], 140 | up_nRrs_t[i] 141 | ]) 142 | 143 | columns = ["type", "wavelen", "m_Rrs", "sd_Rrs", "lo_Rrs", "up_Rrs", "m_nRrs", "sd_nRrs", "lo_nRrs", "up_nRrs"] 144 | df = pd.DataFrame(results, columns=columns) 145 | 146 | output_file = f"pyowt/data/{version_tag}/OWT_mean_spec.csv" 147 | df.to_csv(output_file, index=False) -------------------------------------------------------------------------------- /projects/AquaINFRA/pygeoapi_processes/owt_classification.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.0.1", 3 | "id": "owt-classification", 4 | "use_case": "HEREON", 5 | "title": {"en": "OWT Classification"}, 6 | "description": { 7 | "en": "Optical Water Type classification for ocean, coastal and inland waters." 8 | }, 9 | "jobControlOptions": ["sync-execute", "async-execute"], 10 | "keywords": ["OWT", "classification", "pyOWT", "hereon"], 11 | "links": [{ 12 | "type": "text/html", 13 | "rel": "about", 14 | "title": "information", 15 | "href": "https://github.com/bishun945/pyOWT", 16 | "hreflang": "en-US" 17 | }], 18 | "inputs": { 19 | "input_data_url": { 20 | "title": "Input data", 21 | "description": "URL to your input file. Find example data on https://github.com/bishun945/pyOWT/tree/main/projects/AquaINFRA.", 22 | "schema": { 23 | "type": "string" 24 | }, 25 | "minOccurs": 1, 26 | "maxOccurs": 1, 27 | "keywords": ["hereon", "pyOWT"] 28 | }, 29 | "input_option": { 30 | "title": "Type of input", 31 | "description": "csv: for text data input (first line wavelength, following lines remote-sensing reflectance). sat: for satellite product input containing reflectance (e.g., Sentinel-3 OLCI Level-2)", 32 | "schema": { 33 | "type": "string", 34 | "enum": [ 35 | "csv", 36 | "sat" 37 | ], 38 | "default": "csv" 39 | }, 40 | "minOccurs": 1, 41 | "maxOccurs": 1, 42 | "keywords": ["hereon", "pyOWT"] 43 | }, 44 | "sensor": { 45 | "title": "Sensor name", 46 | "description": "Spectral band configuration of satellite mission (includes adaptation to sensor spectral response functions).", 47 | "schema": { 48 | "type": "string", 49 | "enum": [ 50 | "HYPER", 51 | "AERONET_OC_1", 52 | "AERONET_OC_2", 53 | "CMEMS_BAL_HROC", 54 | "CMEMS_BAL_NRT", 55 | "CMEMS_MED_MYINT", 56 | "CZCS", 57 | "GOCI", 58 | "HawkEye", 59 | "MERIS", 60 | "ODIS_Aqua", 61 | "MODIS_Terra", 62 | "MSI_S2A", 63 | "MSI_S2B", 64 | "OCTS", 65 | "OLCI_S3A", 66 | "OLCI_S3B", 67 | "OLI", 68 | "SeaWiFS", 69 | "VIIRS_JPSS1", 70 | "VIIRS_JPSS2", 71 | "VIIRS_SNPP" 72 | ], 73 | "default": "HYPER" 74 | }, 75 | "minOccurs": 1, 76 | "maxOccurs": 1, 77 | "keywords": ["hereon", "pyOWT"] 78 | }, 79 | "output_option": { 80 | "title": "Output option", 81 | "description": "1: for standard output with five variables. 2: for extensive output including memberships of all water types.", 82 | "schema": { 83 | "type": "string", 84 | "enum": [ 85 | "1", 86 | "2" 87 | ], 88 | "default": "1" 89 | }, 90 | "minOccurs": 1, 91 | "maxOccurs": 1, 92 | "keywords": ["hereon", "pyOWT"] 93 | } 94 | }, 95 | "outputs": { 96 | "owt_classification": { 97 | "title": "Output of the OWT classification depending on input parameters.", 98 | "description": "If input_option == csv, the output is a text file (.csv). If input_option == sat, the output is a NetCDF file (.nc) of the same dimensions and geo-reference as the input file. If output_option == 1, 5 Output variables: AVW: Apparent Visible Wavelength between 400 and 800 nm; Area: Trapezoidal area of remote-sensing reflectance at RGB bands; NDI: Normalized Difference Index of remote-sensing reflectance at green and red bands; Index value of water class; Total membership values from all ten water types. If output_option == 2, 15 Output variables; AVW: Apparent Visible Wavelength between 400 and 800 nm; Area: Trapezoidal area of remote-sensing reflectance at RGB bands; NDI: Normalized Difference Index of remote-sensing reflectance at green and red bands; Index value of water class; Total membership values from all ten water types; Weighted membership in OWT class 1; Weighted membership in OWT class 2; Weighted membership in OWT class 3a; Weighted membership in OWT class 3b; Weighted membership in OWT class 4a; Weighted membership in OWT class 4b; Weighted membership in OWT class 5a; Weighted membership in OWT class 5b; Weighted membership in OWT class 6; Weighted membership in OWT class 7", 99 | "schema": { 100 | "type": "object", 101 | "contentMediaType": "application/json" 102 | } 103 | } 104 | 105 | }, 106 | "example": { 107 | "inputs": { 108 | "input_data_url": "Rrs_demo_AquaINFRA_hyper.csv", 109 | "input_option": "csv", 110 | "sensor": "MSI_S2A", 111 | "output_option": "1" 112 | } 113 | } 114 | } -------------------------------------------------------------------------------- /projects/covm_shrinkage/calculate_OWT_centroids_v01.py: -------------------------------------------------------------------------------- 1 | import xarray as xr 2 | import numpy as np 3 | import pandas as pd 4 | import os 5 | from datetime import datetime 6 | 7 | from pyowt.OpticalVariables import OpticalVariables 8 | from pyowt.OWT import OWT 9 | 10 | # some setups 11 | version_tag = 'v01' 12 | date_tag = datetime.today().strftime("%Y-%m-%d") 13 | # date_tag = '2024-10-21' 14 | VersionLog = ( 15 | "This is the original version of the OWT classification from the paper by Bi and Hieronymi (2024)." 16 | " The centroid data were calculated from the training dataset (https://zenodo.org/records/12803329)" 17 | " based on the labeled water types. Note that this version may differ slightly from the version dated" 18 | " 2023-09-26, which was based on R packages. However, any classification differences resulting from" 19 | " switching to the Python environment are negligible." 20 | ) 21 | 22 | 23 | # load training data set 24 | fn = "projects/zenodo/owt_BH2024_training_data_hyper.nc" 25 | ds = xr.open_dataset(fn) 26 | 27 | wavelen = ds['wavelen'].values # from 400 to 900 nm 28 | Rrs = ds['Rrs'].values 29 | type_define = ds['type'].values # original OWT definition (BH2024) 30 | 31 | lamBC = 0.226754 # Box-Cox transform coefficient, from BH2024 32 | N = 10 # number of OWT, from BH2024 33 | 34 | # calculate optcal variables: avw, area, and ndi 35 | ov = OpticalVariables(Rrs=Rrs, band=wavelen) 36 | AVW, Area, NDI = ov.AVW, ov.Area, ov.NDI 37 | ABC = OWT.trans_boxcox(Area, lamBC) 38 | ov_mat = np.hstack((AVW, ABC, NDI)) 39 | 40 | # select the used index 41 | index_used = range(Rrs.shape[0]) 42 | 43 | # filter selected data 44 | type_used = type_define[index_used] 45 | ov_mat = ov_mat[index_used, :] 46 | unique_types = np.sort(np.unique(type_used)) 47 | 48 | 49 | # keep the same data structure 50 | # - mean: (N, 3) 51 | # - covm: (3, 3, N) 52 | mean_new = np.zeros((N, 3)) 53 | covm_new = np.zeros((3, 3, N)) 54 | 55 | for i, t in enumerate(unique_types): 56 | indices = np.where(type_used == t)[0] 57 | samples = ov_mat[indices, :] 58 | 59 | mean_type = np.mean(samples, axis=0) 60 | covm_type = np.cov(samples, rowvar=False) 61 | 62 | mean_new[i, :] = mean_type 63 | covm_new[:, :, i] = covm_type 64 | 65 | 66 | 67 | # save to centroid netcdf files 68 | 69 | dc_new = xr.Dataset( 70 | { 71 | "mean": (("type", "var1"), mean_new), 72 | "covm": (("var1", "var2", "type"), covm_new), 73 | }, 74 | coords={ 75 | "type": ["1", "2", "3a", "3b", "4a", "4b", "5a", "5b", "6", "7"], 76 | "var1": ["AVW", "ABC", "NDI"], 77 | "var2": ["AVW", "ABC", "NDI"] 78 | }, 79 | ) 80 | 81 | dc_new["mean"].attrs["long_name"] = "Mean matrix of three optical variables for ten types. Shape (10, 3)" 82 | dc_new["mean"].attrs["units"] = "unitless" 83 | 84 | dc_new["covm"].attrs["long_name"] = "Covariance matrix of three optical variables for ten types. Shape (3, 3, 10)" 85 | dc_new["covm"].attrs["units"] = "unitless" 86 | 87 | dc_new.attrs['Description'] = 'Centroids and covariance matrix data for the optical water type classification by Bi and Hieronymi (2024)' 88 | dc_new.attrs['Version'] = version_tag 89 | dc_new.attrs['VersionLog'] = VersionLog 90 | dc_new.attrs['Author'] = 'Shun Bi' 91 | dc_new.attrs['Email'] = 'Shun.Bi@outlook.com' 92 | dc_new.attrs['CreateDate'] = date_tag 93 | dc_new.attrs['lamBC'] = lamBC 94 | dc_new.attrs['lamBC_description'] = 'Lambda coefficient for Box-Cox transformation for Area' 95 | dc_new.attrs['TypeName'] = '1, 2, 3a, 3b, 4a, 4b, 5a, 5b, 6, 7' 96 | dc_new.attrs['TypeColorName'] = 'blue, yellow, orange, cyan, green, purple, darkblue, red, chocolate, darkcyan' 97 | dc_new.attrs['TypeColorHex'] = '#0000FF, #FFFF00, #FFA500, #00FFFF, #00FF00, #A020F0, #00008B, #FF0000, #D2691E, #008B8B' 98 | dc_new.attrs['Reference'] = 'Bi, S., and Hieronymi, M. (2024). Holistic optical water type classification for ocean, coastal, and inland waters. Limnology & Oceanography, lno.12606. doi: 10.1002/lno.12606' 99 | 100 | output_file = f"pyowt/data/{version_tag}/OWT_centroids.nc" 101 | output_folder = os.path.dirname(output_file) 102 | if not os.path.exists(output_folder): 103 | os.makedirs(output_folder) 104 | 105 | dc_new.to_netcdf(output_file) 106 | 107 | 108 | 109 | # Update OWT_mean_spec.csv for plotting 110 | 111 | results = [] 112 | 113 | for t in unique_types: 114 | 115 | indices = np.where(type_used == t)[0] 116 | Rrs_samples = Rrs[indices, :] 117 | Area_samples = Area[indices, 0] 118 | 119 | m_Rrs_t = np.mean(Rrs_samples, axis=0) 120 | sd_Rrs_t = np.std(Rrs_samples, axis=0) 121 | lo_Rrs_t = np.quantile(Rrs_samples, 0.05, axis=0) 122 | up_Rrs_t = np.quantile(Rrs_samples, 0.95, axis=0) 123 | 124 | nRrs_samples = Rrs_samples / Area_samples[:, np.newaxis] 125 | 126 | m_nRrs_t = np.mean(nRrs_samples, axis=0) 127 | sd_nRrs_t = np.std(nRrs_samples, axis=0) 128 | lo_nRrs_t = np.quantile(nRrs_samples, 0.05, axis=0) 129 | up_nRrs_t = np.quantile(nRrs_samples, 0.95, axis=0) 130 | 131 | for i in range(len(wavelen)): 132 | results.append([ 133 | t, 134 | wavelen[i], 135 | 136 | m_Rrs_t[i], 137 | sd_Rrs_t[i], 138 | lo_Rrs_t[i], 139 | up_Rrs_t[i], 140 | 141 | m_nRrs_t[i], 142 | sd_nRrs_t[i], 143 | lo_nRrs_t[i], 144 | up_nRrs_t[i] 145 | ]) 146 | 147 | columns = ["type", "wavelen", "m_Rrs", "sd_Rrs", "lo_Rrs", "up_Rrs", "m_nRrs", "sd_nRrs", "lo_nRrs", "up_nRrs"] 148 | df = pd.DataFrame(results, columns=columns) 149 | 150 | output_file = f"pyowt/data/{version_tag}/OWT_mean_spec.csv" 151 | df.to_csv(output_file, index=False) -------------------------------------------------------------------------------- /projects/covm_shrinkage/calculate_OWT_centroids_v02.py: -------------------------------------------------------------------------------- 1 | import xarray as xr 2 | import numpy as np 3 | import pandas as pd 4 | import os 5 | from datetime import datetime 6 | 7 | from pyowt.OpticalVariables import OpticalVariables 8 | from pyowt.OWT import OWT 9 | 10 | # some setups 11 | version_tag = 'v02' 12 | date_tag = datetime.today().strftime("%Y-%m-%d") 13 | # date_tag = '2024-10-21' 14 | VersionLog = 'Covariance matrix has been shrunk based on the original training data results.' 15 | VersionLog = ( 16 | "This is the shrunk version of the OWT classification (v01) based on the training dataset" 17 | " (https://zenodo.org/records/12803329). The classification was initially performed using the" 18 | " v01 centroids, and samples that retained the same water type labels as in Bi and Hieronymi" 19 | " (2024) were kept. This version is referred to as 'shrunk.' Note that this version still" 20 | " provides nearly identical mean water spectra for each type, but with reduced covariance," 21 | " which may lower the membership values for inputs that are far from the cluster centers." 22 | " The related manuscript is currently under preparation." 23 | ) 24 | 25 | 26 | # load training data set 27 | fn = "projects/zenodo/owt_BH2024_training_data_hyper.nc" 28 | ds = xr.open_dataset(fn) 29 | 30 | wavelen = ds['wavelen'].values # from 400 to 900 nm 31 | Rrs = ds['Rrs'].values 32 | type_define = ds['type'].values # original OWT definition (BH2024) 33 | 34 | lamBC = 0.226754 # Box-Cox transform coefficient, from BH2024 35 | N = 10 # number of OWT, from BH2024 36 | 37 | # calculate optcal variables: avw, area, and ndi 38 | ov = OpticalVariables(Rrs=Rrs, band=wavelen) 39 | AVW, Area, NDI = ov.AVW, ov.Area, ov.NDI 40 | ABC = OWT.trans_boxcox(Area, lamBC) 41 | ov_mat = np.hstack((AVW, ABC, NDI)) 42 | 43 | # perform classifcation based on v01 44 | owt = OWT(ov.AVW, ov.Area, ov.NDI, version='v01') 45 | type_new = owt.type_str 46 | type_new = type_new.reshape(type_new.shape[0]) 47 | 48 | # find unique types 49 | unique_types = np.sort(np.unique(type_define)) 50 | 51 | 52 | # keep the same data structure 53 | # - mean: (N, 3) 54 | # - covm: (3, 3, N) 55 | mean_new = np.zeros((N, 3)) 56 | covm_new = np.zeros((3, 3, N)) 57 | 58 | for i, t in enumerate(unique_types): 59 | indices = np.where((type_define == t) & (type_define == type_new))[0] 60 | samples = ov_mat[indices, :] 61 | 62 | mean_type = np.mean(samples, axis=0) 63 | covm_type = np.cov(samples, rowvar=False) 64 | 65 | mean_new[i, :] = mean_type 66 | covm_new[:, :, i] = covm_type 67 | 68 | 69 | 70 | # save to centroid netcdf files 71 | 72 | dc_new = xr.Dataset( 73 | { 74 | "mean": (("type", "var1"), mean_new), 75 | "covm": (("var1", "var2", "type"), covm_new), 76 | }, 77 | coords={ 78 | "type": ["1", "2", "3a", "3b", "4a", "4b", "5a", "5b", "6", "7"], 79 | "var1": ["AVW", "ABC", "NDI"], 80 | "var2": ["AVW", "ABC", "NDI"] 81 | }, 82 | ) 83 | 84 | dc_new["mean"].attrs["long_name"] = "Mean matrix of three optical variables for ten types. Shape (10, 3)" 85 | dc_new["mean"].attrs["units"] = "unitless" 86 | 87 | dc_new["covm"].attrs["long_name"] = "Covariance matrix of three optical variables for ten types. Shape (3, 3, 10)" 88 | dc_new["covm"].attrs["units"] = "unitless" 89 | 90 | dc_new.attrs['Description'] = 'Centroids and covariance matrix data for the optical water type classification by Bi and Hieronymi (2024)' 91 | dc_new.attrs['Version'] = version_tag 92 | dc_new.attrs['VersionLog'] = VersionLog 93 | dc_new.attrs['Author'] = 'Shun Bi' 94 | dc_new.attrs['Email'] = 'Shun.Bi@outlook.com' 95 | dc_new.attrs['CreateDate'] = date_tag 96 | dc_new.attrs['lamBC'] = lamBC 97 | dc_new.attrs['lamBC_description'] = 'Lambda coefficient for Box-Cox transformation for Area' 98 | dc_new.attrs['TypeName'] = '1, 2, 3a, 3b, 4a, 4b, 5a, 5b, 6, 7' 99 | dc_new.attrs['TypeColorName'] = 'blue, yellow, orange, cyan, green, purple, darkblue, red, chocolate, darkcyan' 100 | dc_new.attrs['TypeColorHex'] = '#0000FF, #FFFF00, #FFA500, #00FFFF, #00FF00, #A020F0, #00008B, #FF0000, #D2691E, #008B8B' 101 | dc_new.attrs['Reference'] = 'Bi, S., and Hieronymi, M. (2024). Holistic optical water type classification for ocean, coastal, and inland waters. Limnology & Oceanography, lno.12606. doi: 10.1002/lno.12606' 102 | 103 | output_file = f"pyowt/data/{version_tag}/OWT_centroids.nc" 104 | output_folder = os.path.dirname(output_file) 105 | if not os.path.exists(output_folder): 106 | os.makedirs(output_folder) 107 | 108 | dc_new.to_netcdf(output_file) 109 | 110 | 111 | 112 | # Update OWT_mean_spec.csv for plotting 113 | 114 | results = [] 115 | 116 | for t in unique_types: 117 | indices = np.where((type_define == t) & (type_define == type_new))[0] 118 | Rrs_samples = Rrs[indices, :] 119 | Area_samples = Area[indices, 0] 120 | 121 | m_Rrs_t = np.mean(Rrs_samples, axis=0) 122 | sd_Rrs_t = np.std(Rrs_samples, axis=0) 123 | lo_Rrs_t = np.quantile(Rrs_samples, 0.05, axis=0) 124 | up_Rrs_t = np.quantile(Rrs_samples, 0.95, axis=0) 125 | 126 | nRrs_samples = Rrs_samples / Area_samples[:, np.newaxis] 127 | 128 | m_nRrs_t = np.mean(nRrs_samples, axis=0) 129 | sd_nRrs_t = np.std(nRrs_samples, axis=0) 130 | lo_nRrs_t = np.quantile(nRrs_samples, 0.05, axis=0) 131 | up_nRrs_t = np.quantile(nRrs_samples, 0.95, axis=0) 132 | 133 | for i in range(len(wavelen)): 134 | results.append([ 135 | t, 136 | wavelen[i], 137 | 138 | m_Rrs_t[i], 139 | sd_Rrs_t[i], 140 | lo_Rrs_t[i], 141 | up_Rrs_t[i], 142 | 143 | m_nRrs_t[i], 144 | sd_nRrs_t[i], 145 | lo_nRrs_t[i], 146 | up_nRrs_t[i] 147 | ]) 148 | 149 | columns = ["type", "wavelen", "m_Rrs", "sd_Rrs", "lo_Rrs", "up_Rrs", "m_nRrs", "sd_nRrs", "lo_nRrs", "up_nRrs"] 150 | df = pd.DataFrame(results, columns=columns) 151 | 152 | output_file = f"pyowt/data/{version_tag}/OWT_mean_spec.csv" 153 | df.to_csv(output_file, index=False) -------------------------------------------------------------------------------- /pyowt/satellite_handlers/srf_convolution.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import xarray as xr 3 | from scipy.interpolate import interp1d 4 | 5 | def gaussian_srf(central_wavelength, fwhm, wavelengths): 6 | # Calculate standard deviation from FWHM 7 | sigma = fwhm / (2 * np.sqrt(2 * np.log(2))) 8 | 9 | # Calculate the spectral response function using a Gaussian model 10 | srf = np.exp(-((wavelengths - central_wavelength) ** 2) / (2 * sigma ** 2)) 11 | 12 | # Normalize the SRF so that its peak value is 1 13 | srf /= np.max(srf) 14 | 15 | return srf 16 | 17 | 18 | def interpolate_rrs(rrs, rrs_wavelength, target_wavelength): 19 | """ 20 | Interpolate Rrs to match target wavelengths. 21 | Values out of the rrs_wavelength range are extrapolated constantly from boundary values. 22 | Optimized to use vectorized operations for efficiency. 23 | """ 24 | target_wavelength = np.atleast_1d(target_wavelength) 25 | interpolated_rrs = np.full(rrs.shape[:-1] + (len(target_wavelength),), np.nan) 26 | 27 | valid_indices = (target_wavelength >= rrs_wavelength[0]) & (target_wavelength <= rrs_wavelength[-1]) 28 | interp_func = interp1d(rrs_wavelength, rrs, kind='linear', axis=-1, bounds_error=False, fill_value='extrapolate') 29 | interpolated_rrs[..., valid_indices] = interp_func(target_wavelength[valid_indices]) 30 | 31 | # Extrapolate constant values for out-of-bounds 32 | extrap_below = target_wavelength < rrs_wavelength[0] 33 | extrap_above = target_wavelength > rrs_wavelength[-1] 34 | interpolated_rrs[..., extrap_below] = rrs[..., 0][..., np.newaxis] 35 | interpolated_rrs[..., extrap_above] = rrs[..., -1][..., np.newaxis] 36 | 37 | return interpolated_rrs 38 | 39 | 40 | def convolute_rrs_to_multispectral(rrs_interp, srf_rsr, delta_lambda): 41 | """ 42 | Convolute the Rrs with the SRF to obtain multispectral Rrs. 43 | Optimized using vectorized numpy operations for efficiency. 44 | """ 45 | nrow, ncol, nband = rrs_interp.shape 46 | rrs_interp_reshape = rrs_interp.reshape(-1, nband) 47 | 48 | numerator = np.dot(rrs_interp_reshape, srf_rsr.T) 49 | denominator = np.sum(srf_rsr, axis=1) 50 | multispectral_rrs = numerator / denominator.reshape(1, denominator.shape[0]) 51 | multispectral_rrs = multispectral_rrs.reshape(nrow, ncol, srf_rsr.shape[0]) 52 | 53 | return multispectral_rrs 54 | 55 | 56 | def srf_convolution(rrs_wavelength, rrs_data, srf_nc_path): 57 | """Perform SRF convolution on input Rrs data. 58 | 59 | Args: 60 | rrs_wavelength (array-like): The wavelengths corresponding to the Rrs data. 61 | rrs_data (array-like): The Rrs data, can be 1D, 2D, or 3D. If 1D or 2D, it will be reshaped to 3D. 62 | srf_nc_path (str): The file path to the SRF netCDF file. 63 | 64 | Return: 65 | - bands (array): The bands information extracted from the netCDF file. 66 | - multispectral_rrs (np.array): The Rrs after SRF convolution, maintaining the same shape as the input Rrs. 67 | - instrument (str): The instrument information from the netCDF file. 68 | - platform (str): The platform information from the netCDF file. 69 | """ 70 | # Load SRF data from netCDF file 71 | ds = xr.open_dataset(srf_nc_path) 72 | srf_wavelength = ds['wavelength'].values 73 | srf_rsr = ds['RSR'].values 74 | srf_rsr = np.nan_to_num(srf_rsr, nan=0.0) 75 | bands = ds['bands'].values 76 | instrument = ds.attrs.get('instrument', 'Unknown') 77 | platform = ds.attrs.get('platform', 'Unknown') 78 | 79 | # Ensure Rrs data is at least 3D 80 | if rrs_data.ndim == 1: 81 | rrs_data = rrs_data[np.newaxis, np.newaxis, :] 82 | elif rrs_data.ndim == 2: 83 | rrs_data = rrs_data[:, np.newaxis, :] 84 | 85 | # Interpolate Rrs to match SRF wavelengths 86 | rrs_interpolated = interpolate_rrs(rrs_data, rrs_wavelength, srf_wavelength) 87 | 88 | # Check wavelength intervals 89 | wavelength_diff = np.diff(rrs_wavelength) 90 | if np.allclose(wavelength_diff, wavelength_diff[0]): 91 | delta_lambda = wavelength_diff[0] 92 | else: 93 | raise ValueError('Interval of RSR wavelength is different. Please check!') 94 | 95 | # Normalize SRF 96 | srf_rsr_normalized = srf_rsr / np.sum(srf_rsr, axis=1, keepdims=True) 97 | 98 | # Perform convolution 99 | multispectral_rrs = convolute_rrs_to_multispectral(rrs_interpolated, srf_rsr_normalized, delta_lambda) 100 | 101 | # Replace Rrs values with NaN where input wavelengths do not cover the band 102 | for i, band in enumerate(bands): 103 | if band < rrs_wavelength[0] or band > rrs_wavelength[-1]: 104 | multispectral_rrs[..., i] = np.nan 105 | 106 | return bands, multispectral_rrs, instrument, platform 107 | 108 | if __name__ == '__main__': 109 | 110 | import matplotlib.pyplot as plt 111 | 112 | fn = "/Users/apple/GitHub/pyOWT/projects/zenodo/owt_BH2024_training_data_hyper.nc" 113 | ds = xr.open_dataset(fn) 114 | wavelen = ds['wavelen'].values 115 | # Rrs = ds['Rrs'].values[85_002, :] 116 | Rrs = ds['Rrs'].values[85_002:85_102, :] 117 | Rrs = np.atleast_2d(Rrs) 118 | 119 | srf_nc_path = '/Users/apple/Satellite_data/SRF/s3a_olci_RSR.nc' 120 | bands, Rrs_multi, instrument, platform = srf_convolution(wavelen, Rrs, srf_nc_path) 121 | 122 | plt.plot(wavelen, Rrs[0,:], label='hyper') 123 | plt.plot(bands, Rrs_multi[0,0,:], 'o', label=f"{instrument}-{platform}") 124 | 125 | srf_nc_path = '/Users/apple/Satellite_data/SRF/sentinel-2a_msi_RSR.nc' 126 | bands, Rrs_multi, instrument, platform = srf_convolution(wavelen, Rrs, srf_nc_path) 127 | 128 | plt.plot(bands, Rrs_multi[0,0,:], '+', label=f"{instrument}-{platform}") 129 | 130 | srf_nc_path = '/Users/apple/Satellite_data/SRF/aqua_modis_RSR.nc' 131 | bands, Rrs_multi, instrument, platform = srf_convolution(wavelen, Rrs, srf_nc_path) 132 | 133 | plt.plot(bands, Rrs_multi[0,0,:], 'x', label=f"{instrument}-{platform}") 134 | 135 | srf_nc_path = '/Users/apple/Satellite_data/SRF/landsat-8_oli_RSR.nc' 136 | bands, Rrs_multi, instrument, platform = srf_convolution(wavelen, Rrs, srf_nc_path) 137 | 138 | plt.plot(bands, Rrs_multi[0,0,:], 'D', label=f"{instrument}-{platform}") 139 | 140 | plt.legend() 141 | plt.show() 142 | 143 | print(Rrs_multi) 144 | -------------------------------------------------------------------------------- /projects/AquaINFRA/run_AquaINFRA.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This script is for AquaINFRA project to integrate pyOWT into the Galaxy platform 3 | 4 | Examples: 5 | 6 | # run in terminal 7 | 8 | # csv as input 9 | python projects/AquaINFRA/run_AquaINFRA.py --input 'projects/AquaINFRA/data/Rrs_demo_AquaINFRA_hyper.csv' --input_option 'csv' --sensor 'HYPER' --output 'projects/AquaINFRA/results/owt_result_hyper.txt' --output_option 1 10 | python projects/AquaINFRA/run_AquaINFRA.py --input 'projects/AquaINFRA/data/Rrs_demo_AquaINFRA_msi.csv' --input_option 'csv' --sensor 'msi-sentine-2a' --output 'projects/AquaINFRA/results/owt_result_hyper.txt' --output_option 1 11 | python projects/AquaINFRA/run_AquaINFRA.py --input 'projects/AquaINFRA/data/Rrs_demo_AquaINFRA_ocli.csv' --input_option 'csv' --sensor 'OLCI_S3A' --output 'projects/AquaINFRA/results/owt_result_hyper.txt' --output_option 1 12 | 13 | # extensive output 14 | python projects/AquaINFRA/run_AquaINFRA.py --input 'projects/AquaINFRA/data/Rrs_demo_AquaINFRA_hyper.csv' --input_option 'csv' --sensor 'HYPER' --output 'projects/AquaINFRA/results/owt_result_hyper.txt' --output_option 2 15 | 16 | # satellite as input 17 | python projects/AquaINFRA/run_AquaINFRA.py --input '/path/S3B_OL_2_WFR____20220703T075301_20220703T075601_20220704T171729_0179_067_363_2160_MAR_O_NT_003.SEN3.zip' --input_option 'sat' --sensor 'OLCI_S3A' --output '/path/to/save' --output_option 1 18 | 19 | Shun Bi, shun.bi@outlook.com 20 | 01.11.2024 21 | ''' 22 | 23 | import argparse 24 | 25 | # common pkg 26 | import numpy as np 27 | import pandas as pd 28 | 29 | # import OWT pkg 30 | # the exception is for pygeoapi importing in AquaINFRA 31 | try: 32 | from pyowt.OpticalVariables import OpticalVariables 33 | from pyowt.OWT import OWT 34 | from pyowt.satellite_handlers.eumetsat_olci_level2 import eumetsat_olci_level2 35 | except ModuleNotFoundError as e: 36 | from pygeoapi.process.pyOWT.pyowt.OpticalVariables import OpticalVariables 37 | from pygeoapi.process.pyOWT.pyowt.OWT import OWT 38 | from pygeoapi.processes.pyOWT.pyowt.satellite_handlers.eumetsat_olci_level2 import eumetsat_olci_level2 39 | 40 | 41 | # satellite data pkg 42 | import xarray as xr 43 | import zipfile 44 | import tempfile 45 | from lxml import etree 46 | import os 47 | from datetime import date 48 | 49 | 50 | def read_csv_with_auto_sep(file_path): 51 | separators = [',', '\t', r'\s+'] 52 | for sep in separators: 53 | try: 54 | df = pd.read_csv(file_path, sep=sep) 55 | if len(df.columns) > 1: 56 | return df 57 | except Exception as e: 58 | continue 59 | raise ValueError('Unable to read file with common separators: ' + ' '.join(separators)) 60 | 61 | def run_owt_csv(input_path_to_csv, input_sensor, output_path, output_option=1): 62 | d = read_csv_with_auto_sep(input_path_to_csv) 63 | Rrs = d.values.reshape(d.shape[0], 1, d.shape[1]) 64 | band = [float(x) for x in d.columns.tolist()] 65 | 66 | input_sensor = None if input_sensor == "HYPER" else input_sensor 67 | ov = OpticalVariables(Rrs=Rrs, band=band, sensor=input_sensor) 68 | owt = OWT(ov.AVW, ov.Area, ov.NDI) 69 | u_total = np.sum(owt.u, axis=-1) 70 | 71 | if output_option == 1: 72 | owt_result = pd.DataFrame({ 73 | 'OWT': owt.type_str.flatten(), 74 | 'AVW': owt.AVW.flatten().round(3), 75 | 'Area': owt.Area.flatten().round(3), 76 | 'NDI': owt.NDI.flatten().round(3), 77 | 'Utot': u_total.flatten().round(3), 78 | }) 79 | 80 | elif output_option == 2: 81 | u = np.round(owt.u, 5) 82 | owt_result = pd.DataFrame({ 83 | 'OWT': owt.type_str.flatten(), 84 | 'AVW': owt.AVW.flatten().round(3), 85 | 'Area': owt.Area.flatten().round(3), 86 | 'NDI': owt.NDI.flatten().round(3), 87 | 'Utot': u_total.flatten().round(3), 88 | 'U_OWT1': u[:,:,0].flatten(), 89 | 'U_OWT2': u[:,:,1].flatten(), 90 | 'U_OWT3a': u[:,:,2].flatten(), 91 | 'U_OWT3b': u[:,:,3].flatten(), 92 | 'U_OWT4a': u[:,:,4].flatten(), 93 | 'U_OWT4b': u[:,:,5].flatten(), 94 | 'U_OWT5a': u[:,:,6].flatten(), 95 | 'U_OWT5b': u[:,:,7].flatten(), 96 | 'U_OWT6': u[:,:,8].flatten(), 97 | 'U_OWT7': u[:,:,9].flatten(), 98 | }) 99 | else: 100 | raise ValueError('The output_option should be either 1 or 2') 101 | 102 | owt_result.to_csv(output_path, index=False) 103 | 104 | return owt_result 105 | 106 | def find_file(directory, filename): 107 | for root, dirs, files in os.walk(directory): 108 | if filename in files: 109 | return os.path.join(root, filename) 110 | return None 111 | 112 | 113 | def run_owt_sat(input_path_to_sat, input_sensor, output_path, output_option=1): 114 | 115 | eumetsat = eumetsat_olci_level2(filename=input_path_to_sat, sensor=input_sensor, save_path=output_path, save=False) 116 | 117 | if output_option == 1: 118 | eumetsat.save_result() 119 | elif output_option == 2: 120 | for sel_type in eumetsat.owt.classInfo.typeName: 121 | eumetsat.ds_new[f'U_OWT{sel_type}'] = ( 122 | ['rows', 'columns'], 123 | eumetsat.u[:,:,eumetsat.owt.classInfo.typeName.index(sel_type)].astype(np.float32), 124 | {'Description': f'Membership values of optical water type {sel_type}'} 125 | ) 126 | eumetsat.save_result() 127 | else: 128 | raise ValueError('The output_option should be either 1 or 2') 129 | 130 | 131 | def main(): 132 | 133 | project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../")) 134 | data_file_path = os.path.join(project_root, 'pyowt', 'data', 'AVW_all_regression_800.txt') 135 | 136 | support_sensors = pd.read_csv(data_file_path).iloc[:, 0].astype(str) 137 | support_sensors = ', '.join(support_sensors) 138 | 139 | parser = argparse.ArgumentParser(description='Perform Optical Water Type classification based on Bi and Hieronymi (2024)') 140 | parser.add_argument('--input', type=str, required=True, help='Path to your input file') 141 | parser.add_argument('--input_option', type=str, default='csv', required=True, help='Input option. "csv" for text data input; "sat" for satellite products input (e.g., Sentinel-3 OLCI Level-2)') 142 | parser.add_argument('--sensor', type=str, required=True, help=f'Name of you sensor. Select from {support_sensors}') 143 | parser.add_argument('--output', type=str, required=True, help='Path to your output file') 144 | parser.add_argument('--output_option', type=int, required=True, default='1', help='Output option. 1: for standard output; 2: for extensive output with memberships of all types') 145 | 146 | args = parser.parse_args() 147 | 148 | if args.input_option == "csv": 149 | run_owt_csv(input_path_to_csv=args.input, input_sensor=args.sensor, output_path=args.output, output_option=args.output_option) 150 | elif args.input_option == "sat": 151 | run_owt_sat(input_path_to_sat=args.input, input_sensor=args.sensor, output_path=args.output, output_option=args.output_option) 152 | else: 153 | raise ValueError('The input_option should be either "csv" or "sat"') 154 | 155 | 156 | 157 | if __name__ == "__main__": 158 | 159 | main() 160 | 161 | -------------------------------------------------------------------------------- /pyowt/satellite_handlers/envi_liu_products.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import xarray as xr 3 | import re 4 | 5 | try: 6 | from osgeo import gdal, osr 7 | osgeo_installed = True 8 | except ImportError: 9 | osgeo_installed = False 10 | 11 | class ENVIImageReader: 12 | def __init__(self, img_file): 13 | if not osgeo_installed: 14 | raise ImportError("The 'osgeo' package is required but not installed. Please install it using 'pip install gdal'.") 15 | 16 | self.img_file = img_file 17 | self.dataset = gdal.Open(img_file, gdal.GA_ReadOnly) 18 | if self.dataset is None: 19 | raise FileNotFoundError(f"Unable to open image file: {img_file}") 20 | 21 | def get_band_info(self): 22 | """ Get number of bands and band names """ 23 | band_count = self.dataset.RasterCount 24 | band_names = [ 25 | self.dataset.GetRasterBand(i).GetDescription() or f"Band_{i}" 26 | for i in range(1, band_count + 1) 27 | ] 28 | return band_count, band_names 29 | 30 | def get_geotransform_info(self): 31 | """ Get geotransformation and projection information """ 32 | geotransform = self.dataset.GetGeoTransform() 33 | projection = self.dataset.GetProjection() 34 | spatial_ref = osr.SpatialReference() 35 | spatial_ref.ImportFromWkt(projection) 36 | return geotransform, spatial_ref 37 | 38 | @staticmethod 39 | def generate_geo_coords(nrows, ncols, geotransform, spatial_ref): 40 | '''This part is quite time-cosuming 41 | ''' 42 | # Create lon/lat coordinate arrays 43 | target_spatial_ref = spatial_ref.CloneGeogCS() 44 | # transform = osr.CoordinateTransformation(spatial_ref, target_spatial_ref) 45 | lon_coords = np.zeros((nrows, ncols)) 46 | lat_coords = np.zeros((nrows, ncols)) 47 | 48 | if spatial_ref.IsGeographic(): 49 | print("This image file doesn't have a projection and is Geographic.") 50 | for i in range(nrows): 51 | for j in range(ncols): 52 | lon = geotransform[0] + j * geotransform[1] + i * geotransform[2] 53 | lat = geotransform[3] + j * geotransform[4] + i * geotransform[5] 54 | lat_coords[i, j] = lat 55 | lon_coords[i, j] = lon 56 | else: 57 | transform = osr.CoordinateTransformation(spatial_ref, target_spatial_ref) 58 | for i in range(nrows): 59 | for j in range(ncols): 60 | x = geotransform[0] + j * geotransform[1] + i * geotransform[2] 61 | y = geotransform[3] + j * geotransform[4] + i * geotransform[5] 62 | lat, lon, _ = transform.TransformPoint(x, y) 63 | lat_coords[i, j] = lat 64 | lon_coords[i, j] = lon 65 | 66 | # for i in range(nrows): 67 | # for j in range(ncols): 68 | # x = geotransform[0] + j * geotransform[1] + i * geotransform[2] 69 | # y = geotransform[3] + j * geotransform[4] + i * geotransform[5] 70 | # lat, lon, _ = transform.TransformPoint(x, y) 71 | # lat_coords[i, j] = lat 72 | # lon_coords[i, j] = lon 73 | return lat_coords, lon_coords 74 | 75 | def to_xarray(self, band_prefix=None, skip_geo_coords=False): 76 | """ 77 | Convert image data to xarray Dataset. 78 | Optionally filter bands by prefix (e.g., 'rhos', 'Rrs'). 79 | """ 80 | band_count, band_names = self.get_band_info() 81 | 82 | pattern = re.compile(rf'.*{band_prefix}_\d+.*') 83 | matched_org_names = [name for name in band_names if pattern.search(name)] 84 | 85 | # Read band data and create xarray variables 86 | data_vars = {} 87 | mask = None 88 | 89 | for band_name in matched_org_names: 90 | band_number = band_names.index(band_name) + 1 91 | band = self.dataset.GetRasterBand(band_number) 92 | band_array = band.ReadAsArray() 93 | 94 | match = re.search(rf'({band_prefix}_\d+)', band_name) 95 | if match: 96 | var_name = match.group(1) 97 | wavelength = var_name.split('_')[1] 98 | else: 99 | var_name = band_name 100 | wavelength = 'unknown' 101 | 102 | if mask is None: 103 | mask = np.zeros_like(band_array, dtype=bool) 104 | mask |= (band_array != 0) 105 | 106 | data_vars[var_name] = xr.DataArray( 107 | band_array, 108 | dims=('y', 'x'), 109 | attrs={ 110 | 'radiation_wavelength': wavelength, 111 | 'radiation_wavelength_unit': 'nm', 112 | 'description': band_name 113 | } 114 | ) 115 | 116 | # Image dimensions 117 | ncols = self.dataset.RasterXSize 118 | nrows = self.dataset.RasterYSize 119 | 120 | base_coords = { 121 | 'x': ('x', np.arange(ncols)), 122 | 'y': ('y', np.arange(nrows)) 123 | } 124 | 125 | # Calculate lat and lon coords for netcdf variables (if needed) 126 | if not skip_geo_coords: 127 | geotransform, spatial_ref = self.get_geotransform_info() 128 | lat_coords, lon_coords = self.generate_geo_coords(nrows, ncols, geotransform, spatial_ref) 129 | 130 | base_coords.update({ 131 | 'lon': (('y', 'x'), lon_coords), 132 | 'lat': (('y', 'x'), lat_coords), 133 | }) 134 | 135 | # Set pixels with all-zero values to NaN 136 | for band_name in data_vars: 137 | data_vars[band_name] = data_vars[band_name].where(mask, np.nan) 138 | 139 | # Create xarray Dataset with latitude/longitude coordinates 140 | xr_dataset = xr.Dataset( 141 | data_vars, 142 | coords=base_coords, 143 | attrs={ 144 | 'crs': spatial_ref.ExportToWkt() if not skip_geo_coords else 'None', 145 | 'transform': geotransform if not skip_geo_coords else 'None' 146 | } 147 | ) 148 | 149 | if not skip_geo_coords: 150 | xr_dataset['lat'].attrs = { 151 | 'units': 'degrees_north', 152 | 'standard_name': 'latitude', 153 | 'long_name': 'Latitude' 154 | } 155 | xr_dataset['lon'].attrs = { 156 | 'units': 'degrees_east', 157 | 'standard_name': 'longitude', 158 | 'long_name': 'Longitude' 159 | } 160 | 161 | return xr_dataset 162 | 163 | def save_as_netcdf(self, output_file, band_prefix=None, skip_geo_coords=False): 164 | """ Save image data as a NetCDF file """ 165 | xr_dataset = self.to_xarray(band_prefix=band_prefix, skip_geo_coords=skip_geo_coords) 166 | xr_dataset.to_netcdf(output_file, format='NETCDF4') 167 | print(f"Saved as NetCDF file: {output_file}") 168 | 169 | # Example usage 170 | if __name__ == "__main__": 171 | img_file = '/Users/apple/Satellite_data/Liu/504.shp/S3A_OL_1_EFR____20160429T013941_20160429T014241_20180205T153045_0180_003_288_2340_LR2_R_NT_002_x.Rrs.img' 172 | output_file = '/Users/apple/Satellite_data/Liu/gdal_test.nc' 173 | 174 | reader = ENVIImageReader(img_file) 175 | reader.save_as_netcdf(output_file, band_prefix='Rrs') 176 | 177 | -------------------------------------------------------------------------------- /projects/AquaINFRA/pygeoapi_processes/hereon_pyowt.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from pygeoapi.process.base import BaseProcessor, ProcessorExecuteError 4 | from pygeoapi.process.pyOWT.run_AquaINFRA import run_owt_csv 5 | from pygeoapi.process.pyOWT.run_AquaINFRA import run_owt_sat 6 | import os 7 | import json 8 | import requests 9 | 10 | 11 | LOGGER = logging.getLogger(__name__) 12 | 13 | 14 | ''' 15 | 16 | ### Reading input csv from any URL: 17 | # Example 1: HYPER, Rrs_demo_AquaINFRA_hyper.csv 18 | curl -X POST "http://localhost:5000/processes/hereon-pyowt/execution" -H "Content-Type: application/json" -d "{\"inputs\":{\"input_data_url\": \"https://raw.githubusercontent.com/bishun945/pyOWT/refs/heads/AquaINFRA/data/Rrs_demo_AquaINFRA_hyper.csv\", \"input_option\":\"csv\", \"sensor\":\"HYPER\", \"output_option\": 1}}" 19 | 20 | # Example 2: MSI_S2A, Rrs_demo_AquaINFRA_msi.csv 21 | curl -X POST "http://localhost:5000/processes/hereon-pyowt/execution" -H "Content-Type: application/json" -d "{\"inputs\":{\"input_data_url\": \"https://raw.githubusercontent.com/bishun945/pyOWT/refs/heads/AquaINFRA/data/Rrs_demo_AquaINFRA_msi.csv\", \"input_option\":\"csv\", \"sensor\":\"MSI_S2A\", \"output_option\": 1}}" 22 | 23 | # Example 3: OLCI_S3A, Rrs_demo_AquaINFRA_olci.csv 24 | curl -X POST "http://localhost:5000/processes/hereon-pyowt/execution" -H "Content-Type: application/json" -d "{\"inputs\":{\"input_data_url\": \"https://raw.githubusercontent.com/bishun945/pyOWT/refs/heads/AquaINFRA/data/Rrs_demo_AquaINFRA_olci.csv\", \"input_option\":\"csv\", \"sensor\":\"OLCI_S3A\", \"output_option\": 1}}" 25 | 26 | ### Reading the example files from server: 27 | # Example 1: HYPER, Rrs_demo_AquaINFRA_hyper.csv 28 | curl -X POST "http://localhost:5000/processes/hereon-pyowt/execution" -H "Content-Type: application/json" -d "{\"inputs\":{\"input_data_url\": \"Rrs_demo_AquaINFRA_hyper.csv\", \"input_option\":\"csv\", \"sensor\":\"HYPER\", \"output_option\": 1}}" 29 | 30 | # Example 2: MSI_S2A, Rrs_demo_AquaINFRA_msi.csv 31 | curl -X POST "http://localhost:5000/processes/hereon-pyowt/execution" -H "Content-Type: application/json" -d "{\"inputs\":{\"input_data_url\": \"Rrs_demo_AquaINFRA_msi.csv\", \"input_option\":\"csv\", \"sensor\":\"MSI_S2A\", \"output_option\": 1}}" 32 | 33 | # Example 3: OLCI_S3A, Rrs_demo_AquaINFRA_olci.csv 34 | curl -X POST "http://localhost:5000/processes/hereon-pyowt/execution" -H "Content-Type: application/json" -d "{\"inputs\":{\"input_data_url\": \"Rrs_demo_AquaINFRA_olci.csv\", \"input_option\":\"csv\", \"sensor\":\"OLCI_S3A\", \"output_option\": 1}}" 35 | 36 | ### Extensive output: 37 | curl -X POST "http://localhost:5000/processes/hereon-pyowt/execution" -H "Content-Type: application/json" -d "{\"inputs\":{\"input_data_url\": \"Rrs_demo_AquaINFRA_hyper.csv\", \"input_option\":\"csv\", \"sensor\":\"HYPER\", \"output_option\": 2}}" 38 | 39 | ''' 40 | 41 | # Process metadata and description 42 | # Has to be in a JSON file of the same name, in the same dir! 43 | script_title_and_path = __file__ 44 | metadata_title_and_path = script_title_and_path.replace('.py', '.json') 45 | PROCESS_METADATA = json.load(open(metadata_title_and_path)) 46 | 47 | 48 | 49 | class HEREON_PyOWT_Processor(BaseProcessor): 50 | 51 | def __init__(self, processor_def): 52 | super().__init__(processor_def, PROCESS_METADATA) 53 | self.supports_outputs = True 54 | self.job_id = None 55 | self.config = None 56 | 57 | # Set config: 58 | config_file_path = os.environ.get('PYOWT_CONFIG_FILE', "./config.json") 59 | with open(config_file_path, 'r') as config_file: 60 | self.config = json.load(config_file) 61 | 62 | def __repr__(self): 63 | return f' {self.name}' 64 | 65 | def set_job_id(self, job_id: str): 66 | self.job_id = job_id 67 | 68 | def execute(self, data, outputs=None): 69 | input_data_url = data.get('input_data_url', 'Rrs_demo_AquaINFRA_hyper.csv') 70 | input_option = data.get('input_option') 71 | sensor = data.get('sensor') 72 | output_option = int(data.get('output_option')) 73 | 74 | # Check sensor: 75 | support_sensors = set(['HYPER', 'AVW700', 'MODIS_Aqua', 'MODIS_Terra', 'OLCI_S3A', 'OLCI_S3B', 'MERIS', 'SeaWiFS', 'HawkEye', 'OCTS', 'GOCI', 'VIIRS_NPP', 'VIIRS_JPSS1', 'VIIRS_JPSS2', 'CZCS', 'MSI_S2A', 'MSI_S2B', 'OLI', 'ENMAP_HSI', 'CMEMS_HROC_L3_optics', 'cmems_P1D400', 'AERONET_OC_1', 'AERONET_OC_2']) 76 | #support_sensors = pd.read_csv("data/AVW_all_regression_800.txt").iloc[:, 0].astype(str) 77 | # TODO: Read this from AVW_all_regression_800.txt everytime. Add HYPER manually. 78 | # TODO: Ask Shun Bi: Why is HYPER included? 79 | if not sensor in support_sensors: 80 | raise ProcessorExecuteError('Sensor not supported: "%s". Please pick one of: %s' % (sensor, ', '.join(support_sensors))) 81 | 82 | # Use example input file: 83 | input_path = None 84 | if input_data_url in ['Rrs_demo_AquaINFRA_hyper.csv', 'Rrs_demo_AquaINFRA_msi.csv', 'Rrs_demo_AquaINFRA_olci.csv']: 85 | LOGGER.info('Using example input data file: %s' % input_data_url) 86 | input_dir = self.config['pyowt']['example_input_data_dir'] 87 | input_path = input_dir.rstrip('/')+os.sep+input_data_url 88 | 89 | # ... Or download input file: 90 | # TODO: Also allow user to paste CSV as POST request body/payload! - Check size though! 91 | else: 92 | LOGGER.info('Downloading input data file: %s' % input_data_url) 93 | resp = requests.get(input_data_url) 94 | if resp.status_code == 200: 95 | input_dir = self.config['pyowt']['input_data_dir'] 96 | input_path = input_dir.rstrip('/')+os.sep+'inputs_%s' % self.job_id 97 | LOGGER.debug('Writing input data file to: %s' % input_path) 98 | with open(input_path, 'w') as myfile: 99 | myfile.write(resp.text) 100 | else: 101 | raise ProcessorExecuteError('Could not download input file (HTTP status %s): %s' % (resp.status_code, input_data_url)) 102 | 103 | # Where to store output 104 | downloadfilename = 'pyowt_output_%s-%s.txt' % (sensor.lower(), self.job_id) 105 | downloadfilepath = self.config['download_dir']+downloadfilename 106 | 107 | # https://github.com/bishun945/pyOWT/blob/AquaINFRA/run_AquaINFRA.py 108 | if input_option.lower() == 'csv': 109 | run_owt_csv(input_path_to_csv=input_path, input_sensor=sensor, output_path=downloadfilepath, output_option=output_option) 110 | elif input_option.lower() == 'sat': 111 | run_owt_sat(input_path_to_sat=input_path, input_sensor=sensor, output_path=downloadfilepath, output_option=output_option) 112 | else: 113 | err_msg = 'The input_option should be either "csv" or "sat"' 114 | raise ProcessorExecuteError(err_msg) 115 | 116 | # Create download link: 117 | downloadlink = self.config['download_url'] + downloadfilename 118 | 119 | # Build response containing the link 120 | # TODO Better naming 121 | response_object = { 122 | "outputs": { 123 | "some_output": { 124 | 'title': self.metadata['outputs']["some_output"]['title'], 125 | 'description': self.metadata['outputs']["some_output"]['description'], 126 | "href": downloadlink 127 | } 128 | } 129 | } 130 | LOGGER.debug('Built response including link: %s' % response_object) 131 | 132 | return 'application/json', response_object 133 | 134 | 135 | -------------------------------------------------------------------------------- /projects/AquaINFRA/pygeoapi_processes/owt_classification.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from pygeoapi.process.base import BaseProcessor, ProcessorExecuteError 4 | from pygeoapi.process.pyOWT.projects.AquaINFRA.run_AquaINFRA import run_owt_csv 5 | from pygeoapi.process.pyOWT.projects.AquaINFRA.run_AquaINFRA import run_owt_sat 6 | import os 7 | import json 8 | import requests 9 | 10 | 11 | LOGGER = logging.getLogger(__name__) 12 | 13 | 14 | ''' 15 | 16 | ### Reading input csv from any URL: 17 | # Example 1: HYPER, Rrs_demo_AquaINFRA_hyper.csv 18 | curl -X POST "http://localhost:5000/processes/hereon-pyowt/execution" -H "Content-Type: application/json" -d "{\"inputs\":{\"input_data_url\": \"https://raw.githubusercontent.com/bishun945/pyOWT/refs/heads/AquaINFRA/data/Rrs_demo_AquaINFRA_hyper.csv\", \"input_option\":\"csv\", \"sensor\":\"HYPER\", \"output_option\": 1}}" 19 | 20 | # Example 2: MSI_S2A, Rrs_demo_AquaINFRA_msi.csv 21 | curl -X POST "http://localhost:5000/processes/hereon-pyowt/execution" -H "Content-Type: application/json" -d "{\"inputs\":{\"input_data_url\": \"https://raw.githubusercontent.com/bishun945/pyOWT/refs/heads/AquaINFRA/data/Rrs_demo_AquaINFRA_msi.csv\", \"input_option\":\"csv\", \"sensor\":\"MSI_S2A\", \"output_option\": 1}}" 22 | 23 | # Example 3: OLCI_S3A, Rrs_demo_AquaINFRA_olci.csv 24 | curl -X POST "http://localhost:5000/processes/hereon-pyowt/execution" -H "Content-Type: application/json" -d "{\"inputs\":{\"input_data_url\": \"https://raw.githubusercontent.com/bishun945/pyOWT/refs/heads/AquaINFRA/data/Rrs_demo_AquaINFRA_olci.csv\", \"input_option\":\"csv\", \"sensor\":\"OLCI_S3A\", \"output_option\": 1}}" 25 | 26 | ### Reading the example files from server: 27 | # Example 1: HYPER, Rrs_demo_AquaINFRA_hyper.csv 28 | curl -X POST "http://localhost:5000/processes/hereon-pyowt/execution" -H "Content-Type: application/json" -d "{\"inputs\":{\"input_data_url\": \"Rrs_demo_AquaINFRA_hyper.csv\", \"input_option\":\"csv\", \"sensor\":\"HYPER\", \"output_option\": 1}}" 29 | 30 | # Example 2: MSI_S2A, Rrs_demo_AquaINFRA_msi.csv 31 | curl -X POST "http://localhost:5000/processes/hereon-pyowt/execution" -H "Content-Type: application/json" -d "{\"inputs\":{\"input_data_url\": \"Rrs_demo_AquaINFRA_msi.csv\", \"input_option\":\"csv\", \"sensor\":\"MSI_S2A\", \"output_option\": 1}}" 32 | 33 | # Example 3: OLCI_S3A, Rrs_demo_AquaINFRA_olci.csv 34 | curl -X POST "http://localhost:5000/processes/hereon-pyowt/execution" -H "Content-Type: application/json" -d "{\"inputs\":{\"input_data_url\": \"Rrs_demo_AquaINFRA_olci.csv\", \"input_option\":\"csv\", \"sensor\":\"OLCI_S3A\", \"output_option\": 1}}" 35 | 36 | ### Extensive output: 37 | curl -X POST "http://localhost:5000/processes/hereon-pyowt/execution" -H "Content-Type: application/json" -d "{\"inputs\":{\"input_data_url\": \"Rrs_demo_AquaINFRA_hyper.csv\", \"input_option\":\"csv\", \"sensor\":\"HYPER\", \"output_option\": 2}}" 38 | 39 | ''' 40 | 41 | # Process metadata and description 42 | # Has to be in a JSON file of the same name, in the same dir! 43 | script_title_and_path = __file__ 44 | metadata_title_and_path = script_title_and_path.replace('.py', '.json') 45 | PROCESS_METADATA = json.load(open(metadata_title_and_path)) 46 | 47 | 48 | 49 | class OwtClassificationProcessor(BaseProcessor): 50 | 51 | def __init__(self, processor_def): 52 | super().__init__(processor_def, PROCESS_METADATA) 53 | self.supports_outputs = True 54 | self.job_id = None 55 | self.config = None 56 | 57 | # Set config: 58 | config_file_path = os.environ.get('PYOWT_CONFIG_FILE', "./config.json") 59 | with open(config_file_path, 'r') as config_file: 60 | self.config = json.load(config_file) 61 | 62 | def __repr__(self): 63 | return f' {self.name}' 64 | 65 | def set_job_id(self, job_id: str): 66 | self.job_id = job_id 67 | 68 | def execute(self, data, outputs=None): 69 | input_data_url = data.get('input_data_url', 'Rrs_demo_AquaINFRA_hyper.csv') 70 | input_option = data.get('input_option') 71 | sensor = data.get('sensor') 72 | output_option = int(data.get('output_option')) 73 | 74 | # Check sensor: 75 | support_sensors = set(['HYPER', 'AVW700', 'MODIS_Aqua', 'MODIS_Terra', 'OLCI_S3A', 'OLCI_S3B', 'MERIS', 'SeaWiFS', 'HawkEye', 'OCTS', 'GOCI', 'VIIRS_NPP', 'VIIRS_JPSS1', 'VIIRS_JPSS2', 'CZCS', 'MSI_S2A', 'MSI_S2B', 'OLI', 'ENMAP_HSI', 'CMEMS_HROC_L3_optics', 'cmems_P1D400', 'AERONET_OC_1', 'AERONET_OC_2']) 76 | #support_sensors = pd.read_csv("data/AVW_all_regression_800.txt").iloc[:, 0].astype(str) 77 | # TODO: Read this from AVW_all_regression_800.txt everytime. Add HYPER manually. 78 | # TODO: Ask Shun Bi: Why is HYPER included? 79 | if not sensor in support_sensors: 80 | raise ProcessorExecuteError('Sensor not supported: "%s". Please pick one of: %s' % (sensor, ', '.join(support_sensors))) 81 | 82 | # Use example input file: 83 | input_path = None 84 | if input_data_url in ['Rrs_demo_AquaINFRA_hyper.csv', 'Rrs_demo_AquaINFRA_msi.csv', 'Rrs_demo_AquaINFRA_olci.csv']: 85 | LOGGER.info('Using example input data file: %s' % input_data_url) 86 | input_dir = self.config['pyowt']['example_input_data_dir'] 87 | input_path = input_dir.rstrip('/')+os.sep+input_data_url 88 | 89 | # ... Or download input file: 90 | # TODO: Also allow user to paste CSV as POST request body/payload! - Check size though! 91 | else: 92 | LOGGER.info('Downloading input data file: %s' % input_data_url) 93 | resp = requests.get(input_data_url) 94 | if resp.status_code == 200: 95 | input_dir = self.config['pyowt']['input_data_dir'] 96 | input_path = input_dir.rstrip('/')+os.sep+'inputs_%s' % self.job_id 97 | LOGGER.debug('Writing input data file to: %s' % input_path) 98 | with open(input_path, 'w') as myfile: 99 | myfile.write(resp.text) 100 | else: 101 | raise ProcessorExecuteError('Could not download input file (HTTP status %s): %s' % (resp.status_code, input_data_url)) 102 | 103 | # Where to store output 104 | downloadfilename = 'pyowt_output_%s-%s.txt' % (sensor.lower(), self.job_id) 105 | downloadfilepath = self.config['download_dir']+downloadfilename 106 | 107 | # https://github.com/bishun945/pyOWT/blob/AquaINFRA/run_AquaINFRA.py 108 | if input_option.lower() == 'csv': 109 | run_owt_csv(input_path_to_csv=input_path, input_sensor=sensor, output_path=downloadfilepath, output_option=output_option) 110 | elif input_option.lower() == 'sat': 111 | run_owt_sat(input_path_to_sat=input_path, input_sensor=sensor, output_path=downloadfilepath, output_option=output_option) 112 | else: 113 | err_msg = 'The input_option should be either "csv" or "sat"' 114 | raise ProcessorExecuteError(err_msg) 115 | 116 | # Create download link: 117 | downloadlink = self.config['download_url'] + downloadfilename 118 | 119 | # Build response containing the link 120 | # TODO Better naming 121 | response_object = { 122 | "outputs": { 123 | "owt_classification": { 124 | 'title': self.metadata['outputs']["owt_classification"]['title'], 125 | 'description': self.metadata['outputs']["owt_classification"]['description'], 126 | "href": downloadlink 127 | } 128 | } 129 | } 130 | LOGGER.debug('Built response including link: %s' % response_object) 131 | 132 | return 'application/json', response_object 133 | 134 | 135 | -------------------------------------------------------------------------------- /pyowt/satellite_handlers/lakecci_products.py: -------------------------------------------------------------------------------- 1 | import xarray as xr 2 | import numpy as np 3 | import os 4 | import re 5 | import dask 6 | 7 | from pyowt.OpticalVariables import OpticalVariables 8 | from pyowt.OWT import OWT 9 | 10 | def owt_classification_on_chunk(rrs_chunk, band_wavelengths, sensor_name): 11 | """ 12 | A wrapper function to run the complete OWT classification process on a data chunk. 13 | This function is called by xarray.apply_ufunc for each chunk. 14 | """ 15 | if np.all(np.isnan(rrs_chunk)): 16 | nan_shape = rrs_chunk.shape[:-1] # Get lat, lon shape 17 | return (np.full(nan_shape, np.nan, dtype=np.float32), 18 | np.full(nan_shape, np.nan, dtype=np.float32), 19 | np.full(nan_shape, np.nan, dtype=np.float32), 20 | np.full(nan_shape, -1, dtype=np.int32)) 21 | 22 | # Calculate optical variables from the Rrs chunk. 23 | ov = OpticalVariables(Rrs=rrs_chunk, band=band_wavelengths, sensor=sensor_name) 24 | 25 | # Perform the OWT classification. 26 | owt = OWT(ov.AVW, ov.Area, ov.NDI) 27 | 28 | # Clip the optical variable values to their valid ranges. 29 | avw_clipped = np.where((owt.AVW >= 400) & (owt.AVW <= 800), owt.AVW, np.nan) 30 | ndi_clipped = np.where((owt.NDI >= -1) & (owt.NDI <= 1), owt.NDI, np.nan) 31 | 32 | return (avw_clipped.astype(np.float32), 33 | owt.Area.astype(np.float32), 34 | ndi_clipped.astype(np.float32), 35 | owt.type_idx.astype(np.int32)) 36 | 37 | class LakeCCIProcessor: 38 | """ 39 | A class to process a single ESA Lake CCI NetCDF file. 40 | It applies the OWT classification using Dask for out-of-core processing 41 | and outputs a single, complete NetCDF file with the results. 42 | """ 43 | def __init__( 44 | self, 45 | filename, 46 | output_dir=None, 47 | chunk_sizes={"lat": 1000, "lon": 1000}, 48 | keep_rrs_bands=True, 49 | verbose=True, 50 | ): 51 | """ 52 | Initializes and runs the processing workflow. 53 | 54 | Args: 55 | filename (str): Path to the input Lake CCI NetCDF file. 56 | chunk_sizes (dict): A dictionary specifying the chunk sizes for Dask. 57 | keep_rrs_bands (bool): If True, the output file will include the Rrs bands. 58 | If False (default), only classification results are saved. 59 | """ 60 | if not os.path.exists(filename): 61 | print(f"Error: Input file not found at '{filename}'.") 62 | return 63 | 64 | self.filename = filename 65 | if output_dir is None: 66 | self.output_filename = self.filename.replace('.nc', '_owt_result.nc') 67 | else: 68 | basename = os.path.basename(self.filename) 69 | self.output_filename = os.path.join(output_dir, basename.replace('.nc', '_owt_result.nc')) 70 | 71 | self.chunk_sizes = chunk_sizes 72 | self.keep_rrs_bands = keep_rrs_bands 73 | self.verbose = verbose 74 | 75 | self.sensor = 'LakeCCI-MERIS' 76 | self.predefined_bands = {'LakeCCI-MERIS': [413, 443, 490, 510, 560, 620, 665, 681, 709, 754, 779, 885]} 77 | 78 | self.process() 79 | 80 | def process(self): 81 | """ 82 | Executes the main data processing workflow. 83 | """ 84 | required_wavelengths = self.predefined_bands[self.sensor] 85 | 86 | with xr.open_dataset(self.filename, chunks=self.chunk_sizes) as ds: 87 | available_rw_vars = [var for var in ds.data_vars if var.startswith('Rw') and var[2:].isdigit()] 88 | available_wavelength_map = {int(re.search(r'\d+', var).group()): var for var in available_rw_vars} 89 | available_wavelengths = np.array(list(available_wavelength_map.keys())) 90 | 91 | wavelen_sel, final_wavelengths = [], [] 92 | for req_wl in required_wavelengths: 93 | closest_idx = np.argmin(np.abs(available_wavelengths - req_wl)) 94 | closest_wl = available_wavelengths[closest_idx] 95 | wavelen_sel.append(available_wavelength_map[closest_wl]) 96 | final_wavelengths.append(closest_wl) 97 | 98 | # the input is called water reflectance not rrs? 99 | ds_rw = ds[wavelen_sel] 100 | ds_rrs = ds_rw / np.pi 101 | 102 | # set all required attributes for Rrs bands 103 | for var_name in ds_rrs.data_vars: 104 | # Use the actual wavelength found for the attributes 105 | wavelength = available_wavelength_map[final_wavelengths[wavelen_sel.index(var_name)]] 106 | attrs = ds_rrs[var_name].attrs 107 | attrs['long_name'] = f"Remote sensing reflectance at {wavelength} nm" 108 | attrs['units'] = 'sr-1' 109 | attrs['radiation_wavelength'] = float(wavelength[2:]) 110 | attrs['radiation_wavelength_unit'] = 'nm' 111 | 112 | rrs_dask_array_unchunked_vars = ds_rrs.to_array(dim='variable') 113 | rrs_dask_array = rrs_dask_array_unchunked_vars.chunk({**self.chunk_sizes, 'variable': -1}) 114 | rrs_dask_array = rrs_dask_array.transpose('time', 'lat', 'lon', 'variable').isel(time=0, drop=True) 115 | 116 | if self.verbose: 117 | print("Setting up Dask computation graph...") 118 | 119 | avw, area, ndi, type_idx = xr.apply_ufunc( 120 | owt_classification_on_chunk, 121 | rrs_dask_array, 122 | input_core_dims=[["variable"]], 123 | output_core_dims=[[], [], [], []], 124 | exclude_dims=set(("variable",)), 125 | dask="parallelized", 126 | output_dtypes=[np.float32, np.float32, np.float32, np.int32], 127 | kwargs={ 128 | "band_wavelengths": final_wavelengths, 129 | "sensor_name": self.sensor, 130 | }, 131 | ) 132 | 133 | if self.verbose: 134 | print("Building the final output dataset...") 135 | # Conditionally build the output dataset 136 | if self.keep_rrs_bands: 137 | # Start with the newly created Rrs variables 138 | ds_out = ds_rrs.isel(time=0, drop=True) 139 | else: 140 | # Start with an empty dataset containing only coordinates 141 | ds_out = xr.Dataset(coords=ds_rrs.coords) 142 | 143 | # Add the new classification variables. 144 | ds_out['type_idx'] = type_idx 145 | ds_out['AVW'] = avw 146 | ds_out['Area'] = area 147 | ds_out['NDI'] = ndi 148 | 149 | # Set attributes for the new variables. 150 | ds_out['type_idx'].attrs = {'long_name': 'Optical Water Type Index', '_FillValue': -1} 151 | ds_out['AVW'].attrs = {'long_name': 'Apparent Visible Wavelength', 'units': 'nm'} 152 | ds_out['Area'].attrs = {'long_name': 'Trapezoidal area of Rrs at RGB bands', 'units': 'sr-1 nm'} 153 | ds_out['NDI'].attrs = {'long_name': 'Normalized Difference Index', 'units': '1'} 154 | 155 | if 'unlimited_dims' in ds_out.encoding: 156 | del ds_out.encoding['unlimited_dims'] 157 | 158 | encoding = {var: {'zlib': True, 'complevel': 5} for var in ds_out.data_vars} 159 | 160 | if self.verbose: 161 | print(f"Starting computation and writing to a single file: {self.output_filename}") 162 | 163 | with dask.config.set(scheduler='synchronous'): 164 | ds_out.to_netcdf(self.output_filename, compute=True, encoding=encoding) 165 | 166 | if self.verbose: 167 | print(f"\n--- Global data processing complete! ---") 168 | 169 | return None 170 | 171 | 172 | if __name__ == "__main__": 173 | # Example of how to use the class. 174 | input_file = '/media/elbe/Data/LakeCCI/dap.ceda.ac.uk/neodc/esacci/lakes/data/lake_products/L3S/v2.1/merged_product/2009/01/ESACCI-LAKES-L3S-LK_PRODUCTS-MERGED-20090101-fv2.1.0.nc' 175 | 176 | # Define the chunk size for processing. 177 | # Adjust based on your available RAM. Smaller chunks use less memory. 178 | processing_chunks = {'lat': 10_000, 'lon': 10_000} 179 | 180 | # Create an instance of the processor, which will automatically run the workflow. 181 | LakeCCIProcessor(filename=input_file, chunk_sizes=processing_chunks) 182 | -------------------------------------------------------------------------------- /pyowt/satellite_handlers/eumetsat_olci_level2.py: -------------------------------------------------------------------------------- 1 | import xarray as xr 2 | import zipfile 3 | import tempfile 4 | import os 5 | from datetime import date 6 | import numpy as np 7 | from pyowt.OpticalVariables import OpticalVariables 8 | from pyowt.OWT import OWT 9 | 10 | try: 11 | from lxml import etree 12 | lxml_installed = True 13 | except ImportError: 14 | lxml_installed = False 15 | 16 | 17 | class eumetsat_olci_level2: 18 | 19 | def __init__(self, filename, sensor='OLCI_S3A', save_path=None, save=True): 20 | if not lxml_installed: 21 | raise ImportError("The 'lxml' package is required but not installed. Please install it using 'pip install lxml'.") 22 | 23 | self.filename = filename 24 | self.sensor = sensor 25 | 26 | if save_path is None: 27 | self.save_path = os.path.dirname(self.filename) 28 | else: 29 | self.save_path = save_path 30 | 31 | self.create_temporary_dir() 32 | 33 | try: 34 | self.parse_xml() 35 | self.apply_flag() 36 | self.read_geo_coordinates() 37 | self.read_reflectance() 38 | self.classification() 39 | self.prepare_nc() 40 | if save: 41 | self.save_result() 42 | finally: 43 | self.temp_dir.cleanup() 44 | self.zip_ref.close() 45 | 46 | def create_temporary_dir(self): 47 | # TODO: create under the satellite file path? 48 | zip_path = self.filename 49 | self.temp_dir = tempfile.TemporaryDirectory() 50 | self.temp_path = self.temp_dir.name 51 | self.zip_ref = zipfile.ZipFile(zip_path, 'r') 52 | self.zip_ref.extractall(self.temp_path) 53 | self.basename = os.path.basename(os.path.splitext(os.path.basename(zip_path))[0]) 54 | 55 | @staticmethod 56 | def find_file(directory, filename): 57 | for root, dirs, files in os.walk(directory): 58 | if filename in files: 59 | return os.path.join(root, filename) 60 | return None 61 | 62 | def parse_xml(self): 63 | self.path_xfdu = self.find_file(self.temp_path, 'xfdumanifest.xml') 64 | self.path = os.path.dirname(self.path_xfdu) 65 | tree = etree.parse(self.path_xfdu) 66 | namespaces = tree.getroot().nsmap 67 | wavelengths = tree.xpath('//sentinel3:centralWavelength/text()', namespaces=namespaces) 68 | wavelengths = [float(wl) for wl in wavelengths] 69 | bandnames = tree.xpath('//sentinel3:band/@name', namespaces=namespaces) 70 | 71 | self.wavelengths = wavelengths 72 | self.bandnames = bandnames 73 | 74 | def apply_flag(self): 75 | ds = xr.open_dataset(os.path.join(self.path, 'wqsf.nc')) 76 | wqsf = ds['WQSF'] 77 | attrs = wqsf.attrs 78 | 79 | flag_masks = attrs.get('flag_masks', []) 80 | flag_meanings = attrs.get('flag_meanings', '').split() 81 | 82 | # WATER or INLAND_WATER 83 | WATER_mask = flag_masks[flag_meanings.index('WATER')] 84 | INLAND_WATER_mask = flag_masks[flag_meanings.index('INLAND_WATER')] 85 | water_mask = (wqsf & WATER_mask) | (wqsf & INLAND_WATER_mask) 86 | 87 | # Initialize mask as True for everywhere 88 | final_mask = water_mask.astype(bool) 89 | 90 | # Exclude the listed flags 91 | exclude_flags = [ 92 | 'CLOUD', 'CLOUD_AMBIGUOUS', 'CLOUD_MARGIN', 'INVALID', 'COSMETIC', 93 | 'SATURATED', 'SUSPECT', 'HISOLZEN', 'HIGHGLINT', 'SNOW_ICE', 94 | 'AC_FAIL', 'WHITECAPS', 'ADJAC', 'RWNEG_O2', 'RWNEG_O3', 95 | 'RWNEG_O4', 'RWNEG_O5', 'RWNEG_O6', 'RWNEG_O7', 'RWNEG_O8' 96 | ] 97 | 98 | for flag in exclude_flags: 99 | mask = flag_masks[flag_meanings.index(flag)] 100 | final_mask &= ~(wqsf & mask).astype(bool) 101 | 102 | self.WQSF_REFLECTANCE_RECOM = final_mask.astype(int) 103 | 104 | def read_geo_coordinates(self): 105 | ds = xr.open_dataset(os.path.join(self.path, 'geo_coordinates.nc')) 106 | self.lon = ds['longitude'] 107 | self.lat = ds['latitude'] 108 | 109 | def read_reflectance(self): 110 | Ref_list = [] 111 | for i, bandname in enumerate(self.bandnames): 112 | nc_file_path = os.path.join(self.path, f"{bandname}_reflectance.nc") 113 | 114 | if os.path.exists(nc_file_path): 115 | ds = xr.open_dataset(nc_file_path) 116 | Ref_data = ds[f"{bandname}_reflectance"] 117 | Ref_data.attrs['radiation_wavelength'] = self.wavelengths[i] 118 | Ref_data.attrs['radiation_wavelength_unit'] = 'nm' 119 | Ref_data = Ref_data.assign_coords(longitude = self.lon, latitude = self.lat) 120 | 121 | Ref_list.append(Ref_data) 122 | else: 123 | print(f"File {nc_file_path} does not exist.") 124 | 125 | ds_new = xr.merge(Ref_list) 126 | 127 | reflectance_vars = np.array([ds_new[f"{bandname}_reflectance"].data for bandname in self.bandnames]).transpose(1, 2, 0) 128 | Rrs_vars = reflectance_vars / np.pi 129 | 130 | self.Rrs_vars = Rrs_vars 131 | self.ds_new = ds_new 132 | 133 | def classification(self): 134 | ov = OpticalVariables(Rrs=self.Rrs_vars, band=self.wavelengths, sensor=self.sensor) 135 | owt = OWT(ov.AVW, ov.Area, ov.NDI) 136 | self.ov = ov 137 | self.owt = owt 138 | 139 | def prepare_nc(self): 140 | self.ds_new = self.ds_new.drop_vars([f"{bandname}_reflectance" for bandname in self.bandnames]) 141 | 142 | today = date.today() 143 | 144 | self.ds_new.attrs = { 145 | 'Description': 'This dataset contains reflectance data and flags for ocean color remote sensing.', 146 | 'Source': os.path.basename(self.path), 147 | "Author": "Shun Bi, shun.bi@outlook.com", 148 | "CreatedDate": today.strftime("%d/%m/%Y"), 149 | } 150 | 151 | self.ds_new['flag'] = (['rows', 'columns'], self.WQSF_REFLECTANCE_RECOM.data.astype(np.int32)) 152 | 153 | self.ds_new['type_idx'] = ( 154 | ['rows', 'columns'], 155 | self.owt.type_idx.astype(np.int32), 156 | {'Description': ( 157 | 'Index value for optical water types. ' 158 | '-1: No data; ' 159 | '0: OWT 1; ' 160 | '1: OWT 2; ' 161 | '2: OWT 3a; ' 162 | '3: OWT 3b; ' 163 | '4: OWT 4a; ' 164 | '5: OWT 4b; ' 165 | '6: OWT 5a; ' 166 | '7: OWT 5b; ' 167 | '8: OWT 6; ' 168 | '9: OWT 7; ' 169 | )} 170 | ) 171 | 172 | AVW_clipped = np.where((self.owt.AVW >= 400) & (self.owt.AVW <= 800), self.owt.AVW, np.nan) 173 | 174 | self.ds_new['AVW'] = ( 175 | ['rows', 'columns'], 176 | AVW_clipped.astype(np.float32), 177 | {'Description': 'Apparent Visible Wavelength 400-800 nm'} 178 | ) 179 | 180 | self.ds_new['Area'] = ( 181 | ['rows', 'columns'], 182 | self.owt.Area.astype(np.float32), 183 | {'Description': 'Trapezoidal area of Rrs at RGB bands'} 184 | ) 185 | 186 | NDI_clipped = np.where((self.owt.NDI >= -1) & (self.owt.NDI <= 1), self.owt.NDI, np.nan) 187 | self.ds_new['NDI'] = ( 188 | ['rows', 'columns'], 189 | NDI_clipped.astype(np.float32), 190 | {'Description': 'Normalized Difference Index of Rrs at G and B bands'} 191 | ) 192 | 193 | # save membership matrix 194 | self.u = self.owt.u 195 | self.utot = self.owt.utot 196 | 197 | self.ds_new['utot'] = ( 198 | ['rows', 'columns'], 199 | self.utot.astype(np.float32), 200 | {'Description': 'Total membership values of ten water types'} 201 | ) 202 | 203 | def save_result(self): 204 | encoding = { 205 | var: { 206 | 'zlib': True, 207 | 'complevel': 5, 208 | 'shuffle': True 209 | } 210 | for var in list(self.ds_new.data_vars) 211 | } 212 | 213 | self.ds_new.to_netcdf(os.path.join(self.save_path, f"{self.basename}_owt.nc"), encoding=encoding) 214 | 215 | 216 | if __name__ == "__main__": 217 | 218 | fn = '/Users/apple/Satellite_data/S3B_OL_2_WFR____20220703T075301_20220703T075601_20220704T171729_0179_067_363_2160_MAR_O_NT_003.SEN3.zip' 219 | eumetsat = eumetsat_olci_level2(filename=fn, sensor='OLCI_S3B') 220 | -------------------------------------------------------------------------------- /projects/zenodo/data_reshape_to_netcdf_olci.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | import xarray as xr 4 | 5 | # read single variables, ID 6 | d_sing = pd.read_csv("test_running/reshape_netcdf_temp_data/single_variable.csv") 7 | 8 | # read spectrum data, ID ~ wavelen 9 | d_ad = pd.read_csv("test_running/reshape_netcdf_temp_data/spec_ad.csv") 10 | d_agp = pd.read_csv("test_running/reshape_netcdf_temp_data/spec_agp.csv") 11 | d_aph = pd.read_csv("test_running/reshape_netcdf_temp_data/spec_aph.csv") 12 | d_aw = pd.read_csv("test_running/reshape_netcdf_temp_data/spec_aw.csv") 13 | d_bbp = pd.read_csv("test_running/reshape_netcdf_temp_data/spec_bbp.csv") 14 | d_bp = pd.read_csv("test_running/reshape_netcdf_temp_data/spec_bp.csv") 15 | d_bw = pd.read_csv("test_running/reshape_netcdf_temp_data/spec_bw.csv") 16 | d_Rrs = pd.read_csv("test_running/reshape_netcdf_temp_data/spec_Rrs.csv") 17 | 18 | # slice wavelen number 19 | wavelen = np.array( 20 | [400, 412, 442, 490, 510, 560, 620, 666, 674, 682, 708, 754, 21 | 760, 764, 768, 778, 866, 886] 22 | ) 23 | wavelen_sel = [str(i) for i in wavelen] 24 | 25 | # get dimension numbers 26 | ID = np.arange(1, d_sing.shape[0] + 1) 27 | 28 | # convert to numpy array 29 | ad = d_ad[wavelen_sel].to_numpy().round(8) 30 | agp = d_agp[wavelen_sel].to_numpy().round(8) 31 | aph = d_aph[wavelen_sel].to_numpy().round(8) 32 | aw = d_aw[wavelen_sel].to_numpy().round(8) 33 | bbp = d_bbp[wavelen_sel].to_numpy().round(8) 34 | bp = d_bp[wavelen_sel].to_numpy().round(8) 35 | bw = d_bw[wavelen_sel].to_numpy().round(8) 36 | Rrs = d_Rrs[wavelen_sel].to_numpy().round(8) 37 | 38 | 39 | # create netcdf file 40 | from datetime import date 41 | 42 | today = date.today() 43 | 44 | data = xr.Dataset( 45 | { 46 | "ad": (["ID", "wavelen"], ad), 47 | "agp": (["ID", "wavelen"], agp), 48 | "aph": (["ID", "wavelen"], aph), 49 | "aw": (["ID", "wavelen"], aw), 50 | "bbp": (["ID", "wavelen"], bbp), 51 | "bp": (["ID", "wavelen"], bp), 52 | "bw": (["ID", "wavelen"], bw), 53 | "Rrs": (["ID", "wavelen"], Rrs), 54 | "ID_ref": (["ID"], d_sing["ID"].to_numpy()), 55 | "source": (["ID"], d_sing["source"].to_numpy()), 56 | "type": (["ID"], d_sing["type"].to_numpy()), 57 | "Chl": (["ID"], d_sing["Chl"].to_numpy().round(3)), 58 | "ISM": (["ID"], d_sing["ISM"].to_numpy().round(3)), 59 | "ag440": (["ID"], d_sing["ag440"].to_numpy().round(8)), 60 | "A_d": (["ID"], d_sing["A_d"].to_numpy().round(8)), 61 | "G_d": (["ID"], d_sing["G_d"].to_numpy().round(8)), 62 | "Sal": (["ID"], d_sing["Sal"].to_numpy().round(2)), 63 | "Temp": (["ID"], d_sing["Temp"].to_numpy().round(2)), 64 | "a_frac": (["ID"], d_sing["a_frac"].to_numpy().round(3)), 65 | "cocco_frac": (["ID"], d_sing["cocco_frac"].to_numpy().round(3)), 66 | }, 67 | coords={"ID": ID, "wavelen": wavelen}, 68 | attrs={ 69 | "description": "Training data for the OWT classification framework by Bi and Hieronymi (2024) - Sentinel-3/OLCI band setup", 70 | "reference": ( 71 | "OWT framework: Bi, S., and Hieronymi, M. (2024). Holistic optical water type classification" 72 | "for ocean, coastal, and inland waters. Limnology & Oceanography, lno.12606. doi: 10.1002/lno.12606" 73 | "\n" 74 | "Component IOP model: Bi, S., Hieronymi, M., and Röttgers, R. (2023). Bio-geo-optical modelling of natural waters." 75 | "Front. Mar. Sci. 10, 1196352. doi: 10.3389/fmars.2023.1196352" 76 | "\n" 77 | "Pure water IOP model: Röttgers, R., Doerffer, R., McKee, D., and Schönfeld, W. (2016)." 78 | "The Water Optical Properties Processor (WOPP): Pure Water Spectral Absorption, Scattering and" 79 | "Real Part of Refractive Index Model. Technical Report No WOPP-ATBD/WRD6." 80 | "Available at: https://calvalportal.ceos.org/tools" 81 | "\n" 82 | "Rrs model: Lee, Z., Du, K., Voss, K. J., Zibordi, G., Lubac, B., Arnone, R., et al. (2011). " 83 | "An inherent-optical-property-centered approach to correct the angular effects in water-leaving radiance." 84 | "Appl. Opt. 50, 3155. doi: 10.1364/AO.50.003155" 85 | ), 86 | "Author": "Shun Bi, Martin Hieronymi, Rüdiger Röttgers", 87 | "Creator": "Shun Bi, Shun.Bi@heren.de", 88 | "CreatedDate": today.strftime("%d/%m/%Y"), 89 | }, 90 | ) 91 | 92 | # add variable and dimension description (according to Bi et al. 2023 IOP Paper Table 1) 93 | data.coords["ID"].attrs["description"] = "Sample ID number" 94 | data.coords["wavelen"].attrs["description"] = "Nominal Sentinel-3 OLCI waveband" 95 | data.coords["wavelen"].attrs["units"] = "nm" 96 | 97 | 98 | data["ad"].attrs["description"] = "Absorption coefficient of detritus" 99 | data["ad"].attrs["units"] = "1/m" 100 | 101 | data["agp"].attrs["description"] = "Total absorption coefficient without pure water" 102 | data["agp"].attrs["units"] = "1/m" 103 | 104 | data["aph"].attrs["description"] = "Absorption coefficient of phytoplankton" 105 | data["aph"].attrs["units"] = "1/m" 106 | 107 | data["aw"].attrs["description"] = "Absorption coefficient of pure water" 108 | data["aw"].attrs["units"] = "1/m" 109 | 110 | data["bbp"].attrs["description"] = "Backscattering coefficient of total particulate matter" 111 | data["bbp"].attrs["units"] = "1/m" 112 | 113 | data["bp"].attrs["description"] = "Scattering coefficient of total particulate matter" 114 | data["bp"].attrs["units"] = "1/m" 115 | 116 | data["bw"].attrs["description"] = "Scattering coefficient of pure water" 117 | data["bw"].attrs["units"] = "1/m" 118 | 119 | data["Rrs"].attrs["description"] = ( 120 | "Remote-sensing reflectance (above water surface). Calculated by Lee et al. (2011) model. " 121 | "Note that the inelastic scattering were not considered in the model." 122 | ) 123 | data["Rrs"].attrs["units"] = "1/sr" 124 | 125 | data["ID_ref"].attrs["description"] = "Reference identity from original data sources" 126 | data["source"].attrs["description"] = "Data source from Global or Specific. See Bi and Hieronymi (2024) for details." 127 | 128 | data["type"].attrs["description"] = "Optical water type for 1, 2, 3a, 3b, 4a, 4b, 5a, 5b, 6, and 7. See Bi and Hieronymi (2024) for detailed descriptions." 129 | 130 | data["Chl"].attrs["description"] = "Concentration of chlorophyll a" 131 | data["Chl"].attrs["units"] = "mg/m^3" 132 | 133 | data["ISM"].attrs["description"] = "Concentration of inorganic suspended matter" 134 | data["ISM"].attrs["units"] = "g/m^3" 135 | 136 | data["ag440"].attrs["description"] = "Colored dissolved organic matter absorption coefficient at 440 nm" 137 | data["ag440"].attrs["units"] = "1/m" 138 | 139 | data["A_d"].attrs["description"] = "Single-scattering albedo of detritus at 550 nm. Albedo_d(550) = 1 - 10^A_d" 140 | data["G_d"].attrs["description"] = "Power law exponent of attenuation of detritus. c_d(lam) = c_d(550) * (lam0/lam) ^ G_d" 141 | 142 | data["Sal"].attrs["description"] = "Water salinity" 143 | data["Sal"].attrs["units"] = "PSU" 144 | 145 | data["Temp"].attrs["description"] = "Water temperature" 146 | data["Temp"].attrs["units"] = "degC" 147 | 148 | data["a_frac"].attrs["description"] = "Fraction for diminished Coccolithophore absorption - minic its lifecycle from bloom to cocclith-detached." 149 | data["cocco_frac"].attrs["description"] = "Fraction of Coccolithophore group" 150 | 151 | # Define the encoding dictionary for compression 152 | complevel_sel = 5 153 | encoding = { 154 | "ad": {"zlib": True, "complevel": complevel_sel}, 155 | "agp": {"zlib": True, "complevel": complevel_sel}, 156 | "aph": {"zlib": True, "complevel": complevel_sel}, 157 | "aw": {"zlib": True, "complevel": complevel_sel}, 158 | "bbp": {"zlib": True, "complevel": complevel_sel}, 159 | "bp": {"zlib": True, "complevel": complevel_sel}, 160 | "bw": {"zlib": True, "complevel": complevel_sel}, 161 | "Rrs": {"zlib": True, "complevel": complevel_sel}, 162 | "Chl": {"zlib": True, "complevel": complevel_sel}, 163 | "ISM": {"zlib": True, "complevel": complevel_sel}, 164 | "ag440": {"zlib": True, "complevel": complevel_sel}, 165 | "A_d": {"zlib": True, "complevel": complevel_sel}, 166 | "G_d": {"zlib": True, "complevel": complevel_sel}, 167 | "Sal": {"zlib": True, "complevel": complevel_sel}, 168 | "Temp": {"zlib": True, "complevel": complevel_sel}, 169 | "a_frac": {"zlib": True, "complevel": complevel_sel}, 170 | "cocco_frac": {"zlib": True, "complevel": complevel_sel}, 171 | # No compression for object data types 172 | "ID_ref": {}, 173 | "source": {}, 174 | "type": {}, 175 | } 176 | 177 | # save data 178 | data.to_netcdf( 179 | "/Users/Bi/Documents/GitHub/pyOWT/test_running/owt_BH2024_training_data_olci.nc", 180 | encoding=encoding, 181 | ) 182 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # **pyOWT**: python library for Optical Water Type classification 2 | 3 | Note: this repo is translated from the R repo [`OWT`](https://github.com/bishun945/OWT) for the water type classification and has been maintained independently from its original version. 4 | 5 | # Install 6 | 7 | Clone the whole repo to the your path and install it: 8 | 9 | ```console 10 | pip install /path/to/pyowt 11 | ``` 12 | 13 | # License 14 | 15 | See the [LICENSE](/LICENSE) file. 16 | 17 | # How to use it 18 | 19 | Two classes, `OWT` and `OpticalVariables` are needed to perform the OWT classification. 20 | 21 | ```python 22 | from pyowt.OWT import OWT 23 | from pyowt.OpticalVariables import OpticalVariables 24 | from pyowt.PlotOWT import PlotOV, PlotSpec 25 | 26 | # first calculate three optical variables from Rrs 27 | # `sensor` should be specified for satellite data 28 | ov = OpticalVariables(Rrs=Rrs_data, band=Band_list, sensor=Sensor_str) 29 | 30 | # feed data into classification 31 | owt = OWT(AVW=ov.AVW, Area=ov.Area, NDI=ov.NDI) 32 | 33 | # show classification results 34 | print(owt.type_str) 35 | 36 | # show plots 37 | 38 | ## scatter plot 39 | PlotOV(owt) 40 | 41 | ## spectral distribution 42 | PlotSpec(owt, ov, norm=False) 43 | 44 | ## spectral distribution - normalized 45 | PlotSpec(owt, ov, norm=True) 46 | ``` 47 | 48 | Check the [example](/run_examples.py) file for more detailed demo runs: 49 | 50 | 1) Some hyperspectral Remote-sensing reflectance data simulated by Bi et al. (2023) 51 | 52 | 2) Satellite data with atmosphericly corrected by A4O (Hieronymi et al. 2023) 53 | 54 | # Short description of OWTs 55 | 56 | |OWT | Desciption | 57 | |----|------------| 58 | | 1 | Extremely clear and oligotrophic indigo-blue waters with high reflectance in the short visible wavelengths. | 59 | | 2 | Blue waters with similar biomass level as OWT 1 but with slightly higher detritus and CDOM content. | 60 | | 3a | Turquoise waters with slightly higher phytoplankton, detritus, and CDOM compared to the first two types. | 61 | | 3b | A special case of OWT 3a with similar detritus and CDOM distribution but with strong scattering and little absorbing particles like in the case of Coccolithophore blooms. This type usually appears brighter and exhibits a remarkable ~490 nm reflectance peak. | 62 | | 4a | Greenish water found in coastal and inland environments, with higher biomass compared to the previous water types. Reflectance in short wavelengths is usually depressed by the absorption of particles and CDOM. | 63 | | 4b | A special case of OWT 4a, sharing similar detritus and CDOM distribution, exhibiting phytoplankton blooms with higher scattering coefficients, e.g., Coccolithophore bloom. The color of this type shows a very bright green. | 64 | | 5a | Green eutrophic water, with significantly higher phytoplankton biomass, exhibiting a bimodal reflectance shape with typical peaks at ~560 and ~709 nm.| 65 | | 5b | Green hyper-eutrophic water, with even higher biomass than that of OWT 5a (over several orders of magnitude), displaying a reflectance plateau in the Near Infrared Region, NIR (vegetation-like spectrum). | 66 | | 6 | Bright brown water with high detritus concentrations, which has a high reflectance determined by scattering. | 67 | | 7 | Dark brown to black water with very high CDOM concentration, which has low reflectance in the entire visible range and is dominated by absorption. | 68 | 69 |
70 | 71 |
72 | 73 | owt_spec 74 | 75 |
76 | 77 | *Mean spectrum of simulated spectra for optical water types. Panel (A) displays the raw remote-sensing reflectance (unscaled), while Panel (B) shows the spectral normalized by trapezoidal-area. The positions of RGB bands are marked on the x-axis. Image Source: Bi and Hieronymi (2024)* 78 | 79 | # Supported band configurations 80 | 81 | The Spectral Response Functions (SRFs) for the satellite sensors used in `pyOWT` were obtained from the NASA [Ocean Color website](https://oceancolor.gsfc.nasa.gov/resources/docs/rsr_tables/). The naming convention for these SRFs follows the format `instrument-platform`, which is directly derived from corresponding NetCDF files associated with each sensor. 82 | 83 | - "HSI-EnMAP": from 420.9 to 797.0 nm 84 | - "OCI-PACE": from 400.2 to 799.4 nm 85 | - "HSI-PRISMA" from 407.0 to 796.1 nm 86 | - "OCTS-adeos": [412, 443, 490, 516, 565, 667, 862] 87 | - "modis-aqua": [412, 443, 469, 488, 531, 547, 555, 645, 667, 678, 748, 859] 88 | - "GOCI-coms": [412, 443, 490, 555, 660, 680, 745, 865] 89 | - "MERIS-envisat": [413, 443, 490, 510, 560, 620, 665, 681, 709, 754, 779, 865] 90 | - "viirs-jpss-1": [411, 445, 489, 556, 667, 746, 868] 91 | - "viirs-jpss-2": [411, 445, 488, 555, 671, 747, 868] 92 | - "OLI-landsat-8": [443, 482, 561, 655, 865] 93 | - "SeaWiFS-orbview-2": [412, 443, 490, 510, 555, 670, 865] 94 | - "olci-s3a": [400, 412, 443, 490, 510, 560, 620, 665, 674, 682, 709, 754, 779, 866] 95 | - "olci-s3b": [400, 412, 443, 490, 510, 560, 620, 665, 674, 681, 709, 754, 779, 866] 96 | - "HAWKEYE-seahawk1": [412, 447, 488, 510, 556, 670, 752, 867] 97 | - "msi-sentinel-2a": [443, 492, 560, 665, 704, 740, 783, 835, 865] 98 | - "msi-sentinel-2b": [442, 492, 559, 665, 704, 739, 780, 835, 864] 99 | - "viirs-suomi-npp": [410, 443, 486, 551, 671, 745, 862] 100 | - "modis-terra": [412, 443, 469, 488, 531, 547, 555, 645, 667, 678, 748, 859] 101 | 102 | CMEMS setups 103 | - "CMEMS_BAL_HROC": [443, 492, 560, 665, 704, 740, 783, 865] # likely msi-sentinel-2a 104 | - "CMEMS_BAL_NRT": [400, 412, 443, 490, 510, 560, 620, 665, 674, 682, 709, 779, 866] # likely olci-s3a 105 | - "CMEMS_MED_MYINT": [400, 412, 443, 490, 510, 560, 620, 665, 674, 682, 709, 779, 866] # likely olci-s3a 106 | 107 | AERONET-OC setups 108 | - "AERONET_OC_1": [400, 412, 443, 490, 510, 560, 620, 665, 779, 866] # likely olci-s3a 109 | - "AERONET_OC_2": [412, 443, 490, 532, 551, 667, 870] # likely modis-aqua 110 | 111 | LakeCCI setups 112 | - "LakeCCI-MERIS": [413, 443, 490, 510, 560, 620, 665, 681, 709, 754, 779, 885] # likely MERIS-envisat 113 | 114 | # Bug rerport 115 | 116 | When you find any issues or bugs while running the module, please [open an issue](https://github.com/bishun945/pyOWT/issues) or directly contact [Shun Bi](Shun.Bi@outlook.com) with a reproducible script with data. 117 | 118 | # Projects 119 | 120 | The `projects` directory contains tasks that rely on the `pyowt` package but are maintained independently of `pyowt`. 121 | 122 | ## AquaINFRA 123 | 124 | This part is supported by [Merret Buurman](merret.buurman@igb-berlin.de) 125 | 126 | AquaINFRA related scripts can be found in `projects/AquaINFRA` 127 | 128 | When running this as OGC-compliant web service in an installation of pygeoapi, please create a json config file `config.json` with the below contents (or add it to the general AquaINFRA config file), and define an environment variable called `PYOWT_CONFIG_FILE` that contains the path to it. 129 | 130 | ``` 131 | { 132 | "download_dir": "/var/www/nginx/download/", 133 | "download_url": "https://someserver/download/", 134 | "pyowt": { 135 | "example_input_data_dir": ".../pygeoapi/process/pyOWT/data/", # this is where the process will try to find existing example inputs 136 | "input_data_dir": "/.../inputs/", # this is where the process will try to store downloaded inputs 137 | "path_sensor_band_library": ".../pygeoapi/process/pyOWT/data/sensor_band_library.yaml" 138 | } 139 | } 140 | ``` 141 | 142 | ## zenodo 143 | 144 | This part is supported by Shun Bi and Martin Hieronymi. 145 | 146 | The `projects/zenodo` directory contains R and python scripts used to generate NetCDF files for the training data set in Bi and Hieronymi (2024). These data sets, available in both hyperspectral and Sentinel-3 OLCI band configurations, are published on [zenodo](https://zenodo.org/records/12803329). Note that some files are too large to be uploaded to GitHub, but they are available upon request. 147 | 148 | # Contributors 149 | 150 | - Dr. Shun Bi - Project maintainer, developer 151 | - Dr. Martin Hieronymi (Hereon) - Reviewer, Support 152 | - Merret Buurman (IGB-Berlin) - Developer for AquaINFRA 153 | - Dr. Markus Konkol (52north) - Developer for AquaINFRA 154 | 155 | # References 156 | 157 | - Bi and Hieronymi (2024). Holistic optical water type classification for ocean, coastal, and inland waters. Limnol Oceanogr. https://doi.org/10.1002/lno.12606 158 | 159 | - Bi et al. (2023). Bio-geo-optical modelling of natural waters. Front. Mar. Sci. 10, 1196352. https://doi.org/10.3389/fmars.2023.1196352 160 | 161 | - Hieronymi et al. (2023). Ocean color atmospheric correction methods in view of usability for different optical water types. Front. Mar. Sci. 10, 1129876. https://doi.org/10.3389/fmars.2023.1129876 162 | 163 | - Bi et al. (2024). Supplementary dataset to the publication "Bi, S., and Hieronymi, M. (2024). Holistic optical water type classification for ocean, coastal, and inland waters. Limnology & Oceanography" [Data set]. Zenodo. https://doi.org/10.5281/zenodo.12803329 -------------------------------------------------------------------------------- /projects/zenodo/data_reshape_to_netcdf_hyper.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | import xarray as xr 4 | 5 | # read single variables, ID 6 | d_sing = pd.read_csv("test_running/reshape_netcdf_temp_data/single_variable.csv") 7 | 8 | # read spectrum data, ID ~ wavelen 9 | d_ad = pd.read_csv("test_running/reshape_netcdf_temp_data/spec_ad.csv") 10 | d_agp = pd.read_csv("test_running/reshape_netcdf_temp_data/spec_agp.csv") 11 | d_aph = pd.read_csv("test_running/reshape_netcdf_temp_data/spec_aph.csv") 12 | d_aw = pd.read_csv("test_running/reshape_netcdf_temp_data/spec_aw.csv") 13 | d_bbp = pd.read_csv("test_running/reshape_netcdf_temp_data/spec_bbp.csv") 14 | d_bp = pd.read_csv("test_running/reshape_netcdf_temp_data/spec_bp.csv") 15 | d_bw = pd.read_csv("test_running/reshape_netcdf_temp_data/spec_bw.csv") 16 | d_Rrs = pd.read_csv("test_running/reshape_netcdf_temp_data/spec_Rrs.csv") 17 | 18 | # slice wavelen number 19 | wavelen_sel = [str(i) for i in range(400, 901, 2)] 20 | 21 | 22 | # get dimension numbers 23 | wavelen = np.arange(400, 901, step=2) 24 | ID = np.arange(1, d_sing.shape[0] + 1) 25 | 26 | 27 | # convert to numpy array 28 | ad = d_ad[wavelen_sel].to_numpy().round(8) 29 | agp = d_agp[wavelen_sel].to_numpy().round(8) 30 | aph = d_aph[wavelen_sel].to_numpy().round(8) 31 | aw = d_aw[wavelen_sel].to_numpy().round(8) 32 | bbp = d_bbp[wavelen_sel].to_numpy().round(8) 33 | bp = d_bp[wavelen_sel].to_numpy().round(8) 34 | bw = d_bw[wavelen_sel].to_numpy().round(8) 35 | Rrs = d_Rrs[wavelen_sel].to_numpy().round(8) 36 | 37 | # create netcdf file 38 | from datetime import date 39 | 40 | today = date.today() 41 | 42 | data = xr.Dataset( 43 | { 44 | "ad": (["ID", "wavelen"], ad), 45 | "agp": (["ID", "wavelen"], agp), 46 | "aph": (["ID", "wavelen"], aph), 47 | "aw": (["ID", "wavelen"], aw), 48 | "bbp": (["ID", "wavelen"], bbp), 49 | "bp": (["ID", "wavelen"], bp), 50 | "bw": (["ID", "wavelen"], bw), 51 | "Rrs": (["ID", "wavelen"], Rrs), 52 | "ID_ref": (["ID"], d_sing["ID"].to_numpy()), 53 | "source": (["ID"], d_sing["source"].to_numpy()), 54 | "type": (["ID"], d_sing["type"].to_numpy()), 55 | "Chl": (["ID"], d_sing["Chl"].to_numpy().round(3)), 56 | "ISM": (["ID"], d_sing["ISM"].to_numpy().round(3)), 57 | "ag440": (["ID"], d_sing["ag440"].to_numpy().round(8)), 58 | "A_d": (["ID"], d_sing["A_d"].to_numpy().round(8)), 59 | "G_d": (["ID"], d_sing["G_d"].to_numpy().round(8)), 60 | "Sal": (["ID"], d_sing["Sal"].to_numpy().round(2)), 61 | "Temp": (["ID"], d_sing["Temp"].to_numpy().round(2)), 62 | "a_frac": (["ID"], d_sing["a_frac"].to_numpy().round(3)), 63 | "cocco_frac": (["ID"], d_sing["cocco_frac"].to_numpy().round(3)), 64 | }, 65 | coords={"ID": ID, "wavelen": wavelen}, 66 | attrs={ 67 | "description": "Training data for the OWT classification framework by Bi and Hieronymi (2024) - Hyperspectral version 400 to 900 nm (step 2 nm)", 68 | "reference": ( 69 | "OWT framework: Bi, S., and Hieronymi, M. (2024). Holistic optical water type classification" 70 | " for ocean, coastal, and inland waters. Limnology & Oceanography, lno.12606. doi: 10.1002/lno.12606" 71 | "\n" 72 | "Component IOP model: Bi, S., Hieronymi, M., and Röttgers, R. (2023). Bio-geo-optical modelling of natural waters." 73 | " Front. Mar. Sci. 10, 1196352. doi: 10.3389/fmars.2023.1196352" 74 | "\n" 75 | "Pure water IOP model: Röttgers, R., Doerffer, R., McKee, D., and Schönfeld, W. (2016)." 76 | " The Water Optical Properties Processor (WOPP): Pure Water Spectral Absorption, Scattering and" 77 | " Real Part of Refractive Index Model. Technical Report No WOPP-ATBD/WRD6." 78 | " Available at: https://calvalportal.ceos.org/tools" 79 | "\n" 80 | "Rrs model: Lee, Z., Du, K., Voss, K. J., Zibordi, G., Lubac, B., Arnone, R., et al. (2011)." 81 | " An inherent-optical-property-centered approach to correct the angular effects in water-leaving radiance." 82 | " Appl. Opt. 50, 3155. doi: 10.1364/AO.50.003155" 83 | ), 84 | "Author": "Shun Bi, Martin Hieronymi, Rüdiger Röttgers", 85 | "Creator": "Shun Bi, Shun.Bi@heren.de", 86 | "CreatedDate": today.strftime("%d/%m/%Y"), 87 | }, 88 | ) 89 | 90 | # add variable and dimension description (according to Bi et al. 2023 IOP Paper Table 1) 91 | data.coords["ID"].attrs["description"] = "Sample ID number" 92 | data.coords["wavelen"].attrs["description"] = "Wavelength of light" 93 | data.coords["wavelen"].attrs["units"] = "nm" 94 | 95 | 96 | data["ad"].attrs["description"] = "Absorption coefficient of detritus" 97 | data["ad"].attrs["units"] = "1/m" 98 | 99 | data["agp"].attrs["description"] = "Total absorption coefficient without pure water" 100 | data["agp"].attrs["units"] = "1/m" 101 | 102 | data["aph"].attrs["description"] = "Absorption coefficient of phytoplankton" 103 | data["aph"].attrs["units"] = "1/m" 104 | 105 | data["aw"].attrs["description"] = "Absorption coefficient of pure water" 106 | data["aw"].attrs["units"] = "1/m" 107 | 108 | data["bbp"].attrs["description"] = "Backscattering coefficient of total particulate matter" 109 | data["bbp"].attrs["units"] = "1/m" 110 | 111 | data["bp"].attrs["description"] = "Scattering coefficient of total particulate matter" 112 | data["bp"].attrs["units"] = "1/m" 113 | 114 | data["bw"].attrs["description"] = "Scattering coefficient of pure water" 115 | data["bw"].attrs["units"] = "1/m" 116 | 117 | data["Rrs"].attrs["description"] = ( 118 | "Remote-sensing reflectance (above water surface). Calculated by Lee et al. (2011) model. " 119 | "Note that the inelastic scattering were not considered in the model." 120 | ) 121 | data["Rrs"].attrs["units"] = "1/sr" 122 | 123 | data["ID_ref"].attrs["description"] = "Reference identity from original data sources" 124 | data["source"].attrs["description"] = "Data source from Global or Specific. See Bi and Hieronymi (2024) for details." 125 | 126 | data["type"].attrs["description"] = "Optical water type for 1, 2, 3a, 3b, 4a, 4b, 5a, 5b, 6, and 7. See Bi and Hieronymi (2024) for detailed descriptions." 127 | 128 | data["Chl"].attrs["description"] = "Concentration of chlorophyll a" 129 | data["Chl"].attrs["units"] = "mg/m^3" 130 | 131 | data["ISM"].attrs["description"] = "Concentration of inorganic suspended matter" 132 | data["ISM"].attrs["units"] = "g/m^3" 133 | 134 | data["ag440"].attrs["description"] = "Colored dissolved organic matter absorption coefficient at 440 nm" 135 | data["ag440"].attrs["units"] = "1/m" 136 | 137 | data["A_d"].attrs["description"] = "Single-scattering albedo of detritus at 550 nm. Albedo_d(550) = 1 - 10^A_d" 138 | data["G_d"].attrs["description"] = "Power law exponent of attenuation of detritus. c_d(lam) = c_d(550) * (lam0/lam) ^ G_d" 139 | 140 | data["Sal"].attrs["description"] = "Water salinity" 141 | data["Sal"].attrs["units"] = "PSU" 142 | 143 | data["Temp"].attrs["description"] = "Water temperature" 144 | data["Temp"].attrs["units"] = "degC" 145 | 146 | data["a_frac"].attrs["description"] = "Fraction for diminished Coccolithophore absorption - minic its lifecycle from bloom to cocclith-detached." 147 | data["cocco_frac"].attrs["description"] = "Fraction of Coccolithophore group" 148 | 149 | # Define the encoding dictionary for compression 150 | complevel_sel = 5 151 | encoding = { 152 | "ad": {"zlib": True, "complevel": complevel_sel}, 153 | "agp": {"zlib": True, "complevel": complevel_sel}, 154 | "aph": {"zlib": True, "complevel": complevel_sel}, 155 | "aw": {"zlib": True, "complevel": complevel_sel}, 156 | "bbp": {"zlib": True, "complevel": complevel_sel}, 157 | "bp": {"zlib": True, "complevel": complevel_sel}, 158 | "bw": {"zlib": True, "complevel": complevel_sel}, 159 | "Rrs": {"zlib": True, "complevel": complevel_sel}, 160 | "Chl": {"zlib": True, "complevel": complevel_sel}, 161 | "ISM": {"zlib": True, "complevel": complevel_sel}, 162 | "ag440": {"zlib": True, "complevel": complevel_sel}, 163 | "A_d": {"zlib": True, "complevel": complevel_sel}, 164 | "G_d": {"zlib": True, "complevel": complevel_sel}, 165 | "Sal": {"zlib": True, "complevel": complevel_sel}, 166 | "Temp": {"zlib": True, "complevel": complevel_sel}, 167 | "a_frac": {"zlib": True, "complevel": complevel_sel}, 168 | "cocco_frac": {"zlib": True, "complevel": complevel_sel}, 169 | # No compression for object data types 170 | "ID_ref": {}, 171 | "source": {}, 172 | "type": {}, 173 | } 174 | 175 | # save data 176 | data.to_netcdf( 177 | "/Users/Bi/Documents/GitHub/pyOWT/test_running/owt_BH2024_training_data_hyper.nc", 178 | encoding=encoding, 179 | ) 180 | 181 | 182 | # # plot test 183 | # # test plotting for check 184 | # import matplotlib.pyplot as plt 185 | # import random 186 | # import seaborn as sns 187 | # 188 | # ind = random.randint(0, len(ID)-1) 189 | # plt.figure(figsize=(10, 6)) 190 | # plt.plot(wavelen, ad[ind,:], label='ad') 191 | # plt.plot(wavelen, agp[ind,:], label='agp') 192 | # plt.plot(wavelen, aph[ind,:], label='aph') 193 | # # plt.plot(wavelen, aw[ind,:], label='aw') 194 | # plt.legend() 195 | # plt.show() 196 | # 197 | # plt.figure(figsize=(10, 6)) 198 | # sns.boxplot(x='type', y='ag440', data=d_sing) 199 | # plt.yscale('log') 200 | # plt.ylabel('Value (log scale)') 201 | # plt.title('Boxplot with log10 scale on y-axis') 202 | # plt.show() 203 | -------------------------------------------------------------------------------- /pyowt/OpticalVariables.py: -------------------------------------------------------------------------------- 1 | from pandas import read_csv 2 | import numpy as np 3 | import os 4 | import yaml 5 | import json 6 | from scipy.interpolate import interp1d 7 | 8 | class OpticalVariables(): 9 | 10 | def __init__(self, Rrs, band, sensor=None, version='v01'): 11 | 12 | if not isinstance(Rrs, np.ndarray): 13 | 14 | raise TypeError("Input 'Rrs' should be np.ndarray type.") 15 | 16 | # manipulate dimension of input Rrs as we always assume it is 3d nparray (raster-like) 17 | # the wavelength should be on the shape[2] dim 18 | 19 | if np.ndim(Rrs) == 1: 20 | # here assume a signle spectrum 21 | self.Rrs = Rrs.reshape(1, 1, Rrs.shape[0]) 22 | elif np.ndim(Rrs) == 2: 23 | # here assume shape[0] is sample and shape[1] is wavelength 24 | self.Rrs = Rrs.reshape(Rrs.shape[0], 1, Rrs.shape[1]) 25 | elif np.ndim(Rrs) == 3: 26 | # here assume shape[2] is wavelength 27 | self.Rrs = Rrs 28 | elif np.ndim(Rrs) == 4: 29 | # here assume (wavelen, time, lat, lon) 30 | self.original_shape = Rrs.shape[1:] # shape of (time, lat, lon) 31 | Rrs_ = Rrs.transpose(1, 2, 3, 0).reshape(-1, Rrs.shape[0]) 32 | self.Rrs = Rrs_.reshape(Rrs_.shape[0], 1, Rrs_.shape[1]) 33 | else: 34 | raise ValueError("Input 'Rrs' should only have 1-4 dims!") 35 | 36 | self.band = np.array(band) 37 | 38 | self.sensor = sensor 39 | self.version = version 40 | 41 | self.AVW = None 42 | self.Area = None 43 | self.NDI = None 44 | 45 | # TODO: check the input band fits the selected sensor range 46 | base_dir = os.path.dirname(os.path.abspath(__file__)) 47 | data_dir = os.path.join(base_dir, 'data') 48 | path_sensor_band_library = os.path.join(data_dir, 'sensor_band_library.yaml') 49 | 50 | if not os.path.isfile(path_sensor_band_library): 51 | # we're obviously in a different env with a different cwd, so read path from config 52 | # config may be in cwd, or in a file referenced by env var, to be consistent with 53 | # other AquaINFRA processes. 54 | config_file_path = os.environ.get('PYOWT_CONFIG_FILE', "./config.json") 55 | with open(config_file_path, 'r') as config_file: 56 | config = json.load(config_file) 57 | path_sensor_band_library = config['pyowt']['path_sensor_band_library'] 58 | 59 | 60 | with open(path_sensor_band_library, 'r') as file: 61 | sensor_lib = yaml.load(file, Loader=yaml.FullLoader) 62 | 63 | 64 | self.sensor_AVW_bands_library = sensor_lib['lib_800']['sensor_AVW_bands_library'] 65 | self.sensor_RGB_bands_library = sensor_lib['lib_800']['sensor_RGB_bands_library'] 66 | # dont_TODO: if AVW ends by 700 nm, this list has to be modified 67 | self.sensor_RGB_min_max = sensor_lib['lib_800']['sensor_RGB_min_max'] 68 | self.AVW_regression_coef = sensor_lib['lib_800']['AVW_regression_coef'] 69 | 70 | self.available_sensors = ', '.join(self.sensor_AVW_bands_library.keys()) 71 | 72 | if self.sensor is None: 73 | 74 | # the input Rrs are assumed to be hyperspectral 75 | self.spectral_attr = "hyper" 76 | ref_sensor_RGB_bands = [443, 560, 665] 77 | self.sensor_RGB_bands = [self.band[np.argmin(abs(self.band - v))] for v in ref_sensor_RGB_bands] 78 | self.sensor_band_min = 400 79 | self.sensor_band_max = 800 80 | self.AVW_conver_coef = [0, 1, 0, 0, 0, 0] 81 | 82 | if self.band.min() > self.sensor_band_min or self.band.max() < self.sensor_band_max: 83 | raise ValueError( 84 | "Wavelength range should be 400 to 800 nm for hyperspectral Rrs. " 85 | f"The input is {self.band.min()} - {self.band.max()}" 86 | ) 87 | 88 | else: 89 | 90 | if self.sensor not in self.sensor_AVW_bands_library.keys(): 91 | 92 | raise ValueError(f"The input `sensor` couldn't be found in the library: {self.available_sensors}") 93 | 94 | # if `sensor` specifized, trigger `conver_AVW_multi_to_hyper` 95 | self.spectral_attr = "multi" 96 | 97 | # define RGB bands, by B, R, G order 98 | ref_sensor_RGB_bands = self.sensor_RGB_bands_library[self.sensor] 99 | self.sensor_RGB_bands = [self.band[np.argmin(abs(self.band - v))] for v in ref_sensor_RGB_bands] 100 | 101 | # define min max ranges 102 | self.sensor_band_min = self.sensor_RGB_min_max[self.sensor]["min"] 103 | self.sensor_band_max = self.sensor_RGB_min_max[self.sensor]["max"] 104 | 105 | # read coefficients to convert AVW_multi to AVW_hyper 106 | proj_root = os.path.dirname(os.path.abspath(__file__)) 107 | fn = os.path.join(proj_root, self.AVW_regression_coef) 108 | d = read_csv(fn) 109 | self.AVW_convert_coef = d[d["sensor"] == sensor][["0", "1", "2", "3", "4", "5"]].values.tolist()[0] 110 | 111 | 112 | # run calculation 113 | self.calculate_AVW() 114 | self.calculate_Area() 115 | self.calculate_NDI() 116 | 117 | class ArrayWithAttributes: 118 | '''This subclass add attributes to np.array 119 | 120 | Outputs of `OpticalVariables` (AVW, Area, NDI) can have attributes indicating 121 | some key features during the calculation. For example, we should know which 122 | bands were used to calcualte AVW, Area, and NDI. 123 | 124 | Examples: 125 | 126 | A = ArrayWithAttributes(np.array([1, 2, 3]), author='Shun') 127 | print(A) # Output: [1 2 3] 128 | print(A.author) # Output: Shun 129 | ''' 130 | def __init__(self, array, **attributes): 131 | self.array = np.asarray(array) 132 | self.__dict__.update(attributes) 133 | 134 | def __getitem__(self, item): 135 | return self.array[item] 136 | 137 | def __setitem__(self, key, value): 138 | self.array[key] = value 139 | 140 | def __repr__(self): 141 | return repr(self.array) 142 | 143 | def __str__(self): 144 | return str(self.array) 145 | 146 | def __getattr__(self, name): 147 | return getattr(self.array, name) 148 | 149 | def __array__(self): 150 | return self.array 151 | 152 | 153 | def convert_AVW_multi_to_hyper(self): 154 | self.AVW_hyper = np.zeros(self.AVW_multi.shape) 155 | for i in range(len(self.AVW_convert_coef)): 156 | self.AVW_hyper += self.AVW_convert_coef[i] * (self.AVW_multi**i) 157 | 158 | 159 | def calculate_AVW(self): 160 | # idx_for_AVW = (self.band >= self.sensor_band_min) & (self.band <= self.sensor_band_max) 161 | # bands_for_AVW = self.band[idx_for_AVW] 162 | if self.spectral_attr == "hyper": 163 | bands_for_AVW = self.band 164 | Rrs_for_AVW = self.Rrs 165 | 166 | # check if is 1 nm interval from 400 to 800 nm 167 | target_bands = np.arange(self.sensor_band_min, self.sensor_band_max + 1, 1) 168 | is_1nm_interval = np.all(np.diff(bands_for_AVW) == 1) 169 | 170 | if not is_1nm_interval: 171 | X, Y, _ = Rrs_for_AVW.shape 172 | Rrs_new = np.zeros((X, Y, len(target_bands))) 173 | interp_func = interp1d(bands_for_AVW, Rrs_for_AVW, kind='linear', axis=-1, 174 | bounds_error=False, fill_value='extrapolate') 175 | Rrs_new = interp_func(target_bands) 176 | 177 | bands_for_AVW = target_bands 178 | Rrs_for_AVW = Rrs_new 179 | 180 | else: 181 | bands_for_AVW = [self.band[np.argmin(abs(self.band - v))].item() for v in self.sensor_AVW_bands_library[self.sensor]] 182 | bands_for_AVW = np.array(bands_for_AVW) 183 | idx_for_AVW = [np.where(self.band == band)[0][0].item() for band in bands_for_AVW if band in self.band] 184 | Rrs_for_AVW = self.Rrs[:, :, idx_for_AVW] 185 | 186 | self.AVW_init = np.sum(Rrs_for_AVW, axis=-1) / np.sum(Rrs_for_AVW / bands_for_AVW[None, None, :], axis=-1) 187 | 188 | if self.spectral_attr == "hyper": 189 | self.AVW_hyper = self.AVW_init 190 | else: 191 | self.AVW_multi = self.AVW_init 192 | self.convert_AVW_multi_to_hyper() 193 | 194 | self.AVW = self.AVW_hyper 195 | 196 | 197 | def calculate_Area(self): 198 | bands_for_Area = np.array(self.sensor_RGB_bands) 199 | Rrs_for_Area = self.Rrs[:, :, np.where(np.isin(self.band, bands_for_Area))[0]] 200 | self.Area = np.trapz(x=bands_for_Area, y=Rrs_for_Area, axis=-1) 201 | 202 | 203 | def calculate_NDI(self): 204 | r_blue = self.Rrs[:, :, np.where(np.isin(self.band, self.sensor_RGB_bands[0]))[0]] 205 | r_green = self.Rrs[:, :, np.where(np.isin(self.band, self.sensor_RGB_bands[1]))[0]] 206 | r_red = self.Rrs[:, :, np.where(np.isin(self.band, self.sensor_RGB_bands[2]))[0]] 207 | 208 | if self.version == 'v99': 209 | r_1 = np.maximum(r_blue, r_green) 210 | else: 211 | r_1 = r_green 212 | 213 | NDI = (r_1 - r_red) / (r_1 + r_red) 214 | self.NDI = NDI[:, :, 0] 215 | 216 | 217 | def run(self): 218 | # TODO: deprecate this func in the future 219 | import warnings 220 | warnings.warn( 221 | "No need to calculate via 'ov.run()'. " 222 | "The optical variables will be calculated directly once you create this instance. " 223 | "This function is deprecated and will be removed in the future versions.", 224 | DeprecationWarning, 225 | stacklevel=2 226 | ) 227 | 228 | self.calculate_AVW() 229 | self.calculate_Area() 230 | self.calculate_NDI() 231 | 232 | def shape_reverse(self, arr): 233 | if hasattr(self, 'original_shape'): 234 | arr = arr.reshape(self.original_shape) 235 | return arr 236 | else: 237 | raise ValueError("Not 4d arr input and no original_shape") 238 | 239 | 240 | 241 | if __name__ == "__main__": 242 | 243 | # ov = OpticalVariables(Rrs=1, band=1, sensor="OLCI_S3B") 244 | # print(ov.sensor_RGB_bands) 245 | band = np.arange(400, 801, step = 2) 246 | # Rrs = np.random.normal(loc = 0, scale = 1, size = len(band)) 247 | Rrs = np.full(len(band), 1) 248 | ov = OpticalVariables(Rrs=Rrs, band=band) 249 | print(ov.AVW) 250 | 251 | -------------------------------------------------------------------------------- /pyowt/OWT.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import xarray as xr 3 | from types import SimpleNamespace 4 | from scipy.stats import chi2 5 | 6 | import os 7 | 8 | class OWT(): 9 | 10 | def __init__(self, AVW=None, Area=None, NDI=None, version='v01', thres_u=0.0001): 11 | """Initialize three optical variables for spectral classification 12 | 13 | Args: 14 | AVW (np.array, ndim <= 2): Apparent Visible Wavelength 400-800 nm 15 | Area (np.array, ndim <= 2): Trapezoidal area of Rrs at RGB bands 16 | NDI (np.array, ndim <= 2): Normalized Difference Index of Rrs at G and B bands 17 | version (str): Version of the classification centroids. Default as 'v01'. 18 | Two version are now included: 19 | - v01: original from Bi and Hieronymi (2024) paper 20 | - v02: revised version of v01 which covariance matrix were shrinked 21 | thre_u (numeric): the threshold of membership (u) to mask out non-classifiable inputs. 22 | Default as 0.0001 from Hieronymi et al. (2023) Table 3. 23 | 24 | Return: 25 | u (np.array, ndim = 3): the first and second dims are from AVW or np.atleast_2d(AVW). 26 | Its last dim is water type (N = 10) 27 | """ 28 | 29 | self.AVW = AVW 30 | self.Area = Area 31 | self.NDI = NDI 32 | 33 | if any(np.ndim(arr) > 2 for arr in [self.AVW, self.Area, self.NDI]): 34 | 35 | raise ValueError("AVW, Area, and NDI with more than two dims are not supported!") 36 | 37 | self.AVW = np.atleast_2d(self.AVW) 38 | self.Area = np.atleast_2d(self.Area) 39 | self.NDI = np.atleast_2d(self.NDI) 40 | 41 | if not (self.AVW.shape == self.Area.shape == self.NDI.shape): 42 | 43 | raise ValueError("The shapes of AVW, Area, and NDI must be the same!") 44 | 45 | # set the threshold of membership values for classifiable results 46 | self.thres_u = thres_u 47 | 48 | # load pre-trained centroids 49 | self.version = version 50 | 51 | # warning of version 52 | if self.version != 'v01': 53 | import warnings 54 | warnings.warn( 55 | f"Version '{self.version}' is not the original set from Bi and Hieronymi (2024). " 56 | "These centroids are under testing. Please use with caution.", 57 | UserWarning, 58 | stacklevel=2 59 | ) 60 | 61 | self.classInfo = self.load_centroids_version(version=self.version) 62 | # classInfo = self.load_centroids() 63 | # self.mean_OWT = classInfo["mean_OWT"] 64 | # self.covm_OWT = classInfo["covm_OWT"] 65 | # self.lamBC = classInfo["lamBC"] 66 | # self.typeName = classInfo["typeName"] 67 | # self.typeNumb = classInfo["typeNumb"] 68 | # self.typeColor = classInfo["typeColHex"] 69 | 70 | dict_idx_name = {i: self.classInfo.typeName[i] for i in range(self.classInfo.typeNumb)} # for mapping 71 | dict_idx_name[-1] = "NaN" # once not classifiable 72 | self.dict_idx_name = {key: dict_idx_name[key] for key in sorted(dict_idx_name)} 73 | 74 | dict_idx_color = {i: self.classInfo.typeColHex[i] for i in range(self.classInfo.typeNumb)} # for mapping 75 | dict_idx_color[-1] = "#808080" # once not classifiable 76 | self.dict_idx_color = {key: dict_idx_color[key] for key in sorted(dict_idx_color)} 77 | 78 | 79 | # placeholder 80 | self.u = np.full(self.AVW.shape + (self.classInfo.typeNumb,), None) 81 | self.type_idx = np.full(self.AVW.shape, None) 82 | self.type_str = np.full(self.AVW.shape, None) 83 | 84 | # run classification 85 | self.run_classification() 86 | 87 | # Use Hieronymi et al. (2023) Table 3 to mask out non-classifiable inputs 88 | # if utot < thres_u, classifiability = 0; else = 1 89 | self.classifiability = np.full(self.AVW.shape, 1) 90 | self.classifiability[self.utot < self.thres_u] = 0 91 | self.classifiability[self.type_idx == -1] = 0 92 | 93 | 94 | def update_type_idx(self): 95 | """Update the type (index of `typeName`) based on the current membership value. 96 | The function will be updated once `run_classification` is performed. 97 | OWT will be asigned to -1 if all memberships are zero or below the threshold `thres_u`. 98 | """ 99 | if self.u is not None: 100 | idx_max = np.argmax(self.u, axis=-1) 101 | self.type_idx = idx_max 102 | mask_all_zero = np.all(self.u <= 0, axis=-1) 103 | self.type_idx[mask_all_zero] = -1 104 | mask_all_nan = np.all(np.isnan(self.u), axis=-1) 105 | self.type_idx[mask_all_nan] = -1 106 | mask_blt_thres = np.all(self.utot <= self.thres_u, axis=-1) 107 | self.type_idx[mask_blt_thres] = -1 108 | else: 109 | raise ValueError("Membership values have not been calculated! Run the classification first") 110 | 111 | 112 | def update_type_str(self): 113 | """Update the type (name in `typeName`) based on `self.type_idx` 114 | """ 115 | if self.u is not None: 116 | vectorized_map = np.vectorize(lambda x: self.dict_idx_name.get(x, '')) 117 | self.type_str = vectorized_map(self.type_idx) 118 | else: 119 | raise ValueError("Membership values have not been calculated! Run the classification first") 120 | 121 | 122 | def run_classification(self): 123 | """Run the classification procedure 124 | 125 | Returns: 126 | np.array, ndim <= 3: membership values for pre-defined 10 types 127 | """ 128 | 129 | import warnings 130 | warnings.warn( 131 | "No need to calculate via 'owt.run_classification()'. " 132 | "The classification will be performed directly once you create this instance. " 133 | "This function is deprecated and will be removed in the future versions.", 134 | DeprecationWarning, 135 | stacklevel=2 136 | ) 137 | 138 | self.ABC = self.trans_boxcox(self.Area, self.classInfo.lamBC) 139 | 140 | x = np.array([self.AVW, self.ABC, self.NDI]).transpose(1, 2, 0) 141 | d = np.zeros((x.shape[0], x.shape[1], self.classInfo.typeNumb)) 142 | 143 | for i in range(self.classInfo.typeNumb): 144 | 145 | y = self.classInfo.mean_OWT[i, :][None, None, :] 146 | covm = self.classInfo.covm_OWT[:, :, i] 147 | covm_inv = np.linalg.inv(covm) 148 | diff = x - y 149 | d[:, :, i] = np.einsum("...i,ij,...j->...", diff, covm_inv, diff) 150 | 151 | u = np.round(1 - chi2.cdf(d, df=x.shape[2]), 6) 152 | 153 | self.u = u 154 | self.utot = np.sum(u, axis=-1) 155 | 156 | self.update_type_idx() 157 | self.update_type_str() 158 | 159 | def load_centroids_version(self, version): 160 | """ 161 | load the centroids for classification 162 | For the "data/OWT_centroids.nc" file, three variables included: 163 | - [mean] mean of each optical water type (OWT) at three dims 164 | - [covm] covariance matrix (3x3) of each OWT 165 | - [lamBC] lambda coeffcient for the Box-Cox transformation 166 | Dimensions: [AVW, Area, NDI] 167 | Note: Area in the nc lib is after Box-Cox transformation 168 | """ 169 | proj_root = os.path.dirname(os.path.abspath(__file__)) 170 | fn = os.path.join(proj_root, f"data/{version}/OWT_centroids.nc") 171 | ds = xr.open_dataset(fn) 172 | mean_OWT = ds['mean'].values 173 | covm_OWT = ds['covm'].values 174 | lamBC = ds.attrs['lamBC'] 175 | typeName = ds.attrs['TypeName'].split(", ") 176 | typeNumb = len(typeName) 177 | typeColName = ds.attrs['TypeColorName'].split(", ") 178 | typeColHex = ds.attrs['TypeColorHex'].split(", ") 179 | 180 | # mean_OWT[0,:] returns 1x3 matrix for the first OWT 181 | # covm_OWT[:,:,0] returns 3x3 matrix for the first OWT 182 | result = { 183 | "mean_OWT": mean_OWT, 184 | "covm_OWT": covm_OWT, 185 | "lamBC": lamBC, 186 | "typeName": typeName, 187 | "typeNumb": typeNumb, 188 | "typeColName": typeColName, 189 | "typeColHex": typeColHex, 190 | } 191 | result = SimpleNamespace(**result) 192 | 193 | return result 194 | 195 | @staticmethod 196 | def load_centroids_dep(): 197 | """ 198 | WARNING: This function has been deprecated, please use `load_centroids_version()' 199 | load the centroids for classification 200 | For the "data/OWT_centroids.nc" file, three variables included: 201 | - [mean] mean of each optical water type (OWT) at three dims 202 | - [covm] covariance matrix (3x3) of each OWT 203 | - [lamBC] lambda coeffcient for the Box-Cox transformation 204 | Dimensions: [AVW, Area, NDI] 205 | Note: Area in the nc lib is after Box-Cox transformation 206 | """ 207 | import netCDF4 as nc 208 | import warnings 209 | warnings.warn( 210 | "This function has been deprecated and will be deleted in the future, " 211 | "please use `load_centroids_version()'", 212 | DeprecationWarning, 213 | stacklevel=2 214 | ) 215 | proj_root = os.path.dirname(os.path.abspath(__file__)) 216 | fn = os.path.join(proj_root, "data/v01/OWT_centroids.nc") 217 | ds = nc.Dataset(fn) 218 | mean_OWT = ds.variables["mean"][:] 219 | covm_OWT = ds.variables["covm"][:] 220 | lamBC = ds.variables["lamBC"][:].__float__() 221 | typeName = ds.getncattr("TypeName").split(", ") 222 | typeNumb = len(typeName) 223 | typeColName = ds.getncattr("TypeColorName").split(", ") 224 | typeColHex = ds.getncattr("TypeColorHex").split(", ") 225 | 226 | # mean_OWT[0,:] returns 1x3 matrix for the first OWT 227 | # covm_OWT[0,:,:] returns 3x3 matrix for the first OWT 228 | result = { 229 | "mean_OWT": mean_OWT, 230 | "covm_OWT": covm_OWT, 231 | "lamBC": lamBC, 232 | "typeName": typeName, 233 | "typeNumb": typeNumb, 234 | "typeColName": typeColName, 235 | "typeColHex": typeColHex, 236 | } 237 | return result 238 | 239 | @staticmethod 240 | def trans_boxcox(x, lamb): 241 | """Box-Cox transformation 242 | 243 | Args: 244 | x (float): Input value 245 | lamb (float): One value for the lambda coefficient 246 | 247 | Returns: 248 | float: Transformed value 249 | """ 250 | mask = (x > 0) & np.isfinite(x) 251 | y = np.full_like(x, np.nan) 252 | y[mask] = (x[mask]**lamb - 1) / lamb 253 | # y = (x**lamb - 1) / lamb 254 | return y 255 | 256 | @staticmethod 257 | def trans_boxcox_rev(y, lamb): 258 | """Reversed Box-Cox transformation 259 | 260 | Args: 261 | y (float): Input value 262 | lamb (float): One value for the lambda coefficient 263 | 264 | Returns: 265 | float: Transformed value 266 | """ 267 | x = (y * lamb + 1) ** (1 / lamb) 268 | return x 269 | 270 | 271 | 272 | 273 | 274 | if __name__ == "__main__": 275 | 276 | owt = OWT(AVW = [560, 400], Area = [1, 0.9], NDI = [0.2, 0.0]) 277 | print(owt.type_idx) 278 | print(owt.type_str) 279 | print(owt.u.shape[0:2]) 280 | 281 | -------------------------------------------------------------------------------- /pyowt/data/sensor_band_library.yaml: -------------------------------------------------------------------------------- 1 | lib_800: 2 | AVW_regression_coef: data/AVW_all_regression_800.txt 3 | sensor_AVW_bands_library: 4 | AERONET_OC_1: 5 | - 400 6 | - 412 7 | - 443 8 | - 490 9 | - 510 10 | - 560 11 | - 620 12 | - 665 13 | - 779 14 | - 866 15 | AERONET_OC_2: 16 | - 412 17 | - 443 18 | - 490 19 | - 532 20 | - 551 21 | - 667 22 | - 870 23 | CMEMS_BAL_HROC: 24 | - 443 25 | - 492 26 | - 560 27 | - 665 28 | - 704 29 | - 740 30 | - 783 31 | - 865 32 | CMEMS_BAL_NRT: 33 | - 400 34 | - 412 35 | - 443 36 | - 490 37 | - 510 38 | - 560 39 | - 620 40 | - 665 41 | - 674 42 | - 682 43 | - 709 44 | - 779 45 | - 866 46 | CMEMS_MED_MYINT: 47 | - 400 48 | - 412 49 | - 443 50 | - 490 51 | - 510 52 | - 560 53 | - 620 54 | - 665 55 | - 674 56 | - 682 57 | - 709 58 | - 779 59 | - 866 60 | GOCI-coms: 61 | - 412 62 | - 443 63 | - 490 64 | - 555 65 | - 660 66 | - 680 67 | - 745 68 | - 865 69 | HAWKEYE-seahawk1: 70 | - 412 71 | - 447 72 | - 488 73 | - 510 74 | - 556 75 | - 670 76 | - 752 77 | - 867 78 | HSI-EnMAP: 79 | - 420.9 80 | - 426.5 81 | - 431.8 82 | - 437.0 83 | - 442.0 84 | - 446.9 85 | - 451.7 86 | - 456.4 87 | - 461.2 88 | - 465.9 89 | - 470.5 90 | - 475.2 91 | - 479.9 92 | - 484.5 93 | - 489.2 94 | - 493.9 95 | - 498.6 96 | - 503.4 97 | - 508.2 98 | - 513.0 99 | - 517.9 100 | - 522.8 101 | - 527.7 102 | - 532.7 103 | - 537.7 104 | - 542.8 105 | - 547.9 106 | - 553.0 107 | - 558.2 108 | - 563.5 109 | - 568.8 110 | - 574.2 111 | - 579.6 112 | - 585.1 113 | - 590.7 114 | - 596.3 115 | - 602.0 116 | - 607.8 117 | - 613.7 118 | - 619.6 119 | - 625.7 120 | - 631.8 121 | - 637.9 122 | - 644.1 123 | - 650.4 124 | - 656.7 125 | - 663.1 126 | - 669.6 127 | - 676.1 128 | - 682.7 129 | - 689.3 130 | - 696.1 131 | - 702.9 132 | - 709.7 133 | - 716.7 134 | - 723.7 135 | - 730.7 136 | - 737.9 137 | - 745.1 138 | - 752.3 139 | - 759.6 140 | - 767.0 141 | - 774.4 142 | - 781.9 143 | - 789.5 144 | - 797.0 145 | HSI-PRISMA: 146 | - 407.0 147 | - 415.8 148 | - 423.8 149 | - 431.3 150 | - 438.7 151 | - 446.0 152 | - 453.4 153 | - 460.7 154 | - 468.1 155 | - 475.3 156 | - 482.5 157 | - 489.8 158 | - 497.1 159 | - 504.5 160 | - 512.0 161 | - 519.5 162 | - 527.3 163 | - 535.1 164 | - 542.9 165 | - 550.9 166 | - 559.0 167 | - 567.2 168 | - 575.5 169 | - 583.8 170 | - 592.3 171 | - 601.0 172 | - 610.0 173 | - 618.7 174 | - 627.8 175 | - 636.7 176 | - 646.0 177 | - 655.4 178 | - 664.9 179 | - 674.5 180 | - 684.1 181 | - 694.1 182 | - 703.7 183 | - 713.7 184 | - 723.9 185 | - 734.0 186 | - 744.1 187 | - 754.5 188 | - 764.9 189 | - 775.3 190 | - 785.7 191 | - 796.1 192 | LakeCCI-MERIS: 193 | - 413 194 | - 443 195 | - 490 196 | - 510 197 | - 560 198 | - 620 199 | - 665 200 | - 681 201 | - 709 202 | - 754 203 | - 779 204 | - 885 205 | MERIS-envisat: 206 | - 413 207 | - 443 208 | - 490 209 | - 510 210 | - 560 211 | - 620 212 | - 665 213 | - 681 214 | - 709 215 | - 754 216 | - 779 217 | - 865 218 | OCI-PACE: 219 | - 400.2 220 | - 402.7 221 | - 405.1 222 | - 407.6 223 | - 410.1 224 | - 412.6 225 | - 415.0 226 | - 417.5 227 | - 420.0 228 | - 422.5 229 | - 424.9 230 | - 427.4 231 | - 429.9 232 | - 432.4 233 | - 434.9 234 | - 437.4 235 | - 439.8 236 | - 442.3 237 | - 444.8 238 | - 447.3 239 | - 449.8 240 | - 452.3 241 | - 454.8 242 | - 457.3 243 | - 459.8 244 | - 462.2 245 | - 464.7 246 | - 467.2 247 | - 469.7 248 | - 472.2 249 | - 474.7 250 | - 477.2 251 | - 479.7 252 | - 482.2 253 | - 484.7 254 | - 487.2 255 | - 489.7 256 | - 492.2 257 | - 494.7 258 | - 497.2 259 | - 499.7 260 | - 502.2 261 | - 504.7 262 | - 507.2 263 | - 509.7 264 | - 512.2 265 | - 514.7 266 | - 517.2 267 | - 519.7 268 | - 522.2 269 | - 524.8 270 | - 527.3 271 | - 529.8 272 | - 532.3 273 | - 534.9 274 | - 537.3 275 | - 539.9 276 | - 542.4 277 | - 544.9 278 | - 547.4 279 | - 550.0 280 | - 552.5 281 | - 555.0 282 | - 557.6 283 | - 560.1 284 | - 562.6 285 | - 565.2 286 | - 567.7 287 | - 570.3 288 | - 572.8 289 | - 575.3 290 | - 577.9 291 | - 580.5 292 | - 583.0 293 | - 585.6 294 | - 588.1 295 | - 590.5 296 | - 593.1 297 | - 595.7 298 | - 598.3 299 | - 600.9 300 | - 603.3 301 | - 605.5 302 | - 600.5 303 | - 602.9 304 | - 605.5 305 | - 608.0 306 | - 610.4 307 | - 612.7 308 | - 615.1 309 | - 617.6 310 | - 620.1 311 | - 622.5 312 | - 625.0 313 | - 627.4 314 | - 629.9 315 | - 632.4 316 | - 634.8 317 | - 637.3 318 | - 639.8 319 | - 641.0 320 | - 642.3 321 | - 643.5 322 | - 644.7 323 | - 646.0 324 | - 647.2 325 | - 648.4 326 | - 649.7 327 | - 650.9 328 | - 652.2 329 | - 653.4 330 | - 654.6 331 | - 655.9 332 | - 657.1 333 | - 658.3 334 | - 659.6 335 | - 660.8 336 | - 662.1 337 | - 663.3 338 | - 664.6 339 | - 665.8 340 | - 667.0 341 | - 668.3 342 | - 669.5 343 | - 670.8 344 | - 672.0 345 | - 673.2 346 | - 674.5 347 | - 675.7 348 | - 677.0 349 | - 678.2 350 | - 679.4 351 | - 680.7 352 | - 681.9 353 | - 683.2 354 | - 684.4 355 | - 685.7 356 | - 686.9 357 | - 688.1 358 | - 689.4 359 | - 690.7 360 | - 691.9 361 | - 693.1 362 | - 694.4 363 | - 695.6 364 | - 696.9 365 | - 698.1 366 | - 699.4 367 | - 700.6 368 | - 701.9 369 | - 703.1 370 | - 704.4 371 | - 705.6 372 | - 706.8 373 | - 708.1 374 | - 709.3 375 | - 710.6 376 | - 711.8 377 | - 713.1 378 | - 714.3 379 | - 716.8 380 | - 719.3 381 | - 721.8 382 | - 724.3 383 | - 726.8 384 | - 729.3 385 | - 731.8 386 | - 734.3 387 | - 736.8 388 | - 739.3 389 | - 740.5 390 | - 741.8 391 | - 743.0 392 | - 744.3 393 | - 745.5 394 | - 746.8 395 | - 748.0 396 | - 749.3 397 | - 750.5 398 | - 751.8 399 | - 753.0 400 | - 754.3 401 | - 755.5 402 | - 756.8 403 | - 758.1 404 | - 759.3 405 | - 760.6 406 | - 761.8 407 | - 763.1 408 | - 764.3 409 | - 765.6 410 | - 766.8 411 | - 768.1 412 | - 769.3 413 | - 770.6 414 | - 771.8 415 | - 773.1 416 | - 774.3 417 | - 776.8 418 | - 779.3 419 | - 781.8 420 | - 784.4 421 | - 786.9 422 | - 789.4 423 | - 791.9 424 | - 794.4 425 | - 796.9 426 | - 799.4 427 | OCTS-adeos: 428 | - 412 429 | - 443 430 | - 490 431 | - 516 432 | - 565 433 | - 667 434 | - 862 435 | OLI-landsat-8: 436 | - 443 437 | - 482 438 | - 561 439 | - 655 440 | - 865 441 | SeaWiFS-orbview-2: 442 | - 412 443 | - 443 444 | - 490 445 | - 510 446 | - 555 447 | - 670 448 | - 865 449 | modis-aqua: 450 | - 412 451 | - 443 452 | - 469 453 | - 488 454 | - 531 455 | - 547 456 | - 555 457 | - 645 458 | - 667 459 | - 678 460 | - 748 461 | - 859 462 | modis-gee: 463 | - 412 464 | - 443 465 | - 488 466 | - 531 467 | - 555 468 | - 667 469 | - 678 470 | - 748 471 | - 859 472 | modis-terra: 473 | - 412 474 | - 443 475 | - 469 476 | - 488 477 | - 531 478 | - 547 479 | - 555 480 | - 645 481 | - 667 482 | - 678 483 | - 748 484 | - 859 485 | msi-sentinel-2a: 486 | - 443 487 | - 492 488 | - 560 489 | - 665 490 | - 704 491 | - 740 492 | - 783 493 | - 865 494 | msi-sentinel-2b: 495 | - 442 496 | - 492 497 | - 559 498 | - 665 499 | - 704 500 | - 739 501 | - 780 502 | - 864 503 | olci-s3a: 504 | - 400 505 | - 412 506 | - 443 507 | - 490 508 | - 510 509 | - 560 510 | - 620 511 | - 665 512 | - 674 513 | - 682 514 | - 709 515 | - 754 516 | - 779 517 | - 866 518 | olci-s3b: 519 | - 400 520 | - 412 521 | - 443 522 | - 490 523 | - 510 524 | - 560 525 | - 620 526 | - 665 527 | - 674 528 | - 681 529 | - 709 530 | - 754 531 | - 779 532 | - 866 533 | viirs-jpss-1: 534 | - 411 535 | - 445 536 | - 489 537 | - 556 538 | - 667 539 | - 746 540 | - 868 541 | viirs-jpss-2: 542 | - 411 543 | - 445 544 | - 488 545 | - 555 546 | - 671 547 | - 747 548 | - 868 549 | viirs-suomi-npp: 550 | - 410 551 | - 443 552 | - 486 553 | - 551 554 | - 671 555 | - 745 556 | - 862 557 | sensor_RGB_bands_library: 558 | AERONET_OC_1: 559 | - 443 560 | - 560 561 | - 665 562 | AERONET_OC_2: 563 | - 443 564 | - 551 565 | - 667 566 | CMEMS_BAL_HROC: 567 | - 443 568 | - 560 569 | - 665 570 | CMEMS_BAL_NRT: 571 | - 443 572 | - 560 573 | - 665 574 | CMEMS_MED_MYINT: 575 | - 443 576 | - 560 577 | - 665 578 | GOCI-coms: 579 | - 443 580 | - 555 581 | - 660 582 | HAWKEYE-seahawk1: 583 | - 447 584 | - 556 585 | - 670 586 | HSI-EnMAP: 587 | - 442.0 588 | - 558.2 589 | - 663.1 590 | HSI-PRISMA: 591 | - 446.0 592 | - 559.0 593 | - 664.9 594 | LakeCCI-MERIS: 595 | - 443 596 | - 560 597 | - 665 598 | MERIS-envisat: 599 | - 443 600 | - 560 601 | - 665 602 | OCI-PACE: 603 | - 442.3 604 | - 560.1 605 | - 664.6 606 | OCTS-adeos: 607 | - 443 608 | - 565 609 | - 667 610 | OLI-landsat-8: 611 | - 443 612 | - 561 613 | - 655 614 | SeaWiFS-orbview-2: 615 | - 443 616 | - 555 617 | - 670 618 | modis-aqua: 619 | - 443 620 | - 555 621 | - 667 622 | modis-gee: 623 | - 443 624 | - 555 625 | - 667 626 | modis-terra: 627 | - 443 628 | - 555 629 | - 667 630 | msi-sentinel-2a: 631 | - 443 632 | - 560 633 | - 665 634 | msi-sentinel-2b: 635 | - 442 636 | - 559 637 | - 665 638 | olci-s3a: 639 | - 443 640 | - 560 641 | - 665 642 | olci-s3b: 643 | - 443 644 | - 560 645 | - 665 646 | viirs-jpss-1: 647 | - 445 648 | - 556 649 | - 667 650 | viirs-jpss-2: 651 | - 445 652 | - 555 653 | - 671 654 | viirs-suomi-npp: 655 | - 443 656 | - 551 657 | - 671 658 | sensor_RGB_min_max: 659 | AERONET_OC_1: 660 | max: 866 661 | min: 400 662 | AERONET_OC_2: 663 | max: 870 664 | min: 412 665 | CMEMS_BAL_HROC: 666 | max: 865 667 | min: 443 668 | CMEMS_BAL_NRT: 669 | max: 866 670 | min: 400 671 | CMEMS_MED_MYINT: 672 | max: 866 673 | min: 400 674 | GOCI-coms: 675 | max: 865 676 | min: 412 677 | HAWKEYE-seahawk1: 678 | max: 867 679 | min: 412 680 | HSI-EnMAP: 681 | max: 797.0 682 | min: 420.9 683 | HSI-PRISMA: 684 | max: 796.1 685 | min: 407.0 686 | LakeCCI-MERIS: 687 | max: 885 688 | min: 413 689 | MERIS-envisat: 690 | max: 865 691 | min: 413 692 | OCI-PACE: 693 | max: 799.4 694 | min: 400.2 695 | OCTS-adeos: 696 | max: 862 697 | min: 412 698 | OLI-landsat-8: 699 | max: 865 700 | min: 443 701 | SeaWiFS-orbview-2: 702 | max: 865 703 | min: 412 704 | modis-aqua: 705 | max: 859 706 | min: 412 707 | modis-gee: 708 | max: 859 709 | min: 412 710 | modis-terra: 711 | max: 859 712 | min: 412 713 | msi-sentinel-2a: 714 | max: 865 715 | min: 443 716 | msi-sentinel-2b: 717 | max: 864 718 | min: 442 719 | olci-s3a: 720 | max: 866 721 | min: 400 722 | olci-s3b: 723 | max: 866 724 | min: 400 725 | viirs-jpss-1: 726 | max: 868 727 | min: 411 728 | viirs-jpss-2: 729 | max: 868 730 | min: 411 731 | viirs-suomi-npp: 732 | max: 862 733 | min: 410 734 | -------------------------------------------------------------------------------- /pyowt/PlotOWT.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | import pandas as pd 4 | from matplotlib.patches import Ellipse 5 | import os 6 | 7 | from pyowt.OWT import OWT 8 | from pyowt.OpticalVariables import OpticalVariables 9 | 10 | 11 | # Scatter plot ax limits 12 | AVW_MIN = 430 13 | AVW_MAX = 730 14 | ABC_MIN = -3.6 15 | ABC_MAX = 5 16 | NDI_MIN = -1.0 17 | NDI_MAX = 1.0 18 | 19 | 20 | def _is_instance_of_OWT(variable): 21 | return isinstance(variable, OWT) 22 | 23 | 24 | def _is_instance_of_OpticalVariables(variable): 25 | return isinstance(variable, OpticalVariables) 26 | 27 | 28 | class PlotOV: 29 | """Scatter plot of optical variables, AVW, ABC, and NDI. 30 | 31 | Args: 32 | owt (OWT): OWT instance 33 | """ 34 | 35 | def __init__(self, owt, show_label=True, abc_ndi=False, show=True): 36 | 37 | if not _is_instance_of_OWT(owt): 38 | raise ValueError( 39 | "The input 'owt' should be an 'OWT' class to get AVW, ABC, and NDI." 40 | ) 41 | 42 | # copy OWT attributes 43 | self.owt = owt 44 | 45 | # for background ellipse 46 | # from spectral library 47 | self.mean_OWT = owt.classInfo.mean_OWT 48 | self.covm_OWT = owt.classInfo.covm_OWT 49 | 50 | # from name and color code 51 | self.color_OWT = owt.classInfo.typeColHex 52 | self.name_OWT = owt.classInfo.typeName 53 | 54 | # add NaN for unclassified inputs 55 | # for those type_idx = -1 and type_str = 'NaN' 56 | # New element appended for color_OWT and name_OWT 57 | self.color_OWT.append("#000000") # black 58 | self.name_OWT.append("NaN") # unclassified 59 | 60 | # convert code lists to np.array for indexing 61 | self.color_OWT = np.array(self.color_OWT) 62 | self.name_OWT = np.array(self.name_OWT) 63 | 64 | if not hasattr(owt, "ABC"): 65 | raise ValueError( 66 | "No attribute 'ABC' found in 'owt'. Please use 'owt.run_classification()' to obtain ABC." 67 | ) 68 | 69 | # copy Optical Variable values for scattering plot 70 | self.AVW = owt.AVW.flatten() 71 | self.ABC = owt.ABC.flatten() 72 | self.NDI = owt.NDI.flatten() 73 | 74 | # copy classification results (type name and index) 75 | self.type_str = owt.type_str.flatten() 76 | self.type_idx = owt.type_idx.flatten() 77 | # type_idx = -1 should be replaced as 10 for correct indexing 78 | self.type_idx = np.where(self.type_idx == -1, 10, self.type_idx) 79 | 80 | # variables for scatter plotting 81 | plt_color = self.color_OWT[self.type_idx] 82 | type_marker = np.full(10, "o") 83 | type_marker = np.append(type_marker, "x") 84 | plt_marker = type_marker[self.type_idx] 85 | unique_markers = np.unique(plt_marker) 86 | 87 | ######## 88 | # Plot # 89 | ######## 90 | 91 | # Create figure with two subplots 92 | if abc_ndi: 93 | self.fig, self.axs = plt.subplots(1, 3, figsize=(15, 5)) 94 | else: 95 | self.fig, self.axs = plt.subplots(1, 2, figsize=(12, 5)) 96 | # TODO return fig? and plt.show() it ? 97 | 98 | # Panel 1: AVW vs ABC 99 | ax = self.axs[0] 100 | ax.set_xlim(AVW_MIN, AVW_MAX) 101 | ax.set_ylim(ABC_MIN, ABC_MAX) 102 | 103 | for i in range(len(self.mean_OWT)): 104 | mean = self.mean_OWT[i, [0, 1]] 105 | # cov = self.covm_OWT[i, :2, :2] 106 | cov = self.covm_OWT[:2, :2, i] 107 | color = self.color_OWT[i] 108 | type_name = self.name_OWT[i] 109 | if show_label: 110 | self.draw_ellipse(ax, mean[0], mean[1], cov, color, f"{type_name}") 111 | else: 112 | self.draw_ellipse(ax, mean[0], mean[1], cov, color, "") 113 | 114 | for marker in unique_markers: 115 | mask = plt_marker == marker 116 | ax.scatter( 117 | self.AVW[mask], 118 | self.ABC[mask], 119 | color=plt_color[mask], 120 | marker=marker, 121 | zorder=5, 122 | ) 123 | 124 | ax.set_xlabel("AVW [nm]") 125 | ax.set_ylabel(r"$\mathrm{A}_\mathrm{BC}$") 126 | 127 | # Panel 2: AVW vs NDI 128 | ax = self.axs[1] 129 | ax.set_xlim(AVW_MIN, AVW_MAX) 130 | ax.set_ylim(NDI_MIN, NDI_MAX) 131 | 132 | for i in range(len(self.mean_OWT)): 133 | mean = self.mean_OWT[i, [0, 2]] 134 | # cov = self.covm_OWT[i, [0, 2]][:, [0, 2]] 135 | cov = self.covm_OWT[[0, 2], :][:, [0, 2], i] 136 | color = self.color_OWT[i] 137 | type_name = self.name_OWT[i] 138 | if show_label: 139 | self.draw_ellipse(ax, mean[0], mean[1], cov, color, f"{type_name}") 140 | else: 141 | self.draw_ellipse(ax, mean[0], mean[1], cov, color, "") 142 | 143 | for marker in unique_markers: 144 | mask = plt_marker == marker 145 | ax.scatter( 146 | self.AVW[mask], 147 | self.NDI[mask], 148 | color=plt_color[mask], 149 | marker=marker, 150 | zorder=5, 151 | ) 152 | 153 | ax.set_xlabel("AVW [nm]") 154 | ax.set_ylabel("NDI") 155 | 156 | if abc_ndi==True: 157 | # Panel 3: ABC vs NDI 158 | ax = self.axs[2] 159 | ax.set_xlim(ABC_MIN, ABC_MAX) 160 | ax.set_ylim(NDI_MIN, NDI_MAX) 161 | 162 | for i in range(len(self.mean_OWT)): 163 | mean = self.mean_OWT[i, [1, 2]] 164 | cov = self.covm_OWT[[1, 2], :][:, [1, 2], i] 165 | color = self.color_OWT[i] 166 | type_name = self.name_OWT[i] 167 | if show_label: 168 | self.draw_ellipse(ax, mean[0], mean[1], cov, color, f"{type_name}") 169 | else: 170 | self.draw_ellipse(ax, mean[0], mean[1], cov, color, "") 171 | 172 | for marker in unique_markers: 173 | mask = plt_marker == marker 174 | ax.scatter( 175 | self.ABC[mask], 176 | self.NDI[mask], 177 | color=plt_color[mask], 178 | marker=marker, 179 | zorder=5, 180 | ) 181 | 182 | ax.set_xlabel(r"$\mathrm{A}_\mathrm{BC}$") 183 | ax.set_ylabel("NDI") 184 | 185 | plt.tight_layout() 186 | 187 | if show: 188 | plt.show() 189 | 190 | @staticmethod 191 | def draw_ellipse(ax, mean_x, mean_y, cov, color, label, n_std=1.0): 192 | # TODO check why ellipse here is different from that in R... 193 | # Calculate the eigenvalues and eigenvectors of the covariance matrix 194 | v, w = np.linalg.eigh(cov) 195 | order = v.argsort()[::-1] 196 | v = v[order] 197 | w = w[:, order] 198 | 199 | angle = np.degrees(np.arctan2(*w[:, 0][::-1])) 200 | 201 | # Width and height are "full" widths, not radius 202 | width, height = 2 * n_std * np.sqrt(v) 203 | ellipse = Ellipse( 204 | xy=(mean_x, mean_y), 205 | width=width, 206 | height=height, 207 | angle=angle, 208 | edgecolor="#000000", 209 | facecolor="none", 210 | label=label, 211 | zorder=6, 212 | ) 213 | 214 | ax.add_patch(ellipse) 215 | ax.text( 216 | mean_x, 217 | mean_y, 218 | label, 219 | fontsize=12, 220 | color=color, 221 | bbox=dict(facecolor="white", alpha=0.3, edgecolor="none"), 222 | ha="center", 223 | va="center", 224 | zorder=7, 225 | ) 226 | 227 | 228 | class PlotSpec: 229 | 230 | def __init__( 231 | self, 232 | owt, 233 | ov, 234 | norm=False, 235 | thre_u=None, 236 | fill_alpha=0.2, 237 | spec_alpha=0.8, 238 | figsize=(12, 8), 239 | show=True, 240 | ): 241 | """Plot spectral distribution of OWT classification results 242 | 243 | Args: 244 | owt (OWT): OWT instance 245 | ov (OpticalVariables): OpticalVariables instance 246 | norm (bool, optional): Plot normalized spectrum or not. Defaults to False. 247 | thre_u (float, optional): If None, input Rrs will be plotted by classified types; 248 | otherwise, spectra with memberships greater than thre_u will be plotted for each OWT 249 | fill_alpha (float, optional): Alpha value for the ribbon of upper and lower ranges of OWT means. 250 | Default to 0.8 251 | spec_alpha (float, optional): Alpha value for input Rrs spectra. Default to 0.8. 252 | figsize (tuple, optional): Figure size for plotting. Default to (12, 8) 253 | show (bool, optional): if True, the figure will be plotted directly; 254 | otherwise, the fig and axs can be assessed from the PlotSpec instance. 255 | """ 256 | 257 | if not _is_instance_of_OWT(owt): 258 | raise ValueError( 259 | "The input 'owt' should be an 'OWT' class to get AVW, ABC, and NDI." 260 | ) 261 | 262 | if not _is_instance_of_OpticalVariables(ov): 263 | raise ValueError( 264 | "The input 'ov' should be an 'OpticalVariables' class to get Rrs data." 265 | ) 266 | 267 | # copy OWT attributes 268 | self.owt = owt 269 | 270 | self.version = owt.version 271 | spec_lib_file = f"data/{self.version}/OWT_mean_spec.csv" 272 | 273 | # for background spectra 274 | # from spectral library 275 | proj_root = os.path.dirname(os.path.abspath(__file__)) 276 | self.spec_lib_file = os.path.join(proj_root, spec_lib_file) 277 | self.spec_lib = pd.read_csv(self.spec_lib_file) 278 | 279 | # from name and color code 280 | self.color_OWT = owt.classInfo.typeColHex 281 | self.name_OWT = owt.classInfo.typeName 282 | 283 | # add NaN for unclassified inputs 284 | # for those type_idx = -1 and type_str = 'NaN' 285 | # New element appended for color_OWT and name_OWT 286 | self.color_OWT.append("#000000") # black 287 | self.name_OWT.append("NaN") # unclassified 288 | 289 | # convert code lists to np.array for indexing 290 | self.color_OWT = np.array(self.color_OWT) 291 | self.name_OWT = np.array(self.name_OWT) 292 | 293 | # copy classification results (type name and index) 294 | self.type_str = owt.type_str.flatten() 295 | self.type_idx = owt.type_idx.flatten() 296 | # type_idx = -1 should be replaced as 10 for correct indexing 297 | self.type_idx = np.where(self.type_idx == -1, 10, self.type_idx) 298 | 299 | # variables for scatter plotting 300 | plt_color = self.color_OWT[:-1].tolist() 301 | type_marker = np.full(10, "o") 302 | type_marker = np.append(type_marker, "x") 303 | 304 | # get Rrs from ov class 305 | Rrs = ov.Rrs 306 | self.band = ov.band 307 | self.Rrs = Rrs.reshape(-1, Rrs.shape[2]) 308 | self.nRrs = self.Rrs / owt.Area 309 | 310 | # get membership from owt class 311 | self.u = owt.u.reshape(-1, owt.u.shape[2]) 312 | 313 | ######## 314 | # Plot # 315 | ######## 316 | 317 | types = self.spec_lib["type"].unique() 318 | 319 | n_rows = 3 320 | n_cols = 4 321 | 322 | self.fig, self.axs = plt.subplots( 323 | n_rows, n_cols, figsize=figsize, sharex=True, sharey=True 324 | ) 325 | 326 | ax_unclassified = self.fig.add_subplot(n_rows, n_cols, n_rows * n_cols - 1) 327 | 328 | axs = self.axs.flatten() 329 | 330 | # Original scale on Rrs [sr^-1]# 331 | # loop mean and shadow of spectra for each type 332 | i_owt = 0 333 | for ax, t, col in zip(axs, types, plt_color): 334 | 335 | subset = self.spec_lib[self.spec_lib["type"] == t] 336 | 337 | if norm: 338 | ax.plot(subset["wavelen"], subset["m_nRrs"], color=col, alpha=1) 339 | ax.fill_between( 340 | subset["wavelen"], 341 | subset["lo_nRrs"], 342 | subset["up_nRrs"], 343 | color=col, 344 | alpha=fill_alpha, 345 | ) 346 | else: 347 | ax.plot(subset["wavelen"], subset["m_Rrs"], color=col, alpha=1) 348 | ax.fill_between( 349 | subset["wavelen"], 350 | subset["lo_Rrs"], 351 | subset["up_Rrs"], 352 | color=col, 353 | alpha=fill_alpha, 354 | ) 355 | 356 | # only add labels for left and bottom margins 357 | # if i_owt % n_cols == 0: 358 | if i_owt == n_cols: 359 | if norm: 360 | ax.set_ylabel(r"Normalized Rrs [$1/\text{nm}^{2}$]") 361 | else: 362 | ax.set_ylabel(r"Rrs [$\text{sr}^{-1}$]") 363 | 364 | if i_owt >= ((n_rows - 1) * n_cols): 365 | ax.set_xlabel("Wavelength [nm]") 366 | 367 | ax.text( 368 | 0.05, 369 | 0.95, 370 | f"OWT {t}", 371 | transform=ax.transAxes, 372 | fontsize=12, 373 | verticalalignment="top", 374 | bbox=dict(facecolor="white", alpha=0.5, edgecolor="none"), 375 | ) 376 | 377 | # check if input Rrs corresponds to the current OWT type and plot in the subpanel 378 | if thre_u is None: 379 | # plot for types determined by max memebership 380 | select_ind = np.where(self.type_str == t)[0] 381 | else: 382 | # plot for types that are filttered by membership thresholds 383 | select_ind = np.where( 384 | (self.type_idx < 10) & (self.u[:, i_owt] > thre_u) 385 | )[0] 386 | 387 | if len(select_ind) > 0: 388 | if norm: 389 | ax.plot( 390 | self.band, 391 | self.nRrs[select_ind].T, 392 | color="black", 393 | alpha=spec_alpha, 394 | ) 395 | else: 396 | ax.plot( 397 | self.band, 398 | self.Rrs[select_ind].T, 399 | color="black", 400 | alpha=spec_alpha, 401 | ) 402 | 403 | i_owt += 1 404 | 405 | # delete empty ax 406 | self.fig.delaxes(axs[-1]) 407 | self.fig.delaxes(axs[-2]) 408 | 409 | # Plot unclassified Rrs in the last ax without sharing y-axis 410 | unclassified_ind = np.where(self.type_idx == 10)[0] 411 | if len(unclassified_ind) > 0: 412 | 413 | if norm: 414 | ax_unclassified.plot( 415 | self.band, 416 | self.nRrs[unclassified_ind].T, 417 | color="black", 418 | alpha=spec_alpha, 419 | ) 420 | else: 421 | ax_unclassified.plot( 422 | self.band, 423 | self.Rrs[unclassified_ind].T, 424 | color="black", 425 | alpha=spec_alpha, 426 | ) 427 | 428 | # Set labels and title for the unclassified Rrs plot 429 | ax_unclassified.set_xlabel("Wavelength [nm]") 430 | ax_unclassified.set_ylabel("") 431 | ax_unclassified.text( 432 | 0.05, 433 | 0.95, 434 | "Unclassified", 435 | transform=ax_unclassified.transAxes, 436 | fontsize=12, 437 | verticalalignment="top", 438 | bbox=dict(facecolor="white", alpha=0.5, edgecolor="none"), 439 | ) 440 | 441 | ax_unclassified.set_xlim(axs[0].get_xlim()) 442 | ax_unclassified.yaxis.set_label_position('right') 443 | ax_unclassified.yaxis.tick_right() 444 | 445 | plt.tight_layout() 446 | plt.subplots_adjust(wspace=0.1) 447 | 448 | if show: 449 | plt.show() 450 | 451 | 452 | if __name__ == "__main__": 453 | 454 | """Test 1""" 455 | if False: 456 | AVW_test = np.array([460, 600, 580]) 457 | Area_test = np.array([0.5, 0.4, 0.6]) 458 | NDI_test = np.array([-0.5, 0.2, 0.4]) 459 | 460 | owt = OWT(AVW=AVW_test, Area=Area_test, NDI=NDI_test) 461 | # print(owt.type_str) 462 | PlotOV(owt) 463 | 464 | """Test 2""" 465 | if True: 466 | d0 = pd.read_csv("./data/Rrs_demo.csv") 467 | d = d0.pivot_table(index="SampleID", columns="wavelen", values="Rrs") 468 | owt_train = d0[d0["wavelen"] == 350].sort_values(by="SampleID").type.values 469 | 470 | # data preparation for `ov` and `owt` classes 471 | Rrs = d.values 472 | band = d.columns.tolist() 473 | 474 | # add an unclassified spectrum for testing 475 | Rrs = np.append(Rrs, np.repeat(0.5, len(band)).reshape(1, -1), axis=0) 476 | 477 | # create `ov` class to calculate three optical variables 478 | ov = OpticalVariables(Rrs=Rrs, band=band) 479 | 480 | # create `owt` class to run optical classification 481 | owt = OWT(ov.AVW, ov.Area, ov.NDI) 482 | 483 | PlotSpec(owt, ov, norm=True, thre_u=0.5) 484 | -------------------------------------------------------------------------------- /projects/AquaINFRA/data/Rrs_demo_AquaINFRA_hyper.csv: -------------------------------------------------------------------------------- 1 | 350,352,354,356,358,360,362,364,366,368,370,372,374,376,378,380,382,384,386,388,390,392,394,396,398,400,402,404,406,408,410,412,414,416,418,420,422,424,426,428,430,432,434,436,438,440,442,444,446,448,450,452,454,456,458,460,462,464,466,468,470,472,474,476,478,480,482,484,486,488,490,492,494,496,498,500,502,504,506,508,510,512,514,516,518,520,522,524,526,528,530,532,534,536,538,540,542,544,546,548,550,552,554,556,558,560,562,564,566,568,570,572,574,576,578,580,582,584,586,588,590,592,594,596,598,600,602,604,606,608,610,612,614,616,618,620,622,624,626,628,630,632,634,636,638,640,642,644,646,648,650,652,654,656,658,660,662,664,666,668,670,672,674,676,678,680,682,684,686,688,690,692,694,696,698,700,702,704,706,708,710,712,714,716,718,720,722,724,726,728,730,732,734,736,738,740,742,744,746,748,750,752,754,756,758,760,762,764,766,768,770,772,774,776,778,780,782,784,786,788,790,792,794,796,798,800,802,804,806,808,810,812,814,816,818,820,822,824,826,828,830,832,834,836,838,840,842,844,846,848,850,852,854,856,858,860,862,864,866,868,870,872,874,876,878,880,882,884,886,888,890,892,894,896,898,900 2 | 0.0031255,0.0031771,0.0032275,0.0032763,0.0033229,0.0033676,0.0034108,0.003453,0.0034948,0.0035365,0.0035785,0.003621,0.0036638,0.0037072,0.0037508,0.0037947,0.0038391,0.0038837,0.0039276,0.0039703,0.0040109,0.0040471,0.0040812,0.0041144,0.004144,0.0041708,0.0041956,0.0042196,0.0042463,0.0042744,0.0043043,0.0043365,0.0043687,0.0043997,0.0044304,0.0044613,0.0044922,0.0045237,0.0045534,0.0045819,0.004606,0.0046237,0.0046331,0.0046298,0.004614,0.0045756,0.0045227,0.0044726,0.0044123,0.004353,0.0043068,0.0042667,0.0042442,0.0042329,0.0042209,0.0042123,0.0042016,0.0041819,0.0041583,0.0041239,0.0040831,0.004038,0.0039868,0.0039304,0.0038716,0.0038057,0.003739,0.0036625,0.0035775,0.0034787,0.0033592,0.0032257,0.0030791,0.0029211,0.0027539,0.0025835,0.0024052,0.002211,0.0020284,0.0018478,0.0016906,0.0015678,0.0014632,0.0013878,0.0013425,0.0013096,0.0012867,0.0012678,0.0012449,0.0012245,0.0012013,0.0011755,0.0011476,0.0011182,0.0010864,0.0010514,0.0010155,0.000976,0.0009372,0.0008986,0.000862,0.0008312,0.0008039,0.0007827,0.0007649,0.0007454,0.0007275,0.0007083,0.0006869,0.0006627,0.0006357,0.0006074,0.0005738,0.0005376,0.0004994,0.00046,0.0004206,0.0003827,0.0003476,0.000314,0.0002809,0.0002514,0.0002234,0.0002,0.0001802,0.0001643,0.0001526,0.0001437,0.0001383,0.0001345,0.0001324,0.0001309,0.0001292,0.0001277,0.000126,0.0001239,0.0001218,0.0001196,0.0001175,0.0001155,0.0001135,0.0001113,0.0001092,0.0001071,0.000105,0.0001027,0.0001004,9.81e-05,9.55e-05,9.27e-05,8.96e-05,8.59e-05,8.22e-05,7.82e-05,7.45e-05,7.14e-05,6.87e-05,6.68e-05,6.52e-05,6.4e-05,6.3e-05,6.19e-05,6.1e-05,6e-05,5.89e-05,5.77e-05,5.64e-05,5.49e-05,5.34e-05,5.19e-05,5.01e-05,4.83e-05,4.64e-05,4.44e-05,4.21e-05,3.96e-05,3.71e-05,3.45e-05,3.19e-05,2.94e-05,2.69e-05,2.45e-05,2.24e-05,2.04e-05,1.85e-05,1.69e-05,1.52e-05,1.38e-05,1.24e-05,1.12e-05,1.03e-05,9.4e-06,8.8e-06,8.4e-06,8.1e-06,7.9e-06,7.7e-06,7.6e-06,7.5e-06,7.5e-06,7.4e-06,7.4e-06,7.4e-06,7.4e-06,7.4e-06,7.4e-06,7.4e-06,7.5e-06,7.5e-06,7.6e-06,7.7e-06,7.8e-06,8e-06,8.1e-06,8.2e-06,8.4e-06,8.5e-06,8.7e-06,8.9e-06,9e-06,9.1e-06,9.2e-06,9.3e-06,9.3e-06,9.3e-06,9.2e-06,9.1e-06,9e-06,8.8e-06,8.6e-06,8.3e-06,8e-06,7.7e-06,7.3e-06,6.9e-06,6.6e-06,6.2e-06,5.8e-06,5.5e-06,5.2e-06,4.9e-06,4.7e-06,4.5e-06,4.4e-06,4.2e-06,4.1e-06,3.9e-06,3.8e-06,3.7e-06,3.5e-06,3.4e-06,3.3e-06,3.2e-06,3.2e-06,3.1e-06,3e-06,3e-06,2.9e-06,2.8e-06,2.8e-06,2.7e-06,2.7e-06,2.6e-06,2.6e-06,2.5e-06,2.5e-06,2.4e-06,2.4e-06,2.4e-06,2.3e-06,2.3e-06,2.2e-06,2.2e-06,2.1e-06,2.1e-06,2e-06 3 | 0.0026068,0.0026321,0.0026553,0.0026772,0.0026981,0.0027182,0.0027379,0.0027573,0.0027768,0.0027963,0.0028155,0.0028341,0.0028515,0.0028675,0.0028809,0.0028901,0.0028917,0.0028751,0.0028571,0.0028378,0.0028169,0.0027936,0.0027683,0.0027416,0.0027129,0.0026823,0.0026496,0.0026127,0.0025742,0.002535,0.0024964,0.0024629,0.0024318,0.0024048,0.0023818,0.0023625,0.0023433,0.0023245,0.0023043,0.0022832,0.002262,0.0022423,0.0022263,0.0022176,0.0022179,0.0022269,0.00225,0.0022858,0.0023298,0.0023809,0.0024349,0.0024835,0.0025289,0.0025679,0.0026009,0.0026292,0.0026527,0.0026726,0.0026933,0.0027116,0.0027317,0.0027553,0.0027761,0.002795,0.0028087,0.002819,0.0028245,0.002826,0.0028279,0.0028332,0.0028334,0.0028376,0.0028459,0.0028575,0.0028737,0.0028872,0.0029038,0.0029132,0.0029163,0.0029039,0.0028867,0.0028717,0.002853,0.0028518,0.0028629,0.0028799,0.0028969,0.0029084,0.0029102,0.0029036,0.0028849,0.0028633,0.0028293,0.002791,0.0027443,0.0026907,0.002634,0.0025646,0.0024996,0.002433,0.0023658,0.0023136,0.0022652,0.0022245,0.0021898,0.0021504,0.002116,0.0020792,0.0020472,0.0020161,0.0019891,0.0019741,0.0019551,0.0019326,0.0019008,0.0018543,0.0017846,0.0017004,0.001605,0.0014989,0.0013791,0.0012642,0.0011488,0.0010484,0.0009618,0.0008907,0.0008392,0.0008002,0.0007773,0.0007617,0.0007531,0.0007472,0.0007396,0.0007336,0.0007269,0.0007171,0.0007071,0.0006971,0.0006872,0.0006785,0.0006701,0.0006611,0.0006524,0.0006434,0.0006339,0.0006234,0.0006132,0.0006025,0.0005899,0.0005757,0.0005599,0.0005404,0.00052,0.0004978,0.0004762,0.0004575,0.0004407,0.0004278,0.0004164,0.0004073,0.0003999,0.0003933,0.0003884,0.0003836,0.0003792,0.0003753,0.0003724,0.00037,0.0003676,0.000365,0.0003607,0.0003555,0.0003482,0.000339,0.0003271,0.0003124,0.0002968,0.0002796,0.0002617,0.0002435,0.0002254,0.0002079,0.0001913,0.0001757,0.0001612,0.0001477,0.0001342,0.0001222,0.0001109,0.0001012,9.32e-05,8.63e-05,8.12e-05,7.77e-05,7.51e-05,7.34e-05,7.23e-05,7.14e-05,7.09e-05,7.05e-05,7.02e-05,7.02e-05,7.02e-05,7.03e-05,7.04e-05,7.07e-05,7.11e-05,7.17e-05,7.24e-05,7.33e-05,7.43e-05,7.56e-05,7.69e-05,7.84e-05,8e-05,8.17e-05,8.34e-05,8.52e-05,8.7e-05,8.85e-05,9e-05,9.12e-05,9.22e-05,9.28e-05,9.31e-05,9.3e-05,9.24e-05,9.14e-05,8.99e-05,8.81e-05,8.58e-05,8.32e-05,8.02e-05,7.68e-05,7.33e-05,6.98e-05,6.63e-05,6.3e-05,6e-05,5.72e-05,5.48e-05,5.28e-05,5.1e-05,4.93e-05,4.76e-05,4.6e-05,4.44e-05,4.28e-05,4.14e-05,4.01e-05,3.9e-05,3.79e-05,3.69e-05,3.6e-05,3.52e-05,3.45e-05,3.39e-05,3.32e-05,3.27e-05,3.21e-05,3.15e-05,3.1e-05,3.05e-05,3e-05,2.95e-05,2.9e-05,2.85e-05,2.81e-05,2.77e-05,2.72e-05,2.68e-05,2.63e-05,2.58e-05,2.54e-05,2.5e-05,2.46e-05 4 | 0.0029558,0.0030158,0.0030755,0.0031348,0.0031946,0.0032546,0.0033147,0.0033752,0.0034361,0.0034974,0.0035588,0.0036199,0.0036808,0.0037413,0.003801,0.003859,0.0039139,0.0039604,0.0040063,0.0040514,0.0040953,0.0041373,0.0041776,0.0042166,0.0042535,0.004288,0.0043196,0.0043467,0.0043715,0.0043949,0.0044177,0.0044431,0.0044706,0.0045009,0.0045336,0.0045687,0.0046036,0.0046371,0.0046675,0.0046956,0.0047234,0.0047516,0.004782,0.0048199,0.0048678,0.0049261,0.0049991,0.0050867,0.0051844,0.00529,0.0053986,0.0055,0.0055976,0.0056914,0.0057781,0.00586,0.0059394,0.0060186,0.006097,0.0061794,0.0062656,0.0063567,0.0064506,0.0065434,0.0066385,0.0067311,0.0068265,0.0069226,0.0070207,0.007122,0.0072377,0.0073629,0.0075025,0.0076613,0.0078304,0.0080131,0.0082022,0.0083919,0.0085768,0.0087482,0.008918,0.0090774,0.0092364,0.0094029,0.0096126,0.0098206,0.0100219,0.0102258,0.0103956,0.0105577,0.0106976,0.0108158,0.010917,0.0109919,0.011055,0.011095,0.011116,0.0111226,0.0111002,0.0110666,0.0110392,0.0110193,0.0110026,0.0110048,0.0110071,0.0109993,0.0109786,0.0109609,0.0109134,0.0108555,0.0107822,0.010717,0.0106153,0.0104796,0.0103012,0.0100629,0.0097545,0.0093901,0.0090035,0.0085665,0.008075,0.0075783,0.00706,0.0065903,0.0061614,0.0057919,0.0055169,0.0052963,0.0051728,0.0050948,0.005064,0.0050547,0.0050339,0.0050244,0.0049985,0.004967,0.0049346,0.0048982,0.0048627,0.0048309,0.0047975,0.004757,0.0047166,0.0046707,0.0046191,0.0045602,0.0044912,0.0044167,0.0043334,0.0042406,0.0041337,0.0040017,0.0038629,0.0037069,0.0035534,0.0034128,0.0032875,0.0031907,0.0031077,0.0030438,0.0029909,0.0029482,0.0029189,0.0028966,0.0028803,0.0028698,0.002868,0.0028723,0.0028801,0.0028859,0.0028764,0.0028568,0.0028179,0.0027616,0.0026756,0.0025582,0.002433,0.0022934,0.0021431,0.0019898,0.0018364,0.001687,0.0015465,0.0014156,0.0012956,0.0011841,0.0010746,0.0009766,0.0008838,0.0008036,0.0007375,0.0006804,0.0006383,0.0006103,0.0005905,0.0005786,0.0005715,0.0005665,0.0005647,0.0005641,0.0005649,0.0005668,0.0005693,0.0005729,0.0005767,0.0005811,0.0005872,0.0005948,0.0006034,0.0006135,0.0006249,0.000638,0.0006524,0.0006674,0.0006834,0.0007007,0.0007184,0.0007363,0.0007543,0.0007702,0.0007851,0.0007977,0.0008077,0.0008147,0.0008182,0.0008174,0.0008118,0.0008021,0.0007873,0.0007696,0.0007479,0.0007226,0.0006948,0.0006634,0.0006302,0.0005987,0.000566,0.0005344,0.0005055,0.0004787,0.0004564,0.0004373,0.0004214,0.0004077,0.0003945,0.0003822,0.0003696,0.0003578,0.0003474,0.0003376,0.0003292,0.0003213,0.0003139,0.0003072,0.000301,0.0002955,0.0002908,0.0002862,0.0002819,0.0002777,0.0002734,0.0002693,0.0002652,0.0002611,0.0002572,0.0002534,0.0002497,0.0002461,0.0002424,0.0002387,0.0002348,0.0002308,0.0002267,0.000223,0.0002191,0.0002155 5 | 1.25e-05,1.3e-05,1.35e-05,1.4e-05,1.46e-05,1.51e-05,1.57e-05,1.62e-05,1.69e-05,1.75e-05,1.82e-05,1.88e-05,1.96e-05,2.03e-05,2.1e-05,2.18e-05,2.26e-05,2.34e-05,2.42e-05,2.51e-05,2.59e-05,2.68e-05,2.77e-05,2.86e-05,2.95e-05,3.04e-05,3.13e-05,3.23e-05,3.32e-05,3.42e-05,3.52e-05,3.63e-05,3.74e-05,3.86e-05,3.99e-05,4.12e-05,4.26e-05,4.41e-05,4.56e-05,4.71e-05,4.87e-05,5.04e-05,5.22e-05,5.4e-05,5.6e-05,5.81e-05,6.04e-05,6.29e-05,6.54e-05,6.81e-05,7.08e-05,7.36e-05,7.64e-05,7.92e-05,8.2e-05,8.49e-05,8.79e-05,9.09e-05,9.4e-05,9.73e-05,0.0001006,0.0001041,0.0001077,0.0001113,0.0001151,0.0001188,0.0001225,0.0001263,0.0001301,0.000134,0.0001378,0.0001418,0.0001459,0.0001501,0.0001546,0.0001592,0.0001641,0.0001692,0.0001746,0.0001801,0.0001857,0.0001915,0.0001974,0.0002034,0.0002097,0.000216,0.0002224,0.0002288,0.0002353,0.0002419,0.0002484,0.0002551,0.0002618,0.0002685,0.0002752,0.000282,0.0002889,0.0002958,0.0003027,0.0003097,0.0003168,0.0003241,0.0003315,0.000339,0.0003466,0.0003543,0.0003622,0.0003704,0.0003791,0.0003886,0.0003992,0.0004117,0.0004258,0.0004413,0.000458,0.0004754,0.0004922,0.0005086,0.0005242,0.0005387,0.0005517,0.0005634,0.0005739,0.0005839,0.0005933,0.0006023,0.0006121,0.0006221,0.0006332,0.0006446,0.000656,0.0006671,0.0006782,0.0006895,0.0007008,0.0007116,0.0007221,0.0007329,0.0007443,0.0007566,0.0007702,0.0007842,0.0007992,0.0008151,0.0008316,0.0008488,0.0008683,0.0008884,0.0009081,0.0009266,0.0009426,0.0009534,0.0009596,0.0009596,0.0009547,0.0009471,0.0009366,0.0009254,0.0009131,0.0009016,0.0008924,0.0008875,0.0008869,0.00089,0.0008977,0.000912,0.0009371,0.000971,0.0010134,0.0010633,0.0011163,0.0011697,0.0012196,0.001264,0.0012963,0.0013142,0.0013201,0.0013127,0.0012937,0.0012645,0.0012263,0.0011807,0.0011307,0.0010772,0.001022,0.0009654,0.0009035,0.0008443,0.0007844,0.0007301,0.0006838,0.0006423,0.0006112,0.0005906,0.0005761,0.0005679,0.0005631,0.0005599,0.0005592,0.0005595,0.0005608,0.0005631,0.000566,0.0005696,0.0005736,0.0005781,0.0005841,0.0005915,0.0005999,0.0006096,0.0006206,0.000633,0.0006466,0.0006611,0.0006763,0.0006928,0.0007096,0.0007269,0.000744,0.0007595,0.0007743,0.000787,0.0007975,0.0008055,0.0008106,0.000812,0.0008091,0.0008026,0.0007917,0.0007776,0.0007598,0.0007384,0.0007144,0.0006863,0.0006561,0.0006271,0.0005968,0.0005675,0.0005409,0.000516,0.0004952,0.0004773,0.000462,0.0004483,0.0004345,0.0004212,0.0004074,0.0003944,0.0003828,0.0003717,0.0003621,0.0003531,0.0003449,0.0003375,0.0003306,0.0003247,0.0003192,0.000314,0.0003091,0.0003044,0.0002997,0.0002951,0.0002905,0.0002861,0.0002818,0.0002777,0.0002737,0.0002699,0.000266,0.0002621,0.0002581,0.0002539,0.0002497,0.000246,0.000242,0.0002385 6 | 0.0086194,0.0087865,0.0089557,0.0091254,0.0092987,0.0094726,0.009647,0.0098213,0.0099951,0.0101676,0.0103384,0.0105067,0.0106707,0.0108309,0.0109847,0.0111308,0.0112624,0.0113549,0.0114472,0.0115335,0.0116112,0.0116807,0.0117451,0.0118078,0.0118605,0.0119063,0.0119455,0.0119697,0.0119857,0.0119961,0.012007,0.0120304,0.0120588,0.0120967,0.0121454,0.0122019,0.0122534,0.0123028,0.012344,0.012376,0.0123959,0.0124122,0.0124386,0.0124841,0.0125603,0.0126662,0.0128178,0.0130191,0.0132438,0.0134892,0.0137394,0.0139838,0.0142191,0.0144203,0.0146107,0.0147992,0.0149733,0.0151314,0.0153249,0.0154933,0.0156741,0.0158762,0.0160649,0.0162692,0.0164357,0.0165897,0.0166875,0.0167425,0.0167839,0.0168332,0.0167585,0.0166833,0.0165933,0.0164529,0.0163212,0.0161302,0.0159214,0.0156086,0.0152675,0.0148232,0.0143639,0.0140225,0.0136898,0.0135276,0.013448,0.0134546,0.0135135,0.0135425,0.0135551,0.0135368,0.0134683,0.013385,0.0132471,0.013106,0.0129095,0.0126791,0.0124418,0.0121192,0.0118441,0.0115476,0.0112272,0.0109885,0.0107746,0.010611,0.0104877,0.0103156,0.010177,0.0099882,0.0098099,0.0095937,0.0093545,0.0091109,0.0087959,0.0084404,0.0080373,0.0075871,0.0070908,0.0065844,0.0060776,0.0055757,0.0050505,0.0045705,0.0041002,0.0036973,0.0033551,0.0030798,0.0028824,0.0027375,0.0026545,0.0026011,0.0025765,0.0025641,0.0025439,0.0025281,0.0025107,0.0024813,0.0024515,0.002422,0.0023921,0.0023652,0.0023376,0.0023077,0.0022781,0.0022486,0.0022183,0.0021846,0.0021507,0.0021155,0.0020727,0.0020234,0.0019672,0.0018954,0.0018198,0.0017369,0.0016575,0.0015916,0.0015353,0.0014951,0.0014616,0.0014361,0.0014155,0.0013954,0.0013809,0.0013658,0.0013502,0.0013346,0.0013189,0.0013025,0.0012842,0.0012634,0.0012352,0.0012046,0.0011682,0.0011265,0.0010768,0.0010186,0.0009597,0.0008975,0.0008341,0.0007711,0.0007096,0.0006505,0.0005955,0.0005444,0.0004976,0.0004543,0.0004116,0.0003737,0.000338,0.0003072,0.000282,0.0002603,0.0002445,0.000234,0.0002265,0.0002221,0.0002192,0.0002171,0.0002161,0.0002155,0.0002153,0.0002155,0.000216,0.0002168,0.0002177,0.0002189,0.0002206,0.0002229,0.0002256,0.0002288,0.0002325,0.0002367,0.0002414,0.0002464,0.0002517,0.0002574,0.0002632,0.0002692,0.000275,0.0002802,0.000285,0.0002889,0.000292,0.000294,0.0002949,0.0002943,0.000292,0.0002884,0.0002831,0.0002767,0.000269,0.0002601,0.0002504,0.0002393,0.0002276,0.0002165,0.000205,0.0001939,0.0001838,0.0001744,0.0001666,0.00016,0.0001545,0.0001497,0.0001449,0.0001403,0.0001355,0.000131,0.000127,0.0001232,0.0001198,0.0001167,0.0001138,0.0001112,0.0001088,0.0001067,0.0001048,0.0001029,0.0001012,9.95e-05,9.78e-05,9.61e-05,9.45e-05,9.29e-05,9.14e-05,8.99e-05,8.85e-05,8.71e-05,8.57e-05,8.43e-05,8.29e-05,8.14e-05,7.99e-05,7.86e-05,7.71e-05,7.59e-05 7 | 0.0016405,0.0016848,0.0017293,0.0017734,0.0018184,0.0018634,0.0019084,0.0019535,0.0019987,0.0020438,0.0020887,0.0021332,0.0021772,0.0022204,0.0022624,0.002302,0.0023376,0.0023633,0.0023884,0.0024122,0.0024344,0.0024549,0.0024735,0.0024904,0.0025052,0.0025174,0.0025266,0.0025306,0.002531,0.0025287,0.0025253,0.0025228,0.0025227,0.0025261,0.0025328,0.0025423,0.0025528,0.0025636,0.0025727,0.0025803,0.0025875,0.0025944,0.0026041,0.0026221,0.002652,0.0026967,0.0027595,0.0028387,0.0029315,0.0030345,0.0031402,0.0032408,0.0033351,0.0034227,0.0035044,0.0035828,0.0036603,0.0037417,0.0038287,0.0039229,0.0040246,0.0041326,0.0042446,0.0043574,0.0044691,0.0045763,0.0046781,0.0047745,0.0048668,0.00496,0.0050606,0.0051732,0.0053065,0.0054671,0.0056578,0.0058802,0.0061391,0.0064398,0.0067762,0.0071505,0.0075614,0.0079949,0.0084517,0.0089286,0.0094208,0.0099171,0.0104034,0.0108804,0.0113355,0.0117636,0.0121622,0.0125309,0.0128798,0.0132044,0.0135122,0.0138079,0.0140866,0.0143512,0.0146057,0.0148472,0.0150734,0.0152817,0.0154688,0.0156344,0.015774,0.0158768,0.0159331,0.0159586,0.0159586,0.0159286,0.0158659,0.0157706,0.0156525,0.0155227,0.0153963,0.0152689,0.0151306,0.0149969,0.0148799,0.0147779,0.0146764,0.0145738,0.0144675,0.0143586,0.0142326,0.0140854,0.0139193,0.0137144,0.0134831,0.0132308,0.0129833,0.0127373,0.012486,0.0122527,0.0120335,0.0118261,0.0116297,0.0114476,0.0112924,0.011182,0.0111257,0.0111154,0.011149,0.0112269,0.0113535,0.0115295,0.0117467,0.0119851,0.0122214,0.0124117,0.0125138,0.0124857,0.0122996,0.011923,0.0113726,0.0106758,0.0099176,0.0091856,0.008511,0.0079354,0.0074778,0.0071416,0.0069232,0.0068268,0.0068619,0.0070517,0.0074286,0.0080184,0.0088635,0.0099665,0.0112902,0.0127657,0.0143716,0.0160588,0.0176607,0.0191112,0.0203175,0.0212157,0.0218049,0.0220932,0.0220888,0.021822,0.0213442,0.0206851,0.0198895,0.0189783,0.0179003,0.0168178,0.0156893,0.0146537,0.0137712,0.012955,0.0123226,0.0118681,0.0115145,0.0112792,0.0111117,0.0109788,0.0109005,0.0108425,0.0108112,0.0108023,0.0108041,0.0108228,0.0108488,0.0108888,0.0109532,0.0110452,0.0111579,0.011295,0.0114535,0.0116375,0.0118419,0.0120626,0.0122976,0.0125482,0.0128063,0.0130719,0.0133349,0.0135693,0.0137921,0.0139786,0.0141327,0.0142457,0.0143157,0.0143311,0.014279,0.0141718,0.0139958,0.0137682,0.0134794,0.0131297,0.0127295,0.0122564,0.0117433,0.0112479,0.0107345,0.0102436,0.0098114,0.0094081,0.0090641,0.0087439,0.0084423,0.0081564,0.007863,0.0075804,0.0072945,0.0070261,0.0067879,0.0065638,0.0063649,0.0061843,0.0060192,0.0058727,0.005739,0.0056169,0.0055091,0.0054054,0.0053101,0.0052192,0.0051275,0.0050392,0.004953,0.0048692,0.0047891,0.0047143,0.0046416,0.0045716,0.0045022,0.0044321,0.0043613,0.0042884,0.0042143,0.00415,0.0040801,0.0040193 8 | 0.0129435,0.0129235,0.012896,0.0128634,0.0128286,0.0127904,0.0127503,0.0127082,0.0126666,0.0126232,0.0125806,0.0125343,0.0124809,0.0124214,0.0123511,0.0122672,0.0121634,0.0120145,0.0118602,0.0117019,0.0115362,0.0113514,0.011172,0.011016,0.0108534,0.0106798,0.0105001,0.0103103,0.0101268,0.0099453,0.0097698,0.0096109,0.0094575,0.0093063,0.0091613,0.0090228,0.0088816,0.0087391,0.0085862,0.0084252,0.0082517,0.008069,0.0078832,0.0076953,0.00751,0.0073123,0.0071273,0.0069813,0.0068376,0.0067093,0.0066059,0.0065114,0.0064424,0.0063852,0.006328,0.0062755,0.0062185,0.0061498,0.0060846,0.0060034,0.0059187,0.0058358,0.0057465,0.0056548,0.0055565,0.0054495,0.0053375,0.0052128,0.0050807,0.0049373,0.0047628,0.0045773,0.0043802,0.0041698,0.0039523,0.0037283,0.0034936,0.0032335,0.0029834,0.0027271,0.0024989,0.0023203,0.0021713,0.0020716,0.0020167,0.0019827,0.0019624,0.0019458,0.0019233,0.0019029,0.001877,0.0018464,0.0018102,0.00177,0.0017241,0.0016723,0.0016195,0.0015603,0.0015024,0.0014426,0.0013848,0.0013375,0.0012968,0.0012668,0.0012422,0.0012137,0.0011884,0.0011602,0.0011283,0.0010916,0.0010505,0.0010072,0.000954,0.0008957,0.0008333,0.0007675,0.000701,0.0006367,0.0005775,0.0005217,0.0004668,0.0004177,0.0003711,0.0003318,0.0002983,0.0002713,0.0002518,0.0002371,0.0002286,0.0002233,0.0002209,0.00022,0.0002183,0.0002171,0.0002152,0.0002129,0.0002105,0.0002082,0.0002058,0.0002036,0.0002012,0.0001984,0.0001956,0.0001927,0.0001897,0.0001865,0.0001829,0.0001792,0.0001751,0.0001705,0.0001652,0.0001586,0.0001516,0.0001439,0.0001365,0.0001304,0.0001255,0.0001221,0.0001194,0.0001175,0.0001158,0.0001141,0.0001128,0.0001115,0.0001098,0.000108,0.000106,0.0001039,0.0001016,9.92e-05,9.62e-05,9.32e-05,8.99e-05,8.63e-05,8.2e-05,7.69e-05,7.2e-05,6.69e-05,6.16e-05,5.65e-05,5.16e-05,4.68e-05,4.25e-05,3.86e-05,3.51e-05,3.19e-05,2.88e-05,2.6e-05,2.34e-05,2.11e-05,1.92e-05,1.76e-05,1.64e-05,1.57e-05,1.51e-05,1.48e-05,1.47e-05,1.46e-05,1.46e-05,1.46e-05,1.46e-05,1.46e-05,1.47e-05,1.48e-05,1.49e-05,1.5e-05,1.52e-05,1.54e-05,1.56e-05,1.58e-05,1.61e-05,1.64e-05,1.68e-05,1.72e-05,1.76e-05,1.8e-05,1.84e-05,1.88e-05,1.93e-05,1.96e-05,1.99e-05,2.02e-05,2.04e-05,2.05e-05,2.05e-05,2.03e-05,2.01e-05,1.97e-05,1.92e-05,1.87e-05,1.8e-05,1.73e-05,1.65e-05,1.57e-05,1.48e-05,1.4e-05,1.31e-05,1.23e-05,1.15e-05,1.08e-05,1.02e-05,9.7e-06,9.3e-06,9e-06,8.7e-06,8.5e-06,8.2e-06,8e-06,7.8e-06,7.6e-06,7.4e-06,7.2e-06,7e-06,6.9e-06,6.7e-06,6.6e-06,6.5e-06,6.4e-06,6.3e-06,6.2e-06,6.1e-06,6e-06,5.9e-06,5.8e-06,5.7e-06,5.6e-06,5.5e-06,5.4e-06,5.3e-06,5.2e-06,5.1e-06,5e-06,4.9e-06,4.8e-06,4.7e-06,4.6e-06 9 | 0.0055125,0.0056733,0.0058353,0.0059969,0.0061623,0.0063282,0.0064948,0.0066616,0.0068281,0.0069932,0.0071549,0.0073115,0.0074616,0.0076058,0.0077427,0.0078726,0.007992,0.0080853,0.0081774,0.0082635,0.0083419,0.0084136,0.0084791,0.0085408,0.0085921,0.0086336,0.0086642,0.00868,0.0086864,0.0086861,0.0086823,0.0086861,0.0086961,0.0087143,0.0087394,0.0087706,0.0088024,0.0088325,0.0088558,0.0088747,0.0088936,0.0089138,0.0089419,0.008994,0.0090776,0.0091969,0.0093625,0.0095694,0.0098066,0.0100665,0.0103291,0.0105715,0.0107983,0.0110103,0.0112095,0.0114025,0.0115972,0.0118044,0.0120292,0.0122765,0.0125498,0.0128505,0.0131632,0.0134783,0.0137895,0.0140859,0.0143643,0.0146225,0.0148657,0.0151051,0.0153518,0.0156169,0.0159186,0.0162742,0.0166816,0.017134,0.0176491,0.0182192,0.0188133,0.0194209,0.0200474,0.0206581,0.0212621,0.0218726,0.0224971,0.0230937,0.0236401,0.024148,0.0245805,0.0249493,0.0252469,0.0255096,0.025722,0.0258906,0.0260307,0.0261394,0.0262235,0.026271,0.0263024,0.0263204,0.0263355,0.0263785,0.0264105,0.026445,0.0264712,0.026465,0.0264411,0.0264208,0.0264181,0.0264507,0.0265408,0.0267645,0.0270442,0.0273621,0.0276842,0.0279458,0.028022,0.0279411,0.0277134,0.0272928,0.0266323,0.0258407,0.0248947,0.0239524,0.0230191,0.0221523,0.0214753,0.0208991,0.0205316,0.0202451,0.0200496,0.0198846,0.0196861,0.0195207,0.0193272,0.0190993,0.0188666,0.0186325,0.0184053,0.0182086,0.0180365,0.0178634,0.0177122,0.0175698,0.0174323,0.0172923,0.0171639,0.0170233,0.0168353,0.01658,0.0162361,0.0157614,0.0151997,0.0145251,0.0138054,0.0130877,0.0123959,0.0117894,0.0112396,0.0107812,0.0104119,0.0101287,0.0099426,0.0098399,0.0098258,0.0099056,0.0100967,0.0103781,0.0107324,0.0111166,0.0114475,0.011711,0.0118606,0.0118914,0.0117564,0.0114515,0.0110487,0.0105354,0.0099504,0.0093218,0.0086692,0.0080128,0.0073784,0.0067705,0.0061997,0.0056613,0.0051236,0.0046426,0.0041903,0.0038041,0.0034939,0.0032239,0.0030239,0.0028864,0.0027844,0.002719,0.0026741,0.0026394,0.0026195,0.0026054,0.0025981,0.0025967,0.0025982,0.0026044,0.0026122,0.0026235,0.0026418,0.0026673,0.0026983,0.0027359,0.0027794,0.0028301,0.0028866,0.0029474,0.0030121,0.0030822,0.0031544,0.0032287,0.0033024,0.0033682,0.0034303,0.003482,0.0035234,0.0035526,0.0035681,0.0035665,0.0035445,0.0035058,0.0034466,0.0033732,0.0032831,0.0031768,0.0030589,0.0029234,0.0027798,0.0026432,0.0025033,0.0023701,0.0022519,0.0021424,0.0020507,0.0019693,0.0018969,0.0018311,0.0017651,0.0017021,0.0016382,0.0015784,0.0015253,0.0014756,0.0014317,0.0013916,0.0013548,0.0013219,0.0012918,0.0012648,0.0012406,0.0012174,0.001196,0.0011754,0.0011547,0.0011347,0.0011151,0.001096,0.0010777,0.0010604,0.0010435,0.0010273,0.0010111,0.0009948,0.0009782,0.0009611,0.0009438,0.0009286,0.0009122,0.0008977 10 | 0.0035346,0.003627,0.0037207,0.003815,0.003911,0.004008,0.004106,0.0042052,0.0043055,0.0044068,0.0045088,0.0046115,0.004715,0.0048193,0.0049243,0.0050299,0.0051359,0.0052406,0.0053468,0.0054539,0.0055614,0.0056693,0.0057774,0.0058857,0.0059939,0.0061017,0.006209,0.0063153,0.0064213,0.0065274,0.0066343,0.0067437,0.0068554,0.00697,0.0070873,0.0072066,0.0073268,0.0074474,0.0075678,0.007688,0.0078071,0.0079262,0.0080482,0.0081764,0.0083136,0.0084615,0.008622,0.0087944,0.0089757,0.0091624,0.0093502,0.0095324,0.0097088,0.0098818,0.0100536,0.0102274,0.0104059,0.0105912,0.0107861,0.0109919,0.0112082,0.0114343,0.0116664,0.0119034,0.012144,0.0123847,0.0126229,0.0128594,0.0130945,0.0133298,0.0135679,0.0138136,0.0140701,0.0143384,0.01462,0.0149149,0.0152258,0.0155482,0.0158784,0.016212,0.0165487,0.016885,0.0172215,0.0175597,0.0179004,0.0182392,0.0185725,0.0189009,0.0192196,0.0195291,0.0198292,0.0201267,0.0204166,0.0207017,0.0209849,0.0212664,0.0215475,0.0218208,0.0220918,0.0223597,0.022625,0.0228948,0.0231633,0.0234327,0.0237005,0.0239576,0.0242131,0.0244687,0.0247274,0.0249907,0.0252626,0.0255582,0.0258608,0.0261693,0.0264771,0.0267685,0.0270166,0.0272326,0.0274193,0.0275682,0.0276612,0.0277205,0.0277333,0.0277308,0.0277111,0.0276881,0.0277021,0.027735,0.0278265,0.0279433,0.0280893,0.0282447,0.0283855,0.0285285,0.0286504,0.0287407,0.0288117,0.0288691,0.0289152,0.028963,0.0290165,0.0290693,0.0291317,0.0291996,0.0292752,0.0293542,0.0294412,0.0295225,0.0295814,0.0295953,0.0295381,0.0293816,0.0291362,0.0287701,0.0283171,0.0278084,0.027243,0.026693,0.0261465,0.0256449,0.0252097,0.0248559,0.0246193,0.024505,0.0245308,0.0247143,0.0250746,0.0255795,0.0262237,0.0268992,0.0274897,0.0279932,0.0283733,0.0286226,0.028695,0.028578,0.0283324,0.0279304,0.0273919,0.0267332,0.0259659,0.0251063,0.0241868,0.0232145,0.0222115,0.0211737,0.0200329,0.0189154,0.017763,0.0166925,0.0157569,0.0148977,0.0142363,0.013779,0.0134423,0.0132366,0.0131042,0.0130071,0.0129633,0.0129412,0.0129426,0.0129661,0.013,0.0130518,0.0131088,0.013179,0.0132764,0.0134013,0.013546,0.0137155,0.0139066,0.0141237,0.0143607,0.0146103,0.0148714,0.0151499,0.0154325,0.0157176,0.0159969,0.0162454,0.0164781,0.0166728,0.0168302,0.016946,0.0170139,0.0170233,0.0169635,0.0168467,0.0166571,0.0164151,0.0161102,0.0157403,0.0153205,0.0148226,0.0142781,0.0137445,0.0131778,0.0126192,0.0121049,0.0116164,0.0112006,0.0108326,0.0105086,0.0102149,0.0099169,0.0096275,0.009328,0.0090426,0.0087866,0.0085432,0.0083278,0.0081283,0.0079434,0.0077766,0.0076232,0.0074872,0.0073629,0.0072439,0.0071332,0.0070265,0.0069182,0.0068128,0.0067089,0.0066067,0.0065083,0.0064149,0.0063236,0.0062351,0.0061465,0.0060562,0.005964,0.0058685,0.0057708,0.0056853,0.005592,0.0055099 11 | 0.0015979,0.0016306,0.001662,0.0016926,0.0017229,0.0017531,0.0017831,0.0018133,0.0018438,0.0018746,0.0019052,0.0019354,0.0019649,0.0019935,0.0020208,0.002046,0.0020674,0.0020797,0.0020906,0.0021001,0.0021081,0.0021135,0.0021165,0.0021177,0.0021163,0.0021119,0.002104,0.0020923,0.0020785,0.002063,0.0020461,0.0020311,0.002019,0.0020095,0.0020017,0.0019956,0.0019904,0.0019846,0.0019761,0.0019659,0.0019568,0.001948,0.0019404,0.0019392,0.0019461,0.0019622,0.0019898,0.0020272,0.0020715,0.0021208,0.0021691,0.002213,0.0022546,0.0022935,0.0023298,0.0023649,0.0024005,0.0024398,0.002483,0.0025302,0.0025829,0.0026414,0.0027028,0.0027641,0.0028239,0.0028802,0.0029342,0.0029851,0.0030331,0.003082,0.0031355,0.0031943,0.0032644,0.0033509,0.0034527,0.003567,0.0036999,0.0038552,0.0040209,0.0041999,0.0043933,0.0045915,0.0047948,0.0050035,0.0052181,0.0054314,0.0056403,0.0058461,0.0060375,0.0062151,0.0063771,0.0065313,0.0066808,0.0068178,0.0069478,0.0070705,0.0071867,0.0072991,0.0074109,0.0075214,0.0076348,0.0077496,0.0078575,0.0079593,0.0080502,0.0081196,0.0081607,0.0081919,0.0082164,0.0082295,0.0082274,0.0082223,0.0082111,0.0081932,0.0081734,0.0081372,0.0080612,0.0079614,0.0078528,0.0077362,0.0075963,0.0074431,0.0072751,0.0071062,0.0069244,0.0067441,0.0065875,0.0064421,0.0063244,0.0062097,0.0061149,0.006026,0.005926,0.0058341,0.005741,0.0056503,0.0055652,0.0054813,0.005401,0.0053367,0.0052885,0.0052472,0.0052208,0.0052047,0.0051998,0.0052061,0.0052203,0.0052344,0.0052429,0.0052235,0.0051571,0.0050309,0.0048394,0.0045742,0.0042648,0.003935,0.0036198,0.0033392,0.0030949,0.0028971,0.0027472,0.0026412,0.002577,0.0025565,0.0025821,0.002656,0.0027895,0.002988,0.0032451,0.0035571,0.0038953,0.0042314,0.0045413,0.0048026,0.0049539,0.0049831,0.0049349,0.0048022,0.0045954,0.0043391,0.0040503,0.0037476,0.0034512,0.003168,0.0029036,0.0026557,0.002412,0.0021914,0.0019787,0.0017909,0.0016335,0.001499,0.0014011,0.0013385,0.0012967,0.0012743,0.0012629,0.0012565,0.0012569,0.0012601,0.0012658,0.0012739,0.0012835,0.0012954,0.0013075,0.0013209,0.0013385,0.0013592,0.0013824,0.001409,0.0014387,0.0014725,0.0015094,0.0015474,0.0015872,0.0016308,0.0016751,0.0017194,0.0017635,0.0018022,0.0018377,0.0018672,0.0018896,0.0019039,0.0019084,0.0019011,0.0018818,0.0018517,0.0018088,0.0017593,0.0017004,0.0016333,0.0015619,0.0014831,0.0014009,0.0013231,0.0012424,0.0011635,0.0010898,0.0010218,0.0009661,0.0009209,0.0008858,0.0008577,0.0008319,0.0008079,0.0007832,0.0007601,0.0007395,0.0007201,0.0007039,0.0006881,0.0006731,0.0006592,0.0006463,0.0006351,0.0006256,0.0006161,0.000607,0.0005981,0.0005891,0.0005802,0.0005713,0.0005624,0.0005537,0.0005452,0.0005367,0.0005285,0.00052,0.0005112,0.0005022,0.0004927,0.000483,0.0004743,0.0004648,0.0004561 12 | --------------------------------------------------------------------------------