├── controlModisExample.sh ├── controlViirsEdrExample.sh ├── README.md ├── masterControlExample.sh ├── makeTimeList.py ├── aotConf.py ├── aeronetDataLev15.py ├── aeroPyAqua.py ├── aeroPyVirrsEdr.py └── aotLib.py /controlModisExample.sh: -------------------------------------------------------------------------------- 1 | year=$1 2 | months=('01' '02' '03' '04') 3 | lev=$2 4 | 5 | for month in ${months[@]} 6 | do 7 | # echo 2014$month $month 8 | ./aeroPyAqua.py $year$month $month $lev 9 | done 10 | 11 | -------------------------------------------------------------------------------- /controlViirsEdrExample.sh: -------------------------------------------------------------------------------- 1 | year=$1 2 | months=('01' '02' '03') 3 | lev=$2 4 | 5 | for month in ${months[@]} 6 | do 7 | # echo 2014$month $month 8 | ./aeroPyVirrsEdr.py $year$month $month $lev 9 | done 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # aotPy 2 | This repository contain a set of Python and Shell files to extract Aerosol Optical Depth from VIIRS, MODIS, and AERONET data set. I am working to improve documentation. 3 | 4 | Next step: 5 | 6 | * To include a test case 7 | 8 | -------------------------------------------------------------------------------- /masterControlExample.sh: -------------------------------------------------------------------------------- 1 | year=('2014' '2015') 2 | #year=('2014') 3 | 4 | lev=lev15 5 | #lev=lev20 6 | 7 | for year in ${year[@]} 8 | do 9 | 10 | echo $lev 11 | 12 | # ./control.sh $year $lev 13 | # ./control2.sh $year $lev 14 | 15 | # ./controlNppEdr.sh $year $lev 16 | # ./controlNppEdr2.sh $year $lev 17 | 18 | # ./makeTimeList.py $year aqua_nasa 19 | # ./makeTimeList.py $year edrNppNasa 20 | 21 | ./aeronetDataLev15.py $year aqua_nasa $lev 22 | ./aeronetDataLev15.py $year edrNppNasa $lev 23 | done 24 | -------------------------------------------------------------------------------- /makeTimeList.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python 2 | 3 | import numpy as np 4 | from aotConf import aeronetPath, nppPath 5 | import h5py as h5 6 | from sys import argv 7 | 8 | inputAeronetDataPath,outputAeronetDataPath=aeronetPath() 9 | outputNppDataPath=nppPath()[1] 10 | 11 | 12 | ##### creat time appended ##### 13 | nppPath=outputNppDataPath+'/'+'aotDataSet.h5' 14 | nppData=h5.File(nppPath) 15 | year=argv[1] 16 | satellite=argv[2] 17 | 18 | 19 | timeSatelliteArr=np.array([]) 20 | allTimeSatelliteArr=np.array([]) 21 | 22 | #months=['01','02','03','04','05','06','07'] 23 | months=['01','02','03','04','05','06','07','08','09','10','11','12'] 24 | 25 | #months=['09'] 26 | 27 | for month in months: 28 | try: 29 | timeSatelliteArr=np.array(nppData['/time/'+year+'/'+satellite+month]) 30 | allTimeSatelliteArr=np.append(allTimeSatelliteArr,np.unique(timeSatelliteArr)) 31 | 32 | except: 33 | print 'without '+'/time/'+year+'/'+satellite+month 34 | 35 | #allTimeSatelliteArr=np.append(allTimeSatelliteArr,np.unique(timeSatelliteArr)) 36 | 37 | try: 38 | del nppData['time'+'/'+year+'/'+satellite] 39 | except: 40 | print 'without timeSerie' 41 | 42 | nppData['time'+'/'+year+'/'+satellite]=allTimeSatelliteArr 43 | nppData.close() 44 | -------------------------------------------------------------------------------- /aotConf.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python 2 | 3 | 4 | #this file is used to set path to use at input and output data 5 | 6 | inputDataPath='/Users/josedias/Documents/aotData' 7 | outputDataPath='/Users/josedias/Documents/aotOutData' 8 | 9 | def nppPath(): 10 | 11 | inputNppDataPath=inputDataPath+'/npp' 12 | outputNppDataPath=outputDataPath 13 | return inputNppDataPath, outputNppDataPath 14 | 15 | def edrNppPath(): 16 | 17 | inputNppDataPath=inputDataPath+'/npp/edr' 18 | outputNppDataPath=outputDataPath 19 | return inputNppDataPath, outputNppDataPath 20 | 21 | def edrNppPathNasa(): 22 | 23 | inputNppDataPath=inputDataPath+'/npp/edrNasaAsc' 24 | outputNppDataPath=outputDataPath 25 | return inputNppDataPath, outputNppDataPath 26 | 27 | def edrCsppPath(): 28 | 29 | inputNppDataPath=inputDataPath+'/npp/edrCsppAsc' 30 | outputNppDataPath=outputDataPath 31 | return inputNppDataPath, outputNppDataPath 32 | 33 | def aeronetPath(): 34 | 35 | inputAeronetDataPath=inputDataPath+'/aeronet' 36 | outputAeronetDataPath=outputDataPath 37 | return inputAeronetDataPath, outputAeronetDataPath 38 | 39 | 40 | def aquaPath(): 41 | 42 | inputAquaDataPath=inputDataPath+'/aqua' 43 | outputAquaDataPath=outputDataPath 44 | return inputAquaDataPath, outputAquaDataPath 45 | 46 | 47 | -------------------------------------------------------------------------------- /aeronetDataLev15.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python 2 | 3 | #from pandas import Series, DataFrame 4 | #import pandas as pd 5 | #import glob 6 | 7 | import numpy as np 8 | from aotConf import aeronetPath 9 | import h5py as h5 10 | import aotLib 11 | from sys import argv 12 | 13 | ############ main code ############ 14 | aotAeroMean=np.array([]) 15 | aotAeroStd=np.array([]) 16 | aotAero550Mean=np.array([]) 17 | aotAero550Std=np.array([]) 18 | newTimeArr=np.array([]) 19 | 20 | inputAeronetDataPath,outputAeronetDataPath=aeronetPath() 21 | 22 | year=argv[1] 23 | 24 | #get time array from aotDataSet.h5 25 | #satellite=argv[1] 26 | #satellite='npp' 27 | satellite=argv[2] 28 | #satellite='edrNppNasa' 29 | #satellite='aqua' 30 | #satellite='aqua_nasa' 31 | 32 | dataLev = argv[3] 33 | 34 | 35 | pathOut=aeronetPath()[1] 36 | dataPath=pathOut+'/'+'aotDataSet.h5' 37 | aotData=h5.File(dataPath,'r') 38 | timeSatelliteArr=np.array(aotData['/time/'+year+'/'+satellite]) 39 | aotData.close() 40 | device='aeronet' 41 | 42 | #get list of aeronet archive 43 | 44 | if dataLev == 'lev20': 45 | filePathArr=aotLib.getAeronetPathArr(inputAeronetDataPath + '/allDataL2',dataLev) 46 | 47 | 48 | else: 49 | dataLev = 'lev15' 50 | filePathArr=aotLib.getAeronetPathArr(inputAeronetDataPath + '/allDataL15',dataLev) 51 | 52 | 53 | #print filePathArr 54 | stationNameArr=aotLib.returnAeronetCoordenateArr(filePathArr)[0] 55 | 56 | 57 | for filePath in range(len(filePathArr)): 58 | 59 | print filePathArr[filePath] 60 | table,aotWlenHeader=aotLib.getAeronetDataTable(filePathArr[filePath]) 61 | 62 | table = table[(table.Date_dd_mm_yy_.str.split(':').str[2] == year)] 63 | 64 | 65 | for timeSat in timeSatelliteArr: 66 | aot=table[(table.Julian_Day>timeSat-30/1440.)&(table.Julian_Day=0: 75 | 76 | aotAero550Mean=np.append(aotAero550Mean,aot550Arr.mean()) 77 | aotAero550Std=np.append(aotAero550Std,aot550Arr.std()) 78 | aotAeroMean=np.append(aotAeroMean,aot.mean()) 79 | aotAeroStd=np.append(aotAeroStd,aot.std()) 80 | newTimeArr=np.append(newTimeArr,timeSat) 81 | 82 | 83 | 84 | aotLib.writeAeroData(pathOut,device,satellite,stationNameArr[filePath],year,newTimeArr,aotAeroMean,aotAeroStd,'500') 85 | aotLib.writeAeroData(pathOut,device,satellite,stationNameArr[filePath],year,newTimeArr,aotAero550Mean,aotAero550Std,'550') 86 | aotAeroMean=np.array([]) 87 | aotAeroStd=np.array([]) 88 | newTimeArr=np.array([]) 89 | aotAero550Mean=np.array([]) 90 | aotAero550Std=np.array([]) 91 | 92 | 93 | -------------------------------------------------------------------------------- /aeroPyAqua.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python 2 | 3 | import numpy as np 4 | import h5py as h5 5 | import gc 6 | import aotLib 7 | from colorama import Fore, Back, Style 8 | from aotConf import aquaPath 9 | from aotConf import aeronetPath 10 | from sys import argv 11 | 12 | def aotProcess(filePath,aotFile,julianDay,aeronetLat,aeronetLon): 13 | 14 | aerosolFile=h5.File(aotFile,'r') 15 | 16 | try: 17 | aeroOptDep=np.array(aerosolFile['/Optical_Depth_Land_And_Ocean']) 18 | 19 | except: 20 | aeroOptDep=np.array(aerosolFile['/mod04/Data Fields/Optical_Depth_Land_And_Ocean']) 21 | 22 | aeroOptDep=np.ma.masked_where(aeroOptDep<0,aeroOptDep) 23 | 24 | try: 25 | lat=np.array(aerosolFile['/Latitude']) 26 | lon=np.array(aerosolFile['/Longitude']) 27 | 28 | except: 29 | lat=np.array(aerosolFile['/mod04/Geolocation Fields/Latitude']) 30 | lon=np.array(aerosolFile['/mod04/Geolocation Fields/Longitude']) 31 | 32 | aerosolFile.close() 33 | 34 | while True: 35 | lat,lon,aeroOptDep,parameter=aotLib.removeValues(0.,lat,lon,aeroOptDep) 36 | 37 | #print parameter 38 | if parameter==0: 39 | break 40 | 41 | while True: 42 | lat,lon,aeroOptDep,parameter=aotLib.removeValues(-999.0,lat,lon,aeroOptDep) 43 | 44 | #print parameter 45 | if parameter==0: 46 | break 47 | 48 | 49 | aeroOptDep=aeroOptDep*10**(-3) 50 | 51 | 52 | 53 | #-----aeronet position-------- 54 | #aeronet= -10.0,-54.5 55 | #aeronet=-15.729500,-56.021000 56 | earthRay= 6371 ## km 57 | length= 40 ## km 58 | 59 | beginLatReg,endLatReg,beginLonReg,endLonReg=aotLib.getComparisonRegion(aeronetLat,aeronetLon,length,earthRay) 60 | rowMin,rowMax,colMin,colMax=aotLib.getRegionIndexMatrix(beginLatReg,endLatReg,lat,beginLonReg,endLonReg,lon) 61 | 62 | aeroOptDep=aotLib.getNewMatrix(rowMin,rowMax,colMin,colMax,aeroOptDep) 63 | lat=aotLib.getNewMatrix(rowMin,rowMax,colMin,colMax,lat) 64 | lon=aotLib.getNewMatrix(rowMin,rowMax,colMin,colMax,lon) 65 | 66 | print aeroOptDep.mean() 67 | 68 | 69 | return aeroOptDep.mean(),aeroOptDep.std(),aeroOptDep,lat,lon 70 | 71 | 72 | ################################## 73 | # main code 74 | # 75 | #--------------------------------- 76 | #aqua path work 77 | # 78 | yearMonth=argv[1] 79 | #yearMonth='201401' 80 | inputAquaDataPath, outputAquaDataPath=aquaPath() 81 | #filePath=inputAquaDataPath+'/'+yearMonth+'/' 82 | #filePath=inputAquaDataPath+'/'+yearMonth+'Nasa'+'/' 83 | filePath=inputAquaDataPath+'/Nasa'+yearMonth+'/' 84 | 85 | processId=argv[2] 86 | 87 | rec='nasa' 88 | satellite='aqua_'+rec 89 | 90 | 91 | 92 | #--------------------------------- 93 | print filePath 94 | 95 | #--------------------------------- 96 | # geometric parameters 97 | # 98 | nameStationArr=[] 99 | aeronetLatArr=[] 100 | aeronetLonArr=[] 101 | 102 | inputAeronetDataPath,outputAeronetDataPath=aeronetPath() 103 | 104 | dataLev = argv[3] 105 | 106 | if dataLev == 'lev20': 107 | filePathArr=aotLib.getAeronetPathArr(inputAeronetDataPath + '/allDataL2',dataLev) 108 | 109 | else: 110 | dataLev = 'lev15' 111 | filePathArr=aotLib.getAeronetPathArr(inputAeronetDataPath + '/allDataL15',dataLev) 112 | 113 | nameStationArr,aeronetLatArr,aeronetLonArr=aotLib.returnAeronetCoordenateArr(filePathArr) 114 | 115 | #print nameStationArr 116 | #--------------------------------- 117 | 118 | 119 | #-------------------------------------- 120 | # array variables 121 | # 122 | lenList=[] 123 | aerosolFileList=[] 124 | timeArr=np.array([]) 125 | aotMeanArr=np.array([]) 126 | aotMeanList=[] 127 | aotStdArr=np.array([]) 128 | aotStdList=[] 129 | aotArr40=[] 130 | latArr40=[] 131 | lonArr40=[] 132 | nameStationAux=[] 133 | #-------------------------------------- 134 | 135 | 136 | 137 | #-------------------------------------- 138 | # 139 | 140 | lenPathList, aerosolFilePathList=aotLib.getAotAquaFileList(filePath) 141 | 142 | #aerosolFilePathList=['/Users/josedias/Documents/aotData/aqua/Nasa201410/MYD04_L2.A2014279.1830.051.2014283080904.h5'] 143 | 144 | #for i in range(len(aerosolFilePathList)): 145 | # print aerosolFilePathList[i] 146 | 147 | 148 | for aotFilePath in aerosolFilePathList: 149 | 150 | print aotFilePath 151 | fileName=aotFilePath.split('/')[-1] 152 | 153 | 154 | if rec == 'dsa': 155 | idFileName=fileName.split('.')[1] 156 | #print idFileName 157 | 158 | year='20'+idFileName[:2] 159 | julianDay=idFileName[2:5] 160 | hour=idFileName[5:7] 161 | minute=idFileName[7:9] 162 | 163 | else: 164 | 165 | idFileName=fileName.split('.') 166 | #print idFileName 167 | 168 | year=idFileName[1][1:5] 169 | julianDay=idFileName[1][5:] 170 | hour=idFileName[2][:2] 171 | minute=idFileName[2][2:] 172 | 173 | 174 | decimalTime=aotLib.getDecimalTimeMinute(minute,hour) 175 | time=float(julianDay)+round(decimalTime,5) 176 | 177 | #print year, time 178 | # print nameStationArr 179 | 180 | 181 | for nameStation in range(len(nameStationArr)): 182 | 183 | # print nameStationArr[nameStation] 184 | try: 185 | aotMean,aotStd,aotMatrix,latMatrix,lonMatrix=aotProcess(filePath,aotFilePath,\ 186 | str(time),aeronetLatArr[nameStation],aeronetLonArr[nameStation]) 187 | 188 | 189 | #print aotMean 190 | 191 | if aotMean >=0: 192 | nameStationAux=nameStationAux+[nameStationArr[nameStation]] 193 | aotArr40=aotArr40+[aotMatrix] 194 | latArr40=latArr40+[latMatrix] 195 | lonArr40=lonArr40+[lonMatrix] 196 | 197 | else: 198 | pass 199 | 200 | except: 201 | # print "I/O error({0}): {1}".format(e.errno, e.strerror) 202 | 203 | 204 | # except: 205 | print 'Error' 206 | pass 207 | #-------------------------------------- 208 | 209 | 210 | #-------------------------------------- 211 | # write data in hdf5 file 212 | # 213 | # nameStationAux=['Santiago_eauchef'] 214 | 215 | try: 216 | 217 | aotLib.writeData(outputAquaDataPath,satellite,nameStationAux,year,time,aotArr40,latArr40,lonArr40) 218 | timeArr=np.append(timeArr,time) 219 | 220 | print(Fore.YELLOW + 'DATA WROTE') 221 | print(Styele.RESET_ALL) 222 | 223 | 224 | except: 225 | print(Fore.RED + 'NO DATA TO WROTE') 226 | print(Style.RESET_ALL) 227 | 228 | #print outputAquaDataPath,satellite,nameStationAux,year,time,aotArr40,latArr40,lonArr40 229 | #-------------------------------------- 230 | #aotMeanArr=aotMeanArr.reshape(len(timeArr),3)#numberStation) 231 | #print aotMeanArr 232 | # aotMeanList=aotMeanList+[aotMeanArr] 233 | # aotStdList=aotStdList+[aotStdArr] 234 | aotArr40=[] 235 | latArr40=[] 236 | lonArr40=[] 237 | nameStationAux=[] 238 | 239 | try: 240 | 241 | aotLib.writeDataTime(outputAquaDataPath,year,timeArr,satellite,processId) 242 | 243 | print(Fore.YELLOW + 'TIME WROTE') 244 | print(Styele.RESET_ALL) 245 | 246 | except: 247 | 248 | print(Fore.RED + 'NO TIME TO WRITE') 249 | print(Style.RESET_ALL) 250 | -------------------------------------------------------------------------------- /aeroPyVirrsEdr.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python 2 | 3 | import numpy as np 4 | import h5py as h5 5 | import gc 6 | import aotLib 7 | from colorama import Fore, Back, Style 8 | #from aotConf import edrNppPath 9 | from aotConf import aeronetPath 10 | from sys import argv 11 | 12 | def aotProcess(filePath,gaeroFile,vaoooFile,julianDay,aeronetLat,aeronetLon): 13 | 14 | date=vaoooFile[56:64] 15 | vaooo=h5.File(vaoooFile,'r') 16 | factors=np.array(vaooo['/All_Data/VIIRS-Aeros-EDR_All/AerosolOpticalDepthFactors']) 17 | aeroOptDep=np.array(vaooo['/All_Data/VIIRS-Aeros-EDR_All/AerosolOpticalDepth_at_550nm']) 18 | qualFlag1=np.array(vaooo['/All_Data/VIIRS-Aeros-EDR_All/QF1_VIIRSAEROEDR']) 19 | qualFlag4=np.array(vaooo['/All_Data/VIIRS-Aeros-EDR_All/QF4_VIIRSAEROEDR']) 20 | vaooo.close() 21 | del vaooo 22 | 23 | #--data cleaner----------- 24 | aeroOptDep10=aeroOptDep*factors[0]+factors[1] 25 | #------------------------- 26 | 27 | del aeroOptDep 28 | gc.collect() 29 | 30 | gaero=h5.File(gaeroFile,'r') 31 | lat=np.array(gaero['/All_Data/VIIRS-Aeros-EDR-GEO_All/Latitude']) 32 | lon=np.array(gaero['/All_Data/VIIRS-Aeros-EDR-GEO_All/Longitude']) 33 | gaero.close() 34 | 35 | del gaero 36 | gc.collect() 37 | 38 | #-----aeronet position-------- 39 | #aeronet= -10.0,-54.5 40 | #aeronet=-15.729500,-56.021000 41 | earthRay= 6371 ## km 42 | length= 40 ## km 43 | 44 | beginLatReg,endLatReg,beginLonReg,endLonReg=aotLib.getComparisonRegion(aeronetLat,aeronetLon,length,earthRay) 45 | rowMin,rowMax,colMin,colMax=aotLib.getRegionIndexMatrix(beginLatReg,endLatReg,lat,beginLonReg,endLonReg,lon) 46 | qualFlag1=aotLib.getNewMatrix(rowMin,rowMax,colMin,colMax,qualFlag1) 47 | aeroOptDep10=aotLib.getNewMatrix(rowMin,rowMax,colMin,colMax,aeroOptDep10) 48 | lat=aotLib.getNewMatrix(rowMin,rowMax,colMin,colMax,lat) 49 | lon=aotLib.getNewMatrix(rowMin,rowMax,colMin,colMax,lon) 50 | qualFlag4=aotLib.getNewMatrix(rowMin,rowMax,colMin,colMax,qualFlag4) 51 | 52 | # aeroOptDep10=aotLib.RemoveBadData2(qualFlag1,aeroOptDep10,'10','00') 53 | aeroOptDep10=aotLib.removeBadDataBitwise(qualFlag1,aeroOptDep10,3,3) 54 | 55 | print aeroOptDep10.mean() 56 | 57 | # #--do aotLib.plot2---------------- 58 | # #---coordenates around station 59 | # 60 | # length=100 #len 61 | # coordenates=aotLib.getComparisonRegion(aeronetLat,aeronetLon,length,earthRay) 62 | # 63 | # beginLatFig,endLatFig=coordenates[0],coordenates[1] 64 | # beginLonFig,endLonFig=coordenates[2],coordenates[3] 65 | # figName=(filePath+julianDay+'bit10') 66 | # title=('AOT em NPP bit 10 '+date) 67 | # aotLib.doPlot2(lon,lat,aeroOptDep10,title,figName,beginLatFig,endLatFig,beginLonFig,endLonFig)#,\ 68 | # #beginLatReg,endLatReg,beginLonReg,endLonReg) 69 | 70 | gc.collect() 71 | return aeroOptDep10.mean(),aeroOptDep10.std(),aeroOptDep10,lat,lon,qualFlag4,qualFlag1 72 | #-------------------------------------- 73 | 74 | 75 | #################################### 76 | # main code 77 | # 78 | #----------------------------------- 79 | # edr work path 80 | # 81 | yearMonth=argv[1] 82 | 83 | #satellite='edrNpp' 84 | satellite='edrNppNasa' 85 | 86 | if satellite == 'edrNppNasa': 87 | from aotConf import edrNppPathNasa 88 | inputNppDataPath,outputNppDataPath=edrNppPathNasa() 89 | 90 | else: 91 | from aotConf import edrNppPath 92 | inputNppDataPath,outputNppDataPath=edrNppPath() 93 | 94 | filePath=inputNppDataPath+'/'+yearMonth+'/' 95 | pathOut=outputNppDataPath 96 | 97 | 98 | processId=argv[2] 99 | #---------------------------------- 100 | 101 | print filePath 102 | #---------------------------------- 103 | # geometric paramenters 104 | # 105 | nameStationArr=[] 106 | aeronetLatArr=[] 107 | aeronetLonArr=[] 108 | inputAeronetDataPath,outputAeronetDataPath=aeronetPath() 109 | 110 | dataLev = argv[3] 111 | 112 | if dataLev == 'lev20': 113 | filePathArr=aotLib.getAeronetPathArr(inputAeronetDataPath + '/allDataL2',dataLev) 114 | 115 | else: 116 | dataLev = 'lev15' 117 | filePathArr=aotLib.getAeronetPathArr(inputAeronetDataPath + '/allDataL15',dataLev) 118 | 119 | #print 'arr',inputAeronetDataPath 120 | nameStationArr,aeronetLatArr,aeronetLonArr=aotLib.returnAeronetCoordenateArr(filePathArr) 121 | #---------------------------------- 122 | 123 | #-------------------------------------- 124 | # array variables 125 | # 126 | lenList=[] 127 | gaeroFileList=[] 128 | vaoooFileList=[] 129 | timeArr=np.array([]) 130 | aotMeanArr=np.array([]) 131 | aotMeanList=[] 132 | aotStdArr=np.array([]) 133 | aotStdList=[] 134 | aotArr40=[] 135 | latArr40=[] 136 | lonArr40=[] 137 | aotModel=[] 138 | qFlagMat=[] 139 | nameStationAux=[] 140 | #-------------------------------------- 141 | 142 | #-------------------------------------- 143 | # it is used to retrieve all available 144 | # data 145 | if satellite == 'edrNppNasa': 146 | lenList,gaeroFileList,vaoooFileList=aotLib.getEdrAotFileListNasa(filePath) 147 | # print 'here' 148 | 149 | else: 150 | lenList,gaeroFileList,vaoooFileList=aotLib.getEdrAotFileList(filePath) 151 | 152 | print lenList 153 | 154 | for i in range(lenList): 155 | 156 | print vaoooFileList[i] 157 | 158 | fileName=vaoooFileList[i].split('/')[-1] 159 | fildName=fileName.split('_') 160 | 161 | year=fildName[2][1:5] 162 | month=fildName[2][5:7] 163 | day=fildName[2][7:] 164 | hour=fildName[3][1:3] 165 | minute=fildName[3][3:5] 166 | 167 | julianDay=aotLib.getJulianDay(year,month,day) 168 | decimalTime=aotLib.getDecimalTimeMinute(minute,hour) 169 | time=julianDay+round(decimalTime,5) 170 | # print 'time', time 171 | 172 | for nameStation in range(len(nameStationArr)): 173 | # for nameStation in range(13,14): 174 | 175 | 176 | # print nameStationArr[nameStation] 177 | try: 178 | aotMean, aotStd, aotMatrix, latMatrix, lonMatrix, aotModelMatrix,flagMatrix = aotProcess( 179 | filePath, gaeroFileList[i], vaoooFileList[i], str(time), 180 | aeronetLatArr[nameStation], aeronetLonArr[nameStation]) 181 | 182 | 183 | if aotMean >= 0: 184 | nameStationAux = nameStationAux + [nameStationArr[nameStation]] 185 | aotArr40 = aotArr40 + [aotMatrix] 186 | latArr40 = latArr40 + [latMatrix] 187 | lonArr40 = lonArr40 + [lonMatrix] 188 | aotModel = aotModel + [aotModelMatrix] 189 | qFlagMat = qFlagMat + [flagMatrix] 190 | 191 | 192 | # print time,aotMean 193 | else: 194 | pass 195 | 196 | except: 197 | 198 | print 'Error' 199 | # pass 200 | #-------------------------------------- 201 | # print year 202 | 203 | #-------------------------------------- 204 | # write data in hdf5 file 205 | # 206 | try: 207 | aotLib.writeDataEdr(pathOut,satellite,nameStationAux,year,time,aotArr40,latArr40,lonArr40,aotModel, qFlagMat) 208 | timeArr=np.append(timeArr,time) 209 | 210 | print(Fore.YELLOW + 'DATA WROTE') 211 | print(Style.RESET_ALL) 212 | 213 | 214 | except: 215 | print(Fore.RED + 'NO DATA TO WRITE') 216 | print(Style.RESET_ALL) 217 | #-------------------------------------- 218 | #aotMeanArr=aotMeanArr.reshape(len(timeArr),3)#numberStation) 219 | #print aotMeanArr 220 | # aotMeanList=aotMeanList+[aotMeanArr] 221 | # aotStdList=aotStdList+[aotStdArr] 222 | aotArr40=[] 223 | latArr40=[] 224 | lonArr40=[] 225 | nameStationAux=[] 226 | qFlagMat=[] 227 | 228 | #print year 229 | 230 | try: 231 | aotLib.writeDataTime(pathOut,year,timeArr,satellite,processId) 232 | 233 | print(Fore.YELLOW + 'TIME WROTE') 234 | print(Style.RESET_ALL) 235 | 236 | except: 237 | print(Fore.RED + 'NO TIME TO WRITE') 238 | print(Style.RESET_ALL) 239 | 240 | -------------------------------------------------------------------------------- /aotLib.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import datetime 4 | import math 5 | from pandas import Series, DataFrame 6 | from matplotlib.patches import Polygon 7 | from mpl_toolkits.basemap import Basemap 8 | from scipy.optimize import curve_fit 9 | import glob 10 | import pandas as pd 11 | import h5py as h5 12 | 13 | 14 | #################### SATELLITE ##################### 15 | 16 | 17 | #------------------------------------------ 18 | #A is quality flag matrix, B is AOT matrix, binNumber is bit group 19 | #flag 20 | def RemoveBadData(A,B,binNumber): 21 | 22 | intNumber=np.uint8(binNumber) 23 | 24 | indexBadData=np.argwhere(A!=intNumber) 25 | 26 | for i in indexBadData: 27 | B[i[0],i[1]]=-1. 28 | B = np.ma.masked_where(B<0,B) 29 | #np.ma.set_fill_value(B,np.nan) 30 | return B 31 | #------------------------------------------ 32 | 33 | 34 | #------------------------------------------ 35 | #'A' is quality flag matrix. 'B' is AOT matrix. 'bit' is pair of 36 | # numbers that cam be 76, 54, 32 e 10. 'valBit' is value associated 37 | # to 'bit' 38 | def RemoveBadData2(A,B,bit,valBit): 39 | #varBit=str(78-bit) 40 | varBit0,varBit1=7-int(bit[0]),8-int(bit[1]) 41 | badMatrixData=[] 42 | for i in range(A.shape[0]): 43 | for j in range(A.shape[1]): 44 | binNumber='{0:08b}'.format(A[i,j],2) 45 | 46 | bit76=binNumber[0:2] 47 | bit54=binNumber[2:4] 48 | bit32=binNumber[4:6] 49 | bit10=binNumber[6:8] 50 | 51 | #if binNumber[varBit0:varBit1]==valBit: 52 | # print binNumber+' bit10: '+str(bit10)+' bit32: '+str(bit32)+' bit54: '+str(bit54)+' bit76: '+str(bit76)+' dado bom' 53 | 54 | if binNumber[varBit0:varBit1]!=valBit: 55 | badMatrixData+=[[i,j]] 56 | # print binNumber+' bit10: '+str(bit10)+' bit32: '+str(bit32)+' bit54: '+str(bit54)+' bit76: '+str(bit76) 57 | 58 | badMatrixData=np.array(badMatrixData) 59 | for i in badMatrixData: 60 | B[i[0],i[1]]=-1. 61 | B = np.ma.masked_where(B<0,B) 62 | print 'end bit '+bit 63 | return B 64 | 65 | 66 | def removeBadDataBitwise(A,B,maskBit,valBit): 67 | 68 | # for index in np.argwhere(A & 3 !=0): 69 | # B[index[0],index[1]]=-1 70 | # B = np.ma.masked_where(A & maskBit !=valBit,B) 71 | 72 | #print 'end bit' 73 | # return B 74 | return np.ma.masked_where(A & maskBit !=valBit,B) 75 | # return np.ma.masked_array(B,A & maskBit != valBit) 76 | 77 | 78 | #------------------------------------------ 79 | 80 | 81 | #------------------------------------------ 82 | # this function return a index list for values between two values 83 | # ('begin' and 'end') in 2d array 84 | 85 | def getValBetAt2dArr(begin,end,Arr): 86 | 87 | indexBegin=np.argwhere(Arr>=begin) 88 | indexEnd=np.argwhere(Arr<=end) 89 | listBegin=[] 90 | listEnd=[] 91 | 92 | for i in indexBegin: 93 | listBegin+=[str(i[0])+','+str(i[1])] 94 | #print Arr[i[0],i[1]] 95 | #print latListBegin 96 | for i in indexEnd: 97 | listEnd+=[str(i[0])+','+str(i[1])] 98 | 99 | #print listBegin,listEnd 100 | interBeginEnd=np.intersect1d(listBegin,listEnd) 101 | 102 | # print interBeginEnd 103 | return interBeginEnd 104 | #------------------------------------------ 105 | 106 | 107 | #------------------------------------------ 108 | # this function return a index list for values between two values 109 | # ('begin' and 'end') in 2d array 110 | 111 | def getValBetAt2dArrB(rowMina,colMina,begin,end,Arr): 112 | 113 | indexBegin=np.argwhere(Arr>=begin) 114 | indexEnd=np.argwhere(Arr<=end) 115 | listBegin=[] 116 | listEnd=[] 117 | 118 | for i in indexBegin: 119 | listBegin+=[str(i[0]+rowMina)+','+str(i[1]+colMina)] 120 | #print Arr[i[0],i[1]] 121 | #print latListBegin 122 | for i in indexEnd: 123 | listEnd+=[str(i[0]+rowMina)+','+str(i[1]+colMina)] 124 | 125 | #print listBegin,listEnd 126 | interBeginEnd=np.intersect1d(listBegin,listEnd) 127 | 128 | # print interBeginEnd 129 | return interBeginEnd 130 | #------------------------------------------ 131 | 132 | 133 | #------------------------------------------ 134 | #this function creat a color screen over an area 135 | # 136 | def drawScreen(beginLat,endLat,beginLon,endLon,m): 137 | lats = [endLat,beginLat,beginLat,endLat]#beginLat,endLat,endLat,beginLat 138 | lons = [endLon,endLon,beginLon,beginLon]#beginLon,beginLon,endLon,endLon 139 | x, y = m( lons, lats ) 140 | xy = zip(x,y) 141 | poly = Polygon( xy, facecolor='red', alpha=0.5 ) 142 | plt.gca().add_patch(poly) 143 | #----------------------------------------- 144 | 145 | 146 | #----------------------------------------- 147 | #this function do average in 2d array using index list 148 | #variables 'lat' and 'lon' are array 149 | def regionAverage(latBegin,latEnd,lat,lonBegin,lonEnd,lon,A): 150 | sumation=np.array([]) 151 | #print A 152 | #value=np.array([]) 153 | #get intersection between two 2d array (lat and lon) 154 | interLatBeginEnd=getValBetAt2dArr(latBegin,latEnd,lat) 155 | interLonBeginEnd=getValBetAt2dArr(lonBegin,lonEnd,lon) 156 | interLatLon=np.intersect1d(interLatBeginEnd,interLonBeginEnd) 157 | #print interLatLon 158 | #----do average of values 159 | for i in interLatLon: 160 | i0,i2=i.split(',') 161 | sumation=np.append(sumation,A[i0,i2]) 162 | value=np.sort(sumation) 163 | #print 'size', sumation.size 164 | #print value 165 | #print sumation 166 | average=np.mean(sumation) 167 | #print sumAot, len(interLatLon), averageAot 168 | return average,value#,interLatLon 169 | 170 | #----------------------------------------- 171 | 172 | 173 | #----------------------------------------- 174 | # 175 | # 176 | def getNewMatrix(rowMin,rowMax,colMin,colMax,matrix): 177 | 178 | matrix=matrix[rowMin:rowMax+1,colMin:colMax+1] 179 | 180 | return matrix 181 | #----------------------------------------- 182 | 183 | 184 | #----------------------------------------- 185 | # 186 | # 187 | def getMatrixCorner(arrayIntersec): 188 | 189 | row=np.array([]) 190 | col=np.array([]) 191 | 192 | #row=[] 193 | #col=[] 194 | 195 | for i in arrayIntersec: 196 | i0,i1=i.split(',') 197 | row=np.append(row,int(i0)) 198 | col=np.append(col,int(i1)) 199 | #row=row+[i0] 200 | #col=col+[i1] 201 | 202 | #row=np.array(row,dtype='int') 203 | #col=np.array(col,dtype='int') 204 | 205 | row=np.sort(row) 206 | col=np.sort(col) 207 | 208 | return int(row[0]),int(row[-1]),int(col[0]),int(col[-1]) 209 | 210 | #return row.min(),row.max(),col.min(),col.max() 211 | #----------------------------------------- 212 | 213 | 214 | #----------------------------------------- 215 | # 216 | # 217 | def getRegionIndexMatrixOld(latBegin,latEnd,lat,lonBegin,lonEnd,lon): 218 | 219 | try: 220 | interLatBeginEnd=getValBetAt2dArr(latBegin,latEnd,lat) 221 | #print interLatBeginEnd 222 | except: 223 | print 'out of latitude' 224 | 225 | rowMina,rowMaxa,colMina,colMaxa=getMatrixCorner(interLatBeginEnd) 226 | 227 | if(lonBegin>=lon[rowMaxa][colMaxa] and lonEnd<=lon[rowMina][colMina]): 228 | 229 | lon=getNewMatrix(rowMina,rowMaxa,colMina,colMaxa,lon) 230 | 231 | try: 232 | interLonBeginEnd=getValBetAt2dArrB(rowMina,colMina,lonBegin,lonEnd,lon) 233 | #print interLonBeginEnd 234 | interLatLon=np.intersect1d(interLatBeginEnd,interLonBeginEnd) 235 | 236 | except: 237 | print 'out of longitude' 238 | 239 | try: 240 | rowMin,rowMax,colMin,colMax=getMatrixCorner(interLatLon) 241 | 242 | except: 243 | print 'without longitude values' 244 | 245 | else: 246 | print 'out of longitude 2' 247 | 248 | try: 249 | return rowMin,rowMax,colMin,colMax 250 | 251 | except: 252 | print 'no values' 253 | #----------------------------------------- 254 | 255 | 256 | #----------------------------------------- 257 | # 258 | # 259 | def getRegionIndexMatrix(latBegin,latEnd,lat,lonBegin,lonEnd,lon): 260 | 261 | try: 262 | 263 | ind=((lat>=latBegin) & (lat<=latEnd) & (lon>=lonBegin) & (lon<=lonEnd)) 264 | ind=np.argwhere(ind==True) 265 | 266 | rowMax=ind[:,0].max() 267 | rowMin=ind[:,0].min() 268 | 269 | colMax=ind[:,1].max() 270 | colMin=ind[:,1].min() 271 | 272 | #ind=ind.transpose() 273 | #rowMax=ind[0].max() 274 | #rowMin=ind[0].min() 275 | 276 | #colMax=ind[1].max() 277 | #colMin=ind[1].min() 278 | 279 | except: 280 | print 'out of lat lon' 281 | return None 282 | 283 | return rowMin,rowMax,colMin,colMax 284 | 285 | #----------------------------------------- 286 | 287 | 288 | #----------------------------------------- 289 | def getJulianDay(strYear,strMonth,strDay): 290 | 291 | # strDay=line[9:11] 292 | # strMonth=line[6:8] 293 | # strYear=line[1:5] 294 | strTime=strYear+" "+strMonth+" "+strDay 295 | julianDay=datetime.datetime.strptime(strTime, '%Y %m %d').strftime('%j') 296 | #print strDay, strMonth, strYear, julianDay 297 | #print julianDay 298 | return float(julianDay) 299 | #----------------------------------------- 300 | 301 | 302 | 303 | #----------------------------------------- 304 | # 305 | # 306 | def getDecimalTimeMinute(minute,hour): 307 | 308 | decimalMinute=float(minute)/60.0**2 309 | decimalTime=float(hour)/24.0+decimalMinute 310 | 311 | #print hourNumber, minuteNumber, decimalMinute, decimalTime 312 | return(decimalTime) 313 | #----------------------------------------- 314 | 315 | 316 | #----------------------------------------- 317 | #this function get a list of npp files to 318 | #plot aot npp 319 | # 320 | def getAotFileList(filePath): 321 | 322 | # print filePath 323 | pathIvaotList=[] 324 | pathGmtcoList=[] 325 | pathIvaot=filePath+'IVAOT*.h5' 326 | pathList=glob.glob(pathIvaot) 327 | for ivaotFile in pathList: 328 | 329 | dateTime=ivaotFile[55:82] 330 | pathGmtco=glob.glob(filePath+'GMTCO*'+dateTime+'*.h5') 331 | 332 | if len(pathGmtco)<2: 333 | #print dateTime, len(d) 334 | #print ivaotFile 335 | #print pathGmtco[0] 336 | 337 | # if len(pathGmtco)<1: 338 | # break 339 | try: 340 | pathGmtcoList.append(pathGmtco[0]) 341 | pathIvaotList.append(ivaotFile) 342 | 343 | except: 344 | pass 345 | 346 | # print pathGmtcoList 347 | return len(pathGmtcoList),pathGmtcoList,pathIvaotList 348 | #----------------------------------------- 349 | 350 | 351 | #----------------------------------------- 352 | #this function remove zero values from AOT 353 | #array 354 | 355 | def removeZeroValues(arr): 356 | index=np.argwhere(arr==0.) 357 | arr=np.delete(arr,index) 358 | return arr 359 | 360 | #----------------------------------------- 361 | 362 | 363 | 364 | #----------------------------------------- 365 | #Function to get cordenates using Aeronte 366 | #lat and long reference 367 | 368 | def getComparisonRegion(lat,lon,length,ray): 369 | 370 | beginLatReg=lat-length*180/(2.*ray*math.pi) 371 | endLatReg=lat+length*180/(2.*ray*math.pi) 372 | beginLonReg=lon-length*180/(2.*ray*math.pi) 373 | endLonReg=lon+length*180/(2.*ray*math.pi) 374 | 375 | return beginLatReg,endLatReg,beginLonReg,endLonReg 376 | #----------------------------------------- 377 | 378 | 379 | #----------------------------------------- 380 | #This function do a plot of AOT data 381 | # 382 | def doPlotHistogram(valueArr): 383 | 384 | plt.hist(valueArr,12,normed=1,facecolor='green',alpha=0.5) 385 | plt.show() 386 | 387 | 388 | #----------------------------------------- 389 | 390 | 391 | #----------------------------------------- 392 | #This function do a plot of AOT data 393 | # 394 | def doPlot2(lon,lat,aeroOptDep,title,figName,beginLatFig=-60,endLatFig=20,beginLonFig=-96,endLonFig=-13,\ 395 | beginLatReg=0,endLatReg=0,beginLonReg=0,endLonReg=0): 396 | 397 | m=Basemap(projection='cyl',resolution='l',llcrnrlat=beginLatFig,\ 398 | urcrnrlat=endLatFig,llcrnrlon=beginLonFig,urcrnrlon=endLonFig) 399 | #lon,lat=m(lon,lat) 400 | 401 | print figName 402 | parallels = np.arange(beginLatFig,endLatFig,(endLatFig-beginLatFig)/8.) 403 | m.drawparallels(parallels,labels=[1,0,0,1]) 404 | meridians = np.arange(beginLonFig,endLonFig,(endLonFig-beginLonFig)/4.) 405 | m.drawmeridians(meridians,labels=[1,0,0,1]) 406 | m.drawcountries(linewidth=0.5) 407 | m.drawcoastlines(linewidth=0.5) 408 | m.pcolormesh(lon,lat,aeroOptDep,vmin=0.,vmax=1) 409 | if beginLatReg==0 and endLatReg==0 and beginLonReg==0 and\ 410 | endLonReg==0: 411 | pass 412 | 413 | else: 414 | drawScreen(beginLatReg,endLatReg,beginLonReg,endLonReg,m) 415 | #print beginLatReg,endLatReg,beginLonReg,endLonReg 416 | cb=m.colorbar() 417 | plt.title(title) 418 | plt.savefig(figName+'.png',format='png',dpi=150) 419 | plt.clf() 420 | #plt.show() 421 | #------------------------- 422 | 423 | ##################### MODIS ######################### 424 | 425 | 426 | def getAotAquaFileList(filePath): 427 | 428 | aerosolList=[] 429 | 430 | try: 431 | pathAerosol=filePath+'AEROSOL*.h5' 432 | 433 | except: 434 | print('Without AEROSOL*.h5\n') 435 | print('Searching for MYD04*.h5') 436 | 437 | try: 438 | 439 | pathAerosol=filePath+'MYD04*.h5' 440 | except: 441 | print('Wthiout AOT files') 442 | 443 | pathList=glob.glob(pathAerosol) 444 | 445 | return len(pathList), pathList 446 | 447 | def removeValues(arg,lat,lon,aot): 448 | inter=np.argwhere(lat==arg) 449 | line=np.array([],dtype=int) 450 | 451 | for i in inter: 452 | line=np.append(line,i[0]) 453 | 454 | line=np.unique(line) 455 | 456 | if line.size == 0: 457 | parameter=0 458 | #print 'empty' 459 | 460 | elif line.size == 1: 461 | if line[0] lat.shape[0]/2: 484 | lat=lat[:line[-1]] 485 | lon=lon[:line[-1]] 486 | aot=aot[:line[-1]] 487 | #print '4' 488 | parameter=1 489 | 490 | else: 491 | lat=lat[line[0]+1:line[-1]] 492 | lon=lon[line[0]+1:line[-1]] 493 | aot=aot[line[0]+1:line[-1]] 494 | #print '5' 495 | parameter=1 496 | 497 | return lat,lon,aot,parameter 498 | 499 | 500 | 501 | 502 | 503 | 504 | 505 | 506 | #################### AERONET ######################## 507 | 508 | #------------------------------------------ 509 | #this funtion get aeronet Dic file 510 | # 511 | def getAeronetDirectoryFile(aeronetPathDir): 512 | 513 | try: 514 | aeronetFile=aeronetPathDir.split('/')[-1].split('.')[0] 515 | #print aeronetPathList[0] 516 | date=aeronetFile[0:14] 517 | stationName=aeronetFile[14:] 518 | #print stationName, date 519 | return stationName, date 520 | 521 | except: 522 | print 'was not possible to open' 523 | #------------------------------------------ 524 | 525 | 526 | #------------------------------------------ 527 | #this funcition get aeronet coordenate from 528 | #.lev15 data table 529 | def getAeronetCoordenates(dataTable): 530 | geome=(dataTable.loc[[0],:])[0][0] 531 | geome=geome.split(',') 532 | station=geome[0].split('=')[1];# print station 533 | lon=float(geome[1].split('=')[1]);# print lon 534 | lat=float(geome[2].split('=')[1]);# print lat 535 | 536 | return station, lat, lon 537 | #------------------------------------------ 538 | 539 | 540 | def getAeronetPathArr(inputAeronetDataPath, dataLev): 541 | 542 | filePathArr=[] 543 | 544 | aeronetPathList=glob.glob(inputAeronetDataPath+'/*.'+dataLev) 545 | 546 | for dirStation in aeronetPathList: 547 | stationName,date=getAeronetDirectoryFile(dirStation) 548 | filePath=inputAeronetDataPath+'/'+date+stationName+'.'+dataLev 549 | filePathArr.append(filePath) 550 | 551 | return filePathArr 552 | 553 | def returnAeronetCoordenateArr(filePathArr): 554 | 555 | stationNameArr=[] 556 | latArr=[] 557 | lonArr=[] 558 | 559 | for aeronetFilePath in filePathArr: 560 | dataTable=pd.read_table(aeronetFilePath,skiprows=2,header=None) 561 | stationName,lat,lon=getAeronetCoordenates(dataTable) 562 | if '_' in stationName: 563 | stationName=stationName.replace('_','_') 564 | if '-' in stationName: 565 | stationName=stationName.replace('-','_') 566 | stationNameArr.append(stationName) 567 | latArr.append(lat) 568 | lonArr.append(lon) 569 | 570 | return stationNameArr, latArr, lonArr 571 | 572 | 573 | def getAeronetDataTable(filePath): 574 | 575 | table=pd.read_table(filePath,skiprows=4,header=None) 576 | headerN=(table.loc[[0],:])[0][0] 577 | headerN=headerN.replace('-','_').replace('(','_').replace(')','_').replace('/','_').replace('[','_').replace(']','_').replace(':','_') 578 | headerN=headerN.split(',') 579 | table=pd.read_table(filePath,skiprows=5,sep=',',header=None,names=headerN) 580 | 581 | return table,headerN[3:19] 582 | 583 | def writeAeroData(pathOut,device,satellite,nameStation,year,time,aotAeroMean,aotAeroStd,wLen): 584 | 585 | aotOut=h5.File(pathOut+'/'+'aotDataSet.h5') 586 | 587 | try: 588 | del aotOut[device+'/'+satellite+'/'+nameStation+'/'+year+'/aotMean/aotMean'+wLen] 589 | except: 590 | print 'without '+nameStation+' aotMean' 591 | 592 | try: 593 | del aotOut[device+'/'+satellite+'/'+nameStation+'/'+year+'/aotStd/aotStd'+wLen] 594 | except: 595 | print 'without '+nameStation+' aotStd' 596 | 597 | try: 598 | del aotOut[device+'/'+satellite+'/'+nameStation+'/'+year+'/aotTime/aotTime'+wLen] 599 | except: 600 | print 'without '+nameStation+' time' 601 | 602 | aotOut[device+'/'+satellite+'/'+nameStation+'/'+year+'/aotMean/aotMean'+wLen]=aotAeroMean 603 | aotOut[device+'/'+satellite+'/'+nameStation+'/'+year+'/aotStd/aotStd'+wLen]=aotAeroStd 604 | aotOut[device+'/'+satellite+'/'+nameStation+'/'+year+'/aotTime/aotTime'+wLen]=time 605 | 606 | aotOut.close() 607 | 608 | def fitFunc2(lam,a,b,c,d): 609 | return (a/lam)+b**(c*lam)+d 610 | 611 | 612 | def getInterpolatedAot(table,aotLenHeader,line): 613 | aotValueArr=np.array([]) 614 | waveLenList=[] 615 | waveLenArr=([]) 616 | for aotWlen in aotLenHeader: 617 | #print aotWlen 618 | aotValue=np.array(DataFrame(table,columns=[aotWlen]).loc[[line]]) 619 | if aotValue>=0: 620 | aotValueArr=np.append(aotValueArr,aotValue) 621 | waveLen=aotWlen.split('_')[1] 622 | waveLenList.append(waveLen) 623 | waveLenArr=np.array(waveLenList,dtype='float64') 624 | 625 | fitParams, fitCovariances = curve_fit(fitFunc2, waveLenArr, aotValueArr) 626 | aot550=fitFunc2(550, fitParams[0], fitParams[1],fitParams[2],fitParams[3]) 627 | return aot550 628 | 629 | ##################### edrViirs data ########################### 630 | 631 | #----------------------------------------- 632 | #this function get a list of npp edr files to 633 | #plot aot edr npp 634 | # 635 | def getEdrAotFileList(filePath): 636 | 637 | pathVaoooList=[] 638 | pathGaeroList=[] 639 | pathVaooo=filePath+'VAOOO*.h5' 640 | pathList=glob.glob(pathVaooo) 641 | 642 | for vaoooFile in pathList: 643 | 644 | dateTime=vaoooFile[54:] 645 | pathGaero=glob.glob(filePath+'GAERO*'+dateTime) 646 | 647 | if len(pathGaero)<2: 648 | 649 | try: 650 | pathGaeroList.append(pathGaero[0]) 651 | pathVaoooList.append(vaoooFile) 652 | 653 | except: 654 | pass 655 | 656 | return len(pathGaeroList),pathGaeroList,pathVaoooList 657 | #----------------------------------------- 658 | 659 | 660 | #----------------------------------------- 661 | #this function get a list of npp edr files to 662 | #plot aot edr npp from nasa 663 | # 664 | def getEdrAotFileListNasa(filePath): 665 | 666 | pathGaeroList=[] 667 | pathGaero=filePath+'GAERO*.h5' 668 | pathGaeroList=glob.glob(pathGaero) 669 | 670 | return len(pathGaeroList),pathGaeroList,pathGaeroList 671 | #----------------------------------------- 672 | 673 | 674 | #----------------------------------------- 675 | # 676 | # 677 | # 678 | def writeDataEdr(pathOut,satellite,nameStation,year,time,aotArr40,latArr40,lonArr40,aotModel,qFlagMat): 679 | 680 | aotOut=h5.File(pathOut+'/'+'aotDataSet.h5') 681 | 682 | for i in range(len(nameStation)): 683 | 684 | try: 685 | del aotOut[satellite+'/'+nameStation[i]+'/'+year+'/aot40x40/'+str(time)+'aot40x40'] 686 | except: 687 | print 'without '+nameStation[i]+' '+str(time)+'aot40x40' 688 | 689 | try: 690 | del aotOut[satellite+'/'+nameStation[i]+'/'+year+'/lat40x40/'+str(time)+'lat40x40'] 691 | except: 692 | print 'without '+nameStation[i]+' '+str(time)+'lat40x40' 693 | 694 | try: 695 | del aotOut[satellite+'/'+nameStation[i]+'/'+year+'/lon40x40/'+str(time)+'lon40x40'] 696 | except: 697 | print 'without '+nameStation[i]+' '+str(time)+'lon40x40' 698 | 699 | try: 700 | del aotOut[satellite+'/'+nameStation[i]+'/'+year+'/aotModel/'+str(time)+'aotModel'] 701 | except: 702 | print 'without '+nameStation[i]+' '+str(time)+'aotModel' 703 | 704 | try: 705 | del aotOut[satellite+'/'+nameStation[i]+'/'+year+'/qFlagMat/'+str(time)+'qFlagMat'] 706 | except: 707 | print 'without '+nameStation[i]+' '+str(time)+'qFlagMat' 708 | 709 | 710 | 711 | 712 | print 'saida -> '+satellite+'/'+nameStation[i]+'/'+year+'/aot40x40/'+str(time)+'aot40x40' 713 | 714 | # print aotModel[i] 715 | 716 | aotOut[satellite+'/'+str(nameStation[i])+'/'+year+'/aot40x40/'+str(time)+'aot40x40']=aotArr40[i] 717 | aotOut[satellite+'/'+str(nameStation[i])+'/'+year+'/lat40x40/'+str(time)+'lat40x40']=latArr40[i] 718 | aotOut[satellite+'/'+str(nameStation[i])+'/'+year+'/lon40x40/'+str(time)+'lon40x40']=lonArr40[i] 719 | 720 | #print aotModel[i] 721 | aotOut[satellite+'/'+str(nameStation[i])+'/'+year+'/aotModel/'+str(time)+'aotModel']=aotModel[i] 722 | aotOut[satellite+'/'+str(nameStation[i])+'/'+year+'/qFlagMat/'+str(time)+'qFlagMat']=qFlagMat[i] 723 | 724 | 725 | aotOut.close() 726 | 727 | 728 | 729 | 730 | 731 | #----------------------------------------- 732 | 733 | 734 | ##################### general ########################### 735 | 736 | #------------------------------------------ 737 | # write file .h5 file 738 | # 739 | def writeData(pathOut,satellite,nameStation,year,time,aotArr40,latArr40,lonArr40): 740 | 741 | aotOut=h5.File(pathOut+'/'+'aotDataSet.h5') 742 | 743 | for i in range(len(nameStation)): 744 | 745 | try: 746 | del aotOut[satellite+'/'+nameStation[i]+'/'+year+'/aot40x40/'+str(time)+'aot40x40'] 747 | except: 748 | print 'without '+nameStation[i]+' '+str(time)+'aot40x40' 749 | 750 | try: 751 | del aotOut[satellite+'/'+nameStation[i]+'/'+year+'/lat40x40/'+str(time)+'lat40x40'] 752 | except: 753 | print 'without '+nameStation[i]+' '+str(time)+'lat40x40' 754 | 755 | try: 756 | del aotOut[satellite+'/'+nameStation[i]+'/'+year+'/lon40x40/'+str(time)+'lon40x40'] 757 | except: 758 | print 'without '+nameStation[i]+' '+str(time)+'lon40x40' 759 | 760 | print 'saida -> '+satellite+'/'+nameStation[i]+'/'+year+'/aot40x40/'+str(time)+'aot40x40' 761 | # print aotArr40[i] 762 | aotOut[satellite+'/'+str(nameStation[i])+'/'+year+'/aot40x40/'+str(time)+'aot40x40']=aotArr40[i] 763 | aotOut[satellite+'/'+str(nameStation[i])+'/'+year+'/lat40x40/'+str(time)+'lat40x40']=latArr40[i] 764 | aotOut[satellite+'/'+str(nameStation[i])+'/'+year+'/lon40x40/'+str(time)+'lon40x40']=lonArr40[i] 765 | 766 | 767 | aotOut.close() 768 | #-------------------------------------- 769 | 770 | 771 | def writeDataTime(pathOut,year,timeArr,satellite,process): 772 | 773 | aotOut=h5.File(pathOut+'/'+'aotDataSet.h5') 774 | 775 | try: 776 | del aotOut['time'+'/'+year+'/'+satellite+process] 777 | except: 778 | print 'without timeSerie' 779 | 780 | aotOut['time'+'/'+year+'/'+satellite+process]=timeArr 781 | 782 | aotOut.close() 783 | 784 | 785 | --------------------------------------------------------------------------------