├── LICENSE ├── README.md ├── SAR.py ├── addCovariates.py ├── assemblage.py ├── diffMedian.py ├── gapfilling.py ├── landsat.py ├── landsat8.py ├── landsatYearlyComposite.py ├── landsat_cloudScore.py ├── misc ├── annualMedian.py └── landsat_cloudScore.py ├── mosaicAndUpdate.py ├── paramsTemplate.py ├── rfClassification.py ├── sentineLandsat.py ├── sentinel2.py ├── sun_angles.py ├── utils.py └── view_angles.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Spatial Informatics Group, LLC 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Ecuador_SEPAL 2 | ------ 3 | This repository contains the preprocessing code for landsat-8 and and sentinel-2 to create biweekly composites. Both scripts are based upon the Google Earth engine (GEE). 4 | 5 | 6 | ## sentinel-2 7 | ------ 8 | Sentinel-2 data are available in Google Earth Engine in the “Sentinel-2 MSI: MultiSpectral Instrument, Level-1C” image collection (ID: “COPERNICUS/S2”). These data are available as top-of-atmosphere radiometrically-corrected imagery. 9 | 10 | Processing is done through: 11 | 12 | **Cloud Masking:** The QA60 band is used for cloud removal and a custom build cloud masking algorithm is used to remove clouds from the images. 13 | 14 | **Cloud shadow masking:** The Temporal Dark-Outlier Mask algorithm (TDOM) approachis used to remove cloud shadows. The algorithm identifies cloud shadows as pixels that are both dark in the infrared bands and that are dark statistical outliers across the time series. 15 | 16 | **BRDF correction:** bidirectional reflectance distribution function is applied to for the correction of view and illumination angle effects. 17 | 18 | **illumination correction:** correction for topographic effects on illumation. 19 | 20 | **Atmospheric correction:** Atmospheric corresion is done using the 6s emulator. The 6S emulator is an open-source atmospheric correction tool. Follow the steps on here (https://github.com/samsammurphy/6S_emulator ) to install the 6s emulator. 21 | 22 | ## Landsat-8 23 | ------ 24 | Landsat 8 data used to create the cloud free annual composites or biweekly mosaics are available in Google Earth Engine in the “USGS Landsat 8 Surface Reflectance Tier 1” image collection (ID: “LANDSAT/LC08/C01/T1_SR”). These data have been atmospherically corrected from top-of-atmosphere to surface reflectance using the Landsat 8 Surface Reflectance Code (LaSRC), produced by the U.S. Geological Survey (USGS). Surface Reflectance Landsat data is then corrected for topographic, radiometric, and atmospheric distortion; and clouds and cloud-shadows are masked. 25 | 26 | **Cloud Masking:** The QA60 band is used for cloud removal and a custom build cloud masking algorithm is used to remove clouds from the images. 27 | 28 | **Cloud shadow masking:** The Temporal Dark-Outlier Mask algorithm (TDOM) approachis used to remove cloud shadows. The algorithm identifies cloud shadows as pixels that are both dark in the infrared bands and that are dark statistical outliers across the time series. 29 | 30 | **BRDF correction:** bidirectional reflectance distribution function is applied to for the correction of view and illumination angle effects. 31 | 32 | **illumination correction:** correction for topographic effects on illumation. 33 | -------------------------------------------------------------------------------- /SAR.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import ee 4 | import math 5 | from utils import * 6 | 7 | class env(object): 8 | 9 | def __init__(self): 10 | """Initialize the environment""" 11 | self.studyArea = ee.FeatureCollection("projects/Sacha/AncillaryData/StudyRegions/Ecuador_EcoRegions_Complete") 12 | self.polar = ["VV","VH"] 13 | self.direction = 'ASCENDING'; 14 | self.startDate = ee.Date.fromYMD(2017,1,1) 15 | self.endDate = ee.Date.fromYMD(2017,12,31) 16 | 17 | self.ksize = 3 18 | self.enl = 7; 19 | 20 | 21 | class functions(): 22 | def __init__(self): 23 | """Initialize the Surfrace Reflectance app.""" 24 | 25 | # get the environment 26 | self.env = env() 27 | 28 | def main(self): 29 | collection = self.getSAR() 30 | 31 | def applySpeckleFilter(img): 32 | vv = img.select('VV'); 33 | vv = self.speckleFilter(img).rename(['VV']); 34 | 35 | if len(self.env.polar) > 1: 36 | vh = self.speckleFilter(img).rename(['VH']); 37 | return ee.Image.cat(vv,vh).copyProperties(img,['system:time_start']); 38 | 39 | else: 40 | return ee.Image.cat(vv).copyProperties(img,['system:time_start']); 41 | 42 | # Filter speckle using Gamma Map algorithm 43 | collection = collection.map(applySpeckleFilter) 44 | 45 | 46 | if len(self.env.polar) > 1: 47 | collection = collection.map(self.addRatio) 48 | 49 | print ee.Image(collection.first()).bandNames().getInfo() 50 | 51 | 52 | 53 | def getSAR(self): 54 | """ get the Sentinel 1 data collection""" 55 | # Inventory of S1 dual polarization images. 56 | collection = ee.ImageCollection('COPERNICUS/S1_GRD').filterBounds(self.env.studyArea)\ 57 | .filter(ee.Filter.eq('instrumentMode','IW'))\ 58 | .filter(ee.Filter.eq('orbitProperties_pass', self.env.direction))\ 59 | .filter(ee.Filter.eq('transmitterReceiverPolarisation', self.env.polar))\ 60 | .filter(ee.Filter.date(self.env.startDate, self.env.endDate)); 61 | 62 | return collection 63 | 64 | def speckleFilter(self,image): 65 | """ apply the speckle filter """ 66 | # Convert image from dB to natural values 67 | nat_img = self.toNatural(image); 68 | 69 | # Square kernel, ksize should be odd (typically 3, 5 or 7) 70 | weights = ee.List.repeat(ee.List.repeat(1,self.env.ksize),self.env.ksize); 71 | 72 | # ~~(ksize/2) does integer division in JavaScript 73 | kernel = ee.Kernel.fixed(self.env.ksize,self.env.ksize, weights, ~~(self.env.ksize/2), ~~(self.env.ksize/2), False); 74 | 75 | # Get mean and variance 76 | mean = nat_img.reduceNeighborhood(ee.Reducer.mean(), kernel); 77 | variance = nat_img.reduceNeighborhood(ee.Reducer.variance(), kernel); 78 | 79 | # "Pure speckle" threshold 80 | ci = variance.sqrt().divide(mean);# square root of inverse of enl 81 | 82 | # If ci <= cu, the kernel lies in a "pure speckle" area -> return simple mean 83 | cu = 1.0/math.sqrt(self.env.enl); 84 | 85 | # If cu < ci < cmax the kernel lies in the low textured speckle area 86 | # -> return the filtered value 87 | cmax = math.sqrt(2.0) * cu; 88 | 89 | alpha = ee.Image(1.0 + cu*cu).divide(ci.multiply(ci).subtract(cu*cu)); 90 | b = alpha.subtract(self.env.enl + 1.0); 91 | d = mean.multiply(mean).multiply(b).multiply(b).add(alpha.multiply(mean).multiply(nat_img).multiply(4.0*self.env.enl)); 92 | f = b.multiply(mean).add(d.sqrt()).divide(alpha.multiply(2.0)); 93 | 94 | # If ci > cmax do not filter at all (i.e. we don't do anything, other then masking) 95 | 96 | # Compose a 3 band image with the mean filtered "pure speckle", 97 | # the "low textured" filtered and the unfiltered portions 98 | out = ee.Image.cat(self.toDB(mean.updateMask(ci.lte(cu))),self.toDB(f.updateMask(ci.gt(cu)).updateMask(ci.lt(cmax))),image.updateMask(ci.gte(cmax))); 99 | 100 | return out.reduce(ee.Reducer.sum()); 101 | 102 | def addRatio(self,img): 103 | vv = self.toNatural(img.select('VV')).rename('VV'); 104 | vh = self.toNatural(img.select('VH')).rename('VH'); 105 | ratio = vh.divide(vv).rename('ratio'); 106 | return ee.Image.cat(vv,vh,ratio).copyProperties(img,['system:time_start']); 107 | 108 | 109 | def toNatural(self,img): 110 | """Function to convert from dB to natural""" 111 | return ee.Image(10.0).pow(img.select(0).divide(10.0)); 112 | 113 | def toDB(self,img): 114 | """ Function to convert from natural to dB """ 115 | return ee.Image(img).log10().multiply(10.0); 116 | 117 | if __name__ == "__main__": 118 | ee.Initialize() 119 | functions().main() 120 | 121 | 122 | -------------------------------------------------------------------------------- /addCovariates.py: -------------------------------------------------------------------------------- 1 | import ee 2 | import math 3 | 4 | ee.Initialize() 5 | class addCovariates(): 6 | def __init__(self): 7 | self.jrcImage = ee.Image("JRC/GSW1_0/GlobalSurfaceWater") 8 | self.elevation = ee.Image("USGS/SRTMGL1_003"); 9 | 10 | self.ndCovariatesList = [['blue', 'green'],['blue', 'red'],['blue', 'nir'],['blue', 'swir1'],['blue', 'swir2'], 11 | ['green', 'red'],['green', 'nir'],['green', 'swir1'],['green', 'swir2'],['red', 'swir1'],['red', 'swir2'], 12 | ['nir', 'red'],['nir', 'swir1'],['nir', 'swir2'],['swir1', 'swir2']] 13 | 14 | 15 | rCovariatesList = [['swir1', 'nir'],['red', 'swir1']]; 16 | 17 | def ComputeNDCovariatesList(self,season): 18 | l = []; 19 | for index in self.ndCovariatesList: 20 | currentIndex = self.ndCovariatesList.index(index) 21 | list_ = [season + self.ndCovariatesList[currentIndex][0], season + self.ndCovariatesList[currentIndex][1]]; 22 | l.append(list_); 23 | 24 | return l; 25 | 26 | 27 | def addNDCovariates(self, season, image): 28 | l = self.ComputeNDCovariatesList(season); 29 | for index in l: 30 | currentIndex = l.index(index) 31 | image = image.addBands(image.normalizedDifference(index).rename(season + 'ND_'+ self.ndCovariatesList[currentIndex][0] + '_' + self.ndCovariatesList[currentIndex][1])); 32 | 33 | return image; 34 | 35 | 36 | def ComputeRCovariatesList(season): 37 | l = []; 38 | for index in rCovariatesList: 39 | list_ = [season + rCovariatesList[index][0], season + rCovariatesList[index][1]]; 40 | l.append(list_); 41 | 42 | return l; 43 | 44 | 45 | def addRCovariates(season, image): 46 | l = ComputeRCovariatesList(season); 47 | for index in l: 48 | image = image.addBands(image.select(l[index][0]).divide(image.select(l[index][1])) 49 | .rename(season + '_R_' + rCovariatesList[index][0] + '_' + rCovariatesList[index][1])); 50 | 51 | return image; 52 | 53 | 54 | def addEVI(season, image): 55 | evi = image.expression('2.5 * ((NIR - RED) / (NIR + 6 * RED - 7.5 * BLUE + 1))', { 56 | 'NIR' : image.select(season + '_nir'), 57 | 'RED' : image.select(season + '_red'), 58 | 'BLUE': image.select(season + '_blue')}).float(); 59 | return image.addBands(evi.rename(season + '_EVI')); 60 | 61 | 62 | def addSAVI(season, image): 63 | #// Add Soil Adjust Vegetation Index (SAVI) 64 | #// using L = 0.5; 65 | savi = image.expression('(NIR - RED) * (1 + 0.5)/(NIR + RED + 0.5)', { 66 | 'NIR': image.select(season + '_nir'), 67 | 'RED': image.select(season + '_red')}).float(); 68 | return image.addBands(savi.rename(season + '_SAVI')); 69 | 70 | 71 | def addIBI(season, image): 72 | # // Add Index-Based Built-Up Index (IBI) 73 | ibiA = image.expression('2 * SWIR1 / (SWIR1 + NIR)', { 74 | 'SWIR1': image.select(season + '_swir1'), 75 | 'NIR' : image.select(season + '_nir') 76 | }).rename(['IBI_A']); 77 | 78 | ibiB = image.expression('(NIR / (NIR + RED)) + (GREEN / (GREEN + SWIR1))', { 79 | 'NIR' : image.select(season + '_nir'), 80 | 'RED' : image.select(season + '_red'), 81 | 'GREEN': image.select(season + '_green'), 82 | 'SWIR1': image.select(season + '_swir1') 83 | }).rename(['IBI_B']); 84 | 85 | ibiAB = ibiA.addBands(ibiB); 86 | ibi = ibiAB.normalizedDifference(['IBI_A', 'IBI_B']); 87 | return image.addBands(ibi.rename([season + '_IBI'])); 88 | 89 | 90 | # // Function to compute the Tasseled Cap transformation and return an image 91 | def getTassledCapComponents(season, image): 92 | 93 | coefficients = ee.Array([ 94 | [0.3037, 0.2793, 0.4743, 0.5585, 0.5082, 0.1863], 95 | [-0.2848, -0.2435, -0.5436, 0.7243, 0.0840, -0.1800], 96 | [0.1509, 0.1973, 0.3279, 0.3406, -0.7112, -0.4572], 97 | [-0.8242, 0.0849, 0.4392, -0.0580, 0.2012, -0.2768], 98 | [-0.3280, 0.0549, 0.1075, 0.1855, -0.4357, 0.8085], 99 | [0.1084, -0.9022, 0.4120, 0.0573, -0.0251, 0.0238]]); 100 | bands = ee.List([season + '_blue', season + '_green', season + '_red', season + '_nir', season + '_swir1', season + '_swir2']); 101 | 102 | # // Make an Array Image, with a 1-D Array per pixel. 103 | arrayImage1D = image.select(bands).toArray(); 104 | 105 | # // Make an Array Image with a 2-D Array per pixel, 6 x 1 106 | arrayImage2D = arrayImage1D.toArray(1); 107 | 108 | componentsImage = ee.Image(coefficients).matrixMultiply(arrayImage2D).arrayProject([0]).arrayFlatten([[season + '_brightness', season + '_greenness', season + '_wetness', season + '_fourth', season + '_fifth', season + '_sixth']]).float(); 109 | # // Get a multi-band image with TC-named bands 110 | return image.addBands(componentsImage); 111 | 112 | 113 | # // Function to add Tasseled Cap angles and distances to an image. Assumes image has bands: 'brightness', 'greenness', and 'wetness'. 114 | def getTassledCapAngleAndDistance (season, image): 115 | 116 | brightness = image.select(season + '_brightness'); 117 | greenness = image.select(season + '_greenness'); 118 | wetness = image.select(season + '_wetness'); 119 | 120 | # // Calculate tassled cap angles and distances 121 | tcAngleBG = brightness.atan2(greenness).divide(Math.PI).rename([season + '_tcAngleBG']); 122 | tcAngleGW = greenness.atan2(wetness).divide(Math.PI).rename([season + '_tcAngleGW']); 123 | tcAngleBW = brightness.atan2(wetness).divide(Math.PI).rename([season + '_tcAngleBW']); 124 | 125 | tcDistanceBG = brightness.hypot(greenness).rename([season + '_tcDistanceBG']); 126 | tcDistanceGW = greenness.hypot(wetness).rename([season + '_tcDistanceGW']); 127 | tcDistanceBW = brightness.hypot(wetness).rename([season + '_tcDistanceBW']); 128 | 129 | image = image.addBands(tcAngleBG).addBands(tcAngleGW).addBands(tcAngleBW).addBands(tcDistanceBG).addBands(tcDistanceGW).addBands(tcDistanceBW); 130 | 131 | return image; 132 | 133 | 134 | def computeTassledCap(season, image): 135 | image = getTassledCapComponents(season, image); 136 | image = getTassledCapAngleAndDistance(season, image); 137 | return image; 138 | 139 | 140 | def addTopography (self,image): 141 | 142 | # // Calculate slope, aspect and hillshade 143 | topo = ee.Algorithms.Terrain(self.elevation); 144 | 145 | # // From aspect (a), calculate eastness (sin a), northness (cos a) 146 | deg2rad = ee.Number(math.pi).divide(180); 147 | aspect = topo.select(['aspect']); 148 | aspect_rad = aspect.multiply(deg2rad); 149 | eastness = aspect_rad.sin().rename(['eastness']).float(); 150 | northness = aspect_rad.cos().rename(['northness']).float(); 151 | 152 | # // Add topography bands to image 153 | topo = topo.select(['elevation','slope','aspect']).addBands(eastness).addBands(northness); 154 | image = image.addBands(topo); 155 | return image; 156 | 157 | 158 | def addJRCDataset (self,image): 159 | # // Update the mask. 160 | jrcImage = self.jrcImage.unmask(0); 161 | 162 | image = image.addBands(jrcImage.select(['occurrence']).rename(['occurrence'])); 163 | image = image.addBands(jrcImage.select(['change_abs']).rename(['change_abs'])); 164 | image = image.addBands(jrcImage.select(['change_norm']).rename(['change_norm'])); 165 | image = image.addBands(jrcImage.select(['seasonality']).rename(['seasonality'])); 166 | image = image.addBands(jrcImage.select(['transition']).rename(['transition'])); 167 | image = image.addBands(jrcImage.select(['max_extent']).rename(['max_extent'])); 168 | 169 | return image; 170 | 171 | def addCovariates (self,season, image): 172 | image = self.addNDCovariates(season, image); 173 | # /*image = addRCovariates(season, image); 174 | # image = addEVI(season, image); 175 | # image = addSAVI(season, image); 176 | # image = addIBI(season, image); 177 | # image = computeTassledCap(season, image);*/ 178 | return image; 179 | 180 | 181 | def addJRCAndTopo(self,image): 182 | image = self.addTopography(image); 183 | image = self.addJRCDataset(image); 184 | return image; 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | -------------------------------------------------------------------------------- /assemblage.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | class assemblage(): 4 | 5 | def __init__(self): 6 | pass 7 | 8 | def createAssemblage(self,image,nodeStruct): 9 | 10 | names = image.bandNames().getInfo() 11 | classes = ['other'] 12 | 13 | for name in names: 14 | classes.append(name.encode('ascii')) 15 | numbers = range(0,image.bandNames().length().getInfo()+1,1) 16 | 17 | classStruct = {} 18 | for i in numbers: 19 | classStruct[classes[i]] = {'number' : i} 20 | 21 | # The starting id, i.e. the first decision 22 | startId = 'key1'; 23 | 24 | # The initial decision tree string (DO NOT CHANGE) 25 | DTstring = ['1) root 9999 9999 9999']; 26 | # Call the function to construct the decision tree string (DO NOT CHANGE) 27 | DTstring = "\n".join(self.decision(nodeStruct,classStruct,startId,1,DTstring))#.join("\n"); 28 | 29 | classifier = ee.Classifier.decisionTree(DTstring) 30 | 31 | sd = 10 32 | nIter = 100 33 | nBands = ee.List.sequence(0,image.bandNames().length().getInfo()-1,1) 34 | 35 | 36 | def monteCarlo(i): 37 | def createRand(j): 38 | rand = ee.Image(image.select(0)).gt(-1).multiply(ee.Image.random(ee.Number(i).multiply(j))).subtract(0.5).multiply(2) 39 | random = rand.multiply(sd) 40 | return ee.Image(image.select(ee.Number(j))).add(random) 41 | img = ee.ImageCollection(nBands.map(createRand)) 42 | img = self.collectionToImage(img) 43 | classified = img.classify(classifier); 44 | return classified 45 | 46 | 47 | iters = ee.List.sequence(1,nIter) 48 | assemblage = ee.ImageCollection(iters.map(monteCarlo)) 49 | 50 | mode = ee.Image(assemblage.mode()) 51 | 52 | def uncertainty(img): 53 | return img.eq(mode) 54 | 55 | prob = ee.Image(assemblage.map(uncertainty).sum()).rename('prob') 56 | 57 | #region = ee.Geometry.Polygon([[103.876,18.552],[105.806,18.552],[105.806,19.999],[103.876,19.999],[103.876,18.552]]) 58 | 59 | #task_ordered = ee.batch.Export.image.toAsset(image=ee.Image(prob), description='test', assetId='users/servirmekong/temp/outputAssemblage4',region=region['coordinates'], maxPixels=1e13,scale=300) 60 | #task_ordered.start() 61 | 62 | #print(classified.getInfo()) 63 | 64 | return mode, prob 65 | 66 | # Function to convert a dictionary of nodes into a decision tree string 67 | def decision(self,nodeStruct,classStruct,id1,node,DTstring): 68 | # Extract parameters 69 | lnode = 2*node; #// left child node number 70 | rnode = lnode + 1; #// right child node number 71 | dict1 = nodeStruct[id1]; #// current dictionary 72 | band = dict1['band']; #// decision band 73 | threshold = dict1['threshold']; #// decision threshold 74 | left = dict1['left']; #// left result (either 'terminal' or new id) 75 | right = dict1['right']; #// right result (either 'terminal' or new id) 76 | leftName = dict1['leftName']; #// left class name (if 'terminal') 77 | if right == 'terminal': 78 | rightName = dict1['rightName']; #// right class name (if 'terminal') 79 | 80 | leftLine = '' 81 | rightLine = ''; 82 | leftNumber = 0; 83 | rightNumber = 0; 84 | 85 | # Add the left condition and right condition strings to the current decision 86 | # tree string. If either condition is non-terminal, recursively call the function. 87 | if (left == 'terminal'): # left terminal condition 88 | leftNumber = str(classStruct[leftName]['number']); 89 | leftLine = str(lnode) + ') ' + str(band) + '>=' + str(threshold) + ' 9999 9999 ' + str(leftNumber) + ' *'; 90 | DTstring.append(leftLine); 91 | if (right == 'terminal'): #// right terminal condition 92 | rightNumber = classStruct[rightName]['number']; 93 | rightLine = str(rnode) + ') ' + str(band) + '<' + str(threshold) + ' 9999 9999 ' + str(rightNumber) + ' *'; 94 | DTstring.append(rightLine); 95 | return DTstring; 96 | else: # right non-terminal condition 97 | rightLine = str(rnode) + ') ' + str(band) + '<' + str(threshold) + ' 9999 9999 9999'; 98 | DTstring.append(rightLine); 99 | return self.decision(nodeStruct,classStruct,right,rnode,DTstring); 100 | 101 | else: # left non-terminal condition 102 | leftLine = str(lnode) + ') ' + str(band) + '>=' + str(threshold) + ' 9999 9999 9999'; 103 | DTstring.append(leftLine); 104 | DTstring = self.decision(nodeStruct,classStruct,left,lnode,DTstring); 105 | 106 | if (right == 'terminal'): # // right terminal condition 107 | rightNumber = classStruct[rightName]['number']; 108 | rightLine = str(rnode) + ') ' + str(band) + '<' + str(threshold) + ' 9999 9999 ' + str(rightNumber) + ' *'; 109 | DTstring.append(rightLine); 110 | return DTstring; 111 | else: # right non-terminal 112 | rightLine = str(rnode) + ') ' + str(band) + '<' + str(threshold) + ' 9999 9999 9999'; 113 | DTstring.append(rightLine); 114 | return self.decision(nodeStruct,classStruct,right,rnode,DTstring); 115 | 116 | 117 | return DTstring; 118 | 119 | def collectionToImage(self,collection): 120 | 121 | def iterate(img,prev): 122 | return ee.Image(prev).addBands(img) 123 | 124 | stack = ee.Image(collection.iterate(iterate,ee.Image(1))) 125 | 126 | stack = stack.select(ee.List.sequence(1, stack.bandNames().size().subtract(1))); 127 | 128 | return stack; 129 | 130 | 131 | import ee 132 | ee.Initialize() 133 | 134 | 135 | aquaculture = ee.Image(ee.ImageCollection("projects/servir-mekong/yearly_primitives_smoothed/aquaculture").first()).rename('aquaculture') 136 | barren = ee.Image(ee.ImageCollection("projects/servir-mekong/yearly_primitives_smoothed/barren").first()).rename('barren') 137 | cropland = ee.Image(ee.ImageCollection("projects/servir-mekong/yearly_primitives_smoothed/cropland").first()).rename('cropland') 138 | deciduous = ee.Image(ee.ImageCollection("projects/servir-mekong/yearly_primitives_smoothed/deciduous").first()).rename('forest') 139 | 140 | image = aquaculture.addBands(barren).addBands(cropland).addBands(deciduous) 141 | 142 | nodeStruct = { 'key1': {'band': 'aquaculture','threshold': 50, 'left': 'terminal', 'leftName': 'aquaculture', 'right': 'key2'}, 143 | 'key2': {'band': 'barren', 'threshold': 40, 'left': 'terminal', 'leftName': 'barren', 'right': 'key3'}, 144 | 'key3': {'band': 'cropland', 'threshold': 60, 'left': 'terminal', 'leftName': 'cropland', 'right': 'key4'}, 145 | 'key4': {'band': 'forest', 'threshold': 5, 'left': 'terminal', 'leftName': 'other', 'right': 'terminal', 'rightName': 'forest'} }; 146 | 147 | m,p = assemblage().createAssemblage(image,nodeStruct) 148 | print(m.getInfo()) 149 | print(p.getInfo()) 150 | -------------------------------------------------------------------------------- /diffMedian.py: -------------------------------------------------------------------------------- 1 | import ee 2 | 3 | class diffMedian(): 4 | def __init__(self): 5 | self.exportPath = 'users/TEST/' 6 | self.epsg = "EPSG:32717" 7 | self.ecoregions = ee.FeatureCollection("projects/Sacha/AncillaryData/StudyRegions/Ecuador_EcoRegions_Buffered") 8 | 9 | self.nDayBuffer = 7*10 10 | self.diffCountMin = 3 11 | self.biweeklyIC = 'projects/Sacha/PreprocessedData/L8_Biweekly_V6' 12 | 13 | 14 | def smartJoin(self,primary,secondary,julianDiff): 15 | """Function for joining based on max julian difference. Assumes startJulian and endJulian are set.""" 16 | #Create a time filter to define a match as overlapping timestamps. 17 | maxDiffFilter = ee.Filter.Or( 18 | ee.Filter.maxDifference( 19 | difference = julianDiff, 20 | leftField = 'startJulian', 21 | rightField = 'endJulian'), 22 | ee.Filter.maxDifference( 23 | difference = julianDiff, 24 | leftField = 'startJulian', 25 | rightField = 'endJulian' 26 | ) 27 | ) 28 | 29 | # Define the join. 30 | saveAllJoin = ee.Join.saveAll( 31 | matchesKey = 'matches', 32 | measureKey = 'dayDiff' 33 | ) 34 | 35 | #Apply the join. 36 | joined = saveAllJoin.apply(primary, secondary, maxDiffFilter) 37 | 38 | return joined 39 | 40 | def mergeMany(self,img,secondaryProperty,sortProperty): 41 | """Function to get the many secondaries and choose the first non null value""" 42 | img = ee.Image(img) 43 | secondaries = img.get(secondaryProperty) 44 | secondaries = ee.ImageCollection.fromImages(secondaries).sort(sortProperty) 45 | secondaries1 = secondaries.filter(ee.Filter.calendarRange(preYearPrimary,preYearPrimary,'year')) 46 | secondaries2 = secondaries.filter(ee.Filter.calendarRange(preYearSecondary,preYearSecondary,'year')) 47 | 48 | secondary1Composite = ee.Image(self.weightedCombiner(secondaries1)) 49 | secondary2Composite = ee.Image(self.weightedCombiner(secondaries2)) 50 | secondariesMosaiced = ee.ImageCollection([secondary1Composite,secondary2Composite]).mosaic() 51 | 52 | return img.addBands(secondariesMosaiced) 53 | 54 | def weightedCombiner(self,matches): 55 | """Function to take a set of matches and create a weighted median composite. Assumes the dayDiff property is set.""" 56 | 57 | #Find the unique dayDiffs. 58 | matchesHist = ee.Dictionary(matches.aggregate_histogram('dayDiff')) 59 | 60 | #Convert it back to a number. 61 | def convertkeys(n): 62 | return ee.Number.parse(n).float() 63 | 64 | keys = matchesHist.keys().map(convertkeys) 65 | 66 | #Find the min and max of the dayDiffs and min max 0-1 stretch. Then reverse it and add 1 sdo the repeated values are from 1-20. 67 | minKey = keys.reduce(ee.Reducer.min()) 68 | maxKey = keys.reduce(ee.Reducer.max()) 69 | 70 | def normedn(n): 71 | return (ee.Number(n).subtract(minKey)).divide(ee.Number(minKey).add(maxKey)) 72 | 73 | def normedreverse(n): 74 | return ee.Number(n).multiply(-1).add(1).multiply(20).int16() 75 | 76 | normedkeys = keys.map(normedn) 77 | normed = normedkeys.map(normedreverse) 78 | 79 | #Zip them together 80 | zipped = keys.zip(normed) 81 | 82 | def keyWeight(kw): 83 | keyWeight = ee.List(kw) 84 | key = keyWeight.get(0) 85 | weight = keyWeight.get(1) 86 | 87 | #Get images for given dayDiff. 88 | imgs = matches.filter(ee.Filter.eq('dayDiff',ee.Number(key))) 89 | 90 | def keyweightrepeat(img): 91 | return ee.ImageCollection(ee.List.repeat(ee.Image(img),ee.Number(weight))) 92 | 93 | #Repeat the images based on the weight. 94 | rep = ee.ImageCollection(ee.FeatureCollection(imgs.map(keyweightrepeat)).flatten()) 95 | 96 | return rep 97 | 98 | repeated = zipped.map(keyWeight) 99 | 100 | #Flatten and compute median 101 | out = ee.ImageCollection(ee.FeatureCollection(repeated).flatten()).median() 102 | 103 | return ee.Image(out) 104 | 105 | def setJulian(self,img): 106 | """Function for setting start and end julians based on system:time_start. Assumes a 14 day diff inclusive of the first day.""" 107 | d = ee.Date(img.get('system:time_start')) 108 | startJulian = d.getRelative('day','year') 109 | endJulian = startJulian.add(13) 110 | return img.set({'startJulian':startJulian,'endJulian':endJulian}) 111 | 112 | def simpleAddIndices(self,in_image): 113 | """Function for only adding common indices.""" 114 | in_image = in_image.addBands(in_image.normalizedDifference(['nir','red']).select([0],['NDVI'])) 115 | in_image = in_image.addBands(in_image.normalizedDifference(['nir','swir2']).select([0], ['NBR'])) 116 | in_image = in_image.addBands(in_image.normalizedDifference(['nir','swir1']).select([0], ['NDMI'])) 117 | in_image = in_image.addBands(in_image.normalizedDifference(['green','swir1']).select([0], ['NDSI'])) 118 | return in_image 119 | 120 | def cReducer(self,img): 121 | m = img.mask().reduce(ee.Reducer.min()).focal_min(3.5) 122 | return img.updateMask(m) 123 | 124 | def joinedmerge(self,img): 125 | return ee.Image(self.mergeMany(img,'matches','dayDiff')) 126 | 127 | def joinedmerge2(self,l): 128 | def joinedmerge(img): 129 | return ee.Image(self.mergeMany(img,'matches','dayDiff')) 130 | l = l.map(joinedmerge) 131 | return ee.ImageCollection(l) 132 | 133 | #Find the t2-t1 difference for each time period. 134 | def joineddiff(self,img): 135 | t1T = img.select(['.*_2014']) 136 | t2T = img.select(['.*_2016']) 137 | return img.addBands(t2T.subtract(t1T).rename(self.bnsDiff)) 138 | 139 | def addsuffix(self,l,suffix): 140 | def base(bn): 141 | return ee.String(bn).cat(suffix) 142 | return l.map(base) 143 | 144 | def exportMap(self,img,studyArea): 145 | img = img 146 | ed = str(postYear) 147 | sd = str(preYearPrimary) 148 | regionName = ProcessingRegion.replace(" ",'_') + "_" 149 | 150 | task_ordered= ee.batch.Export.image.toAsset(image=img.clip(studyArea), 151 | description = regionName + '_Diff_Comp_rSA_2lst_' + sd + '_' + ed, 152 | assetId = self.exportPath + regionName + '_Diff_Comp' + sd + '_' + ed, 153 | region = studyArea.bounds().getInfo()['coordinates'], 154 | maxPixels = 1e13, 155 | crs = self.epsg, 156 | scale = 30) 157 | 158 | task_ordered.start() 159 | print('Export Started: ',self.exportPath + regionName + '_Diff_Comp' + sd + '_' + ed) 160 | 161 | def main(self,ProcessingRegion,postYear,preYearPrimary,preYearSecondary,exportImg=False): 162 | studyArea = self.ecoregions.filter(ee.Filter.eq("PROVINCIA", ProcessingRegion)).geometry().buffer(1000) 163 | 164 | c = ee.ImageCollection(self.biweeklyIC).filter(ee.Filter.eq('regionName',ProcessingRegion)).map(self.cReducer).map(self.simpleAddIndices) 165 | 166 | bns = ee.List(['blue','green','red','nir','swir1','swir2','NDVI','NBR','NDMI']) 167 | 168 | c = c.select(bns) 169 | 170 | #Append endings to band names 171 | bnsT1 = self.addsuffix(bns,'_2014') 172 | bnsT2 = self.addsuffix(bns,'_2016') 173 | self.bnsDiff = self.addsuffix(bns,'_2014_2016_diff') 174 | 175 | #Filter off the two years of data 176 | 177 | t1 = c.filter(ee.Filter.calendarRange(preYearPrimary,preYearSecondary,'year')).select(bns,bnsT1).map(self.setJulian) 178 | t2 = c.filter(ee.Filter.calendarRange(postYear,postYear,'year')).select(bns,bnsT2).map(self.setJulian) 179 | print(t2.first().bandNames().getInfo()) 180 | 181 | joined = ee.ImageCollection(self.smartJoin(t2,t1,self.nDayBuffer)) 182 | 183 | joined = joined.toList(500)#.map(self.joinedmerge) 184 | joined = self.joinedmerge2(joined) 185 | print(joined.first().bandNames().getInfo(),'join') 186 | 187 | diff = joined.map(self.joineddiff) 188 | 189 | diffMedian = diff.median() 190 | diffCount = diff.select(['.*_diff']).count().reduce(ee.Reducer.min()) 191 | diffMedian = diffMedian.updateMask(diffCount.gte(self.diffCountMin)) 192 | 193 | 194 | print(diffMedian.getInfo()) 195 | if exportImg: 196 | self.exportMap(diffMedian,studyArea) 197 | 198 | return diffMedian 199 | 200 | if __name__ == "__main__": 201 | ee.Initialize() 202 | 203 | ProcessingRegion = 'GALAPAGOS' 204 | 205 | postYear = 2016 206 | preYearPrimary = 2014 207 | preYearSecondary = 2013 208 | 209 | exportImg = True 210 | 211 | diffMedian().main(ProcessingRegion,postYear,preYearPrimary,preYearSecondary,exportImg) 212 | -------------------------------------------------------------------------------- /gapfilling.py: -------------------------------------------------------------------------------- 1 | 2 | import ee 3 | ee.Initialize() 4 | 5 | filez = ["LS_BW_200901402","LS_BW_200904205"] #,"LS_BW_200928029","LS_BW_200929430","LS_BW_200930832","LS_BW_200932233","LS_BW_200933634","LS_BW_200935036","LS_BW_200936401","LS_BW_201001302","LS_BW_201002704","LS_BW_201004105","LS_BW_201005506","LS_BW_201006908","LS_BW_201008309","LS_BW_201009711","LS_BW_201011112","LS_BW_201012513","LS_BW_201013915","LS_BW_201015316","LS_BW_201016718","LS_BW_201018119","LS_BW_201019520","LS_BW_201020922","LS_BW_201022323","LS_BW_201023725","LS_BW_201025126","LS_BW_201026527","LS_BW_201027929","LS_BW_201029330","LS_BW_201030732","LS_BW_201032133","LS_BW_201033534","LS_BW_201034936","LS_BW_201036301","LS_BW_201101202","LS_BW_201102603","LS_BW_201104005","LS_BW_201105406","LS_BW_201106808","LS_BW_201108209","LS_BW_201109610","LS_BW_201111012","LS_BW_201112413","LS_BW_201113815","LS_BW_201115216","LS_BW_201116617","LS_BW_201118019","LS_BW_201119420","LS_BW_201120822","LS_BW_201122223","LS_BW_201123624","LS_BW_201125026","LS_BW_201126427","LS_BW_201127829","LS_BW_201129230","LS_BW_201130631","LS_BW_201132033","LS_BW_201133434","LS_BW_201134836","LS_BW_201136201","LS_BW_201201102","LS_BW_201202503","LS_BW_201203905","LS_BW_201205306","LS_BW_201206708","LS_BW_201208109","LS_BW_201209510","LS_BW_201210912","LS_BW_201212313","LS_BW_201213715","LS_BW_201215116","LS_BW_201216517","LS_BW_201217919","LS_BW_201219320","LS_BW_201220722","LS_BW_201222123","LS_BW_201223524","LS_BW_201224926","LS_BW_201226327","LS_BW_201227729","LS_BW_201229130","LS_BW_201230531","LS_BW_201231933","LS_BW_201233334","LS_BW_201234736","LS_BW_201236100","LS_BW_201300902","LS_BW_201302303","LS_BW_201303705","LS_BW_201305106","LS_BW_201306507","LS_BW_201307909","LS_BW_201309310","LS_BW_201310712","LS_BW_201312113","LS_BW_201313514","LS_BW_201314916","LS_BW_201316317","LS_BW_201317719","LS_BW_201319120","LS_BW_201320521","LS_BW_201321923","LS_BW_201323324","LS_BW_201324726","LS_BW_201326127","LS_BW_201327528","LS_BW_201328930","LS_BW_201330331","LS_BW_201331733","LS_BW_201333134","LS_BW_201334535","LS_BW_201335900","LS_BW_201400802","LS_BW_201402203","LS_BW_201403604","LS_BW_201405006","LS_BW_201406407","LS_BW_201407809","LS_BW_201409210","LS_BW_201410611","LS_BW_201412013","LS_BW_201413414","LS_BW_201414816","LS_BW_201416217","LS_BW_201417618","LS_BW_201419020","LS_BW_201420421","LS_BW_201421823","LS_BW_201423224","LS_BW_201424625","LS_BW_201426027","LS_BW_201427428","LS_BW_201428830","LS_BW_201430231","LS_BW_201431632","LS_BW_201433034","LS_BW_201434435","LS_BW_201435800","LS_BW_201500702","LS_BW_201502103","LS_BW_201503504","LS_BW_201504906","LS_BW_201506307","LS_BW_201507709","LS_BW_201509110","LS_BW_201510511","LS_BW_201511913","LS_BW_201513314","LS_BW_201514716","LS_BW_201516117","LS_BW_201517518","LS_BW_201518920","LS_BW_201520321","LS_BW_201521723","LS_BW_201523124","LS_BW_201524525","LS_BW_201525927","LS_BW_201527328","LS_BW_201528730","LS_BW_201530131","LS_BW_201531532","LS_BW_201532934","LS_BW_201534335","LS_BW_201535700","LS_BW_201600601","LS_BW_201602003","LS_BW_201603404","LS_BW_201604806","LS_BW_201606207","LS_BW_201607608","LS_BW_201609010","LS_BW_201610411","LS_BW_201611813","LS_BW_201613214","LS_BW_201614615","LS_BW_201616017","LS_BW_201617418","LS_BW_201618820","LS_BW_201620221","LS_BW_201621622","LS_BW_201623024","LS_BW_201624425","LS_BW_201625827","LS_BW_201627228","LS_BW_201628629","LS_BW_201630031","LS_BW_201631432","LS_BW_201632834","LS_BW_201634235","LS_BW_201635600","LS_BW_201700401","LS_BW_201701803","LS_BW_201703204","LS_BW_201704605","LS_BW_201706007","LS_BW_201707408","LS_BW_201708810","LS_BW_201710211","LS_BW_201711612","LS_BW_201713014","LS_BW_201714415","LS_BW_201715817","LS_BW_201717218","LS_BW_201718619","LS_BW_201720021","LS_BW_201721422","LS_BW_201722824","LS_BW_201724225","LS_BW_201725626","LS_BW_201727028","LS_BW_201728429","LS_BW_201729831","LS_BW_201731232","LS_BW_201732633","LS_BW_201734035"] 6 | 7 | def calculateGaps(collection): 8 | 9 | # make a gap 1 and data 0 10 | def unmaskNoData(img): 11 | im = img.select("red").gt(9999).rename(["gaps"]) 12 | gaps = ee.Image(im.unmask(1).set('system:time_start', img.get('system:time_start'))).float() 13 | return gaps 14 | 15 | # iterate over the collection and count the number of consequetive gaps 16 | def gapsCumulative(img,mylist): 17 | previous = ee.Image(ee.List(mylist).get(-1)); 18 | gaps = img.add(previous) 19 | gaps = gaps.where(gaps.eq(previous),0).set('system:time_start', img.get('system:time_start')) 20 | return ee.List(mylist).add(gaps) 21 | 22 | gapCollection = ee.ImageCollection(collection.map(unmaskNoData)) 23 | 24 | # count the number of gaps 25 | first = ee.List([ee.Image(gapCollection.first())]); 26 | previous = ee.Image(ee.List(first).get(-1)); 27 | gaps = ee.ImageCollection(ee.List(gapCollection.iterate(gapsCumulative, first))); 28 | 29 | return ee.ImageCollection(gaps); 30 | 31 | 32 | def fillCollectionYear(gaps,inCollection,year): 33 | 34 | stDay = year*365+8 35 | endDay = year*365-8 36 | # appy gap filling using the year before 37 | def fillGaps(img): 38 | date = ee.Date(img.get("system:time_start")) 39 | startDate = date.advance(-8,"day") 40 | endDate = date.advance(8,"day") 41 | thisyear = ee.Image(inCollection.filterDate(startDate,endDate).first()) 42 | startDate = date.advance(-stDay,"day") 43 | endDate = date.advance(-endDay,"day") 44 | previousYear = ee.ImageCollection(ic.filterDate(startDate,endDate)) 45 | 46 | def returnMask(img,previousYear): 47 | mask = img.gt(2) 48 | previousYear =previousYear.updateMask(mask) 49 | img = thisyear.unmask(previousYear) 50 | return img 51 | 52 | img = ee.Algorithms.If(previousYear.size().gt(0),returnMask(img,ee.Image(previousYear.first())),thisyear) 53 | return img 54 | collection = gaps.map(fillGaps) 55 | 56 | return ee.ImageCollection(collection) 57 | 58 | 59 | 60 | ic = ee.ImageCollection("projects/Sacha/PreprocessedData/L8_Biweekly_V5_Mosaiced_2") 61 | 62 | namez = [] 63 | 64 | 65 | myGaps = calculateGaps(ic) 66 | myCollection = ee.ImageCollection(fillCollectionYear(myGaps,ic,1)) 67 | 68 | myGaps = calculateGaps(myCollection) 69 | myCollection = fillCollectionYear(myGaps,myCollection,2) 70 | 71 | myGaps = calculateGaps(myCollection) 72 | myCollection = fillCollectionYear(myGaps,myCollection,3) 73 | 74 | myList = myCollection.toList(500) 75 | 76 | 77 | bandNames = ee.List(['blue', 'green', 'red', 'nir', 'swir1', 'swir2','blue_stdDev', 'red_stdDev', 'green_stdDev', 'nir_stdDev', 'swir1_stdDev', 'swir2_stdDev', 'ND_green_swir1_stdDev', 'ND_nir_red_stdDev', 'ND_nir_swir2_stdDev', 'svvi', 'thermal', 'date', 'year', 'cloudMask', 'TDOMMask', 'pixel_qa']) 78 | 79 | 80 | studyArea = ee.FeatureCollection("projects/Sacha/AncillaryData/StudyRegions/Ecuador_EcoRegions_Complete") 81 | studyArea = studyArea.geometry().bounds() 82 | 83 | geom = studyArea.getInfo() 84 | 85 | 86 | for i in range(0,100,1): 87 | im = ee.Image(myList.get(i)) 88 | name = filez[i] 89 | print name 90 | im = im.select(bandNames) 91 | task_ordered= ee.batch.Export.image.toAsset(image=im, 92 | description = name, 93 | assetId= "projects/Sacha/PreprocessedData/L8_Biweekly_V5_Mosaiced_gapfilled/"+ name, 94 | region=geom['coordinates'], 95 | maxPixels=1e13, 96 | crs="epsg:4326", 97 | scale=30) 98 | 99 | task_ordered.start() 100 | 101 | 102 | 103 | 104 | 105 | -------------------------------------------------------------------------------- /landsat.py: -------------------------------------------------------------------------------- 1 | # Sentinel-2 package 2 | from paramsTemplate import * 3 | import ee 4 | from Py6S import * 5 | import math 6 | import datetime 7 | import os, sys 8 | from utils import * 9 | import sun_angles 10 | import view_angles 11 | import time 12 | 13 | class env(object): 14 | 15 | def __init__(self): 16 | """Initialize the environment.""" 17 | 18 | # Initialize the Earth Engine object, using the authentication credentials. 19 | ee.Initialize() 20 | 21 | self.dem = ee.Image("JAXA/ALOS/AW3D30_V1_1").select(["AVE"]) 22 | self.epsg = "EPSG:32717" 23 | 24 | ########################################## 25 | # variable for the landsat data request # 26 | ########################################## 27 | self.metadataCloudCoverMax = 80; 28 | 29 | ########################################## 30 | # Export variables # 31 | ########################################## 32 | 33 | self.assetId ="projects/Sacha/PreprocessedData/L8_Biweekly_V6/" 34 | self.name = "LS_BW_" 35 | 36 | 37 | self.exportScale = 20 38 | 39 | ########################################## 40 | # variable for the shadowMask algorithm # 41 | ########################################## 42 | 43 | # zScoreThresh: Threshold for cloud shadow masking- lower number masks out 44 | # less. Between -0.8 and -1.2 generally works well 45 | self.zScoreThresh = -0.9 46 | 47 | # shadowSumThresh: Sum of IR bands to include as shadows within TDOM and the 48 | # shadow shift method (lower number masks out less) 49 | self.shadowSumThresh = 0.4; 50 | 51 | # contractPixels: The radius of the number of pixels to contract (negative buffer) clouds and cloud shadows by. Intended to eliminate smaller cloud 52 | # patches that are likely errors (1.5 results in a -1 pixel buffer)(0.5 results in a -0 pixel buffer) 53 | # (1.5 or 2.5 generally is sufficient) 54 | self.contractPixels = 1.5; 55 | 56 | # dilatePixels: The radius of the number of pixels to dilate (buffer) clouds 57 | # and cloud shadows by. Intended to include edges of clouds/cloud shadows 58 | # that are often missed (1.5 results in a 1 pixel buffer)(0.5 results in a 0 pixel buffer) 59 | # (2.5 or 3.5 generally is sufficient) 60 | self.dilatePixels = 3.25; 61 | 62 | 63 | ########################################## 64 | # variable for cloudScore algorithm # 65 | ########################################## 66 | 67 | # 9. Cloud and cloud shadow masking parameters. 68 | # If cloudScoreTDOM is chosen 69 | # cloudScoreThresh: If using the cloudScoreTDOMShift method-Threshold for cloud 70 | # masking (lower number masks more clouds. Between 10 and 30 generally works best) 71 | self.cloudScoreThresh = 1; 72 | 73 | # Percentile of cloud score to pull from time series to represent a minimum for 74 | # the cloud score over time for a given pixel. Reduces commission errors over 75 | # cool bright surfaces. Generally between 5 and 10 works well. 0 generally is a bit noisy 76 | self.cloudScorePctl = 8 77 | self.hazeThresh = 195 78 | 79 | ########################################## 80 | # variable for terrain algorithm # 81 | ########################################## 82 | 83 | self.terrainScale = 600 84 | 85 | ########################################## 86 | # variable band selection # 87 | ########################################## 88 | self.percentiles = [25,75] 89 | self.medianPercentileBands = ee.List(['blue','green','red','nir','swir1','swir2','date','pixel_qa','cloudScore']) 90 | 91 | self.divideBands = ee.List(['blue','green','red','nir','swir1','swir2']) 92 | self.medoidBands = ee.List(['blue','green','red','nir','swir1','swir2']) 93 | self.medoidIncludeBands = ee.List(['blue','green','red','nir','swir1','swir2','pixel_qa']) 94 | 95 | self.noScaleBands = ee.List(['date','year','cloudMask','count','TDOMMask','pixel_qa','cloudScore']) 96 | 97 | self.bandNamesLandsat = ee.List(['blue','green','red','nir','swir1','thermal','swir2','sr_atmos_opacity','pixel_qa','radsat_qa']) 98 | self.sensorBandDictLandsatSR = ee.Dictionary({'L8' : ee.List([1,2,3,4,5,7,6,9,10,11]),\ 99 | 'L7' : ee.List([0,1,2,3,4,5,6,7,9,10]),\ 100 | 'L5' : ee.List([0,1,2,3,4,5,6,7,9,10]),\ 101 | 'L4' : ee.List([0,1,2,3,4,5,6,7,9,10])}) 102 | 103 | 104 | ########################################## 105 | # enable / disable modules # 106 | ########################################## 107 | self.maskSR = True 108 | self.cloudMask = False 109 | self.hazeMask = False 110 | self.shadowMask = False 111 | self.brdfCorrect = True 112 | self.terrainCorrection = True 113 | self.includePercentiles = True 114 | self.compositingMethod = 'Medoid' 115 | 116 | 117 | class functions(): 118 | def __init__(self): 119 | """Initialize the Surfrace Reflectance app.""" 120 | 121 | # get the environment 122 | self.env = env() 123 | 124 | def main(self,studyArea,startDate,endDate,startDay,endDay,week,regionName): 125 | 126 | self.env.startDate = startDate 127 | self.env.endDate = endDate 128 | 129 | self.env.startDoy = startDay 130 | self.env.endDoy = endDay 131 | self.env.regionName = regionName 132 | 133 | self.studyArea = studyArea 134 | # Set cloud score and tdomm paramters based of region 135 | self.paramSwitch = Switcher().paramSelect(self.env.regionName) 136 | self.env.cloudScoreThresh = self.paramSwitch[0] 137 | self.env.cloudScorePctl= self.paramSwitch[1] 138 | self.env.zScoreThresh= self.paramSwitch[2] 139 | self.env.shadowSumThresh= self.paramSwitch[3] 140 | self.env.contractPixels= self.paramSwitch[4] 141 | self.env.dilatePixels= self.paramSwitch[5] 142 | 143 | # Set correct no scale bands depending on enabled / disabled modules 144 | self.updateNoScaleBands() 145 | print(self.env.noScaleBands.getInfo()) 146 | 147 | landsat8 = ee.ImageCollection('LANDSAT/LC08/C01/T1_SR').filterDate(self.env.startDate,self.env.endDate).filterBounds(studyArea) 148 | landsat8 = landsat8.filterMetadata('CLOUD_COVER','less_than',self.env.metadataCloudCoverMax) 149 | landsat8 = landsat8.select(self.env.sensorBandDictLandsatSR.get('L8'),self.env.bandNamesLandsat) 150 | 151 | landsat5 = ee.ImageCollection('LANDSAT/LT05/C01/T1_SR').filterDate(self.env.startDate,self.env.endDate).filterBounds(studyArea) 152 | landsat5 = landsat5.filterMetadata('CLOUD_COVER','less_than',self.env.metadataCloudCoverMax) 153 | landsat5 = landsat5.select(self.env.sensorBandDictLandsatSR.get('L5'),self.env.bandNamesLandsat).map(self.defringe) 154 | 155 | landsat7 = ee.ImageCollection('LANDSAT/LE07/C01/T1_SR').filterDate(self.env.startDate,self.env.endDate).filterBounds(studyArea) 156 | landsat7 = landsat7.filterMetadata('CLOUD_COVER','less_than',self.env.metadataCloudCoverMax) 157 | landsat7 = landsat7.select(self.env.sensorBandDictLandsatSR.get('L7'),self.env.bandNamesLandsat) 158 | 159 | 160 | landsat = landsat5.merge(landsat7).merge(landsat8) 161 | 162 | if landsat.size().getInfo() > 0: 163 | 164 | # mask clouds using the QA band 165 | if self.env.maskSR == True: 166 | print "removing clouds" 167 | landsat = landsat.map(self.CloudMaskSRL8) 168 | 169 | 170 | # mask clouds using cloud mask function 171 | if self.env.hazeMask == True: 172 | print "removing haze" 173 | landsat = landsat.map(self.maskHaze) 174 | 175 | landsat = landsat.map(self.scaleLandsat).map(self.addDateYear) 176 | 177 | if self.env.cloudMask == True: 178 | print "removing some more clouds" 179 | landsat = landsat.map(self.maskClouds) 180 | 181 | # mask clouds using cloud mask function 182 | if self.env.shadowMask == True: 183 | print "shadow masking" 184 | landsat = self.maskShadows(landsat) 185 | 186 | if self.env.brdfCorrect == True: 187 | landsat = landsat.map(self.brdf) 188 | 189 | if self.env.terrainCorrection == True: 190 | landsat = ee.ImageCollection(landsat.map(self.terrain)) 191 | 192 | if self.env.compositingMethod == 'Medoid': 193 | print("calculating medoid") 194 | mosaic = self.medoidMosaic(landsat) 195 | if self.env.includePercentiles: 196 | medoidDown = ee.Image(self.medoidMosaicPercentiles(landsat,self.env.percentiles[0])) 197 | medoidUp = self.medoidMosaicPercentiles(landsat,self.env.percentiles[1]) 198 | stdevBands = self.addSTDdev(landsat) 199 | mosaic = mosaic.addBands(medoidDown).addBands(medoidUp).addBands(stdevBands) 200 | 201 | 202 | if self.env.compositingMethod == 'Median': 203 | print("calculating Median") 204 | mosaic = self.medianMosaic(landsat) 205 | if self.env.includePercentiles: 206 | imgPercentials = self.medianPercentiles(landsat, self.env.percentiles) 207 | stdevBands = self.addSTDdev(landsat) 208 | mosaic = img.addBands(imgPercentials).addBands(stdevBands) 209 | 210 | print("rescale") 211 | mosaic = self.reScaleLandsat(mosaic) 212 | 213 | print("set MetaData") 214 | mosaic = self.setMetaData(mosaic) 215 | 216 | print("exporting composite") 217 | self.exportMap(mosaic,studyArea,week) 218 | print(mosaic.getInfo()['bands']) 219 | return mosaic 220 | def medoidMosaicPercentiles(self,inCollection,p): 221 | ' calculate the medoid of a percentile' 222 | 223 | inCollection = inCollection.select(self.env.medoidBands) 224 | 225 | p1 = p 226 | p2 = 100 -p 227 | 228 | med1 = self.medoidPercentiles(inCollection,p1).select(["green","nir"]) 229 | med2 = self.medoidPercentiles(inCollection,p2).select(["blue","red","swir1","swir2"]) 230 | 231 | medoidP = self.renameBands(ee.Image(med1).addBands(med2),str("p")+str(p)) 232 | return medoidP 233 | 234 | def medoidPercentiles(self,inCollection,p): 235 | 236 | # Find band names in first image 237 | bandNumbers = ee.List.sequence(1,self.env.medoidBands.length()); 238 | 239 | # Find the median 240 | percentile = inCollection.select(self.env.medoidBands).reduce(ee.Reducer.percentile([p])); 241 | 242 | def subtractPercentile(img): 243 | diff = ee.Image(img).subtract(percentile).pow(ee.Image.constant(2)); 244 | return diff.reduce('sum').addBands(img); 245 | 246 | percentile = inCollection.map(subtractPercentile) 247 | 248 | percentile = ee.ImageCollection(percentile).reduce(ee.Reducer.min(self.env.medoidBands.length().add(1))).select(bandNumbers,self.env.medoidBands); 249 | 250 | return percentile; 251 | 252 | def renameBands(self,image,prefix): 253 | 'rename bands with prefix' 254 | 255 | bandnames = image.bandNames(); 256 | 257 | def mapBands(band): 258 | band = ee.String(prefix).cat('_').cat(band); 259 | return band; 260 | 261 | bandnames = bandnames.map(mapBands) 262 | 263 | image = image.rename(bandnames); 264 | 265 | return image; 266 | 267 | def addSTDdev(self,collection): 268 | 269 | def addSTDdevIndices(img): 270 | """ Function to add common (and less common) spectral indices to an image. 271 | Includes the Normalized Difference Spectral Vector from (Angiuli and Trianni, 2014) """ 272 | img = img.addBands(img.normalizedDifference(['green','swir1']).rename(['ND_green_swir1'])); # NDSI, MNDWI 273 | img = img.addBands(img.normalizedDifference(['nir','red']).rename(['ND_nir_red'])); # NDVI 274 | img = img.addBands(img.normalizedDifference(['nir','swir2']).rename(['ND_nir_swir2'])); # NBR, MNDVI 275 | 276 | return img; 277 | 278 | 279 | 280 | blue_stdDev = collection.select(["blue"]).reduce(ee.Reducer.stdDev()).rename(['blue_stdDev']) 281 | red_stdDev = collection.select(["red"]).reduce(ee.Reducer.stdDev()).rename(['red_stdDev']) 282 | green_stdDev = collection.select(["green"]).reduce(ee.Reducer.stdDev()).rename(['green_stdDev']) 283 | nir_stdDev = collection.select(["nir"]).reduce(ee.Reducer.stdDev()).rename(['nir_stdDev']) 284 | swir1_stdDev = collection.select(["swir1"]).reduce(ee.Reducer.stdDev()).rename(['swir1_stdDev']) 285 | swir2_stdDev = collection.select(["swir2"]).reduce(ee.Reducer.stdDev()).rename(['swir2_stdDev']) 286 | 287 | col = collection.map(addSTDdevIndices) 288 | 289 | ND_green_swir1 = col.select(['ND_green_swir1']).reduce(ee.Reducer.stdDev()).rename(['ND_green_swir1_stdDev']); 290 | ND_nir_red = col.select(['ND_nir_red']).reduce(ee.Reducer.stdDev()).rename(['ND_nir_red_stdDev']); 291 | ND_nir_swir2 = col.select(['ND_nir_swir2']).reduce(ee.Reducer.stdDev()).rename(['ND_nir_swir2_stdDev']); 292 | # svvi = sd(1,2,3,4,5,6,7)-sd(5,6,7) 293 | irStd = nir_stdDev.add(swir1_stdDev).add(swir2_stdDev) 294 | allStd = blue_stdDev.add(red_stdDev).add(green_stdDev).add(nir_stdDev).add(swir1_stdDev).add(swir2_stdDev) 295 | svvi = allStd.subtract(irStd).rename(['svvi']) 296 | 297 | stdevBands = ee.Image(blue_stdDev.addBands(red_stdDev).addBands(green_stdDev).addBands(nir_stdDev).addBands(swir1_stdDev).addBands(swir2_stdDev)\ 298 | .addBands(ND_green_swir1).addBands(ND_nir_red).addBands(ND_nir_swir2).addBands(svvi)) 299 | 300 | return stdevBands 301 | 302 | def addDateYear(self,img): 303 | #add a date and year band 304 | date = ee.Date(img.get("system:time_start")) 305 | 306 | day = date.getRelative('day','year').add(1); 307 | yr = date.get('year'); 308 | mk = img.mask().reduce(ee.Reducer.min()); 309 | 310 | img = img.addBands(ee.Image.constant(day).mask(mk).uint16().rename('date')); 311 | img = img.addBands(ee.Image.constant(yr).mask(mk).uint16().rename('year')); 312 | 313 | return img; 314 | 315 | def CloudMaskSRL8(self,img): 316 | """apply cf-mask Landsat""" 317 | QA = img.select("pixel_qa") 318 | 319 | shadow = QA.bitwiseAnd(8).neq(0); 320 | cloud = QA.bitwiseAnd(32).neq(0); 321 | return img.updateMask(shadow.Not()).updateMask(cloud.Not()).copyProperties(img) 322 | 323 | def scaleLandsat(self,img): 324 | """Landast is scaled by factor 0.0001 """ 325 | thermal = img.select(ee.List(['thermal'])).multiply(0.1) 326 | scaled = ee.Image(img).select(self.env.divideBands).multiply(ee.Number(0.0001)) 327 | 328 | return img.select(['pixel_qa']).addBands(scaled).addBands(thermal) 329 | 330 | def reScaleLandsat(self,img): 331 | """Landast is scaled by factor 0.0001 """ 332 | noScaleBands = self.env.noScaleBands #ee.List(['date','year','cloudMask','count','TDOMMask','pixel_qa','cloudScore'])# ee.List(['date','year','TDOMMask','cloudMask','count']) 333 | noScale = ee.Image(img).select(noScaleBands) 334 | thermalBand = ee.List(['thermal']) 335 | thermal = ee.Image(img).select(thermalBand).multiply(10) 336 | 337 | otherBands = ee.Image(img).bandNames().removeAll(thermalBand).removeAll(noScaleBands) 338 | scaled = ee.Image(img).select(otherBands).divide(0.0001) 339 | 340 | image = ee.Image(scaled.addBands([thermal,noScale])).int16() 341 | 342 | return image.copyProperties(img) 343 | 344 | def updateNoScaleBands(self): 345 | """ removes bands if not being used from no scale bands list """ 346 | if self.env.cloudMask != True : 347 | self.env.noScaleBands = self.env.noScaleBands.remove('cloudMask').remove('cloudScore') 348 | self.env.medianPercentileBands = self.env.medianPercentileBands.remove('cloudScore') 349 | if self.env.shadowMask != True : self.env.noScaleBands = self.env.noScaleBands.remove('TDOMMask') 350 | 351 | def maskHaze(self,img): 352 | """ mask haze """ 353 | opa = ee.Image(img.select(['sr_atmos_opacity'])) 354 | haze = opa.gt(self.env.hazeThresh) 355 | return img.updateMask(haze.Not()) 356 | 357 | def maskClouds(self,img): 358 | """ 359 | Computes spectral indices of cloudyness and take the minimum of them. 360 | 361 | Each spectral index is fairly lenient because the group minimum 362 | is a somewhat stringent comparison policy. side note -> this seems like a job for machine learning :) 363 | originally written by Matt Hancher for Landsat imageryadapted to Sentinel by Chris Hewig and Ian Housman 364 | """ 365 | 366 | score = ee.Image(1.0); 367 | # Clouds are reasonably bright in the blue band. 368 | blue_rescale = img.select('blue').subtract(ee.Number(0.1)).divide(ee.Number(0.3).subtract(ee.Number(0.1))) 369 | score = score.min(blue_rescale); 370 | 371 | # Clouds are reasonably bright in all visible bands. 372 | visible = img.select('red').add(img.select('green')).add(img.select('blue')) 373 | visible_rescale = visible.subtract(ee.Number(0.2)).divide(ee.Number(0.8).subtract(ee.Number(0.2))) 374 | score = score.min(visible_rescale); 375 | 376 | # Clouds are reasonably bright in all infrared bands. 377 | infrared = img.select('nir').add(img.select('swir1')).add(img.select('swir2')) 378 | infrared_rescale = infrared.subtract(ee.Number(0.3)).divide(ee.Number(0.8).subtract(ee.Number(0.3))) 379 | score = score.min(infrared_rescale); 380 | 381 | # Clouds are reasonably cool in temperature. 382 | temp_rescale = img.select('thermal').subtract(ee.Number(300)).divide(ee.Number(290).subtract(ee.Number(300))) 383 | score = score.min(temp_rescale); 384 | 385 | # However, clouds are not snow. 386 | ndsi = img.normalizedDifference(['green', 'swir1']); 387 | ndsi_rescale = ndsi.subtract(ee.Number(0.8)).divide(ee.Number(0.6).subtract(ee.Number(0.8))) 388 | score = score.min(ndsi_rescale).multiply(100).byte().rename('cloudScore'); 389 | mask = score.lt(self.env.cloudScoreThresh).rename(['cloudMask']); 390 | img = img.updateMask(mask).addBands([mask]).addBands(score); 391 | 392 | return img; 393 | 394 | def maskShadows(self,collection): 395 | 396 | def TDOM(image): 397 | zScore = image.select(shadowSumBands).subtract(irMean).divide(irStdDev) 398 | irSum = image.select(shadowSumBands).reduce(ee.Reducer.sum()) 399 | TDOMMask = zScore.lt(self.env.zScoreThresh).reduce(ee.Reducer.sum()).eq(2)\ 400 | .And(irSum.lt(self.env.shadowSumThresh)).Not() 401 | TDOMMask = TDOMMask.focal_min(self.env.contractPixels).focal_max(self.env.dilatePixels).rename(['TDOMMask']) 402 | 403 | image = image.addBands([TDOMMask]) 404 | 405 | return image.updateMask(TDOMMask) 406 | 407 | shadowSumBands = ['nir','swir1'] 408 | 409 | # Get some pixel-wise stats for the time series 410 | irStdDev = ee.Image('projects/Sacha/AncillaryData/TDOM/irStdDev_jd') 411 | irMean = ee.Image('projects/Sacha/AncillaryData/TDOM/irMean_jd') 412 | 413 | # Mask out dark dark outliers 414 | collection_tdom = collection.map(TDOM) 415 | 416 | return collection_tdom 417 | 418 | def terrain(self,img): 419 | 420 | 421 | degree2radian = 0.01745; 422 | otherBands = img.select(self.env.noScaleBands.add('thermal').remove('count'))#['thermal','date','year','TDOMMask','cloudMask','pixel_qa','cloudScore']) 423 | 424 | def topoCorr_IC(img): 425 | 426 | dem = ee.Image("USGS/SRTMGL1_003") 427 | 428 | 429 | # Extract image metadata about solar position 430 | SZ_rad = ee.Image.constant(ee.Number(img.get('SOLAR_ZENITH_ANGLE'))).multiply(degree2radian).clip(img.geometry().buffer(10000)); 431 | SA_rad = ee.Image.constant(ee.Number(img.get('SOLAR_AZIMUTH_ANGLE'))).multiply(degree2radian).clip(img.geometry().buffer(10000)); 432 | 433 | 434 | # Creat terrain layers 435 | slp = ee.Terrain.slope(dem).clip(img.geometry().buffer(10000)); 436 | slp_rad = ee.Terrain.slope(dem).multiply(degree2radian).clip(img.geometry().buffer(10000)); 437 | asp_rad = ee.Terrain.aspect(dem).multiply(degree2radian).clip(img.geometry().buffer(10000)); 438 | 439 | 440 | 441 | # Calculate the Illumination Condition (IC) 442 | # slope part of the illumination condition 443 | cosZ = SZ_rad.cos(); 444 | cosS = slp_rad.cos(); 445 | slope_illumination = cosS.expression("cosZ * cosS", \ 446 | {'cosZ': cosZ, 'cosS': cosS.select('slope')}); 447 | 448 | 449 | # aspect part of the illumination condition 450 | sinZ = SZ_rad.sin(); 451 | sinS = slp_rad.sin(); 452 | cosAziDiff = (SA_rad.subtract(asp_rad)).cos(); 453 | aspect_illumination = sinZ.expression("sinZ * sinS * cosAziDiff", \ 454 | {'sinZ': sinZ, \ 455 | 'sinS': sinS, \ 456 | 'cosAziDiff': cosAziDiff}); 457 | 458 | # full illumination condition (IC) 459 | ic = slope_illumination.add(aspect_illumination); 460 | 461 | 462 | 463 | # Add IC to original image 464 | img_plus_ic = ee.Image(img.addBands(ic.rename(['IC'])).addBands(cosZ.rename(['cosZ'])).addBands(cosS.rename(['cosS'])).addBands(slp.rename(['slope']))); 465 | 466 | return ee.Image(img_plus_ic); 467 | 468 | def topoCorr_SCSc(img): 469 | img_plus_ic = img; 470 | mask1 = img_plus_ic.select('nir').gt(-0.1); 471 | mask2 = img_plus_ic.select('slope').gte(5) \ 472 | .And(img_plus_ic.select('IC').gte(0)) \ 473 | .And(img_plus_ic.select('nir').gt(-0.1)); 474 | 475 | img_plus_ic_mask2 = ee.Image(img_plus_ic.updateMask(mask2)); 476 | 477 | bandList = ['blue', 'green', 'red', 'nir', 'swir1', 'swir2']; # Specify Bands to topographically correct 478 | 479 | 480 | def applyBands(image): 481 | blue = apply_SCSccorr('blue').select(['blue']) 482 | green = apply_SCSccorr('green').select(['green']) 483 | red = apply_SCSccorr('red').select(['red']) 484 | nir = apply_SCSccorr('nir').select(['nir']) 485 | swir1 = apply_SCSccorr('swir1').select(['swir1']) 486 | swir2 = apply_SCSccorr('swir2').select(['swir2']) 487 | return replace_bands(image, [blue, green, red, nir, swir1, swir2]) 488 | 489 | def apply_SCSccorr(band): 490 | method = 'SCSc'; 491 | 492 | out = ee.Image(1).addBands(img_plus_ic_mask2.select('IC', band)).reduceRegion(reducer= ee.Reducer.linearRegression(2,1), \ 493 | geometry= ee.Geometry(img.geometry().buffer(-5000)), \ 494 | scale= self.env.terrainScale, \ 495 | bestEffort =True, 496 | maxPixels=1e10) 497 | 498 | 499 | #out_a = ee.Number(out.get('scale')); 500 | #out_b = ee.Number(out.get('offset')); 501 | #out_c = ee.Number(out.get('offset')).divide(ee.Number(out.get('scale'))); 502 | 503 | 504 | fit = out.combine({"coefficients": ee.Array([[1],[1]])}, False); 505 | 506 | #Get the coefficients as a nested list, 507 | #cast it to an array, and get just the selected column 508 | out_a = (ee.Array(fit.get('coefficients')).get([0,0])); 509 | out_b = (ee.Array(fit.get('coefficients')).get([1,0])); 510 | out_c = out_a.divide(out_b) 511 | 512 | 513 | # apply the SCSc correction 514 | SCSc_output = img_plus_ic_mask2.expression("((image * (cosB * cosZ + cvalue)) / (ic + cvalue))", { 515 | 'image': img_plus_ic_mask2.select([band]), 516 | 'ic': img_plus_ic_mask2.select('IC'), 517 | 'cosB': img_plus_ic_mask2.select('cosS'), 518 | 'cosZ': img_plus_ic_mask2.select('cosZ'), 519 | 'cvalue': out_c }); 520 | 521 | return ee.Image(SCSc_output); 522 | 523 | #img_SCSccorr = ee.Image([apply_SCSccorr(band) for band in bandList]).addBands(img_plus_ic.select('IC')); 524 | img_SCSccorr = applyBands(img).select(bandList).addBands(img_plus_ic.select('IC')) 525 | 526 | bandList_IC = ee.List([bandList, 'IC']).flatten(); 527 | 528 | img_SCSccorr = img_SCSccorr.unmask(img_plus_ic.select(bandList_IC)).select(bandList); 529 | 530 | return img_SCSccorr.unmask(img_plus_ic.select(bandList)) 531 | 532 | 533 | 534 | img = topoCorr_IC(img) 535 | img = topoCorr_SCSc(img) 536 | 537 | return img.addBands(otherBands) 538 | 539 | def defringe(self,img): 540 | 541 | # threshold for defringing landsat5 and 7 542 | fringeCountThreshold = 279 543 | 544 | k = ee.Kernel.fixed(41, 41, 545 | [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 546 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 547 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 548 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 549 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 550 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 551 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 552 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 553 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 554 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 555 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 556 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 557 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 558 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 559 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 560 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 561 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 562 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 563 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 564 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 565 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 566 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 567 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 568 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 569 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 570 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 571 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 572 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 573 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 574 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 575 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 576 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 577 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 578 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 579 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 580 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 581 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 582 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 583 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 584 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 585 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]); 586 | 587 | 588 | m = ee.Image(img).mask().reduce(ee.Reducer.min()) 589 | sum = m.reduceNeighborhood(ee.Reducer.sum(), k, 'kernel') 590 | mask = sum.gte(fringeCountThreshold) 591 | 592 | return img.updateMask(mask) 593 | 594 | def brdf(self,img): 595 | 596 | import sun_angles 597 | import view_angles 598 | 599 | 600 | def _apply(image, kvol, kvol0): 601 | blue = _correct_band(image, 'blue', kvol, kvol0, f_iso=0.0774, f_geo=0.0079, f_vol=0.0372) 602 | green = _correct_band(image, 'green', kvol, kvol0, f_iso=0.1306, f_geo=0.0178, f_vol=0.0580) 603 | red = _correct_band(image, 'red', kvol, kvol0, f_iso=0.1690, f_geo=0.0227, f_vol=0.0574) 604 | nir = _correct_band(image, 'nir', kvol, kvol0, f_iso=0.3093, f_geo=0.0330, f_vol=0.1535) 605 | swir1 = _correct_band(image, 'swir1', kvol, kvol0, f_iso=0.3430, f_geo=0.0453, f_vol=0.1154) 606 | swir2 = _correct_band(image, 'swir2', kvol, kvol0, f_iso=0.2658, f_geo=0.0387, f_vol=0.0639) 607 | return replace_bands(image, [blue, green, red, nir, swir1, swir2]) 608 | 609 | 610 | def _correct_band(image, band_name, kvol, kvol0, f_iso, f_geo, f_vol): 611 | """fiso + fvol * kvol + fgeo * kgeo""" 612 | iso = ee.Image(f_iso) 613 | geo = ee.Image(f_geo) 614 | vol = ee.Image(f_vol) 615 | pred = vol.multiply(kvol).add(geo.multiply(kvol)).add(iso).rename(['pred']) 616 | pred0 = vol.multiply(kvol0).add(geo.multiply(kvol0)).add(iso).rename(['pred0']) 617 | cfac = pred0.divide(pred).rename(['cfac']) 618 | corr = image.select(band_name).multiply(cfac).rename([band_name]) 619 | return corr 620 | 621 | 622 | def _kvol(sunAz, sunZen, viewAz, viewZen): 623 | """Calculate kvol kernel. 624 | From Lucht et al. 2000 625 | Phase angle = cos(solar zenith) cos(view zenith) + sin(solar zenith) sin(view zenith) cos(relative azimuth)""" 626 | 627 | relative_azimuth = sunAz.subtract(viewAz).rename(['relAz']) 628 | pa1 = viewZen.cos() \ 629 | .multiply(sunZen.cos()) 630 | pa2 = viewZen.sin() \ 631 | .multiply(sunZen.sin()) \ 632 | .multiply(relative_azimuth.cos()) 633 | phase_angle1 = pa1.add(pa2) 634 | phase_angle = phase_angle1.acos() 635 | p1 = ee.Image(PI().divide(2)).subtract(phase_angle) 636 | p2 = p1.multiply(phase_angle1) 637 | p3 = p2.add(phase_angle.sin()) 638 | p4 = sunZen.cos().add(viewZen.cos()) 639 | p5 = ee.Image(PI().divide(4)) 640 | 641 | kvol = p3.divide(p4).subtract(p5).rename(['kvol']) 642 | 643 | viewZen0 = ee.Image(0) 644 | pa10 = viewZen0.cos() \ 645 | .multiply(sunZen.cos()) 646 | pa20 = viewZen0.sin() \ 647 | .multiply(sunZen.sin()) \ 648 | .multiply(relative_azimuth.cos()) 649 | phase_angle10 = pa10.add(pa20) 650 | phase_angle0 = phase_angle10.acos() 651 | p10 = ee.Image(PI().divide(2)).subtract(phase_angle0) 652 | p20 = p10.multiply(phase_angle10) 653 | p30 = p20.add(phase_angle0.sin()) 654 | p40 = sunZen.cos().add(viewZen0.cos()) 655 | p50 = ee.Image(PI().divide(4)) 656 | 657 | kvol0 = p30.divide(p40).subtract(p50).rename(['kvol0']) 658 | 659 | return (kvol, kvol0) 660 | 661 | date = img.date() 662 | footprint = determine_footprint(img) 663 | (sunAz, sunZen) = sun_angles.create(date, footprint) 664 | (viewAz, viewZen) = view_angles.create(footprint) 665 | (kvol, kvol0) = _kvol(sunAz, sunZen, viewAz, viewZen) 666 | return _apply(img, kvol.multiply(PI()), kvol0.multiply(PI())) 667 | 668 | def medoidMosaic(self,collection): 669 | """ medoid composite with equal weight among indices """ 670 | nImages = ee.ImageCollection(collection).select([0]).count().rename('count') 671 | bandNames = ee.Image(collection.first()).bandNames() 672 | otherBands = bandNames.removeAll(self.env.medoidIncludeBands) 673 | 674 | others = collection.select(otherBands).reduce(ee.Reducer.mean()).rename(otherBands); 675 | 676 | collection = collection.select(self.env.medoidIncludeBands) 677 | 678 | bandNumbers = ee.List.sequence(1,self.env.medoidIncludeBands.length()); 679 | 680 | median = ee.ImageCollection(collection).select(self.env.divideBands).median() 681 | 682 | def subtractmedian(img): 683 | diff = ee.Image(img).select(self.env.divideBands).subtract(median).pow(ee.Image.constant(2)); 684 | return diff.reduce('sum').addBands(img); 685 | 686 | medoid = collection.map(subtractmedian) 687 | 688 | medoid = ee.ImageCollection(medoid).reduce(ee.Reducer.min(self.env.medoidIncludeBands.length().add(1))).select(bandNumbers,self.env.medoidIncludeBands); 689 | 690 | return medoid.addBands(others).addBands(nImages); 691 | 692 | def medianMosaic(self,collection): 693 | 694 | """ median composite """ 695 | nImages = ee.ImageCollection(collection).select([0]).count().rename('count') 696 | bandNames = collection.first().bandNames() 697 | bandNumbers = ee.List.sequence(1,self.env.medoidBands.length()); 698 | 699 | median = ee.ImageCollection(collection).select(self.env.medoidBands).reduce(ee.Reducer.median()).rename(self.env.medoidBands); 700 | othersBands = bandNames.removeAll(self.env.medoidBands); 701 | 702 | others = collection.select(othersBands).reduce(ee.Reducer.mean()).rename(othersBands); 703 | 704 | return median.addBands(others).addBands(nImages) 705 | 706 | def medianPercentiles(self, collection, p): 707 | 708 | ''' Build Meidan Perntiles: 709 | Takes an Image Collection, and a list of percentiles. 710 | ''' 711 | collection = collection.select(self.env.medianPercentileBands).reduce(ee.Reducer.percentile(p)) 712 | 713 | return collection 714 | 715 | def setMetaData(self,img): 716 | 717 | img = ee.Image(img).set({'regionName': str(self.env.regionName), 718 | 'system:time_start':ee.Date(self.env.startDate).millis(), 719 | 'startDOY':str(self.env.startDoy), 720 | 'endDOY':str(self.env.endDoy), 721 | 'assetId':str(self.env.assetId), 722 | 'compositingMethod':self.env.compositingMethod, 723 | 'toaOrSR':'SR', 724 | 'epsg':str(self.env.epsg), 725 | 'exportScale':str(self.env.exportScale), 726 | 'shadowSumThresh':str(self.env.shadowSumThresh), 727 | 'maskSR':str(self.env.maskSR), 728 | 'cloudMask':str(self.env.cloudMask), 729 | 'hazeMask':str(self.env.hazeMask), 730 | 'shadowMask':str(self.env.shadowMask), 731 | 'brdfCorrect':str(self.env.brdfCorrect), 732 | 'terrainCorrection':str(self.env.terrainCorrection), 733 | 'contractPixels':str(self.env.contractPixels), 734 | 'dilatePixels':str(self.env.dilatePixels), 735 | 'zScoreThresh':str(self.env.zScoreThresh), 736 | 'metadataCloudCoverMax':str(self.env.metadataCloudCoverMax), 737 | 'cloudScorePctl':str(self.env.cloudScorePctl), 738 | 'hazeThresh':str(self.env.hazeThresh), 739 | 'terrainScale':str(self.env.terrainScale)}) 740 | 741 | return img 742 | 743 | def exportMap(self,img,studyArea,week): 744 | 745 | geom = studyArea.getInfo(); 746 | sd = str(self.env.startDate.getRelative('day','year').add(1).getInfo()).zfill(3); 747 | ed = str(self.env.endDate.getRelative('day','year').add(1).getInfo()).zfill(3); 748 | year = str(self.env.startDate.get('year').getInfo()); 749 | regionName = self.env.regionName.replace(" ",'_') + "_" 750 | 751 | task_ordered= ee.batch.Export.image.toAsset(image=img, 752 | description = self.env.name + regionName + str(week).zfill(3) +'_'+ year + sd + ed, 753 | assetId= self.env.assetId + self.env.name + regionName + str(week).zfill(3)+'_'+ year + sd + ed, 754 | region=geom['coordinates'], 755 | maxPixels=1e13, 756 | crs=self.env.epsg, 757 | scale=self.env.exportScale) 758 | 759 | task_ordered.start() 760 | print(self.env.assetId + self.env.name + regionName + str(week).zfill(3)+'_'+ year + sd + ed) 761 | 762 | 763 | 764 | if __name__ == "__main__": 765 | 766 | ee.Initialize() 767 | start = 0 768 | # choose if you export biweekly or yearly 769 | yearly = False 770 | 771 | for i in range(0,1,1): 772 | 773 | #2017 starts at week 212 774 | startWeek = start+ i 775 | print startWeek 776 | 777 | year = ee.Date("2009-01-01") 778 | 779 | if yearly: 780 | startDay = 0 781 | endDay = 364 782 | 783 | startDate = year.advance(startDay,'day').advance(i,'year') 784 | endDate = year.advance(endDay,'day').advance(i,'year') 785 | else: 786 | startDay = (startWeek -1) *14 787 | endDay = (startWeek) *14 -1 788 | startDate = year.advance(startDay,'day') 789 | endDate = year.advance(endDay,'day') 790 | 791 | regionName = 'ANDES DEL NORTE' 792 | studyArea = ee.FeatureCollection("projects/Sacha/AncillaryData/StudyRegions/Ecuador_EcoRegions_Complete") 793 | studyArea = studyArea.filterMetadata('PROVINCIA','equals',regionName).geometry().bounds() 794 | 795 | functions().main(studyArea,startDate,endDate,startDay,endDay,startWeek,regionName) 796 | -------------------------------------------------------------------------------- /landsat8.py: -------------------------------------------------------------------------------- 1 | # Sentinel-2 package 2 | 3 | import ee 4 | from Py6S import * 5 | import math 6 | import datetime 7 | import os, sys 8 | from utils import * 9 | import sun_angles 10 | import view_angles 11 | import time 12 | 13 | class env(object): 14 | 15 | def __init__(self): 16 | """Initialize the environment.""" 17 | 18 | # Initialize the Earth Engine object, using the authentication credentials. 19 | ee.Initialize() 20 | 21 | self.dem = ee.Image("USGS/SRTMGL1_003") 22 | self.epsg = "EPSG:32717" 23 | 24 | ########################################## 25 | # variable for the landsat data request # 26 | ########################################## 27 | self.metadataCloudCoverMax = 60; 28 | 29 | ########################################## 30 | # Export variables # 31 | ########################################## 32 | 33 | self.assetId ="projects/Sacha/L8/" 34 | self.name = "landsat_SR_Biweek_" 35 | self.exportScale = 30 36 | 37 | ########################################## 38 | # variable for the shadowMask algorithm # 39 | ########################################## 40 | 41 | # zScoreThresh: Threshold for cloud shadow masking- lower number masks out 42 | # less. Between -0.8 and -1.2 generally works well 43 | self.zScoreThresh = -1 44 | 45 | # shadowSumThresh: Sum of IR bands to include as shadows within TDOM and the 46 | # shadow shift method (lower number masks out less) 47 | self.shadowSumThresh = 0.35; 48 | 49 | # contractPixels: The radius of the number of pixels to contract (negative buffer) clouds and cloud shadows by. Intended to eliminate smaller cloud 50 | # patches that are likely errors (1.5 results in a -1 pixel buffer)(0.5 results in a -0 pixel buffer) 51 | # (1.5 or 2.5 generally is sufficient) 52 | self.contractPixels = 1.5; 53 | 54 | # dilatePixels: The radius of the number of pixels to dilate (buffer) clouds 55 | # and cloud shadows by. Intended to include edges of clouds/cloud shadows 56 | # that are often missed (1.5 results in a 1 pixel buffer)(0.5 results in a 0 pixel buffer) 57 | # (2.5 or 3.5 generally is sufficient) 58 | self.dilatePixels = 2.5; 59 | 60 | 61 | ########################################## 62 | # variable for cloudScore algorithm # 63 | ########################################## 64 | 65 | # 9. Cloud and cloud shadow masking parameters. 66 | # If cloudScoreTDOM is chosen 67 | # cloudScoreThresh: If using the cloudScoreTDOMShift method-Threshold for cloud 68 | # masking (lower number masks more clouds. Between 10 and 30 generally works best) 69 | self.cloudScoreThresh = 20; 70 | 71 | # Percentile of cloud score to pull from time series to represent a minimum for 72 | # the cloud score over time for a given pixel. Reduces commission errors over 73 | # cool bright surfaces. Generally between 5 and 10 works well. 0 generally is a bit noisy 74 | self.cloudScorePctl = 5; 75 | self.hazeThresh = 200 76 | 77 | ########################################## 78 | # variable for terrain algorithm # 79 | ########################################## 80 | 81 | self.terrainScale = 300 82 | 83 | ########################################## 84 | # variable band selection # 85 | ########################################## 86 | 87 | self.divideBands = ee.List(['blue','green','red','nir','swir1','swir2']) 88 | self.bandNamesLandsat = ee.List(['blue','green','red','nir','swir1','thermal','swir2','sr_atmos_opacity','pixel_qa','radsat_qa']) 89 | self.sensorBandDictLandsatSR = ee.Dictionary({'L8' : ee.List([1,2,3,4,5,7,6,9,10,11])}) 90 | 91 | 92 | ########################################## 93 | # enable / disable modules # 94 | ########################################## 95 | self.maskSR = True 96 | self.cloudMask = True 97 | self.hazeMask = True 98 | self.shadowMask = True 99 | self.brdfCorrect = True 100 | self.terrainCorrection = True 101 | 102 | class functions(): 103 | def __init__(self): 104 | """Initialize the Surfrace Reflectance app.""" 105 | 106 | # get the environment 107 | self.env = env() 108 | 109 | def main(self,studyArea,startDate,endDate,startDay,endDay,week): 110 | 111 | self.env.startDate = startDate 112 | self.env.endDate = endDate 113 | 114 | self.env.startDoy = startDay 115 | self.env.endDoy = endDay 116 | 117 | #studyArea = ee.FeatureCollection("users/apoortinga/countries/Ecuador_nxprovincias").geometry().bounds(); 118 | 119 | landsat8 = ee.ImageCollection('LANDSAT/LC08/C01/T1_SR').filterDate(self.env.startDate,self.env.endDate).filterBounds(studyArea) 120 | landsat8 = landsat8.filterMetadata('CLOUD_COVER','less_than',self.env.metadataCloudCoverMax) 121 | landsat8 = landsat8.select(self.env.sensorBandDictLandsatSR.get('L8'),self.env.bandNamesLandsat) 122 | 123 | print(landsat8.size().getInfo()) 124 | 125 | if landsat8.size().getInfo() > 0: 126 | 127 | # mask clouds using the QA band 128 | if self.env.maskSR == True: 129 | print("removing clouds") 130 | landsat8 = landsat8.map(self.CloudMaskSRL8) 131 | 132 | # mask clouds using cloud mask function 133 | if self.env.hazeMask == True: 134 | print("removing haze") 135 | landsat8 = landsat8.map(self.maskHaze) 136 | 137 | # mask clouds using cloud mask function 138 | if self.env.shadowMask == True: 139 | print("shadow masking") 140 | landsat8 = self.maskShadows(landsat8,studyArea) 141 | 142 | landsat8 = landsat8.map(self.scaleLandsat) 143 | 144 | # mask clouds using cloud mask function 145 | if self.env.cloudMask == True: 146 | print("removing some more clouds") 147 | landsat8 = landsat8.map(self.maskClouds) 148 | 149 | if self.env.brdfCorrect == True: 150 | landsat8 = landsat8.map(self.brdf) 151 | 152 | if self.env.terrainCorrection == True: 153 | print("terrain correction") 154 | landsat8 = ee.ImageCollection(landsat8.map(self.terrain)) 155 | 156 | print("calculating medoid") 157 | img = self.medoidMosaic(landsat8) 158 | 159 | print("rescale") 160 | img = self.reScaleLandsat(img) 161 | 162 | print("set MetaData") 163 | img = self.setMetaData(img) 164 | 165 | print("exporting composite") 166 | self.exportMap(img,studyArea,week) 167 | 168 | def CloudMaskSRL8(self,img): 169 | """apply cf-mask Landsat""" 170 | QA = img.select("pixel_qa") 171 | 172 | shadow = QA.bitwiseAnd(8).neq(0); 173 | cloud = QA.bitwiseAnd(32).neq(0); 174 | return img.updateMask(shadow.Not()).updateMask(cloud.Not()).copyProperties(img) 175 | 176 | def scaleLandsat(self,img): 177 | """Landast is scaled by factor 0.0001 """ 178 | thermal = img.select(ee.List(['thermal'])).multiply(0.1) 179 | scaled = ee.Image(img).select(self.env.divideBands).multiply(ee.Number(0.0001)) 180 | 181 | return img.select([]).addBands(scaled).addBands(thermal) 182 | 183 | def reScaleLandsat(self,img): 184 | """Landast is scaled by factor 0.0001 """ 185 | 186 | thermalBand = ee.List(['thermal']) 187 | thermal = ee.Image(img).select(thermalBand).multiply(10) 188 | 189 | otherBands = ee.Image(img).bandNames().removeAll(thermalBand) 190 | scaled = ee.Image(img).select(otherBands).divide(0.0001) 191 | 192 | image = ee.Image(scaled.addBands(thermal)).int16() 193 | 194 | return image.copyProperties(img) 195 | 196 | def maskHaze(self,img): 197 | """ mask haze """ 198 | opa = ee.Image(img.select(['sr_atmos_opacity']).multiply(0.001)) 199 | haze = opa.gt(self.env.hazeThresh) 200 | return img.updateMask(haze.Not()) 201 | 202 | 203 | def maskClouds(self,img): 204 | """ 205 | Computes spectral indices of cloudyness and take the minimum of them. 206 | 207 | Each spectral index is fairly lenient because the group minimum 208 | is a somewhat stringent comparison policy. side note -> this seems like a job for machine learning :) 209 | originally written by Matt Hancher for Landsat imageryadapted to Sentinel by Chris Hewig and Ian Housman 210 | """ 211 | 212 | score = ee.Image(1.0); 213 | # Clouds are reasonably bright in the blue band. 214 | blue_rescale = img.select('blue').subtract(ee.Number(0.1)).divide(ee.Number(0.3).subtract(ee.Number(0.1))) 215 | score = score.min(blue_rescale); 216 | 217 | # Clouds are reasonably bright in all visible bands. 218 | visible = img.select('red').add(img.select('green')).add(img.select('blue')) 219 | visible_rescale = visible.subtract(ee.Number(0.2)).divide(ee.Number(0.8).subtract(ee.Number(0.2))) 220 | score = score.min(visible_rescale); 221 | 222 | # Clouds are reasonably bright in all infrared bands. 223 | infrared = img.select('nir').add(img.select('swir1')).add(img.select('swir2')) 224 | infrared_rescale = infrared.subtract(ee.Number(0.3)).divide(ee.Number(0.8).subtract(ee.Number(0.3))) 225 | score = score.min(infrared_rescale); 226 | 227 | # Clouds are reasonably cool in temperature. 228 | temp_rescale = img.select('thermal').subtract(ee.Number(300)).divide(ee.Number(290).subtract(ee.Number(300))) 229 | score = score.min(temp_rescale); 230 | 231 | # However, clouds are not snow. 232 | ndsi = img.normalizedDifference(['green', 'swir1']); 233 | ndsi_rescale = ndsi.subtract(ee.Number(0.8)).divide(ee.Number(0.6).subtract(ee.Number(0.8))) 234 | score = score.min(ndsi_rescale).multiply(100).byte(); 235 | mask = score.lt(self.env.cloudScoreThresh).rename(['cloudMask']); 236 | img = img.updateMask(mask); 237 | 238 | return img; 239 | 240 | def maskShadows(self,collection,studyArea): 241 | 242 | def TDOM(image): 243 | zScore = image.select(shadowSumBands).subtract(irMean).divide(irStdDev) 244 | irSum = image.select(shadowSumBands).reduce(ee.Reducer.sum()) 245 | TDOMMask = zScore.lt(self.env.zScoreThresh).reduce(ee.Reducer.sum()).eq(2)\ 246 | .And(irSum.lt(self.env.shadowSumThresh)).Not() 247 | TDOMMask = TDOMMask.focal_min(self.env.dilatePixels) 248 | 249 | return image.updateMask(TDOMMask) 250 | 251 | shadowSumBands = ['nir','swir1'] 252 | 253 | self.fullCollection = ee.ImageCollection('LANDSAT/LC08/C01/T1_SR').filterBounds(studyArea).select(self.env.sensorBandDictLandsatSR.get('L8'),self.env.bandNamesLandsat) 254 | 255 | # Get some pixel-wise stats for the time series 256 | irStdDev = self.fullCollection.select(shadowSumBands).reduce(ee.Reducer.stdDev()) 257 | irMean = self.fullCollection.select(shadowSumBands).reduce(ee.Reducer.mean()) 258 | 259 | # Mask out dark dark outliers 260 | collection_tdom = collection.map(TDOM) 261 | 262 | return collection_tdom 263 | 264 | 265 | def terrain(self,img): 266 | degree2radian = 0.01745; 267 | 268 | thermalBand = img.select(['thermal']) 269 | 270 | def topoCorr_IC(img): 271 | 272 | dem = ee.Image("USGS/SRTMGL1_003") 273 | 274 | 275 | # Extract image metadata about solar position 276 | SZ_rad = ee.Image.constant(ee.Number(img.get('SOLAR_ZENITH_ANGLE'))).multiply(degree2radian).clip(img.geometry().buffer(10000)); 277 | SA_rad = ee.Image.constant(ee.Number(img.get('SOLAR_AZIMUTH_ANGLE'))).multiply(degree2radian).clip(img.geometry().buffer(10000)); 278 | 279 | 280 | # Creat terrain layers 281 | slp = ee.Terrain.slope(dem).clip(img.geometry().buffer(10000)); 282 | slp_rad = ee.Terrain.slope(dem).multiply(degree2radian).clip(img.geometry().buffer(10000)); 283 | asp_rad = ee.Terrain.aspect(dem).multiply(degree2radian).clip(img.geometry().buffer(10000)); 284 | 285 | 286 | 287 | # Calculate the Illumination Condition (IC) 288 | # slope part of the illumination condition 289 | cosZ = SZ_rad.cos(); 290 | cosS = slp_rad.cos(); 291 | slope_illumination = cosS.expression("cosZ * cosS", \ 292 | {'cosZ': cosZ, 'cosS': cosS.select('slope')}); 293 | 294 | 295 | # aspect part of the illumination condition 296 | sinZ = SZ_rad.sin(); 297 | sinS = slp_rad.sin(); 298 | cosAziDiff = (SA_rad.subtract(asp_rad)).cos(); 299 | aspect_illumination = sinZ.expression("sinZ * sinS * cosAziDiff", \ 300 | {'sinZ': sinZ, \ 301 | 'sinS': sinS, \ 302 | 'cosAziDiff': cosAziDiff}); 303 | 304 | # full illumination condition (IC) 305 | ic = slope_illumination.add(aspect_illumination); 306 | 307 | 308 | 309 | # Add IC to original image 310 | img_plus_ic = ee.Image(img.addBands(ic.rename(['IC'])).addBands(cosZ.rename(['cosZ'])).addBands(cosS.rename(['cosS'])).addBands(slp.rename(['slope']))); 311 | 312 | return ee.Image(img_plus_ic); 313 | 314 | def topoCorr_SCSc(img): 315 | img_plus_ic = img; 316 | mask1 = img_plus_ic.select('nir').gt(-0.1); 317 | mask2 = img_plus_ic.select('slope').gte(5) \ 318 | .And(img_plus_ic.select('IC').gte(0)) \ 319 | .And(img_plus_ic.select('nir').gt(-0.1)); 320 | 321 | img_plus_ic_mask2 = ee.Image(img_plus_ic.updateMask(mask2)); 322 | 323 | bandList = ['blue', 'green', 'red', 'nir', 'swir1', 'swir2']; # Specify Bands to topographically correct 324 | 325 | 326 | def applyBands(image): 327 | blue = apply_SCSccorr('blue').select(['blue']) 328 | green = apply_SCSccorr('green').select(['green']) 329 | red = apply_SCSccorr('red').select(['red']) 330 | nir = apply_SCSccorr('nir').select(['nir']) 331 | swir1 = apply_SCSccorr('swir1').select(['swir1']) 332 | swir2 = apply_SCSccorr('swir2').select(['swir2']) 333 | return replace_bands(image, [blue, green, red, nir, swir1, swir2]) 334 | 335 | def apply_SCSccorr(band): 336 | method = 'SCSc'; 337 | 338 | out = img_plus_ic_mask2.select('IC', band).reduceRegion(reducer= ee.Reducer.linearFit(), \ 339 | geometry= ee.Geometry(img.geometry().buffer(-5000)), \ 340 | scale= self.env.terrainScale, \ 341 | maxPixels = 1e13); 342 | 343 | out_a = ee.Number(out.get('scale')); 344 | out_b = ee.Number(out.get('offset')); 345 | out_c = ee.Number(out.get('offset')).divide(ee.Number(out.get('scale'))); 346 | 347 | # apply the SCSc correction 348 | SCSc_output = img_plus_ic_mask2.expression("((image * (cosB * cosZ + cvalue)) / (ic + cvalue))", { 349 | 'image': img_plus_ic_mask2.select([band]), 350 | 'ic': img_plus_ic_mask2.select('IC'), 351 | 'cosB': img_plus_ic_mask2.select('cosS'), 352 | 'cosZ': img_plus_ic_mask2.select('cosZ'), 353 | 'cvalue': out_c }); 354 | 355 | return ee.Image(SCSc_output); 356 | 357 | #img_SCSccorr = ee.Image([apply_SCSccorr(band) for band in bandList]).addBands(img_plus_ic.select('IC')); 358 | img_SCSccorr = applyBands(img).select(bandList).addBands(img_plus_ic.select('IC')) 359 | 360 | bandList_IC = ee.List([bandList, 'IC']).flatten(); 361 | 362 | img_SCSccorr = img_SCSccorr.unmask(img_plus_ic.select(bandList_IC)).select(bandList); 363 | 364 | return img_SCSccorr.unmask(img_plus_ic.select(bandList)) 365 | 366 | 367 | 368 | img = topoCorr_IC(img) 369 | img = topoCorr_SCSc(img) 370 | 371 | return img.addBands(thermalBand) 372 | 373 | 374 | def brdf(self,img): 375 | 376 | import sun_angles 377 | import view_angles 378 | 379 | 380 | def _apply(image, kvol, kvol0): 381 | blue = _correct_band(image, 'blue', kvol, kvol0, f_iso=0.0774, f_geo=0.0079, f_vol=0.0372) 382 | green = _correct_band(image, 'green', kvol, kvol0, f_iso=0.1306, f_geo=0.0178, f_vol=0.0580) 383 | red = _correct_band(image, 'red', kvol, kvol0, f_iso=0.1690, f_geo=0.0227, f_vol=0.0574) 384 | nir = _correct_band(image, 'nir', kvol, kvol0, f_iso=0.3093, f_geo=0.0330, f_vol=0.1535) 385 | swir1 = _correct_band(image, 'swir1', kvol, kvol0, f_iso=0.3430, f_geo=0.0453, f_vol=0.1154) 386 | swir2 = _correct_band(image, 'swir2', kvol, kvol0, f_iso=0.2658, f_geo=0.0387, f_vol=0.0639) 387 | return replace_bands(image, [blue, green, red, nir, swir1, swir2]) 388 | 389 | 390 | def _correct_band(image, band_name, kvol, kvol0, f_iso, f_geo, f_vol): 391 | """fiso + fvol * kvol + fgeo * kgeo""" 392 | iso = ee.Image(f_iso) 393 | geo = ee.Image(f_geo) 394 | vol = ee.Image(f_vol) 395 | pred = vol.multiply(kvol).add(geo.multiply(kvol)).add(iso).rename(['pred']) 396 | pred0 = vol.multiply(kvol0).add(geo.multiply(kvol0)).add(iso).rename(['pred0']) 397 | cfac = pred0.divide(pred).rename(['cfac']) 398 | corr = image.select(band_name).multiply(cfac).rename([band_name]) 399 | return corr 400 | 401 | 402 | def _kvol(sunAz, sunZen, viewAz, viewZen): 403 | """Calculate kvol kernel. 404 | From Lucht et al. 2000 405 | Phase angle = cos(solar zenith) cos(view zenith) + sin(solar zenith) sin(view zenith) cos(relative azimuth)""" 406 | 407 | relative_azimuth = sunAz.subtract(viewAz).rename(['relAz']) 408 | pa1 = viewZen.cos() \ 409 | .multiply(sunZen.cos()) 410 | pa2 = viewZen.sin() \ 411 | .multiply(sunZen.sin()) \ 412 | .multiply(relative_azimuth.cos()) 413 | phase_angle1 = pa1.add(pa2) 414 | phase_angle = phase_angle1.acos() 415 | p1 = ee.Image(PI().divide(2)).subtract(phase_angle) 416 | p2 = p1.multiply(phase_angle1) 417 | p3 = p2.add(phase_angle.sin()) 418 | p4 = sunZen.cos().add(viewZen.cos()) 419 | p5 = ee.Image(PI().divide(4)) 420 | 421 | kvol = p3.divide(p4).subtract(p5).rename(['kvol']) 422 | 423 | viewZen0 = ee.Image(0) 424 | pa10 = viewZen0.cos() \ 425 | .multiply(sunZen.cos()) 426 | pa20 = viewZen0.sin() \ 427 | .multiply(sunZen.sin()) \ 428 | .multiply(relative_azimuth.cos()) 429 | phase_angle10 = pa10.add(pa20) 430 | phase_angle0 = phase_angle10.acos() 431 | p10 = ee.Image(PI().divide(2)).subtract(phase_angle0) 432 | p20 = p10.multiply(phase_angle10) 433 | p30 = p20.add(phase_angle0.sin()) 434 | p40 = sunZen.cos().add(viewZen0.cos()) 435 | p50 = ee.Image(PI().divide(4)) 436 | 437 | kvol0 = p30.divide(p40).subtract(p50).rename(['kvol0']) 438 | 439 | return (kvol, kvol0) 440 | 441 | date = img.date() 442 | footprint = determine_footprint(img) 443 | (sunAz, sunZen) = sun_angles.create(date, footprint) 444 | (viewAz, viewZen) = view_angles.create(footprint) 445 | (kvol, kvol0) = _kvol(sunAz, sunZen, viewAz, viewZen) 446 | return _apply(img, kvol.multiply(PI()), kvol0.multiply(PI())) 447 | 448 | 449 | def medoidMosaic(self,collection): 450 | """ medoid composite with equal weight among indices """ 451 | 452 | bandNames = ee.Image(collection.first()).bandNames() 453 | otherBands = bandNames.removeAll(self.env.divideBands) 454 | 455 | others = collection.select(otherBands).reduce(ee.Reducer.mean()).rename(otherBands); 456 | 457 | collection = collection.select(self.env.divideBands) 458 | 459 | bandNumbers = ee.List.sequence(1,self.env.divideBands.length()); 460 | 461 | median = ee.ImageCollection(collection).median() 462 | 463 | def subtractmedian(img): 464 | diff = ee.Image(img).subtract(median).pow(ee.Image.constant(2)); 465 | return diff.reduce('sum').addBands(img); 466 | 467 | medoid = collection.map(subtractmedian) 468 | 469 | medoid = ee.ImageCollection(medoid).reduce(ee.Reducer.min(self.env.divideBands.length().add(1))).select(bandNumbers,self.env.divideBands); 470 | 471 | return medoid.addBands(others); 472 | 473 | def medianMosaic(self,collection): 474 | 475 | """ median composite """ 476 | median = collection.select(medianIncludeBands).median(); 477 | othersBands = bandNames.removeAll(medianIncludeBands); 478 | others = collection.select(otherBands).mean(); 479 | 480 | return median.addBands(others) 481 | 482 | 483 | def setMetaData(self,img): 484 | """ add metadata to image """ 485 | 486 | img = ee.Image(img).set({'system:time_start':ee.Date(self.env.startDate).millis(), \ 487 | 'startDOY':str(self.env.startDoy), \ 488 | 'endDOY':str(self.env.endDoy), \ 489 | 'useCloudScore':str(self.env.cloudMask), \ 490 | 'useTDOM':str(self.env.shadowMask), \ 491 | 'useSRmask':str(self.env.maskSR ), \ 492 | 'useCloudProject':str(self.env.cloudMask), \ 493 | 'terrain':str(self.env.terrainCorrection), \ 494 | 'cloudScoreThresh':str(self.env.cloudScoreThresh), \ 495 | 'cloudScorePctl':str(self.env.cloudScorePctl), \ 496 | 'zScoreThresh':str(self.env.zScoreThresh), \ 497 | 'shadowSumThresh':str(self.env.shadowSumThresh), \ 498 | 'contractPixels':str(self.env.contractPixels), \ 499 | 'cloudFilter':str(self.env.metadataCloudCoverMax),\ 500 | 'crs':str(self.env.epsg), \ 501 | 'dilatePixels':str(self.env.dilatePixels)}) 502 | 503 | return img 504 | 505 | def exportMap(self,img,studyArea,week): 506 | 507 | geom = studyArea.geometry().bounds().getInfo(); 508 | 509 | task_ordered= ee.batch.Export.image.toAsset(image=img.clip(studyArea), 510 | description = self.env.name + str(week), 511 | assetId= self.env.assetId + self.env.name + str(week).zfill(3), 512 | region=geom['coordinates'], 513 | maxPixels=1e13, 514 | crs=self.env.epsg, 515 | scale=self.env.exportScale) 516 | 517 | task_ordered.start() 518 | 519 | 520 | 521 | if __name__ == "__main__": 522 | 523 | ee.Initialize() 524 | 525 | studyArea = ee.FeatureCollection("users/apoortinga/countries/Ecuador_nxprovincias") #.geometry() #.bounds(); 526 | 527 | 528 | # 2015 529 | year = ee.Date("2016-01-01") 530 | startWeek = 39 531 | startDay = [168,182,196,210,224,238,252,266,280,294,308,322,336,350,364] 532 | endDay = [181,195,209,223,237,251,265,279,293,307,321,335,349,363,377] 533 | 534 | # 2016 535 | year = ee.Date("2016-01-01") 536 | startWeek = 54 537 | startDay = [13,27,41,55,69,83,97,111,125,139,153,167,181,195,209,223,237,251,265,279,293,307,321,335,349,363] 538 | endDay = [26,40,54,68,82,96,110,124,138,152,166,180,194,208,222,236,250,264,278,292,306,320,334,348,362,376] 539 | 540 | # 2017 541 | year = ee.Date("2017-01-01") 542 | startWeek = 80 543 | startDay = [11,25,39,53,67,81,95,109,123,137,151,165,179,193,207,221,235,249,263,277,291,305,319,333,347,361] 544 | endDay = [24,38,52,66,80,94,108,122,136,150,164,178,192,206,220,234,248,262,276,290,304,318,332,346,360,374] 545 | 546 | # 2018 547 | year = ee.Date("2018-01-01") 548 | startWeek = 106 549 | startDay = [10,24,38,52,66,80,94,108,122,136,150,164,178,192,206,220,234,248,262,276,290,304,318,332,346,360] 550 | endDay = [23,37,51,65,79,93,107,121,135,149,163,177,191,205,219,233,247,261,275,289,303,317,331,345,359,373] 551 | 552 | 553 | for i in range(2,3,1): 554 | startDate = year.advance(startDay[i],"day") 555 | endDate = year.advance(endDay[i],"day") 556 | 557 | functions().main(studyArea,startDate,endDate,startDay[i],endDay[i],startWeek+i) 558 | -------------------------------------------------------------------------------- /landsatYearlyComposite.py: -------------------------------------------------------------------------------- 1 | # Landsat package 2 | 3 | 4 | import ee 5 | import math 6 | from utils import * 7 | from paramsTemplate import * 8 | 9 | class env(object): 10 | 11 | def __init__(self): 12 | """Initialize the environment.""" 13 | ########################################## 14 | # Export variables # 15 | ########################################## 16 | 17 | self.assetId ="projects/Sacha/PreprocessedData/L8_Annual_V4_QBands/" 18 | self.name = "LS_AN_" 19 | self.exportScale = 20 20 | self.epsg = "EPSG:32717" 21 | 22 | # Initialize the Earth Engine object, using the authentication credentials. 23 | self.startDate = "" 24 | self.endDate = "" 25 | self.location = ee.Geometry.Polygon([[103.876,18.552],[105.806,18.552],[105.806,19.999],[103.876,19.999],[103.876,18.552]]) 26 | 27 | self.metadataCloudCoverMax = 80 28 | self.cloudThreshold = 10 29 | self.hazeThresh = 200 30 | 31 | self.maskSR = True 32 | self.cloudMask = True 33 | self.hazeMask = True 34 | self.shadowMask = True 35 | self.brdfCorrect = True 36 | self.terrainCorrection = True 37 | 38 | self.terrainScale = 600 39 | self.dem = ee.Image("JAXA/ALOS/AW3D30_V1_1").select("MED") 40 | 41 | self.cloudScoreThresh= 2 42 | self.cloudScorePctl= 11 43 | self.zScoreThresh= -0.8 44 | self.shadowSumThresh= 0.15 45 | self.contractPixels= 1.2 46 | self.dilatePixels= 2.5 47 | 48 | 49 | self.SLC = False 50 | self.percentiles = [20,80] 51 | 52 | self.medoidBands = ee.List(['blue','green','red','nir','swir1','swir2']) 53 | self.divideBands = ee.List(['blue','green','red','nir','swir1','swir2']) 54 | self.bandNamesLandsat = ee.List(['blue','green','red','nir','swir1','thermal','swir2','sr_atmos_opacity','pixel_qa','radsat_qa']) 55 | self.sensorBandDictLandsatSR = ee.Dictionary({'L8' : ee.List([1,2,3,4,5,7,6,9,10,11]),\ 56 | 'L7' : ee.List([0,1,2,3,4,5,6,7,9,10]),\ 57 | 'L5' : ee.List([0,1,2,3,4,5,6,7,9,10]),\ 58 | 'L4' : ee.List([0,1,2,3,4,5,6,7,9,10])}) 59 | 60 | 61 | class functions(): 62 | def __init__(self): 63 | """Initialize the Surfrace Reflectance app.""" 64 | 65 | # get the environment 66 | self.env = env() 67 | 68 | def getLandsat(self,aoi,year,regionName): 69 | self.env.regionName = regionName 70 | self.env.location = aoi 71 | self.env.startDate = ee.Date.fromYMD(year,1,1) 72 | self.env.endDate = ee.Date.fromYMD(year+1,1,1) 73 | 74 | self.paramSwitch = Switcher().paramSelect(self.env.regionName) 75 | 76 | self.env.cloudScoreThresh = self.paramSwitch[0] 77 | self.env.cloudScorePctl= self.paramSwitch[1] 78 | self.env.zScoreThresh= self.paramSwitch[2] 79 | self.env.shadowSumThresh= self.paramSwitch[3] 80 | self.env.contractPixels= self.paramSwitch[4] 81 | self.env.dilatePixels= self.paramSwitch[5] 82 | 83 | 84 | landsat5 = ee.ImageCollection('LANDSAT/LT05/C01/T1_SR').filterDate(self.env.startDate,self.env.endDate).filterBounds(self.env.location) 85 | landsat5 = landsat5.filterMetadata('CLOUD_COVER','less_than',self.env.metadataCloudCoverMax) 86 | landsat5 = landsat5.select(self.env.sensorBandDictLandsatSR.get('L5'),self.env.bandNamesLandsat).map(self.defringe) 87 | 88 | landsat7 = ee.ImageCollection('LANDSAT/LE07/C01/T1_SR').filterDate(self.env.startDate,self.env.endDate).filterBounds(self.env.location) 89 | landsat7 = landsat7.filterMetadata('CLOUD_COVER','less_than',self.env.metadataCloudCoverMax) 90 | landsat7 = landsat7.select(self.env.sensorBandDictLandsatSR.get('L7'),self.env.bandNamesLandsat) 91 | 92 | 93 | landsat8 = ee.ImageCollection('LANDSAT/LC08/C01/T1_SR').filterDate(self.env.startDate,self.env.endDate).filterBounds(self.env.location) 94 | landsat8 = landsat8.filterMetadata('CLOUD_COVER','less_than',self.env.metadataCloudCoverMax) 95 | landsat8 = landsat8.select(self.env.sensorBandDictLandsatSR.get('L8'),self.env.bandNamesLandsat) 96 | 97 | if (self.env.SLC or year < 2004): 98 | landsat = landsat5.merge(landsat7).merge(landsat8) 99 | else: 100 | landsat = landsat5.merge(landsat8) 101 | 102 | print landsat.size().getInfo() 103 | if landsat.size().getInfo() > 0: 104 | 105 | # mask clouds using the QA band 106 | if self.env.maskSR == True: 107 | #print "removing clouds" 108 | landsat = landsat.map(self.CloudMaskSRL8) 109 | 110 | 111 | # mask clouds using cloud mask function 112 | if self.env.hazeMask == True: 113 | #print "removing haze" 114 | landsat = landsat.map(self.maskHaze) 115 | 116 | 117 | # mask clouds using cloud mask function 118 | if self.env.shadowMask == True: 119 | #print "shadow masking" 120 | self.fullCollection = ee.ImageCollection('LANDSAT/LC08/C01/T1_SR').filterBounds(self.env.location).select(self.env.sensorBandDictLandsatSR.get('L8'),self.env.bandNamesLandsat) 121 | landsat = self.maskShadows(landsat) 122 | 123 | 124 | landsat = landsat.map(self.scaleLandsat).map(self.addDateYear) 125 | 126 | # mask clouds using cloud mask function 127 | if self.env.cloudMask == True: 128 | #print "removing some more clouds" 129 | landsat = landsat.map(self.maskClouds) 130 | 131 | if self.env.brdfCorrect == True: 132 | landsat = landsat.map(self.brdf) 133 | 134 | if self.env.terrainCorrection == True: 135 | landsat = ee.ImageCollection(landsat.map(self.terrain)) 136 | 137 | medoid = self.medoidMosaic(landsat) 138 | medoidDown = ee.Image(self.medoidMosaicPercentiles(landsat,self.env.percentiles[0])) 139 | medoidUp = self.medoidMosaicPercentiles(landsat,self.env.percentiles[1]) 140 | stdevBands = self.addSTDdev(landsat) 141 | 142 | mosaic = medoid.addBands(medoidDown).addBands(medoidUp).addBands(stdevBands) 143 | 144 | 145 | 146 | mosaic = self.reScaleLandsat(mosaic) 147 | 148 | mosaic = self.setMetaData(mosaic) 149 | 150 | self.exportMap(mosaic,self.env.location) 151 | return mosaic 152 | 153 | def addDateYear(self,img): 154 | """add a date and year bands""" 155 | date = ee.Date(img.get("system:time_start")) 156 | 157 | day = date.getRelative('day','year').add(1); 158 | yr = date.get('year'); 159 | mk = img.mask().reduce(ee.Reducer.min()); 160 | 161 | img = img.addBands(ee.Image.constant(day).mask(mk).uint16().rename('date')); 162 | img = img.addBands(ee.Image.constant(yr).mask(mk).uint16().rename('year')); 163 | 164 | return img; 165 | 166 | def CloudMaskSRL8(self,img): 167 | """apply cf-mask Landsat""" 168 | QA = img.select("pixel_qa") 169 | 170 | shadow = QA.bitwiseAnd(8).neq(0); 171 | cloud = QA.bitwiseAnd(32).neq(0); 172 | return img.updateMask(shadow.Not()).updateMask(cloud.Not()).copyProperties(img) 173 | 174 | def scaleLandsat(self,img): 175 | """Landast is scaled by factor 0.0001 """ 176 | thermal = img.select(ee.List(['thermal'])).multiply(0.1) 177 | scaled = ee.Image(img).select(self.env.divideBands).multiply(ee.Number(0.0001)) 178 | 179 | return img.select(['TDOMMask']).addBands(scaled).addBands(thermal) 180 | 181 | def reScaleLandsat(self,img): 182 | """Landast is scaled by factor 0.0001 """ 183 | 184 | noScaleBands = ee.List(['date','year','TDOMMask','cloudMask','count']) 185 | noScale = ee.Image(img).select(noScaleBands) 186 | 187 | thermalBand = ee.List(['thermal']) 188 | thermal = ee.Image(img).select(thermalBand).multiply(10) 189 | 190 | otherBands = ee.Image(img).bandNames().removeAll(thermalBand).removeAll(noScaleBands) 191 | scaled = ee.Image(img).select(otherBands).divide(0.0001) 192 | 193 | image = ee.Image(scaled.addBands([thermal,noScale])).int16() 194 | 195 | return image.copyProperties(img) 196 | 197 | 198 | def maskHaze(self,img): 199 | """ mask haze """ 200 | opa = ee.Image(img.select(['sr_atmos_opacity']).multiply(0.001)) 201 | haze = opa.gt(self.env.hazeThresh) 202 | return img.updateMask(haze.Not()) 203 | 204 | 205 | def maskClouds(self,img): 206 | """ 207 | Computes spectral indices of cloudyness and take the minimum of them. 208 | 209 | Each spectral index is fairly lenient because the group minimum 210 | is a somewhat stringent comparison policy. side note -> this seems like a job for machine learning :) 211 | originally written by Matt Hancher for Landsat imageryadapted to Sentinel by Chris Hewig and Ian Housman 212 | """ 213 | 214 | score = ee.Image(1.0); 215 | # Clouds are reasonably bright in the blue band. 216 | blue_rescale = img.select('blue').subtract(ee.Number(0.1)).divide(ee.Number(0.3).subtract(ee.Number(0.1))) 217 | score = score.min(blue_rescale); 218 | 219 | # Clouds are reasonably bright in all visible bands. 220 | visible = img.select('red').add(img.select('green')).add(img.select('blue')) 221 | visible_rescale = visible.subtract(ee.Number(0.2)).divide(ee.Number(0.8).subtract(ee.Number(0.2))) 222 | score = score.min(visible_rescale); 223 | 224 | # Clouds are reasonably bright in all infrared bands. 225 | infrared = img.select('nir').add(img.select('swir1')).add(img.select('swir2')) 226 | infrared_rescale = infrared.subtract(ee.Number(0.3)).divide(ee.Number(0.8).subtract(ee.Number(0.3))) 227 | score = score.min(infrared_rescale); 228 | 229 | # Clouds are reasonably cool in temperature. 230 | temp_rescale = img.select('thermal').subtract(ee.Number(300)).divide(ee.Number(290).subtract(ee.Number(300))) 231 | score = score.min(temp_rescale); 232 | 233 | # However, clouds are not snow. 234 | ndsi = img.normalizedDifference(['green', 'swir1']); 235 | ndsi_rescale = ndsi.subtract(ee.Number(0.8)).divide(ee.Number(0.6).subtract(ee.Number(0.8))) 236 | score = score.min(ndsi_rescale).multiply(100).byte(); 237 | mask = score.lt(self.env.cloudScoreThresh).rename(['cloudMask']); 238 | img = img.updateMask(mask).addBands([mask]); 239 | 240 | return img; 241 | 242 | def maskShadows(self,collection): 243 | 244 | def TDOM(image): 245 | zScore = image.select(shadowSumBands).subtract(irMean).divide(irStdDev) 246 | irSum = image.select(shadowSumBands).reduce(ee.Reducer.sum()) 247 | TDOMMask = zScore.lt(self.env.zScoreThresh).reduce(ee.Reducer.sum()).eq(2)\ 248 | .And(irSum.lt(self.env.shadowSumThresh)).Not() 249 | TDOMMask = TDOMMask.focal_min(self.env.contractPixels).focal_max(self.env.dilatePixels).rename(['TDOMMask']) 250 | 251 | image = image.addBands([TDOMMask]) 252 | 253 | return image.updateMask(TDOMMask) 254 | 255 | shadowSumBands = ['nir','swir1'] 256 | 257 | # Get some pixel-wise stats for the time series 258 | irStdDev = self.fullCollection.select(shadowSumBands).reduce(ee.Reducer.stdDev()) 259 | irMean = self.fullCollection.select(shadowSumBands).reduce(ee.Reducer.mean()) 260 | 261 | # Mask out dark dark outliers 262 | collection_tdom = collection.map(TDOM) 263 | 264 | return collection_tdom 265 | 266 | def addSTDdev(self,collection): 267 | 268 | def addSTDdevIndices(img): 269 | """ Function to add common (and less common) spectral indices to an image. 270 | Includes the Normalized Difference Spectral Vector from (Angiuli and Trianni, 2014) """ 271 | img = img.addBands(img.normalizedDifference(['green','swir1']).rename(['ND_green_swir1'])); # NDSI, MNDWI 272 | img = img.addBands(img.normalizedDifference(['nir','red']).rename(['ND_nir_red'])); # NDVI 273 | img = img.addBands(img.normalizedDifference(['nir','swir2']).rename(['ND_nir_swir2'])); # NBR, MNDVI 274 | 275 | return img; 276 | 277 | 278 | 279 | blue_stdDev = collection.select(["blue"]).reduce(ee.Reducer.stdDev()).rename(['blue_stdDev']) 280 | red_stdDev = collection.select(["red"]).reduce(ee.Reducer.stdDev()).rename(['red_stdDev']) 281 | green_stdDev = collection.select(["green"]).reduce(ee.Reducer.stdDev()).rename(['green_stdDev']) 282 | nir_stdDev = collection.select(["nir"]).reduce(ee.Reducer.stdDev()).rename(['nir_stdDev']) 283 | swir1_stdDev = collection.select(["swir1"]).reduce(ee.Reducer.stdDev()).rename(['swir1_stdDev']) 284 | swir2_stdDev = collection.select(["swir2"]).reduce(ee.Reducer.stdDev()).rename(['swir2_stdDev']) 285 | 286 | col = collection.map(addSTDdevIndices) 287 | 288 | ND_green_swir1 = col.select(['ND_green_swir1']).reduce(ee.Reducer.stdDev()).rename(['ND_green_swir1_stdDev']); 289 | ND_nir_red = col.select(['ND_nir_red']).reduce(ee.Reducer.stdDev()).rename(['ND_nir_red_stdDev']); 290 | ND_nir_swir2 = col.select(['ND_nir_swir2']).reduce(ee.Reducer.stdDev()).rename(['ND_nir_swir2_stdDev']); 291 | 292 | stdevBands = ee.Image(blue_stdDev.addBands(red_stdDev).addBands(green_stdDev).addBands(nir_stdDev).addBands(swir1_stdDev).addBands(swir2_stdDev)\ 293 | .addBands(ND_green_swir1).addBands(ND_nir_red).addBands(ND_nir_swir2)) 294 | 295 | return stdevBands 296 | 297 | 298 | 299 | def defringe(self,img): 300 | 301 | # threshold for defringing landsat5 and 7 302 | fringeCountThreshold = 279 303 | 304 | k = ee.Kernel.fixed(41, 41, 305 | [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 306 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 307 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 308 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 309 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 310 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 311 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 312 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 313 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 314 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 315 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 316 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 317 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 318 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 319 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 320 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 321 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 322 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 323 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 324 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 325 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 326 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 327 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 328 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 329 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 330 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 331 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 332 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 333 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 334 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 335 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 336 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 337 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 338 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 339 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 340 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 341 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 342 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 343 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 344 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 345 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]); 346 | 347 | 348 | m = ee.Image(img).mask().reduce(ee.Reducer.min()) 349 | sum = m.reduceNeighborhood(ee.Reducer.sum(), k, 'kernel') 350 | mask = sum.gte(fringeCountThreshold) 351 | 352 | return img.updateMask(mask) 353 | 354 | 355 | def terrain(self,img): 356 | 357 | 358 | degree2radian = 0.01745; 359 | otherBands = img.select(['thermal','date','year','TDOMMask','cloudMask']) 360 | 361 | def topoCorr_IC(img): 362 | 363 | dem = self.env.dem 364 | 365 | 366 | # Extract image metadata about solar position 367 | SZ_rad = ee.Image.constant(ee.Number(img.get('SOLAR_ZENITH_ANGLE'))).multiply(degree2radian).clip(img.geometry().buffer(10000)); 368 | SA_rad = ee.Image.constant(ee.Number(img.get('SOLAR_AZIMUTH_ANGLE'))).multiply(degree2radian).clip(img.geometry().buffer(10000)); 369 | 370 | 371 | # Creat terrain layers 372 | slp = ee.Terrain.slope(dem).clip(img.geometry().buffer(10000)); 373 | slp_rad = ee.Terrain.slope(dem).multiply(degree2radian).clip(img.geometry().buffer(10000)); 374 | asp_rad = ee.Terrain.aspect(dem).multiply(degree2radian).clip(img.geometry().buffer(10000)); 375 | 376 | 377 | 378 | # Calculate the Illumination Condition (IC) 379 | # slope part of the illumination condition 380 | cosZ = SZ_rad.cos(); 381 | cosS = slp_rad.cos(); 382 | slope_illumination = cosS.expression("cosZ * cosS", \ 383 | {'cosZ': cosZ, 'cosS': cosS.select('slope')}); 384 | 385 | 386 | # aspect part of the illumination condition 387 | sinZ = SZ_rad.sin(); 388 | sinS = slp_rad.sin(); 389 | cosAziDiff = (SA_rad.subtract(asp_rad)).cos(); 390 | aspect_illumination = sinZ.expression("sinZ * sinS * cosAziDiff", \ 391 | {'sinZ': sinZ, \ 392 | 'sinS': sinS, \ 393 | 'cosAziDiff': cosAziDiff}); 394 | 395 | # full illumination condition (IC) 396 | ic = slope_illumination.add(aspect_illumination); 397 | 398 | 399 | 400 | # Add IC to original image 401 | img_plus_ic = ee.Image(img.addBands(ic.rename(['IC'])).addBands(cosZ.rename(['cosZ'])).addBands(cosS.rename(['cosS'])).addBands(slp.rename(['slope']))); 402 | 403 | return ee.Image(img_plus_ic); 404 | 405 | def topoCorr_SCSc(img): 406 | img_plus_ic = img; 407 | mask1 = img_plus_ic.select('nir').gt(-0.1); 408 | mask2 = img_plus_ic.select('slope').gte(5) \ 409 | .And(img_plus_ic.select('IC').gte(0)) \ 410 | .And(img_plus_ic.select('nir').gt(-0.1)); 411 | 412 | img_plus_ic_mask2 = ee.Image(img_plus_ic.updateMask(mask2)); 413 | 414 | bandList = ['blue', 'green', 'red', 'nir', 'swir1', 'swir2']; # Specify Bands to topographically correct 415 | 416 | 417 | def applyBands(image): 418 | blue = apply_SCSccorr('blue').select(['blue']) 419 | green = apply_SCSccorr('green').select(['green']) 420 | red = apply_SCSccorr('red').select(['red']) 421 | nir = apply_SCSccorr('nir').select(['nir']) 422 | swir1 = apply_SCSccorr('swir1').select(['swir1']) 423 | swir2 = apply_SCSccorr('swir2').select(['swir2']) 424 | return replace_bands(image, [blue, green, red, nir, swir1, swir2]) 425 | 426 | def apply_SCSccorr(band): 427 | method = 'SCSc'; 428 | 429 | out = ee.Image(1).addBands(img_plus_ic_mask2.select('IC', band)).reduceRegion(reducer= ee.Reducer.linearRegression(2,1), \ 430 | geometry= ee.Geometry(img.geometry().buffer(-5000)), \ 431 | scale= self.env.terrainScale, \ 432 | bestEffort =True, 433 | maxPixels=1e10) 434 | 435 | 436 | #out_a = ee.Number(out.get('scale')); 437 | #out_b = ee.Number(out.get('offset')); 438 | #out_c = ee.Number(out.get('offset')).divide(ee.Number(out.get('scale'))); 439 | 440 | 441 | fit = out.combine({"coefficients": ee.Array([[1],[1]])}, False); 442 | 443 | #Get the coefficients as a nested list, 444 | #cast it to an array, and get just the selected column 445 | out_a = (ee.Array(fit.get('coefficients')).get([0,0])); 446 | out_b = (ee.Array(fit.get('coefficients')).get([1,0])); 447 | out_c = out_a.divide(out_b) 448 | 449 | 450 | # apply the SCSc correction 451 | SCSc_output = img_plus_ic_mask2.expression("((image * (cosB * cosZ + cvalue)) / (ic + cvalue))", { 452 | 'image': img_plus_ic_mask2.select([band]), 453 | 'ic': img_plus_ic_mask2.select('IC'), 454 | 'cosB': img_plus_ic_mask2.select('cosS'), 455 | 'cosZ': img_plus_ic_mask2.select('cosZ'), 456 | 'cvalue': out_c }); 457 | 458 | return ee.Image(SCSc_output); 459 | 460 | #img_SCSccorr = ee.Image([apply_SCSccorr(band) for band in bandList]).addBands(img_plus_ic.select('IC')); 461 | img_SCSccorr = applyBands(img).select(bandList).addBands(img_plus_ic.select('IC')) 462 | 463 | bandList_IC = ee.List([bandList, 'IC']).flatten(); 464 | 465 | img_SCSccorr = img_SCSccorr.unmask(img_plus_ic.select(bandList_IC)).select(bandList); 466 | 467 | return img_SCSccorr.unmask(img_plus_ic.select(bandList)) 468 | 469 | 470 | 471 | img = topoCorr_IC(img) 472 | img = topoCorr_SCSc(img) 473 | 474 | return img.addBands(otherBands) 475 | 476 | 477 | 478 | def brdf(self,img): 479 | 480 | import sun_angles 481 | import view_angles 482 | 483 | 484 | def _apply(image, kvol, kvol0): 485 | blue = _correct_band(image, 'blue', kvol, kvol0, f_iso=0.0774, f_geo=0.0079, f_vol=0.0372) 486 | green = _correct_band(image, 'green', kvol, kvol0, f_iso=0.1306, f_geo=0.0178, f_vol=0.0580) 487 | red = _correct_band(image, 'red', kvol, kvol0, f_iso=0.1690, f_geo=0.0227, f_vol=0.0574) 488 | nir = _correct_band(image, 'nir', kvol, kvol0, f_iso=0.3093, f_geo=0.0330, f_vol=0.1535) 489 | swir1 = _correct_band(image, 'swir1', kvol, kvol0, f_iso=0.3430, f_geo=0.0453, f_vol=0.1154) 490 | swir2 = _correct_band(image, 'swir2', kvol, kvol0, f_iso=0.2658, f_geo=0.0387, f_vol=0.0639) 491 | return replace_bands(image, [blue, green, red, nir, swir1, swir2]) 492 | 493 | 494 | def _correct_band(image, band_name, kvol, kvol0, f_iso, f_geo, f_vol): 495 | """fiso + fvol * kvol + fgeo * kgeo""" 496 | iso = ee.Image(f_iso) 497 | geo = ee.Image(f_geo) 498 | vol = ee.Image(f_vol) 499 | pred = vol.multiply(kvol).add(geo.multiply(kvol)).add(iso).rename(['pred']) 500 | pred0 = vol.multiply(kvol0).add(geo.multiply(kvol0)).add(iso).rename(['pred0']) 501 | cfac = pred0.divide(pred).rename(['cfac']) 502 | corr = image.select(band_name).multiply(cfac).rename([band_name]) 503 | return corr 504 | 505 | 506 | def _kvol(sunAz, sunZen, viewAz, viewZen): 507 | """Calculate kvol kernel. 508 | From Lucht et al. 2000 509 | Phase angle = cos(solar zenith) cos(view zenith) + sin(solar zenith) sin(view zenith) cos(relative azimuth)""" 510 | 511 | relative_azimuth = sunAz.subtract(viewAz).rename(['relAz']) 512 | pa1 = viewZen.cos() \ 513 | .multiply(sunZen.cos()) 514 | pa2 = viewZen.sin() \ 515 | .multiply(sunZen.sin()) \ 516 | .multiply(relative_azimuth.cos()) 517 | phase_angle1 = pa1.add(pa2) 518 | phase_angle = phase_angle1.acos() 519 | p1 = ee.Image(PI().divide(2)).subtract(phase_angle) 520 | p2 = p1.multiply(phase_angle1) 521 | p3 = p2.add(phase_angle.sin()) 522 | p4 = sunZen.cos().add(viewZen.cos()) 523 | p5 = ee.Image(PI().divide(4)) 524 | 525 | kvol = p3.divide(p4).subtract(p5).rename(['kvol']) 526 | 527 | viewZen0 = ee.Image(0) 528 | pa10 = viewZen0.cos() \ 529 | .multiply(sunZen.cos()) 530 | pa20 = viewZen0.sin() \ 531 | .multiply(sunZen.sin()) \ 532 | .multiply(relative_azimuth.cos()) 533 | phase_angle10 = pa10.add(pa20) 534 | phase_angle0 = phase_angle10.acos() 535 | p10 = ee.Image(PI().divide(2)).subtract(phase_angle0) 536 | p20 = p10.multiply(phase_angle10) 537 | p30 = p20.add(phase_angle0.sin()) 538 | p40 = sunZen.cos().add(viewZen0.cos()) 539 | p50 = ee.Image(PI().divide(4)) 540 | 541 | kvol0 = p30.divide(p40).subtract(p50).rename(['kvol0']) 542 | 543 | return (kvol, kvol0) 544 | 545 | date = img.date() 546 | footprint = determine_footprint(img) 547 | (sunAz, sunZen) = sun_angles.create(date, footprint) 548 | (viewAz, viewZen) = view_angles.create(footprint) 549 | (kvol, kvol0) = _kvol(sunAz, sunZen, viewAz, viewZen) 550 | return _apply(img, kvol.multiply(PI()), kvol0.multiply(PI())) 551 | 552 | def medoidMosaic(self,collection): 553 | """ medoid composite with equal weight among indices """ 554 | nImages = ee.ImageCollection(collection).select([0]).count().rename('count') 555 | bandNames = ee.Image(collection.first()).bandNames() 556 | otherBands = bandNames.removeAll(self.env.divideBands) 557 | 558 | others = collection.select(otherBands).reduce(ee.Reducer.mean()).rename(otherBands); 559 | 560 | collection = collection.select(self.env.divideBands) 561 | 562 | bandNumbers = ee.List.sequence(1,self.env.divideBands.length()); 563 | 564 | median = ee.ImageCollection(collection).median() 565 | 566 | def subtractmedian(img): 567 | diff = ee.Image(img).subtract(median).pow(ee.Image.constant(2)); 568 | return diff.reduce('sum').addBands(img); 569 | 570 | medoid = collection.map(subtractmedian) 571 | 572 | medoid = ee.ImageCollection(medoid).reduce(ee.Reducer.min(self.env.divideBands.length().add(1))).select(bandNumbers,self.env.divideBands); 573 | 574 | return medoid.addBands(others).addBands(nImages); 575 | 576 | 577 | def medoidMosaicPercentiles(self,inCollection,p): 578 | ' calculate the medoid of a percentile' 579 | 580 | inCollection = inCollection.select(self.env.medoidBands) 581 | 582 | p1 = p 583 | p2 = 100 -p 584 | 585 | med1 = self.medoidPercentiles(inCollection,p1).select(["green","nir"]) 586 | med2 = self.medoidPercentiles(inCollection,p2).select(["blue","red","swir1","swir2"]) 587 | 588 | medoidP = self.renameBands(ee.Image(med1).addBands(med2),str("p")+str(p)) 589 | return medoidP 590 | 591 | 592 | def medoidPercentiles(self,inCollection,p): 593 | 594 | # Find band names in first image 595 | bandNumbers = ee.List.sequence(1,self.env.medoidBands.length()); 596 | 597 | # Find the median 598 | percentile = inCollection.select(self.env.medoidBands).reduce(ee.Reducer.percentile([p])); 599 | 600 | def subtractPercentile(img): 601 | diff = ee.Image(img).subtract(percentile).pow(ee.Image.constant(2)); 602 | return diff.reduce('sum').addBands(img); 603 | 604 | percentile = inCollection.map(subtractPercentile) 605 | 606 | percentile = ee.ImageCollection(percentile).reduce(ee.Reducer.min(self.env.medoidBands.length().add(1))).select(bandNumbers,self.env.medoidBands); 607 | 608 | return percentile; 609 | 610 | 611 | def renameBands(self,image,prefix): 612 | 'rename bands with prefix' 613 | 614 | bandnames = image.bandNames(); 615 | 616 | def mapBands(band): 617 | band = ee.String(prefix).cat('_').cat(band); 618 | return band; 619 | 620 | bandnames = bandnames.map(mapBands) 621 | 622 | image = image.rename(bandnames); 623 | 624 | return image; 625 | 626 | def setMetaData(self,img): 627 | 628 | img = ee.Image(img).set({'regionName': str(self.env.regionName), 629 | 'system:time_start':ee.Date(self.env.startDate).millis(), 630 | 'compositingMethod':'medoid', 631 | 'toaOrSR':'SR', 632 | 'epsg':str(self.env.epsg), 633 | 'exportScale':str(self.env.exportScale), 634 | 'shadowSumThresh':str(self.env.shadowSumThresh), 635 | 'maskSR':str(self.env.maskSR), 636 | 'cloudMask':str(self.env.cloudMask), 637 | 'hazeMask':str(self.env.hazeMask), 638 | 'shadowMask':str(self.env.shadowMask), 639 | 'brdfCorrect':str(self.env.brdfCorrect), 640 | 'terrainCorrection':str(self.env.terrainCorrection), 641 | 'contractPixels':str(self.env.contractPixels), 642 | 'dilatePixels':str(self.env.dilatePixels), 643 | 'metadataCloudCoverMax':str(self.env.metadataCloudCoverMax), 644 | 'hazeThresh':str(self.env.hazeThresh), 645 | 'terrainScale':str(self.env.terrainScale)}) 646 | 647 | return img 648 | def exportMap(self,img,studyArea): 649 | 650 | geom = studyArea.getInfo(); 651 | sd = str(self.env.startDate.getRelative('day','year').getInfo()).zfill(3); 652 | ed = str(self.env.endDate.getRelative('day','year').getInfo()).zfill(3); 653 | year = str(self.env.startDate.get('year').getInfo()); 654 | regionName = self.env.regionName.replace(" ",'_') + "_" 655 | 656 | task_ordered= ee.batch.Export.image.toAsset(image=img, 657 | description = self.env.name + regionName + year + sd + ed, 658 | assetId= self.env.assetId + self.env.name + regionName + year + sd + ed, 659 | region=geom['coordinates'], 660 | maxPixels=1e13, 661 | crs=self.env.epsg, 662 | scale=self.env.exportScale) 663 | 664 | task_ordered.start() 665 | print(self.env.assetId + self.env.name + regionName + year + sd + ed) 666 | 667 | #def composite(aoi,year): 668 | # img = ee.Image(functions().getLandsat(aoi,year)) 669 | # return img 670 | 671 | 672 | if __name__ == "__main__": 673 | #def composite(aoi,year): 674 | 675 | ee.Initialize() 676 | for i in range(0,10): 677 | year = 2009 + i 678 | 679 | regionName = 'AMAZONIA NOROCCIDENTAL' 680 | studyArea = ee.FeatureCollection("projects/Sacha/AncillaryData/StudyRegions/Ecuador_EcoRegions_Complete") 681 | studyArea = studyArea.filterMetadata('PROVINCIA','equals',regionName).geometry().bounds() 682 | 683 | print(functions().getLandsat(studyArea,year,regionName)) 684 | 685 | -------------------------------------------------------------------------------- /landsat_cloudScore.py: -------------------------------------------------------------------------------- 1 | # Sentinel-2 package 2 | from paramsTemplate import * 3 | import ee 4 | from Py6S import * 5 | import math 6 | import datetime 7 | import os, sys 8 | from utils import * 9 | import sun_angles 10 | import view_angles 11 | import time 12 | 13 | class env(object): 14 | 15 | def __init__(self): 16 | """Initialize the environment.""" 17 | 18 | # Initialize the Earth Engine object, using the authentication credentials. 19 | ee.Initialize() 20 | 21 | self.epsg = "EPSG:32717" 22 | 23 | ########################################## 24 | # variable for the landsat data request # 25 | ########################################## 26 | self.metadataCloudCoverMax = 100; 27 | 28 | ########################################## 29 | # Export variables # 30 | ########################################## 31 | 32 | self.assetId ="projects/Sacha/PreprocessedData/L8_Annual_CloudScore/" 33 | self.name = "LS_CS_" 34 | 35 | 36 | self.exportScale = 20 37 | 38 | self.cloudScoreThresh = 1; 39 | 40 | ########################################## 41 | # variable band selection # 42 | ########################################## 43 | self.percentiles = [25,75] 44 | 45 | 46 | self.divideBands = ee.List(['blue','green','red','nir','swir1','swir2']) 47 | 48 | self.bandNamesLandsat = ee.List(['blue','green','red','nir','swir1','thermal','swir2','sr_atmos_opacity','pixel_qa','radsat_qa']) 49 | self.sensorBandDictLandsatSR = ee.Dictionary({'L8' : ee.List([1,2,3,4,5,7,6,9,10,11]),\ 50 | 'L7' : ee.List([0,1,2,3,4,5,6,7,9,10]),\ 51 | 'L5' : ee.List([0,1,2,3,4,5,6,7,9,10]),\ 52 | 'L4' : ee.List([0,1,2,3,4,5,6,7,9,10])}) 53 | 54 | 55 | ########################################## 56 | # enable / disable modules # 57 | ########################################## 58 | self.cloudMask = True 59 | 60 | 61 | class functions(): 62 | def __init__(self): 63 | """Initialize the Surfrace Reflectance app.""" 64 | 65 | # get the environment 66 | self.env = env() 67 | 68 | def main(self,studyArea,startDate,endDate,startDay,endDay,week,regionName): 69 | 70 | self.env.startDate = startDate 71 | self.env.endDate = endDate 72 | 73 | 74 | self.env.startDoy = startDay 75 | self.env.endDoy = endDay 76 | self.env.regionName = regionName 77 | 78 | self.studyArea = studyArea 79 | 80 | 81 | landsat8 = ee.ImageCollection('LANDSAT/LC08/C01/T1_SR').filterDate(self.env.startDate,self.env.endDate).filterBounds(studyArea) 82 | landsat8 = landsat8.filterMetadata('CLOUD_COVER','less_than',self.env.metadataCloudCoverMax) 83 | landsat8 = landsat8.select(self.env.sensorBandDictLandsatSR.get('L8'),self.env.bandNamesLandsat) 84 | 85 | landsat5 = ee.ImageCollection('LANDSAT/LT05/C01/T1_SR').filterDate(self.env.startDate,self.env.endDate).filterBounds(studyArea) 86 | landsat5 = landsat5.filterMetadata('CLOUD_COVER','less_than',self.env.metadataCloudCoverMax) 87 | landsat5 = landsat5.select(self.env.sensorBandDictLandsatSR.get('L5'),self.env.bandNamesLandsat).map(self.defringe) 88 | 89 | landsat7 = ee.ImageCollection('LANDSAT/LE07/C01/T1_SR').filterDate(self.env.startDate,self.env.endDate).filterBounds(studyArea) 90 | landsat7 = landsat7.filterMetadata('CLOUD_COVER','less_than',self.env.metadataCloudCoverMax) 91 | landsat7 = landsat7.select(self.env.sensorBandDictLandsatSR.get('L7'),self.env.bandNamesLandsat) 92 | 93 | 94 | landsat = landsat5.merge(landsat7).merge(landsat8) 95 | 96 | 97 | if landsat.size().getInfo() > 0: 98 | 99 | 100 | landsat = landsat.map(self.scaleLandsat) 101 | 102 | # mask clouds using cloud mask function 103 | if self.env.cloudMask == True: 104 | #print "removing some more clouds" 105 | landsat = landsat.map(self.maskClouds) 106 | 107 | landsat = landsat.select(['cloudScore','pixel_qa']) 108 | landsat = self.percentile(landsat,self.env.percentiles) 109 | landsat = landsat.set('system:time_start',ee.Date(self.env.startDate).millis()) 110 | self.exportMap(landsat,studyArea,week) 111 | print(landsat.getInfo()) 112 | 113 | return landsat 114 | 115 | 116 | def scaleLandsat(self,img): 117 | """Landast is scaled by factor 0.0001 """ 118 | thermal = img.select(ee.List(['thermal'])).multiply(0.1) 119 | scaled = ee.Image(img).select(self.env.divideBands).multiply(ee.Number(0.0001)) 120 | 121 | return img.select(['pixel_qa']).addBands(scaled).addBands(thermal) 122 | 123 | def maskClouds(self,img): 124 | """ 125 | Computes spectral indices of cloudyness and take the minimum of them. 126 | 127 | Each spectral index is fairly lenient because the group minimum 128 | is a somewhat stringent comparison policy. side note -> this seems like a job for machine learning :) 129 | originally written by Matt Hancher for Landsat imageryadapted to Sentinel by Chris Hewig and Ian Housman 130 | """ 131 | 132 | score = ee.Image(1.0); 133 | # Clouds are reasonably bright in the blue band. 134 | blue_rescale = img.select('blue').subtract(ee.Number(0.1)).divide(ee.Number(0.3).subtract(ee.Number(0.1))) 135 | score = score.min(blue_rescale); 136 | 137 | # Clouds are reasonably bright in all visible bands. 138 | visible = img.select('red').add(img.select('green')).add(img.select('blue')) 139 | visible_rescale = visible.subtract(ee.Number(0.2)).divide(ee.Number(0.8).subtract(ee.Number(0.2))) 140 | score = score.min(visible_rescale); 141 | 142 | # Clouds are reasonably bright in all infrared bands. 143 | infrared = img.select('nir').add(img.select('swir1')).add(img.select('swir2')) 144 | infrared_rescale = infrared.subtract(ee.Number(0.3)).divide(ee.Number(0.8).subtract(ee.Number(0.3))) 145 | score = score.min(infrared_rescale); 146 | 147 | # Clouds are reasonably cool in temperature. 148 | temp_rescale = img.select('thermal').subtract(ee.Number(300)).divide(ee.Number(290).subtract(ee.Number(300))) 149 | score = score.min(temp_rescale); 150 | 151 | # However, clouds are not snow. 152 | ndsi = img.normalizedDifference(['green', 'swir1']); 153 | ndsi_rescale = ndsi.subtract(ee.Number(0.8)).divide(ee.Number(0.6).subtract(ee.Number(0.8))) 154 | score = score.min(ndsi_rescale).multiply(100).byte().rename(['cloudScore']); 155 | mask = score.lt(self.env.cloudScoreThresh).rename(['cloudMask']); 156 | img = img.updateMask(mask).addBands([mask]).addBands([score]); 157 | 158 | return img; 159 | 160 | 161 | def defringe(self,img): 162 | 163 | # threshold for defringing landsat5 and 7 164 | fringeCountThreshold = 279 165 | 166 | k = ee.Kernel.fixed(41, 41, 167 | [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 168 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 169 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 170 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 171 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 172 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 173 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 174 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 175 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 176 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 177 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 178 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 179 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 180 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 181 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 182 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 183 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 184 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 185 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 186 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 187 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 188 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 189 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 190 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 191 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 192 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 193 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 194 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 195 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 196 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 197 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 198 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 199 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 200 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 201 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 202 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 203 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 204 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 205 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 206 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 207 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]); 208 | 209 | 210 | m = ee.Image(img).mask().reduce(ee.Reducer.min()) 211 | sum = m.reduceNeighborhood(ee.Reducer.sum(), k, 'kernel') 212 | mask = sum.gte(fringeCountThreshold) 213 | 214 | return img.updateMask(mask) 215 | 216 | 217 | def percentile(self,collection,p): 218 | median = ee.ImageCollection(collection).reduce(ee.Reducer.median()).rename(['cloudScore','pixel_qa']); 219 | percentiles = collection.reduce(ee.Reducer.percentile(p)) 220 | return median.addBands(percentiles) 221 | 222 | def exportMap(self,img,studyArea,week): 223 | 224 | geom = studyArea.getInfo(); 225 | sd = str(self.env.startDate.getRelative('day','year').getInfo()).zfill(3); 226 | ed = str(self.env.endDate.getRelative('day','year').getInfo()).zfill(3); 227 | year = str(self.env.startDate.get('year').getInfo()); 228 | regionName = self.env.regionName.replace(" ",'_') + "_" 229 | 230 | task_ordered= ee.batch.Export.image.toAsset(image=img, 231 | description = self.env.name + regionName + str(week).zfill(3) +'_'+ year + sd + ed, 232 | assetId= self.env.assetId + self.env.name + regionName + str(week).zfill(3)+'_'+ year + sd + ed, 233 | region=geom['coordinates'], 234 | maxPixels=1e13, 235 | crs=self.env.epsg, 236 | scale=self.env.exportScale) 237 | 238 | task_ordered.start() 239 | print(self.env.assetId + self.env.name + regionName + str(week).zfill(3)+'_'+ year + sd + ed) 240 | 241 | 242 | 243 | if __name__ == "__main__": 244 | 245 | ee.Initialize() 246 | 247 | start = 0 248 | for i in range(0,2,1): 249 | 250 | #2018 starts at week 104 251 | runNumber = start+ i 252 | print runNumber 253 | 254 | year = ee.Date("2009-01-01") 255 | 256 | startDay = 0 257 | endDay = 364 258 | 259 | startDate = year.advance(startDay,'day').advance(i,'year') 260 | endDate = year.advance(endDay,'day').advance(i,'year') 261 | 262 | regionName = 'ECUADOR' 263 | studyArea = ee.FeatureCollection("projects/Sacha/AncillaryData/StudyRegions/Ecuador_EcoRegions_Complete") 264 | studyArea = studyArea.geometry().bounds() 265 | 266 | functions().main(studyArea,startDate,endDate,startDay,endDay,runNumber,regionName) 267 | 268 | 269 | 270 | -------------------------------------------------------------------------------- /misc/annualMedian.py: -------------------------------------------------------------------------------- 1 | import ee 2 | from landsat import * 3 | ee.Initialize() 4 | 5 | func = functions() 6 | p = [25,75] 7 | 8 | geom = ee.FeatureCollection("projects/Sacha/AncillaryData/StudyRegions/Ecuador_EcoRegions_Complete").geometry().bounds().getInfo() 9 | collection = ee.ImageCollection("projects/Sacha/PreprocessedData/L8_Biweekly_V5_Mosaiced_2") 10 | # remove bands of biweekly percentiles 11 | newNames = ee.List(['p25_green','p25_nir','p25_blue','p25_red','p25_swir1','p25_swir2','p75_green','p75_nir','p75_blue','p75_red','p75_swir1','p75_swir2']) 12 | badNames = ee.List(['green_p25','nir_p25','blue_p25','red_p25','swir1_p25','swir2_p25','green_p75','nir_p75','blue_p75','red_p75','swir1_p75','swir2_p75']) 13 | badStdDev = ee.List(['count','blue_stdDev','red_stdDev','green_stdDev','nir_stdDev','swir1_stdDev','swir2_stdDev','ND_green_swir1','ND_nir_red_stdDev','ND_nir_swir2_stdDev','svvi']) 14 | 15 | bn = collection.first().bandNames() 16 | sn = bn.removeAll(newNames.cat(badStdDev)) 17 | collection = collection.select(sn) 18 | 19 | exportLive = False 20 | start = 0 21 | sy = 2009 22 | for i in range(2,3,1): 23 | 24 | year = ee.Date("2009-01-01") 25 | startDay = 0 26 | endDay = 364 27 | 28 | startDate = year.advance(startDay,'day').advance(i,'year') 29 | endDate = year.advance(endDay,'day').advance(i,'year') 30 | 31 | tcollection = collection.filterDate(startDate,endDate) 32 | 33 | medianMosaic = func.medianMosaic(tcollection) 34 | medianPercentiles = func.medianPercentiles(tcollection,p) 35 | medianPercentiles = medianPercentiles.select(badNames,newNames) 36 | stdDevBands = func.addSTDdev(tcollection) 37 | 38 | median = medianMosaic.addBands(medianPercentiles).addBands(stdDevBands) 39 | median = median.set('system:time_start',startDate) 40 | nameYr = sy+i 41 | 42 | name = 'LS_AN_' + str(nameYr) +'000365' 43 | 44 | if exportLive == True: 45 | task_ordered= ee.batch.Export.image.toAsset(image=median, 46 | description = name, 47 | assetId= "projects/Sacha/PreprocessedData/L8_Annual_V5/" + name, 48 | region=geom['coordinates'], 49 | maxPixels=1e13, 50 | crs="epsg:32717", 51 | scale=30) 52 | print(name) 53 | task_ordered.start() 54 | else: 55 | print(str(median.getInfo()['bands']).split("u'id"),name) -------------------------------------------------------------------------------- /misc/landsat_cloudScore.py: -------------------------------------------------------------------------------- 1 | # Sentinel-2 package 2 | from paramsTemplate import * 3 | import ee 4 | from Py6S import * 5 | import math 6 | import datetime 7 | import os, sys 8 | from utils import * 9 | import sun_angles 10 | import view_angles 11 | import time 12 | 13 | class env(object): 14 | 15 | def __init__(self): 16 | """Initialize the environment.""" 17 | 18 | # Initialize the Earth Engine object, using the authentication credentials. 19 | ee.Initialize() 20 | 21 | self.epsg = "EPSG:32717" 22 | 23 | ########################################## 24 | # variable for the landsat data request # 25 | ########################################## 26 | self.metadataCloudCoverMax = 100; 27 | 28 | ########################################## 29 | # Export variables # 30 | ########################################## 31 | 32 | self.assetId ="projects/Sacha/PreprocessedData/L8_Annual_CloudScore/" 33 | self.name = "LS_CS_" 34 | 35 | 36 | self.exportScale = 20 37 | 38 | self.cloudScoreThresh = 1; 39 | 40 | ########################################## 41 | # variable band selection # 42 | ########################################## 43 | self.percentiles = [25,75] 44 | 45 | 46 | self.divideBands = ee.List(['blue','green','red','nir','swir1','swir2']) 47 | 48 | self.bandNamesLandsat = ee.List(['blue','green','red','nir','swir1','thermal','swir2','sr_atmos_opacity','pixel_qa','radsat_qa']) 49 | self.sensorBandDictLandsatSR = ee.Dictionary({'L8' : ee.List([1,2,3,4,5,7,6,9,10,11]),\ 50 | 'L7' : ee.List([0,1,2,3,4,5,6,7,9,10]),\ 51 | 'L5' : ee.List([0,1,2,3,4,5,6,7,9,10]),\ 52 | 'L4' : ee.List([0,1,2,3,4,5,6,7,9,10])}) 53 | 54 | 55 | ########################################## 56 | # enable / disable modules # 57 | ########################################## 58 | self.cloudMask = True 59 | 60 | 61 | class functions(): 62 | def __init__(self): 63 | """Initialize the Surfrace Reflectance app.""" 64 | 65 | # get the environment 66 | self.env = env() 67 | 68 | def main(self,studyArea,startDate,endDate,startDay,endDay,week,regionName): 69 | 70 | self.env.startDate = startDate 71 | self.env.endDate = endDate 72 | 73 | 74 | self.env.startDoy = startDay 75 | self.env.endDoy = endDay 76 | self.env.regionName = regionName 77 | 78 | self.studyArea = studyArea 79 | 80 | 81 | landsat8 = ee.ImageCollection('LANDSAT/LC08/C01/T1_SR').filterDate(self.env.startDate,self.env.endDate).filterBounds(studyArea) 82 | landsat8 = landsat8.filterMetadata('CLOUD_COVER','less_than',self.env.metadataCloudCoverMax) 83 | landsat8 = landsat8.select(self.env.sensorBandDictLandsatSR.get('L8'),self.env.bandNamesLandsat) 84 | 85 | landsat5 = ee.ImageCollection('LANDSAT/LT05/C01/T1_SR').filterDate(self.env.startDate,self.env.endDate).filterBounds(studyArea) 86 | landsat5 = landsat5.filterMetadata('CLOUD_COVER','less_than',self.env.metadataCloudCoverMax) 87 | landsat5 = landsat5.select(self.env.sensorBandDictLandsatSR.get('L5'),self.env.bandNamesLandsat).map(self.defringe) 88 | 89 | landsat7 = ee.ImageCollection('LANDSAT/LE07/C01/T1_SR').filterDate(self.env.startDate,self.env.endDate).filterBounds(studyArea) 90 | landsat7 = landsat7.filterMetadata('CLOUD_COVER','less_than',self.env.metadataCloudCoverMax) 91 | landsat7 = landsat7.select(self.env.sensorBandDictLandsatSR.get('L7'),self.env.bandNamesLandsat) 92 | 93 | 94 | landsat = landsat5.merge(landsat7).merge(landsat8) 95 | 96 | 97 | if landsat.size().getInfo() > 0: 98 | 99 | 100 | landsat = landsat.map(self.scaleLandsat) 101 | 102 | # mask clouds using cloud mask function 103 | if self.env.cloudMask == True: 104 | landsat = landsat.map(self.maskClouds) 105 | 106 | landsat = landsat.select(['cloudScore','pixel_qa']) 107 | landsat = self.percentile(landsat,self.env.percentiles) 108 | landsat = landsat.set('system:time_start',ee.Date(self.env.startDate).millis()) 109 | self.exportMap(landsat,studyArea,week) 110 | print(landsat.getInfo()) 111 | 112 | return landsat 113 | 114 | 115 | def scaleLandsat(self,img): 116 | """Landast is scaled by factor 0.0001 """ 117 | thermal = img.select(ee.List(['thermal'])).multiply(0.1) 118 | scaled = ee.Image(img).select(self.env.divideBands).multiply(ee.Number(0.0001)) 119 | 120 | return img.select(['pixel_qa']).addBands(scaled).addBands(thermal) 121 | 122 | def maskClouds(self,img): 123 | """ 124 | Computes spectral indices of cloudyness and take the minimum of them. 125 | 126 | Each spectral index is fairly lenient because the group minimum 127 | is a somewhat stringent comparison policy. side note -> this seems like a job for machine learning :) 128 | originally written by Matt Hancher for Landsat imageryadapted to Sentinel by Chris Hewig and Ian Housman 129 | """ 130 | 131 | score = ee.Image(1.0); 132 | # Clouds are reasonably bright in the blue band. 133 | blue_rescale = img.select('blue').subtract(ee.Number(0.1)).divide(ee.Number(0.3).subtract(ee.Number(0.1))) 134 | score = score.min(blue_rescale); 135 | 136 | # Clouds are reasonably bright in all visible bands. 137 | visible = img.select('red').add(img.select('green')).add(img.select('blue')) 138 | visible_rescale = visible.subtract(ee.Number(0.2)).divide(ee.Number(0.8).subtract(ee.Number(0.2))) 139 | score = score.min(visible_rescale); 140 | 141 | # Clouds are reasonably bright in all infrared bands. 142 | infrared = img.select('nir').add(img.select('swir1')).add(img.select('swir2')) 143 | infrared_rescale = infrared.subtract(ee.Number(0.3)).divide(ee.Number(0.8).subtract(ee.Number(0.3))) 144 | score = score.min(infrared_rescale); 145 | 146 | # Clouds are reasonably cool in temperature. 147 | temp_rescale = img.select('thermal').subtract(ee.Number(300)).divide(ee.Number(290).subtract(ee.Number(300))) 148 | score = score.min(temp_rescale); 149 | 150 | # However, clouds are not snow. 151 | ndsi = img.normalizedDifference(['green', 'swir1']); 152 | ndsi_rescale = ndsi.subtract(ee.Number(0.8)).divide(ee.Number(0.6).subtract(ee.Number(0.8))) 153 | score = score.min(ndsi_rescale).multiply(100).byte().rename(['cloudScore']); 154 | mask = score.lt(self.env.cloudScoreThresh).rename(['cloudMask']); 155 | img = img.updateMask(mask).addBands([mask]).addBands([score]); 156 | 157 | return img; 158 | 159 | 160 | def defringe(self,img): 161 | 162 | # threshold for defringing landsat5 and 7 163 | fringeCountThreshold = 279 164 | 165 | k = ee.Kernel.fixed(41, 41, 166 | [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 167 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 168 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 169 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 170 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 171 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 172 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 173 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 174 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 175 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 176 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 177 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 178 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 179 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 180 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 181 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 182 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 183 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 184 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 185 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 186 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 187 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 188 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 189 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 190 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 191 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 192 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 193 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 194 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 195 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 196 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 197 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 198 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 199 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 200 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 201 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 202 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 203 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 204 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 205 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 206 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]); 207 | 208 | 209 | m = ee.Image(img).mask().reduce(ee.Reducer.min()) 210 | sum = m.reduceNeighborhood(ee.Reducer.sum(), k, 'kernel') 211 | mask = sum.gte(fringeCountThreshold) 212 | 213 | return img.updateMask(mask) 214 | 215 | 216 | def percentile(self,collection,p): 217 | median = ee.ImageCollection(collection).reduce(ee.Reducer.median()).rename(['cloudScore','pixel_qa']); 218 | percentiles = collection.reduce(ee.Reducer.percentile(p)) 219 | return median.addBands(percentiles) 220 | 221 | def exportMap(self,img,studyArea,week): 222 | 223 | geom = studyArea.getInfo(); 224 | sd = str(self.env.startDate.getRelative('day','year').getInfo()).zfill(3); 225 | ed = str(self.env.endDate.getRelative('day','year').getInfo()).zfill(3); 226 | year = str(self.env.startDate.get('year').getInfo()); 227 | regionName = self.env.regionName.replace(" ",'_') + "_" 228 | 229 | task_ordered= ee.batch.Export.image.toAsset(image=img, 230 | description = self.env.name + regionName + str(week).zfill(3) +'_'+ year + sd + ed, 231 | assetId= self.env.assetId + self.env.name + regionName + str(week).zfill(3)+'_'+ year + sd + ed, 232 | region=geom['coordinates'], 233 | maxPixels=1e13, 234 | crs=self.env.epsg, 235 | scale=self.env.exportScale) 236 | 237 | task_ordered.start() 238 | print(self.env.assetId + self.env.name + regionName + str(week).zfill(3)+'_'+ year + sd + ed) 239 | 240 | 241 | 242 | if __name__ == "__main__": 243 | 244 | ee.Initialize() 245 | 246 | start = 0 247 | for i in range(0,2,1): 248 | 249 | #2018 starts at week 104 250 | runNumber = start+ i 251 | print runNumber 252 | 253 | year = ee.Date("2009-01-01") 254 | 255 | startDay = 0 256 | endDay = 364 257 | 258 | startDate = year.advance(startDay,'day').advance(i,'year') 259 | endDate = year.advance(endDay,'day').advance(i,'year') 260 | 261 | regionName = 'ECUADOR' 262 | studyArea = ee.FeatureCollection("projects/Sacha/AncillaryData/StudyRegions/Ecuador_EcoRegions_Complete") 263 | studyArea = studyArea.geometry().bounds() 264 | 265 | functions().main(studyArea,startDate,endDate,startDay,endDay,runNumber,regionName) 266 | 267 | 268 | 269 | -------------------------------------------------------------------------------- /mosaicAndUpdate.py: -------------------------------------------------------------------------------- 1 | from rfClassification import * 2 | import ee 3 | 4 | 5 | def mosaicAndUpdate(collection,year,exportPath,reduceClass): 6 | def clipRegion(img): 7 | aoi = img.get('regionName'); 8 | studyRegion = ecoregions.filterMetadata('PROVINCIA','equals',aoi); 9 | return img.clip(studyRegion); 10 | 11 | rf = randomForest() 12 | 13 | ecoregions = ee.FeatureCollection('projects/Sacha/AncillaryData/StudyRegions/Ecuador_EcoRegions_Buffered') 14 | 15 | lcToUpdate = collection.filter(ee.Filter.calendarRange(year,year,'year')) 16 | 17 | mmu = lcToUpdate.filterMetadata('MMU','equals','1 ha') 18 | print(mmu.size().getInfo()) 19 | if mmu.size().getInfo() < 6: 20 | no_mmu = lcToUpdate.filterMetadata('MMU','equals','None') 21 | def sieveImg(img): 22 | img = rf.sieve(img,11) 23 | return img.rename('Mode') 24 | no_mmu = no_mmu.map(sieveImg) 25 | 26 | 27 | lcToUpdate = ee.ImageCollection(no_mmu).merge(ee.ImageCollection(mmu)).map(clipRegion).mosaic() 28 | else: 29 | lcToUpdate = mmu.map(clipRegion).mosaic() 30 | 31 | print(lcToUpdate.bandNames().getInfo()) 32 | # Hansen should be updated to relevent year 33 | hansen = ee.Image("UMD/hansen/global_forest_change_2018_v1_6"); 34 | # // MAE layers for updating LC 35 | rice = ee.FeatureCollection("projects/Sacha/RICE2013") 36 | shrimp = ee.FeatureCollection("projects/Sacha/shrimpFarm2013") 37 | 38 | # // MAE layers for updating LC 39 | aw1 = ee.FeatureCollection("projects/Sacha/oriente_a_inudaciont") 40 | aw2 = ee.FeatureCollection("projects/Sacha/sierra_a_inudacion") 41 | aw3 = ee.FeatureCollection("projects/Sacha/costa_a_inudacion") 42 | inf = ee.FeatureCollection("projects/Sacha/Infrastructure") 43 | 44 | # v3 land cover of sierra -used for paramo updating 45 | v3Sierra = ee.Image('projects/Sacha/Production/LandCover/LC_Nivel2_2016v3/SIERRA_LC_mmu1ha2016') 46 | 47 | 48 | # // paint the FCs 49 | emptImg = ee.Image().byte(); 50 | artificalW = aw1.merge(aw2).merge(aw3); 51 | riceAg = artificalW.filterBounds(rice) 52 | shrimpAw = inf.filterBounds(shrimp) 53 | 54 | riceImg = emptImg.paint(riceAg); 55 | shrimpImg = emptImg.paint(shrimpAw) 56 | artificalWImg = emptImg.paint(artificalW); 57 | infra = emptImg.paint(inf); 58 | 59 | forestloss = hansen.select('loss'); 60 | # update using artifical water, forestloss, and infrastructure 61 | lcToUpdate = lcToUpdate.where(lcToUpdate.eq(44),5).where(forestloss.eq(1).And(lcToUpdate.eq(40)),43).where(artificalWImg.eq(0),30).where(infra.eq(0),10); 62 | # //update paramo from v3, shrimp, and rice 63 | lcToUpdate = lcToUpdate.where(v3Sierra.eq(60),60).where(riceImg.eq(0),5).where(shrimpImg.eq(0),30) 64 | lcToUpdate = lcToUpdate.set('system:time_start',ee.Date(year)) 65 | print(lcToUpdate.get('system:time_start').getInfo()) 66 | task_ordered= ee.batch.Export.image.toAsset(image=lcToUpdate, 67 | description = 'LandCover_' + str(year), 68 | assetId = exportPath + 'LandCover_' + str(year), 69 | region = ecoregions.geometry().bounds().getInfo()['coordinates'], 70 | maxPixels = 1e13, 71 | crs = "EPSG:32717", 72 | scale = 30) 73 | 74 | task_ordered.start() 75 | 76 | if reduceClass: 77 | def reduceClasses(img): 78 | img = img.where(img.eq(10).Or(img.eq(1)), 1).where(img.eq(20).Or(img.eq(21)), 2)\ 79 | .where(img.eq(30).Or(img.eq(3)), 3).where(img.eq(40).Or(img.eq(41)).Or(img.eq(42))\ 80 | .Or(img.eq(43)), 4).where(img.eq(50).Or(img.eq(51)).Or(img.eq(52)).Or(img.eq(53)), 5)\ 81 | .where(img.eq(60).Or(img.eq(61)).Or(img.eq(62)), 6); 82 | 83 | mask = img.gt(6); 84 | 85 | return img.mask(mask.Not()); 86 | 87 | lcToUpdate = reduceClasses(lcToUpdate) 88 | 89 | task_ordered= ee.batch.Export.image.toAsset(image=lcToUpdate, 90 | description = 'LandCover_ReduceClasses_' + str(year), 91 | assetId = exportPath + 'LandCover_ReduceClasses_' + str(year), 92 | region = ecoregions.geometry().bounds().getInfo()['coordinates'], 93 | maxPixels = 1e13, 94 | crs = "EPSG:32717", 95 | scale = 30) 96 | 97 | # task_ordered.start() 98 | 99 | 100 | 101 | 102 | 103 | if __name__ == '__main__': 104 | ee.Initialize() 105 | year = 2018 106 | collection = ee.ImageCollection('projects/Sacha/Production/LandCover/LC_Nivel2_V8') 107 | exportPath = 'users/TEST/' 108 | reduceClass = True 109 | 110 | mosaicAndUpdate(collection, year, exportPath, reduceClass) -------------------------------------------------------------------------------- /paramsTemplate.py: -------------------------------------------------------------------------------- 1 | class Switcher(object): 2 | def paramSelect(self, argument): 3 | """Dispatch method""" 4 | method_name = str(argument).replace(" ","_") 5 | # Get the method from 'self'. Default to a lambda. 6 | method = getattr(self, method_name, lambda: "nothing") 7 | 8 | return method() 9 | 10 | def AMAZONIA_NOROCCIDENTAL(self): 11 | cloudScoreThresh= 1 12 | cloudScorePctl= 8 13 | zScoreThresh= -0.9 14 | shadowSumThresh= 0.4 15 | contractPixels= 1.5 16 | dilatePixels= 3.25 17 | 18 | return cloudScoreThresh, cloudScorePctl, zScoreThresh, shadowSumThresh, contractPixels, dilatePixels 19 | 20 | def ANDES_DEL_NORTE(self): 21 | cloudScoreThresh= 11 22 | cloudScorePctl= 8 23 | zScoreThresh= -0.8 24 | shadowSumThresh= 0.15 25 | contractPixels= 1.25 26 | dilatePixels= 2.75 27 | 28 | return cloudScoreThresh, cloudScorePctl, zScoreThresh, shadowSumThresh, contractPixels, dilatePixels 29 | 30 | def CHOCO(self): 31 | cloudScoreThresh= 1 32 | cloudScorePctl= 8 33 | zScoreThresh= -0.9 34 | shadowSumThresh= 0.35 35 | contractPixels= 0.86 36 | dilatePixels= 2 37 | 38 | return cloudScoreThresh, cloudScorePctl, zScoreThresh, shadowSumThresh, contractPixels, dilatePixels 39 | 40 | def GALAPAGOS(self): 41 | cloudScoreThresh= 1 42 | cloudScorePctl= 8 43 | zScoreThresh= -1.11 44 | shadowSumThresh= 0.25 45 | contractPixels= 1.18 46 | dilatePixels= 2 47 | 48 | return cloudScoreThresh, cloudScorePctl, zScoreThresh, shadowSumThresh, contractPixels, dilatePixels 49 | 50 | def PACIFICO_ECUATORIAL(self): 51 | cloudScoreThresh= 1 52 | cloudScorePctl= 8 53 | zScoreThresh= -0.9 54 | shadowSumThresh= 0.35 55 | contractPixels= 0.86 56 | dilatePixels= 2 57 | 58 | return cloudScoreThresh, cloudScorePctl, zScoreThresh, shadowSumThresh, contractPixels, dilatePixels 59 | 60 | def SIERRA(self): 61 | cloudScoreThresh= 2 62 | cloudScorePctl= 11 63 | zScoreThresh= -0.8 64 | shadowSumThresh= 0.15 65 | contractPixels= 1.2 66 | dilatePixels= 2.5 67 | 68 | return cloudScoreThresh, cloudScorePctl, zScoreThresh, shadowSumThresh, contractPixels, dilatePixels 69 | -------------------------------------------------------------------------------- /rfClassification.py: -------------------------------------------------------------------------------- 1 | import ee 2 | from addCovariates import * 3 | 4 | class randomForest(): 5 | 6 | def __init__(self): 7 | ee.Initialize() 8 | 9 | self.exportPath = 'users/TEST/' 10 | self.epsg = "EPSG:32717" 11 | # set number of trees for the random forest classifier 12 | self.trees = 40 13 | 14 | # set the version number of the product 15 | self.versionNumber = 9; 16 | 17 | # specify the training data set 18 | self.TrainingID = 'CompiledData02102019' 19 | 20 | # // specify the Landsat and Sentinel 1 data set 21 | self.LandsatID = 'TOA_composites_V2' 22 | self.DualS1_ID = 'S1Dual_Annual_V2' 23 | # specify the sample dataset 24 | self.trainingData = ee.FeatureCollection("projects/Sacha/AncillaryData/RefData/CompiledData02102019") 25 | # // specify the Sentinel 1 and Landsat S2 data set 26 | self.Dual_Annual = ee.ImageCollection("projects/Sacha/PreprocessedData/S1Dual_Annual_V2") 27 | self.S2LSTOA = ee.ImageCollection("projects/Sacha/PreprocessedData/TOA_composites_V2") 28 | # Specify the regions feature collection 29 | self.ecoregions = ee.FeatureCollection("projects/Sacha/AncillaryData/StudyRegions/Ecuador_EcoRegions_Complete") 30 | 31 | # todo need to convert covariates into py 32 | self.covariates = addCovariates() 33 | self.bandNames = ee.List(["red", "nir", "swir1", "swir2", 34 | 35 | "ND_nir_red", "ND_nir_swir1", "ND_nir_swir2", 36 | "ND_red_swir1", "ND_red_swir2", 37 | "ND_swir1_swir2", 38 | 39 | "p25_nir", "p25_red", "p25_swir1", "p25_swir2", 40 | "p25_ND_nir_red", "p25_ND_nir_swir1", "p25_ND_nir_swir2", 41 | "p25_ND_red_swir1", "p25_ND_red_swir2", 42 | "p25_ND_swir1_swir2", 43 | 44 | "p75_nir", "p75_red", "p75_swir1", "p75_swir2", 45 | "p75_ND_nir_red", "p75_ND_nir_swir1", "p75_ND_nir_swir2", 46 | "p75_ND_red_swir1", "p75_ND_red_swir2", 47 | "p75_ND_swir1_swir2", 48 | 49 | "VH", "VH_amplitude_1", "VH_phase_1", "VV", "VV_amplitude_1","VV_phase_1","ratio", 50 | 51 | "aspect", "eastness", "elevation", "northness", "slope", 52 | 53 | "change_abs", "change_norm", "max_extent","occurrence", "transition", "seasonality", 54 | 'svvi','blue_stdDev','red_stdDev','green_stdDev','nir_stdDev','swir1_stdDev','swir2_stdDev', 55 | 'ND_green_swir1_stdDev','ND_nir_red_stdDev','ND_nir_swir2_stdDev','clusters']); 56 | 57 | 58 | def main(self,year,ProcessingRegion,**kwargs): 59 | 60 | studyArea = self.ecoregions.filter(ee.Filter.eq("PROVINCIA", ProcessingRegion)).geometry().buffer(1000); 61 | sampleData = self.trainingData.filterBounds(studyArea) 62 | 63 | imageLandsatS2 = image if kwargs.get('image') else self.S2LSTOA.filterDate(str(year)).first() 64 | trianingImg = self.S2LSTOA.filterDate('2016').first() 65 | 66 | analysisYr = self.addBandsToComposite(imageLandsatS2) 67 | trainingYr = self.addBandsToComposite(trianingImg) 68 | 69 | 70 | data = trainingYr.sampleRegions(sampleData,["niv2_int"],20); 71 | classifier = ee.Classifier.randomForest(self.trees,0).train(ee.FeatureCollection(data),"niv2_int",self.bandNames); 72 | classification = analysisYr.classify(classifier,'Mode'); 73 | 74 | # todo think if we want to include this if its not automated 75 | classification = classification.set({ 76 | 'system:time_start': ee.Date.fromYMD(year,6,1).millis(), 77 | 'version': self.versionNumber, 78 | 'TrainingData': self.TrainingID, 79 | 'LandsatID': self.LandsatID, 80 | 'S1_Dual_ID': self.DualS1_ID, 81 | 'regionName': ProcessingRegion, 82 | }); 83 | 84 | classificationMMU = self.sieve(classification,11) 85 | 86 | self.mmuEx = kwargs.get('mmu') 87 | self.exportMap(classification,studyArea,"") if kwargs.get('exportImg') and self.mmuEx == False or self.mmuEx == 3 else None 88 | self.exportMap(classificationMMU,studyArea,"_MMU") if kwargs.get('exportImg') and self.mmuEx == True or self.mmuEx == 3 else None 89 | 90 | print(classification.bandNames().getInfo()) 91 | return classification, classificationMMU 92 | 93 | def exportMap(self,img,studyArea, suffix): 94 | img = img 95 | ed = str(year) + suffix 96 | regionName = ProcessingRegion.replace(" ",'_') 97 | 98 | task_ordered= ee.batch.Export.image.toAsset(image=img.clip(studyArea), 99 | description = regionName + '_LandCover_' + ed, 100 | assetId = self.exportPath + regionName + '_LandCover_' + ed, 101 | region = studyArea.bounds().getInfo()['coordinates'], 102 | maxPixels = 1e13, 103 | crs = self.epsg, 104 | scale = 30) 105 | 106 | task_ordered.start() 107 | print('Export Started: ',self.exportPath + regionName + '_LandCover_' + ed) 108 | 109 | # in functions 110 | def addBandsToComposite(self,composite): 111 | year = ee.Date(composite.get('system:time_start')).get('year') 112 | 113 | stdBands = composite.select(['svvi','blue_stdDev','red_stdDev','green_stdDev', 114 | 'nir_stdDev','swir1_stdDev','swir2_stdDev', 115 | 'ND_green_swir1_stdDev','ND_nir_red_stdDev','ND_nir_swir2_stdDev']) 116 | img25 = composite.select(['p25_blue','p25_green','p25_red','p25_nir','p25_swir1','p25_swir2']) 117 | img75 = composite.select(['p75_blue','p75_green','p75_red','p75_nir','p75_swir1','p75_swir2']) 118 | 119 | composite = composite.select(['blue','green','red','nir','swir1','swir2']) 120 | # adding abnds to the composite 121 | 122 | addJRCAndTopo = self.covariates.addJRCAndTopo(composite) 123 | # gets nd for selected bands 124 | img = addJRCAndTopo.addBands(self.covariates.addCovariates("",composite)); 125 | 126 | # // gets nd of 25 and 75percentile bands 127 | img = img.addBands(self.covariates.addCovariates("p75_",img75)).addBands(self.covariates.addCovariates("p25_",img25)); 128 | 129 | bands = img.bandNames().removeAll(['blue_1','green_1','red_1','nir_1','swir1_1','swir2_1']); 130 | 131 | img = img.select(bands).addBands(stdBands) 132 | 133 | Seg = ee.Algorithms.Image.Segmentation.SNIC(image = img.select(['ND_nir_red']), 134 | size = 20, 135 | compactness = 0, 136 | connectivity = 8); 137 | 138 | img = img.addBands(Seg.select(['clusters'])) 139 | # the dual pol SAR data is added here 140 | DualSar = ee.Image(self.Dual_Annual.filter(ee.Filter.calendarRange(year,year,'year')).first()).divide(10000); 141 | img = img.addBands(DualSar) 142 | 143 | return img 144 | 145 | def mosaicUniqueDates(self,collection): 146 | allDates = ee.List(collection.aggregate_array('system:time_start')).distinct() 147 | def getUDate(i): 148 | uD = collection.filterDate(i).mosaic() 149 | return uD 150 | out = allDates.map(getUDate) 151 | 152 | return ee.ImageCollection(out) 153 | 154 | def sieve(self,image,mmu): 155 | """function to filter isolated pixels be designated MMU. 156 | MMU value is the pixel count. """ 157 | connected = image.connectedPixelCount(mmu+20); 158 | elim = connected.gt(mmu); 159 | mode = image.focal_mode(mmu/2,'circle'); 160 | mode = mode.mask(image.mask()); 161 | filled = image.where(elim.Not(),mode); 162 | return filled.rename('Mode_MMU') 163 | 164 | if __name__ == "__main__": 165 | 166 | # // EcoRegions include: AMAZONIA NOROCCIDENTAL, ANDES DEL NORTE, GALAPAGOS, SIERRA, 167 | # // CHOCO, PACIFICO ECUATORIAL 168 | ProcessingRegion = 'AMAZONIA NOROCCIDENTAL' 169 | year = 2018; 170 | 171 | ''' main() takes 2 paramters -year,ProcessingRegion- and returns a tulep of classified images (0: no mmu, 1: mmu) 172 | optionally you can choose to export by setting exportImg to true and selecting which product you want with 173 | mmu. By default main will use the Sentinel2/Landsat Collection, but you can pass in another compostie by setting 174 | image = ee.Image('path/to/asset') 175 | mmu can be 0: exports classification without mmu 176 | 1: exports classification with mmu 177 | 3: exports both 178 | example: using default image collection and not exporting 179 | randomForest().main(2016,'AMAZONIA NOROCCIDENTAL') 180 | 181 | randomForest().main(2016,'AMAZONIA NOROCCIDENTAL',exportImg=True,mmu=1) 182 | exporting mmu 183 | 184 | providing your own image and exporting 185 | myImg = ee.Image('projects/Sacha/.../LSS2_ECUADOR_ANNUAL_MEDIAN_CLDMX80_2016_000365_ST') 186 | randomForest().main(2016,'AMAZONIA NOROCCIDENTAL',exportImg=False,mmu=True, image=myImg) ''' 187 | randomForest().main(year,ProcessingRegion,exportImg=True,mmu=3) -------------------------------------------------------------------------------- /sentinel2.py: -------------------------------------------------------------------------------- 1 | # Sentinel-2 package 2 | 3 | import ee 4 | from Py6S import * 5 | import math 6 | import datetime 7 | import os, sys 8 | from utils import * 9 | sys.path.append("/gee-atmcorr-S2/bin") 10 | from atmospheric import Atmospheric 11 | import sun_angles 12 | import view_angles 13 | import time 14 | 15 | class env(object): 16 | 17 | def __init__(self): 18 | """Initialize the environment.""" 19 | 20 | # Initialize the Earth Engine object, using the authentication credentials. 21 | ee.Initialize() 22 | 23 | self.dem = ee.Image("JAXA/ALOS/AW3D30_V1_1").select(["AVE"]) 24 | self.epsg = "EPSG:32717" 25 | self.feature = 0 26 | ########################################## 27 | # variable for the getSentinel algorithm # 28 | ########################################## 29 | 30 | self.metadataCloudCoverMax = 80; 31 | 32 | 33 | ########################################## 34 | # variable for the shadowMask algorithm # 35 | ########################################## 36 | 37 | # zScoreThresh: Threshold for cloud shadow masking- lower number masks out 38 | # less. Between -0.8 and -1.2 generally works well 39 | self.zScoreThresh = -1 40 | 41 | # shadowSumThresh: Sum of IR bands to include as shadows within TDOM and the 42 | # shadow shift method (lower number masks out less) 43 | self.shadowSumThresh = 0.500; 44 | 45 | # contractPixels: The radius of the number of pixels to contract (negative buffer) clouds and cloud shadows by. Intended to eliminate smaller cloud 46 | # patches that are likely errors (1.5 results in a -1 pixel buffer)(0.5 results in a -0 pixel buffer) 47 | # (1.5 or 2.5 generally is sufficient) 48 | self.contractPixels = 1.5; 49 | 50 | # dilatePixels: The radius of the number of pixels to dilate (buffer) clouds 51 | # and cloud shadows by. Intended to include edges of clouds/cloud shadows 52 | # that are often missed (1.5 results in a 1 pixel buffer)(0.5 results in a 0 pixel buffer) 53 | # (2.5 or 3.5 generally is sufficient) 54 | self.dilatePixels = 3.5; 55 | 56 | 57 | ########################################## 58 | # variable for cloudScore algorithm # 59 | ########################################## 60 | 61 | # 9. Cloud and cloud shadow masking parameters. 62 | # If cloudScoreTDOM is chosen 63 | # cloudScoreThresh: If using the cloudScoreTDOMShift method-Threshold for cloud 64 | # masking (lower number masks more clouds. Between 10 and 30 generally works best) 65 | self.cloudScoreThresh = 20; 66 | 67 | # Percentile of cloud score to pull from time series to represent a minimum for 68 | # the cloud score over time for a given pixel. Reduces commission errors over 69 | # cool bright surfaces. Generally between 5 and 10 works well. 0 generally is a bit noisy 70 | self.cloudScorePctl = 5; 71 | 72 | ########################################## 73 | # variable for terrain algorithm # 74 | ########################################## 75 | 76 | self.terrainScale = 1000 77 | 78 | ########################################## 79 | # Export variables # 80 | ########################################## 81 | 82 | self.assetId ="projects/Sacha/PreprocessedData/S2_Biweekly_V3/" 83 | self.name = "S2_BW_" 84 | self.exportScale = 20 85 | 86 | ########################################## 87 | # variable band selection # 88 | ########################################## 89 | 90 | self.s2BandsIn = ee.List(['QA60','B1','B2','B3','B4','B5','B6','B7','B8','B8A','B9','B10','B11','B12','TDOMMask']) 91 | self.s2BandsOut = ee.List(['QA60','cb','blue','green','red','re1','re2','re3','nir','re4','waterVapor','cirrus','swir1','swir2','TDOMMask']) 92 | self.divideBands = ee.List(['blue','green','red','re1','re2','re3','nir','re4','cb','cirrus','swir1','swir2','waterVapor']) 93 | self.medianIncludeBands = ee.List(['blue','green','red','re1','re2','re3','nir','re4','cb','cirrus','swir1','swir2','waterVapor']) 94 | 95 | ########################################## 96 | # enable / disable modules # 97 | ########################################## 98 | self.calcSR = True 99 | self.brdf = False 100 | self.QAcloudMask = True 101 | self.cloudMask = True 102 | self.shadowMask = True 103 | self.terrainCorrection = False 104 | 105 | 106 | class functions(): 107 | def __init__(self): 108 | """Initialize the Surfrace Reflectance app.""" 109 | 110 | # get the environment 111 | self.env = env() 112 | 113 | 114 | def main(self,studyArea,startDate,endDate,startDay,endDay,week,regionName): 115 | 116 | self.env.regionName = regionName 117 | self.env.startDate = startDate 118 | self.env.endDate = endDate 119 | 120 | self.env.startDoy = startDay 121 | self.env.endDoy = endDay 122 | 123 | s2 = self.getSentinel2(startDate,endDate,studyArea); 124 | 125 | #print(s2.size().getInfo()) 126 | if s2.size().getInfo() > 0: 127 | 128 | # print(self.env.startDate.getInfo()) 129 | # print(self.env.endDate.getInfo()) 130 | 131 | 132 | s2 = s2.map(self.scaleS2) 133 | 134 | # masking the shadows 135 | print("Masking shadows..") 136 | if self.env.shadowMask == True: 137 | s2 = self.maskShadows(s2,studyArea) 138 | 139 | self.collectionMeta = s2.getInfo()['features'] 140 | #print(ee.Image(s2.first()).get('system:time_start').getInfo()) 141 | 142 | print("rename bands, add date..") 143 | s2 = s2.select(self.env.s2BandsIn,self.env.s2BandsOut).map(self.addDateYear) 144 | #print(ee.Image(s2.first()).get('system:time_start').getInfo()) 145 | 146 | 147 | if self.env.QAcloudMask == True: 148 | print("use QA band for cloud Masking") 149 | s2 = s2.map(self.QAMaskCloud) 150 | #print(ee.Image(s2.first()).get('system:time_start').getInfo()) 151 | 152 | 153 | if self.env.cloudMask == True: 154 | print("sentinel cloud score...") 155 | s2 = s2.map(self.sentinelCloudScore) 156 | s2 = self.cloudMasking(s2) 157 | #print(ee.Image(s2.first()).get('system:time_start').getInfo()) 158 | 159 | # applying the atmospheric correction 160 | if self.env.calcSR == True: 161 | s2 = s2.select(self.env.s2BandsOut,self.env.s2BandsIn) 162 | print("applying atmospheric correction") 163 | s2 = s2.map(self.TOAtoSR).select(self.env.s2BandsIn,self.env.s2BandsOut) 164 | #print(ee.Image(s2.first()).getInfo()) 165 | 166 | 167 | if self.env.brdf == True: 168 | print("apply brdf correction..") 169 | s2 = s2.map(self.brdf) 170 | 171 | if self.env.terrainCorrection == True: 172 | print("apply terrain correction..") 173 | s2 = s2.map(self.getTopo) 174 | corrected = s2.filter(ee.Filter.gt("slope",20)) 175 | notCorrected = s2.filter(ee.Filter.lt("slope",20)) 176 | s2 = corrected.map(self.terrain).merge(notCorrected) 177 | 178 | 179 | 180 | print("calculating medoid") 181 | img = self.medoidMosaic(s2) 182 | 183 | print("rescale") 184 | img = self.reScaleS2(img) 185 | 186 | print("set MetaData") 187 | img = self.setMetaData(img) 188 | 189 | print("exporting composite") 190 | self.exportMap(img,studyArea,week) 191 | #print(img.getInfo()['properties']) 192 | 193 | def getSentinel2(self,start,end,studyArea): 194 | 195 | s2s = ee.ImageCollection('COPERNICUS/S2').filterDate(start,end) \ 196 | .filterBounds(studyArea) \ 197 | .filter(ee.Filter.lt('CLOUDY_PIXEL_PERCENTAGE',self.env.metadataCloudCoverMax)) \ 198 | .filter(ee.Filter.lt('CLOUD_COVERAGE_ASSESSMENT',self.env.metadataCloudCoverMax))\ 199 | 200 | return s2s 201 | 202 | def addDateYear(self,img): 203 | #add a date and year band 204 | date = ee.Date(img.get("system:time_start")) 205 | 206 | day = date.getRelative('day','year').add(1); 207 | yr = date.get('year'); 208 | mk = img.mask().reduce(ee.Reducer.min()); 209 | 210 | img = img.addBands(ee.Image.constant(day).mask(mk).uint16().rename('date')); 211 | img = img.addBands(ee.Image.constant(yr).mask(mk).uint16().rename('year')); 212 | 213 | return img; 214 | 215 | def maskShadows(self,collection,studyArea): 216 | 217 | def TDOM(image): 218 | zScore = image.select(shadowSumBands).subtract(irMean).divide(irStdDev) 219 | irSum = image.select(shadowSumBands).reduce(ee.Reducer.sum()) 220 | TDOMMask = zScore.lt(self.env.zScoreThresh).reduce(ee.Reducer.sum()).eq(2)\ 221 | .And(irSum.lt(self.env.shadowSumThresh)).Not() 222 | TDOMMask = TDOMMask.focal_min(self.env.dilatePixels) 223 | return image.addBands(TDOMMask.rename(['TDOMMask'])) 224 | 225 | def mask(image): 226 | outimg = image.updateMask(image.select(['TDOMMask'])) 227 | return outimg 228 | 229 | shadowSumBands = ['B8','B11'] 230 | 231 | allCollection = ee.ImageCollection('COPERNICUS/S2').filterBounds(studyArea) 232 | 233 | # Get some pixel-wise stats for the time series 234 | irStdDev = allCollection.select(shadowSumBands).reduce(ee.Reducer.stdDev()) 235 | irMean = allCollection.select(shadowSumBands).reduce(ee.Reducer.mean()) 236 | 237 | # Mask out dark dark outliers 238 | collection_tdom = collection.map(TDOM) 239 | 240 | return collection_tdom.map(mask) 241 | 242 | def getTopo(self,img): 243 | ''' funtion to filter for areas with terrain and areas without''' 244 | dem = self.env.dem.unmask(0) 245 | geom = ee.Geometry(img.get('system:footprint')).bounds() 246 | slp_rad = ee.Terrain.slope(dem).clip(geom); 247 | 248 | slope = slp_rad.reduceRegion(reducer= ee.Reducer.percentile([80]), \ 249 | geometry= geom,\ 250 | scale= 100 ,\ 251 | maxPixels=10000000) 252 | return img.set('slope',slope.get('slope')) 253 | 254 | def scaleS2(self,img): 255 | 256 | divideBands = ['B1','B2','B3','B4','B5','B6','B7','B8','B8A','B9','B10','B11','B12'] 257 | bandNames = img.bandNames() 258 | otherBands = bandNames.removeAll(divideBands) 259 | 260 | others = img.select(otherBands) 261 | out = img.select(divideBands).divide(10000) 262 | return out.addBands(others).copyProperties(img,['system:time_start','system:footprint','MEAN_SOLAR_ZENITH_ANGLE','MEAN_SOLAR_AZIMUTH_ANGLE']).set("centroid",img.geometry().centroid()); 263 | 264 | def reScaleS2(self,img): 265 | 266 | bandNames = img.bandNames() 267 | otherBands = bandNames.removeAll(self.env.divideBands) 268 | others = img.select(otherBands) 269 | 270 | t = img.select(self.env.divideBands); 271 | t = t.multiply(10000) 272 | 273 | out = ee.Image(t.copyProperties(img).copyProperties(img,['system:time_start'])).addBands(others).int16() 274 | 275 | return out; 276 | 277 | def pixelArea(self,img): 278 | geom = ee.Geometry(img.get('system:footprint')).bounds() 279 | 280 | area = img.select(['red']).gt(0).reduceRegion(reducer= ee.Reducer.sum(),\ 281 | geometry= geom,\ 282 | scale= 100,\ 283 | maxPixels=10000000) 284 | 285 | return img.set("pixelArea",area.get("red")) 286 | 287 | def TOAtoSR(self,img): 288 | 289 | TDOMMask = img.select(['TDOMMask']) 290 | info = self.collectionMeta[self.env.feature]['properties'] 291 | scene_date = datetime.datetime.utcfromtimestamp(info['system:time_start']/1000)# i.e. Python uses seconds, EE uses milliseconds 292 | solar_z = info['MEAN_SOLAR_ZENITH_ANGLE'] 293 | 294 | geom = ee.Geometry(info['system:footprint']).centroid() 295 | #geom = ee.Geometry.Point([info['centroid']['coordinates'][0],info['centroid']['coordinates'][1]]) 296 | date = ee.Date.fromYMD(scene_date.year,scene_date.month,scene_date.day) 297 | 298 | h2o = Atmospheric.water(geom,date).getInfo() 299 | o3 = Atmospheric.ozone(geom,date).getInfo() 300 | aot = Atmospheric.aerosol(geom,date).getInfo() 301 | 302 | SRTM = ee.Image('CGIAR/SRTM90_V4')# Shuttle Radar Topography mission covers *most* of the Earth 303 | alt = SRTM.reduceRegion(reducer = ee.Reducer.mean(),geometry = geom).get('elevation').getInfo() 304 | 305 | if alt: 306 | km = alt/1000 # i.e. Py6S uses units of kilometers 307 | 308 | else: 309 | km = 0 310 | # Instantiate 311 | s = SixS() 312 | 313 | # Atmospheric constituents 314 | s.atmos_profile = AtmosProfile.UserWaterAndOzone(h2o,o3) 315 | s.aero_profile = AeroProfile.Continental 316 | s.aot550 = aot 317 | 318 | # Earth-Sun-satellite geometry 319 | s.geometry = Geometry.User() 320 | s.geometry.view_z = 0 # always NADIR (I think..) 321 | s.geometry.solar_z = solar_z # solar zenith angle 322 | s.geometry.month = scene_date.month # month and day used for Earth-Sun distance 323 | s.geometry.day = scene_date.day # month and day used for Earth-Sun distance 324 | s.altitudes.set_sensor_satellite_level() 325 | s.altitudes.set_target_custom_altitude(km) 326 | 327 | sensor = info['SPACECRAFT_NAME'] 328 | def spectralResponseFunction(bandname): 329 | """ 330 | Extract spectral response function for given band name 331 | """ 332 | if sensor == 'Sentinel-2A': 333 | bandSelect = { 334 | 'B1':PredefinedWavelengths.S2A_MSI_01, 335 | 'B2':PredefinedWavelengths.S2A_MSI_02, 336 | 'B3':PredefinedWavelengths.S2A_MSI_03, 337 | 'B4':PredefinedWavelengths.S2A_MSI_04, 338 | 'B5':PredefinedWavelengths.S2A_MSI_05, 339 | 'B6':PredefinedWavelengths.S2A_MSI_06, 340 | 'B7':PredefinedWavelengths.S2A_MSI_07, 341 | 'B8':PredefinedWavelengths.S2A_MSI_08, 342 | 'B8A':PredefinedWavelengths.S2A_MSI_08A, 343 | 'B9':PredefinedWavelengths.S2A_MSI_09, 344 | 'B10':PredefinedWavelengths.S2A_MSI_10, 345 | 'B11':PredefinedWavelengths.S2A_MSI_11, 346 | 'B12':PredefinedWavelengths.S2A_MSI_12} 347 | elif sensor == 'Sentinel-2B': 348 | bandSelect = { 349 | 'B1': PredefinedWavelengths.S2B_MSI_01, 350 | 'B2': PredefinedWavelengths.S2B_MSI_02, 351 | 'B3': PredefinedWavelengths.S2B_MSI_03, 352 | 'B4': PredefinedWavelengths.S2B_MSI_04, 353 | 'B5': PredefinedWavelengths.S2B_MSI_05, 354 | 'B6': PredefinedWavelengths.S2B_MSI_06, 355 | 'B7': PredefinedWavelengths.S2B_MSI_07, 356 | 'B8': PredefinedWavelengths.S2B_MSI_08, 357 | 'B8A': PredefinedWavelengths.S2B_MSI_08A, 358 | 'B9': PredefinedWavelengths.S2B_MSI_09, 359 | 'B10': PredefinedWavelengths.S2B_MSI_10, 360 | 'B11': PredefinedWavelengths.S2B_MSI_11, 361 | 'B12': PredefinedWavelengths.S2B_MSI_12} 362 | else: 363 | assert None, 'Invalid sensor. Detected %s . Supports "Sentinel-2A" or "Sentinel-2B"' % sensor 364 | 365 | return Wavelength(bandSelect[bandname]) 366 | 367 | def toa_to_rad(bandname): 368 | """ 369 | Converts top of atmosphere reflectance to at-sensor radiance 370 | """ 371 | 372 | # solar exoatmospheric spectral irradiance 373 | ESUN = info['SOLAR_IRRADIANCE_'+bandname] 374 | solar_angle_correction = math.cos(math.radians(solar_z)) 375 | 376 | # Earth-Sun distance (from day of year) 377 | doy = scene_date.timetuple().tm_yday 378 | d = 1 - 0.01672 * math.cos(0.9856 * (doy-4))# http://physics.stackexchange.com/questions/177949/earth-sun-distance-on-a-given-day-of-the-year 379 | 380 | # conversion factor 381 | multiplier = ESUN*solar_angle_correction/(math.pi*d**2) 382 | 383 | # at-sensor radiance 384 | rad = img.select(bandname).multiply(multiplier) 385 | return rad 386 | 387 | 388 | def surface_reflectance(bandname): 389 | """ 390 | Calculate surface reflectance from at-sensor radiance given waveband name 391 | """ 392 | 393 | # run 6S for this waveband 394 | s.wavelength = spectralResponseFunction(bandname) 395 | s.run() 396 | 397 | # extract 6S outputs 398 | Edir = s.outputs.direct_solar_irradiance #direct solar irradiance 399 | Edif = s.outputs.diffuse_solar_irradiance #diffuse solar irradiance 400 | Lp = s.outputs.atmospheric_intrinsic_radiance #path radiance 401 | absorb = s.outputs.trans['global_gas'].upward #absorption transmissivity 402 | scatter = s.outputs.trans['total_scattering'].upward #scattering transmissivity 403 | tau2 = absorb*scatter #total transmissivity 404 | 405 | # radiance to surface reflectance 406 | rad = toa_to_rad(bandname) 407 | 408 | ref = rad.subtract(Lp).multiply(math.pi).divide(tau2*(Edir+Edif)) 409 | 410 | return ref 411 | 412 | # all wavebands 413 | output = img.select('QA60') 414 | for band in ['B1','B2','B3','B4','B5','B6','B7','B8','B8A','B9','B10','B11','B12']: 415 | output = output.addBands(surface_reflectance(band)) 416 | 417 | self.env.feature +=1 418 | 419 | return output.addBands(TDOMMask).copyProperties(img,['system:time_start','system:footprint','MEAN_SOLAR_ZENITH_ANGLE','MEAN_SOLAR_AZIMUTH_ANGLE']) 420 | 421 | # Function to mask clouds using the Sentinel-2 QA band. 422 | def QAMaskCloud(self,img): 423 | 424 | bandNames = img.bandNames() 425 | otherBands = bandNames.removeAll(self.env.divideBands) 426 | others = img.select(otherBands) 427 | 428 | 429 | qa = img.select('QA60').int16(); 430 | 431 | img = img.select(self.env.divideBands) 432 | 433 | 434 | # Bits 10 and 11 are clouds and cirrus, respectively. 435 | cloudBitMask = int(math.pow(2, 10)); 436 | cirrusBitMask = int(math.pow(2, 11)); 437 | 438 | # Both flags should be set to zero, indicating clear conditions. 439 | mask = qa.bitwiseAnd(cloudBitMask).eq(0).And(qa.bitwiseAnd(cirrusBitMask).eq(0)); 440 | 441 | img = img.updateMask(mask).addBands(others) 442 | 443 | # Return the masked and scaled data. 444 | return img 445 | 446 | def sentinelCloudScore(self,img): 447 | """ 448 | Computes spectral indices of cloudyness and take the minimum of them. 449 | 450 | Each spectral index is fairly lenient because the group minimum 451 | is a somewhat stringent comparison policy. side note -> this seems like a job for machine learning :) 452 | 453 | originally written by Matt Hancher for Landsat imagery 454 | adapted to Sentinel by Chris Hewig and Ian Housman 455 | """ 456 | 457 | 458 | def rescale(img, thresholds): 459 | """ 460 | Linear stretch of image between two threshold values. 461 | """ 462 | return img.subtract(thresholds[0]).divide(thresholds[1] - thresholds[0]) 463 | 464 | # cloud until proven otherwise 465 | score = ee.Image(1) 466 | blueCirrusScore = ee.Image(0) 467 | 468 | # clouds are reasonably bright 469 | blueCirrusScore = blueCirrusScore.max(rescale(img.select(['blue']), [0.1, 0.5])) 470 | blueCirrusScore = blueCirrusScore.max(rescale(img.select(['cb']), [0.1, 0.5])) 471 | blueCirrusScore = blueCirrusScore.max(rescale(img.select(['cirrus']), [0.1, 0.3])) 472 | score = score.min(blueCirrusScore) 473 | 474 | score = score.min(rescale(img.select(['red']).add(img.select(['green'])).add(img.select('blue')), [0.2, 0.8])) 475 | score = score.min(rescale(img.select(['nir']).add(img.select(['swir1'])).add(img.select('swir2')), [0.3, 0.8])) 476 | 477 | # clouds are moist 478 | ndsi = img.normalizedDifference(['green','swir1']) 479 | score=score.min(rescale(ndsi, [0.8, 0.6])) 480 | score = score.multiply(100).byte(); 481 | score = score.clamp(0,100); 482 | 483 | return img.addBands(score.rename(['cloudScore'])) 484 | 485 | def cloudMasking(self,collection): 486 | 487 | def maskClouds(img): 488 | 489 | cloudMask = img.select(['cloudScore']).lt(self.env.cloudScoreThresh)\ 490 | .focal_min(self.env.dilatePixels) \ 491 | .focal_max(self.env.contractPixels) \ 492 | .rename(['cloudMask']) 493 | 494 | bandNames = img.bandNames() 495 | otherBands = bandNames.removeAll(self.env.divideBands) 496 | others = img.select(otherBands) 497 | 498 | img = img.select(self.env.divideBands).updateMask(cloudMask) 499 | 500 | return img.addBands(cloudMask).addBands(others); 501 | 502 | 503 | # Find low cloud score pctl for each pixel to avoid comission errors 504 | minCloudScore = collection.select(['cloudScore']).reduce(ee.Reducer.percentile([self.env.cloudScorePctl])); 505 | 506 | collection = collection.map(maskClouds) 507 | 508 | return collection 509 | 510 | def brdf(self,img): 511 | 512 | 513 | 514 | def _apply(image, kvol, kvol0): 515 | blue = _correct_band(image, 'blue', kvol, kvol0, f_iso=0.0774, f_geo=0.0079, f_vol=0.0372) 516 | green = _correct_band(image, 'green', kvol, kvol0, f_iso=0.1306, f_geo=0.0178, f_vol=0.0580) 517 | red = _correct_band(image, 'red', kvol, kvol0, f_iso=0.1690, f_geo=0.0227, f_vol=0.0574) 518 | re1 = _correct_band(image, 're1', kvol, kvol0, f_iso=0.2085, f_geo=0.0256, f_vol=0.0845) 519 | re2 = _correct_band(image, 're2', kvol, kvol0, f_iso=0.2316, f_geo=0.0273, f_vol=0.1003) 520 | re3 = _correct_band(image, 're3', kvol, kvol0, f_iso=0.2599, f_geo=0.0294, f_vol=0.1197) 521 | nir = _correct_band(image, 'nir', kvol, kvol0, f_iso=0.3093, f_geo=0.0330, f_vol=0.1535) 522 | re4 = _correct_band(image, 're4', kvol, kvol0, f_iso=0.2907, f_geo=0.0410, f_vol=0.1611) 523 | swir1 = _correct_band(image, 'swir1', kvol, kvol0, f_iso=0.3430, f_geo=0.0453, f_vol=0.1154) 524 | swir2 = _correct_band(image, 'swir2', kvol, kvol0, f_iso=0.2658, f_geo=0.0387, f_vol=0.0639) 525 | return replace_bands(image, [blue, green, red,re1,re2,re3, nir,re4, swir1, swir2]) 526 | 527 | 528 | def _correct_band(image, band_name, kvol, kvol0, f_iso, f_geo, f_vol): 529 | """fiso + fvol * kvol + fgeo * kgeo""" 530 | iso = ee.Image(f_iso) 531 | geo = ee.Image(f_geo) 532 | vol = ee.Image(f_vol) 533 | pred = vol.multiply(kvol).add(geo.multiply(kvol)).add(iso).rename(['pred']) 534 | pred0 = vol.multiply(kvol0).add(geo.multiply(kvol0)).add(iso).rename(['pred0']) 535 | cfac = pred0.divide(pred).rename(['cfac']) 536 | corr = image.select(band_name).multiply(cfac).rename([band_name]) 537 | return corr 538 | 539 | 540 | def _kvol(sunAz, sunZen, viewAz, viewZen): 541 | """Calculate kvol kernel. 542 | From Lucht et al. 2000 543 | Phase angle = cos(solar zenith) cos(view zenith) + sin(solar zenith) sin(view zenith) cos(relative azimuth)""" 544 | 545 | relative_azimuth = sunAz.subtract(viewAz).rename(['relAz']) 546 | pa1 = viewZen.cos() \ 547 | .multiply(sunZen.cos()) 548 | pa2 = viewZen.sin() \ 549 | .multiply(sunZen.sin()) \ 550 | .multiply(relative_azimuth.cos()) 551 | phase_angle1 = pa1.add(pa2) 552 | phase_angle = phase_angle1.acos() 553 | p1 = ee.Image(PI().divide(2)).subtract(phase_angle) 554 | p2 = p1.multiply(phase_angle1) 555 | p3 = p2.add(phase_angle.sin()) 556 | p4 = sunZen.cos().add(viewZen.cos()) 557 | p5 = ee.Image(PI().divide(4)) 558 | 559 | kvol = p3.divide(p4).subtract(p5).rename(['kvol']) 560 | 561 | viewZen0 = ee.Image(0) 562 | pa10 = viewZen0.cos() \ 563 | .multiply(sunZen.cos()) 564 | pa20 = viewZen0.sin() \ 565 | .multiply(sunZen.sin()) \ 566 | .multiply(relative_azimuth.cos()) 567 | phase_angle10 = pa10.add(pa20) 568 | phase_angle0 = phase_angle10.acos() 569 | p10 = ee.Image(PI().divide(2)).subtract(phase_angle0) 570 | p20 = p10.multiply(phase_angle10) 571 | p30 = p20.add(phase_angle0.sin()) 572 | p40 = sunZen.cos().add(viewZen0.cos()) 573 | p50 = ee.Image(PI().divide(4)) 574 | 575 | kvol0 = p30.divide(p40).subtract(p50).rename(['kvol0']) 576 | 577 | return (kvol, kvol0) 578 | 579 | date = img.date() 580 | footprint = ee.List(img.geometry().bounds().bounds().coordinates().get(0)); 581 | (sunAz, sunZen) = sun_angles.create(date, footprint) 582 | (viewAz, viewZen) = view_angles.create(footprint) 583 | (kvol, kvol0) = _kvol(sunAz, sunZen, viewAz, viewZen) 584 | 585 | bandNames = img.bandNames() 586 | otherBands = bandNames.removeAll(self.env.divideBands) 587 | others = img.select(otherBands) 588 | 589 | img = ee.Image(_apply(img, kvol.multiply(PI()), kvol0.multiply(PI()))) 590 | 591 | return img 592 | 593 | def terrain(self,img): 594 | degree2radian = 0.01745; 595 | 596 | geom = ee.Geometry(img.get('system:footprint')).bounds().buffer(10000) 597 | dem = self.env.dem 598 | 599 | bandNames = img.bandNames() 600 | otherBands = bandNames.removeAll(self.env.divideBands) 601 | others = img.select(otherBands) 602 | 603 | bandList = ['blue','green','red','re1','re2','re3','nir','re4','cb','cirrus','swir1','swir2','waterVapor'] 604 | 605 | def topoCorr_IC(img): 606 | 607 | # Extract image metadata about solar position 608 | SZ_rad = ee.Image.constant(ee.Number(img.get('MEAN_SOLAR_ZENITH_ANGLE'))).multiply(degree2radian).clip(geom); 609 | SA_rad = ee.Image.constant(ee.Number(img.get('MEAN_SOLAR_AZIMUTH_ANGLE'))).multiply(degree2radian).clip(geom); 610 | 611 | 612 | # Creat terrain layers 613 | slp = ee.Terrain.slope(dem).clip(geom); 614 | slp_rad = ee.Terrain.slope(dem).multiply(degree2radian).clip(geom); 615 | asp_rad = ee.Terrain.aspect(dem).multiply(degree2radian).clip(geom); 616 | 617 | 618 | # Calculate the Illumination Condition (IC) 619 | # slope part of the illumination condition 620 | cosZ = SZ_rad.cos(); 621 | cosS = slp_rad.cos(); 622 | slope_illumination = cosS.expression("cosZ * cosS", \ 623 | {'cosZ': cosZ, 'cosS': cosS.select('slope')}); 624 | 625 | # aspect part of the illumination condition 626 | sinZ = SZ_rad.sin(); 627 | sinS = slp_rad.sin(); 628 | cosAziDiff = (SA_rad.subtract(asp_rad)).cos(); 629 | aspect_illumination = sinZ.expression("sinZ * sinS * cosAziDiff", \ 630 | {'sinZ': sinZ, \ 631 | 'sinS': sinS, \ 632 | 'cosAziDiff': cosAziDiff}); 633 | 634 | # full illumination condition (IC) 635 | ic = slope_illumination.add(aspect_illumination); 636 | 637 | # Add IC to original image 638 | img_plus_ic = ee.Image(img.addBands(ic.rename(['IC'])).addBands(cosZ.rename(['cosZ'])).addBands(cosS.rename(['cosS'])).addBands(slp.rename(['slope']))); 639 | 640 | return ee.Image(img_plus_ic); 641 | 642 | 643 | def topoCorr_SCSc(img): 644 | img_plus_ic = img; 645 | mask1 = img_plus_ic.select('nir').gt(-0.1); 646 | mask2 = img_plus_ic.select('slope').gte(5) \ 647 | .And(img_plus_ic.select('IC').gte(0)) \ 648 | .And(img_plus_ic.select('nir').gt(-0.1)); 649 | 650 | img_plus_ic_mask2 = ee.Image(img_plus_ic.updateMask(mask2)); 651 | 652 | 653 | 654 | def apply_SCSccorr(band): 655 | method = 'SCSc'; 656 | 657 | out = ee.Image(1).addBands(img_plus_ic_mask2.select('IC', band)).reduceRegion(reducer= ee.Reducer.linearRegression(2,1), \ 658 | geometry= ee.Geometry(img.geometry().buffer(-5000)), \ 659 | scale= 300, \ 660 | bestEffort =True, 661 | maxPixels=1e10) 662 | 663 | 664 | fit = out.combine({"coefficients": ee.Array([[1],[1]])}, False); 665 | 666 | #Get the coefficients as a nested list, 667 | #cast it to an array, and get just the selected column 668 | out_a = (ee.Array(fit.get('coefficients')).get([0,0])); 669 | out_b = (ee.Array(fit.get('coefficients')).get([1,0])); 670 | out_c = out_a.divide(out_b) 671 | 672 | 673 | # apply the SCSc correction 674 | SCSc_output = img_plus_ic_mask2.expression("((image * (cosB * cosZ + cvalue)) / (ic + cvalue))", { 675 | 'image': img_plus_ic_mask2.select([band]), 676 | 'ic': img_plus_ic_mask2.select('IC'), 677 | 'cosB': img_plus_ic_mask2.select('cosS'), 678 | 'cosZ': img_plus_ic_mask2.select('cosZ'), 679 | 'cvalue': out_c }); 680 | 681 | return ee.Image(SCSc_output); 682 | 683 | img_SCSccorr = ee.Image([apply_SCSccorr(band) for band in bandList]).addBands(img_plus_ic.select('IC')); 684 | 685 | bandList_IC = ee.List([bandList, 'IC']).flatten(); 686 | 687 | img_SCSccorr = img_SCSccorr.unmask(img_plus_ic.select(bandList_IC)).select(bandList); 688 | 689 | return img_SCSccorr.unmask(img_plus_ic.select(bandList)) 690 | 691 | img = topoCorr_IC(img) 692 | img = topoCorr_SCSc(img).addBands(others ) 693 | 694 | return img 695 | 696 | def medoidMosaic(self,collection): 697 | """ medoid composite with equal weight among indices """ 698 | 699 | bandNames = ee.Image(collection.first()).bandNames() 700 | otherBands = bandNames.removeAll(self.env.divideBands) 701 | 702 | others = collection.select(otherBands).reduce(ee.Reducer.mean()).rename(otherBands); 703 | 704 | collection = collection.select(self.env.divideBands) 705 | 706 | bandNumbers = ee.List.sequence(1,self.env.divideBands.length()); 707 | 708 | median = ee.ImageCollection(collection).median() 709 | 710 | def subtractmedian(img): 711 | diff = ee.Image(img).subtract(median).pow(ee.Image.constant(2)); 712 | return diff.reduce('sum').addBands(img); 713 | 714 | medoid = collection.map(subtractmedian) 715 | 716 | medoid = ee.ImageCollection(medoid).reduce(ee.Reducer.min(self.env.divideBands.length().add(1))).select(bandNumbers,self.env.divideBands); 717 | 718 | return medoid.addBands(others); 719 | 720 | def medianMosaic(self,collection): 721 | 722 | """ median composite """ 723 | median = collection.select(medianIncludeBands).median(); 724 | othersBands = bandNames.removeAll(medianIncludeBands); 725 | others = collection.select(otherBands).mean(); 726 | 727 | return median.addBands(others) 728 | 729 | def setMetaData(self,img): 730 | """ add metadata to image """ 731 | 732 | img = ee.Image(img).set({'system:time_start':ee.Date(self.env.startDate).millis(), \ 733 | 'regionName': str(self.env.regionName), \ 734 | 'assetId':str(self.env.assetId), \ 735 | 'compositingMethod':'medoid', \ 736 | 'exportScale':str(self.env.exportScale), \ 737 | 'startDOY':str(self.env.startDoy), \ 738 | 'endDOY':str(self.env.endDoy), \ 739 | 'useCloudScore':str(self.env.cloudMask), \ 740 | 'useTDOM':str(self.env.shadowMask), \ 741 | 'useQAmask':str(self.env.QAcloudMask), \ 742 | 'brdf':str(self.env.brdf), \ 743 | 'useCloudProject':str(self.env.cloudMask), \ 744 | 'terrain':str(self.env.terrainCorrection), \ 745 | 'surfaceReflectance':str(self.env.calcSR), \ 746 | 'cloudScoreThresh':str(self.env.cloudScoreThresh), \ 747 | 'cloudScorePctl':str(self.env.cloudScorePctl), \ 748 | 'zScoreThresh':str(self.env.zScoreThresh), \ 749 | 'shadowSumThresh':str(self.env.shadowSumThresh), \ 750 | 'contractPixels':str(self.env.contractPixels), \ 751 | 'epsg':self.env.epsg, \ 752 | 'cloudFilter':str(self.env.metadataCloudCoverMax),\ 753 | 'terrainScale':str(self.env.terrainScale), \ 754 | 'dilatePixels':str(self.env.dilatePixels)}) 755 | 756 | return img 757 | 758 | def exportMap(self,img,studyArea,week): 759 | 760 | geom = studyArea.bounds().getInfo(); 761 | sd = str(self.env.startDate.getRelative('day','year').getInfo()).zfill(3); 762 | ed = str(self.env.endDate.getRelative('day','year').getInfo()).zfill(3); 763 | year = str(self.env.startDate.get('year').getInfo()); 764 | regionName = self.env.regionName.replace(" ",'_') + "_" 765 | 766 | task_ordered= ee.batch.Export.image.toAsset(image=img.clip(studyArea.buffer(10000)), 767 | description = self.env.name + regionName + str(week).zfill(3) +'_'+ year + sd + ed, 768 | assetId= self.env.assetId + self.env.name + regionName + str(week).zfill(3)+'_'+ year + sd + ed, 769 | region=geom['coordinates'], 770 | maxPixels=1e13, 771 | crs=self.env.epsg, 772 | scale=self.env.exportScale) 773 | 774 | task_ordered.start() 775 | 776 | if __name__ == "__main__": 777 | 778 | ee.Initialize() 779 | 780 | start = 1 781 | 782 | for i in range(38,105,1): 783 | #Jun 2015 starts at biweek 38 784 | startWeek = start+ i 785 | print(startWeek) 786 | 787 | year = ee.Date("2014-01-01") 788 | startDay = (startWeek -1) *14 789 | endDay = (startWeek) *14 -1 790 | 791 | startDate = year.advance(startDay,"day") 792 | endDate = year.advance(endDay,"day") 793 | 794 | regionName = 'AMAZONIA NOROCCIDENTAL' 795 | studyArea = ee.FeatureCollection("projects/Sacha/AncillaryData/StudyRegions/Ecuador_EcoRegions_Complete") 796 | studyArea = studyArea.filterMetadata('PROVINCIA','equals',regionName).geometry().bounds() 797 | studyArea = studyArea.geometry().bounds() 798 | 799 | print(functions().main(studyArea,startDate,endDate,startDay,endDay,startWeek,regionName)) 800 | -------------------------------------------------------------------------------- /sun_angles.py: -------------------------------------------------------------------------------- 1 | 2 | from utils import * 3 | 4 | def create(date, footprint): 5 | jdp = date.getFraction('year') 6 | seconds_in_hour = 3600 7 | hourGMT = ee.Number(date.getRelative('second', 'day')) \ 8 | .divide(seconds_in_hour) 9 | 10 | latRad = degToRad(ee.Image.pixelLonLat().select('latitude')) 11 | longDeg = ee.Image.pixelLonLat().select('longitude') 12 | 13 | # Julian day proportion in radians 14 | jdpr = jdp.multiply(PI()).multiply(2) 15 | 16 | a = ee.List([0.000075, 0.001868, 0.032077, 0.014615, 0.040849]) 17 | meanSolarTime = longDeg.divide(15.0).add(ee.Number(hourGMT)) 18 | localSolarDiff1 = value(a, 0) \ 19 | .add(value(a, 1).multiply(jdpr.cos())) \ 20 | .subtract(value(a, 2).multiply(jdpr.sin())) \ 21 | .subtract(value(a, 3).multiply(jdpr.multiply(2).cos())) \ 22 | .subtract(value(a, 4).multiply(jdpr.multiply(2).sin())) 23 | 24 | localSolarDiff2 = localSolarDiff1.multiply(12 * 60) 25 | 26 | localSolarDiff = localSolarDiff2.divide(PI()) 27 | trueSolarTime = meanSolarTime \ 28 | .add(localSolarDiff.divide(60)) \ 29 | .subtract(12.0) 30 | 31 | # Hour as an angle 32 | ah = trueSolarTime.multiply(degToRad(ee.Number(MAX_SATELLITE_ZENITH * 2))) 33 | b = ee.List([0.006918, 0.399912, 0.070257, 0.006758, 0.000907, 0.002697, 0.001480]) 34 | delta = value(b, 0) \ 35 | .subtract(value(b, 1).multiply(jdpr.cos())) \ 36 | .add(value(b, 2).multiply(jdpr.sin())) \ 37 | .subtract(value(b, 3).multiply(jdpr.multiply(2).cos())) \ 38 | .add(value(b, 4).multiply(jdpr.multiply(2).sin())) \ 39 | .subtract(value(b, 5).multiply(jdpr.multiply(3).cos())) \ 40 | .add(value(b, 6).multiply(jdpr.multiply(3).sin())) 41 | cosSunZen = latRad.sin().multiply(delta.sin()) \ 42 | .add(latRad.cos().multiply(ah.cos()).multiply(delta.cos())) 43 | sunZen = cosSunZen.acos() 44 | 45 | # sun azimuth from south, turning west 46 | sinSunAzSW = ah.sin().multiply(delta.cos()).divide(sunZen.sin()) 47 | sinSunAzSW = sinSunAzSW.clamp(-1.0, 1.0) 48 | 49 | cosSunAzSW = (latRad.cos().multiply(-1).multiply(delta.sin()) 50 | .add(latRad.sin().multiply(delta.cos()).multiply(ah.cos()))) \ 51 | .divide(sunZen.sin()) 52 | sunAzSW = sinSunAzSW.asin() 53 | 54 | sunAzSW = where(cosSunAzSW.lte(0), sunAzSW.multiply(-1).add(PI()), sunAzSW) 55 | sunAzSW = where(cosSunAzSW.gt(0).And(sinSunAzSW.lte(0)), sunAzSW.add(PI().multiply(2)), sunAzSW) 56 | 57 | sunAz = sunAzSW.add(PI()) 58 | # Keep within [0, 2pi] range 59 | sunAz = where(sunAz.gt(PI().multiply(2)), sunAz.subtract(PI().multiply(2)), sunAz) 60 | 61 | footprint_polygon = ee.Geometry.Polygon(footprint) 62 | sunAz = sunAz.clip(footprint_polygon) 63 | sunAz = sunAz.rename(['sunAz']) 64 | sunZen = sunZen.clip(footprint_polygon).rename(['sunZen']) 65 | 66 | return (sunAz, sunZen) 67 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | # gee helpers 2 | 3 | import ee 4 | import math 5 | 6 | UPPER_LEFT = 0 7 | LOWER_LEFT = 1 8 | LOWER_RIGHT = 2 9 | UPPER_RIGHT = 3 10 | PI = lambda: ee.Number(math.pi) 11 | MAX_SATELLITE_ZENITH = 7.5 12 | 13 | def line_from_coords(coordinates, fromIndex, toIndex): 14 | return ee.Geometry.LineString(ee.List([ 15 | coordinates.get(fromIndex), 16 | coordinates.get(toIndex)])) 17 | 18 | 19 | def line(start, end): 20 | return ee.Geometry.LineString(ee.List([start, end])) 21 | 22 | 23 | def degToRad(deg): 24 | return deg.multiply(PI().divide(180)) 25 | 26 | 27 | def value(list, index): 28 | return ee.Number(list.get(index)) 29 | 30 | 31 | def radToDeg(rad): 32 | return rad.multiply(180).divide(PI()) 33 | 34 | 35 | def where(condition, trueValue, falseValue): 36 | trueMasked = trueValue.mask(condition) 37 | falseMasked = falseValue.mask(invertMask(condition)) 38 | return trueMasked.unmask(falseMasked) 39 | 40 | 41 | def invertMask(mask): 42 | return mask.multiply(-1).add(1) 43 | 44 | 45 | def x(point): 46 | return ee.Number(ee.List(point).get(0)) 47 | 48 | 49 | def y(point): 50 | return ee.Number(ee.List(point).get(1)) 51 | 52 | 53 | def determine_footprint(image): 54 | footprint = ee.Geometry(image.get('system:footprint')) 55 | bounds = ee.List(footprint.bounds().coordinates().get(0)) 56 | coords = footprint.coordinates() 57 | 58 | xs = coords.map(lambda item: x(item)) 59 | ys = coords.map(lambda item: y(item)) 60 | 61 | def findCorner(targetValue, values): 62 | diff = values.map(lambda value: ee.Number(value).subtract(targetValue).abs()) 63 | minValue = diff.reduce(ee.Reducer.min()) 64 | idx = diff.indexOf(minValue) 65 | return coords.get(idx) 66 | 67 | lowerLeft = findCorner(x(bounds.get(0)), xs) 68 | lowerRight = findCorner(y(bounds.get(1)), ys) 69 | upperRight = findCorner(x(bounds.get(2)), xs) 70 | upperLeft = findCorner(y(bounds.get(3)), ys) 71 | 72 | return ee.List([upperLeft, lowerLeft, lowerRight, upperRight, upperLeft]) 73 | 74 | 75 | def replace_bands(image, bands): 76 | result = image 77 | for band in bands: 78 | result = result.addBands(band, overwrite=True) 79 | return result 80 | 81 | 82 | 83 | 84 | -------------------------------------------------------------------------------- /view_angles.py: -------------------------------------------------------------------------------- 1 | import ee 2 | 3 | from utils import * 4 | 5 | MAX_DISTANCE = 1000000 6 | 7 | def create(footprint): 8 | return (azimuth(footprint), zenith(footprint)) 9 | 10 | 11 | def azimuth(footprint): 12 | upperCenter = line_from_coords(footprint, UPPER_LEFT, UPPER_RIGHT).centroid().coordinates() 13 | lowerCenter = line_from_coords(footprint, LOWER_LEFT, LOWER_RIGHT).centroid().coordinates() 14 | slope = ((y(lowerCenter)).subtract(y(upperCenter))).divide((x(lowerCenter)).subtract(x(upperCenter))) 15 | slopePerp = ee.Number(-1).divide(slope) 16 | azimuthLeft = ee.Image(PI().divide(2).subtract((slopePerp).atan())) 17 | return azimuthLeft.rename(['viewAz']) 18 | 19 | 20 | def zenith(footprint): 21 | leftLine = line_from_coords(footprint, UPPER_LEFT, LOWER_LEFT) 22 | rightLine = line_from_coords(footprint, UPPER_RIGHT, LOWER_RIGHT) 23 | leftDistance = ee.FeatureCollection(leftLine).distance(MAX_DISTANCE) 24 | rightDistance = ee.FeatureCollection(rightLine).distance(MAX_DISTANCE) 25 | viewZenith = rightDistance.multiply(ee.Number(MAX_SATELLITE_ZENITH * 2)) \ 26 | .divide(rightDistance.add(leftDistance)) \ 27 | .subtract(ee.Number(MAX_SATELLITE_ZENITH)) \ 28 | .clip(ee.Geometry.Polygon(footprint)) \ 29 | .rename(['viewZen']) 30 | return degToRad(viewZenith) 31 | --------------------------------------------------------------------------------