├── Extract ├── __pycache__ │ ├── Extract.cpython-312.pyc │ └── _extract_s1.cpython-312.pyc ├── Extract.py ├── _extract_s1.py └── _extract_s2.py ├── ETL.py ├── Load └── Load.py ├── Transform ├── Transform.py └── calculate_interval_mean.py └── README.md /Extract/__pycache__/Extract.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/parvvaresh/ETL-gee/main/Extract/__pycache__/Extract.cpython-312.pyc -------------------------------------------------------------------------------- /Extract/__pycache__/_extract_s1.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/parvvaresh/ETL-gee/main/Extract/__pycache__/_extract_s1.cpython-312.pyc -------------------------------------------------------------------------------- /ETL.py: -------------------------------------------------------------------------------- 1 | import ee 2 | 3 | from Extract._extract_s2 import extract_Sentinel2 4 | from Extract.Extract import Extract 5 | 6 | from Transform.Transform import Transform 7 | from Load.Load import Load 8 | 9 | def gee(aoi : ee.Geometry, 10 | start_date: str, 11 | end_date: str, 12 | cloudy_pixel : int, 13 | interval: int, 14 | name_file: str, 15 | name_folder: str): 16 | 17 | 18 | sentinel1 = Extract(aoi, 19 | start_date, 20 | end_date) 21 | 22 | print("Extracted Sentinel-1 and Sentinel-2 data.") 23 | 24 | 25 | reduced_image = Transform(aoi, 26 | sentinel1, 27 | extract_Sentinel2, 28 | interval, 29 | start_date, 30 | end_date, 31 | cloudy_pixel) 32 | print("Transformed Sentinel-1 and Sentinel-2 data.") 33 | 34 | 35 | 36 | Load(reduced_image, 37 | name_file, 38 | name_folder) 39 | 40 | print("Data saved to path.") 41 | -------------------------------------------------------------------------------- /Extract/Extract.py: -------------------------------------------------------------------------------- 1 | import ee 2 | 3 | from ._extract_s1 import extract_Sentinel1 4 | from ._extract_s2 import extract_Sentinel2 5 | 6 | 7 | 8 | 9 | 10 | 11 | def Extract(aoi : ee.Geometry, start_date : str, end_date : str) -> dict: 12 | """ 13 | Main function for the Extract phase. 14 | 15 | Extracts Sentinel-2 and Sentinel-1 data for the given area of interest (AOI) 16 | and date range, and returns the collections. 17 | 18 | Parameters: 19 | - aoi: ee.Geometry 20 | Area of interest in Earth Engine Geometry format. 21 | - start_date: str 22 | Start date for data extraction in 'YYYY-MM-DD' format. 23 | - end_date: str 24 | End date for data extraction in 'YYYY-MM-DD' format. 25 | 26 | Returns: 27 | - dict containing Sentinel-2 and Sentinel-1 collections. 28 | """ 29 | try: 30 | print("Extracting Sentinel-2 data...") 31 | sentinel2_collection = extract_Sentinel2(aoi, start_date, end_date) 32 | print(f"Extracted {sentinel2_collection.size().getInfo()} Sentinel-2 images.") 33 | 34 | print("Extracting Sentinel-1 data...") 35 | sentinel1_collection = extract_Sentinel1(aoi, start_date, end_date) 36 | print(f"Extracted {sentinel1_collection.size().getInfo()} Sentinel-1 images.") 37 | 38 | return { 39 | "Sentinel2": sentinel2_collection, 40 | "Sentinel1": sentinel1_collection 41 | } 42 | except Exception as e: 43 | print(f"Error during data extraction: {str(e)}") 44 | return None 45 | -------------------------------------------------------------------------------- /Extract/_extract_s1.py: -------------------------------------------------------------------------------- 1 | import ee 2 | 3 | 4 | def extract_Sentinel1(aoi: ee.Geometry, start_date: str, end_date: str) -> ee.ImageCollection: 5 | """ 6 | Extract Sentinel-1 radar data within a specified area and time range. 7 | 8 | This function retrieves Sentinel-1 Ground Range Detected (GRD) data from the 9 | 'COPERNICUS/S1_GRD' collection. The images are filtered based on the specified 10 | area of interest (AOI) and date range. The function selects polarization bands 11 | suitable for analysis: Vertical-Vertical (VV) and Vertical-Horizontal (VH) polarizations. 12 | 13 | Parameters: 14 | - aoi (ee.Geometry): Area of interest in Earth Engine Geometry format. 15 | - start_date (str): Start date for data extraction in 'YYYY-MM-DD' format. 16 | - end_date (str): End date for data extraction in 'YYYY-MM-DD' format. 17 | 18 | Returns: 19 | - ee.ImageCollection: A collection of Sentinel-1 radar images with selected polarization bands. 20 | 21 | Raises: 22 | - ValueError: If no images are available for the specified parameters. 23 | 24 | Notes: 25 | - Sentinel-1 provides radar data, which is unaffected by cloud cover, making it useful 26 | for all-weather monitoring. 27 | - The selected bands: 28 | - VV (Vertical transmit and Vertical receive): Useful for surface water and urban mapping. 29 | - VH (Vertical transmit and Horizontal receive): Sensitive to vegetation structure. 30 | """ 31 | sentinel1 = ee.ImageCollection('COPERNICUS/S1_GRD') \ 32 | .filterDate(start_date, end_date) \ 33 | .filterBounds(aoi) \ 34 | .select(['VV', 'VH']) 35 | 36 | if sentinel1.size().getInfo() == 0: 37 | raise ValueError("Sentinel-1 collection is empty. Check your date range or AOI.") 38 | 39 | return sentinel1 40 | 41 | -------------------------------------------------------------------------------- /Load/Load.py: -------------------------------------------------------------------------------- 1 | import ee 2 | import time 3 | 4 | def load(reduced_image: ee.Image, aoi: ee.Geometry, name_file: str, name_folder: str) -> None: 5 | """ 6 | Export the processed image to Google Drive. 7 | 8 | This function exports a given Earth Engine image (reduced_image) to the user's Google Drive. 9 | The exported image is clipped to the specified area of interest (AOI) and saved with the 10 | specified file name and folder. 11 | 12 | Parameters: 13 | - reduced_image (ee.Image): 14 | The processed image to be exported. 15 | - aoi (ee.Geometry): 16 | The area of interest in Earth Engine Geometry format. The exported image will be clipped to this AOI. 17 | - name_file (str): 18 | The name of the output file in Google Drive. 19 | - name_folder (str): 20 | The folder name in Google Drive where the file will be saved. 21 | 22 | Returns: 23 | - None: 24 | This function does not return any value. The exported image is saved to Google Drive. 25 | 26 | Raises: 27 | - ValueError: 28 | If the input image (reduced_image) is empty or invalid. 29 | - Exception: 30 | If an error occurs during the export process. 31 | 32 | Notes: 33 | - The export task uses a spatial resolution (scale) of 10 meters. 34 | - The coordinate reference system (CRS) used is EPSG:4326 (WGS84). 35 | - The maximum number of pixels allowed for export is set to 1e13. 36 | """ 37 | # Check if the input image is valid 38 | if not reduced_image.getInfo(): 39 | raise ValueError("Reduced image dictionary is empty. Nothing to export.") 40 | 41 | # Initialize the export task to Google Drive 42 | task = ee.batch.Export.image.toDrive( 43 | image=reduced_image, 44 | description=name_file, 45 | folder=name_folder, 46 | region=aoi, 47 | scale=10, # Spatial resolution in meters 48 | crs='EPSG:4326', # Coordinate Reference System 49 | maxPixels=1e13 # Maximum number of pixels allowed for export 50 | ) 51 | 52 | # Start the export task 53 | task.start() 54 | 55 | # Monitor the task status 56 | while task.active(): 57 | print("Export task status:", task.status()) 58 | time.sleep(10) # Wait 10 seconds before checking again 59 | 60 | # Print final task status 61 | print("Export task completed:", task.status()) 62 | -------------------------------------------------------------------------------- /Transform/Transform.py: -------------------------------------------------------------------------------- 1 | import ee 2 | from .calculate_interval_mean import * 3 | 4 | def transform_sentinel2(aoi: ee.Geometry, start_date: str, end_date: str, interval_days: int, collectionS2: ee.ImageCollection, collectionS1: ee.ImageCollection) -> ee.ImageCollection: 5 | """ 6 | Transform Sentinel-2 and Sentinel-1 data into aggregated intervals. 7 | 8 | This function processes Sentinel-2 and Sentinel-1 collections by aggregating them 9 | over a specified time interval. It calculates mean images for each interval, adds 10 | spectral indices and auxiliary data (terrain and MODIS LST), and combines the results 11 | into a single ImageCollection. 12 | 13 | Parameters: 14 | - aoi (ee.Geometry): 15 | Area of interest in Earth Engine Geometry format. 16 | - start_date (str): 17 | Start date for data processing in 'YYYY-MM-DD' format. 18 | - end_date (str): 19 | End date for data processing in 'YYYY-MM-DD' format. 20 | - interval_days (int): 21 | The duration of each time interval in days. 22 | - collectionS2 (ee.ImageCollection): 23 | Pre-filtered Sentinel-2 image collection. 24 | - collectionS1 (ee.ImageCollection): 25 | Pre-filtered Sentinel-1 image collection. 26 | 27 | Returns: 28 | - ee.ImageCollection: 29 | A collection of images where each image represents aggregated data for one interval, 30 | including Sentinel-2 and Sentinel-1 bands, spectral indices, terrain data, and MODIS LST. 31 | """ 32 | # Convert start_date and end_date to ee.Date objects 33 | start_date = ee.Date(start_date) 34 | end_date = ee.Date(end_date) 35 | 36 | # Calculate the number of intervals 37 | num_intervals = end_date.difference(start_date, 'day').subtract(interval_days).divide(interval_days).int().getInfo() 38 | 39 | # Initialize a list to store interval mean images 40 | interval_means = [] 41 | 42 | # Iterate through the intervals and calculate mean images 43 | for i in range(num_intervals): 44 | # Define the start date for the current interval 45 | start = start_date.advance(i * interval_days, 'day') 46 | 47 | # Calculate the mean image for the current interval 48 | interval_mean = calculate_interval_mean(start, interval_days, collectionS2, collectionS1, aoi) 49 | 50 | # Append the calculated mean image to the list 51 | interval_means.append(interval_mean) 52 | 53 | # Combine the interval images into a single ImageCollection and convert to bands 54 | return ee.ImageCollection(interval_means).toBands().toFloat() 55 | -------------------------------------------------------------------------------- /Extract/_extract_s2.py: -------------------------------------------------------------------------------- 1 | import ee 2 | 3 | def s2_clear_sky(image: ee.Image) -> ee.Image: 4 | """ 5 | Masks clear sky pixels in a Sentinel-2 image using the SCL band. 6 | 7 | Parameters: 8 | - image (ee.Image): Sentinel-2 image containing the 'SCL' band. 9 | 10 | Returns: 11 | - ee.Image: Sentinel-2 image masked for clear sky pixels, scaled to reflectance values. 12 | """ 13 | scl = image.select('SCL') 14 | clear_sky_pixels = scl.eq(4).Or(scl.eq(5)).Or(scl.eq(6)).Or(scl.eq(11)) 15 | return image.updateMask(clear_sky_pixels).divide(10000).copyProperties(image, ["system:time_start"]) 16 | 17 | 18 | def extract_Sentinel2(aoi: ee.Geometry, start_date: str, end_date: str) -> ee.ImageCollection: 19 | """ 20 | Extract Sentinel-2 surface reflectance data within a specified area and time range. 21 | 22 | This function retrieves Sentinel-2 data from the Harmonized Sentinel-2 Surface Reflectance collection 23 | ('COPERNICUS/S2_SR_HARMONIZED'). It filters the data based on the specified area of interest (AOI), 24 | date range, and a cloud coverage threshold. The function applies a clear sky mask using the SCL (Scene 25 | Classification Layer) band to retain only high-quality pixels and selects key spectral bands for analysis. 26 | 27 | Parameters: 28 | - aoi (ee.Geometry): Area of interest in Earth Engine Geometry format. 29 | - start_date (str): Start date for data extraction in 'YYYY-MM-DD' format. 30 | - end_date (str): End date for data extraction in 'YYYY-MM-DD' format. 31 | 32 | Returns: 33 | - ee.ImageCollection: A collection of Sentinel-2 images with clear sky pixels and selected spectral bands. 34 | 35 | Raises: 36 | - ValueError: If no images are available for the specified parameters. 37 | 38 | Notes: 39 | - The selected bands include: B1 (Coastal aerosol), B2 (Blue), B3 (Green), B4 (Red), 40 | B5–B7 (Vegetation Red Edge), B8 (NIR), B8A (Narrow NIR), B9 (Water vapor), B11 (SWIR1), 41 | and B12 (SWIR2). 42 | - Cloudy pixels are filtered using the CLOUDY_PIXEL_PERCENTAGE metadata. 43 | """ 44 | sentinel2 = ee.ImageCollection('COPERNICUS/S2_SR_HARMONIZED') \ 45 | .filterDate(start_date, end_date) \ 46 | .filter(ee.Filter.lt('CLOUDY_PIXEL_PERCENTAGE', 100)) \ 47 | .filterBounds(aoi) \ 48 | .map(s2_clear_sky) \ 49 | .select(['B1', 'B2', 'B3', 'B4', 'B5', 'B6', 'B7', 'B8', 'B8A', 'B9', 'B11', 'B12']) 50 | 51 | if sentinel2.size().getInfo() == 0: 52 | raise ValueError("Sentinel-2 collection is empty. Check your date range or AOI.") 53 | 54 | return sentinel2 55 | -------------------------------------------------------------------------------- /Transform/calculate_interval_mean.py: -------------------------------------------------------------------------------- 1 | import ee 2 | 3 | def calculate_interval_mean(start: ee.Date, interval: int, collectionS2: ee.ImageCollection, collectionS1: ee.ImageCollection, aoi: ee.Geometry) -> ee.Image: 4 | """ 5 | Calculate the mean image for a given time interval from Sentinel-2 and Sentinel-1 collections. 6 | 7 | This function computes the temporal mean of Sentinel-2 and Sentinel-1 images for a specified 8 | time interval. It also enhances the merged dataset by adding spectral indices, terrain 9 | information, and MODIS Land Surface Temperature (LST) data. 10 | 11 | Parameters: 12 | - start (ee.Date): The start date for the interval. 13 | - interval (int): The interval duration in days. 14 | - collectionS2 (ee.ImageCollection): Sentinel-2 image collection. 15 | - collectionS1 (ee.ImageCollection): Sentinel-1 image collection. 16 | - aoi (ee.Geometry): Area of interest in Earth Engine Geometry format. 17 | 18 | Returns: 19 | - ee.Image: An image with averaged bands from Sentinel-2 and Sentinel-1, spectral indices, 20 | terrain data, and MODIS LST bands clipped to the AOI. 21 | """ 22 | # Compute the end date for the interval 23 | end = start.advance(interval, 'day') 24 | 25 | # Compute the mean image for Sentinel-2 within the interval 26 | filteredS2 = collectionS2.filterDate(start, end).mean().set('system:time_start', start.millis()) 27 | 28 | # Compute the mean image for Sentinel-1 within the interval 29 | filteredS1 = collectionS1.filterDate(start, end).mean().set('system:time_start', start.millis()) 30 | 31 | # Merge Sentinel-2 and Sentinel-1 bands 32 | mergedImage = filteredS2.addBands(filteredS1) 33 | 34 | # Function to add spectral indices (NDVI, EVI, SAVI, BSI) 35 | def add_indices(image: ee.Image) -> ee.Image: 36 | """ 37 | Add spectral indices (NDVI, EVI, SAVI, BSI) as bands to the image. 38 | 39 | Parameters: 40 | - image (ee.Image): The input image. 41 | 42 | Returns: 43 | - ee.Image: The image with added spectral indices. 44 | """ 45 | # Calculate NDVI 46 | ndvi = image.normalizedDifference(['B8', 'B4']).rename('NDVI') 47 | 48 | # Calculate EVI 49 | evi = image.expression( 50 | '2.5 * ((NIR - RED) / (NIR + 6 * RED - 7.5 * BLUE + 1))', { 51 | 'NIR': image.select('B8'), 52 | 'RED': image.select('B4'), 53 | 'BLUE': image.select('B2') 54 | }).rename('EVI') 55 | 56 | # Calculate SAVI 57 | savi = image.expression( 58 | '((NIR - RED) / (NIR + RED + 0.5)) * 1.5', { 59 | 'NIR': image.select('B8'), 60 | 'RED': image.select('B4') 61 | }).rename('SAVI') 62 | 63 | # Calculate BSI 64 | bsi = image.expression( 65 | '((SWIR + RED) - (NIR + BLUE)) / ((SWIR + RED) + (NIR + BLUE))', { 66 | 'SWIR': image.select('B11'), 67 | 'RED': image.select('B4'), 68 | 'NIR': image.select('B8'), 69 | 'BLUE': image.select('B2') 70 | }).rename('BSI') 71 | 72 | # Add indices as bands 73 | return image.addBands([ndvi, evi, savi, bsi]) 74 | 75 | # Add spectral indices to the merged image 76 | mergedImage = add_indices(mergedImage) 77 | 78 | # Add elevation bands (mean, min, max) from GMTED2010 dataset 79 | gmted = ee.Image('USGS/GMTED2010_FULL').select(['mea', 'min', 'max']).clip(aoi) 80 | mergedImage = mergedImage.addBands(gmted) 81 | 82 | # Add slope derived from GMTED2010 mean elevation 83 | slope = ee.Terrain.slope(gmted.select('mea')).rename('slope').clip(aoi) 84 | mergedImage = mergedImage.addBands(slope) 85 | 86 | # Add MODIS Land Surface Temperature (LST) and emissivity bands 87 | modis = ee.ImageCollection('MODIS/061/MOD11A1') \ 88 | .filterDate(start, end) \ 89 | .filterBounds(aoi) \ 90 | .median() \ 91 | .select(['Emis_32', 'Emis_31', 'LST_Day_1km', 'LST_Night_1km']) 92 | mergedImage = mergedImage.addBands(modis) 93 | 94 | # Clip the final image to the AOI 95 | return mergedImage.clip(aoi) 96 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Google Earth Engine 2 | 3 | # 1. Get Data 4 | 5 | ## Overview 6 | 7 | The `gee` library is a Python package designed for processing and analyzing Sentinel-1 and Sentinel-2 satellite imagery using Google Earth Engine (GEE). It allows for the loading, processing, and combining of Sentinel-1 and Sentinel-2 data, as well as the calculation of various vegetation indices. Additionally, it supports exporting the processed data to Google Drive. 8 | 9 | ## Installation 10 | 11 | Before using the `gee` library, you need to set up the Google Earth Engine Python API. Follow the official installation guide [here](https://developers.google.com/earth-engine/guides/python_install). 12 | 13 | ## Usage 14 | 15 | Here's a step-by-step guide on how to use the `gee` library. 16 | 17 | ### 1. Import the Library 18 | 19 | ```python 20 | import ee 21 | from gee import gee 22 | 23 | # Initialize the Earth Engine library 24 | ee.Initialize() 25 | ee.Initialize(project='name of project') 26 | ``` 27 | 28 | 29 | ### 2. Clone form github 30 | 31 | This code can only be used in Google Colab 32 | 33 | ```python 34 | !git clone https://github.com/parvvaresh/google-earth-engine 35 | %cd google-earth-engine 36 | 37 | from get_data.gee import gee 38 | ``` 39 | ### 3. Define the Area of Interest (AOI) 40 | 41 | You need to define your area of interest (AOI) as an `ee.Geometry`. For example, to define a rectangular AOI: 42 | 43 | ```python 44 | aoi = ee.Geometry.Rectangle([xmin, ymin, xmax, ymax]) 45 | ``` 46 | 47 | ### 4. Create an Instance of the `gee` Class 48 | 49 | ```python 50 | gee_instance = gee(aoi) 51 | ``` 52 | 53 | Optionally, you can provide a `table_clip` parameter if you want to clip the results to a specific geometry. 54 | 55 | ### 5. Run the Data Pipeline 56 | 57 | Call the `pipeline_data` method with the required parameters to process and export the data: 58 | 59 | ```python 60 | gee_instance.pipeline_data( 61 | start_date='YYYY-MM-DD', # Start date of the data collection period 62 | end_date='YYYY-MM-DD', # End date of the data collection period 63 | name_file='exported_data', # Name of the exported file 64 | name_folder='GEE_exports' # Name of the folder in Google Drive where the file will be saved 65 | ) 66 | ``` 67 | 68 | ### Example Usage 69 | 70 | ```python 71 | import ee 72 | from gee import gee 73 | 74 | # Initialize the Earth Engine library 75 | ee.Initialize() 76 | ee.Initialize(project='name of project') 77 | 78 | 79 | !git clone https://github.com/parvvaresh/google-earth-engine 80 | %cd google-earth-engine 81 | 82 | from get_data.gee import gee 83 | 84 | # Define your AOI (example coordinates) 85 | aoi = ee.Geometry.Rectangle([-10, 35, 10, 45]) 86 | 87 | # Create an instance of the gee class 88 | gee_instance = gee(aoi) 89 | 90 | # Run the data pipeline 91 | gee_instance.pipeline_data( 92 | start_date='2023-01-01', 93 | end_date='2023-01-31', 94 | name_file='sentinel_data', 95 | name_folder='GEE_exports' 96 | ) 97 | ``` 98 | 99 | ## Methods 100 | 101 | ### `_load_Sentinel1(self, start_date: str, end_date: str) -> None` 102 | 103 | Loads the Sentinel-1 ImageCollection within the specified date range and AOI. 104 | 105 | ### `_process_Sentinel1(self) -> None` 106 | 107 | Processes the loaded Sentinel-1 data, including filtering and creating mosaics based on ascending and descending orbit passes. 108 | 109 | ### `_load_Sentinel2(self, start_date: str, end_date: str, cloudy_pixel: int) -> None` 110 | 111 | Loads the Sentinel-2 ImageCollection within the specified date range, AOI, and cloud cover percentage. 112 | 113 | ### `_process_Sentinel2(self, interval: int, start_date: str, end_date: str) -> None` 114 | 115 | Processes the loaded Sentinel-2 data, creating composites at specified intervals and calculating NDVI, EVI, and SAVI indices. 116 | 117 | ### `_combine_sentinel1_sentinel2(self) -> None` 118 | 119 | Combines the processed Sentinel-1 and Sentinel-2 data into a single dataset and clips it if a `table_clip` is provided. 120 | 121 | ### `_export_data(self, name_file: str, name_folder: str) -> None` 122 | 123 | Exports the processed and combined data to Google Drive as a CSV file. 124 | 125 | ### `pipeline_data(self, start_date: str, end_date: str, name_file: str, name_folder: str) -> None` 126 | 127 | Runs the entire data pipeline, from loading and processing Sentinel-1 and Sentinel-2 data to exporting the results. 128 | 129 | ## Notes 130 | 131 | - Ensure that you have sufficient permissions and quota in your Google Earth Engine account to run the processing tasks. 132 | - The export task may take some time depending on the size of the AOI and the date range specified. 133 | 134 | 135 | # 2. Convert to csv 136 | 137 | ## License 138 | 139 | This code is provided under the MIT License. Feel free to use and modify it as needed. 140 | 141 | --- 142 | --------------------------------------------------------------------------------