├── GNSSR_MERRByS.ipynb ├── GNSSR_Python ├── CoastalDistanceMap.py ├── GNSSR.py ├── MapPlotter.py └── landDistGrid_0.10LLRes_hGSHHSres.nc └── readme.md /GNSSR_Python/CoastalDistanceMap.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import h5py 3 | import matplotlib.pyplot as plt 4 | from GNSSR import * 5 | from CoastalDistanceMap import * 6 | from MapPlotter import * 7 | 8 | class CoastalDistanceMap: 9 | """A class that can load a map of the distance to the nearest coast-line 10 | """ 11 | 12 | def __init__(self): 13 | """Constructor""" 14 | 15 | def loadMap(self, filePath): 16 | """Load the distance to the coast map""" 17 | try: 18 | f = h5py.File(filePath, 'r') 19 | except OSError as e: 20 | #File does not exist 21 | #As TDS-1 is run periodically, many time segment files are not populated 22 | print("Could not find coastal distance map " + filePath) 23 | return 24 | 25 | self.coastalData = np.array(f['/array']) 26 | self.lats = np.array(f['/lats']) 27 | self.lons = np.array(f['/lons']) 28 | self.maxkm = np.array(f['/maxkm']) 29 | self.res = np.array(f['/res']) 30 | 31 | NaN = float('nan'); 32 | 33 | def getDistanceToCoast(self, longitudeArray, latitudeArray): 34 | """ Function that takes a numpy 1D array of [lat, lon] and 35 | returns the distance to the coast""" 36 | 37 | # Generate the map indexes 38 | longIdx = np.zeros(longitudeArray.size) 39 | latIdx = np.zeros(latitudeArray.size) 40 | 41 | it = np.nditer([longitudeArray, latitudeArray, None, None]) 42 | for longitude, latitude, longIdx, latIdx in it: 43 | longIdx[...] = np.argmin(np.abs(self.lons - longitude)) 44 | latIdx[...] = np.argmin(np.abs(self.lats - latitude)) 45 | 46 | #Lookup into map 47 | distanceArray = self.coastalData[it.operands[2].astype(int), it.operands[3].astype(int)] 48 | 49 | return distanceArray 50 | 51 | def displayMapTest(self): 52 | """ Test display the map over a grid of latitude-longitude points 53 | """ 54 | 55 | #To run the test do: 56 | #coastalDistanceMap = CoastalDistanceMap() 57 | #coastalDistanceMap.loadMap(os.path.join(os.getcwd(), 'GNSSR_Python', 'landDistGrid_0.10LLRes_hGSHHSres.nc')) 58 | #coastalDistanceMap.DisplayMapTest() 59 | 60 | mapPlotter = MapPlotter(200e3) #Map grid in km (at equator) 61 | 62 | coastDistance = np.zeros((mapPlotter.sizeLat, mapPlotter.sizeLon)) 63 | lons = np.zeros((mapPlotter.sizeLat, mapPlotter.sizeLon)) 64 | lats = np.zeros((mapPlotter.sizeLat, mapPlotter.sizeLon)) 65 | 66 | for indexes, x in np.ndenumerate(coastDistance): 67 | lon = np.array(mapPlotter.scaleLon[indexes[1]]) 68 | lat = np.array(mapPlotter.scaleLat[indexes[0]]) 69 | 70 | # Fill in output table 71 | coastDistance[indexes[0]][indexes[1]] = self.getDistanceToCoast(lon, lat) 72 | 73 | #Reshape to 2D map 74 | np.reshape(coastDistance, (mapPlotter.sizeLon, mapPlotter.sizeLat)) 75 | #Plot 76 | mapPlotter.plotMapStatic(coastDistance) -------------------------------------------------------------------------------- /GNSSR_Python/GNSSR.py: -------------------------------------------------------------------------------- 1 | '''Set of functions for processing GNSS-R data from www.MERRByS.co.uk''' 2 | 3 | 4 | def FindFiles(startTime, stopTime): 5 | '''Returns a list of all file names that would be present between the startTime and stopTime 6 | File names would be in format yyyy-mm/dd/Hhh which are segmented into 6 hour segments''' 7 | 8 | import datetime 9 | 10 | thisStartTime = startTime 11 | processStopTime = stopTime 12 | 13 | segmentationTimeHours = datetime.timedelta(hours=6) 14 | 15 | thisStopTime = thisStartTime + segmentationTimeHours 16 | # Loop through all time segments from processStartTime 17 | timeRangeList = []; 18 | while thisStopTime <= processStopTime: 19 | # Add to list for processing 20 | timeRangeList.append((thisStartTime, thisStopTime)) 21 | 22 | # Move to the next step 23 | thisStartTime = thisStartTime + segmentationTimeHours 24 | thisStopTime = thisStartTime + segmentationTimeHours 25 | 26 | return timeRangeList 27 | 28 | def FolderFromTimeStamp(timeRange): 29 | '''Get the folder that the data will be stored in from the time range ''' 30 | import datetime 31 | 32 | monthlyFolderName = str(timeRange[0].year).zfill(4) + "-" + str(timeRange[0].month).zfill(2) 33 | if timeRange[1] - timeRange[0] < datetime.timedelta(hours=23, minutes=59, seconds=58): 34 | midPointTime = (timeRange[1] - timeRange[0]) / 2 + timeRange[0] 35 | dailyFolderName = str(midPointTime.day).zfill(2) + "/H" + str(midPointTime.hour).zfill(2) 36 | else: 37 | dailyFolderName = str(timeRange[0].day).zfill(2) 38 | 39 | return monthlyFolderName + "/" + dailyFolderName 40 | 41 | def MatlabToPythonDateNum(matlab_datenum): 42 | '''Convert Matlab datenum to python datetime''' 43 | import datetime 44 | 45 | day = datetime.datetime.fromordinal(int(matlab_datenum)) 46 | dayfrac = datetime.timedelta(days=matlab_datenum%1) - datetime.timedelta(days = 366) 47 | return day + dayfrac 48 | 49 | def DownloadData(startTime, endTime, destination, ftpServer, userName, passWord, dataLevels, ftpDataFolder): 50 | '''Download data from MERRByS server over the time-range 51 | Provided the following parameters: 52 | startTime, endTime: The time range to download 53 | destination: The folder where the data should be stored 54 | ftpServer: The string: 'ftp.merrbys.co.uk' 55 | userName, passWord: The credentials given to you on registration 56 | dataLevels: The data levels to download {'L1B': True, 'L2_FDI': True, 'L2_CBRE_v0_5': True} 57 | ftpDataFolder: The FTP data access folder 'Data' for regular users or 'DataFast' for low latency access for approved users 58 | ''' 59 | 60 | from ftplib import FTP, error_perm 61 | import os 62 | import sys 63 | import datetime 64 | 65 | #Ensure destination folder exists 66 | if not os.path.exists(destination): 67 | os.makedirs(destination) 68 | 69 | #List of files to collect from server 70 | #'L2_FDI.nc' is only in L2 data, and the rest only in L1. Script could be optimised to remove these checks 71 | fileList = [] 72 | if dataLevels['L1B'] == True: 73 | fileList.append('metadata.nc') 74 | fileList.append('ddms.nc') 75 | fileList.append('directSignalPower.nc') 76 | fileList.append('blackbodyNadir.nc') 77 | fileList.append('blackbodyZenith.nc') 78 | if dataLevels['L2_FDI'] == True: 79 | fileList.append('L2_FDI.nc') 80 | if dataLevels['L2_CBRE_v0_5'] == True: 81 | fileList.append('L2_CBRE_v0_5.nc') 82 | 83 | ftp = FTP(ftpServer) 84 | ftp.login(user=userName, passwd = passWord) 85 | 86 | #Generate a list of possible files 87 | dataList = FindFiles(startTime, endTime) 88 | 89 | print ('Starting download') 90 | itemsDownloaded = 0 91 | 92 | from ipywidgets import FloatProgress 93 | from IPython.display import display 94 | f = FloatProgress(min=1, max=len(dataList)) 95 | display(f) 96 | 97 | #Main loop 98 | for entry in dataList: 99 | entryFolder = FolderFromTimeStamp(entry) 100 | for dataLevel in dataLevels: 101 | try: 102 | ftp.cwd('/Data/' + dataLevel + '/' + entryFolder) 103 | except Exception as e: 104 | # Continue on error as data may not be found 105 | continue 106 | 107 | itemsDownloaded = itemsDownloaded + 1 108 | f.value += 1 109 | 110 | #print(entryFolder) 111 | 112 | for fileName in fileList: 113 | #Check that the file exists before opening it, as this creates empty files otherwise 114 | try: 115 | fileSize = ftp.size(fileName) 116 | except error_perm as e: 117 | #Send e to an error file as needed 118 | continue 119 | 120 | #Split the data and recombine as a valid path to the destination folder 121 | ymd = entryFolder.split('/') 122 | fullPath = os.path.join(destination, dataLevel, ymd[0], ymd[1], ymd[2]) 123 | 124 | #Create folders as needed 125 | if not os.path.exists(fullPath): 126 | os.makedirs(fullPath) 127 | 128 | #Create full path 129 | filePath = os.path.join(fullPath, fileName) 130 | 131 | if os.path.isfile(filePath): 132 | #If the file exists, do not redownload 133 | continue 134 | else: 135 | ftp.retrbinary("RETR " + fileName, open(filePath, 'wb').write) 136 | 137 | print ('Complete. Got: ' + str(itemsDownloaded) + ' segments') 138 | f.value = len(dataList) 139 | -------------------------------------------------------------------------------- /GNSSR_Python/MapPlotter.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import math 3 | import matplotlib.pyplot as plt 4 | 5 | class MapPlotter: 6 | """A class that will plot a map in Plate Carree projection. 7 | Internally the map is represented as a 1D Array to ease matrix operations 8 | """ 9 | 10 | def __init__(self, boxSize): 11 | """Generate the map grid points 12 | Parameter: boxSize: Map grid in km (at equator)""" 13 | 14 | radiusOfEarth = 6371e3; 15 | self.sizeLon = round(2*math.pi*radiusOfEarth / boxSize); 16 | self.xLim = 180; 17 | self.offsetLon = self.sizeLon / 2; 18 | self.scaleLon = np.linspace(-self.xLim, self.xLim, self.sizeLon); 19 | self.sizeLat = round(2*math.pi*radiusOfEarth / boxSize / 2); 20 | self.yLim = 90; 21 | self.offsetLat = self.sizeLat / 2; 22 | self.scaleLat = np.linspace(-self.yLim, self.yLim, self.sizeLat); 23 | 24 | self.accum = np.zeros(self.sizeLon * self.sizeLat); 25 | self.count = np.zeros(self.sizeLon * self.sizeLat); 26 | 27 | def accumulateDataToMap(self, longitudeArray, latitudeArray, vals): 28 | """ Function that adds data to map accumulation and count of data 29 | so that average can be calculated. 30 | Inputs: 1D Numpy array of each of latitude, longitude and values 31 | """ 32 | #longitudeArray = np.zeros(1); 33 | #latitudeArray = np.zeros(1); 34 | #vals = np.ones(1); 35 | xScaled = np.round(longitudeArray / self.xLim * self.sizeLon/2 + self.offsetLon); 36 | yScaled = np.round(latitudeArray / self.yLim * self.sizeLat/2 + self.offsetLat); 37 | #print(self.sizeLon) 38 | #print(self.sizeLat) 39 | #print(xScaled) 40 | #print(np.shape(self.accum)) 41 | linearisedLocation = (xScaled-1) + self.sizeLon * (yScaled-1); 42 | #Saturate the rounding 43 | #linearisedLocation[linearisedLocation > np.shape(self.accum)[0]] = np.shape(self.accum)[0]; 44 | 45 | #Average across duplicates 46 | uniqueLocations = np.unique(linearisedLocation); 47 | for i in range(0, np.shape(uniqueLocations)[0]-1): 48 | location = uniqueLocations[i].astype(int); 49 | #Increase the accum and count ignoring data that are NaNs 50 | addCount = np.sum(np.isfinite(vals[linearisedLocation == location])); 51 | addValue = np.nansum(vals[linearisedLocation == location]); 52 | 53 | self.accum[location] = self.accum[location] + addValue; 54 | self.count[location] = self.count[location] + addCount; 55 | 56 | def plotMap(self): 57 | '''Plot the map''' 58 | 59 | accum2D = np.reshape(self.accum, (self.sizeLat, self.sizeLon)) 60 | count2D = np.reshape(self.count, (self.sizeLat, self.sizeLon)) 61 | 62 | mapAverage = np.divide(accum2D, count2D) 63 | 64 | mapAverage = np.flipud(mapAverage) 65 | 66 | # Mask array to only where there is data 67 | #mapAverage = np.ma.masked_where(count2D < 0, mapAverage) 68 | 69 | #mapAverage = np.random.random((200,100)) 70 | 71 | plt.figure(figsize=(10,10)) 72 | plt.imshow(mapAverage, cmap='jet', interpolation='none', extent=[-self.xLim, self.xLim, -self.yLim, self.yLim]) 73 | 74 | plt.xticks(np.arange(-180,181,45), fontsize=10) 75 | plt.yticks(np.arange(-90,91,45), fontsize=10) 76 | 77 | plt.tight_layout() 78 | plt.show() 79 | 80 | def plotMapStatic(self, map): 81 | '''Plot the map''' 82 | 83 | map = np.flipud(map) 84 | 85 | plt.figure(figsize=(10,10)) 86 | plt.imshow(map, cmap='jet', interpolation='none', extent=[-self.xLim, self.xLim, -self.yLim, self.yLim]) 87 | 88 | plt.xticks(np.arange(-180,181,45), fontsize=10) 89 | plt.yticks(np.arange(-90,91,45), fontsize=10) 90 | 91 | plt.tight_layout() 92 | plt.show() 93 | -------------------------------------------------------------------------------- /GNSSR_Python/landDistGrid_0.10LLRes_hGSHHSres.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pjalesSSTL/GNSSR_MERRByS_Python/54588350e79a23d5fed6bb2815662259cd84d95a/GNSSR_Python/landDistGrid_0.10LLRes_hGSHHSres.nc -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | ## GNSS-R MERRByS Python Example code 2 | This Jupyter Notebook contains some examples for the processing of the GNSS-Reflectometry (GNSS-R) data from www.merbys.co.uk. 3 | 4 | Surrey Satellite Technology Ltd provides these functions under a permissive MIT license to make it easier for people to get started with this data source. MERRByS Dataset from SSTL is licensed under a Creative Commons Attribution-NonCommercial 4.0 International License. 5 | 6 | ![CC-BY-NC](https://i.creativecommons.org/l/by-nc/4.0/88x31.png) 7 | 8 | The Jupyter notebook can be viewed here by clicking on the notebook above: `GNSSR_MERRByS.ipynb` 9 | 10 | The notebook uses some helper functions in the `/GNSSR_Python` folder. 11 | --------------------------------------------------------------------------------