├── README.md ├── pyhrrr.pyc ├── notebooks ├── pyhrrr.pyc ├── useful snippets.ipynb ├── pyhrrr.py ├── module file testing.ipynb ├── saveplothrrrfiles testing.ipynb ├── column extraction.ipynb ├── plot from readHrrr.ipynb ├── max plot test.ipynb ├── read Hrrr test2.ipynb ├── SGP plotting.ipynb ├── readHrrr test.ipynb ├── images to video.ipynb ├── netcdf variable recovery.ipynb ├── final testing of spec_plotting.ipynb ├── NetCDF Stuff-June2.ipynb ├── .ipynb_checkpoints │ └── NetCDF Stuff-June2-checkpoint.ipynb └── spec_plotting.ipynb ├── functions ├── HRRR_inventory.py ├── make_video.py ├── NetCDFvariablegrab.py ├── plotfromhrrrread.py ├── SGP_spec_plot.py ├── saveplothrrrfiles.py ├── SGPplotting.py ├── specificlocationdataextraction.py ├── readHrrr.py ├── datafileset.py └── gatherHrrrFiles.py ├── pyhrrr.py └── iastate_fetch_hrrr.py /README.md: -------------------------------------------------------------------------------- 1 | HRRR 2 | ==== 3 | -------------------------------------------------------------------------------- /pyhrrr.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scollis/HRRR/HEAD/pyhrrr.pyc -------------------------------------------------------------------------------- /notebooks/pyhrrr.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scollis/HRRR/HEAD/notebooks/pyhrrr.pyc -------------------------------------------------------------------------------- /functions/HRRR_inventory.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Fri Jun 6 09:35:18 2014 4 | 5 | @author: mattjohnson 6 | """ 7 | 8 | def HRRR_inventory(in_dir): 9 | """ 10 | finds the number of HRRR files in a given directory that correspond to each date 11 | returns that in a list along with a list of the corresponding dates. 12 | """ 13 | x = gather_hrrr_files(in_dir) 14 | x = x[0] 15 | y = [len(x[i]) for i in range(len(x))] 16 | return [y, x[1]] -------------------------------------------------------------------------------- /notebooks/useful snippets.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "name": "", 4 | "signature": "sha256:5fa76ca6c69137ac322720a0a7bd1904e224075a7daabddfc81162d9cca5c11d" 5 | }, 6 | "nbformat": 3, 7 | "nbformat_minor": 0, 8 | "worksheets": [ 9 | { 10 | "cells": [ 11 | { 12 | "cell_type": "code", 13 | "collapsed": false, 14 | "input": [ 15 | "import sys\n", 16 | "sys.path.append(\"/home/me/mypy\") \n", 17 | "\n", 18 | "\n", 19 | "\n", 20 | "\n" 21 | ], 22 | "language": "python", 23 | "metadata": {}, 24 | "outputs": [] 25 | } 26 | ], 27 | "metadata": {} 28 | } 29 | ] 30 | } -------------------------------------------------------------------------------- /pyhrrr.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Jun 9 11:46:41 2014 4 | 5 | @author: mattjohnson 6 | """ 7 | """ 8 | Imports all functions in the functions folder of our project. As a script 9 | works for any directory, but can only be directly called for import in the 10 | HRRR directory. 11 | """ 12 | 13 | import os 14 | 15 | wkdir = os.getcwd() 16 | 17 | directory = wkdir[:] 18 | 19 | while "HRRR" in directory: 20 | os.chdir(os.path.abspath('..')) 21 | directory = os.getcwd() 22 | 23 | dirpath = os.path.abspath("HRRR") 24 | 25 | dirpath2 = dirpath+'/functions/' 26 | 27 | filenames = os.listdir(dirpath2) 28 | 29 | for name in filenames: 30 | execfile(dirpath2+'/'+name) 31 | 32 | os.chdir(wkdir) 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /notebooks/pyhrrr.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Jun 9 11:46:41 2014 4 | 5 | @author: mattjohnson 6 | """ 7 | """ 8 | Imports all functions in the functions folder of our project. As a script 9 | works for any directory, but can only be directly called for import in the 10 | HRRR directory. 11 | """ 12 | 13 | import os 14 | 15 | wkdir = os.getcwd() 16 | 17 | directory = wkdir[:] 18 | 19 | while "HRRR" in directory: 20 | os.chdir(os.path.abspath('..')) 21 | directory = os.getcwd() 22 | 23 | dirpath = os.path.abspath("HRRR") 24 | 25 | dirpath2 = dirpath+'/functions/' 26 | 27 | filenames = os.listdir(dirpath2) 28 | 29 | for name in filenames: 30 | execfile(dirpath2+'/'+name) 31 | 32 | os.chdir(wkdir) 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /functions/make_video.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Jun 9 13:31:58 2014 4 | 5 | @author: mattjohnson 6 | """ 7 | """ 8 | Note: requires Image Magick 9 | """ 10 | import os 11 | 12 | def make_gif_video(imagefilelist,directory = os.getcwd(),name = 'video',inputtype = '.png',outputtype = '.gif'): 13 | """ 14 | This converts a series of images in alphabetical order existing in a given directory, for which they 15 | are the only images of their type into a .gif file. Other outputtypes may not be valid. 16 | """ 17 | import os 18 | 19 | wkdir = os.getcwd() 20 | os.chdir(directory) 21 | name = name+outputtype 22 | string = 'convert *'+inputtype+' '+name 23 | os.system(string) 24 | os.chdir(wkdir) 25 | 26 | return name 27 | 28 | 29 | -------------------------------------------------------------------------------- /functions/NetCDFvariablegrab.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Wed Jun 4 15:58:05 2014 4 | 5 | @author: mattjohnson 6 | """ 7 | 8 | def get_netCDF_variables(filename, variablelist = []): 9 | """ 10 | Accesses a specified netCDF file and recovers specified variables (if variablelist is empty will return all variables) 11 | returns a list of data arrays, dimensions and units for each variable in a list with the date of the file. 12 | """ 13 | import numpy as np 14 | import datetime 15 | from scipy.io import netcdf 16 | f = netcdf.netcdf_file(filename, 'r') 17 | 18 | if variablelist == []: 19 | variablelist = f.variables 20 | 21 | date = datetime(filename[-19:-15],filename[-15:-13],filename[-13:-11]) 22 | 23 | data = [] 24 | units = [] 25 | dim = [] 26 | 27 | for i in range(len(variablelist)): 28 | data.append(f.variables(variablelist[i]).data) 29 | units.append(f.variables(variablelist[i]).units) 30 | dim.append(f.variables(variablelist[i]).dimensions) 31 | 32 | return [[data,dim,units],date] -------------------------------------------------------------------------------- /iastate_fetch_hrrr.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #Fetch grb files from ia state website. 3 | import sys, os, urllib, urllib2 4 | 5 | def fetch_HRRR(forecast_time = 'f000', dimensionality = '3d', outdir = '/data/HRRR/'): 6 | HRRR_url = 'http://hrrr.agron.iastate.edu/data/hrrr/' 7 | all_files = urllib.urlopen(HRRR_url).read() 8 | all_times = [] 9 | for item in all_files.split("\n"): 10 | if "a href" in item: 11 | idx = item.find('a href="') 12 | all_times.append(item[idx + 8 : idx + 20]) 13 | last_dir = HRRR_url + all_times[-1] 14 | print last_dir 15 | filename = 'hrrr.' + dimensionality + '.' + all_times[-1] + forecast_time + '.grib2' 16 | exising_hrrr_files = os.listdir(outdir) 17 | outfile = outdir + filename 18 | if filename in exising_hrrr_files: 19 | print('File exists, saving bandwidth') 20 | else: 21 | print('fetching') 22 | try: 23 | req = urllib2.urlopen(last_dir + '/' + filename) 24 | outobj=open(outfile, 'wb') 25 | outobj.write(req.read()) 26 | outobj.close() 27 | except: 28 | outfile = 'null' 29 | print('File not found, skipping') 30 | return outfile 31 | 32 | if __name__ == '__main__': 33 | print('HRRR fetching') 34 | r_outdir = sys.argv[1] 35 | n_hours = int(sys.argv[2]) 36 | forecast_str = 'f%(dd)03d' 37 | f_strs = [forecast_str % {'dd' : k } for k in range(n_hours)] 38 | print(f_strs) 39 | fnames = [fetch_HRRR(forecast_time = fs, outdir = r_outdir) for fs in f_strs] 40 | -------------------------------------------------------------------------------- /functions/plotfromhrrrread.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Wed Jun 4 14:03:36 2014 4 | 5 | @author: mattjohnson 6 | """ 7 | 8 | from matplotlib import pyplot as plt 9 | from matplotlib import colors 10 | import numpy as np 11 | from mpl_toolkits.basemap import Basemap, addcyclic 12 | import pygrib 13 | 14 | def plothrrrfromfile(filename,parameter,hinp='',scaling=1,final_unit = '',vmax=None,vmin=None): #US 15 | """ 16 | Plots a given HRRR file over a given parameter and height in hPa over the US. If the height is left blank 17 | it will plot the maximum values of the parameter over all locations. 18 | 19 | The scaling, final_unit vmax and vmin parameters allow fine tuning of the plot to make differences more visible. 20 | """ 21 | 22 | if hinp != '': 23 | [data,parameterlist,datah,dataloc,units] = read_Hrrr(filename,[parameter]) 24 | else: 25 | [data,parameterlist,datah,dataloc,units] = read_Hrrr(filename,[parameter],max=True) 26 | 27 | if hinp !='': 28 | datah = datah.tolist() 29 | hindex = datah.index(hinp) 30 | 31 | if final_unit == '': 32 | final_unit = units[0] 33 | 34 | f = plt.figure(figsize=[12,10]) 35 | m = Basemap(llcrnrlon = -130,llcrnrlat = 20, urcrnrlon = -70, 36 | urcrnrlat = 52 , projection = 'mill', area_thresh =10000 , 37 | resolution='l') 38 | 39 | 40 | x, y = m(dataloc[1],dataloc[0]) 41 | data = np.array(data) 42 | 43 | if hinp != '': 44 | newdata = data[0][hindex][:][:] 45 | else: 46 | newdata = data[0] 47 | 48 | my_mesh = m.pcolormesh(x, y, newdata*scaling, vmax = vmax,vmin = vmin) 49 | my_coast = m.drawcoastlines(linewidth=1.25) 50 | my_states = m.drawstates() 51 | my_p = m.drawparallels(np.arange(20,80,4),labels=[1,1,0,0]) 52 | my_m = m.drawmeridians(np.arange(-140,-60,4),labels=[0,0,0,1]) 53 | 54 | plt.colorbar(label=final_unit) 55 | plt.show() 56 | return 0 57 | 58 | -------------------------------------------------------------------------------- /functions/SGP_spec_plot.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Jun 10 14:51:46 2014 4 | 5 | @author: mattjohnson 6 | """ 7 | def spec_hrrr_plot(directory,parameter,datetimestart = None,datetimeend = None,hinp = None,hour=0,loc = [-97.485,36.605]): 8 | """ 9 | Plots a given parameter over a given timespan for a given parameter, modelhour, height, location and directory of 10 | HRRR files. Leaving hinp empty will cause it to plot the maximum values over all heights. 11 | """ 12 | import numpy as np 13 | import matplotlib 14 | import os 15 | import matplotlib.pyplot as plt 16 | %matplotlib inline 17 | 18 | x = gather_hrrr_files(directory) 19 | y = np.array(x[0]) 20 | y = y.transpose() 21 | y = y[hour] 22 | 23 | dates = x[1] 24 | 25 | if datetimestart == None != datetimeend: 26 | print 'error datetimestart and datetimeend can only be none if both are' 27 | return 28 | elif datetimestart == None: 29 | startindex = 0 30 | endindex = len(dates) 31 | else: 32 | startindex = dates.index(datetimestart) 33 | endindex = dates.index(datetimeend) 34 | 35 | values = [] 36 | times = [] 37 | 38 | y = y[startindex:endindex] 39 | 40 | wkdir = os.getcwd() 41 | os.chdir(directory) 42 | 43 | for i in range(len(y)): 44 | 45 | if hinp != None: 46 | info = read_Hrrr_spec_loc(y[i], parameter,loc = [-97.485,36.605], max = False) 47 | index = info[2].tolist().index(hinp) 48 | datart = info[0][0] 49 | datarts = datart[index] 50 | else: 51 | info = read_Hrrr_spec_loc(y[i], parameter,loc = [-97.485,36.605], max = True) 52 | datarts = info[0][0] 53 | values.append(datarts) 54 | times.append(x[1][i]) 55 | 56 | units = info[-1] 57 | 58 | plt.plot(times, values) 59 | 60 | plt.xlabel('Time') 61 | plt.ylabel(parameter[0]+units) 62 | 63 | os.chdir(wkdir) 64 | 65 | return -------------------------------------------------------------------------------- /functions/saveplothrrrfiles.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Jun 9 15:13:18 2014 4 | 5 | @author: mattjohnson 6 | """ 7 | 8 | from matplotlib import pyplot as plt 9 | from matplotlib import colors 10 | import numpy as np 11 | from mpl_toolkits.basemap import Basemap, addcyclic 12 | import pygrib 13 | import os 14 | 15 | def saveplothrrrfiles(filename,parameter,fdirectory = os.getcwd()+'/images',imname = 'img1',filetype = '.png',size = (5,5),dpi = 600,hinp='',scaling=1,final_unit = '',vmax=None,vmin=None): #US 16 | """ 17 | Plots and saves a given HRRR file over a given parameter and height in hPa over the US. If the height is left blank 18 | it will plot the maximum values of the parameter over all locations. 19 | 20 | The scaling, final_unit vmax and vmin parameters allow fine tuning of the plot to make differences more visible. 21 | 22 | A .png file is saved in the folder existing or created that corresponds to fdirectory and named imname. 23 | """ 24 | 25 | if not os.path.exists(fdirectory): 26 | os.makedirs(fdirectory) 27 | 28 | 29 | if hinp != '': 30 | [data,parameterlist,datah,dataloc,units] = read_Hrrr(filename,[parameter]) 31 | else: 32 | [data,parameterlist,datah,dataloc,units] = read_Hrrr(filename,[parameter],max=True) 33 | 34 | 35 | if hinp !='': 36 | datah = datah.tolist() 37 | hindex = datah.index(hinp) 38 | 39 | if final_unit == '': 40 | final_unit = units[0] 41 | 42 | f = plt.figure(figsize=[12,10]) 43 | m = Basemap(llcrnrlon = -130,llcrnrlat = 20, urcrnrlon = -70, 44 | urcrnrlat = 52 , projection = 'mill', area_thresh =10000 , 45 | resolution='l') 46 | 47 | 48 | x, y = m(dataloc[1],dataloc[0]) 49 | data = np.array(data) 50 | 51 | if hinp != '': 52 | newdata = data[0][hindex][:][:] 53 | else: 54 | newdata = data[0] 55 | 56 | my_mesh = m.pcolormesh(x, y, newdata*scaling, vmax = vmax,vmin = vmin) 57 | my_coast = m.drawcoastlines(linewidth=1.25) 58 | my_states = m.drawstates() 59 | my_p = m.drawparallels(np.arange(20,80,4),labels=[1,1,0,0]) 60 | my_m = m.drawmeridians(np.arange(-140,-60,4),labels=[0,0,0,1]) 61 | 62 | plt.colorbar(label=final_unit) 63 | 64 | 65 | plt.savefig(fdirectory+'/'+imname+filetype,figsize = size,dpi=dpi) 66 | 67 | return (imname+filetype) -------------------------------------------------------------------------------- /functions/SGPplotting.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Thu Jun 5 08:32:23 2014 4 | 5 | @author: mattjohnson 6 | """ 7 | 8 | import pygrib 9 | from matplotlib import pyplot as plt 10 | from matplotlib import colors 11 | import numpy as np 12 | from mpl_toolkits.basemap import Basemap, addcyclic 13 | 14 | def plot_sgp_hrrr(filename,parameter,hinp = '', scaling = 1, final_unit = '', margin = 10, vmax = None, vmin = None): 15 | 16 | """ 17 | Plots an hrrr file focused on the SGP site for a given hrrr filename, parameter and height in hPa. 18 | Labels the SGP site. 19 | 20 | Leaving the height blank will cause it to plot the maximum values of the file. The margin defines half the length of 21 | the plot area in degrees latitude and longitude. Scaling and final_unit can be used to tweak the data in order to 22 | make it more visible on the graph. vmax and vmin function the same as in pcolormesh from Basemap. 23 | """ 24 | 25 | if hinp != '': 26 | [data,parameterlist,datah,dataloc,units] = read_Hrrr(filename,[parameter]) 27 | else: 28 | [data,parameterlist,datah,dataloc,units] = read_Hrrr(filename,[parameter],max=True) 29 | 30 | if hinp !='': 31 | datah = datah.tolist() 32 | hindex = datah.index(hinp) 33 | 34 | if final_unit == '': 35 | final_unit = units[0] 36 | 37 | 38 | f = plt.figure(figsize=[12,10]) 39 | m = Basemap(llcrnrlon = -97.485-margin,llcrnrlat = 36.605-margin, urcrnrlon = -97.485+margin, 40 | urcrnrlat = 36.605+margin, projection = 'mill', area_thresh =10000 , 41 | resolution='l') 42 | 43 | latlonsgp = [-97.485,36.605] 44 | cities = ['Lamont,OK'] 45 | sgpx,sgpy = m(latlonsgp[0],latlonsgp[1]) 46 | m.plot(sgpx,sgpy,'bo') 47 | plt.text(sgpx+50000,sgpy+50000,'SGP site') 48 | 49 | x, y = m(dataloc[1],dataloc[0]) 50 | data = np.array(data) 51 | 52 | 53 | if hinp != '': 54 | newdata = data[0][hindex][:][:] 55 | else: 56 | newdata = data[0] 57 | 58 | my_mesh = m.pcolormesh(x, y, newdata, vmax = .05, norm = colors.LogNorm()) 59 | my_coast = m.drawcoastlines(linewidth=1.25) 60 | my_states = m.drawstates() 61 | my_p = m.drawparallels(np.arange(20,80,4),labels=[1,1,0,0]) 62 | my_m = m.drawmeridians(np.arange(-140,-60,4),labels=[0,0,0,1]) 63 | 64 | plt.colorbar(label=final_unit) 65 | plt.show() 66 | return 0 -------------------------------------------------------------------------------- /notebooks/module file testing.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "name": "", 4 | "signature": "sha256:cda589aac9ab5dd40f9289073146e52bdc25fdb3c1de8b900c3e73de01a388ee" 5 | }, 6 | "nbformat": 3, 7 | "nbformat_minor": 0, 8 | "worksheets": [ 9 | { 10 | "cells": [ 11 | { 12 | "cell_type": "code", 13 | "collapsed": false, 14 | "input": [ 15 | "import os\n", 16 | "\n", 17 | "wkdir = os.getcwd()\n", 18 | "directory = wkdir\n", 19 | "\n", 20 | "while \"HRRR\" in directory:\n", 21 | " os.chdir(os.path.abspath('..'))\n", 22 | "\n", 23 | "dirpath = os.path.abspath(\"HRRR\")\n", 24 | "\n", 25 | "dirpath2 = dirpath+'/functions/'\n", 26 | " \n", 27 | "filenames = os.listdir(dirpath2)\n", 28 | " \n", 29 | "for name in filenames:\n", 30 | " execfile(dirpath2+'/'+name)\n", 31 | "\n", 32 | "os.chdir(wkdir)" 33 | ], 34 | "language": "python", 35 | "metadata": {}, 36 | "outputs": [], 37 | "prompt_number": 5 38 | }, 39 | { 40 | "cell_type": "code", 41 | "collapsed": false, 42 | "input": [], 43 | "language": "python", 44 | "metadata": {}, 45 | "outputs": [ 46 | { 47 | "metadata": {}, 48 | "output_type": "pyout", 49 | "prompt_number": 2, 50 | "text": [ 51 | "" 52 | ] 53 | } 54 | ], 55 | "prompt_number": 2 56 | }, 57 | { 58 | "cell_type": "code", 59 | "collapsed": false, 60 | "input": [], 61 | "language": "python", 62 | "metadata": {}, 63 | "outputs": [ 64 | { 65 | "ename": "ImportError", 66 | "evalue": "No module named pyhrrr", 67 | "output_type": "pyerr", 68 | "traceback": [ 69 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)", 70 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mpyhrrr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 71 | "\u001b[0;31mImportError\u001b[0m: No module named pyhrrr" 72 | ] 73 | } 74 | ], 75 | "prompt_number": 4 76 | }, 77 | { 78 | "cell_type": "code", 79 | "collapsed": false, 80 | "input": [], 81 | "language": "python", 82 | "metadata": {}, 83 | "outputs": [] 84 | } 85 | ], 86 | "metadata": {} 87 | } 88 | ] 89 | } -------------------------------------------------------------------------------- /functions/specificlocationdataextraction.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Fri Jun 6 09:43:46 2014 4 | 5 | @author: mattjohnson 6 | """ 7 | from matplotlib import pyplot as plt 8 | from matplotlib import colors 9 | import numpy as np 10 | from mpl_toolkits.basemap import Basemap, addcyclic 11 | import pygrib 12 | 13 | def read_Hrrr_spec_loc(filename, parameters = [''],loc = [-97.485,36.605], max = False): 14 | 15 | """ 16 | With an option for returning just the maximum values of a given list of parameters at a specific location, this 17 | function takes in a filename, list of parameters (all parameters if left blank) and returns the ndarray of data, 18 | the list of parameters, list of heights in hPa, location in latitude, longitude and the list of units 19 | corresponding to each parameter in a list. 20 | """ 21 | 22 | myfile = pygrib.open(filename) 23 | parameterlist = ['Geopotential Height','Temperature','Relative humidity','Dew point temperature', 24 | 'Specific humidity','Vertical velocity','U component of wind','V component of wind', 25 | 'Absolute vorticity','Cloud mixing ratio','Cloud Ice','Rain mixing ratio','Snow mixing ratio', 26 | 'Graupel (snow pellets)'] 27 | 28 | if parameters != ['']: 29 | for i in range(len(parameters)): 30 | x = parameterlist.count(parameters[i]) 31 | if x == 0: 32 | print 'requested parameter not in list' 33 | print parameters[i] 34 | parameterlist = parameters[:] 35 | 36 | 37 | data = [] 38 | grb = myfile.select(name = parameterlist[0]) 39 | grb_cube = grb_to_grid(grb) 40 | dataloc = np.array(grb[0].latlons()) 41 | datah = grb_cube['levels'] 42 | units = [] 43 | 44 | x = abs(dataloc[0]-loc[0]) 45 | y = abs(dataloc[1]-loc[1]) 46 | xy = x+y 47 | xymin = min(xy.flatten()) 48 | xy2 = xy.flatten().tolist() 49 | xyflatindex = xy2.index(xymin) 50 | [ysize,xsize] = dataloc[0].shape 51 | zsize = len(grb_cube['levels']) 52 | xyindex = [xyflatindex/xsize, xyflatindex%xsize] 53 | 54 | 55 | for p in parameterlist: 56 | grb = myfile.select(name = p) 57 | grb_cube = grb_to_grid(grb) 58 | if not max: 59 | newshape = grb_cube['data'].reshape([ysize,xsize,zsize]) 60 | data.append(newshape[xyindex[0]][xyindex[1]][:]) 61 | else: 62 | newshape = grb_cube['data'].reshape([ysize,xsize,zsize]) 63 | data.append(newshape[xyindex[0]][xyindex[1]][:].max(axis=0)) 64 | units.append(grb_cube['units']) 65 | 66 | 67 | return [data,parameterlist,datah,loc,units] -------------------------------------------------------------------------------- /functions/readHrrr.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Jun 3 13:33:41 2014 4 | 5 | @author: mattjohnson 6 | """ 7 | 8 | from matplotlib import pyplot as plt 9 | from matplotlib import colors 10 | import numpy as np 11 | from mpl_toolkits.basemap import Basemap, addcyclic 12 | import pygrib 13 | 14 | def grb_to_grid(grb_obj): 15 | #from scollis 16 | """Takes a single grb object containing multiple 17 | levels. Assumes same time, pressure levels. Compiles to a cube""" 18 | n_levels = len(grb_obj) 19 | levels = np.array([grb_element['level'] for grb_element in grb_obj]) 20 | indexes = np.argsort(levels)[::-1] # highest pressure first 21 | cube = np.zeros([n_levels, grb_obj[0].values.shape[0], grb_obj[1].values.shape[1]]) 22 | for i in range(n_levels): 23 | cube[i,:,:] = grb_obj[indexes[i]].values 24 | cube_dict = {'data' : cube, 'units' : grb_obj[0]['units'], 25 | 'levels' : levels[indexes]} 26 | return cube_dict 27 | 28 | def read_Hrrr(filename, parameters = [''],max = False): 29 | 30 | """ 31 | With an option for returning just the maximum values of a given list of parameters at each location, this function 32 | takes in a filename, list of parameters (all parameters if left blank) and returns the ndarray of data, the list 33 | of parameters, list of heights in hPa, ndarray of locations and the list of units corresponding to each parameter 34 | in a list. 35 | """ 36 | 37 | myfile = pygrib.open(filename) 38 | parameterlist = ['Geopotential Height','Temperature','Relative humidity','Dew point temperature', 39 | 'Specific humidity','Vertical velocity','U component of wind','V component of wind', 40 | 'Absolute vorticity','Cloud mixing ratio','Cloud Ice','Rain mixing ratio','Snow mixing ratio', 41 | 'Graupel (snow pellets)'] 42 | 43 | if parameters != ['']: 44 | for i in range(len(parameters)): 45 | x = parameterlist.count(parameters[i]) 46 | if x == 0: 47 | print 'requested parameter not in list' 48 | print parameters[i] 49 | return 0 50 | parameterlist = parameters[:] 51 | 52 | 53 | data = [] 54 | grb = myfile.select(name = parameterlist[0]) 55 | grb_cube = grb_to_grid(grb) 56 | dataloc = np.array(grb[0].latlons()) 57 | datah = grb_cube['levels'] 58 | units = [] 59 | 60 | for p in parameterlist: 61 | grb = myfile.select(name = p) 62 | grb_cube = grb_to_grid(grb) 63 | if not max: 64 | data.append(grb_cube['data']) 65 | else: 66 | data.append(grb_cube['data'].max(axis=0)) 67 | units.append(grb_cube['units']) 68 | 69 | return [data,parameterlist,datah,dataloc,units] 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /functions/datafileset.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Jun 2 09:23:00 2014 4 | 5 | @author: mattjohnson 6 | """ 7 | 8 | class datafileset: 9 | 10 | 11 | import datetime 12 | import numpy as np 13 | 14 | def __init__(self,path): 15 | self.filesets = [] 16 | self.dates = [] 17 | self.filetype = [] 18 | self.dataname = [] 19 | 20 | #hrrrfiles 21 | [hrrrfiles, dates] = gatherHrrrFiles(path) 22 | 23 | hrrrfiles = np.array(hrrrfiles) 24 | lens = len(hrrrfiles[0]) 25 | newdates = dates[:] 26 | for i in range(lens): 27 | for j in range(len(dates)): 28 | newdates[j] = datetime.datetime(dates[j].year,dates[j].month,dates[j].day,dates[j].hour+i,dates[j].minute) 29 | 30 | temphrrrfiles = [hrrrfiles[j][i] for j in range(len(hrrrfiles))] 31 | self.combine(temphrrrfiles,newdates) 32 | self.filetype.append('hrrr') 33 | self.dataname.append('hrrr'+str(i)) 34 | 35 | "..." 36 | 37 | def combine(self,files,dates): 38 | if self.filesets == []: 39 | self.filesets = [[files[i]] for i in range(len(files))] 40 | self.dates = dates[:] 41 | return 42 | else: 43 | for i in range(len(files)): 44 | if self.dates.count(dates[i]) > 0: 45 | self.filesets[self.dates.index(dates[i])].append(files[i]) 46 | else: 47 | j = 0 48 | size = len(self.filesets[0]) 49 | temparray = [None for k in range(size)] 50 | temparray.append(files[i]) 51 | if dates[i]>self.dates[len(dates)-1]: 52 | self.dates.append(dates[i]) 53 | self.filesets.append(temparray) 54 | continue 55 | while dates[i]>self.dates[j]: 56 | j = j+1 57 | 58 | self.dates.insert(j,dates[i]) 59 | self.filesets.insert(j,temparray) 60 | lengths = [len(self.filesets[i]) for i in range(len(self.filesets))] 61 | maxlength = max(lengths) 62 | for i in range(len(self.filesets)): 63 | for j in range(maxlength-lengths[i]): 64 | self.filesets[i].append(None) 65 | 66 | 67 | def accessfilelist(self,date): 68 | if self.dates.count(date)>0: 69 | index = self.dates.index(date) 70 | filelist = self.filesets[index] 71 | return filelist 72 | else: 73 | print 'invalid date input' 74 | return 75 | 76 | def readdata(self): 77 | values = [[] for k in range(len(self.dates))] 78 | for j in range(len(self.dates)): 79 | files = self.accessfilelist(self.dates[j]) 80 | for i in range(len(self.filetype)): 81 | fcn = 'read'+self.filetype[i]+'('+'files'+')' 82 | values[j].append(eval(fcn)) 83 | 84 | return values 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | -------------------------------------------------------------------------------- /notebooks/saveplothrrrfiles testing.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "name": "", 4 | "signature": "sha256:7150d23ab60328e047982a61cfa4727708928b95f7093b44c395bcadd2c7810b" 5 | }, 6 | "nbformat": 3, 7 | "nbformat_minor": 0, 8 | "worksheets": [ 9 | { 10 | "cells": [ 11 | { 12 | "cell_type": "code", 13 | "collapsed": false, 14 | "input": [ 15 | "from matplotlib import pyplot as plt\n", 16 | "from matplotlib import colors\n", 17 | "import numpy as np\n", 18 | "from mpl_toolkits.basemap import Basemap, addcyclic\n", 19 | "import pygrib\n", 20 | "import os\n", 21 | "\n", 22 | "filename = os.getcwd()+'/HRRRs/'+'hrrr.3d.201405291000f000.grib2'\n", 23 | "parameter = 'Relative humidity'\n", 24 | "fdirectory = os.getcwd()+'/images'\n", 25 | "size = (5,5)\n", 26 | "dpi = 600\n", 27 | "hinp=''\n", 28 | "scaling=1\n", 29 | "final_unit = ''\n", 30 | "vmax=None\n", 31 | "vmin=None\n", 32 | "\n", 33 | "if not os.path.exists(fdirectory):\n", 34 | " os.makedirs(fdirectory)\n", 35 | " \n", 36 | "if hinp != '':\n", 37 | " [data,parameterlist,datah,dataloc,units] = read_Hrrr(filename,[parameter])\n", 38 | "else:\n", 39 | " [data,parameterlist,datah,dataloc,units] = read_Hrrr(filename,[parameter],max=True)\n", 40 | " \n", 41 | "if hinp !='':\n", 42 | " datah = datah.tolist()\n", 43 | " hindex = datah.index(hinp)\n", 44 | " \n", 45 | "if final_unit == '':\n", 46 | " final_unit = units[0]\n", 47 | " \n", 48 | "f = plt.figure(figsize=[12,10])\n", 49 | "m = Basemap(llcrnrlon = -130,llcrnrlat = 20, urcrnrlon = -70,\n", 50 | " urcrnrlat = 52 , projection = 'mill', area_thresh =10000 ,\n", 51 | " resolution='l')\n", 52 | " \n", 53 | " \n", 54 | "x, y = m(dataloc[1],dataloc[0])\n", 55 | "data = np.array(data)\n", 56 | " \n", 57 | "if hinp != '':\n", 58 | " newdata = data[0][hindex][:][:]\n", 59 | "else:\n", 60 | " newdata = data[0]\n", 61 | " \n", 62 | "my_mesh = m.pcolormesh(x, y, newdata*scaling, vmax = vmax,vmin = vmin)\n", 63 | "my_coast = m.drawcoastlines(linewidth=1.25)\n", 64 | "my_states = m.drawstates()\n", 65 | "my_p = m.drawparallels(np.arange(20,80,4),labels=[1,1,0,0])\n", 66 | "my_m = m.drawmeridians(np.arange(-140,-60,4),labels=[0,0,0,1])\n", 67 | " \n", 68 | "plt.colorbar(label=final_unit)\n", 69 | "plt.savefig(fdirectory+'.png',figsize = size,dpi=dpi)\n", 70 | " " 71 | ], 72 | "language": "python", 73 | "metadata": {}, 74 | "outputs": [], 75 | "prompt_number": 5 76 | }, 77 | { 78 | "cell_type": "code", 79 | "collapsed": false, 80 | "input": [ 81 | "import os\n", 82 | "\n", 83 | "wkdir = os.getcwd()\n", 84 | "\n", 85 | "directory = wkdir\n", 86 | "\n", 87 | "while \"HRRR\" in directory:\n", 88 | " os.chdir(os.path.abspath('..'))\n", 89 | "\n", 90 | "dirpath = os.path.abspath(\"HRRR\")\n", 91 | "\n", 92 | "dirpath2 = dirpath+'/functions/'\n", 93 | " \n", 94 | "filenames = os.listdir(dirpath2)\n", 95 | " \n", 96 | "for name in filenames:\n", 97 | " execfile(dirpath2+'/'+name)\n", 98 | "\n", 99 | "os.chdir(wkdir)" 100 | ], 101 | "language": "python", 102 | "metadata": {}, 103 | "outputs": [], 104 | "prompt_number": 2 105 | }, 106 | { 107 | "cell_type": "code", 108 | "collapsed": false, 109 | "input": [ 110 | "fdirectory" 111 | ], 112 | "language": "python", 113 | "metadata": {}, 114 | "outputs": [ 115 | { 116 | "metadata": {}, 117 | "output_type": "pyout", 118 | "prompt_number": 6, 119 | "text": [ 120 | "'/Users/mattjohnson/images'" 121 | ] 122 | } 123 | ], 124 | "prompt_number": 6 125 | }, 126 | { 127 | "cell_type": "code", 128 | "collapsed": false, 129 | "input": [], 130 | "language": "python", 131 | "metadata": {}, 132 | "outputs": [] 133 | } 134 | ], 135 | "metadata": {} 136 | } 137 | ] 138 | } -------------------------------------------------------------------------------- /notebooks/column extraction.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "name": "", 4 | "signature": "sha256:e18cf9272c5c95112581f5b447aa8d64ce8f5ae106e389bc650220f7aece17cf" 5 | }, 6 | "nbformat": 3, 7 | "nbformat_minor": 0, 8 | "worksheets": [ 9 | { 10 | "cells": [ 11 | { 12 | "cell_type": "code", 13 | "collapsed": false, 14 | "input": [ 15 | "from matplotlib import pyplot as plt\n", 16 | "from matplotlib import colors\n", 17 | "import numpy as np\n", 18 | "from mpl_toolkits.basemap import Basemap, addcyclic\n", 19 | "import pygrib\n", 20 | "\n", 21 | "def grb_to_grid(grb_obj):\n", 22 | " #from scollis\n", 23 | " \"\"\"Takes a single grb object containing multiple\n", 24 | " levels. Assumes same time, pressure levels. Compiles to a cube\"\"\"\n", 25 | " n_levels = len(grb_obj)\n", 26 | " levels = np.array([grb_element['level'] for grb_element in grb_obj])\n", 27 | " indexes = np.argsort(levels)[::-1] # highest pressure first\n", 28 | " cube = np.zeros([n_levels, grb_obj[0].values.shape[0], grb_obj[1].values.shape[1]])\n", 29 | " for i in range(n_levels):\n", 30 | " cube[i,:,:] = grb_obj[indexes[i]].values\n", 31 | " cube_dict = {'data' : cube, 'units' : grb_obj[0]['units'],\n", 32 | " 'levels' : levels[indexes]}\n", 33 | " return cube_dict\n", 34 | "\n", 35 | "filename = '/Users/mattjohnson/HRRRs/hrrr.3d.201405291100f000.grib2'\n", 36 | "parameters = ['Relative humidity']\n", 37 | "loc = [36.605,-97.485]\n", 38 | "max = False\n", 39 | "\n", 40 | "myfile = pygrib.open(filename) \n", 41 | "parameterlist = ['Geopotential Height','Temperature','Relative humidity','Dew point temperature',\n", 42 | " 'Specific humidity','Vertical velocity','U component of wind','V component of wind',\n", 43 | " 'Absolute vorticity','Cloud mixing ratio','Cloud Ice','Rain mixing ratio','Snow mixing ratio',\n", 44 | " 'Graupel (snow pellets)'] \n", 45 | " \n", 46 | "if parameters != ['']:\n", 47 | " for i in range(len(parameters)):\n", 48 | " x = parameterlist.count(parameters[i])\n", 49 | " if x == 0: \n", 50 | " print 'requested parameter not in list'\n", 51 | " print parameters[i] \n", 52 | " parameterlist = parameters[:]\n", 53 | " \n", 54 | " \n", 55 | "data = []\n", 56 | "grb = myfile.select(name = parameterlist[0]) \n", 57 | "grb_cube = grb_to_grid(grb)\n", 58 | "dataloc = np.array(grb[0].latlons())\n", 59 | "datah = grb_cube['levels']\n", 60 | "units = []\n", 61 | "\n", 62 | "x = abs(dataloc[0]-loc[0])\n", 63 | "y = abs(dataloc[1]-loc[1])\n", 64 | "xy = x+y\n", 65 | "xymin = min(xy.flatten())\n", 66 | "xy2 = xy.flatten().tolist()\n", 67 | "xyflatindex = xy2.index(xymin)\n", 68 | "[ysize,xsize] = dataloc[0].shape\n", 69 | "zsize = len(grb_cube['levels'])\n", 70 | "xyindex = [xyflatindex/xsize, xyflatindex%xsize]\n", 71 | "\n", 72 | " \n", 73 | "for p in parameterlist:\n", 74 | " grb = myfile.select(name = p)\n", 75 | " grb_cube = grb_to_grid(grb)\n", 76 | " if not max:\n", 77 | " newshape = grb_cube['data'].reshape([ysize,xsize,zsize])\n", 78 | " data.append(newshape[xyindex[0]][xyindex[1]][:])\n", 79 | " else:\n", 80 | " newshape = grb_cube['data'].reshape([ysize,xsize,zsize])\n", 81 | " data.append(newshape[xyindex[0]][xyindex[1]][:].max(axis=2))\n", 82 | " units.append(grb_cube['units'])\n", 83 | " \n", 84 | "\n", 85 | "\n" 86 | ], 87 | "language": "python", 88 | "metadata": {}, 89 | "outputs": [], 90 | "prompt_number": 168 91 | }, 92 | { 93 | "cell_type": "code", 94 | "collapsed": false, 95 | "input": [ 96 | "dataloc[0][xyindex[0]][xyindex[1]]" 97 | ], 98 | "language": "python", 99 | "metadata": {}, 100 | "outputs": [ 101 | { 102 | "metadata": {}, 103 | "output_type": "pyout", 104 | "prompt_number": 172, 105 | "text": [ 106 | "36.616387052759805" 107 | ] 108 | } 109 | ], 110 | "prompt_number": 172 111 | }, 112 | { 113 | "cell_type": "code", 114 | "collapsed": false, 115 | "input": [ 116 | "dataloc[1][xyindex[0]][xyindex[1]]" 117 | ], 118 | "language": "python", 119 | "metadata": {}, 120 | "outputs": [ 121 | { 122 | "metadata": {}, 123 | "output_type": "pyout", 124 | "prompt_number": 173, 125 | "text": [ 126 | "-97.48858991917993" 127 | ] 128 | } 129 | ], 130 | "prompt_number": 173 131 | } 132 | ], 133 | "metadata": {} 134 | } 135 | ] 136 | } -------------------------------------------------------------------------------- /notebooks/plot from readHrrr.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "name": "", 4 | "signature": "sha256:6888b781443610edb1b24951d1dd02abae06250d22f8b9d88234addcbbb16e6f" 5 | }, 6 | "nbformat": 3, 7 | "nbformat_minor": 0, 8 | "worksheets": [ 9 | { 10 | "cells": [ 11 | { 12 | "cell_type": "code", 13 | "collapsed": false, 14 | "input": [ 15 | "import pygrib\n", 16 | "from matplotlib import pyplot as plt\n", 17 | "from matplotlib import colors\n", 18 | "import numpy as np\n", 19 | "from mpl_toolkits.basemap import Basemap, addcyclic\n", 20 | "\n", 21 | "def grb_to_grid(grb_obj):\n", 22 | " \"\"\"Takes a single grb object containing multiple\n", 23 | " levels. Assumes same time, pressure levels. Compiles to a cube\"\"\"\n", 24 | " n_levels = len(grb_obj)\n", 25 | " levels = np.array([grb_element['level'] for grb_element in grb_obj])\n", 26 | " indexes = np.argsort(levels)[::-1] # highest pressure first\n", 27 | " cube = np.zeros([n_levels, grb_obj[0].values.shape[0], grb_obj[1].values.shape[1]])\n", 28 | " for i in range(n_levels):\n", 29 | " cube[i,:,:] = grb_obj[indexes[i]].values\n", 30 | " cube_dict = {'data' : cube, 'units' : grb_obj[0]['units'],\n", 31 | " 'levels' : levels[indexes]}\n", 32 | " return cube_dict\n", 33 | " \n", 34 | "#adaption\n", 35 | "def readHrrr(filename, parameters = ['']):\n", 36 | " #filename = '/Users/mattjohnson/HRRR/hrrr.3d.201405291100f001.grib2'\n", 37 | " \n", 38 | " myfile = pygrib.open(filename) #issues in script\n", 39 | " parameterlist = ['Geopotential Height','Temperature','Relative humidity','Dew point temperature','Specific humidity','Vertical velocity','U component of wind','V component of wind','Absolute vorticity','Cloud mixing ratio','Cloud Ice','Rain mixing ratio','Snow mixing ratio','Graupel (snow pellets)'] \n", 40 | " \n", 41 | " if parameters != ['']:\n", 42 | " for i in range(len(parameters)):\n", 43 | " x = parameterlist.count(parameters[i])\n", 44 | " if x == 0: \n", 45 | " print 'requested parameter not in list'\n", 46 | " print parameters[i] \n", 47 | " return 0\n", 48 | " parameterlist = parameters[:]\n", 49 | " \n", 50 | " \n", 51 | " data = []\n", 52 | " grb = myfile.select(name = parameterlist[0]) \n", 53 | " grb_cube = grb_to_grid(grb)\n", 54 | " dataloc = np.array(grb[0].latlons())\n", 55 | " datah = grb_cube['levels']\n", 56 | " units = []\n", 57 | " \n", 58 | " for p in parameterlist:\n", 59 | " grb = myfile.select(name = p)\n", 60 | " grb_cube = grb_to_grid(grb)\n", 61 | " data.append(grb_cube['data'])\n", 62 | " units.append(grb_cube['units'])\n", 63 | " \n", 64 | " return [data,parameterlist,datah,dataloc,units]" 65 | ], 66 | "language": "python", 67 | "metadata": {}, 68 | "outputs": [], 69 | "prompt_number": 68 70 | }, 71 | { 72 | "cell_type": "code", 73 | "collapsed": false, 74 | "input": [ 75 | "filename = '/Users/mattjohnson/HRRR/hrrr.3d.201405291100f001.grib2'\n", 76 | "parameter = 'Cloud Ice'\n", 77 | "hinp = 800\n", 78 | "\n", 79 | "[data,parameterlist,datah,dataloc,units] = readHrrr(filename,[parameter])\n", 80 | "\n", 81 | "\n", 82 | "datah = datah.tolist()\n", 83 | "hindex = datah.index(hinp)\n", 84 | "\n", 85 | "\n", 86 | "f = plt.figure(figsize=[12,10])\n", 87 | "m = Basemap(llcrnrlon = -130,llcrnrlat = 20, urcrnrlon = -70,\n", 88 | " urcrnrlat = 52 , projection = 'mill', area_thresh =10000 ,\n", 89 | " resolution='l')\n", 90 | "\n", 91 | "\n", 92 | "x, y = m(dataloc[1],dataloc[0])\n", 93 | "data = np.array(data)\n", 94 | "newdata = data[0][hindex][:][:]\n", 95 | "newdata = data[0][17][:][:]\n", 96 | "\n", 97 | "my_mesh = m.pcolormesh(x, y, newdata*1000000, vmax = .05)\n", 98 | "my_coast = m.drawcoastlines(linewidth=1.25)\n", 99 | "my_states = m.drawstates()\n", 100 | "my_p = m.drawparallels(np.arange(20,80,4),labels=[1,1,0,0])\n", 101 | "my_m = m.drawmeridians(np.arange(-140,-60,4),labels=[0,0,0,1])\n", 102 | "\n", 103 | "plt.colorbar(label=units[0])" 104 | ], 105 | "language": "python", 106 | "metadata": {}, 107 | "outputs": [ 108 | { 109 | "metadata": {}, 110 | "output_type": "pyout", 111 | "prompt_number": 73, 112 | "text": [ 113 | "" 114 | ] 115 | } 116 | ], 117 | "prompt_number": 73 118 | }, 119 | { 120 | "cell_type": "code", 121 | "collapsed": false, 122 | "input": [ 123 | "plt.show()" 124 | ], 125 | "language": "python", 126 | "metadata": {}, 127 | "outputs": [], 128 | "prompt_number": 76 129 | }, 130 | { 131 | "cell_type": "code", 132 | "collapsed": false, 133 | "input": [], 134 | "language": "python", 135 | "metadata": {}, 136 | "outputs": [] 137 | } 138 | ], 139 | "metadata": {} 140 | } 141 | ] 142 | } -------------------------------------------------------------------------------- /notebooks/max plot test.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "name": "", 4 | "signature": "sha256:de7ae8230d76d71b7ecb194106f84452da3154b36a4405c36381342f2bd2a8cc" 5 | }, 6 | "nbformat": 3, 7 | "nbformat_minor": 0, 8 | "worksheets": [ 9 | { 10 | "cells": [ 11 | { 12 | "cell_type": "code", 13 | "collapsed": false, 14 | "input": [ 15 | "import pygrib\n", 16 | "from matplotlib import pyplot as plt\n", 17 | "from matplotlib import colors\n", 18 | "import numpy as np\n", 19 | "from numpy import matlib\n", 20 | "from mpl_toolkits.basemap import Basemap, addcyclic\n", 21 | "\n", 22 | "def grb_to_grid(grb_obj):\n", 23 | " \"\"\"Takes a single grb object containing multiple\n", 24 | " levels. Assumes same time, pressure levels. Compiles to a cube\"\"\"\n", 25 | " n_levels = len(grb_obj)\n", 26 | " levels = np.array([grb_element['level'] for grb_element in grb_obj])\n", 27 | " indexes = np.argsort(levels)[::-1] # highest pressure first\n", 28 | " cube = np.zeros([n_levels, grb_obj[0].values.shape[0], grb_obj[1].values.shape[1]])\n", 29 | " for i in range(n_levels):\n", 30 | " cube[i,:,:] = grb_obj[indexes[i]].values\n", 31 | " cube_dict = {'data' : cube, 'units' : grb_obj[0]['units'],\n", 32 | " 'levels' : levels[indexes]}\n", 33 | " return cube_dict\n", 34 | " \n", 35 | "#adaption\n", 36 | "def readHrrr(filename, parameters = ['']):\n", 37 | " #filename = '/Users/mattjohnson/HRRR/hrrr.3d.201405291100f001.grib2'\n", 38 | " \n", 39 | " myfile = pygrib.open(filename) #issues in script\n", 40 | " parameterlist = ['Geopotential Height','Temperature','Relative humidity','Dew point temperature','Specific humidity','Vertical velocity','U component of wind','V component of wind','Absolute vorticity','Cloud mixing ratio','Cloud Ice','Rain mixing ratio','Snow mixing ratio','Graupel (snow pellets)'] \n", 41 | " \n", 42 | " if parameters != ['']:\n", 43 | " for i in range(len(parameters)):\n", 44 | " x = parameterlist.count(parameters[i])\n", 45 | " if x == 0: \n", 46 | " print 'requested parameter not in list'\n", 47 | " print parameters[i] \n", 48 | " return 0\n", 49 | " parameterlist = parameters[:]\n", 50 | " \n", 51 | " \n", 52 | " data = []\n", 53 | " grb = myfile.select(name = parameterlist[0]) \n", 54 | " grb_cube = grb_to_grid(grb)\n", 55 | " dataloc = np.array(grb[0].latlons())\n", 56 | " datah = grb_cube['levels']\n", 57 | " units = []\n", 58 | " \n", 59 | " for p in parameterlist:\n", 60 | " grb = myfile.select(name = p)\n", 61 | " grb_cube = grb_to_grid(grb)\n", 62 | " data.append(grb_cube['data'])\n", 63 | " units.append(grb_cube['units'])\n", 64 | " \n", 65 | " return [data,parameterlist,datah,dataloc,units]\n", 66 | "\n" 67 | ], 68 | "language": "python", 69 | "metadata": {}, 70 | "outputs": [], 71 | "prompt_number": 18 72 | }, 73 | { 74 | "cell_type": "code", 75 | "collapsed": false, 76 | "input": [ 77 | "filename = '/Users/mattjohnson/HRRR/hrrr.3d.201405291100f001.grib2'\n", 78 | "parameter = 'Cloud Ice'\n", 79 | "hinp = 800 \n", 80 | "final_unit = ''\n", 81 | "vmax = .05\n", 82 | "\n", 83 | "scaling = 1000000\n", 84 | "[data,parameterlist,datah,dataloc,units] = readHrrr(filename,[parameter])\n", 85 | " \n", 86 | "datah = datah.tolist()\n", 87 | "hindex = datah.index(hinp)\n", 88 | " \n", 89 | "if final_unit == '':\n", 90 | " final_unit = units[0]\n", 91 | " \n", 92 | "f = plt.figure(figsize=[12,10])\n", 93 | "m = Basemap(llcrnrlon = -130,llcrnrlat = 20, urcrnrlon = -70,urcrnrlat = 52 , projection = 'mill', area_thresh =10000, resolution='l')\n", 94 | " \n", 95 | " \n", 96 | "x, y = m(dataloc[1],dataloc[0])\n", 97 | "data = np.array(data)\n", 98 | "newdata = data[0][:][:][:]\n", 99 | "sh = data.shape\n", 100 | "maxmatrix = np.zeros((sh[2],sh[3]))\n", 101 | "\n", 102 | "for i in range(sh[1]):\n", 103 | " for j in range(sh[2]):\n", 104 | " for k in range(sh[3]):\n", 105 | " v = newdata[i][j][k]\n", 106 | " if i == 0 or abs(v) > abs(maxmatrix[j][k]):\n", 107 | " maxmatrix[j][k] = v\n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | "my_mesh = m.pcolormesh(x, y, maxmatrix*scaling, vmax = vmax)\n", 112 | "my_coast = m.drawcoastlines(linewidth=1.25)\n", 113 | "my_states = m.drawstates()\n", 114 | "my_p = m.drawparallels(np.arange(20,80,4),labels=[1,1,0,0])\n", 115 | "my_m = m.drawmeridians(np.arange(-140,-60,4),labels=[0,0,0,1])\n", 116 | " \n", 117 | "plt.colorbar(label=final_unit)\n", 118 | "plt.show()" 119 | ], 120 | "language": "python", 121 | "metadata": {}, 122 | "outputs": [], 123 | "prompt_number": 52 124 | }, 125 | { 126 | "cell_type": "code", 127 | "collapsed": false, 128 | "input": [], 129 | "language": "python", 130 | "metadata": {}, 131 | "outputs": [] 132 | } 133 | ], 134 | "metadata": {} 135 | } 136 | ] 137 | } -------------------------------------------------------------------------------- /notebooks/read Hrrr test2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "name": "", 4 | "signature": "sha256:0ded5b904ec3b7be51211744b87123d0e0a99ae56635612ed51236edabc62c05" 5 | }, 6 | "nbformat": 3, 7 | "nbformat_minor": 0, 8 | "worksheets": [ 9 | { 10 | "cells": [ 11 | { 12 | "cell_type": "code", 13 | "collapsed": false, 14 | "input": [ 15 | "import pygrib\n", 16 | "from matplotlib import pyplot as plt\n", 17 | "from matplotlib import colors\n", 18 | "import numpy as np\n", 19 | "from mpl_toolkits.basemap import Basemap, addcyclic\n", 20 | "\n", 21 | "def grb_to_grid(grb_obj):\n", 22 | " \"\"\"Takes a single grb object containing multiple\n", 23 | " levels. Assumes same time, pressure levels. Compiles to a cube\"\"\"\n", 24 | " n_levels = len(grb_obj)\n", 25 | " levels = np.array([grb_element['level'] for grb_element in grb_obj])\n", 26 | " indexes = np.argsort(levels)[::-1] # highest pressure first\n", 27 | " cube = np.zeros([n_levels, grb_obj[0].values.shape[0], grb_obj[1].values.shape[1]])\n", 28 | " for i in range(n_levels):\n", 29 | " cube[i,:,:] = grb_obj[indexes[i]].values\n", 30 | " cube_dict = {'data' : cube, 'units' : grb_obj[0]['units'],\n", 31 | " 'levels' : levels[indexes]}\n", 32 | " return cube_dict\n", 33 | "\n", 34 | "parameters = ['']\n", 35 | "filename = '/Users/mattjohnson/HRRR/hrrr.3d.201405291100f001.grib2'\n", 36 | "\n", 37 | "\n", 38 | "myfile = pygrib.open(filename) #issues in script\n", 39 | "parameterlist = ['Geopotential Height','Temperature','Relative humidity','Dew point temperature','Specific humidity','Vertical velocity','U component of wind','V component of wind','Absolute vorticity','Cloud mixing ratio','Cloud Ice','Rain mixing ratio','Snow mixing ratio','Graupel (snow pellets)'] \n", 40 | " \n", 41 | "if parameters != ['']:\n", 42 | " for i in range(len(parameters)):\n", 43 | " x = parameterlist.count(parameters[i])\n", 44 | " if x == 0: \n", 45 | " print 'requested parameter not in list'\n", 46 | " print parameters[i] \n", 47 | " parameterlist = parameters[:]\n", 48 | " \n", 49 | " \n", 50 | " \n", 51 | " \n", 52 | "data = []\n", 53 | "grb = myfile.select(name = parameterlist[0]) \n", 54 | "grb_cube = grb_to_grid(grb)\n", 55 | "dataloc = np.array(grb[0].latlons())\n", 56 | "datah = grb_cube['levels']\n", 57 | "units = []\n", 58 | "\n", 59 | "for p in parameterlist:\n", 60 | " grb = myfile.select(name = p)\n", 61 | " grb_cube = grb_to_grid(grb)\n", 62 | " data.append(grb_cube['data'])\n", 63 | " units.append(grb_cube['units'])\n", 64 | "\n" 65 | ], 66 | "language": "python", 67 | "metadata": {}, 68 | "outputs": [], 69 | "prompt_number": 28 70 | }, 71 | { 72 | "cell_type": "code", 73 | "collapsed": false, 74 | "input": [ 75 | "parameters" 76 | ], 77 | "language": "python", 78 | "metadata": {}, 79 | "outputs": [ 80 | { 81 | "metadata": {}, 82 | "output_type": "pyout", 83 | "prompt_number": 16, 84 | "text": [ 85 | "['Temperature', 'Specific']" 86 | ] 87 | } 88 | ], 89 | "prompt_number": 16 90 | }, 91 | { 92 | "cell_type": "code", 93 | "collapsed": false, 94 | "input": [ 95 | "parameterlist.count('Specific')" 96 | ], 97 | "language": "python", 98 | "metadata": {}, 99 | "outputs": [ 100 | { 101 | "metadata": {}, 102 | "output_type": "pyout", 103 | "prompt_number": 18, 104 | "text": [ 105 | "1" 106 | ] 107 | } 108 | ], 109 | "prompt_number": 18 110 | }, 111 | { 112 | "cell_type": "code", 113 | "collapsed": false, 114 | "input": [ 115 | "parameterlist.index('Specific')" 116 | ], 117 | "language": "python", 118 | "metadata": {}, 119 | "outputs": [ 120 | { 121 | "metadata": {}, 122 | "output_type": "pyout", 123 | "prompt_number": 19, 124 | "text": [ 125 | "1" 126 | ] 127 | } 128 | ], 129 | "prompt_number": 19 130 | }, 131 | { 132 | "cell_type": "code", 133 | "collapsed": false, 134 | "input": [ 135 | "parameterlist[0]" 136 | ], 137 | "language": "python", 138 | "metadata": {}, 139 | "outputs": [ 140 | { 141 | "metadata": {}, 142 | "output_type": "pyout", 143 | "prompt_number": 20, 144 | "text": [ 145 | "'Temperature'" 146 | ] 147 | } 148 | ], 149 | "prompt_number": 20 150 | }, 151 | { 152 | "cell_type": "code", 153 | "collapsed": false, 154 | "input": [ 155 | "parameterlist[1]" 156 | ], 157 | "language": "python", 158 | "metadata": {}, 159 | "outputs": [ 160 | { 161 | "metadata": {}, 162 | "output_type": "pyout", 163 | "prompt_number": 21, 164 | "text": [ 165 | "'Specific'" 166 | ] 167 | } 168 | ], 169 | "prompt_number": 21 170 | }, 171 | { 172 | "cell_type": "code", 173 | "collapsed": false, 174 | "input": [], 175 | "language": "python", 176 | "metadata": {}, 177 | "outputs": [] 178 | } 179 | ], 180 | "metadata": {} 181 | } 182 | ] 183 | } -------------------------------------------------------------------------------- /notebooks/SGP plotting.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "name": "", 4 | "signature": "sha256:3f0661d261204acb99edf4b1ed4d4e51dd5a7f7a752089bfdec4281466a746e8" 5 | }, 6 | "nbformat": 3, 7 | "nbformat_minor": 0, 8 | "worksheets": [ 9 | { 10 | "cells": [ 11 | { 12 | "cell_type": "code", 13 | "collapsed": false, 14 | "input": [ 15 | "import pygrib\n", 16 | "from matplotlib import pyplot as plt\n", 17 | "from matplotlib import colors\n", 18 | "import numpy as np\n", 19 | "from mpl_toolkits.basemap import Basemap, addcyclic\n", 20 | "\n", 21 | "\n", 22 | "def grb_to_grid(grb_obj):\n", 23 | " \"\"\"Takes a single grb object containing multiple\n", 24 | " levels. Assumes same time, pressure levels. Compiles to a cube\"\"\"\n", 25 | " n_levels = len(grb_obj)\n", 26 | " levels = np.array([grb_element['level'] for grb_element in grb_obj])\n", 27 | " indexes = np.argsort(levels)[::-1] # highest pressure first\n", 28 | " cube = np.zeros([n_levels, grb_obj[0].values.shape[0], grb_obj[1].values.shape[1]])\n", 29 | " for i in range(n_levels):\n", 30 | " cube[i,:,:] = grb_obj[indexes[i]].values\n", 31 | " cube_dict = {'data' : cube, 'units' : grb_obj[0]['units'],\n", 32 | " 'levels' : levels[indexes]}\n", 33 | " return cube_dict\n", 34 | " \n", 35 | "#adaption\n", 36 | "def readHrrr(filename, parameters = [''],max = False):\n", 37 | " #filename = '/Users/mattjohnson/HRRR/hrrr.3d.201405291100f001.grib2'\n", 38 | " \n", 39 | " myfile = pygrib.open(filename) #issues in script\n", 40 | " parameterlist = ['Geopotential Height','Temperature','Relative humidity','Dew point temperature','Specific humidity','Vertical velocity','U component of wind','V component of wind','Absolute vorticity','Cloud mixing ratio','Cloud Ice','Rain mixing ratio','Snow mixing ratio','Graupel (snow pellets)'] \n", 41 | " \n", 42 | " if parameters != ['']:\n", 43 | " for i in range(len(parameters)):\n", 44 | " x = parameterlist.count(parameters[i])\n", 45 | " if x == 0: \n", 46 | " print 'requested parameter not in list'\n", 47 | " print parameters[i] \n", 48 | " return 0\n", 49 | " parameterlist = parameters[:]\n", 50 | " \n", 51 | " \n", 52 | " data = []\n", 53 | " grb = myfile.select(name = parameterlist[0]) \n", 54 | " grb_cube = grb_to_grid(grb)\n", 55 | " dataloc = np.array(grb[0].latlons())\n", 56 | " datah = grb_cube['levels']\n", 57 | " units = []\n", 58 | " \n", 59 | " for p in parameterlist:\n", 60 | " grb = myfile.select(name = p)\n", 61 | " grb_cube = grb_to_grid(grb)\n", 62 | " if not max:\n", 63 | " data.append(grb_cube['data'])\n", 64 | " else:\n", 65 | " data.append(grb_cube['data'].max(axis=0))\n", 66 | " units.append(grb_cube['units'])\n", 67 | " \n", 68 | " return [data,parameterlist,datah,dataloc,units]\n", 69 | "\n", 70 | "filename = '/Users/mattjohnson/HRRR/hrrr.3d.201405291100f001.grib2'\n", 71 | "parameter = 'Cloud mixing ratio'\n", 72 | "hinp = 800\n", 73 | "final_unit = ''\n", 74 | "scaling = 1000\n", 75 | "\n", 76 | "if hinp != '':\n", 77 | " [data,parameterlist,datah,dataloc,units] = readHrrr(filename,[parameter])\n", 78 | "else:\n", 79 | " [data,parameterlist,datah,dataloc,units] = readHrrr(filename,[parameter],max=True)\n", 80 | " \n", 81 | "if hinp !='':\n", 82 | " datah = datah.tolist()\n", 83 | " hindex = datah.index(hinp)\n", 84 | " \n", 85 | "if final_unit == '':\n", 86 | " final_unit = units[0]\n", 87 | " \n" 88 | ], 89 | "language": "python", 90 | "metadata": {}, 91 | "outputs": [], 92 | "prompt_number": 38 93 | }, 94 | { 95 | "cell_type": "code", 96 | "collapsed": false, 97 | "input": [ 98 | "margin = 10\n", 99 | "f = plt.figure(figsize=[12,10])\n", 100 | "m = Basemap(llcrnrlon = -97.485-margin,llcrnrlat = 36.605-margin, urcrnrlon = -97.485+margin,\n", 101 | " urcrnrlat = 36.605+margin, projection = 'mill', area_thresh =10000 ,\n", 102 | " resolution='l')\n", 103 | " \n", 104 | "latlonsgp = [-97.485,36.605]\n", 105 | "cities = ['Lamont,OK']\n", 106 | "sgpx,sgpy = m(latlonsgp[0],latlonsgp[1])\n", 107 | "m.plot(sgpx,sgpy,'bo')\n", 108 | "plt.text(sgpx+50000,sgpy+50000,'SGP site')\n", 109 | " \n", 110 | "x, y = m(dataloc[1],dataloc[0])\n", 111 | "data = np.array(data)\n", 112 | "\n", 113 | "\n", 114 | "if hinp != '':\n", 115 | " newdata = data[0][hindex][:][:]\n", 116 | "else:\n", 117 | " newdata = data[0]\n", 118 | " \n", 119 | "my_mesh = m.pcolormesh(x, y, newdata, vmax = .05,norm=colors.LogNorm())\n", 120 | "my_coast = m.drawcoastlines(linewidth=1.25)\n", 121 | "my_states = m.drawstates()\n", 122 | "my_p = m.drawparallels(np.arange(20,80,4),labels=[1,1,0,0])\n", 123 | "my_m = m.drawmeridians(np.arange(-140,-60,4),labels=[0,0,0,1])\n", 124 | " \n", 125 | "plt.colorbar(label=final_unit)\n", 126 | "plt.show()" 127 | ], 128 | "language": "python", 129 | "metadata": {}, 130 | "outputs": [], 131 | "prompt_number": 39 132 | }, 133 | { 134 | "cell_type": "code", 135 | "collapsed": false, 136 | "input": [], 137 | "language": "python", 138 | "metadata": {}, 139 | "outputs": [] 140 | } 141 | ], 142 | "metadata": {} 143 | } 144 | ] 145 | } -------------------------------------------------------------------------------- /functions/gatherHrrrFiles.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Thu May 29 16:55:37 2014 4 | 5 | @author: mattjohnson 6 | """ 7 | 8 | def gather_hrrr_files(path, datestrings = [], hourslists = []): 9 | 10 | """ 11 | Takes in a string path leading to a set of HRRR files, a list of datetime objects (datestrings) and a list of the same 12 | size containing lists of the corresponding model hours requested (hourslists), if datestrings and hourslists are 13 | omitted or both empty the output will be for all dates and model hours files available in the path, if datestrings 14 | and hourslists are not the same length zero is returned along with an error message. 15 | 16 | The output is a list with the first entry being a list corresponding to each requested date with each entry 17 | containing a list of the hour filenames requested and the second entry contains the list of datetime objects that 18 | correspond to each date in the first entry. The hours are always 0,1,2... While the dates come in the same order as 19 | they were in datestrings. If datestrings and hourslists are omitted or both empty the dates and the hours will all 20 | be sequential. 21 | """ 22 | 23 | import numpy as np 24 | import matplotlib 25 | import matplotlib.pyplot as plt 26 | import os 27 | import glob 28 | import matplotlib.dates 29 | import datetime 30 | 31 | if (len(datestrings) != len(hourslists)): 32 | print 'datetimes and requested hours don\'t correspond (different lengths)' 33 | return 34 | 35 | files = os.listdir(path) 36 | 37 | #removing files without correct format 38 | i = 0 39 | 40 | while i < len(files): 41 | filetemp = files[i] 42 | if (filetemp[0:8] != 'hrrr.3d.'): 43 | files.remove(filetemp) 44 | elif (filetemp[24:30] != '.grib2'): 45 | files.remove(filetemp) 46 | else: 47 | i = i+1 48 | 49 | 50 | length = len(files) 51 | 52 | if length == 0: 53 | print 'no available HRRR files' 54 | return 55 | 56 | year =range(length) 57 | months = range(length) 58 | hours = range(length) 59 | days = range(length) 60 | minutes = range(length) 61 | modelhours = range(length) 62 | 63 | #Getting the datetime objects and model hours from each file 64 | for i in range(length): 65 | 66 | tempfile = files[i] 67 | 68 | for j in range(6): 69 | if j==0: 70 | year[i] = int(tempfile[8:12]) 71 | elif j==1: 72 | months[i] = int(tempfile[12:14]) 73 | elif j==2: 74 | days[i] = int(tempfile[14:16]) 75 | elif j==3: 76 | hours[i] = int(tempfile[16:18]) 77 | elif j==4: 78 | minutes[i] = int(tempfile[18:20]) 79 | elif j==5: 80 | modelhours[i] = int(tempfile[21:24]) 81 | 82 | datetimes = [datetime.datetime(year[i],months[i],days[i],hours[i],minutes[i]) for i in range(length)] 83 | 84 | datenums = matplotlib.dates.date2num(datetimes) 85 | datenums = datenums.tolist() 86 | 87 | #Generating the full datestrings and modelhours if only given path 88 | if datestrings==[] and hourslists==[]: 89 | datestrings = datetimes[:] 90 | datestrings = set(datestrings) 91 | datestrings = list(datestrings) 92 | datestrings.sort() 93 | maxhours = max(modelhours) 94 | temparray = [None for i in range(maxhours+1)] 95 | filelists = [temparray[:] for i in range(len(datestrings))] 96 | 97 | for i in range(len(datetimes)): 98 | date = datetimes[i] 99 | hour = modelhours[i] 100 | index = datestrings.index(date) 101 | filelists[index][hour] = files[i] 102 | return [filelists,datestrings] 103 | 104 | 105 | 106 | #finding the filenames corresponding to each requested date/time and model hours 107 | filelists = [] 108 | 109 | for k in range(len(datestrings)): 110 | datestring = datestrings[k] 111 | hours = hourslists[k] 112 | datenumber = matplotlib.dates.date2num(datestring) 113 | indexnum = datenums.count(datenumber) 114 | 115 | 116 | indexes3 = [] 117 | indexes2 = [] 118 | datenumstemp = datenums[:] 119 | 120 | for i in range(indexnum): 121 | aindex = datenumstemp.index(datenumber) 122 | indexes2.append(aindex+i) 123 | datenumstemp.remove(datenumber) 124 | 125 | 126 | modelhourstemp = [modelhours[i] for i in indexes2] 127 | datefiles = [files[i] for i in indexes2] 128 | hourstemp = hours 129 | 130 | #Checking that requested dates and modelhours exist 131 | if indexes2 == []: 132 | print '\nrequested datetime not in path: ' 133 | print datestring 134 | print '(no files gathered for this date)' 135 | continue 136 | 137 | count = 0 138 | for i in range(len(hourstemp)): 139 | for j in range(len(modelhourstemp)): 140 | if hourstemp[i] == modelhourstemp[j]: 141 | count = count+1 142 | 143 | if count, line 1)", 214 | "output_type": "pyerr", 215 | "traceback": [ 216 | "\u001b[0;36m File \u001b[0;32m\"\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m ffmpeg -i 'video.avi' -acodec aac -strict -2 -b:a