├── LICENSE
├── README.md
├── aia_fits_to_zarr.py
├── eve_npy_to_zarr.py
├── hmi_fits_to_zarr.py
└── save_filelists.py


/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2024 SDO Machine Learning Dataset
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # SDOMLv2
 2 | 
 3 | This gitlab repository contains python scripts for getting and processing SDO/AIA, SDO/HMI, and SDO/EVE files to new cloud-friendly .zarr files (SDOMLv2). The data is essentially the same as previous SDOML. For more details about the SDOML dataset, please refer to the following paper:
 4 | 
 5 | "A Machine-learning Data Set Prepared from the NASA Solar Dynamics Observatory Mission" http://adsabs.harvard.edu/abs/2019ApJS..242....7G
 6 | 
 7 | The major changes in SDOMLv2 include:
 8 | 
 9 | - Using the cloud-friendly Zarr format for the storage of chunked, compressed, N-dimensional arrays (https://zarr.readthedocs.io/en/stable/).
10 | - All fits header information is saved in the meta data.
11 | 
12 | A brief description about each script:
13 | * aia_fits_to_zarr.py for processing the AIA synoptic fits data to zarr files
14 | * hmi_fits_to_zarr.py for processing the HMI fits data to zarr files
15 | * eve_npy_to_zarr.py for processing the EVE npy files (from SDOMLv1) to zarr files
16 | * save_filelists.py for saving file lists of AIA channels
17 | 


--------------------------------------------------------------------------------
/aia_fits_to_zarr.py:
--------------------------------------------------------------------------------
  1 | #Given: 
  2 | #   a folder --target to contain zarr files
  3 | #   an integer --scale (a proper divisor of 1024) containing the target output size
  4 | #Converts the source fits files to target zarr files:
  5 | #   -Rescaling the sun to a constant pixel size and correcting for invalid interpolation
  6 | #   -Applying the degradation constant and accounting for exposure time
  7 | #   -Then downsampling by mean
  8 | #   -Save all fits header information to meta data in zarr
  9 | 
 10 | import os, pdb
 11 | import numpy as np
 12 | import sunpy.io
 13 | from tqdm import tqdm
 14 | import zarr
 15 | from sunpy.map import Map
 16 | import skimage.transform
 17 | import gcsfs
 18 | from numcodecs import Blosc, Delta
 19 | import warnings
 20 | import argparse
 21 | 
 22 | warnings.filterwarnings("ignore")
 23 | 
 24 | CHANNELS = [131,1600,1700,171,193,211,304,335,94,4500]
 25 | trgtAS = 976.0
 26 | 
 27 | def parse_args():
 28 |     parser = argparse.ArgumentParser()
 29 |     parser.add_argument('--target',dest='target',required=True)
 30 |     parser.add_argument('--scale',dest='scale',required=True,type=int)
 31 |     args = parser.parse_args()
 32 |     return args
 33 | 
 34 | def loadAIADegrads(path):
 35 |     #return wavelength -> (date -> degradation dictionary)                                         
 36 |     degrads = {}
 37 |     for wl in CHANNELS:
 38 |         degrads[wl] = getDegrad("%s/degrad_%d.csv" % (path,wl))
 39 |     return degrads
 40 | 
 41 | def getDegrad(fn):
 42 |     #map YYYY-MM-DD -> degradation parameter                                                      
 43 |     lines = open(fn).read().strip().split("\n")
 44 |     degrad = {}
 45 |     for l in lines:
 46 |         d, f = l.split(",")
 47 |         degrad[d[1:11]] = float(f)
 48 |     return degrad
 49 | 
 50 | if __name__ == "__main__":
 51 |     args = parse_args()
 52 | 
 53 |     degrads = loadAIADegrads("degrad/")
 54 |     print(np.shape(degrads))
 55 | 
 56 |     print(args)
 57 |     target, scale = args.target, args.scale
 58 | 
 59 |     if not os.path.exists(target):
 60 |         os.mkdir(target)
 61 | 
 62 |     divideFactor = np.int(1024 / scale)
 63 | 
 64 |     #load file lists for AIA channels
 65 |     filelist_131  = np.load('./filelists_2011/filelist_131.npy')
 66 |     filelist_1600 = np.load('./filelists_2011/filelist_1600.npy')
 67 |     filelist_1700  = np.load('./filelists_2011/filelist_1700.npy')
 68 |     filelist_171  = np.load('./filelists_2011/filelist_171.npy')
 69 |     filelist_193  = np.load('./filelists_2011/filelist_193.npy')
 70 |     filelist_211  = np.load('./filelists_2011/filelist_211.npy')
 71 |     filelist_304  = np.load('./filelists_2011/filelist_304.npy')
 72 |     filelist_335  = np.load('./filelists_2011/filelist_335.npy')
 73 |     filelist_94  = np.load('./filelists_2011/filelist_94.npy')
 74 | 
 75 |     store = zarr.DirectoryStore(target+'sdomlv2_2011.zarr')
 76 |     compressor = Blosc(cname='zstd', clevel=5, shuffle=Blosc.BITSHUFFLE)
 77 |     root = zarr.group(store=store,overwrite=True)
 78 |     year = root.create_group('2011')
 79 |     aia131 = year.create_dataset('131A',shape=(np.shape(filelist_131)[0],scale,scale),chunks=(15,None,None),dtype='f4',compressor=compressor)
 80 |     aia1600 = year.create_dataset('1600A',shape=(np.shape(filelist_1600)[0],scale,scale),chunks=(15,None,None),dtype='f4',compressor=compressor)
 81 |     aia1700 = year.create_dataset('1700A',shape=(np.shape(filelist_1700)[0],scale,scale),chunks=(15,None,None),dtype='f4',compressor=compressor)
 82 |     aia171 = year.create_dataset('171A',shape=(np.shape(filelist_171)[0],scale,scale),chunks=(15,None,None),dtype='f4',compressor=compressor)
 83 |     aia193 = year.create_dataset('193A',shape=(np.shape(filelist_193)[0],scale,scale),chunks=(15,None,None),dtype='f4',compressor=compressor)
 84 |     aia211 = year.create_dataset('211A',shape=(np.shape(filelist_211)[0],scale,scale),chunks=(15,None,None),dtype='f4',compressor=compressor)
 85 |     aia304 = year.create_dataset('304A',shape=(np.shape(filelist_304)[0],scale,scale),chunks=(15,None,None),dtype='f4',compressor=compressor)
 86 |     aia335 = year.create_dataset('335A',shape=(np.shape(filelist_335)[0],scale,scale),chunks=(15,None,None),dtype='f4',compressor=compressor)
 87 |     aia94 = year.create_dataset('94A',shape=(np.shape(filelist_94)[0],scale,scale),chunks=(15,None,None),dtype='f4',compressor=compressor)
 88 | 
 89 |     # Process 131A
 90 |     Xd = Map(filelist_131[0])
 91 |     for key in Xd.meta:
 92 |         if key != 'keycomments' and key != 'simple':
 93 |             vars()[key] = []
 94 |     deg_cor = []
 95 |     pixlunit = []
 96 | 
 97 |     for fn,file in tqdm(enumerate(filelist_131)):
 98 |         Xd = Map(file)
 99 |         fn2 = file[-26:].split("_")[0].replace("AIA","")
100 |         datestring = "%s-%s-%s" % (fn2[:4],fn2[4:6],fn2[6:8])
101 |         wavelength = int(file[-26:].split("_")[-1].replace(".fits",""))
102 |         correction = degrads[wavelength][datestring]
103 | 
104 |         for key in Xd.meta:
105 |             if key != 'keycomments' and key != 'simple':
106 |                 vars()[key].append(Xd.meta[key])
107 |         deg_cor.append(correction)
108 |         pixlunit.append('DN/s')
109 | 
110 |         X = Xd.data
111 |         validMask = 1.0 * (X > 0)
112 |         X[np.where(X<=0.0)] = 0.0
113 |         expTime = max(Xd.meta['EXPTIME'],1e-2)
114 |         rad = Xd.meta['RSUN_OBS']
115 |         scale_factor = trgtAS/rad
116 |         t = (X.shape[0]/2.0)-scale_factor*(X.shape[0]/2.0)
117 |         XForm = skimage.transform.SimilarityTransform(scale=scale_factor,translation=(t,t))
118 |         Xr = skimage.transform.warp(X,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
119 |         Xm = skimage.transform.warp(validMask,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
120 |         Xr = np.divide(Xr,(Xm+1e-8))
121 |         Xr = Xr / (expTime*correction)
122 |         Xr = skimage.transform.downscale_local_mean(Xr,(divideFactor,divideFactor))
123 |         Xr = Xr.astype('float32')
124 |         aia131[fn,:,:]=Xr
125 | 
126 |     for key in Xd.meta:
127 |         if key != 'keycomments' and key != 'simple':
128 |             aia131.attrs[key.upper()]=vars()[key]
129 | 
130 |     aia131.attrs['NAXIS1'] = list(np.asarray(naxis1, dtype=np.float64) / divideFactor)
131 |     aia131.attrs['NAXIS2'] = list(np.asarray(naxis2, dtype=np.float64) / divideFactor)
132 |     aia131.attrs['CDELT1'] = list(np.asarray(cdelt1, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
133 |     aia131.attrs['CDELT2'] = list(np.asarray(cdelt2, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
134 |     aia131.attrs['R_SUN'] = list(np.asarray(r_sun, dtype=np.float64) / (4. * divideFactor) * trgtAS / rsun_obs)
135 |     aia131.attrs['X0_MP'] = list(np.asarray(x0_mp, dtype=np.float64) / (4. * divideFactor))
136 |     aia131.attrs['Y0_MP'] = list(np.asarray(y0_mp, dtype=np.float64) / (4. * divideFactor))
137 |     aia131.attrs['CRPIX1']  = list(np.asarray(crpix1, dtype=np.float64) / divideFactor)
138 |     aia131.attrs['CRPIX2']  = list(np.asarray(crpix2, dtype=np.float64) / divideFactor)
139 |     aia131.attrs['DEG_COR']  = list(np.asarray(deg_cor, dtype=np.float64))
140 |     aia131.attrs['PIXLUNIT']  = list(pixlunit)
141 | 
142 |     # Process 1600A
143 |     Xd = Map(filelist_1600[0])
144 |     for key in Xd.meta:
145 |         if key != 'keycomments' and key != 'simple':
146 |             vars()[key] = []
147 |     deg_cor = []
148 |     pixlunit = []
149 | 
150 |     for fn,file in tqdm(enumerate(filelist_1600)):
151 |         Xd = Map(file)
152 |         fn2 = file[-26:].split("_")[0].replace("AIA","")
153 |         datestring = "%s-%s-%s" % (fn2[:4],fn2[4:6],fn2[6:8])
154 |         wavelength = int(file[-26:].split("_")[-1].replace(".fits",""))
155 |         correction = degrads[wavelength][datestring]
156 | 
157 |         for key in Xd.meta:
158 |             if key != 'keycomments' and key != 'simple':
159 |                 vars()[key].append(Xd.meta[key])
160 |         deg_cor.append(correction)
161 |         pixlunit.append('DN/s')
162 | 
163 |         X = Xd.data
164 |         validMask = 1.0 * (X > 0)
165 |         X[np.where(X<=0.0)] = 0.0
166 |         expTime = max(Xd.meta['EXPTIME'],1e-2)
167 |         rad = Xd.meta['RSUN_OBS']
168 |         scale_factor = trgtAS/rad
169 |         t = (X.shape[0]/2.0)-scale_factor*(X.shape[0]/2.0)
170 |         XForm = skimage.transform.SimilarityTransform(scale=scale_factor,translation=(t,t))
171 |         Xr = skimage.transform.warp(X,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
172 |         Xm = skimage.transform.warp(validMask,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
173 |         Xr = np.divide(Xr,(Xm+1e-8))
174 |         Xr = Xr / (expTime*correction)
175 |         Xr = skimage.transform.downscale_local_mean(Xr,(divideFactor,divideFactor))
176 |         Xr = Xr.astype('float32')
177 |         aia1600[fn,:,:]=Xr
178 | 
179 |     for key in Xd.meta:
180 |         if key != 'keycomments' and key != 'simple':
181 |             aia1600.attrs[key.upper()]=vars()[key]
182 |     
183 |     aia1600.attrs['NAXIS1'] = list(np.asarray(naxis1, dtype=np.float64) / divideFactor)
184 |     aia1600.attrs['NAXIS2'] = list(np.asarray(naxis2, dtype=np.float64) / divideFactor)
185 |     aia1600.attrs['CDELT1'] = list(np.asarray(cdelt1, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
186 |     aia1600.attrs['CDELT2'] = list(np.asarray(cdelt2, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
187 |     aia1600.attrs['R_SUN'] = list(np.asarray(r_sun, dtype=np.float64) / (4. * divideFactor) * trgtAS / rsun_obs)
188 |     aia1600.attrs['X0_MP'] = list(np.asarray(x0_mp, dtype=np.float64) / (4. * divideFactor))
189 |     aia1600.attrs['Y0_MP'] = list(np.asarray(y0_mp, dtype=np.float64) / (4. * divideFactor))
190 |     aia1600.attrs['CRPIX1']  = list(np.asarray(crpix1, dtype=np.float64) / divideFactor)
191 |     aia1600.attrs['CRPIX2']  = list(np.asarray(crpix2, dtype=np.float64) / divideFactor)
192 |     aia1600.attrs['DEG_COR']  = list(np.asarray(deg_cor, dtype=np.float64))
193 |     aia1600.attrs['PIXLUNIT']  = list(pixlunit)
194 | 
195 |     # Process 1700A
196 |     Xd = Map(filelist_1700[0])
197 |     for key in Xd.meta:
198 |         if key != 'keycomments' and key != 'simple':
199 |             vars()[key] = []
200 |     deg_cor = []
201 |     pixlunit = []
202 | 
203 |     for fn,file in tqdm(enumerate(filelist_1700)):
204 |         Xd = Map(file)
205 |         fn2 = file[-26:].split("_")[0].replace("AIA","")
206 |         datestring = "%s-%s-%s" % (fn2[:4],fn2[4:6],fn2[6:8])
207 |         wavelength = int(file[-26:].split("_")[-1].replace(".fits",""))
208 |         correction = degrads[wavelength][datestring]
209 | 
210 |         for key in Xd.meta:
211 |             if key != 'keycomments' and key != 'simple':
212 |                 vars()[key].append(Xd.meta[key])
213 |         deg_cor.append(correction)
214 |         pixlunit.append('DN/s')
215 | 
216 |         X = Xd.data
217 |         validMask = 1.0 * (X > 0)
218 |         X[np.where(X<=0.0)] = 0.0
219 |         expTime = max(Xd.meta['EXPTIME'],1e-2)
220 |         rad = Xd.meta['RSUN_OBS']
221 |         scale_factor = trgtAS/rad
222 |         t = (X.shape[0]/2.0)-scale_factor*(X.shape[0]/2.0)
223 |         XForm = skimage.transform.SimilarityTransform(scale=scale_factor,translation=(t,t))
224 |         Xr = skimage.transform.warp(X,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
225 |         Xm = skimage.transform.warp(validMask,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
226 |         Xr = np.divide(Xr,(Xm+1e-8))
227 |         Xr = Xr / (expTime*correction)
228 |         Xr = skimage.transform.downscale_local_mean(Xr,(divideFactor,divideFactor))
229 |         Xr = Xr.astype('float32')
230 |         aia1700[fn,:,:]=Xr
231 | 
232 |     for key in Xd.meta:
233 |         if key != 'keycomments' and key != 'simple':
234 |             aia1700.attrs[key.upper()]=vars()[key]
235 | 
236 |     aia1700.attrs['NAXIS1'] = list(np.asarray(naxis1, dtype=np.float64) / divideFactor)
237 |     aia1700.attrs['NAXIS2'] = list(np.asarray(naxis2, dtype=np.float64) / divideFactor)
238 |     aia1700.attrs['CDELT1'] = list(np.asarray(cdelt1, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
239 |     aia1700.attrs['CDELT2'] = list(np.asarray(cdelt2, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
240 |     aia1700.attrs['R_SUN'] = list(np.asarray(r_sun, dtype=np.float64) / (4. * divideFactor) * trgtAS / rsun_obs)
241 |     aia1700.attrs['X0_MP'] = list(np.asarray(x0_mp, dtype=np.float64) / (4. * divideFactor))
242 |     aia1700.attrs['Y0_MP'] = list(np.asarray(y0_mp, dtype=np.float64) / (4. * divideFactor))
243 |     aia1700.attrs['CRPIX1']  = list(np.asarray(crpix1, dtype=np.float64) / divideFactor)
244 |     aia1700.attrs['CRPIX2']  = list(np.asarray(crpix2, dtype=np.float64) / divideFactor)
245 |     aia1700.attrs['DEG_COR']  = list(np.asarray(deg_cor, dtype=np.float64))
246 |     aia1700.attrs['PIXLUNIT']  = list(pixlunit)
247 | 
248 |     # Process 171A
249 |     Xd = Map(filelist_171[0])
250 |     for key in Xd.meta:
251 |         if key != 'keycomments' and key != 'simple':
252 |             vars()[key] = []
253 |     deg_cor = []
254 |     pixlunit = []
255 | 
256 |     for fn,file in tqdm(enumerate(filelist_171)):
257 |         Xd = Map(file)
258 |         fn2 = file[-26:].split("_")[0].replace("AIA","")
259 |         datestring = "%s-%s-%s" % (fn2[:4],fn2[4:6],fn2[6:8])
260 |         wavelength = int(file[-26:].split("_")[-1].replace(".fits",""))
261 |         correction = degrads[wavelength][datestring]
262 | 
263 |         for key in Xd.meta:
264 |             if key != 'keycomments' and key != 'simple':
265 |                 vars()[key].append(Xd.meta[key])
266 |         deg_cor.append(correction)
267 |         pixlunit.append('DN/s')
268 | 
269 |         X = Xd.data
270 |         validMask = 1.0 * (X > 0)
271 |         X[np.where(X<=0.0)] = 0.0
272 |         expTime = max(Xd.meta['EXPTIME'],1e-2)
273 |         rad = Xd.meta['RSUN_OBS']
274 |         scale_factor = trgtAS/rad
275 |         t = (X.shape[0]/2.0)-scale_factor*(X.shape[0]/2.0)
276 |         XForm = skimage.transform.SimilarityTransform(scale=scale_factor,translation=(t,t))
277 |         Xr = skimage.transform.warp(X,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
278 |         Xm = skimage.transform.warp(validMask,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
279 |         Xr = np.divide(Xr,(Xm+1e-8))
280 |         Xr = Xr / (expTime*correction)
281 |         Xr = skimage.transform.downscale_local_mean(Xr,(divideFactor,divideFactor))
282 |         Xr = Xr.astype('float32')
283 |         aia171[fn,:,:]=Xr
284 | 
285 |     for key in Xd.meta:
286 |         if key != 'keycomments' and key != 'simple':
287 |             aia171.attrs[key.upper()]=vars()[key]
288 | 
289 |     aia171.attrs['NAXIS1'] = list(np.asarray(naxis1, dtype=np.float64) / divideFactor)
290 |     aia171.attrs['NAXIS2'] = list(np.asarray(naxis2, dtype=np.float64) / divideFactor)
291 |     aia171.attrs['CDELT1'] = list(np.asarray(cdelt1, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
292 |     aia171.attrs['CDELT2'] = list(np.asarray(cdelt2, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
293 |     aia171.attrs['R_SUN'] = list(np.asarray(r_sun, dtype=np.float64) / (4. * divideFactor) * trgtAS / rsun_obs)
294 |     aia171.attrs['X0_MP'] = list(np.asarray(x0_mp, dtype=np.float64) / (4. * divideFactor))
295 |     aia171.attrs['Y0_MP'] = list(np.asarray(y0_mp, dtype=np.float64) / (4. * divideFactor))
296 |     aia171.attrs['CRPIX1']  = list(np.asarray(crpix1, dtype=np.float64) / divideFactor)
297 |     aia171.attrs['CRPIX2']  = list(np.asarray(crpix2, dtype=np.float64) / divideFactor)
298 |     aia171.attrs['DEG_COR']  = list(np.asarray(deg_cor, dtype=np.float64))
299 |     aia171.attrs['PIXLUNIT']  = list(pixlunit)
300 | 
301 |     # Process 193A
302 |     Xd = Map(filelist_193[0])
303 |     for key in Xd.meta:
304 |         if key != 'keycomments' and key != 'simple':
305 |             vars()[key] = []
306 |     deg_cor = []
307 |     pixlunit = []
308 | 
309 |     for fn,file in tqdm(enumerate(filelist_193)):
310 |         Xd = Map(file)
311 |         fn2 = file[-26:].split("_")[0].replace("AIA","")
312 |         datestring = "%s-%s-%s" % (fn2[:4],fn2[4:6],fn2[6:8])
313 |         wavelength = int(file[-26:].split("_")[-1].replace(".fits",""))
314 |         correction = degrads[wavelength][datestring]
315 | 
316 |         for key in Xd.meta:
317 |             if key != 'keycomments' and key != 'simple':
318 |                 vars()[key].append(Xd.meta[key])
319 |         deg_cor.append(correction)
320 |         pixlunit.append('DN/s')
321 | 
322 |         X = Xd.data
323 |         validMask = 1.0 * (X > 0)
324 |         X[np.where(X<=0.0)] = 0.0
325 |         expTime = max(Xd.meta['EXPTIME'],1e-2)
326 |         rad = Xd.meta['RSUN_OBS']
327 |         scale_factor = trgtAS/rad
328 |         t = (X.shape[0]/2.0)-scale_factor*(X.shape[0]/2.0)
329 |         XForm = skimage.transform.SimilarityTransform(scale=scale_factor,translation=(t,t))
330 |         Xr = skimage.transform.warp(X,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
331 |         Xm = skimage.transform.warp(validMask,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
332 |         Xr = np.divide(Xr,(Xm+1e-8))
333 |         Xr = Xr / (expTime*correction)
334 |         Xr = skimage.transform.downscale_local_mean(Xr,(divideFactor,divideFactor))
335 |         Xr = Xr.astype('float32')
336 |         aia193[fn,:,:]=Xr
337 | 
338 |     for key in Xd.meta:
339 |         if key != 'keycomments' and key != 'simple':
340 |             aia193.attrs[key.upper()]=vars()[key]
341 |     
342 |     aia193.attrs['NAXIS1'] = list(np.asarray(naxis1, dtype=np.float64) / divideFactor)
343 |     aia193.attrs['NAXIS2'] = list(np.asarray(naxis2, dtype=np.float64) / divideFactor)
344 |     aia193.attrs['CDELT1'] = list(np.asarray(cdelt1, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
345 |     aia193.attrs['CDELT2'] = list(np.asarray(cdelt2, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
346 |     aia193.attrs['R_SUN'] = list(np.asarray(r_sun, dtype=np.float64) / (4. * divideFactor) * trgtAS / rsun_obs)
347 |     aia193.attrs['X0_MP'] = list(np.asarray(x0_mp, dtype=np.float64) / (4. * divideFactor))
348 |     aia193.attrs['Y0_MP'] = list(np.asarray(y0_mp, dtype=np.float64) / (4. * divideFactor))
349 |     aia193.attrs['CRPIX1']  = list(np.asarray(crpix1, dtype=np.float64) / divideFactor)
350 |     aia193.attrs['CRPIX2']  = list(np.asarray(crpix2, dtype=np.float64) / divideFactor)
351 |     aia193.attrs['DEG_COR']  = list(np.asarray(deg_cor, dtype=np.float64))
352 |     aia193.attrs['PIXLUNIT']  = list(pixlunit)
353 | 
354 |     # Process 211A
355 |     Xd = Map(filelist_211[0])
356 |     for key in Xd.meta:
357 |         if key != 'keycomments' and key != 'simple':
358 |             vars()[key] = []
359 |     deg_cor = []
360 |     pixlunit = []
361 | 
362 |     for fn,file in tqdm(enumerate(filelist_211)):
363 |         Xd = Map(file)
364 |         fn2 = file[-26:].split("_")[0].replace("AIA","")
365 |         datestring = "%s-%s-%s" % (fn2[:4],fn2[4:6],fn2[6:8])
366 |         wavelength = int(file[-26:].split("_")[-1].replace(".fits",""))
367 |         correction = degrads[wavelength][datestring]
368 | 
369 |         for key in Xd.meta:
370 |             if key != 'keycomments' and key != 'simple':
371 |                 vars()[key].append(Xd.meta[key])
372 |         deg_cor.append(correction)
373 |         pixlunit.append('DN/s')
374 | 
375 |         X = Xd.data
376 |         validMask = 1.0 * (X > 0)
377 |         X[np.where(X<=0.0)] = 0.0
378 |         expTime = max(Xd.meta['EXPTIME'],1e-2)
379 |         rad = Xd.meta['RSUN_OBS']
380 |         scale_factor = trgtAS/rad
381 |         t = (X.shape[0]/2.0)-scale_factor*(X.shape[0]/2.0)
382 |         XForm = skimage.transform.SimilarityTransform(scale=scale_factor,translation=(t,t))
383 |         Xr = skimage.transform.warp(X,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
384 |         Xm = skimage.transform.warp(validMask,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
385 |         Xr = np.divide(Xr,(Xm+1e-8))
386 |         Xr = Xr / (expTime*correction)
387 |         Xr = skimage.transform.downscale_local_mean(Xr,(divideFactor,divideFactor))
388 |         Xr = Xr.astype('float32')
389 |         aia211[fn,:,:]=Xr
390 | 
391 |     for key in Xd.meta:
392 |         if key != 'keycomments' and key != 'simple':
393 |             aia211.attrs[key.upper()]=vars()[key]
394 |     
395 |     aia211.attrs['NAXIS1'] = list(np.asarray(naxis1, dtype=np.float64) / divideFactor)
396 |     aia211.attrs['NAXIS2'] = list(np.asarray(naxis2, dtype=np.float64) / divideFactor)
397 |     aia211.attrs['CDELT1'] = list(np.asarray(cdelt1, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
398 |     aia211.attrs['CDELT2'] = list(np.asarray(cdelt2, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
399 |     aia211.attrs['R_SUN'] = list(np.asarray(r_sun, dtype=np.float64) / (4. * divideFactor) * trgtAS / rsun_obs)
400 |     aia211.attrs['X0_MP'] = list(np.asarray(x0_mp, dtype=np.float64) / (4. * divideFactor))
401 |     aia211.attrs['Y0_MP'] = list(np.asarray(y0_mp, dtype=np.float64) / (4. * divideFactor))
402 |     aia211.attrs['CRPIX1']  = list(np.asarray(crpix1, dtype=np.float64) / divideFactor)
403 |     aia211.attrs['CRPIX2']  = list(np.asarray(crpix2, dtype=np.float64) / divideFactor)
404 |     aia211.attrs['DEG_COR']  = list(np.asarray(deg_cor, dtype=np.float64))
405 |     aia211.attrs['PIXLUNIT']  = list(pixlunit)
406 | 
407 |     # Process 304A
408 |     Xd = Map(filelist_304[0])
409 |     for key in Xd.meta:
410 |         if key != 'keycomments' and key != 'simple':
411 |             vars()[key] = []
412 |     deg_cor = []
413 |     pixlunit = []
414 | 
415 |     for fn,file in tqdm(enumerate(filelist_304)):
416 |         Xd = Map(file)
417 |         fn2 = file[-26:].split("_")[0].replace("AIA","")
418 |         datestring = "%s-%s-%s" % (fn2[:4],fn2[4:6],fn2[6:8])
419 |         wavelength = int(file[-26:].split("_")[-1].replace(".fits",""))
420 |         correction = degrads[wavelength][datestring]
421 | 
422 |         for key in Xd.meta:
423 |             if key != 'keycomments' and key != 'simple':
424 |                 vars()[key].append(Xd.meta[key])
425 |         deg_cor.append(correction)
426 |         pixlunit.append('DN/s')
427 | 
428 |         X = Xd.data
429 |         validMask = 1.0 * (X > 0)
430 |         X[np.where(X<=0.0)] = 0.0
431 |         expTime = max(Xd.meta['EXPTIME'],1e-2)
432 |         rad = Xd.meta['RSUN_OBS']
433 |         scale_factor = trgtAS/rad
434 |         t = (X.shape[0]/2.0)-scale_factor*(X.shape[0]/2.0)
435 |         XForm = skimage.transform.SimilarityTransform(scale=scale_factor,translation=(t,t))
436 |         Xr = skimage.transform.warp(X,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
437 |         Xm = skimage.transform.warp(validMask,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
438 |         Xr = np.divide(Xr,(Xm+1e-8))
439 |         Xr = Xr / (expTime*correction)
440 |         Xr = skimage.transform.downscale_local_mean(Xr,(divideFactor,divideFactor))
441 |         Xr = Xr.astype('float32')
442 |         aia304[fn,:,:]=Xr
443 | 
444 |     for key in Xd.meta:
445 |         if key != 'keycomments' and key != 'simple':
446 |             aia304.attrs[key.upper()]=vars()[key]
447 |     
448 |     aia304.attrs['NAXIS1'] = list(np.asarray(naxis1, dtype=np.float64) / divideFactor)
449 |     aia304.attrs['NAXIS2'] = list(np.asarray(naxis2, dtype=np.float64) / divideFactor)
450 |     aia304.attrs['CDELT1'] = list(np.asarray(cdelt1, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
451 |     aia304.attrs['CDELT2'] = list(np.asarray(cdelt2, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
452 |     aia304.attrs['R_SUN'] = list(np.asarray(r_sun, dtype=np.float64) / (4. * divideFactor) * trgtAS / rsun_obs)
453 |     aia304.attrs['X0_MP'] = list(np.asarray(x0_mp, dtype=np.float64) / (4. * divideFactor))
454 |     aia304.attrs['Y0_MP'] = list(np.asarray(y0_mp, dtype=np.float64) / (4. * divideFactor))
455 |     aia304.attrs['CRPIX1']  = list(np.asarray(crpix1, dtype=np.float64) / divideFactor)
456 |     aia304.attrs['CRPIX2']  = list(np.asarray(crpix2, dtype=np.float64) / divideFactor)
457 |     aia304.attrs['DEG_COR']  = list(np.asarray(deg_cor, dtype=np.float64))
458 |     aia304.attrs['PIXLUNIT']  = list(pixlunit)
459 | 
460 |     # Process 335A
461 |     Xd = Map(filelist_335[0])
462 |     for key in Xd.meta:
463 |         if key != 'keycomments' and key != 'simple':
464 |             vars()[key] = []
465 |     deg_cor = []
466 |     pixlunit = []
467 | 
468 |     for fn,file in tqdm(enumerate(filelist_335)):
469 |         Xd = Map(file)
470 |         fn2 = file[-26:].split("_")[0].replace("AIA","")
471 |         datestring = "%s-%s-%s" % (fn2[:4],fn2[4:6],fn2[6:8])
472 |         wavelength = int(file[-26:].split("_")[-1].replace(".fits",""))
473 |         correction = degrads[wavelength][datestring]
474 | 
475 |         for key in Xd.meta:
476 |             if key != 'keycomments' and key != 'simple':
477 |                 vars()[key].append(Xd.meta[key])
478 |         deg_cor.append(correction)
479 |         pixlunit.append('DN/s')
480 | 
481 |         X = Xd.data
482 |         validMask = 1.0 * (X > 0)
483 |         X[np.where(X<=0.0)] = 0.0
484 |         expTime = max(Xd.meta['EXPTIME'],1e-2)
485 |         rad = Xd.meta['RSUN_OBS']
486 |         scale_factor = trgtAS/rad
487 |         t = (X.shape[0]/2.0)-scale_factor*(X.shape[0]/2.0)
488 |         XForm = skimage.transform.SimilarityTransform(scale=scale_factor,translation=(t,t))
489 |         Xr = skimage.transform.warp(X,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
490 |         Xm = skimage.transform.warp(validMask,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
491 |         Xr = np.divide(Xr,(Xm+1e-8))
492 |         Xr = Xr / (expTime*correction)
493 |         Xr = skimage.transform.downscale_local_mean(Xr,(divideFactor,divideFactor))
494 |         Xr = Xr.astype('float32')
495 |         aia335[fn,:,:]=Xr
496 | 
497 |     for key in Xd.meta:
498 |         if key != 'keycomments' and key != 'simple':
499 |             aia335.attrs[key.upper()]=vars()[key]
500 |     
501 |     aia335.attrs['NAXIS1'] = list(np.asarray(naxis1, dtype=np.float64) / divideFactor)
502 |     aia335.attrs['NAXIS2'] = list(np.asarray(naxis2, dtype=np.float64) / divideFactor)
503 |     aia335.attrs['CDELT1'] = list(np.asarray(cdelt1, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
504 |     aia335.attrs['CDELT2'] = list(np.asarray(cdelt2, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
505 |     aia335.attrs['R_SUN'] = list(np.asarray(r_sun, dtype=np.float64) / (4. * divideFactor) * trgtAS / rsun_obs)
506 |     aia335.attrs['X0_MP'] = list(np.asarray(x0_mp, dtype=np.float64) / (4. * divideFactor))
507 |     aia335.attrs['Y0_MP'] = list(np.asarray(y0_mp, dtype=np.float64) / (4. * divideFactor))
508 |     aia335.attrs['CRPIX1']  = list(np.asarray(crpix1, dtype=np.float64) / divideFactor)
509 |     aia335.attrs['CRPIX2']  = list(np.asarray(crpix2, dtype=np.float64) / divideFactor)
510 |     aia335.attrs['DEG_COR']  = list(np.asarray(deg_cor, dtype=np.float64))
511 |     aia335.attrs['PIXLUNIT']  = list(pixlunit)
512 | 
513 |     # Process 94A
514 |     Xd = Map(filelist_94[0])
515 |     for key in Xd.meta:
516 |         if key != 'keycomments' and key != 'simple':
517 |             vars()[key] = []
518 |     deg_cor = []
519 |     pixlunit = []
520 | 
521 |     for fn,file in tqdm(enumerate(filelist_94)):
522 |         Xd = Map(file)
523 |         fn2 = file[-26:].split("_")[0].replace("AIA","")
524 |         datestring = "%s-%s-%s" % (fn2[:4],fn2[4:6],fn2[6:8])
525 |         wavelength = int(file[-26:].split("_")[-1].replace(".fits",""))
526 |         correction = degrads[wavelength][datestring]
527 | 
528 |         for key in Xd.meta:
529 |             if key != 'keycomments' and key != 'simple':
530 |                 vars()[key].append(Xd.meta[key])
531 |         deg_cor.append(correction)
532 |         pixlunit.append('DN/s')
533 | 
534 |         X = Xd.data
535 |         validMask = 1.0 * (X > 0)
536 |         X[np.where(X<=0.0)] = 0.0
537 |         expTime = max(Xd.meta['EXPTIME'],1e-2)
538 |         rad = Xd.meta['RSUN_OBS']
539 |         scale_factor = trgtAS/rad
540 |         t = (X.shape[0]/2.0)-scale_factor*(X.shape[0]/2.0)
541 |         XForm = skimage.transform.SimilarityTransform(scale=scale_factor,translation=(t,t))
542 |         Xr = skimage.transform.warp(X,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
543 |         Xm = skimage.transform.warp(validMask,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
544 |         Xr = np.divide(Xr,(Xm+1e-8))
545 |         Xr = Xr / (expTime*correction)
546 |         Xr = skimage.transform.downscale_local_mean(Xr,(divideFactor,divideFactor))
547 |         Xr = Xr.astype('float32')
548 |         aia94[fn,:,:]=Xr
549 | 
550 |     for key in Xd.meta:
551 |         if key != 'keycomments' and key != 'simple':
552 |             aia94.attrs[key.upper()]=vars()[key]
553 | 
554 |     aia94.attrs['NAXIS1'] = list(np.asarray(naxis1, dtype=np.float64) / divideFactor)
555 |     aia94.attrs['NAXIS2'] = list(np.asarray(naxis2, dtype=np.float64) / divideFactor)
556 |     aia94.attrs['CDELT1'] = list(np.asarray(cdelt1, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
557 |     aia94.attrs['CDELT2'] = list(np.asarray(cdelt2, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
558 |     aia94.attrs['R_SUN'] = list(np.asarray(r_sun, dtype=np.float64) / (4. * divideFactor) * trgtAS / rsun_obs)
559 |     aia94.attrs['X0_MP'] = list(np.asarray(x0_mp, dtype=np.float64) / (4. * divideFactor))
560 |     aia94.attrs['Y0_MP'] = list(np.asarray(y0_mp, dtype=np.float64) / (4. * divideFactor))
561 |     aia94.attrs['CRPIX1']  = list(np.asarray(crpix1, dtype=np.float64) / divideFactor)
562 |     aia94.attrs['CRPIX2']  = list(np.asarray(crpix2, dtype=np.float64) / divideFactor)
563 |     aia94.attrs['DEG_COR']  = list(np.asarray(deg_cor, dtype=np.float64))
564 |     aia94.attrs['PIXLUNIT']  = list(pixlunit)
565 | 


--------------------------------------------------------------------------------
/eve_npy_to_zarr.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/python
  2 | # Convert the SDO/EVE .npy files of SDOML to .zarr files.
  3 | 
  4 | import numpy as np
  5 | from astropy.time import Time
  6 | import matplotlib.pyplot as plt
  7 | import pandas as pd
  8 | import zarr
  9 | from numcodecs import Blosc, Delta
 10 | 
 11 | 
 12 | eve_irradiance = np.load('./EVE/irradiance.npy')
 13 | eve_jd = np.load('./EVE/jd.npy')
 14 | eve_logt = np.load('./EVE/logt.npy')
 15 | eve_name = np.load('./EVE/name.npy', allow_pickle=True)
 16 | eve_wavelength = np.load('./EVE/wavelength.npy')
 17 | 
 18 | t = Time(eve_jd, format='jd')
 19 | t.format = 'iso'
 20 | 
 21 | store = zarr.DirectoryStore('eve.zarr')
 22 | compressor = Blosc(cname='zstd', clevel=5, shuffle=Blosc.BITSHUFFLE)
 23 | root = zarr.group(store=store,overwrite=True)
 24 | ints = root.create_group('MEGS-A')
 25 | time = ints.create_dataset('Time',shape=(np.shape(eve_irradiance)[0]),dtype='<U23',compressor=compressor)
 26 | fe18 = ints.create_dataset('Fe XVIII',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 27 | fe8 = ints.create_dataset('Fe VIII',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 28 | fe20 = ints.create_dataset('Fe XX',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 29 | fe9 = ints.create_dataset('Fe IX',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 30 | fe10 = ints.create_dataset('Fe X',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 31 | fe11 = ints.create_dataset('Fe XI',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 32 | fe12 = ints.create_dataset('Fe XII',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 33 | fe13 = ints.create_dataset('Fe XIII',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 34 | fe14 = ints.create_dataset('Fe XIV',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 35 | he2 = ints.create_dataset('He II',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 36 | fe15 = ints.create_dataset('Fe XV',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 37 | he2_2 = ints.create_dataset('He II_2',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 38 | fe16 = ints.create_dataset('Fe XVI',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 39 | fe16_2 = ints.create_dataset('Fe XVI_2',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 40 | mg9 = ints.create_dataset('Mg IX',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 41 | s14 = ints.create_dataset('S XIV',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 42 | ne7 = ints.create_dataset('Ne VII',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 43 | si12 = ints.create_dataset('Si XII',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 44 | si12_2 = ints.create_dataset('Si XII_2',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 45 | o3 = ints.create_dataset('O III',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 46 | he1 = ints.create_dataset('He I',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 47 | o4 = ints.create_dataset('O IV',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 48 | fe20_2 = ints.create_dataset('Fe XX_2',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 49 | he1_2 = ints.create_dataset('He I_2',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 50 | fe19 = ints.create_dataset('Fe XIX',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 51 | o3_2 = ints.create_dataset('O III_2',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 52 | mg10 = ints.create_dataset('Mg X',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 53 | mg10_2 = ints.create_dataset('Mg X_2',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 54 | o5 = ints.create_dataset('O V',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 55 | o2 = ints.create_dataset('O II',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 56 | fe20_3 = ints.create_dataset('Fe XX_3',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 57 | ne8 = ints.create_dataset('Ne VIII',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 58 | o4_2 = ints.create_dataset('O IV_2',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 59 | o2_2 = ints.create_dataset('O II_2',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 60 | h1 = ints.create_dataset('H I',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 61 | h1_2 = ints.create_dataset('H I_2',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 62 | c3 = ints.create_dataset('C III',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 63 | h1_3 = ints.create_dataset('H I_3',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 64 | o6 = ints.create_dataset('O VI',shape=(np.shape(eve_irradiance)[0]),dtype='f4',compressor=compressor)
 65 | 
 66 | time[:] = t.value
 67 | 
 68 | fe18[:] = eve_irradiance[:,0]
 69 | fe18.attrs['wavelength'] = str(eve_wavelength[0])+' nm'
 70 | fe18.attrs['logT'] = str(eve_logt[0])+' MK'
 71 | fe18.attrs['ion'] = 'Fe XVIII'
 72 | 
 73 | fe8[:] = eve_irradiance[:,1]
 74 | fe8.attrs['wavelength'] = str(eve_wavelength[1])+' nm'
 75 | fe8.attrs['logT'] = str(eve_logt[1])+' MK'
 76 | fe8.attrs['ion'] = 'Fe VIII'
 77 | 
 78 | fe20[:] = eve_irradiance[:,2]
 79 | fe20.attrs['wavelength'] = str(eve_wavelength[2])+' nm'
 80 | fe20.attrs['logT'] = str(eve_logt[2])+' MK'
 81 | fe20.attrs['ion'] = 'Fe XX'
 82 | 
 83 | fe9[:] = eve_irradiance[:,3]
 84 | fe9.attrs['wavelength'] = str(eve_wavelength[3])+' nm'
 85 | fe9.attrs['logT'] = str(eve_logt[3])+' MK'
 86 | fe9.attrs['ion'] = 'Fe IX'
 87 | 
 88 | fe10[:] = eve_irradiance[:,4]
 89 | fe10.attrs['wavelength'] = str(eve_wavelength[4])+' nm'
 90 | fe10.attrs['logT'] = str(eve_logt[4])+' MK'
 91 | fe10.attrs['ion'] = 'Fe X'
 92 | 
 93 | fe11[:] = eve_irradiance[:,5]
 94 | fe11.attrs['wavelength'] = str(eve_wavelength[5])+' nm'
 95 | fe11.attrs['logT'] = str(eve_logt[5])+' MK'
 96 | fe11.attrs['ion'] = 'Fe XI'
 97 | 
 98 | fe12[:] = eve_irradiance[:,6]
 99 | fe12.attrs['wavelength'] = str(eve_wavelength[6])+' nm'
100 | fe12.attrs['logT'] = str(eve_logt[6])+' MK'
101 | fe12.attrs['ion'] = 'Fe XII'
102 | 
103 | fe13[:] = eve_irradiance[:,7]
104 | fe13.attrs['wavelength'] = str(eve_wavelength[7])+' nm'
105 | fe13.attrs['logT'] = str(eve_logt[7])+' MK'
106 | fe13.attrs['ion'] = 'Fe XIII'
107 | 
108 | fe14[:] = eve_irradiance[:,8]
109 | fe14.attrs['wavelength'] = str(eve_wavelength[8])+' nm'
110 | fe14.attrs['logT'] = str(eve_logt[8])+' MK'
111 | fe14.attrs['ion'] = 'Fe XIV'
112 | 
113 | he2[:] = eve_irradiance[:,9]
114 | he2.attrs['wavelength'] = str(eve_wavelength[9])+' nm'
115 | he2.attrs['logT'] = str(eve_logt[9])+' MK'
116 | he2.attrs['ion'] = 'He II'
117 | 
118 | fe15[:] = eve_irradiance[:,10]
119 | fe15.attrs['wavelength'] = str(eve_wavelength[10])+' nm'
120 | fe15.attrs['logT'] = str(eve_logt[10])+' MK'
121 | fe15.attrs['ion'] = 'Fe XV'
122 | 
123 | he2_2[:] = eve_irradiance[:,11]
124 | he2_2.attrs['wavelength'] = str(eve_wavelength[11])+' nm'
125 | he2_2.attrs['logT'] = str(eve_logt[11])+' MK'
126 | he2_2.attrs['ion'] = 'He II'
127 | 
128 | fe16[:] = eve_irradiance[:,12]
129 | fe16.attrs['wavelength'] = str(eve_wavelength[12])+' nm'
130 | fe16.attrs['logT'] = str(eve_logt[12])+' MK'
131 | fe16.attrs['ion'] = 'Fe XVI'
132 | 
133 | fe16_2[:] = eve_irradiance[:,13]
134 | fe16_2.attrs['wavelength'] = str(eve_wavelength[13])+' nm'
135 | fe16_2.attrs['logT'] = str(eve_logt[13])+' MK'
136 | fe16_2.attrs['ion'] = 'Fe XVI'
137 | 
138 | mg9[:] = eve_irradiance[:,14]
139 | mg9.attrs['wavelength'] = str(eve_wavelength[14])+' nm'
140 | mg9.attrs['logT'] = str(eve_logt[14])+' MK'
141 | mg9.attrs['ion'] = 'Mg IX'
142 | 
143 | s14[:] = eve_irradiance[:,15]
144 | s14.attrs['wavelength'] = str(eve_wavelength[15])+' nm'
145 | s14.attrs['logT'] = str(eve_logt[15])+' MK'
146 | s14.attrs['ion'] = 'S XIV'
147 | 
148 | ne7[:] = eve_irradiance[:,16]
149 | ne7.attrs['wavelength'] = str(eve_wavelength[16])+' nm'
150 | ne7.attrs['logT'] = str(eve_logt[16])+' MK'
151 | ne7.attrs['ion'] = 'Ne VII'
152 | 
153 | si12[:] = eve_irradiance[:,17]
154 | si12.attrs['wavelength'] = str(eve_wavelength[17])+' nm'
155 | si12.attrs['logT'] = str(eve_logt[17])+' MK'
156 | si12.attrs['ion'] = 'Si XII'
157 | 
158 | si12_2[:] = eve_irradiance[:,18]
159 | si12_2.attrs['wavelength'] = str(eve_wavelength[18])+' nm'
160 | si12_2.attrs['logT'] = str(eve_logt[18])+' MK'
161 | si12_2.attrs['ion'] = 'Si XII'
162 | 
163 | o3[:] = eve_irradiance[:,19]
164 | o3.attrs['wavelength'] = str(eve_wavelength[19])+' nm'
165 | o3.attrs['logT'] = str(eve_logt[19])+' MK'
166 | o3.attrs['ion'] = 'O III'
167 | 
168 | he1[:] = eve_irradiance[:,20]
169 | he1.attrs['wavelength'] = str(eve_wavelength[20])+' nm'
170 | he1.attrs['logT'] = str(eve_logt[20])+' MK'
171 | he1.attrs['ion'] = 'He I'
172 | 
173 | o4[:] = eve_irradiance[:,21]
174 | o4.attrs['wavelength'] = str(eve_wavelength[21])+' nm'
175 | o4.attrs['logT'] = str(eve_logt[21])+' MK'
176 | o4.attrs['ion'] = 'O IV'
177 | 
178 | fe20_2[:] = eve_irradiance[:,22]
179 | fe20_2.attrs['wavelength'] = str(eve_wavelength[22])+' nm'
180 | fe20_2.attrs['logT'] = str(eve_logt[22])+' MK'
181 | fe20_2.attrs['ion'] = 'Fe XX'
182 | 
183 | he1_2[:] = eve_irradiance[:,23]
184 | he1_2.attrs['wavelength'] = str(eve_wavelength[23])+' nm'
185 | he1_2.attrs['logT'] = str(eve_logt[23])+' MK'
186 | he1_2.attrs['ion'] = 'He I'
187 | 
188 | fe19[:] = eve_irradiance[:,24]
189 | fe19.attrs['wavelength'] = str(eve_wavelength[24])+' nm'
190 | fe19.attrs['logT'] = str(eve_logt[24])+' MK'
191 | fe19.attrs['ion'] = 'Fe XIX'
192 | 
193 | o3_2[:] = eve_irradiance[:,25]
194 | o3_2.attrs['wavelength'] = str(eve_wavelength[25])+' nm'
195 | o3_2.attrs['logT'] = str(eve_logt[25])+' MK'
196 | o3_2.attrs['ion'] = 'O III'
197 | 
198 | mg10[:] = eve_irradiance[:,26]
199 | mg10.attrs['wavelength'] = str(eve_wavelength[26])+' nm'
200 | mg10.attrs['logT'] = str(eve_logt[26])+' MK'
201 | mg10.attrs['ion'] = 'Mg X'
202 | 
203 | mg10_2[:] = eve_irradiance[:,27]
204 | mg10_2.attrs['wavelength'] = str(eve_wavelength[27])+' nm'
205 | mg10_2.attrs['logT'] = str(eve_logt[27])+' MK'
206 | mg10_2.attrs['ion'] = 'Mg X'
207 | 
208 | o5[:] = eve_irradiance[:,28]
209 | o5.attrs['wavelength'] = str(eve_wavelength[28])+' nm'
210 | o5.attrs['logT'] = str(eve_logt[28])+' MK'
211 | o5.attrs['ion'] = 'O V'
212 | 
213 | o2[:] = eve_irradiance[:,29]
214 | o2.attrs['wavelength'] = str(eve_wavelength[29])+' nm'
215 | o2.attrs['logT'] = str(eve_logt[29])+' MK'
216 | o2.attrs['ion'] = 'O II'
217 | 
218 | fe20_3[:] = eve_irradiance[:,30]
219 | fe20_3.attrs['wavelength'] = str(eve_wavelength[30])+' nm'
220 | fe20_3.attrs['logT'] = str(eve_logt[30])+' MK'
221 | fe20_3.attrs['ion'] = 'Fe XX'
222 | 
223 | ne8[:] = eve_irradiance[:,31]
224 | ne8.attrs['wavelength'] = str(eve_wavelength[31])+' nm'
225 | ne8.attrs['logT'] = str(eve_logt[31])+' MK'
226 | ne8.attrs['ion'] = 'Ne VIII'
227 | 
228 | o4_2[:] = eve_irradiance[:,32]
229 | o4_2.attrs['wavelength'] = str(eve_wavelength[32])+' nm'
230 | o4_2.attrs['logT'] = str(eve_logt[32])+' MK'
231 | o4_2.attrs['ion'] = 'O IV'
232 | 
233 | o2_2[:] = eve_irradiance[:,33]
234 | o2_2.attrs['wavelength'] = str(eve_wavelength[33])+' nm'
235 | o2_2.attrs['logT'] = str(eve_logt[33])+' MK'
236 | o2_2.attrs['ion'] = 'O II'
237 | 
238 | h1[:] = eve_irradiance[:,34]
239 | h1.attrs['wavelength'] = str(eve_wavelength[34])+' nm'
240 | h1.attrs['logT'] = str(eve_logt[34])+' MK'
241 | h1.attrs['ion'] = 'H I'
242 | 
243 | h1_2[:] = eve_irradiance[:,35]
244 | h1_2.attrs['wavelength'] = str(eve_wavelength[35])+' nm'
245 | h1_2.attrs['logT'] = str(eve_logt[35])+' MK'
246 | h1_2.attrs['ion'] = 'H I'
247 | 
248 | c3[:] = eve_irradiance[:,36]
249 | c3.attrs['wavelength'] = str(eve_wavelength[36])+' nm'
250 | c3.attrs['logT'] = str(eve_logt[36])+' MK'
251 | c3.attrs['ion'] = 'C III'
252 | 
253 | h1_3[:] = eve_irradiance[:,37]
254 | h1_3.attrs['wavelength'] = str(eve_wavelength[37])+' nm'
255 | h1_3.attrs['logT'] = str(eve_logt[37])+' MK'
256 | h1_3.attrs['ion'] = 'H I'
257 | 
258 | o6[:] = eve_irradiance[:,38]
259 | o6.attrs['wavelength'] = str(eve_wavelength[38])+' nm'
260 | o6.attrs['logT'] = str(eve_logt[38])+' MK'
261 | o6.attrs['ion'] = 'O VI'
262 | 


--------------------------------------------------------------------------------
/hmi_fits_to_zarr.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/python
  2 | #Given:
  3 | #   a folder --src containing fits files
  4 | #   a folder --target to contain zarr files
  5 | #   an integer --scale (a proper divisor of 512) containing the target output size
  6 | #Converts the source fits to target zarr files:
  7 | #   -Rescaling the sun to a constant pixel size and correcting for invalid interpolation
  8 | #   -Save all fits header information to meta data in zarr  
  9 | 
 10 | import os, pdb
 11 | import numpy as np
 12 | import sunpy.io
 13 | import glob
 14 | from tqdm import tqdm
 15 | import zarr
 16 | from sunpy.map import Map
 17 | import skimage.transform
 18 | import gcsfs
 19 | from numcodecs import Blosc, Delta
 20 | import warnings
 21 | import argparse
 22 | 
 23 | warnings.filterwarnings("ignore")
 24 | 
 25 | trgtAS = 976.0
 26 | 
 27 | def parse_args():
 28 |     parser = argparse.ArgumentParser()
 29 |     parser.add_argument('--src',dest='src',required=True)
 30 |     parser.add_argument('--target',dest='target',required=True)
 31 |     parser.add_argument('--scale',dest='scale',required=True,type=int)
 32 |     args = parser.parse_args()
 33 |     return args
 34 | 
 35 | if __name__ == "__main__":
 36 |     args = parse_args()
 37 | 
 38 |     print(args)
 39 |     src, target, scale = args.src, args.target, args.scale
 40 | 
 41 |     if not os.path.exists(target):
 42 |         os.mkdir(target)
 43 | 
 44 |     divideFactor = np.int(512 / scale)
 45 | 
 46 |     filelist_bx  = sorted(glob.glob(src+'*bx.fits'))
 47 |     filelist_by = sorted(glob.glob(src+'*by.fits'))
 48 |     filelist_bz  = sorted(glob.glob(src+'*bz.fits'))
 49 | 
 50 |     store = zarr.DirectoryStore(target+'sdomlv2_hmi_2011.zarr')
 51 |     compressor = Blosc(cname='zstd', clevel=5, shuffle=Blosc.BITSHUFFLE)
 52 |     root = zarr.group(store=store,overwrite=True)
 53 |     year = root.create_group('2011')
 54 |     bx = year.create_dataset('Bx',shape=(np.shape(filelist_bx)[0],scale,scale),chunks=(15,None,None),dtype='f4',compressor=compressor)
 55 |     by = year.create_dataset('By',shape=(np.shape(filelist_by)[0],scale,scale),chunks=(15,None,None),dtype='f4',compressor=compressor)
 56 |     bz = year.create_dataset('Bz',shape=(np.shape(filelist_bz)[0],scale,scale),chunks=(15,None,None),dtype='f4',compressor=compressor)
 57 | 
 58 |     # Process BX
 59 |     Xd = Map(filelist_bx[-1])
 60 |     for key in Xd.meta:
 61 |         if key != 'keycomments' and key != 'simple' and key != 'history':
 62 |             vars()[key] = []
 63 |     pixlunit = []
 64 | 
 65 |     for fn,file in tqdm(enumerate(filelist_bx)):
 66 |         Xd = Map(file)
 67 |         for key in Xd.meta:
 68 |             if key != 'keycomments' and key != 'simple' and key != 'history':
 69 |                 vars()[key].append(Xd.meta[key])
 70 |         pixlunit.append('Gauss')
 71 | 
 72 |         X = Xd.data
 73 |         validMask = 1.0 * (X < 1.E6)
 74 |         rad = Xd.meta['RSUN_OBS']
 75 |         scale_factor = trgtAS/rad
 76 |         t = (X.shape[0]/2.0)-scale_factor*(X.shape[0]/2.0)
 77 |         XForm = skimage.transform.SimilarityTransform(scale=scale_factor,translation=(t,t))
 78 |         Xr = skimage.transform.warp(X,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
 79 |         Xm = skimage.transform.warp(validMask,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
 80 |         Xr = np.divide(Xr,(Xm+1e-8))
 81 |         Xr = skimage.transform.downscale_local_mean(Xr,(divideFactor,divideFactor))
 82 |         Xr = Xr.astype('float32')
 83 |         bx[fn,:,:]=Xr
 84 | 
 85 |     for key in Xd.meta:
 86 |         if key != 'keycomments' and key != 'simple' and key != 'history':
 87 |             bx.attrs[key.upper()]=vars()[key]
 88 | 
 89 |     bx.attrs['NAXIS1'] = list(np.asarray(naxis1, dtype=np.float64) / divideFactor)
 90 |     bx.attrs['NAXIS2'] = list(np.asarray(naxis2, dtype=np.float64) / divideFactor)
 91 |     bx.attrs['CDELT1'] = list(np.asarray(cdelt1, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
 92 |     bx.attrs['CDELT2'] = list(np.asarray(cdelt2, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
 93 |     bx.attrs['R_SUN'] = list(np.asarray(r_sun, dtype=np.float64) / (8. * divideFactor) * trgtAS / rsun_obs)
 94 |     bx.attrs['CRPIX1']  = list(np.asarray(crpix1, dtype=np.float64) / divideFactor)
 95 |     bx.attrs['CRPIX2']  = list(np.asarray(crpix2, dtype=np.float64) / divideFactor)
 96 |     bx.attrs['PIXLUNIT']  = list(pixlunit)
 97 | 
 98 |     # Process BY
 99 |     Xd = Map(filelist_by[-1])
100 |     for key in Xd.meta:
101 |         if key != 'keycomments' and key != 'simple' and key != 'history':
102 |             vars()[key] = []
103 |     pixlunit = []
104 | 
105 |     for fn,file in tqdm(enumerate(filelist_by)):
106 |         Xd = Map(file)
107 |         for key in Xd.meta:
108 |             if key != 'keycomments' and key != 'simple' and key != 'history':
109 |                 vars()[key].append(Xd.meta[key])
110 |         pixlunit.append('Gauss')
111 | 
112 |         X = Xd.data
113 |         validMask = 1.0 * (X < 1.E6)
114 |         rad = Xd.meta['RSUN_OBS']
115 |         scale_factor = trgtAS/rad
116 |         t = (X.shape[0]/2.0)-scale_factor*(X.shape[0]/2.0)
117 |         XForm = skimage.transform.SimilarityTransform(scale=scale_factor,translation=(t,t))
118 |         Xr = skimage.transform.warp(X,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
119 |         Xm = skimage.transform.warp(validMask,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
120 |         Xr = np.divide(Xr,(Xm+1e-8))
121 |         Xr = skimage.transform.downscale_local_mean(Xr,(divideFactor,divideFactor))
122 |         Xr = Xr.astype('float32')
123 |         by[fn,:,:]=Xr
124 | 
125 |     for key in Xd.meta:
126 |         if key != 'keycomments' and key != 'simple' and key != 'history':
127 |             by.attrs[key.upper()]=vars()[key]
128 | 
129 |     by.attrs['NAXIS1'] = list(np.asarray(naxis1, dtype=np.float64) / divideFactor)
130 |     by.attrs['NAXIS2'] = list(np.asarray(naxis2, dtype=np.float64) / divideFactor)
131 |     by.attrs['CDELT1'] = list(np.asarray(cdelt1, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
132 |     by.attrs['CDELT2'] = list(np.asarray(cdelt2, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
133 |     by.attrs['R_SUN'] = list(np.asarray(r_sun, dtype=np.float64) / (8. * divideFactor) * trgtAS / rsun_obs)
134 |     by.attrs['CRPIX1']  = list(np.asarray(crpix1, dtype=np.float64) / divideFactor)
135 |     by.attrs['CRPIX2']  = list(np.asarray(crpix2, dtype=np.float64) / divideFactor)
136 |     by.attrs['PIXLUNIT']  = list(pixlunit)
137 | 
138 |     # Process BZ
139 |     Xd = Map(filelist_bz[-1])
140 |     for key in Xd.meta:
141 |         if key != 'keycomments' and key != 'simple' and key != 'history':
142 |             vars()[key] = []
143 |     pixlunit = []
144 | 
145 |     for fn,file in tqdm(enumerate(filelist_bz)):
146 |         Xd = Map(file)
147 |         for key in Xd.meta:
148 |             if key != 'keycomments' and key != 'simple' and key != 'history':
149 |                 vars()[key].append(Xd.meta[key])
150 |         pixlunit.append('Gauss')
151 | 
152 |         X = Xd.data
153 |         validMask = 1.0 * (X < 1.E6)
154 |         rad = Xd.meta['RSUN_OBS']
155 |         scale_factor = trgtAS/rad
156 |         t = (X.shape[0]/2.0)-scale_factor*(X.shape[0]/2.0)
157 |         XForm = skimage.transform.SimilarityTransform(scale=scale_factor,translation=(t,t))
158 |         Xr = skimage.transform.warp(X,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
159 |         Xm = skimage.transform.warp(validMask,XForm.inverse,preserve_range=True,mode='edge',output_shape=(X.shape[0],X.shape[0]))
160 |         Xr = np.divide(Xr,(Xm+1e-8))
161 |         Xr = skimage.transform.downscale_local_mean(Xr,(divideFactor,divideFactor))
162 |         Xr = Xr.astype('float32')
163 |         bz[fn,:,:]=Xr
164 | 
165 |     for key in Xd.meta:
166 |         if key != 'keycomments' and key != 'simple' and key != 'history':
167 |             bz.attrs[key.upper()]=vars()[key]
168 | 
169 |     bz.attrs['NAXIS1'] = list(np.asarray(naxis1, dtype=np.float64) / divideFactor)
170 |     bz.attrs['NAXIS2'] = list(np.asarray(naxis2, dtype=np.float64) / divideFactor)
171 |     bz.attrs['CDELT1'] = list(np.asarray(cdelt1, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
172 |     bz.attrs['CDELT2'] = list(np.asarray(cdelt2, dtype=np.float64) * divideFactor * rsun_obs / trgtAS)
173 |     bz.attrs['R_SUN'] = list(np.asarray(r_sun, dtype=np.float64) / (8. * divideFactor) * trgtAS / rsun_obs)
174 |     bz.attrs['CRPIX1']  = list(np.asarray(crpix1, dtype=np.float64) / divideFactor)
175 |     bz.attrs['CRPIX2']  = list(np.asarray(crpix2, dtype=np.float64) / divideFactor)
176 |     bz.attrs['PIXLUNIT']  = list(pixlunit)
177 | 


--------------------------------------------------------------------------------
/save_filelists.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/python
  2 | #Save file lists for the AIA channels.
  3 | 
  4 | import os, pdb
  5 | import numpy as np
  6 | import sunpy.io
  7 | import warnings
  8 | import argparse
  9 | 
 10 | warnings.filterwarnings("ignore")
 11 | 
 12 | CHANNELS = [131,1600,1700,171,193,211,304,335,94,4500]
 13 | 
 14 | def parse_args():
 15 |     parser = argparse.ArgumentParser()
 16 |     parser.add_argument('--src',dest='src',required=True)
 17 |     args = parser.parse_args()
 18 |     return args
 19 | 
 20 | if __name__ == "__main__":
 21 |     args = parse_args()
 22 | 
 23 |     print(args)
 24 |     src = args.src
 25 | 
 26 |     if not os.path.exists(target):
 27 |         os.mkdir(target)
 28 | 
 29 |     filelist_131=[]
 30 |     filelist_1600=[]
 31 |     filelist_1700=[]
 32 |     filelist_171=[]
 33 |     filelist_193=[]
 34 |     filelist_211=[]
 35 |     filelist_304=[]
 36 |     filelist_335=[]
 37 |     filelist_94=[]
 38 |     filelist_4500=[]
 39 | 
 40 |     for root, dirs, files in os.walk(src,followlinks=True):
 41 |         for file in tqdm(files):
 42 |             if file.endswith(".fits") and int(file[-12:-10]) % 6 == 0:
 43 |                 if file[-9:-5] == '0131':
 44 |                     try:
 45 |                         Xh = sunpy.io.read_file_header(os.path.join(root,file))
 46 |                         if Xh[1]['QUALITY'] == 0:
 47 |                             filelist_131.append(os.path.join(root,file))
 48 |                     except:
 49 |                         print("FILE CORRUPTED: %s" % file)
 50 |                         continue
 51 |                 elif file[-9:-5] == '1600':
 52 |                     try:
 53 |                         Xh = sunpy.io.read_file_header(os.path.join(root,file))
 54 |                         if Xh[1]['QUALITY'] == 0:
 55 |                             filelist_1600.append(os.path.join(root,file))
 56 |                     except:
 57 |                         print("FILE CORRUPTED: %s" % file)
 58 |                         continue
 59 |                 elif file[-9:-5] == '1700':
 60 |                     try:
 61 |                         Xh = sunpy.io.read_file_header(os.path.join(root,file))
 62 |                         if Xh[1]['QUALITY'] == 0:
 63 |                             filelist_1700.append(os.path.join(root,file))
 64 |                     except:
 65 |                         print("FILE CORRUPTED: %s" % file)
 66 |                         continue
 67 |                 elif file[-9:-5] == '0171':
 68 |                     try:
 69 |                         Xh = sunpy.io.read_file_header(os.path.join(root,file))
 70 |                         if Xh[1]['QUALITY'] == 0:
 71 |                             filelist_171.append(os.path.join(root,file))
 72 |                     except:
 73 |                         print("FILE CORRUPTED: %s" % file)
 74 |                         continue
 75 |                 elif file[-9:-5] == '0193':
 76 |                     try:
 77 |                         Xh = sunpy.io.read_file_header(os.path.join(root,file))
 78 |                         if Xh[1]['QUALITY'] == 0:
 79 |                             filelist_193.append(os.path.join(root,file))
 80 |                     except:
 81 |                         print("FILE CORRUPTED: %s" % file)
 82 |                         continue
 83 |                 elif file[-9:-5] == '0211':
 84 |                     try:
 85 |                         Xh = sunpy.io.read_file_header(os.path.join(root,file))
 86 |                         if Xh[1]['QUALITY'] == 0:
 87 |                             filelist_211.append(os.path.join(root,file))
 88 |                     except:
 89 |                         print("FILE CORRUPTED: %s" % file)
 90 |                         continue
 91 |                 elif file[-9:-5] == '0304':
 92 |                     try:
 93 |                         Xh = sunpy.io.read_file_header(os.path.join(root,file))
 94 |                         if Xh[1]['QUALITY'] == 0:
 95 |                             filelist_304.append(os.path.join(root,file))
 96 |                     except:
 97 |                         print("FILE CORRUPTED: %s" % file)
 98 |                         continue
 99 |                 elif file[-9:-5] == '0335':
100 |                     try:
101 |                         Xh = sunpy.io.read_file_header(os.path.join(root,file))
102 |                         if Xh[1]['QUALITY'] == 0:
103 |                             filelist_335.append(os.path.join(root,file))
104 |                     except:
105 |                         print("FILE CORRUPTED: %s" % file)
106 |                         continue
107 |                 elif file[-9:-5] == '0094':
108 |                     try:
109 |                         Xh = sunpy.io.read_file_header(os.path.join(root,file))
110 |                         if Xh[1]['QUALITY'] == 0:
111 |                             filelist_94.append(os.path.join(root,file))
112 |                     except:
113 |                         print("FILE CORRUPTED: %s" % file)
114 |                         continue
115 |                 else:
116 |                     try:
117 |                         Xh = sunpy.io.read_file_header(os.path.join(root,file))
118 |                         if Xh[1]['QUALITY'] == 0:
119 |                             filelist_4500.append(os.path.join(root,file))
120 |                     except:
121 |                         print("FILE CORRUPTED: %s" % file)
122 |                         continue
123 | 
124 | np.save('filelist_131.npy',filelist_131)
125 | np.save('filelist_1600.npy',filelist_1600)
126 | np.save('filelist_1700.npy',filelist_1700)
127 | np.save('filelist_171.npy',filelist_171)
128 | np.save('filelist_193.npy',filelist_193)
129 | np.save('filelist_211.npy',filelist_211)
130 | np.save('filelist_304.npy',filelist_304)
131 | np.save('filelist_335.npy',filelist_335)
132 | np.save('filelist_94.npy',filelist_94)
133 | np.save('filelist_4500.npy',filelist_4500)
134 | 


--------------------------------------------------------------------------------