├── doc ├── source │ ├── _templates │ │ └── .gitignore │ ├── process.rst │ ├── download.rst │ ├── label.rst │ ├── pipeline.rst │ ├── index.rst │ └── conf.py ├── rtd-pip-requirements ├── Makefile └── make.bat ├── pipeline ├── label │ └── __init__.py ├── stat_utils │ ├── __init__.py │ └── computetotals.py ├── utils │ ├── __init__.py │ ├── transfer_output.py │ ├── dask_examples.py │ ├── make_plots.py │ ├── compute_larmor_radius.py │ ├── check_IR_deposition.py │ ├── generate_catalog.py │ ├── generate_thickness_proxy.py │ ├── smooth_cr_heatmap.py │ ├── metadata.py │ ├── initialize.py │ ├── compute_directionality.py │ └── agu_plots.py ├── __init__.py ├── download │ ├── __init__.py │ └── download.py ├── process │ ├── __init__.py │ ├── process_IR.py │ └── separate_ima.py ├── aws_env_mac.yml ├── aws_env_linux.yml ├── prototyping_download.ipynb ├── prototyping_parameters.ipynb └── prototyping_wfpc2.ipynb ├── data ├── STIS │ └── j3m1403io_crr.fits ├── ACS │ ├── 29p1548cj_crr_WFC.fits │ └── n4e12510j_crr_HRC.fits └── WFC3 │ ├── u6a1748ri_crr_IR.fits │ └── n9i1435li_crr_UVIS.fits ├── AWS_SETUP ├── mk_env.sh ├── install_miniconda3.sh └── setup_ec2.sh ├── analyzing_cr_rejection ├── ocrreject.sh ├── analyze_cr.py ├── organize_tools.py ├── run_cr_rejection.py └── compare_results.py ├── CONFIG ├── NOTES.txt ├── pipeline_config.yaml └── aws_env.txt ├── .gitignore ├── README.md ├── LICENSE.md └── notebooks └── analyzing_cr_rejection.ipynb /doc/source/_templates/.gitignore: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /doc/rtd-pip-requirements: -------------------------------------------------------------------------------- 1 | numpy>=1.13 2 | astropy 3 | sphinx-automodapi -------------------------------------------------------------------------------- /pipeline/label/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | __all__ = ['base', 4 | 'labeler'] 5 | -------------------------------------------------------------------------------- /pipeline/stat_utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | __all__ = ['computetotals', 3 | 'statshandler'] -------------------------------------------------------------------------------- /data/STIS/j3m1403io_crr.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/hst_cosmic_rays/master/data/STIS/j3m1403io_crr.fits -------------------------------------------------------------------------------- /data/ACS/29p1548cj_crr_WFC.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/hst_cosmic_rays/master/data/ACS/29p1548cj_crr_WFC.fits -------------------------------------------------------------------------------- /data/ACS/n4e12510j_crr_HRC.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/hst_cosmic_rays/master/data/ACS/n4e12510j_crr_HRC.fits -------------------------------------------------------------------------------- /data/WFC3/u6a1748ri_crr_IR.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/hst_cosmic_rays/master/data/WFC3/u6a1748ri_crr_IR.fits -------------------------------------------------------------------------------- /data/WFC3/n9i1435li_crr_UVIS.fits: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/hst_cosmic_rays/master/data/WFC3/n9i1435li_crr_UVIS.fits -------------------------------------------------------------------------------- /pipeline/utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | __all__ = ['visualize', 4 | 'metadata', 5 | 'initialize'] -------------------------------------------------------------------------------- /doc/source/process.rst: -------------------------------------------------------------------------------- 1 | ******* 2 | process 3 | ******* 4 | 5 | .. automodapi:: process.process 6 | :no-inheritance-diagram: 7 | :skip: defaultdict -------------------------------------------------------------------------------- /AWS_SETUP/mk_env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Create an environment for the pipeline 4 | conda env create -f ~/hst_cosmic_rays/pipeline/aws_env_linux.yml 5 | -------------------------------------------------------------------------------- /doc/source/download.rst: -------------------------------------------------------------------------------- 1 | ******** 2 | download 3 | ******** 4 | 5 | .. automodapi:: download.download 6 | :no-inheritance-diagram: 7 | :skip: Time, Observations -------------------------------------------------------------------------------- /pipeline/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | 4 | __all__ = ['download', 5 | 'label', 6 | 'initialize', 7 | 'process', 8 | 'stat_utils', 9 | 'utils'] -------------------------------------------------------------------------------- /doc/source/label.rst: -------------------------------------------------------------------------------- 1 | ***** 2 | label 3 | ***** 4 | 5 | .. automodapi:: label.labeler 6 | :skip: sigma_clipped_stats, ImageNormalize, LinearStretch, ZScaleInterval, Iterable, median_absolute_deviation, LogStretch 7 | -------------------------------------------------------------------------------- /pipeline/download/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from astropy.io import fits 4 | import numpy as np 5 | import pandas as pd 6 | 7 | 8 | def main(): 9 | pass 10 | 11 | 12 | if __name__ == '__main__': 13 | main() -------------------------------------------------------------------------------- /pipeline/process/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from astropy.io import fits 4 | import numpy as np 5 | import pandas as pd 6 | 7 | 8 | def main(): 9 | pass 10 | 11 | 12 | if __name__ == '__main__': 13 | main() -------------------------------------------------------------------------------- /doc/source/pipeline.rst: -------------------------------------------------------------------------------- 1 | ******** 2 | pipeline 3 | ******** 4 | 5 | .. argparse:: 6 | :module: pipeline 7 | :func: parser 8 | :prog: python pipeline.py 9 | 10 | .. automodapi:: pipeline 11 | :no-inheritance-diagram: 12 | :skip: defaultdict 13 | -------------------------------------------------------------------------------- /AWS_SETUP/install_miniconda3.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Download the latest linux distro of miniconda 4 | curl -OL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh 5 | 6 | # Install it 7 | bash ./Miniconda3-latest-Linux-x86_64.sh 8 | 9 | 10 | -------------------------------------------------------------------------------- /analyzing_cr_rejection/ocrreject.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | exptime1=$1 4 | exptime2=$2 5 | crsigmas=$3 6 | initgues=$4 7 | python run_cr_rejection.py 20 -dir1 /Users/nmiles/hst_cosmic_rays/analyzing_cr_rejection/"$exptime1"_clean/ -dir2 /Users/nmiles/hst_cosmic_rays/analyzing_cr_rejection/"$exptime2"_clean/ -crsigmas $crsigmas -initgues $initgues -------------------------------------------------------------------------------- /CONFIG/NOTES.txt: -------------------------------------------------------------------------------- 1 | For each instrument the config file contains a date under the astroquery block. 2 | 3 | This date is to be used for generating a list of equally spaced date intervals, 4 | which in turn will be used to programmatically search for 1 month chunks of darks. 5 | 6 | There is no efficient way to do this in a generalized manner without querying 7 | for every dark taken and then using the date of the very first dark as the start 8 | date. 9 | -------------------------------------------------------------------------------- /AWS_SETUP/setup_ec2.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copy credentials to EC2 instanace 4 | ec2_ip=$1 # first arg is IP address of instance 5 | aws_dir=~/.aws/ 6 | git_credentials=~/.ssh/ 7 | aws_ssh_key=$aws_dir.aws_ssh/cosmic_ray_stsci_useast1.pem 8 | 9 | # Copy files credentials to AWS 10 | scp -ri "$aws_ssh_key" $aws_dir ec2-user@$ec2_ip:~ 11 | scp -ri "$aws_ssh_key" $git_credentials ec2-user@$ec2-ip:~ 12 | 13 | # log in to the instance 14 | ssh -i "$aws_ssh_key" ec2-user@$ec2_ip -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /data/ 2 | /pipeline/*png 3 | /pipeline/*gif 4 | /crrejtab/ 5 | /crrejtab/*/*gif 6 | /crrejtab/*/*txt 7 | /crrejtab/WFC3/IR_TEST_DATA/* 8 | /pipeline/*fits 9 | /pipeline/*tra 10 | /pipeline/utils/*png 11 | /plots/*png 12 | /plots/*gif 13 | /doc/build/ 14 | /pipeline/__pycache__/ 15 | /pipeline/*/__pycache__/ 16 | /pipeline/.ipynb_checkpoints/ 17 | /pipeline/.DS_Store 18 | /pipeline/*.ipynb 19 | /pipeline/utils/*txt 20 | .DS_Store 21 | /CONFIG/processed*txt 22 | /CONFIG/failed*txt 23 | /doc/ 24 | *.fits 25 | .idea/ 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HST Cosmic Rays 2 | [API Documentation](http://hst-cosmic-rays.s3-website-us-east-1.amazonaws.com/) 3 | 4 | This repository contains the HSTcosmicrays package that is used in the analysis 5 | of every identifiable cosmic ray in any of the calibration darks taken by the 6 | following imagers on HST: 7 | 8 | **CCD Imagers** 9 | - ACS/WFC 10 | - ACS/HRC 11 | - STIS/CCD 12 | - WFPC2 (all four detectors PC, WF1, WF2, and WF3) 13 | - WFC3/UVIS 14 | 15 | **IR Imagers (Work in progress)** 16 | - NICMOS (all three detectors NIC1, NIC2, and NIC3) 17 | - WFC3/IR 18 | 19 | The package has been designed to work locally and in the cloud on an AWS EC2 20 | instance. 21 | -------------------------------------------------------------------------------- /pipeline/utils/transfer_output.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import argparse 4 | import boto3 5 | import glob 6 | import os 7 | import yaml 8 | 9 | parser = argparse.ArgumentParser() 10 | 11 | parser.add_argument('-instr', 12 | default='acs_wfc', 13 | help='HST instrument to process (acs_wfc, ' 14 | 'wfc3_uvis, stis_ccd, acs_hrc)') 15 | 16 | def transfer(instr): 17 | """ Transfer data files from EC2 instance to S3 storage for download 18 | 19 | Parameters 20 | ---------- 21 | instr 22 | 23 | 24 | Returns 25 | ------- 26 | 27 | """ 28 | with open('./../CONFIG/pipeline_config.yaml', 'r') as fobj: 29 | cfg = yaml.load(fobj) 30 | fnames = cfg[instr]['hdf5_files'] 31 | hdf5_files = [] 32 | for f in fnames: 33 | search_pattern = f.replace('.hdf5','*') 34 | flist = glob.glob(search_pattern) 35 | hdf5_files = hdf5_files + flist 36 | 37 | session = boto3.Session(profile_name='nmiles') 38 | client = session.client('s3', region_name='us-east-1') 39 | for f in hdf5_files: 40 | client.upload_file(f, 'hstcosmicraydata','{}'.format(os.path.basename(f))) 41 | 42 | if __name__ == '__main__': 43 | args = parser.parse_args() 44 | instr = args.instr.upper() 45 | transfer(instr) -------------------------------------------------------------------------------- /pipeline/utils/dask_examples.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import matplotlib.pyplot as plt 4 | plt.style.use('ggplot') 5 | import dask.array as da 6 | import h5py 7 | 8 | 9 | def read_data(fname,subgrp): 10 | tmp =[] 11 | with h5py.File(fname, mode='r') as fobj: 12 | subgrp_ = fobj[subgrp] 13 | print('Total number of datasets: {}'.format(len(subgrp_.keys()))) 14 | for name in subgrp_.keys(): 15 | dset = subgrp_[name][:][1] 16 | tmp.append(da.from_array(dset,chunks=(20000))) 17 | x = da.concatenate(tmp, axis=0) 18 | x = x.rechunk('auto') 19 | masked_x = da.ma.masked_invalid(x) 20 | masked_x = da.ma.fix_invalid(masked_x, fill_value=-99) 21 | return masked_x 22 | 23 | def plot_hist(size_data, shape_data,range=(0,10),bins=100): 24 | size_hist, size_edges = da.histogram(size_data, bins=bins, range=range) 25 | shape_hist, shape_edges = da.histogram(shape_data, bins=80, range=(0,0.9)) 26 | fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2,figsize=(7,4)) 27 | ax1.semilogy(size_edges[:-1], size_hist.compute(), drawstyle='steps-mid') 28 | ax1.set_xlabel('FWHM/(2*sqrt(2*ln(2))) [pixels]') 29 | ax2.set_xlabel('Symmetry') 30 | ax2.semilogy(shape_edges[:-1], shape_hist.compute(), drawstyle='steps-mid') 31 | fig.savefig('size_symmetry_hist.png',format='png',dpi=300) 32 | plt.show() 33 | 34 | 35 | if __name__ == '__main__': 36 | main() -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2018, Association of Universities for Research in Astronomy. 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /pipeline/utils/make_plots.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from astropy.io import fits 4 | from astropy.visualization import LinearStretch, ZScaleInterval 5 | from astropy.visualization.mpl_normalize import ImageNormalize 6 | import matplotlib.pyplot as plt 7 | from mpl_toolkits.axes_grid1 import make_axes_locatable 8 | from scipy import ndimage 9 | 10 | 11 | def main(fname): 12 | with fits.open(fname) as hdu: 13 | data = hdu[0].data 14 | smoothed_data = ndimage.filters.gaussian_filter(data, 15 | sigma=3, 16 | mode='nearest') 17 | norm = ImageNormalize(data, 18 | stretch=LinearStretch(), 19 | interval=ZScaleInterval()) 20 | fig, (ax1, ax2) = plt.subplots(1,2, figsize=(8,5)) 21 | ax1.set_title('Cosmic Ray Incidence Map') 22 | ax2.set_title('Smoothed Cosmic Ray Incidence Map') 23 | im1 = ax1.imshow(data, norm=norm, cmap='gray', origin='lower') 24 | im2 = ax2.imshow(smoothed_data, norm=norm, cmap='gray', origin='lower') 25 | divider = make_axes_locatable(ax2) 26 | for ax in [ax1, ax2]: 27 | ax.get_xaxis().set_visible(False) 28 | ax.get_yaxis().set_visible(False) 29 | cax = fig.add_axes([0.2, 0.15, 0.6, 0.04]) 30 | cbar = fig.colorbar(im2, cax=cax, orientation='horizontal') 31 | cbar.set_label('CR Incidences', fontweight='bold') 32 | fig.savefig('cr_incidence_plot_gauss5sig_smoothed_ACS_WFC.png', 33 | format='png', 34 | dpi=350, 35 | bbox_inches='tight') 36 | plt.show() 37 | 38 | 39 | 40 | if __name__ == '__main__': 41 | main() -------------------------------------------------------------------------------- /pipeline/utils/compute_larmor_radius.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | 4 | import astropy.units as u 5 | 6 | import numpy as np 7 | import astropy.constants as const 8 | from collections import namedtuple 9 | 10 | avg_B = 35e3 * u.nT 11 | 12 | 13 | def compute_larmor_radius(v, Z=1, A=1): 14 | m = Z * const.m_p + (A - Z) * const.m_n 15 | lorentz = lambda v: 1 / np.sqrt(1 - (v.to('km/s') / const.c.to('km/s'))) 16 | print(lorentz(v)) 17 | R = lorentz(v).value * v * m / (Z*const.e.to('C') * avg_B) 18 | E = lorentz(v) * m * const.c**2 19 | return R, E 20 | 21 | 22 | def model(v, Z, A): 23 | # Assign units 24 | v = v * u.kilometer/u.second 25 | radius = compute_larmor_radius(v, Z, A) 26 | 27 | def apply_units(variable): 28 | """ 29 | 30 | Parameters 31 | ---------- 32 | variable : namedtuple 33 | 34 | Returns 35 | ------- 36 | 37 | """ 38 | return u.Quantity(variable.quantity, unit = variable.unit) 39 | 40 | 41 | 42 | 43 | 44 | def compute_relativistic_velocity(E, E_unit, Z=1, A=0): 45 | """ compute the velocity of a cosmic ray with energy E 46 | E = gamma * m * c^2 47 | gamma = 1/sqrt(1-(v/c)^2) 48 | 49 | Parameters 50 | ---------- 51 | E : Energy of particle 52 | E_unit : units of E 53 | Z : Atomic number 54 | A : Mass number 55 | Returns 56 | ------- 57 | 58 | """ 59 | # E = (gamma - 1)mc^2 60 | 61 | E = u.Quantity(E, unit=E_unit) 62 | m = Z * const.m_p.to('kg') + (A - Z) * const.m_n.to('kg') # mass of in kg 63 | rest_E = m * const.c**2 64 | v = const.c.to('m/s') * np.sqrt(1 - (rest_E.to('J') / E.to('J'))**2) 65 | return v 66 | 67 | 68 | 69 | 70 | if __name__ == '__main__': 71 | pass -------------------------------------------------------------------------------- /pipeline/process/process_IR.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from astropy.io import fits 4 | import os 5 | import numpy as np 6 | 7 | 8 | class ProcessIR(object): 9 | def __init__(self, fname): 10 | self.fname = fname 11 | self.dirname = os.path.dirname(fname) 12 | self.nsamp = None 13 | self.exts = None 14 | 15 | def make_exts(self): 16 | with fits.open(self.fname) as hdu: 17 | self.nsamp = hdu[0].header['nsamp'] 18 | print('The number of non-destructive reads is: {}'.format(self.nsamp)) 19 | sci_exts = [('sci', n) for n in range(1, self.nsamp + 1)] 20 | err_exts = [('err', n) for n in range(1, self.nsamp + 1)] 21 | dq_exts = [('dq', n) for n in range(1, self.nsamp + 1)] 22 | samp_exts = [('samp', n) for n in range(1, self.nsamp + 1)] 23 | time_exts = [('time', n) for n in range(1, self.nsamp + 1)] 24 | self.exts = zip(sci_exts, err_exts, dq_exts, samp_exts, time_exts) 25 | 26 | def edit_hdr(self, hdr): 27 | hdr['extver'] = 1 28 | return hdr 29 | 30 | def write_ext(self, f_out='', exts=None): 31 | sci, err, dq, samp, time = next(exts) 32 | with fits.open(self.fname) as hdu: 33 | sci_ext = hdu[hdu.index_of(sci)] 34 | err_ext = hdu[hdu.index_of(err)] 35 | dq_ext = hdu[hdu.index_of(dq)] 36 | samp_ext = hdu[hdu.index_of(samp)] 37 | time_ext = hdu[hdu.index_of(time)] 38 | hdu_list = fits.HDUList() 39 | hdu_list.append(fits.PrimaryHDU(header=hdu[0].header)) 40 | hdu_list.append(fits.ImageHDU(data=sci_ext.data, 41 | header=self.edit_hdr(sci_ext.header))) 42 | hdu_list.append(fits.ImageHDU(data=err_ext.data, 43 | header=self.edit_hdr(err_ext.header))) 44 | hdu_list.append(fits.ImageHDU(data=dq_ext.data, 45 | header=self.edit_hdr(dq_ext.header))) 46 | hdu_list.append(fits.ImageHDU(data=samp_ext.data, 47 | header=self.edit_hdr(samp_ext.header))) 48 | hdu_list.append(fits.ImageHDU(data=time_ext.data, 49 | header=self.edit_hdr(time_ext.header))) 50 | hdu_list.writeto(f_out, overwrite=True) 51 | 52 | def write_out(self): 53 | self.make_exts() 54 | for i in reversed(range(1, self.nsamp + 1)): 55 | self.write_ext(f_out=self.dirname + '/read_{}.fits'.format(i), 56 | exts=self.exts) 57 | 58 | 59 | if __name__ == '__main__': 60 | main() -------------------------------------------------------------------------------- /pipeline/process/separate_ima.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import argparse 4 | from astropy.io import fits 5 | import os 6 | 7 | parser = argparse.ArgumentParser() 8 | parser.add_argument('-fname',help='file to process') 9 | 10 | 11 | def edit_hdr(hdr): 12 | hdr['extver'] = 1 13 | return hdr 14 | 15 | 16 | def write_out(f_in='', f_out='', exts=None): 17 | sci, err, dq, samp, time = next(exts) 18 | with fits.open(f_in) as hdu: 19 | sci_ext = hdu[hdu.index_of(sci)] 20 | err_ext = hdu[hdu.index_of(err)] 21 | dq_ext = hdu[hdu.index_of(dq)] 22 | samp_ext = hdu[hdu.index_of(samp)] 23 | time_ext = hdu[hdu.index_of(time)] 24 | hdu_list = fits.HDUList() 25 | hdu_list.append(fits.PrimaryHDU(header=hdu[0].header)) 26 | hdu_list.append(fits.ImageHDU(data = sci_ext.data, 27 | header = edit_hdr(sci_ext.header))) 28 | hdu_list.append(fits.ImageHDU(data = err_ext.data, 29 | header = edit_hdr(err_ext.header))) 30 | hdu_list.append(fits.ImageHDU(data = dq_ext.data, 31 | header = edit_hdr(dq_ext.header))) 32 | hdu_list.append(fits.ImageHDU(data = samp_ext.data, 33 | header = edit_hdr(samp_ext.header))) 34 | hdu_list.append(fits.ImageHDU(data = time_ext.data, 35 | header = edit_hdr(time_ext.header))) 36 | hdu_list.writeto(f_out,overwrite=True) 37 | 38 | 39 | def mkdir(fname): 40 | dirname = os.path.dirname(fname) 41 | basename = os.path.basename(fname).split('_')[0] 42 | new_dir = dirname+'/'+basename 43 | try: 44 | os.mkdir(new_dir) 45 | except Exception as e: 46 | print(e) 47 | r = input('Overwrite current data? (y/n) ') 48 | if r == 'y': 49 | return new_dir 50 | return -1 51 | else: 52 | return new_dir 53 | 54 | 55 | def main(fname): 56 | dirname = mkdir(fname) 57 | sci_exts = [('sci', n) for n in range(1, 17)] 58 | err_exts = [('err', n) for n in range(1, 17)] 59 | dq_exts = [('dq', n) for n in range(1, 17)] 60 | samp_exts = [('samp', n) for n in range(1, 17)] 61 | time_exts = [('time', n) for n in range(1, 17)] 62 | exts = zip(sci_exts, err_exts, dq_exts, samp_exts, time_exts) 63 | if isinstance(dirname, str): 64 | for i in reversed(range(1, 17)): # [16, 15, ..., 1] 65 | write_out(f_in=fname, f_out='{}/read_{}.fits'.format(dirname, i), 66 | exts=exts) 67 | else: 68 | print('check directory name') 69 | 70 | if __name__ == '__main__': 71 | args = parser.parse_args() 72 | main(args.fname) -------------------------------------------------------------------------------- /pipeline/aws_env_mac.yml: -------------------------------------------------------------------------------- 1 | name: aws_env 2 | channels: 3 | - http://ssb.stsci.edu/astroconda 4 | - conda-forge 5 | - defaults 6 | dependencies: 7 | - acstools=3.0.0 8 | - asn1crypto=0.24.0 9 | - astropy=3.1.2 10 | - atomicwrites=1.3.0 11 | - attrs=19.1.0 12 | - beautifulsoup4=4.7.1 13 | - blas=1.1 14 | - bokeh=1.0.4 15 | - boto3=1.9.111 16 | - botocore=1.12.111 17 | - ca-certificates=2019.3.9 18 | - calcos=3.3.4 19 | - certifi=2019.3.9 20 | - cffi=1.12.2 21 | - cfitsio=3.440 22 | - click=7.0 23 | - cloudpickle=0.8.0 24 | - cryptography=2.5 25 | - cycler=0.10.0 26 | - cython=0.29.6 27 | - cytoolz=0.9.0.1 28 | - dask=1.1.4 29 | - dask-core=1.1.4 30 | - decorator=4.3.2 31 | - distributed=1.26.0 32 | - docutils=0.14 33 | - freetype=2.9.1 34 | - h5py=2.9.0 35 | - hdf5=1.10.4 36 | - heapdict=1.0.0 37 | - hstcal=2.2.0 38 | - idna=2.8 39 | - imageio=2.4.1 40 | - intel-openmp=2019.1 41 | - jinja2=2.10 42 | - jmespath=0.9.3 43 | - jpeg=9c 44 | - kiwisolver=1.0.1 45 | - libcxx=7.0.0 46 | - libedit=3.1.20170329 47 | - libffi=3.2.1 48 | - libgcc=4.8.5 49 | - libopenblas=0.2.20 50 | - libpng=1.6.36 51 | - libtiff=4.0.10 52 | - llvm-meta=7.0.0 53 | - locket=0.2.0 54 | - markupsafe=1.1.1 55 | - matplotlib=3.0.3 56 | - matplotlib-base=3.0.3 57 | - mkl=2019.1 58 | - mkl_fft=1.0.10 59 | - mkl_random=1.0.2 60 | - more-itertools=4.3.0 61 | - msgpack-python=0.6.1 62 | - ncurses=6.1 63 | - networkx=2.2 64 | - numpy=1.16.2 65 | - numpy-base=1.14.3 66 | - olefile=0.46 67 | - openblas=0.3.3 68 | - openssl=1.1.1b 69 | - packaging=19.0 70 | - pandas=0.24.1 71 | - partd=0.3.9 72 | - photutils=0.6 73 | - pillow=5.4.1 74 | - pip=19.0.3 75 | - pluggy=0.9.0 76 | - py=1.8.0 77 | - pycparser=2.19 78 | - pyopenssl=19.0.0 79 | - pyparsing=2.3.1 80 | - pysocks=1.6.8 81 | - pytest=4.3.0 82 | - pytest-arraydiff=0.3 83 | - pytest-astropy=0.4.0 84 | - pytest-doctestplus=0.1.3 85 | - pytest-openfiles=0.3.1 86 | - pytest-remotedata=0.3.1 87 | - python=3.6.7 88 | - python-dateutil=2.8.0 89 | - pytz=2018.9 90 | - pywavelets=1.0.2 91 | - pyyaml=3.13 92 | - readline=7.0 93 | - s3transfer=0.2.0 94 | - scikit-image=0.14.2 95 | - scikit-learn=0.20.3 96 | - scipy=1.2.1 97 | - setuptools=40.8.0 98 | - six=1.12.0 99 | - sortedcontainers=2.1.0 100 | - soupsieve=1.8 101 | - sqlite=3.26.0 102 | - stistools=1.1 103 | - stsci.tools=3.4.13 104 | - tblib=1.3.2 105 | - tk=8.6.9 106 | - toolz=0.9.0 107 | - tornado=6.0.1 108 | - urllib3=1.24.1 109 | - wfc3tools=1.3.5 110 | - wheel=0.33.1 111 | - xz=5.2.4 112 | - yaml=0.1.7 113 | - zict=0.1.4 114 | - zlib=1.2.11 115 | - zstd=1.3.3 116 | - pip: 117 | - astroquery==0.3.10.dev0 118 | - chardet==3.0.4 119 | - entrypoints==0.3 120 | - html5lib==1.0.1 121 | - keyring==18.0.0 122 | - requests==2.21.0 123 | - stsci-tools==3.4.13 124 | - webencodings==0.5.1 125 | prefix: /Users/nmiles/miniconda3/envs/aws_env 126 | 127 | -------------------------------------------------------------------------------- /pipeline/stat_utils/computetotals.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from collections import defaultdict 4 | from collections import namedtuple 5 | import dask 6 | 7 | import h5py 8 | import glob 9 | import pandas as pd 10 | 11 | 12 | 13 | def tally_stats(hdf5_file): 14 | """Tally the statistics in the given HDF5 15 | 16 | For the supplied HDF5 file, this function will compute the: 17 | 18 | * Total number of cosmic rays analyzed 19 | * Total number of images analyzed 20 | * Total exposure time of all the images analyzed 21 | 22 | Parameters 23 | ---------- 24 | hdf5_file : str 25 | Full path to the HDF5 file contains in the `results` directory 26 | 27 | Returns 28 | ------- 29 | result : namedtuple 30 | A namedtuple containing the final tally for each of the parameters 31 | listed above. 32 | 33 | instr : str 34 | Instrument name 35 | 36 | """ 37 | Stat = namedtuple('Stat', ['cr_count', 38 | 'img_count', 39 | 'total_exptime']) 40 | 41 | with h5py.File(hdf5_file,mode='r') as f: 42 | instr = list(f.keys())[0] 43 | print(instr) 44 | grp = f['/{}/sizes'.format(instr)] 45 | num_images = 0 46 | num_cr = 0 47 | total_exptime = 0 48 | for key in grp.keys(): 49 | dset = grp[key][...] 50 | attrs = grp[key].attrs 51 | # print(list(attrs.items())) 52 | num_cr += dset.shape[1] 53 | num_images += 1 54 | total_exptime += attrs['exptime'] 55 | 56 | result = Stat(cr_count=num_cr, 57 | img_count=num_images, 58 | total_exptime=total_exptime) 59 | 60 | return instr, result 61 | 62 | 63 | def compile_global_stats(results_dir='./../data/*/*cr_sizes*hdf5'): 64 | """Parse all files in the results directory and tally the statistics 65 | 66 | 67 | Parameters 68 | ---------- 69 | results_dir : str 70 | Path to the results directory for each instrument 71 | 72 | Returns 73 | ------- 74 | 75 | """ 76 | 77 | flist = glob.glob(results_dir) 78 | output = defaultdict(list) 79 | flist = [f for f in flist if 'nicmos' not in f] 80 | print(flist) 81 | flist.append('./../data/STIS/stis_cr_sizes.hdf5') 82 | results = [dask.delayed(tally_stats)(f) for f in flist] 83 | results = list(dask.compute(*results, scheduler='processes')) 84 | 85 | for instr, data in results: 86 | output[instr].append(data) 87 | 88 | for key in output.keys(): 89 | cr_count = 0 90 | img_count = 0 91 | total_exptime = 0 92 | for val in output[key]: 93 | cr_count += val.cr_count 94 | img_count += val.img_count 95 | total_exptime += val.total_exptime 96 | output[key] = [cr_count, img_count, total_exptime] 97 | 98 | df = pd.DataFrame(output, index=['cr_count', 'img_count', 'total_exptime']) 99 | print(df) 100 | print('Total CR count: {}'.format(df.loc['cr_count', :].sum())) 101 | print('Total number of images analyzed: {}'.format(df.loc['img_count', :].sum())) 102 | print('Cumulative exposure time: {}'.format(df.loc['total_exptime', :].sum())) 103 | 104 | if __name__ == '__main__': 105 | compile_global_stats() 106 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | #SPHINXOPTS = -W 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = build 9 | AUTOMODAPI = source/api 10 | 11 | # Internal variables. 12 | PAPEROPT_a4 = -D latex_paper_size=a4 13 | PAPEROPT_letter = -D latex_paper_size=letter 14 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 15 | 16 | .PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest 17 | 18 | help: 19 | @echo "Please use \`make ' where is one of" 20 | @echo " html to make standalone HTML files" 21 | @echo " dirhtml to make HTML files named index.html in directories" 22 | @echo " pickle to make pickle files" 23 | @echo " json to make JSON files" 24 | @echo " htmlhelp to make HTML files and a HTML help project" 25 | @echo " qthelp to make HTML files and a qthelp project" 26 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 27 | @echo " changes to make an overview of all changed/added/deprecated items" 28 | @echo " linkcheck to check all external links for integrity" 29 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 30 | 31 | clean: 32 | -rm -rf $(BUILDDIR)/* 33 | -rm -rf $(AUTOMODAPI) 34 | 35 | html: 36 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 37 | @echo 38 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 39 | 40 | dirhtml: 41 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 42 | @echo 43 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 44 | 45 | pickle: 46 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 47 | @echo 48 | @echo "Build finished; now you can process the pickle files." 49 | 50 | json: 51 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 52 | @echo 53 | @echo "Build finished; now you can process the JSON files." 54 | 55 | htmlhelp: 56 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 57 | @echo 58 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 59 | ".hhp project file in $(BUILDDIR)/htmlhelp." 60 | 61 | qthelp: 62 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 63 | @echo 64 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 65 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 66 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/acstools.qhcp" 67 | @echo "To view the help file:" 68 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/acstools.qhc" 69 | 70 | latex: 71 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 72 | @echo 73 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 74 | @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ 75 | "run these through (pdf)latex." 76 | 77 | changes: 78 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 79 | @echo 80 | @echo "The overview file is in $(BUILDDIR)/changes." 81 | 82 | linkcheck: 83 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 84 | @echo 85 | @echo "Link check complete; look for any errors in the above output " \ 86 | "or in $(BUILDDIR)/linkcheck/output.txt." 87 | 88 | doctest: 89 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 90 | @echo "Testing of doctests in the sources finished, look at the " \ 91 | "results in $(BUILDDIR)/doctest/output.txt." 92 | -------------------------------------------------------------------------------- /pipeline/utils/check_IR_deposition.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from astropy.io import fits 4 | import matplotlib.pyplot as plt 5 | plt.style.use('ggplot') 6 | import numpy as np 7 | from CosmicRayLabel import CosmicRayLabel 8 | from ComputeStats import ComputeStats 9 | import glob 10 | import itertools 11 | import scipy.ndimage as ndimage 12 | import sys 13 | sys.path.append('/Users/nmiles/animated_fits/lib') 14 | from animate_data.animate_data import AnimateData 15 | 16 | def get_deposition(read1, read2, diff = True): 17 | # grab read1 18 | ('Getting data from {}'.format(read1)) 19 | cr_label_previous = CosmicRayLabel(read1) 20 | cr_label_previous.get_data() 21 | print('DQ Flags present in {}: {}'.format(read1,np.unique(cr_label_previous.dq))) 22 | cr_label_previous.dq = np.bitwise_and(cr_label_previous.dq, 8192) 23 | 24 | # Grab read2 25 | # print('Getting data from {}'.format(read2)) 26 | cr_label = CosmicRayLabel(read2) 27 | cr_label.get_data() 28 | # print('DQ Flags present in {}: {}'.format(read2,np.unique(cr_label.dq))) 29 | cr_label.dq = np.bitwise_and(cr_label.dq, 8192) 30 | # Subtract previous read from current read 31 | if diff: 32 | print('Differencing DQ arrays') 33 | print('CR flags before: {}'.format(len(cr_label.dq[cr_label.dq == 8192]))) 34 | cr_label.dq = cr_label.dq - cr_label_previous.dq 35 | print('CR flags after: {}'.format(len(cr_label.dq[cr_label.dq == 8192]))) 36 | cr_label.get_label() 37 | if not diff: 38 | unique_crs = compare_label(cr_label.label, cr_label_previous.label) 39 | cr_label.label = unique_crs 40 | stats2 = ComputeStats(read2,cr_label.label) 41 | stats1 = ComputeStats(read1, cr_label.label) 42 | stats1.get_data() 43 | stats2.get_data() 44 | stats2.sci = stats2.sci - stats1.sci 45 | total_deposition = stats2.compute_total_cr_deposition() 46 | total_deposition*=fits.getval(read2,keyword='samptime',ext=('sci',1)) # scale by the samples integration time 47 | return total_deposition 48 | 49 | def main(dirname, save=True): 50 | reads = glob.glob('{}/read*.fits'.format(dirname)) 51 | num = [int(f.split('.')[0].split('_')[-1]) for f in reads] 52 | pairs = list(zip(reads, num)) 53 | pairs.sort(key=lambda val: val[1]) 54 | reads = list(zip(*pairs))[0] 55 | 56 | num_cr = [] 57 | hist_data = [] 58 | for i, read1 in enumerate(reads): 59 | if i == len(reads) - 1: 60 | continue 61 | depo = get_deposition(read1, reads[i + 1]) 62 | read_num = fits.getval(reads[i + 1], keyword='sampnum', 63 | ext=('sci', 1)) + 1 64 | hist, edges = np.histogram(np.log10(depo), bins=20, 65 | range=(1.25, 5.25)) 66 | num_cr.append(num) 67 | hist_data.append((edges[:-1], hist)) 68 | with fits.open(reads[0]) as hdu: 69 | hdr = hdu[0].header 70 | 71 | date = fits.getval(reads[0],'date-obs') 72 | args = [np.asarray(hist_data), 'hist', 'hist_{}.gif'.format(date), save] 73 | hist_obj = AnimateData(*args) 74 | hist_obj.xlim = (1, 6) 75 | hist_obj.ylim = (0, 150) 76 | hist_obj.xlabel = 'log10(electron deposition)' 77 | hist_obj.title = 'EXPTIME: {:.3f}, ' \ 78 | 'SAMP_SEQ: {}, DATE-OBS: {}'.format(hdr['exptime'], 79 | hdr['samp_seq'], 80 | hdr['date-obs']) 81 | hist_obj.hist_plot() 82 | 83 | 84 | if __name__ == '__main__': 85 | main() -------------------------------------------------------------------------------- /doc/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | set SPHINXBUILD=sphinx-build 6 | set BUILDDIR=build 7 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source 8 | if NOT "%PAPER%" == "" ( 9 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 10 | ) 11 | 12 | if "%1" == "" goto help 13 | 14 | if "%1" == "help" ( 15 | :help 16 | echo.Please use `make ^` where ^ is one of 17 | echo. html to make standalone HTML files 18 | echo. dirhtml to make HTML files named index.html in directories 19 | echo. pickle to make pickle files 20 | echo. json to make JSON files 21 | echo. htmlhelp to make HTML files and a HTML help project 22 | echo. qthelp to make HTML files and a qthelp project 23 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 24 | echo. changes to make an overview over all changed/added/deprecated items 25 | echo. linkcheck to check all external links for integrity 26 | echo. doctest to run all doctests embedded in the documentation if enabled 27 | goto end 28 | ) 29 | 30 | if "%1" == "clean" ( 31 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 32 | del /q /s %BUILDDIR%\* 33 | goto end 34 | ) 35 | 36 | if "%1" == "html" ( 37 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 38 | echo. 39 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 40 | goto end 41 | ) 42 | 43 | if "%1" == "dirhtml" ( 44 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 45 | echo. 46 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 47 | goto end 48 | ) 49 | 50 | if "%1" == "pickle" ( 51 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 52 | echo. 53 | echo.Build finished; now you can process the pickle files. 54 | goto end 55 | ) 56 | 57 | if "%1" == "json" ( 58 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 59 | echo. 60 | echo.Build finished; now you can process the JSON files. 61 | goto end 62 | ) 63 | 64 | if "%1" == "htmlhelp" ( 65 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 66 | echo. 67 | echo.Build finished; now you can run HTML Help Workshop with the ^ 68 | .hhp project file in %BUILDDIR%/htmlhelp. 69 | goto end 70 | ) 71 | 72 | if "%1" == "qthelp" ( 73 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 74 | echo. 75 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 76 | .qhcp project file in %BUILDDIR%/qthelp, like this: 77 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\acstools.qhcp 78 | echo.To view the help file: 79 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\acstools.ghc 80 | goto end 81 | ) 82 | 83 | if "%1" == "latex" ( 84 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 85 | echo. 86 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 87 | goto end 88 | ) 89 | 90 | if "%1" == "changes" ( 91 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 92 | echo. 93 | echo.The overview file is in %BUILDDIR%/changes. 94 | goto end 95 | ) 96 | 97 | if "%1" == "linkcheck" ( 98 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 99 | echo. 100 | echo.Link check complete; look for any errors in the above output ^ 101 | or in %BUILDDIR%/linkcheck/output.txt. 102 | goto end 103 | ) 104 | 105 | if "%1" == "doctest" ( 106 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 107 | echo. 108 | echo.Testing of doctests in the sources finished, look at the ^ 109 | results in %BUILDDIR%/doctest/output.txt. 110 | goto end 111 | ) 112 | 113 | :end 114 | -------------------------------------------------------------------------------- /analyzing_cr_rejection/analyze_cr.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from collections import defaultdict 3 | import glob 4 | import json 5 | import logging 6 | import os 7 | import shutil 8 | import sys 9 | sys.path.append('/Users/nmiles/hst_cosmic_rays/pipeline') 10 | import warnings 11 | warnings.simplefilter("ignore") 12 | 13 | 14 | from astropy.io import fits 15 | from astropy.time import Time 16 | import numpy as np 17 | import organize_tools as org_tools 18 | import pandas as pd 19 | from pipeline import CosmicRayPipeline 20 | from stistools import ocrreject 21 | from utils import initialize 22 | 23 | 24 | 25 | 26 | 27 | parser = argparse.ArgumentParser() 28 | 29 | parser.add_argument('-dir1', 30 | type=str, 31 | help='first directory containing files to process', 32 | default=None 33 | ) 34 | 35 | parser.add_argument('-dir2', 36 | type=str, 37 | help='second directory containing files to process', 38 | default=None 39 | ) 40 | parser.add_argument('-initialize', 41 | help=('Generate new HDF5 files to store the results in' 42 | '(will overwrite existing files)'), 43 | action='store_true', 44 | default=False 45 | ) 46 | 47 | logging.basicConfig(format='%(levelname)-4s ' 48 | '[%(module)s.%(funcName)s:%(lineno)d]' 49 | ' %(message)s', 50 | ) 51 | LOG = logging.getLogger('analyze_cr') 52 | LOG.setLevel(logging.INFO) 53 | 54 | def run_pipeline(flist=None, dirname=None, **kwargs): 55 | 56 | ocrreject_params = json.load(open(f"{dirname}_summary.json",mode='r')) 57 | fsuffix = f"{ocrreject_params['initgues']}_{ocrreject_params['crsigmas']}" 58 | LOG.info(fsuffix) 59 | 60 | # Create a CosmicRayPipeline Object to perform the labeling 61 | cr_pipe = CosmicRayPipeline(**kwargs) 62 | # Set the chunks attribute to a string to specify the output suffix 63 | # This ensures observations analyzed with the same initgues and crsigmas 64 | # are stored in the same HDF5 files. 65 | cr_pipe.chunks = fsuffix 66 | 67 | # Specify the list of files we are processing 68 | cr_pipe.flist = flist 69 | 70 | if cr_pipe.initialize: 71 | # Create an Initializer 72 | initializer_obj = initialize.Initializer(cr_pipe.instr, cr_pipe.cfg) 73 | 74 | # Set up HDF5 files for saving the data 75 | initializer_obj.initialize_HDF5(chunks=cr_pipe.chunks) 76 | 77 | # Run the pipeline across all the files 78 | cr_pipe.run_labeling_all(chunk_num=cr_pipe.chunks) 79 | 80 | def analyze_cosmicrays(dir1=None, dir2=None, initialize=None, **kwargs): 81 | 82 | # Grab the file sets we processed with stistools.occreject 83 | flist1 = glob.glob(f"{dir1}/*flt.fits") 84 | flist2 = glob.glob(f"{dir2}/*flt.fits") 85 | cr_args = { 86 | 'aws': False, 87 | 'download': False, 88 | 'process': False, 89 | 'ccd': True, 90 | 'ir': False, 91 | 'chunks': None, 92 | 'analyze': True, 93 | 'use_dq': True, 94 | 'instr': 'stis_ccd', 95 | 'initialize': initialize, 96 | 'store_downloads': False, 97 | 'test': True 98 | } 99 | results = defaultdict(list) 100 | for i, (dirname, flist) in enumerate(zip([dir1, dir2], [flist1, flist2])): 101 | LOG.info(f"Processing files in {dirname}") 102 | keyname = dirname.split('/')[-1] 103 | # if i == 0: 104 | # run_pipeline(flist=flist, dirname=dirname, **cr_args) 105 | 106 | run_pipeline(flist=flist, dirname=dirname, **cr_args) 107 | 108 | if __name__ == '__main__': 109 | args = vars(parser.parse_args()) 110 | # Uncomment to set the values of dir1 and dir2 111 | # args['dir1'] = ("/Users/nmiles/hst_cosmic_rays/" 112 | # "analyzing_cr_rejection/60.0_med_6.5,5.5,4.5/") 113 | # args['dir2'] = ("/Users/nmiles/hst_cosmic_rays/" 114 | # "analyzing_cr_rejection/60.0_min_6.5,5.5,4.5/") 115 | analyze_cosmicrays(**args) 116 | -------------------------------------------------------------------------------- /pipeline/aws_env_linux.yml: -------------------------------------------------------------------------------- 1 | name: aws_env 2 | channels: 3 | - conda-forge 4 | - http://ssb.stsci.edu/astroconda 5 | - defaults 6 | dependencies: 7 | - acstools=3.0.0=py36_0 8 | - asn1crypto=0.24.0=py36_1003 9 | - astropy=3.1.2=py36h14c3975_0 10 | - atomicwrites=1.3.0=py_0 11 | - attrs=19.1.0=py_0 12 | - beautifulsoup4=4.7.1=py36_1001 13 | - blas=1.1=openblas 14 | - bokeh=1.0.4=py36_1000 15 | - boto3=1.9.111=py_0 16 | - botocore=1.12.111=py_0 17 | - ca-certificates=2019.3.9=hecc5488_0 18 | - calcos=3.3.4=py36_0 19 | - certifi=2019.3.9=py36_0 20 | - cffi=1.12.2=py36hf0e25f4_1 21 | - cfitsio=3.440=2 22 | - click=7.0=py_0 23 | - cloudpickle=0.8.0=py_0 24 | - cryptography=2.5=py36h9d9f1b6_1 25 | - cycler=0.10.0=py_1 26 | - cython=0.29.6=py36hf484d3e_0 27 | - cytoolz=0.9.0.1=py36h14c3975_1001 28 | - dask=1.1.4=py_0 29 | - dask-core=1.1.4=py_0 30 | - decorator=4.3.2=py_0 31 | - distributed=1.26.0=py36_1 32 | - docutils=0.14=py36_1001 33 | - fontconfig=2.13.1=he4413a7_1000 34 | - freetype=2.9.1=h94bbf69_1005 35 | - h5py=2.9.0=nompi_py36hf008753_1102 36 | - hdf5=1.10.4=nompi_h11e915b_1105 37 | - heapdict=1.0.0=py36_1000 38 | - hstcal=2.2.0=0 39 | - icu=58.2=hf484d3e_1000 40 | - idna=2.8=py36_1000 41 | - imageio=2.4.1=py36_1000 42 | - intel-openmp=2019.1=144 43 | - jinja2=2.10=py_1 44 | - jmespath=0.9.3=py_1 45 | - jpeg=9c=h14c3975_1001 46 | - kiwisolver=1.0.1=py36h6bb024c_1002 47 | - libcxx=7.0.0=h6bb024c_1002 48 | - libedit=3.1.20170329=hf8c457e_1001 49 | - libffi=3.2.1=hf484d3e_1005 50 | - libgcc=4.8.5=2 51 | - libgcc-ng=7.3.0=hdf63c60_0 52 | - libgfortran-ng=7.2.0=hdf63c60_3 53 | - libiconv=1.15=h14c3975_1004 54 | - libopenblas=0.2.20=h9ac9557_7 55 | - libpng=1.6.36=h84994c4_1000 56 | - libstdcxx-ng=7.3.0=hdf63c60_0 57 | - libtiff=4.0.10=h9022e91_1002 58 | - libuuid=2.32.1=h14c3975_1000 59 | - libxml2=2.9.8=h143f9aa_1005 60 | - llvm-meta=7.0.0=0 61 | - locket=0.2.0=py_2 62 | - markupsafe=1.1.1=py36h14c3975_0 63 | - matplotlib=3.0.3=py36_0 64 | - matplotlib-base=3.0.3=py36h167e16e_0 65 | - mkl=2019.1=144 66 | - mkl_fft=1.0.10=py36_0 67 | - mkl_random=1.0.2=py36_0 68 | - more-itertools=4.3.0=py36_1000 69 | - msgpack-python=0.6.1=py36h6bb024c_0 70 | - ncurses=6.1=hf484d3e_1002 71 | - networkx=2.2=py_1 72 | - numpy=1.16.2=py36_blas_openblash1522bff_0 73 | - numpy-base=1.14.3=py36h2b20989_0 74 | - olefile=0.46=py_0 75 | - openblas=0.3.3=h9ac9557_1001 76 | - openssl=1.1.1b=h14c3975_1 77 | - packaging=19.0=py_0 78 | - pandas=0.24.1=py36hf484d3e_0 79 | - partd=0.3.9=py_0 80 | - photutils=0.6=py36h14c3975_1000 81 | - pillow=5.4.1=py36h00a061d_1000 82 | - pip=19.0.3=py36_0 83 | - pluggy=0.9.0=py_0 84 | - psutil=5.5.1=py36h14c3975_0 85 | - py=1.8.0=py_0 86 | - pycparser=2.19=py36_1 87 | - pyopenssl=19.0.0=py36_0 88 | - pyparsing=2.3.1=py_0 89 | - pyqt=4.11.4=py36_3 90 | - pysocks=1.6.8=py36_1002 91 | - pytest=4.3.0=py36_0 92 | - pytest-arraydiff=0.3=py_0 93 | - pytest-astropy=0.4.0=py_0 94 | - pytest-doctestplus=0.1.3=py_0 95 | - pytest-openfiles=0.3.1=py_0 96 | - pytest-remotedata=0.3.1=py_0 97 | - python=3.6.7=h381d211_1004 98 | - python-dateutil=2.8.0=py_0 99 | - pytz=2018.9=py_0 100 | - pywavelets=1.0.2=py36h3010b51_0 101 | - pyyaml=3.13=py36h14c3975_1001 102 | - qt=4.8.7=2 103 | - readline=7.0=hf8c457e_1001 104 | - s3transfer=0.2.0=py36_0 105 | - scikit-image=0.14.2=py36hf484d3e_1 106 | - scikit-learn=0.20.3=py36_blas_openblashebff5e3_0 107 | - scipy=1.2.1=py36_blas_openblash1522bff_0 108 | - setuptools=40.8.0=py36_0 109 | - sip=4.18=py36_1 110 | - six=1.12.0=py36_1000 111 | - sortedcontainers=2.1.0=py_0 112 | - soupsieve=1.8=py36_0 113 | - sqlite=3.26.0=h67949de_1001 114 | - stistools=1.1=py36_2 115 | - stsci.tools=3.4.13=py36_1 116 | - tblib=1.3.2=py_1 117 | - tk=8.6.9=h84994c4_1000 118 | - toolz=0.9.0=py_1 119 | - tornado=6.0.1=py36h14c3975_0 120 | - urllib3=1.24.1=py36_1000 121 | - wfc3tools=1.3.5=py36_0 122 | - wheel=0.33.1=py36_0 123 | - xz=5.2.4=h14c3975_1001 124 | - yaml=0.1.7=h14c3975_1001 125 | - zict=0.1.4=py_0 126 | - zlib=1.2.11=h14c3975_1004 127 | - zstd=1.3.3=1 128 | prefix: /home/ec2-user/miniconda3/envs/aws_env 129 | 130 | -------------------------------------------------------------------------------- /doc/source/index.rst: -------------------------------------------------------------------------------- 1 | .. _hst_cosmic_rays: 2 | 3 | =============== 4 | HSTcosmicrays 5 | =============== 6 | 7 | **Author**: Nathan Miles 8 | 9 | HSTcosmicrays is a package designed to extract morphological properties of cosmic rays identified in calibration dark frames. The package is broken down into a series of modules which are then combined via the :py:class:`~pipeline.CosmicRayPipeline` object contained in the :py:mod:`~pipeline` module. At the present moment, the pipeline will analyze all of the following CCDs imagers: 10 | 11 | * `Advanced Camera for Surveys (ACS) `_ 12 | 13 | * High Resolution Channel (HRC) [inactive] 14 | * Wide Field Channel (WFC) [active] 15 | 16 | * `Space Telescope Imaging Spectrograph (STIS) `_ 17 | 18 | * CCD channel [active] 19 | 20 | * `Wide Field Camera 3 (WFC3) `_ 21 | 22 | * UVIS channel [active] 23 | 24 | * `Wide Field and Planetary Camera 2 (WFPC2) `_ 25 | 26 | * Wide Field Detector 1, 2, and 3 (WF1, WF2, WF3) [inactive] 27 | * Planetary Camera (PC) [inactive] 28 | 29 | In the future, we will extend the pipeline to handle the following IR imagers 30 | as well: 31 | 32 | * `Near Infrared Camera and Multi-Object Spectrometer (NICMOS) `_ 33 | 34 | * Near Infrared Camera 1, 2, and 3 (NIC1, NIC2, NIC3) [inactive] 35 | 36 | 37 | * `Wide Field Camera 3 (WFC3) `_ 38 | 39 | * IR channel [active] 40 | 41 | 42 | Overview 43 | -------- 44 | 45 | 46 | The pipeline uses a configuration file to obtain variety of instrument specific configuration items, as well as some global configuration items that are subsequently used throughout the pipeline. The pipeline is designed to be lightweight with respect to storage requirements of the *downloaded* data. A summary of the steps taken by the pipeline are as follows: 47 | 48 | #. Initialization (:py:mod:`~utils.initialize`) 49 | 50 | * Determine the proper date range based off information stored in configuration file 51 | 52 | * Generate a list of one month intervals and exclude any periods of prolonged inactivity (e.g. instrument failures) 53 | 54 | * Determine the date ranges, if any, that have already been analyzed. These will automatically be skipped by the pipeline 55 | 56 | * If specified, initialize a series of HDF5 files used to store all of the extracted data. 57 | 58 | #. Download (:py:mod:`~download.download`) 59 | 60 | * Submit a query to MAST searching for all files in a one month interval that match the filetypes specified in the configuration file. 61 | 62 | * Download all found files to the directory specified in the configuration file. 63 | 64 | #. Process (:py:mod:`~process.process`) 65 | 66 | * For CCD imagers on active instruments, the dark frames will be processed through the respective instruments cosmic ray rejection routine. 67 | 68 | * For IR imagers, the FITS file containing all of the N inividual reads and there corresponding extensions will be decomposed into a N different 69 | FITS file, one for each read. 70 | 71 | #. Label (:py:mod:`~label.labeler`) 72 | 73 | * Perform a connected-component labeling analysis to identify groups of cosmic ray affected pixels as a single distinct object. 74 | 75 | 76 | #. Analyze 77 | 78 | * Analyze all of the identified cosmic rays and compute a variety of 79 | statistics (:py:class:`~stat_utils.statshandler`) 80 | * For each observation extract relevant metadata (:py:class:`~utils.metadata`) 81 | 82 | 83 | #. Clean Up 84 | 85 | * After processing all dark frames found in the one month interval, 86 | write out the computed statistics and delete the downloaded data. 87 | 88 | 89 | Contents: 90 | 91 | .. toctree:: 92 | :maxdepth: 2 93 | 94 | examples 95 | pipeline 96 | pipeline_config 97 | download 98 | label 99 | process 100 | stat_utils 101 | utils 102 | 103 | 104 | 105 | 106 | Indices and tables 107 | ================== 108 | 109 | * :ref:`genindex` 110 | * :ref:`modindex` 111 | * :ref:`search` 112 | -------------------------------------------------------------------------------- /pipeline/utils/generate_catalog.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import argparse 4 | from collections import defaultdict 5 | import glob 6 | import sys 7 | 8 | 9 | from astropy.time import Time 10 | import datahandler as dh 11 | import h5py 12 | import numpy as np 13 | import pandas as pd 14 | 15 | parser = argparse.ArgumentParser() 16 | 17 | parser.add_argument('-instr', 18 | type=str, 19 | default='ACS_WFC') 20 | 21 | def mk_catalog(instr): 22 | """Generate a catalog of CR data extracted from each image 23 | """ 24 | # Desired information 25 | metadata_keywords = [ 26 | 'date', 27 | 'expend', 28 | 'integration_time', 29 | 'altitude', 30 | 'latitude', 31 | 'longitude', 32 | ] 33 | 34 | keyword_out = [ 35 | 'date_start', 36 | 'date_end' 37 | 'mjd_start', 38 | 'mjd_end', 39 | 'altitude_start', 40 | 'altitude_end', 41 | 'latitude_start', 42 | 'latitude_end', 43 | 'longitude_start', 44 | 'longitude_end', 45 | 'integration_time', 46 | 'incident_cr_rate', 47 | 'cumulative_energy_deposited', 48 | 'energy_per_area', 49 | 'energy_per_area_per_time', 50 | 'average_size_pixels', 51 | 'obs_id' 52 | ] 53 | 54 | output_data = defaultdict(list) 55 | 56 | rates = dh.DataReader(instr=instr.upper(), statistic='incident_cr_rate') 57 | size = dh.DataReader(instr=instr.upper(), statistic='sizes') 58 | energy = dh.DataReader(instr=instr.upper(), statistic='energy_deposited') 59 | 60 | for r in [rates, size, energy]: 61 | r.find_hdf5() 62 | # if 'STIS' in instr.upper(): 63 | # for r in [rates, size, energy]: 64 | # for i,f in enumerate(r.hdf5_files): 65 | # r.hdf5_files[i] = f.replace('STIS','STIS/stis_saa_results/') 66 | 67 | print(rates.hdf5_files) 68 | print(size.hdf5_files) 69 | print(energy.hdf5_files) 70 | #sys.exit() 71 | area = rates.instr_cfg['instr_params']['detector_size'] 72 | 73 | flist_tuple = zip(rates.hdf5_files, size.hdf5_files, energy.hdf5_files) 74 | for (f1, f2, f3) in flist_tuple: 75 | fobj1 = h5py.File(f1, mode='r') 76 | grp1 = fobj1['/incident_cr_rate'] 77 | 78 | fobj2 = h5py.File(f2, mode='r') 79 | grp2 = fobj2['sizes'] 80 | 81 | fobj3 = h5py.File(f3, mode='r') 82 | grp3 = fobj3['energy_deposited'] 83 | print(len(list(grp1.keys()))) 84 | for key in grp1.keys(): 85 | 86 | rate_dset = grp1[key] 87 | missing = False 88 | try: 89 | size_dset = grp2[key] 90 | except Exception as e: 91 | print(e) 92 | missing=True 93 | 94 | try: 95 | energy_dset = grp3[key] 96 | except Exception as e: 97 | print(e) 98 | missing=True 99 | 100 | if missing: 101 | print(missing) 102 | continue 103 | 104 | output_data['obs_id'].append(key) 105 | # get the meta data for the dataset 106 | metadata = rate_dset.attrs 107 | start_date = Time(metadata['date'], format='iso') 108 | end_date = Time(metadata['expend'], format='iso') 109 | output_data['date_start'].append(start_date) 110 | output_data['mjd_start'].append(start_date.mjd) 111 | 112 | output_data['date_end'].append(end_date) 113 | output_data['mjd_end'].append(end_date.mjd) 114 | 115 | output_data['integration_time'].append(metadata['integration_time']) 116 | 117 | 118 | altitude = metadata['altitude'] 119 | try: 120 | output_data['altitude_start'].append(altitude[0]) 121 | output_data['altitude_end'].append(altitude[-1]) 122 | except IndexError: 123 | output_data['altitude_start'].append(altitude) 124 | output_data['altitude_end'].append(altitude) 125 | 126 | 127 | latitude = metadata['latitude'] 128 | try: 129 | output_data['latitude_start'].append(latitude[0]) 130 | output_data['latitude_end'].append(latitude[-1]) 131 | except IndexError: 132 | output_data['latitude_start'].append(latitude) 133 | output_data['latitude_end'].append(latitude) 134 | 135 | longitude = metadata['longitude'] 136 | try: 137 | output_data['longitude_start'].append(longitude[0]) 138 | output_data['longitude_end'].append(longitude[-1]) 139 | except IndexError: 140 | output_data['longitude_start'].append(longitude) 141 | output_data['longitude_end'].append(longitude) 142 | 143 | output_data['incident_cr_rate'].append(rate_dset[()]) 144 | 145 | output_data['cumulative_energy'].append(energy_dset.value.sum()) 146 | output_data['cumulative_energy_per_area'].append( 147 | energy_dset.value.sum()/area 148 | ) 149 | output_data['cumulative_energy_per_area_per_time'].append( 150 | energy_dset.value.sum()/area/metadata['integration_time'] 151 | ) 152 | 153 | output_data['mean_size_pixels'].append(np.nanmean(size_dset[:][1])) 154 | output_data['median_size_pixels'].append(np.nanmedian(size_dset[:][1])) 155 | 156 | print('Number of datasets: {}'.format(len(output_data['date_start']))) 157 | date_index = pd.DatetimeIndex( 158 | [val.iso for val in output_data['date_start']] 159 | ) 160 | df = pd.DataFrame(output_data, index=date_index) 161 | df.sort_index(inplace=True) 162 | df.to_csv('{}_catalog.txt'.format(instr), header=True, index=False) 163 | 164 | if __name__ == '__main__': 165 | args = parser.parse_args() 166 | mk_catalog(args.instr) 167 | -------------------------------------------------------------------------------- /analyzing_cr_rejection/organize_tools.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import argparse 4 | from collections import defaultdict 5 | import glob 6 | import inspect 7 | import json 8 | import logging 9 | import os 10 | import shutil 11 | 12 | from astropy.io import fits 13 | from astropy.time import Time 14 | import numpy as np 15 | import pandas as pd 16 | from tqdm import tqdm 17 | 18 | 19 | logging.basicConfig(format='%(levelname)-4s ' 20 | '[%(module)s.%(funcName)s:%(lineno)d]' 21 | ' %(message)s', 22 | level=logging.INFO) 23 | LOG = logging.getLogger('cr_rejection') 24 | LOG.setLevel(logging.INFO) 25 | 26 | 27 | parser = argparse.ArgumentParser() 28 | 29 | parser.add_argument('-dir1', 30 | type=str, 31 | help='first directory containing files to process') 32 | 33 | parser.add_argument('-dir2', 34 | type=str, 35 | help='second directory containing files to process') 36 | 37 | 38 | 39 | def find_files(longdir=None, shortdir=None, suffix=None): 40 | """ Read in all files in longdir and shortdir with filetype suffix 41 | 42 | Parameters 43 | ---------- 44 | longdir 45 | shortdir 46 | suffix 47 | 48 | Returns 49 | ------- 50 | 51 | """ 52 | flistlong = glob.glob(f'{longdir}/{suffix}') 53 | flistshort = glob.glob(f'{shortdir}/{suffix}') 54 | return flistlong, flistshort 55 | 56 | 57 | def generate_record( 58 | outputdir, 59 | badinpdq=None, 60 | crmask="yes", 61 | crrejtab=None, 62 | crsigmas='6,5,4', 63 | crthresh=0.75, 64 | crradius=1.0, 65 | initgues='med', 66 | scalense=0, 67 | skysub="mode" 68 | ): 69 | """ Generate an output file containing a summary of the input parameters 70 | 71 | Parameters 72 | ---------- 73 | outputdir : str 74 | 75 | crsigmas : list 76 | 77 | crthresh : float 78 | 79 | crradius : float 80 | 81 | 82 | Returns 83 | ------- 84 | 85 | """ 86 | frame = inspect.currentframe() 87 | arginfo = inspect.getargvalues(frame) 88 | args = arginfo.args 89 | values = [arginfo.locals[arg] for arg in args] 90 | fout_json = f"{outputdir}/_summary.json" 91 | LOG.info(f"Writing the input parameters to the following file:\n{fout_json}") 92 | data_dict = {} 93 | for key, val in zip(args, values): 94 | data_dict[key] = val 95 | json.dump(data_dict, open(fout_json, mode='w')) 96 | 97 | 98 | 99 | # noinspection PyTypeChecker 100 | def setup_output(flist1, flist2, dir_suffix=''): 101 | """ Setup output directories for each list of files 102 | 103 | Parameters 104 | ---------- 105 | flist1 : list 106 | 107 | flist2 : list 108 | 109 | 110 | Returns 111 | ------- 112 | 113 | """ 114 | # Generate the output path for each input list 115 | dir1_path = os.path.dirname(flist1[0]) 116 | dir1_name = dir1_path.split('/')[-1] 117 | outdir1 = dir1_path.replace( 118 | dir1_name, 119 | f"{dir1_name.split('_')[0]}_{dir_suffix}" 120 | ) 121 | 122 | try: 123 | os.mkdir(outdir1) 124 | except FileExistsError: 125 | pass 126 | 127 | dir2_path = os.path.dirname(flist2[0]) 128 | dir2_name = dir2_path.split('/')[-1] 129 | outdir2 = dir2_path.replace( 130 | dir2_name, 131 | f"{dir2_name.split('_')[0]}_{dir_suffix}" 132 | ) 133 | try: 134 | os.mkdir(outdir2) 135 | except FileExistsError: 136 | pass 137 | LOG.info( 138 | f"Set up two output directories: \n{outdir1}\n{outdir2}\n {'-'*79}" 139 | ) 140 | 141 | for f1 in tqdm(flist1, desc=f'Copying to {os.path.basename(outdir1)} '): 142 | fits.setval(f1, keyword='CRREJTAB', 143 | value='/Users/nmiles/hst_cosmic_rays/j3m1403io_crr.fits') 144 | shutil.copy(f1, outdir1) 145 | shutil.copy(f1.replace('flt.fits','spt.fits'), outdir1) 146 | 147 | for f2 in tqdm(flist2,desc=f'Copying to {os.path.basename(outdir2)} '): 148 | fits.setval(f2, keyword='CRREJTAB', 149 | value='/Users/nmiles/hst_cosmic_rays/j3m1403io_crr.fits') 150 | shutil.copy(f2, outdir2) 151 | shutil.copy(f2.replace('flt.fits','spt.fits'), outdir2) 152 | 153 | return outdir1, outdir2 154 | 155 | def sort_flist(flist): 156 | date_time = [] 157 | for f in flist: 158 | with fits.open(f) as hdu: 159 | hdr = hdu[0].header 160 | try: 161 | dateobs = hdr['DATE-OBS'] 162 | except KeyError: 163 | dateobs = hdr['TDATEOBS'] 164 | try: 165 | timeobs = hdr['TIME-OBS'] 166 | except KeyError: 167 | timeobs = hdr['TTIMEOBS'] 168 | 169 | date_time.append(Time(f"{dateobs} {timeobs}", 170 | format='iso').to_datetime()) 171 | data = list(zip(flist, date_time)) 172 | data.sort(key=lambda val: val[1]) 173 | flist, date_time = zip(*data) 174 | return flist 175 | 176 | 177 | 178 | def initialize(dir1, dir2, nimages=None, dir_suffix=None): 179 | # Get string to use for recorded the start of processing 180 | if dir_suffix is None: 181 | tday = Time.now().to_datetime() 182 | dir_suffix = tday.strftime('%b%d') 183 | 184 | flist1 = sort_flist(glob.glob(dir1+'*flt.fits')) 185 | flist2 = sort_flist(glob.glob(dir2+'*flt.fits')) 186 | 187 | if nimages is not None: 188 | flist1 = flist1[:nimages] 189 | flist2 = flist2[:nimages] 190 | 191 | outdir1, outdir2 = setup_output(flist1, flist2, dir_suffix=dir_suffix) 192 | return outdir1, outdir2 193 | 194 | def parse_inputs(): 195 | args = vars(parser.parse_args()) 196 | return args 197 | 198 | 199 | if __name__ == '__main__': 200 | # args = vars(parser.parse_args()) 201 | args = { 202 | 'dir1':'/Users/nmiles/hst_cosmic_rays/' 203 | 'analyzing_cr_rejection/1100.0_clean/', 204 | 'dir2':'/Users/nmiles/hst_cosmic_rays/' 205 | 'analyzing_cr_rejection/60.0_clean/' 206 | } 207 | main(**args) -------------------------------------------------------------------------------- /pipeline/utils/generate_thickness_proxy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import argparse 4 | from astropy.io import fits 5 | import h5py 6 | from collections import Counter 7 | from collections import namedtuple 8 | import glob 9 | import matplotlib.pyplot as plt 10 | plt.style.use('ggplot') 11 | import numpy as np 12 | import pandas as pd 13 | import time 14 | import yaml 15 | 16 | # Empirical thresholds derived by analyzing the data interactively 17 | typical_size = 0.6332 18 | sigma_size = 0.0625 19 | size_thresh = 3 20 | 21 | typical_symmetry = 0.4143 22 | sigma_symmetry = 0.0472 23 | sym_thresh = 3 24 | 25 | 26 | 27 | parser = argparse.ArgumentParser() 28 | parser.add_argument('-instr', 29 | default=None, 30 | help='HST instrument to process (acs_wfc, ' 31 | 'wfc3_uvis, stis_ccd, wfpc2/wfc)') 32 | 33 | def get_data(flist, instr, stats, cfg): 34 | start_time = time.time() 35 | c = Counter() 36 | file_counter = Counter() 37 | 38 | instr_name = instr.split('_')[0].lower() 39 | fname_size = cfg[instr]['hdf5_files'][2].replace('.hdf5','_{}.hdf5') 40 | fname_shape = cfg[instr]['hdf5_files'][3].replace('.hdf5','_{}.hdf5') 41 | i = 0 42 | for fname in flist: 43 | print(fname) 44 | fnum = fname.split('_')[-1].split('.')[0] 45 | with h5py.File(fname,mode='r') as f: 46 | grp = f['/{}'.format(instr)] 47 | subgrp = grp['cr_affected_pixels'] 48 | size_file = h5py.File(fname_size.format(fnum),'r') 49 | shape_file = h5py.File(fname_shape.format(fnum),'r') 50 | for dset in subgrp.keys(): 51 | # if dset in bad_files: 52 | # print('skipping {}'.format(dset)) 53 | # file_counter['bad'] += 1 54 | # else: 55 | size_grp = size_file['/{}'.format(instr.upper())] 56 | try: 57 | avg_size = np.nanmean(size_grp['sizes'][dset][:][1]) 58 | except KeyError: 59 | continue 60 | shape_grp = shape_file['/{}'.format(instr.upper())] 61 | try: 62 | avg_symmetry = np.nanmean(shape_grp['shapes'][dset][:][1]) 63 | except KeyError: 64 | continue 65 | print(dset, avg_size, avg_symmetry) 66 | if np.absolute(avg_symmetry - stats.shape_mean) <= sym_thresh*stats.shape_std \ 67 | and np.absolute(avg_size - stats.size_mean) <= size_thresh*stats.size_std: 68 | try: 69 | coords = subgrp[dset][:] 70 | except Exception as e: 71 | file_counter['bad']+=1 72 | else: 73 | file_counter['good']+=1 74 | for coord in coords: 75 | c['{},{}'.format(coord[0],coord[1])]+=1 76 | i +=1 77 | size_file.close() 78 | shape_file.close() 79 | 80 | end_time = time.time() 81 | duration = end_time - start_time 82 | if duration > 3600: 83 | duration /= 3600 84 | units = 'hours' 85 | elif duration < 3600: 86 | duration /= 660 87 | units = 'seconds' 88 | print('It took {} {} to read {} cr-affected pixels'.format(duration, 89 | units, 90 | i)) 91 | return c, i, duration, units, file_counter 92 | 93 | def main(instr): 94 | stats = namedtuple('stats',['size_mean','size_std', 'shape_mean','shape_std']) 95 | 96 | _CFG = { 97 | 'ACS_WFC': {'stat': stats(size_mean=0.513, size_std=0.0625, shape_mean=0.41, shape_std=0.0625), 98 | 'detector_size':[4096, 4096]}, 99 | 'ACS_HRC': {'stat':stats(size_mean=0.513, size_std = 0.0625, shape_mean=0.41, shape_std=0.0625), 100 | 'detector_size':[1024, 1024]}, 101 | 'STIS_CCD': {'stat':stats(size_mean=0.513, size_std = 0.0625, shape_mean=0.41, shape_std=0.0625), 102 | 'detector_size':[1024,1024]}, 103 | 'WFC3_UVIS': {'stat':stats(size_mean=0.513, size_std=0.03125, shape_mean=0.41, shape_std=0.03125), 104 | 'detector_size':[4102, 4096]} 105 | } 106 | 107 | 108 | print('Starting') 109 | with open('./../CONFIG/pipeline_config.yaml', 'r') as fobj: 110 | cfg = yaml.load(fobj) 111 | search_pattern = cfg[instr]['hdf5_files'][0] 112 | search_pattern = search_pattern.replace('.hdf5','_?.hdf5') 113 | flist = glob.glob(search_pattern) 114 | c, num_datapoints, duration, units, file_counter = \ 115 | get_data(flist, instr=instr, cfg=cfg, stats=_CFG[instr]['stat']) 116 | # Make a blank map for CR incidence 117 | counter_array = np.zeros(_CFG[instr]['detector_size']) 118 | hdr = fits.Header() 119 | 120 | for key, val in c.items(): 121 | coords = key.split(',') 122 | coords = (int(float(coords[0])), int(float(coords[1]))) 123 | counter_array[coords[0]][coords[1]] = val 124 | 125 | try: 126 | tmp = instr.split('_') 127 | print(file_counter) 128 | hdr.set(keyword='GOODIMGS', 129 | value =file_counter['good'], 130 | comment='Total number of {} {} images used in '\ 131 | 'this analysis'.format(tmp[0], tmp[1])) 132 | hdr.set(keyword='BADIMGS', 133 | value=file_counter['bad'], 134 | comment='Total number of {} {} images '\ 135 | 'excluded in this analysis'.format(tmp[0], tmp[1])) 136 | hdr.set(keyword='STRIKES', 137 | value=num_datapoints, 138 | comment='Number of CR strikes') 139 | hdr.set(keyword='READTIME', 140 | value=duration, 141 | comment='time to read in data [{}]'.format(units)) 142 | hdu = fits.HDUList() 143 | hdu.append(fits.ImageHDU(header=hdr, data=counter_array)) 144 | hdu.writeto('cosmic_ray_incidence_map_{}.fits'.format(instr), 145 | overwrite=True) 146 | except Exception as e: 147 | print(e) 148 | return counter_array 149 | 150 | if __name__ == '__main__': 151 | args = parser.parse_args() 152 | instr = args.instr.upper() 153 | main(instr) 154 | -------------------------------------------------------------------------------- /pipeline/utils/smooth_cr_heatmap.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import argparse 4 | from astropy.io import fits 5 | from astropy.stats import sigma_clip 6 | import numpy as np 7 | import pandas as pd 8 | import sys 9 | import os 10 | 11 | parser = argparse.ArgumentParser() 12 | 13 | parser.add_argument('-fname', 14 | help='/path/to/file/', 15 | type=str) 16 | parser.add_argument('-fout', 17 | help='filename to write to (default smoothed.fits)', 18 | default='smoothed.fits') 19 | parser.add_argument('-sigma_low', 20 | help='Number of sigma to clip on the ' 21 | 'low end (default is 5)', 22 | type=int, 23 | default=4) 24 | parser.add_argument('-sigma_high', 25 | help='Number of sigma to clip on' 26 | ' the high end (default is 5)', 27 | type=int, 28 | default=5) 29 | parser.add_argument('-box_size', 30 | help='size of box used in smoothing (default is 20x20)', 31 | type=int, 32 | default=20) 33 | parser.add_argument('-num_iter', 34 | help='Number of iterations for smoothing ' 35 | '(default is 0)', 36 | default=None, 37 | type=int) 38 | 39 | 40 | 41 | 42 | def first_pass(chip, threshold=17): 43 | tmp = np.where(chip > threshold) 44 | coords = list(zip(tmp[1], tmp[0])) 45 | return coords 46 | 47 | 48 | def smooth(chip, coords, lowthres=4, highthres=5, dx=10, dy=10): 49 | locally_high_values = [] 50 | locally_low_values = [] 51 | y_max, x_max = chip.shape 52 | for (x, y) in coords: 53 | # Handle edge cases for bottom left corner of image 54 | if y < dy and x < dx: 55 | chip_slice = chip[y:y + 2 * dy, x:x + 2 * dx] 56 | # Handle edge cases for bottom right corner of image 57 | elif x + dx > x_max and y < dy: 58 | chip_slice = chip[y:y + 2 * dy, x:x - 2 * dx] 59 | # Handle edge cases for top left corner of image 60 | elif x < dx and y + dy > y_max: 61 | chip_slice = chip[y:y - 2 * dy, x:x + 2 * dx] 62 | # Handle edge cases for top right corner 63 | elif x + dx > x_max and y + dy > y_max: 64 | chip_slice = chip[y:y - 2 * dy, x:x - 2 * dx] 65 | # Handle edge cases for top border of image 66 | elif x + dx < x_max and x > dx and y + dy > y_max: 67 | chip_slice = chip[y:y - 2 * dy, x - dx:x + dx] 68 | # Handle edge cases for bottom border of image 69 | elif x + dx < x_max and x > dx and y < dy: 70 | chip_slice = chip[y:y + 2 * dy, x - dx:x + dx] 71 | # Handle edge cases for left side of image 72 | elif x < dx and y + dy < y_max: 73 | chip_slice = chip[y - dy:y + dy, x:x + 2 * dx] 74 | # Handle edge cases for right side of image 75 | elif x + dx > x_max and y + dy < y_max: 76 | chip_slice = chip[y - dy: y + dy, x:x - 2 * dx] 77 | # Normal pixels 78 | else: 79 | chip_slice = chip[y - dy:y + dy, x - dx:x + dx] 80 | 81 | median = np.median(chip_slice) 82 | std = np.std(chip_slice.flatten()) 83 | # print(x, y, median, std) 84 | 85 | if chip[y][x] > median + highthres * std: 86 | locally_high_values.append((x, y)) 87 | chip[y][x] = median 88 | elif chip[y][x] < median - lowthres * std: 89 | locally_low_values.append((x, y)) 90 | chip[y][x] = median 91 | return locally_high_values, locally_low_values, chip 92 | 93 | 94 | def mkRegion(coords, fname, color, dq_flag=1): 95 | gr.makeRegion_numpy(coords, os.getcwd() + os.sep + fname, 96 | dq_flag, chip, color=color) 97 | 98 | 99 | def iteratively_smooth(num_iter, data, coords, lowthres=4, 100 | highthres=5, dx=10, dy=10): 101 | i = 0 102 | while i < num_iter: 103 | if i == 0: 104 | high_coords, low_coords, data_smooth = smooth(data, coords, 105 | lowthres,highthres, 106 | dx, dy) 107 | else: 108 | 109 | high_coords, low_coords, data_smooth = smooth(data_smooth, 110 | coords, 111 | lowthres, 112 | highthres, 113 | dx, dy) 114 | i += 1 115 | return high_coords, low_coords, data_smooth 116 | 117 | 118 | def main(fname, sigma_low, sigma_high, box_size, num_iter, fout): 119 | with fits.open(fname) as hdu: 120 | data = hdu[0].data 121 | 122 | # generate a list of coordinates, start with all possible values 123 | coords = np.where(data > 0) 124 | coords = list(zip(coords[1], coords[0])) 125 | if not num_iter: 126 | high_coords, low_coords, chip_smooth = \ 127 | smooth(data, coords=coords, lowthres=sigma_low, 128 | highthres=sigma_high, 129 | dx=round(box_size / 2), 130 | dy=round(box_size / 2)) 131 | else: 132 | high_coords, low_coords, chip_smooth = \ 133 | iteratively_smooth(num_iter=num_iter, 134 | data=data, 135 | coords=coords, 136 | lowthres=sigma_low, 137 | highthres=sigma_high, 138 | dx=round(box_size / 2), 139 | dy=round(box_size / 2)) 140 | 141 | print('Number of smoothed coords {}'.format(len(high_coords + low_coords))) 142 | hdu_o = fits.HDUList() 143 | hdu_o.append(fits.ImageHDU(chip_smooth)) 144 | hdu_o[0].header['sigma_l'] = sigma_low 145 | hdu_o[0].header['sigma_h'] = sigma_high 146 | hdu_o[0].header['box_size'] = box_size 147 | hdu_o.writeto(fout,overwrite=True) 148 | 149 | 150 | if __name__ == '__main__': 151 | args = parser.parse_args() 152 | fname = args.fname 153 | fout = args.fout 154 | sigma_low = args.sigma_low 155 | sigma_high = args.sigma_high 156 | box_size = args.box_size 157 | num_iter = args.num_iter 158 | main(fname, sigma_low, sigma_high, box_size, num_iter, fout) 159 | -------------------------------------------------------------------------------- /CONFIG/pipeline_config.yaml: -------------------------------------------------------------------------------- 1 | ACS_WFC: 2 | search_pattern: '/data/ACS/WFC/mastDownload/HST/*/*flt.fits' 3 | hdf5_files: 4 | cr_affected_pixels: '/results/ACS/acs_wfc_cr_affected_pixels.hdf5' 5 | incident_cr_rate: '/results/ACS/acs_wfc_cr_rate.hdf5' 6 | sizes: '/results/ACS/acs_wfc_cr_sizes.hdf5' 7 | shapes: '/results/ACS/acs_wfc_cr_shapes.hdf5' 8 | energy_deposited: '/results/ACS/acs_wfc_cr_energy_deposited.hdf5' 9 | failed: '/results/ACS/acs_wfc_failed_observations.txt' 10 | astroquery: 11 | date_range: '2002-03-01' 12 | SubGroupDescription: 13 | - 'FLT' 14 | - 'SPT' 15 | download_dir: '/data/ACS/WFC/' 16 | crrejtab: '/data/ACS/29p1548cj_crr_WFC.fits' 17 | instr_params: 18 | extnums: [1,2] 19 | readout_time: 50.0 20 | gain_keyword: 'ATODGN*' 21 | detector_size: 37.748 22 | pixel_size: 15 23 | ACS_HRC: 24 | search_pattern: '/data/ACS/HRC/mastDownload/HST/*/*flt.fits' 25 | hdf5_files: 26 | cr_affected_pixels: '/results/ACS/acs_hrc_cr_affected_pixels.hdf5' 27 | incident_cr_rate: '/results/ACS/acs_hrc_cr_rate.hdf5' 28 | sizes: '/results/ACS/acs_hrc_cr_sizes.hdf5' 29 | shapes: '/results/ACS/acs_hrc_cr_shapes.hdf5' 30 | energy_deposited: '/results/ACS/acs_hrc_cr_energy_deposited.hdf5' 31 | failed: '/results/ACS/acs_hrc_failed_observations.txt' 32 | astroquery: 33 | date_range: 34 | - '2002-03-01' 35 | - '2007-01-27' 36 | SubGroupDescription: 37 | - 'FLT' 38 | - 'SPT' 39 | download_dir: '/data/ACS/HRC/' 40 | crrejtab: '/data/ACS/n4e12510j_crr_HRC.fits' 41 | instr_params: 42 | extnums: [1] 43 | readout_time: 15.0 44 | gain_keyword: 'ATODGN*' 45 | detector_size: 4.624 46 | pixel_size: 21 47 | NICMOS_NIC1: 48 | search_pattern: '/data/NICMOS/NIC1/mastDownload/HST/*/*ima.fits' 49 | hdf5_files: 50 | cr_affected_pixels: '/results/NICMOS/nicmos_nic1_cr_affected_pixels.hdf5' 51 | incident_cr_rate: '/results/NICMOS/nicmos_nic1_cr_rate.hdf5' 52 | sizes: '/results/NICMOS/nicmos_nic1_cr_sizes.hdf5' 53 | shapes: '/results/NICMOS/nicmos_nic1_cr_shapes.hdf5' 54 | energy_deposited: '/results/NICMOS/nicmos_nic1_cr_energy_deposited.hdf5' 55 | failed: '/results/NICMOS/nicmos_nic1_failed_observations.txt' 56 | astroquery: 57 | date_range: 58 | - '1997-02-01' 59 | - '2009-10-22' 60 | SubGroupDescription: 61 | - 'IMA' 62 | - 'SPT' 63 | download_dir: '/data/NICMOS/NIC1/' 64 | NICMOS_NIC2: 65 | search_pattern: '/data/NICMOS/NIC2/mastDownload/HST/*/*ima.fits' 66 | hdf5_files: 67 | cr_affected_pixels: '/results/NICMOS/nicmos_nic2_cr_affected_pixels.hdf5' 68 | incident_cr_rate: '/results/NICMOS/nicmos_nic2_cr_rate.hdf5' 69 | sizes: '/results/NICMOS/nicmos_nic2_cr_sizes.hdf5' 70 | shapes: '/results/NICMOS/nicmos_nic2_cr_shapes.hdf5' 71 | energy_deposited: '/results/NICMOS/nicmos_nic2_cr_energy_deposited.hdf5' 72 | failed: '/results/NICMOS/nicmos_nic2_failed_observations.txt' 73 | astroquery: 74 | date_range: 75 | - '1997-02-01' 76 | - '2009-10-22' 77 | SubGroupDescription: 78 | - 'IMA' 79 | - 'SPT' 80 | download_dir: '/data/NICMOS/NIC2/' 81 | 82 | NICMOS_NIC3: 83 | search_pattern: '/data/NICMOS/NIC3/mastDownload/HST/*/*ima.fits' 84 | hdf5_files: 85 | cr_affected_pixels: '/results/NICMOS/nicmos_nic3_cr_affected_pixels.hdf5' 86 | incident_cr_rate: '/results/NICMOS/nicmos_nic3_cr_rate.hdf5' 87 | sizes: '/results/NICMOS/nicmos_nic3_cr_sizes.hdf5' 88 | shapes: '/results/NICMOS/nicmos_nic3_cr_shapes.hdf5' 89 | energy_deposited: '/results/NICMOS/nicmos_nic3_cr_energy_deposited.hdf5' 90 | failed: '/results/NICMOS/nicmos_nic3_failed_observations.txt' 91 | astroquery: 92 | date_range: 93 | - '1997-02-01' 94 | - '2009-10-22' 95 | SubGroupDescription: 96 | - 'IMA' 97 | - 'SPT' 98 | download_dir: '/data/NICMOS/NIC3/' 99 | 100 | STIS_CCD: 101 | search_pattern: '/data/STIS/CCD/mastDownload/HST/*/*flt.fits' 102 | hdf5_files: 103 | cr_affected_pixels: '/results/STIS/stis_ccd_cr_affected_pixels.hdf5' 104 | incident_cr_rate: '/results/STIS/stis_ccd_cr_rate.hdf5' 105 | sizes: '/results/STIS/stis_ccd_cr_sizes.hdf5' 106 | shapes: '/results/STIS/stis_ccd_cr_shapes.hdf5' 107 | energy_deposited: '/results/STIS/stis_ccd_cr_energy_deposited.hdf5' 108 | failed: '/results/STIS/stis_ccd_failed_observations.txt' 109 | astroquery: 110 | date_range: '1997-02-01' 111 | SubGroupDescription: 112 | - 'FLT' 113 | - 'SPT' 114 | download_dir: '/data/STIS/CCD/' 115 | crrejtab: '/data/STIS/j3m1403io_crr.fits' 116 | instr_params: 117 | extnums: [1] 118 | readout_time: 15.0 119 | gain_keyword: 'ATODGAIN' 120 | detector_size: 4.624 121 | pixel_size: 21 122 | 123 | WFC3_IR: 124 | search_pattern: '/data/WFC3/IR/mastDownload/HST/*/*ima.fits' 125 | hdf5_files: 126 | cr_affected_pixels: '/results/WFC3/wfc3_ir_cr_affected_pixels.hdf5' 127 | incident_cr_rate: '/results/WFC3/wfc3_ir_cr_rate.hdf5' 128 | sizes: ' /results/WFC3/wfc3_ir_cr_shapes.hdf5' 129 | energy_deposited: '/results/WFC3/wfc3_ir_cr_energy_deposited.hdf5' 130 | failed: '/results/wfc3_ir_failed_observations.txt' 131 | astroquery: 132 | date_range: '2009-05-01' 133 | SubGroupDescription: 134 | - 'IMA' 135 | - 'SPT' 136 | download_dir: '/data/WFC3/IR/' 137 | instr_params: 138 | extnums: [1] 139 | readout_time: 1.0 140 | detector_size: 3.331 141 | 142 | WFC3_UVIS: 143 | search_pattern: '/data/WFC3/UVIS/mastDownload/HST/*/*flt.fits' 144 | hdf5_files: 145 | cr_affected_pixels: '/results/WFC3/wfc3_uvis_cr_affected_pixels.hdf5' 146 | incident_cr_rate: '/results/WFC3/wfc3_uvis_cr_rate.hdf5' 147 | sizes: '/results/WFC3/wfc3_uvis_cr_sizes.hdf5' 148 | shapes: '/results/WFC3/wfc3_uvis_cr_shapes.hdf5' 149 | energy_deposited: '/results/WFC3/wfc3_uvis_cr_energy_deposited.hdf5' 150 | failed: '/results/wfc3_uvis_failed_observations.txt' 151 | astroquery: 152 | date_range: '2009-05-01' 153 | SubGroupDescription: 154 | - 'FLT' 155 | - 'SPT' 156 | download_dir: '/data/WFC3/UVIS/' 157 | crrejtab: '/data/WFC3/n9i1435li_crr_UVIS.fits' 158 | instr_params: 159 | extnums: [1,2] 160 | readout_time: 50.0 161 | gain_keyword: 'ATODGN*' 162 | detector_size: 37.804 163 | pixel_size: 15 164 | 165 | 166 | WFPC2: 167 | search_pattern: '/data/WFPC2/mastDownload/HST/*/*c0m.fits' 168 | hdf5_files: 169 | cr_affected_pixels: '/results/WFPC2/wfpc2_cr_affected_pixels.hdf5' 170 | incident_cr_rate: '/results/WFPC2/wfpc2_cr_rate.hdf5' 171 | sizes: '/results/WFPC2/wfpc2_cr_sizes.hdf5' 172 | shapes: '/results/WFPC2/wfpc2_cr_shapes.hdf5' 173 | energy_deposited: '/results/WFPC2/wfpc2_cr_energy_deposited.hdf5' 174 | failed: '/results/wfpc2_failed_observations.txt' 175 | astroquery: 176 | date_range: 177 | - '1994-01-01' 178 | - '2009-06-01' 179 | SubGroupDescription: 180 | - 'C0M' 181 | - 'SHM' 182 | download_dir: '/data/WFPC2/' 183 | instr_params: 184 | extnums: [1,2,3,4] 185 | readout_time: [30,60] 186 | gain_keyword: 'ATODGAIN' 187 | detector_size: 5.76 188 | pixel_size: 15 189 | 190 | grp_names: 191 | cr_affected_pixels: cr_affected_pixels 192 | incident_cr_rate: incident_cr_rate 193 | sizes: sizes 194 | shapes: shapes 195 | energy_deposited: energy_deposited 196 | 197 | email: 198 | username: "" 199 | domain: "" 200 | -------------------------------------------------------------------------------- /pipeline/prototyping_download.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "### The goal of this notebook is to demonstrate the following:\n", 8 | "- programmatic query of calibration files for any of the active instruments\n", 9 | "- using the results, chunk all of the darks into sets of darks covering a ~30 day period\n", 10 | " - these chunks of darks will then be combined to reject cosmic rays\n", 11 | " - For the time being I will **ignore** anneal boundaries. All cosmic rays affecting only a few pixels are rejected and so any hot pixels that are cured will be ignored across the anneal boundary will be removed\n", 12 | "#### Author: Nate Miles" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": null, 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "from astroquery.mast import Observations\n", 22 | "from collections import defaultdict\n", 23 | "from astropy.time import Time\n", 24 | "import pandas as pd" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "metadata": {}, 30 | "source": [ 31 | "- Originally, I was querying for every single dark every taken, then I was filtering that table by extension and date of observations.\n", 32 | "\n", 33 | "- Instead, I should be filtering by date beforehand to get a much smaller table and then filter that table by extension to get only the RAW and SPT dataproducts\n", 34 | "\n", 35 | "- In order to acheive this in a generalized manner without having to hard code dates, I will query for all darks, sort by date and then extract the first and last values to get the start and stop times" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": null, 41 | "metadata": {}, 42 | "outputs": [], 43 | "source": [ 44 | "#Observations.enable_s3_hst_dataset()" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "obsTable = Observations.query_criteria(obs_collection='HST',\n", 54 | " instrument_name = 'WFPC2',\n", 55 | " obstype='cal',\n", 56 | " target_name = 'DARK')\n", 57 | "df = obsTable.to_pandas()" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": null, 63 | "metadata": {}, 64 | "outputs": [], 65 | "source": [ 66 | "df.info()" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": null, 72 | "metadata": {}, 73 | "outputs": [], 74 | "source": [ 75 | "df.head()\n" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": null, 81 | "metadata": {}, 82 | "outputs": [], 83 | "source": [ 84 | "start = Time(min(obsTable['t_min']), format='mjd')\n", 85 | "stop = Time(max(obsTable['t_min']), format='mjd')\n", 86 | "print(start.iso, stop.iso)\n", 87 | "# very roundabout way of generating a list of MJD dates separated by a month\n", 88 | "dates = pd.date_range(start=start.iso, end=stop.iso, freq='1MS')\n", 89 | "\n", 90 | "dates = [Time(date.date().isoformat(),format='iso') for date in dates]" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "metadata": {}, 97 | "outputs": [], 98 | "source": [ 99 | "dates" 100 | ] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "execution_count": null, 105 | "metadata": {}, 106 | "outputs": [], 107 | "source": [ 108 | "date_ranges = zip(dates[::2], dates[1::2])" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": null, 114 | "metadata": {}, 115 | "outputs": [], 116 | "source": [ 117 | "dates = next(date_ranges)" 118 | ] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": null, 123 | "metadata": {}, 124 | "outputs": [], 125 | "source": [ 126 | "obsTable = Observations.query_criteria(project='HST',\n", 127 | " instrument_name='WFPC2',\n", 128 | " obstype ='cal',\n", 129 | " target_name='DARK',\n", 130 | " t_min=[dates[0].mjd, dates[1].mjd],\n", 131 | " t_exptime=[800, 2000])" 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": null, 137 | "metadata": {}, 138 | "outputs": [], 139 | "source": [ 140 | "products = Observations.get_product_list(obsTable)" 141 | ] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": null, 146 | "metadata": {}, 147 | "outputs": [], 148 | "source": [ 149 | "products" 150 | ] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": null, 155 | "metadata": {}, 156 | "outputs": [], 157 | "source": [ 158 | "products.columns" 159 | ] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": null, 164 | "metadata": {}, 165 | "outputs": [], 166 | "source": [ 167 | "filtered_products = Observations.filter_products(products,\n", 168 | " mrp_only=False,\n", 169 | " productSubGroupDescription=['C0M','C1M','SHM'])" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": null, 175 | "metadata": {}, 176 | "outputs": [], 177 | "source": [ 178 | "filtered_products\n" 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": null, 184 | "metadata": { 185 | "scrolled": true 186 | }, 187 | "outputs": [], 188 | "source": [ 189 | "filtered_products.columns" 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": null, 195 | "metadata": {}, 196 | "outputs": [], 197 | "source": [ 198 | "download_list = filtered_products['obsID'].tolist()" 199 | ] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "execution_count": null, 204 | "metadata": {}, 205 | "outputs": [], 206 | "source": [ 207 | "download_list" 208 | ] 209 | }, 210 | { 211 | "cell_type": "code", 212 | "execution_count": null, 213 | "metadata": {}, 214 | "outputs": [], 215 | "source": [ 216 | "manifest = Observations.download_products(download_list[:10],\n", 217 | " download_dir='./../crrejtab/WFPC2/',\n", 218 | " mrp_only=False,\n", 219 | " productSubGroupDescription=['C0M','SHM'])" 220 | ] 221 | }, 222 | { 223 | "cell_type": "code", 224 | "execution_count": null, 225 | "metadata": {}, 226 | "outputs": [], 227 | "source": [] 228 | } 229 | ], 230 | "metadata": { 231 | "kernelspec": { 232 | "display_name": "Python 3", 233 | "language": "python", 234 | "name": "python3" 235 | }, 236 | "language_info": { 237 | "codemirror_mode": { 238 | "name": "ipython", 239 | "version": 3 240 | }, 241 | "file_extension": ".py", 242 | "mimetype": "text/x-python", 243 | "name": "python", 244 | "nbconvert_exporter": "python", 245 | "pygments_lexer": "ipython3", 246 | "version": "3.6.6" 247 | } 248 | }, 249 | "nbformat": 4, 250 | "nbformat_minor": 2 251 | } 252 | -------------------------------------------------------------------------------- /analyzing_cr_rejection/run_cr_rejection.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import argparse 4 | from collections import defaultdict 5 | import glob 6 | import logging 7 | import os 8 | import shutil 9 | 10 | from astropy.io import fits 11 | from astropy.time import Time 12 | import numpy as np 13 | import organize_tools as org_tools 14 | import pandas as pd 15 | from stistools import ocrreject 16 | from tqdm import tqdm 17 | 18 | logging.basicConfig(format='%(levelname)-4s ' 19 | '[%(module)s.%(funcName)s:%(lineno)d]' 20 | ' %(message)s', 21 | ) 22 | LOG = logging.getLogger('cr_rejection') 23 | LOG.setLevel(logging.INFO) 24 | 25 | 26 | parser = argparse.ArgumentParser() 27 | 28 | parser.add_argument('N', 29 | help='Number of images to process', 30 | type=int, 31 | default=20) 32 | 33 | parser.add_argument('-dir1', 34 | type=str, 35 | help='first directory containing files to process', 36 | default=None) 37 | 38 | parser.add_argument('-dir2', 39 | type=str, 40 | help='second directory containing files to process', 41 | default=None) 42 | 43 | parser.add_argument('-crrejtab', 44 | type=str, 45 | help='path to crrejtab reference file', 46 | default=None 47 | ) 48 | 49 | parser.add_argument('-crsigmas', 50 | type=str, 51 | help=("sigma thresholds as a comma separated " 52 | "string. Default to CRREJTAB"), 53 | default=None) 54 | 55 | parser.add_argument('-crradius', 56 | type=float, 57 | help='propagation radius for marking pixels adjacent to \ 58 | ones identified as outliers by ocrreject. \ 59 | Default to CRREJTAB', 60 | default=None) 61 | 62 | parser.add_argument('-crthresh', 63 | type=float, 64 | help='sigma threshold to use for pixels within crradius \ 65 | of those identified as outliers by ocrreject. \ 66 | Default to CRREJTAB', 67 | default=None) 68 | 69 | parser.add_argument('-crmask', 70 | type=str, 71 | help="If 'yes', cosmic rays are marked in the DQ arrays of \ 72 | the input files. Default to CRREJTAB", 73 | default=None) 74 | 75 | parser.add_argument('-initgues', 76 | type=str, 77 | help="Method for computing the initial guess; \ 78 | either 'med' or 'min'. Default to CRREJTAB", 79 | default=None) 80 | 81 | parser.add_argument('-skysub', 82 | type=str, 83 | help="Method for estimating the sky background; \ 84 | either 'none' or 'mode. Default to CRREJTAB", 85 | default=None) 86 | 87 | 88 | 89 | def run_setup(dir1, dir2, date_str=None): 90 | 91 | flist1 = glob.glob(dir1 + '*flt.fits') 92 | flist2 = glob.glob(dir2 + '*flt.fits') 93 | outdir1, outdir2 = org_tools.setup_output(flist1, flist2, 94 | date_str=date_str) 95 | 96 | return outdir1, outdir2 97 | 98 | def run_rejection( 99 | flist, 100 | dirname, 101 | crrejtab=None, 102 | crradius=None, 103 | crsigmas=None, 104 | crthresh=None, 105 | crmask=None, 106 | initgues=None, 107 | scalense=None, 108 | skysub=None, 109 | verbose=None, 110 | ): 111 | """ 112 | 113 | Parameters 114 | ---------- 115 | flist : TYPE 116 | Description 117 | dirname : TYPE 118 | Description 119 | crrejtab : None, optional 120 | Description 121 | verbose : None, optional 122 | Description 123 | crsigmas : None, optional 124 | Description 125 | crthresh : None, optional 126 | Description 127 | crmask : None, optional 128 | Description 129 | initgues : None, optional 130 | Description 131 | skysub : None, optional 132 | Description 133 | """ 134 | os.chdir(dirname) 135 | flist = [os.path.basename(f) for f in flist] 136 | fout = 'combined_crj.fits' 137 | 138 | # Check to see if a file exists for this specified configuation 139 | try: 140 | os.remove(fout) 141 | except FileNotFoundError: 142 | pass 143 | try: 144 | os.remove(fout.replace('crj.fits','spt.fits')) 145 | except FileNotFoundError: 146 | pass 147 | 148 | ocrreject.ocrreject( 149 | ' '.join(flist), 150 | output=fout, 151 | crrejtab=crrejtab, 152 | crsigmas=crsigmas, 153 | crradius=crradius, 154 | crthresh=crthresh, 155 | crmask=crmask, 156 | initgues=initgues, 157 | skysub=skysub, 158 | verbose=True 159 | ) 160 | 161 | def main( 162 | dir1=None, 163 | dir2=None, 164 | badinpdq=None, 165 | crmask=None, 166 | crrejtab=None, 167 | crradius=None, 168 | crsigmas=None, 169 | crthresh=None, 170 | initgues=None, 171 | N=None, 172 | scalense=None, 173 | skysub=None, 174 | 175 | ): 176 | 177 | """ 178 | Parameters 179 | ---------- 180 | dir1 : None, optional 181 | Description 182 | dir2 : None, optional 183 | Description 184 | badinpdq : None, optional 185 | Description 186 | crmask : str, optional 187 | Description 188 | crrejtab : None, optional 189 | Description 190 | crradius : float, optional 191 | Description 192 | crsigmas : str, optional 193 | Description 194 | crthresh : float, optional 195 | Description 196 | initgues : str, optional 197 | Description 198 | N : None, optional 199 | Description 200 | scalense : int, optional 201 | Description 202 | skysub : str, optional 203 | Description 204 | """ 205 | 206 | # Create output testing directories and create a record of the input 207 | # parameters passed to the CR Rejection routine 208 | LOG.info(f"{dir1}, {dir2}") 209 | outdir1, outdir2 = org_tools.initialize( 210 | dir1, dir2, nimages=N, dir_suffix=f"{initgues}_{crsigmas}" 211 | ) 212 | 213 | # Get the lists of files to process 214 | flist1 = glob.glob(f"{outdir1}/*flt.fits") 215 | flist2 = glob.glob(f"{outdir2}/*flt.fits") 216 | 217 | for flist, outdir in zip([flist1, flist2],[outdir1, outdir2]): 218 | # Generate a JSON record for the input parameters 219 | org_tools.generate_record(outputdir=outdir, 220 | badinpdq=badinpdq, 221 | crmask=crmask, 222 | crrejtab=crrejtab, 223 | crsigmas=crsigmas, 224 | crthresh=crthresh, 225 | crradius=crradius, 226 | initgues=initgues, 227 | scalense=scalense, 228 | skysub=skysub) 229 | # Run occreject 230 | run_rejection( 231 | flist=flist, 232 | dirname=outdir, 233 | crmask=crmask, 234 | crrejtab=crrejtab, 235 | crsigmas=crsigmas, 236 | crthresh=crthresh, 237 | crradius=crradius, 238 | initgues=initgues, 239 | scalense=scalense, 240 | skysub=skysub 241 | ) 242 | 243 | if __name__ == '__main__': 244 | args = vars(parser.parse_args()) 245 | # Uncomment to set the values of dir1 and dir2 246 | # args['dir1'] = ("/Users/nmiles/hst_cosmic_rays/" 247 | # "analyzing_cr_rejection/1100.0_clean/") 248 | # args['dir2'] = ("/Users/nmiles/hst_cosmic_rays/" 249 | # "analyzing_cr_rejection/60.0_clean/") 250 | main(**args) -------------------------------------------------------------------------------- /doc/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # acstools documentation build configuration file, created by 4 | # sphinx-quickstart on Thu Sep 30 11:14:51 2010. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | 15 | import sys 16 | sys.path.append('/Users/nmiles/hst_cosmic_rays/pipeline') 17 | 18 | import pipeline 19 | 20 | # If extensions (or modules to document with autodoc) are in another directory, 21 | # add these directories to sys.path here. If the directory is relative to the 22 | # documentation root, use os.path.abspath to make it absolute, like shown here. 23 | #sys.path.append(os.path.abspath('.')) 24 | 25 | # -- General configuration ----------------------------------------------------- 26 | 27 | # Add any Sphinx extension module names here, as strings. They can be extensions 28 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 29 | extensions = [ 30 | 'sphinx.ext.autodoc', 31 | 'sphinxarg.ext', 32 | 'sphinx.ext.mathjax', 33 | 'sphinx.ext.napoleon', 34 | 'sphinx.ext.intersphinx', 35 | 'sphinx.ext.autosummary', 36 | 'sphinx_automodapi.automodapi'] 37 | 38 | numpydoc_show_class_members = False 39 | numpydoc_class_members_toctree = False 40 | 41 | # Add any paths that contain templates here, relative to this directory. 42 | templates_path = ['_templates'] 43 | 44 | # The suffix of source filenames. 45 | source_suffix = '.rst' 46 | 47 | # The encoding of source files. 48 | #source_encoding = 'utf-8' 49 | 50 | # The master toctree document. 51 | master_doc = 'index' 52 | 53 | # General information about the project. 54 | project = u'hst_cosmic_rays' 55 | copyright = u'2019, STScI' 56 | 57 | # The version info for the project you're documenting, acts as replacement for 58 | # |version| and |release|, also used in various other places throughout the 59 | # built documents. 60 | # 61 | 62 | # The language for content autogenerated by Sphinx. Refer to documentation 63 | # for a list of supported languages. 64 | #language = None 65 | 66 | # There are two options for replacing |today|: either, you set today to some 67 | # non-false value, then it is used: 68 | #today = '' 69 | # Else, today_fmt is used as the format for a strftime call. 70 | #today_fmt = '%B %d, %Y' 71 | 72 | # List of documents that shouldn't be included in the build. 73 | #unused_docs = [] 74 | 75 | # List of directories, relative to source directory, that shouldn't be searched 76 | # for source files. 77 | exclude_trees = [] 78 | 79 | # The reST default role (used for this markup: `text`) to use for all documents. 80 | default_role = 'py:obj' 81 | 82 | # If true, '()' will be appended to :func: etc. cross-reference text. 83 | #add_function_parentheses = False 84 | 85 | # If true, the current module name will be prepended to all description 86 | # unit titles (such as .. function::). 87 | #add_module_names = True 88 | 89 | # If true, sectionauthor and moduleauthor directives will be shown in the 90 | # output. They are ignored by default. 91 | #show_authors = False 92 | 93 | # The name of the Pygments (syntax highlighting) style to use. 94 | pygments_style = 'sphinx' 95 | 96 | # A list of ignored prefixes for module index sorting. 97 | # modindex_common_prefix = ['acstools.'] 98 | 99 | 100 | # -- Options for HTML output --------------------------------------------------- 101 | 102 | # The theme to use for HTML and HTML Help pages. Major themes that come with 103 | # Sphinx are currently 'default' and 'sphinxdoc'. 104 | html_theme = 'sphinx_rtd_theme' 105 | 106 | # Theme options are theme-specific and customize the look and feel of a theme 107 | # further. For a list of options available for each theme, see the 108 | # documentation. 109 | #html_theme_options = {} 110 | 111 | # Add any paths that contain custom themes here, relative to this directory. 112 | #html_theme_path = [] 113 | html_static_path = ['_static'] 114 | 115 | # The name for this set of Sphinx documents. If None, it defaults to 116 | # " v documentation". 117 | #html_title = None 118 | 119 | # A shorter title for the navigation bar. Default is the same as html_title. 120 | #html_short_title = None 121 | 122 | # The name of an image file (relative to this directory) to place at the top 123 | # of the sidebar. 124 | # html_logo = '_static/logo_cutout with_text with_crs_highlighted.png' 125 | 126 | # The name of an image file (within the static path) to use as favicon of the 127 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 128 | # pixels large. 129 | #html_favicon = None 130 | 131 | # Add any paths that contain custom static files (such as style sheets) here, 132 | # relative to this directory. They are copied after the builtin static files, 133 | # so a file named "default.css" will overwrite the builtin "default.css". 134 | html_static_path = ['_static'] 135 | 136 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 137 | # using the given strftime format. 138 | #html_last_updated_fmt = '%b %d, %Y' 139 | 140 | # If true, SmartyPants will be used to convert quotes and dashes to 141 | # typographically correct entities. 142 | #html_use_smartypants = True 143 | 144 | # Custom sidebar templates, maps document names to template names. 145 | #html_sidebars = {} 146 | 147 | # Additional templates that should be rendered to pages, maps page names to 148 | # template names. 149 | #html_additional_pages = {} 150 | 151 | # If false, no module index is generated. 152 | #html_use_modindex = True 153 | 154 | # If false, no index is generated. 155 | #html_use_index = True 156 | 157 | # If true, the index is split into individual pages for each letter. 158 | #html_split_index = False 159 | 160 | # If true, links to the reST sources are added to the pages. 161 | #html_show_sourcelink = True 162 | 163 | # If true, an OpenSearch description file will be output, and all pages will 164 | # contain a tag referring to it. The value of this option must be the 165 | # base URL from which the finished HTML is served. 166 | #html_use_opensearch = '' 167 | 168 | # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). 169 | #html_file_suffix = '' 170 | 171 | # Output file base name for HTML help builder. 172 | htmlhelp_basename = 'hst_cosmic_rays' 173 | 174 | 175 | # -- Options for LaTeX output -------------------------------------------------- 176 | 177 | # The paper size ('letter' or 'a4'). 178 | #latex_paper_size = 'letter' 179 | 180 | # The font size ('10pt', '11pt' or '12pt'). 181 | #latex_font_size = '10pt' 182 | 183 | # Grouping the document tree into LaTeX files. List of tuples 184 | # (source start file, target name, title, author, documentclass [howto/manual]). 185 | latex_documents = [ 186 | ('index', 'hst_cosmic_rays.tex', u'hst_cosmic_rays Documentation', 187 | u'STScI', 'manual'), 188 | ] 189 | 190 | # The name of an image file (relative to this directory) to place at the top of 191 | # the title page. 192 | #latex_logo = None 193 | 194 | # For "manual" documents, if this is true, then toplevel headings are parts, 195 | # not chapters. 196 | #latex_use_parts = False 197 | 198 | # Additional stuff for the LaTeX preamble. 199 | #latex_preamble = '' 200 | 201 | # Documents to append as an appendix to all manuals. 202 | #latex_appendices = [] 203 | 204 | # If false, no module index is generated. 205 | #latex_use_modindex = True 206 | 207 | # Example configuration for intersphinx: refer to the Python standard library. 208 | intersphinx_mapping = { 209 | 'python': ('http://docs.python.org/', None), 210 | 'numpy': ('http://docs.scipy.org/doc/numpy/', None), 211 | 'scipy': ('http://docs.scipy.org/doc/scipy/reference/', None), 212 | 'skimage': ('http://scikit-image.org/docs/0.11.x/', None), 213 | 'matplotlib': ('http://matplotlib.org/', None), 214 | 'astroquery':('https://astroquery.readthedocs.io/en/latest/', None), 215 | 'astropy': ('http://docs.astropy.org/en/stable/', None), 216 | 'pandas': ('http://pandas.pydata.org/pandas-docs/stable/', None), 217 | 'dask': ('https://docs.dask.org/en/latest/', None) 218 | } 219 | -------------------------------------------------------------------------------- /CONFIG/aws_env.txt: -------------------------------------------------------------------------------- 1 | # This file may be used to create an environment using: 2 | # $ conda create --name --file 3 | # platform: osx-64 4 | @EXPLICIT 5 | https://repo.continuum.io/pkgs/main/osx-64/blas-1.0-mkl.tar.bz2 6 | https://repo.continuum.io/pkgs/main/osx-64/ca-certificates-2018.03.07-0.tar.bz2 7 | https://repo.continuum.io/pkgs/main/osx-64/intel-openmp-2019.0-118.tar.bz2 8 | https://repo.continuum.io/pkgs/main/osx-64/jpeg-9b-he5867d9_2.tar.bz2 9 | https://repo.continuum.io/pkgs/main/osx-64/libcxxabi-4.0.1-hcfea43d_1.tar.bz2 10 | https://repo.continuum.io/pkgs/main/osx-64/libgfortran-3.0.1-h93005f0_2.tar.bz2 11 | https://repo.continuum.io/pkgs/main/osx-64/xz-5.2.4-h1de35cc_4.tar.bz2 12 | https://repo.continuum.io/pkgs/main/osx-64/yaml-0.1.7-hc338f04_2.tar.bz2 13 | https://repo.continuum.io/pkgs/main/osx-64/zlib-1.2.11-hf3cbc9b_2.tar.bz2 14 | https://repo.continuum.io/pkgs/main/osx-64/libcxx-4.0.1-hcfea43d_1.tar.bz2 15 | https://repo.continuum.io/pkgs/main/osx-64/libgcc-4.8.5-hdbeacc1_10.tar.bz2 16 | https://repo.anaconda.com/pkgs/main/osx-64/libpng-1.6.35-ha441bb4_0.tar.bz2 17 | https://repo.continuum.io/pkgs/main/osx-64/mkl-2019.0-118.tar.bz2 18 | https://repo.continuum.io/pkgs/main/osx-64/openssl-1.1.1-h1de35cc_0.tar.bz2 19 | https://repo.continuum.io/pkgs/main/osx-64/tk-8.6.8-ha441bb4_0.tar.bz2 20 | http://ssb.stsci.edu/astroconda/osx-64/cfitsio-3.440-1.tar.bz2 21 | https://repo.continuum.io/pkgs/main/osx-64/freetype-2.9.1-hb4e5f40_0.tar.bz2 22 | https://repo.continuum.io/pkgs/main/osx-64/hdf5-1.10.2-hfa1e0ec_1.tar.bz2 23 | https://repo.continuum.io/pkgs/main/osx-64/libffi-3.2.1-h475c297_4.tar.bz2 24 | https://repo.continuum.io/pkgs/main/osx-64/libtiff-4.0.9-hcb84e12_2.tar.bz2 25 | https://repo.continuum.io/pkgs/main/osx-64/ncurses-6.1-h0a44026_0.tar.bz2 26 | http://ssb.stsci.edu/astroconda/osx-64/hstcal-2.2.0-0.tar.bz2 27 | https://repo.continuum.io/pkgs/main/osx-64/libedit-3.1.20170329-hb402a30_2.tar.bz2 28 | https://repo.continuum.io/pkgs/main/osx-64/readline-7.0-h1de35cc_5.tar.bz2 29 | https://repo.continuum.io/pkgs/main/osx-64/sqlite-3.25.2-ha441bb4_0.tar.bz2 30 | https://repo.continuum.io/pkgs/main/osx-64/python-3.6.7-haf84260_0.tar.bz2 31 | https://repo.continuum.io/pkgs/main/osx-64/asn1crypto-0.24.0-py36_0.tar.bz2 32 | https://repo.continuum.io/pkgs/main/osx-64/atomicwrites-1.2.1-py36_0.tar.bz2 33 | https://repo.continuum.io/pkgs/main/osx-64/attrs-18.2.0-py36h28b3542_0.tar.bz2 34 | https://repo.continuum.io/pkgs/main/osx-64/certifi-2018.10.15-py36_0.tar.bz2 35 | https://repo.anaconda.com/pkgs/main/osx-64/click-7.0-py36_0.tar.bz2 36 | https://repo.anaconda.com/pkgs/main/osx-64/cloudpickle-0.6.1-py36_0.tar.bz2 37 | https://repo.anaconda.com/pkgs/main/osx-64/dask-core-0.20.0-py36_0.tar.bz2 38 | https://repo.continuum.io/pkgs/main/osx-64/decorator-4.3.0-py36_0.tar.bz2 39 | https://repo.continuum.io/pkgs/main/osx-64/docutils-0.14-py36hbfde631_0.tar.bz2 40 | https://repo.continuum.io/pkgs/main/osx-64/heapdict-1.0.0-py36_2.tar.bz2 41 | https://repo.continuum.io/pkgs/main/osx-64/idna-2.7-py36_0.tar.bz2 42 | https://repo.continuum.io/pkgs/main/osx-64/jmespath-0.9.3-py36_0.tar.bz2 43 | https://repo.continuum.io/pkgs/main/osx-64/kiwisolver-1.0.1-py36h0a44026_0.tar.bz2 44 | https://repo.continuum.io/pkgs/main/osx-64/locket-0.2.0-py36hca03003_1.tar.bz2 45 | https://repo.continuum.io/pkgs/main/osx-64/markupsafe-1.0-py36h1de35cc_1.tar.bz2 46 | https://repo.continuum.io/pkgs/main/osx-64/msgpack-python-0.5.6-py36h04f5b5a_1.tar.bz2 47 | https://repo.anaconda.com/pkgs/main/osx-64/numpy-base-1.15.4-py36h8a80b8c_0.tar.bz2 48 | https://repo.continuum.io/pkgs/main/osx-64/olefile-0.46-py36_0.tar.bz2 49 | https://repo.anaconda.com/pkgs/main/osx-64/pluggy-0.8.0-py36_0.tar.bz2 50 | https://repo.anaconda.com/pkgs/main/osx-64/psutil-5.4.8-py36h1de35cc_0.tar.bz2 51 | https://repo.anaconda.com/pkgs/main/osx-64/py-1.7.0-py36_0.tar.bz2 52 | https://repo.continuum.io/pkgs/main/osx-64/pycparser-2.19-py36_0.tar.bz2 53 | https://repo.anaconda.com/pkgs/main/osx-64/pyparsing-2.3.0-py36_0.tar.bz2 54 | https://repo.continuum.io/pkgs/main/osx-64/pysocks-1.6.8-py36_0.tar.bz2 55 | https://repo.anaconda.com/pkgs/main/osx-64/pytz-2018.7-py36_0.tar.bz2 56 | https://repo.continuum.io/pkgs/main/osx-64/pyyaml-3.13-py36h1de35cc_0.tar.bz2 57 | https://repo.continuum.io/pkgs/main/osx-64/six-1.11.0-py36_1.tar.bz2 58 | https://repo.continuum.io/pkgs/main/osx-64/sortedcontainers-2.0.5-py36_0.tar.bz2 59 | https://repo.continuum.io/pkgs/main/osx-64/tblib-1.3.2-py36hda67792_0.tar.bz2 60 | https://repo.continuum.io/pkgs/main/osx-64/toolz-0.9.0-py36_0.tar.bz2 61 | https://repo.anaconda.com/pkgs/main/osx-64/tornado-5.1.1-py36h1de35cc_0.tar.bz2 62 | https://repo.continuum.io/pkgs/main/osx-64/cffi-1.11.5-py36h6174b99_1.tar.bz2 63 | https://repo.continuum.io/pkgs/main/osx-64/cycler-0.10.0-py36hfc81398_0.tar.bz2 64 | https://repo.continuum.io/pkgs/main/osx-64/cytoolz-0.9.0.1-py36h1de35cc_1.tar.bz2 65 | https://repo.continuum.io/pkgs/main/osx-64/more-itertools-4.3.0-py36_0.tar.bz2 66 | https://repo.anaconda.com/pkgs/main/osx-64/packaging-18.0-py36_0.tar.bz2 67 | https://repo.anaconda.com/pkgs/main/osx-64/partd-0.3.9-py36_0.tar.bz2 68 | https://repo.anaconda.com/pkgs/main/osx-64/pillow-5.3.0-py36hb68e598_0.tar.bz2 69 | https://repo.anaconda.com/pkgs/main/osx-64/python-dateutil-2.7.5-py36_0.tar.bz2 70 | https://repo.anaconda.com/pkgs/main/osx-64/setuptools-40.5.0-py36_0.tar.bz2 71 | https://repo.continuum.io/pkgs/main/osx-64/zict-0.1.3-py36_0.tar.bz2 72 | https://repo.continuum.io/pkgs/main/osx-64/cryptography-2.3.1-py36ha12b0ac_2.tar.bz2 73 | https://repo.anaconda.com/pkgs/main/osx-64/cython-0.29-py36h0a44026_0.tar.bz2 74 | https://repo.anaconda.com/pkgs/main/osx-64/distributed-1.24.0-py36_0.tar.bz2 75 | https://repo.continuum.io/pkgs/main/osx-64/jinja2-2.10-py36_0.tar.bz2 76 | https://repo.anaconda.com/pkgs/main/osx-64/networkx-2.2-py36_1.tar.bz2 77 | https://repo.anaconda.com/pkgs/main/osx-64/pytest-3.10.0-py36_0.tar.bz2 78 | https://repo.continuum.io/pkgs/main/osx-64/wheel-0.32.2-py36_0.tar.bz2 79 | https://repo.anaconda.com/pkgs/main/osx-64/pip-18.1-py36_0.tar.bz2 80 | https://repo.continuum.io/pkgs/main/osx-64/pyopenssl-18.0.0-py36_0.tar.bz2 81 | https://repo.continuum.io/pkgs/main/osx-64/pytest-openfiles-0.3.0-py36_0.tar.bz2 82 | https://repo.anaconda.com/pkgs/main/osx-64/pytest-remotedata-0.3.1-py36_0.tar.bz2 83 | https://repo.anaconda.com/pkgs/main/osx-64/urllib3-1.24.1-py36_0.tar.bz2 84 | https://repo.anaconda.com/pkgs/main/osx-64/botocore-1.12.35-py36_0.tar.bz2 85 | https://repo.continuum.io/pkgs/main/osx-64/s3transfer-0.1.13-py36_0.tar.bz2 86 | https://repo.anaconda.com/pkgs/main/osx-64/boto3-1.9.35-py36_0.tar.bz2 87 | https://repo.anaconda.com/pkgs/main/osx-64/bokeh-1.0.1-py36_0.tar.bz2 88 | https://repo.continuum.io/pkgs/main/osx-64/h5py-2.8.0-py36h878fce3_3.tar.bz2 89 | https://repo.continuum.io/pkgs/main/osx-64/imageio-2.4.1-py36_0.tar.bz2 90 | https://repo.anaconda.com/pkgs/main/osx-64/matplotlib-3.0.1-py36h54f8f79_0.tar.bz2 91 | https://repo.anaconda.com/pkgs/main/osx-64/mkl_fft-1.0.6-py36hb8a8100_0.tar.bz2 92 | https://repo.continuum.io/pkgs/main/osx-64/mkl_random-1.0.1-py36h5d10147_1.tar.bz2 93 | https://repo.anaconda.com/pkgs/main/osx-64/numpy-1.15.4-py36h6a91979_0.tar.bz2 94 | https://repo.continuum.io/pkgs/main/osx-64/pandas-0.23.4-py36h6440ff4_0.tar.bz2 95 | https://repo.continuum.io/pkgs/main/osx-64/pytest-arraydiff-0.2-py36h39e3cac_0.tar.bz2 96 | https://repo.continuum.io/pkgs/main/osx-64/pytest-doctestplus-0.1.3-py36_0.tar.bz2 97 | https://repo.anaconda.com/pkgs/main/osx-64/pywavelets-1.0.1-py36h1d22016_0.tar.bz2 98 | https://repo.anaconda.com/pkgs/main/osx-64/scipy-1.1.0-py36h28f7352_1.tar.bz2 99 | https://repo.anaconda.com/pkgs/main/osx-64/dask-0.20.0-py36_0.tar.bz2 100 | https://repo.continuum.io/pkgs/main/osx-64/pytest-astropy-0.4.0-py36_0.tar.bz2 101 | https://repo.continuum.io/pkgs/main/osx-64/scikit-image-0.14.0-py36h0a44026_1.tar.bz2 102 | https://repo.anaconda.com/pkgs/main/osx-64/scikit-learn-0.20.0-py36h4f467ca_1.tar.bz2 103 | https://repo.anaconda.com/pkgs/main/osx-64/astropy-3.0.5-py36h1de35cc_0.tar.bz2 104 | http://ssb.stsci.edu/astroconda/osx-64/photutils-0.5-py36_0.tar.bz2 105 | http://ssb.stsci.edu/astroconda/osx-64/stsci.tools-3.4.13-py36_0.tar.bz2 106 | http://ssb.stsci.edu/astroconda/osx-64/acstools-2.1.0-py36_0.tar.bz2 107 | http://ssb.stsci.edu/astroconda/osx-64/calcos-3.3.4-py36_0.tar.bz2 108 | http://ssb.stsci.edu/astroconda/osx-64/stistools-1.1-py36_2.tar.bz2 109 | http://ssb.stsci.edu/astroconda/osx-64/wfc3tools-1.3.4-py36_2.tar.bz2 110 | -------------------------------------------------------------------------------- /pipeline/prototyping_parameters.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "### Generating a training set\n", 8 | "- The goal of this notebook is to generate a training set for a ML algorithm.\n", 9 | "- The first approach will be simple.\n", 10 | " - Using the DQ as the label, train up a binary classifier to find cosmic rays" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": null, 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "%matplotlib notebook\n", 20 | "from astropy.io import fits\n", 21 | "import pandas as pd\n", 22 | "import numpy as np\n", 23 | "import os\n", 24 | "import glob\n", 25 | "import datetime as dt\n", 26 | "import matplotlib.pyplot as plt\n", 27 | "import matplotlib.patches as patches\n", 28 | "import numpy as np\n", 29 | "from astropy.visualization import SqrtStretch\n", 30 | "from astropy.visualization import LogStretch, LinearStretch, ZScaleInterval\n", 31 | "from astropy.visualization.mpl_normalize import ImageNormalize\n", 32 | "from sklearn.preprocessing import StandardScaler\n", 33 | "from scipy import ndimage\n", 34 | "import sys\n", 35 | "import skimage.segmentation as segment\n", 36 | "from photutils.utils import random_cmap\n", 37 | "plt.style.use('ggplot')" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": null, 43 | "metadata": {}, 44 | "outputs": [], 45 | "source": [ 46 | "sys.path.append('/Users/nmiles/hst_cosmic_rays/lib/')\n", 47 | "from CosmicRayLabel import CosmicRayLabel\n", 48 | "from ComputeStats import ComputeStats" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": null, 54 | "metadata": {}, 55 | "outputs": [], 56 | "source": [ 57 | "fname = './../data/jd4wemc8q_flt.fits'" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": null, 63 | "metadata": {}, 64 | "outputs": [], 65 | "source": [ 66 | "c = CosmicRayLabel(fname)" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": null, 72 | "metadata": {}, 73 | "outputs": [], 74 | "source": [ 75 | "c.generate_label()" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": null, 81 | "metadata": {}, 82 | "outputs": [], 83 | "source": [ 84 | "with fits.open(fname) as hdu:\n", 85 | " sci2 = hdu[1].data\n", 86 | " sci1 = hdu[4].data\n", 87 | "sci = np.concatenate([sci2, sci1])" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": null, 93 | "metadata": {}, 94 | "outputs": [], 95 | "source": [ 96 | "sci" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": null, 102 | "metadata": {}, 103 | "outputs": [], 104 | "source": [ 105 | "stats = ComputeStats(fname,c.label)" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": null, 111 | "metadata": { 112 | "scrolled": true 113 | }, 114 | "outputs": [], 115 | "source": [ 116 | "sizes = stats.compute_size()" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": null, 122 | "metadata": {}, 123 | "outputs": [], 124 | "source": [ 125 | "sizes" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": null, 131 | "metadata": { 132 | "scrolled": true 133 | }, 134 | "outputs": [], 135 | "source": [ 136 | "max_size_idx = np.where(np.asarray(list(sizes.values())) > 15.)" 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": null, 142 | "metadata": {}, 143 | "outputs": [], 144 | "source": [ 145 | "max_size_idx = max_size_idx[0] + 1" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": null, 151 | "metadata": {}, 152 | "outputs": [], 153 | "source": [ 154 | "idx = max_size_idx[1]" 155 | ] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "execution_count": null, 160 | "metadata": {}, 161 | "outputs": [], 162 | "source": [ 163 | "sizes[idx]" 164 | ] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "execution_count": null, 169 | "metadata": {}, 170 | "outputs": [], 171 | "source": [ 172 | "stats.cr_locs[idx-1]" 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": null, 178 | "metadata": {}, 179 | "outputs": [], 180 | "source": [ 181 | "box_data = sci[stats.cr_locs[idx-1]]" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "metadata": {}, 188 | "outputs": [], 189 | "source": [ 190 | "box_data.shape" 191 | ] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "execution_count": null, 196 | "metadata": {}, 197 | "outputs": [], 198 | "source": [ 199 | "coords = np.where(box_data > 4*np.median(box_data))\n", 200 | "coords = list(zip(coords[0], coords[1]))" 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": null, 206 | "metadata": {}, 207 | "outputs": [], 208 | "source": [ 209 | "box_data_dq = c.dq[stats.cr_locs[idx-1]]" 210 | ] 211 | }, 212 | { 213 | "cell_type": "code", 214 | "execution_count": null, 215 | "metadata": {}, 216 | "outputs": [], 217 | "source": [ 218 | "def mk_patch(r, c='red'):\n", 219 | " CR_center = patches.Rectangle((r[1]-0.5,r[0]-0.5), \n", 220 | " width=1, height=1, \n", 221 | " alpha=1.0, fill=False,\n", 222 | " linewidth=1.75, color=c)\n", 223 | " return CR_center" 224 | ] 225 | }, 226 | { 227 | "cell_type": "code", 228 | "execution_count": null, 229 | "metadata": {}, 230 | "outputs": [], 231 | "source": [ 232 | "norm = ImageNormalize(sci, stretch=LogStretch(a=5.), interval=ZScaleInterval())\n", 233 | "fig = plt.figure(figsize=(5,3))\n", 234 | "ax1 = fig.add_subplot(1,2,1)\n", 235 | "ax2 = fig.add_subplot(1,2,2,sharex=ax1, sharey=ax1)\n", 236 | "ax1.imshow(box_data, cmap='gray', origin='lower', norm=norm)\n", 237 | "ax2.imshow(box_data_dq, cmap='bone', interpolation='nearest', origin='lower')\n", 238 | "for coord in coords:\n", 239 | " patch1 = mk_patch(coord)\n", 240 | " patch2 = mk_patch(coord)\n", 241 | " ax1.add_patch(patch1)\n", 242 | " ax2.add_patch(patch2)" 243 | ] 244 | }, 245 | { 246 | "cell_type": "code", 247 | "execution_count": null, 248 | "metadata": {}, 249 | "outputs": [], 250 | "source": [ 251 | "np.where(box_data < -10)" 252 | ] 253 | }, 254 | { 255 | "cell_type": "code", 256 | "execution_count": null, 257 | "metadata": {}, 258 | "outputs": [], 259 | "source": [ 260 | "num_pix = len(box_data.flatten())" 261 | ] 262 | }, 263 | { 264 | "cell_type": "code", 265 | "execution_count": null, 266 | "metadata": {}, 267 | "outputs": [], 268 | "source": [ 269 | "data = {'sci':box_data.flatten(),'dq':box_data_dq.flatten(),'pix':np.linspace(1, num_pix, num_pix)} " 270 | ] 271 | }, 272 | { 273 | "cell_type": "code", 274 | "execution_count": null, 275 | "metadata": {}, 276 | "outputs": [], 277 | "source": [ 278 | "data" 279 | ] 280 | }, 281 | { 282 | "cell_type": "code", 283 | "execution_count": null, 284 | "metadata": {}, 285 | "outputs": [], 286 | "source": [ 287 | "print(len(data['sci']), len(data['dq']), len(data['pix']))" 288 | ] 289 | }, 290 | { 291 | "cell_type": "code", 292 | "execution_count": null, 293 | "metadata": {}, 294 | "outputs": [], 295 | "source": [ 296 | "df = pd.DataFrame(data)" 297 | ] 298 | }, 299 | { 300 | "cell_type": "code", 301 | "execution_count": null, 302 | "metadata": {}, 303 | "outputs": [], 304 | "source": [ 305 | "df.plot(kind='scatter',x='pix',y='sci',c='dq',colormap=plt.get_cmap('inferno_r'), alpha=0.65)" 306 | ] 307 | }, 308 | { 309 | "cell_type": "code", 310 | "execution_count": null, 311 | "metadata": {}, 312 | "outputs": [], 313 | "source": [ 314 | "d = [[1,2,3,4,5],[6,7,8,9,10]]\n", 315 | "d = np.asarray(d)" 316 | ] 317 | }, 318 | { 319 | "cell_type": "code", 320 | "execution_count": null, 321 | "metadata": {}, 322 | "outputs": [], 323 | "source": [ 324 | "d.flatten()" 325 | ] 326 | }, 327 | { 328 | "cell_type": "markdown", 329 | "metadata": {}, 330 | "source": [ 331 | "### Generate a multipage pdf of cutouts for cosmic rays to show what cosmic rays of different sizes look like" 332 | ] 333 | }, 334 | { 335 | "cell_type": "code", 336 | "execution_count": null, 337 | "metadata": {}, 338 | "outputs": [], 339 | "source": [ 340 | "avg_size = np.nanmean(list(sizes.values()))\n", 341 | "std_size = np.nanstd(list(sizes.values()))" 342 | ] 343 | }, 344 | { 345 | "cell_type": "code", 346 | "execution_count": null, 347 | "metadata": {}, 348 | "outputs": [], 349 | "source": [ 350 | "avg_size, std_size" 351 | ] 352 | }, 353 | { 354 | "cell_type": "code", 355 | "execution_count": null, 356 | "metadata": {}, 357 | "outputs": [], 358 | "source": [ 359 | "max_size_idx = np.where(np.asarray(list(sizes.values())) > 5.)" 360 | ] 361 | }, 362 | { 363 | "cell_type": "code", 364 | "execution_count": null, 365 | "metadata": {}, 366 | "outputs": [], 367 | "source": [] 368 | } 369 | ], 370 | "metadata": { 371 | "kernelspec": { 372 | "display_name": "Python 3", 373 | "language": "python", 374 | "name": "python3" 375 | }, 376 | "language_info": { 377 | "codemirror_mode": { 378 | "name": "ipython", 379 | "version": 3 380 | }, 381 | "file_extension": ".py", 382 | "mimetype": "text/x-python", 383 | "name": "python", 384 | "nbconvert_exporter": "python", 385 | "pygments_lexer": "ipython3", 386 | "version": "3.6.6" 387 | } 388 | }, 389 | "nbformat": 4, 390 | "nbformat_minor": 2 391 | } 392 | -------------------------------------------------------------------------------- /pipeline/download/download.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | This module utilizes the `astroquery.mast` package to programmatically download 4 | observations from the Mikulski Archive for Space Telescopes (MAST). 5 | """ 6 | import logging 7 | import os 8 | import warnings 9 | 10 | from astropy.time import Time 11 | from astroquery.mast import Observations 12 | import yaml 13 | 14 | 15 | 16 | __taskname__ = "download" 17 | __author__ = "Nathan Miles" 18 | __version__ = "1.0" 19 | __vdate__ = "22-Jan-2019" 20 | 21 | 22 | logging.basicConfig(format='%(levelname)-4s ' 23 | '[%(module)s:%(funcName)s:%(lineno)d]' 24 | ' %(message)s') 25 | LOG = logging.getLogger('Downloader') 26 | LOG.setLevel(logging.INFO) 27 | 28 | 29 | 30 | class Downloader(object): 31 | """ 32 | Downloader class for handling downloads locally and on AWS 33 | 34 | Parameters 35 | ---------- 36 | instr : str 37 | Instrument to download data for 38 | 39 | instr_cfg : dict 40 | Instrument configuration 41 | 42 | """ 43 | 44 | def __init__(self, instr, instr_cfg=None): 45 | 46 | 47 | self._mod_dir = os.path.dirname(os.path.abspath(__file__)) 48 | 49 | self._base = os.path.join('/', 50 | *self._mod_dir.split('/')[:-2]) 51 | 52 | if instr_cfg is None: 53 | cfg_file = os.path.join(self._base, 54 | 'CONFIG', 55 | 'pipeline_config.yaml') 56 | 57 | with open(cfg_file, 'r') as fobj: 58 | cfg = yaml.load(fobj) 59 | 60 | self._instr_cfg = cfg[instr] 61 | else: 62 | self._instr_cfg = instr_cfg 63 | 64 | 65 | self._dates = None 66 | 67 | self._download_dir = os.path.join( 68 | self._base, 69 | *self._instr_cfg['astroquery']['download_dir'].split('/') 70 | ) 71 | 72 | self._inactive_range = { 73 | 'ACS': [ 74 | Time('2007-01-27', format='iso'), 75 | Time('2009-05-01', format='iso') 76 | ], 77 | 'STIS': [ 78 | Time('2004-08-03', format='iso'), 79 | Time('2009-05-01', format='iso') 80 | ] 81 | } 82 | self._filtered_table = None 83 | self._instr = instr.replace('_', '/') # put into format for astroquery 84 | self._msg_div = '-' * 79 85 | self._obstype = 'calibration' 86 | self._product_type = ['image', 'spectrum'] # SPECTRUM is for STIS 87 | self._products = {} 88 | self._project = 'HST' 89 | self._start_date = None 90 | self._stop_date = None 91 | self._SubGroupDescription = self._instr_cfg['astroquery']['SubGroup' \ 92 | 'Description'] 93 | self._target_name = 'DARK*' 94 | self._t_exptime = [0.5, 10000] # Exposure times to include 95 | 96 | 97 | @property 98 | def instr_cfg(self): 99 | return self._instr_cfg 100 | 101 | @instr_cfg.getter 102 | def instr_cfg(self): 103 | """Configuration object 104 | 105 | Corresponds to the configuration object stored in the 106 | :py:attr:`~pipeline_updated.CosmicRayPipeline.cfg` attribute 107 | 108 | """ 109 | return self._instr_cfg 110 | 111 | 112 | @property 113 | def download_dir(self): 114 | return self._download_dir 115 | 116 | @download_dir.getter 117 | def download_dir(self): 118 | """Download directory for the instrument being analyzed 119 | 120 | Corresponds to the value stored in the given instruments info stored in 121 | the :py:attr:`~pipeline_updated.CosmicRayPipeline.cfg` attribute 122 | 123 | """ 124 | return self._download_dir 125 | 126 | @download_dir.setter 127 | def download_dir(self, value): 128 | self._download_dir = value 129 | 130 | @property 131 | def inactive_range(self): 132 | return self._inactive_range 133 | 134 | @inactive_range.getter 135 | def inactive_range(self): 136 | """Periods of inactivity for each instrument if they exists""" 137 | return self._inactive_range 138 | 139 | @property 140 | def instr(self): 141 | return self._instr 142 | 143 | @instr.getter 144 | def instr(self): 145 | """Name of the instrument that is going to be analyzed""" 146 | return self._instr 147 | 148 | @property 149 | def obstype(self): 150 | return self._obstype 151 | 152 | @obstype.getter 153 | def obstype(self): 154 | """`obstype` kwarg for MAST query (str)""" 155 | return self._obstype 156 | 157 | @property 158 | def products(self): 159 | return self._products 160 | 161 | @products.getter 162 | def products(self): 163 | """Filtered version of data products returned by MAST query""" 164 | return self._products 165 | 166 | @products.setter 167 | def products(self, value): 168 | self._products = value 169 | 170 | @property 171 | def product_type(self): 172 | return self._product_type 173 | 174 | @product_type.getter 175 | def product_type(self): 176 | """Product types to download """ 177 | return self._product_type 178 | 179 | @property 180 | def project(self): 181 | return self._project 182 | 183 | @project.getter 184 | def project(self): 185 | """`project` kwarg for MAST query (str)""" 186 | return self._project 187 | 188 | @property 189 | def SubGroupDescription(self): 190 | return self._SubGroupDescription 191 | 192 | @SubGroupDescription.getter 193 | def SubGroupDescription(self): 194 | """`SubGroupDescription` kwarg for filtering MAST products (list)""" 195 | return self._SubGroupDescription 196 | 197 | @property 198 | def t_exptime(self): 199 | return self._t_exptime 200 | 201 | @t_exptime.getter 202 | def t_exptime(self): 203 | """Range of exposure times to download""" 204 | return self._t_exptime 205 | 206 | @property 207 | def target_name(self): 208 | return self._target_name 209 | 210 | @target_name.getter 211 | def target_name(self): 212 | """Name of target to download""" 213 | return self._target_name 214 | 215 | def query(self, date_range, aws=False): 216 | """ Submit a query to MAST for observations in the date range 217 | 218 | Parameters 219 | ---------- 220 | date_range : tuple 221 | Tuple of `astropy.time.Time` objects correspond to the beginning 222 | and end of a one month interval 223 | 224 | aws : bool 225 | If True, query returns references to data hosted in S3. 226 | 227 | """ 228 | LOG.info('Submitting query to MAST') 229 | if aws: 230 | Observations.enable_s3_hst_dataset() 231 | start, stop = date_range 232 | # there shouldn't be any data taken after the most recent file 233 | query_params = { 234 | 'project': self.project, 235 | 'dataproduct_type': self.product_type, 236 | 'intentType': self.obstype, 237 | 'target_name': self.target_name, 238 | 'instrument_name': self.instr, 239 | 't_min': [start.mjd, stop.mjd], 240 | 't_exptime': self.t_exptime 241 | } 242 | with warnings.catch_warnings(): 243 | warnings.simplefilter('error') 244 | try: 245 | obsTable = Observations.query_criteria(**query_params) 246 | except Exception as e: 247 | msg = ('{}\n Date range [{}, {}]\n {}'.format(e, 248 | start.iso, 249 | stop.iso, 250 | self._msg_div)) 251 | LOG.error(msg) 252 | else: 253 | LOG.info('Filtering observations...') 254 | products = Observations.get_product_list(obsTable) 255 | filter_params = { 256 | 'mrp_only': False, 257 | 'productSubGroupDescription':self.SubGroupDescription 258 | } 259 | 260 | filt_products = Observations.filter_products(products, 261 | **filter_params) 262 | 263 | key = start.datetime.date().isoformat() # 'YYYY-MM-DD' 264 | self.products[key] = filt_products 265 | 266 | def download(self, key): 267 | """Download the data 268 | 269 | Only download the observations contained in the interval specified by 270 | the `key`. The `key` argument must correspond to one of the keys in the 271 | :py:attr:`~download.Downloader.products` attribute. If it is not, 272 | then a KeyError will be raised and the download will be skipped. 273 | 274 | 275 | Parameters 276 | ---------- 277 | key : str 278 | Date in ISO format (YYYY-MM-DD) of a given intervals start time 279 | 280 | Returns 281 | ------- 282 | None 283 | Downloaded data will be stored in directory specified by the 284 | :py:attr:`~download.Downloader.download_dir` attribute 285 | """ 286 | msg = ('Downloading data...\n ' 287 | 'Download Directory: {}\n {}'.format(self.download_dir, 288 | self._msg_div)) 289 | LOG.info(msg) 290 | download_params = { 291 | 'download_dir': self.download_dir, 292 | 'mrp_only': False, 293 | 'dataproduct_type': self.product_type, 294 | 'productSubGroupDescription': self.SubGroupDescription 295 | } 296 | try: 297 | download_list = self.products[key]['obsID'].tolist() 298 | except KeyError as e: 299 | LOG.error('{}\n{}'.format(e, self._msg_div)) 300 | else: 301 | Observations.download_products(download_list, **download_params) 302 | 303 | -------------------------------------------------------------------------------- /pipeline/utils/metadata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | This module contains the :py:class:`~utils.metadata.GenerateMetadata` 4 | that is used to extract the following information from each observation: 5 | 6 | * Date of the observation (YYYY-MM-DD HH:MM:SS) 7 | * Latitude, longitude, and altitude of HST through the course of the observation 8 | * WCS information for determining the pointing of HST during the observation 9 | * The total integration time of the observation (i.e. total exposure time plus additional factor to account for readout) 10 | 11 | """ 12 | from collections.abc import Iterable 13 | import logging 14 | import os 15 | 16 | from astropy.io import fits 17 | from astropy.time import Time 18 | from astropy.wcs import WCS 19 | from astropy.constants import R_earth 20 | from calcos import orbit 21 | from calcos.timeline import gmst, DEGtoRAD, rectToSph 22 | import numpy as np 23 | import yaml 24 | 25 | 26 | logging.basicConfig(format='%(levelname)-4s ' 27 | '[%(module)s.%(funcName)s:%(lineno)d]' 28 | ' %(message)s', 29 | ) 30 | 31 | LOG = logging.getLogger('CosmicRayPipeline') 32 | 33 | LOG.setLevel(logging.INFO) 34 | 35 | class GenerateMetadata(object): 36 | """ 37 | Class for generating and storing relevant metadata for each file 38 | 39 | Parameters 40 | ---------- 41 | fname : str 42 | Name of FITS file 43 | 44 | instr : 45 | Instrument to analyze (e.g. STIS_CCD, ACS_HRC, ACS_WFC, WFPC2, WFC3_UVIS) 46 | 47 | instr_cfg : dict 48 | Instrument specific configuration object 49 | 50 | """ 51 | def __init__(self, fname, instr, instr_cfg=None): 52 | self._fname = fname # file name will always be the FLT 53 | 54 | self._mod_dir = os.path.dirname(os.path.abspath(__file__)) 55 | 56 | self._base = os.path.join('/', 57 | *self._mod_dir.split('/')[:-2]) 58 | 59 | if instr_cfg is None: 60 | cfg_file = os.path.join(self._base, 61 | 'CONFIG', 62 | 'pipeline_config.yaml') 63 | 64 | with open(cfg_file, 'r') as fobj: 65 | cfg = yaml.load(fobj) 66 | self._instr_cfg = cfg[instr] 67 | else: 68 | self._instr_cfg = instr_cfg 69 | self._telemetry_file = None 70 | self._date = None 71 | self._metadata = {} 72 | self._instr = instr 73 | 74 | 75 | @property 76 | def fname(self): 77 | """Name of FITS file""" 78 | return self._fname 79 | 80 | @property 81 | def instr(self): 82 | """Instrument name""" 83 | return self._instr 84 | 85 | @instr.setter 86 | def instr(self, value): 87 | self._instr = value 88 | 89 | @property 90 | def instr_cfg(self): 91 | """Instrument specific configuration object""" 92 | return self._instr_cfg 93 | 94 | @property 95 | def telemetry_file(self): 96 | """Full path to corresponding telemetry file for :py:attr:`fname`""" 97 | return self._telemetry_file 98 | 99 | @telemetry_file.setter 100 | def telemetry_file(self, value): 101 | self._telemetry_file = value 102 | 103 | @property 104 | def metadata(self): 105 | """Dictionary used to store relevant metadata""" 106 | return self._metadata 107 | 108 | @metadata.setter 109 | def metadata(self, value): 110 | self._metadata = value 111 | 112 | def _engineering_file(self): 113 | """ Generate path to telemetry file 114 | 115 | Returns 116 | ------- 117 | 118 | """ 119 | input_suffix = \ 120 | self.instr_cfg['astroquery']['SubGroupDescription'][0].lower() 121 | telemetry_suffix = \ 122 | self.instr_cfg['astroquery']['SubGroupDescription'][1].lower() 123 | 124 | self.telemetry_file = self.fname.replace(input_suffix, 125 | telemetry_suffix) 126 | 127 | 128 | def get_wcs_info(self): 129 | """ Parse the WCS information to determine the telescope pointing 130 | 131 | This method will record all of the WCS information stored in the 132 | header. 133 | 134 | Return 135 | ------- 136 | 137 | """ 138 | with fits.open(self.fname) as hdu: 139 | try: 140 | wcs_obj = WCS(fobj = hdu, header = hdu[1].header) 141 | except (MemoryError, ValueError, KeyError) as e: 142 | LOG.error(e) 143 | else: 144 | wcs_header = wcs_obj.to_header() 145 | for key in wcs_header.keys(): 146 | self.metadata[key] = wcs_header[key] 147 | 148 | def get_image_data(self): 149 | """Parse the FITS header and retrieve important keywords 150 | 151 | This will store the following keywords: 152 | 153 | - `date-obs` 154 | - `expstart` 155 | - `expend` 156 | - `exptime` 157 | - `flashdur` (if it exists) 158 | - `time-obs` 159 | 160 | `date-obs` and `time-obs` are combined into a single string in ISO 161 | format (YYYY-MM-DD HH:MM:SS). 162 | 163 | `exptime` and `flashdur` are combined with the `readout_time` of the 164 | detector to compute the total integration time. 165 | 166 | 167 | Returns 168 | ------- 169 | 170 | """ 171 | header_data = {'date-obs': None, 172 | 'expstart': None, 173 | 'expend': None, 174 | 'exptime': None, 175 | 'flashdur': 0, 176 | 'time-obs': None} 177 | 178 | with fits.open(self.fname) as hdu: 179 | prhdr = hdu[0].header 180 | scihdr = hdu[1].header 181 | for key in header_data.keys(): 182 | try: 183 | header_data[key] = prhdr[key] 184 | except KeyError as e: 185 | # LOG.warning('{}\n Searching SCI header\n'.format(e)) 186 | try: 187 | header_data[key] = scihdr[key] 188 | except KeyError as e: 189 | LOG.warning('{}'.format(e)) 190 | 191 | 192 | date = Time( 193 | '{} {}'.format(header_data['date-obs'], header_data['time-obs']), 194 | format='iso' 195 | ) 196 | readout_time = self.instr_cfg['instr_params']['readout_time'] 197 | # Check to see if there are multiple readout times (WFPC2 only) 198 | if isinstance(readout_time, Iterable): 199 | if header_data['exptime'] <= 180: 200 | LOG.info('WFPC2 observation with EXPTIME <= 180\n' 201 | f"Using readtime time of {readout_time[0]:0.0f}s" 202 | ) 203 | readout_time = readout_time[0] 204 | else: 205 | LOG.info('WFPC2 observation with EXPTIME > 180\n' 206 | f"Using readout time of {readout_time[1]:0.0f}" 207 | ) 208 | readout_time = readout_time[1] 209 | 210 | self.metadata['date'] = date.iso 211 | self.metadata['expstart'] = Time(header_data['expstart'], format='mjd') 212 | self.metadata['expend'] = Time(header_data['expend'], format='mjd') 213 | self.metadata['integration_time'] = \ 214 | header_data['exptime'] + header_data['flashdur'] + readout_time 215 | 216 | def get_observatory_info(self, time_delta=None): 217 | """Compute the lat/lon and altitude of HST. 218 | 219 | Using the expstart and expend, generate a series MJD dates that correspond 220 | to 1 minute intervals. Calculate the lat/lon and altitude at each 221 | time step. 222 | 223 | """ 224 | 225 | altitude_list = [] 226 | lat_list = [] 227 | lon_list = [] 228 | 229 | # Break up the exposure into one minute intervals 230 | if time_delta is None: 231 | time_delta = self.metadata['expend'] - self.metadata['expstart'] 232 | num_intervals = 2*int(time_delta.to('minute').value) 233 | expend = self.metadata['expend'].mjd 234 | else: 235 | expend= self.metadata['expstart'].mjd + time_delta/86400.0 236 | num_intervals = int(np.round(time_delta/60)) 237 | 238 | # If the number of one minute intervals is less than 2, set the number 239 | # of intervals to 5 (arbitrarily chosen) 240 | if num_intervals < 2: 241 | num_intervals = 5 242 | # Generate MJD dates correspond to these one minute intervals 243 | time_intervals = np.linspace(self.metadata['expstart'].mjd, 244 | expend, 245 | num_intervals, 246 | endpoint=True) 247 | 248 | # Generate the path to the engineering file 249 | self._engineering_file() 250 | # Using the telemetry data for the SPT file, compute HST (lon, lat, z) 251 | if os.path.isfile(self.telemetry_file): 252 | orbital_params = orbit.HSTOrbit(self.telemetry_file) 253 | # compute coords at beginning and end of exposure 254 | for t in time_intervals: 255 | rect, vel = orbital_params.getPos(t) 256 | r, ra, dec = rectToSph(rect) 257 | altitude_list.append(r - R_earth.to('km').value) 258 | lat = dec 259 | lon = ra - 2 * np.pi * gmst(t) 260 | if lon < 0: 261 | lon += 2 * np.pi 262 | lon /= DEGtoRAD 263 | lat /= DEGtoRAD 264 | lat_list.append(lat) 265 | lon_list.append(lon) 266 | 267 | self.metadata['latitude'] = np.asarray(lat_list) 268 | self.metadata['longitude'] = np.asarray(lon_list) 269 | self.metadata['altitude'] = np.asarray(altitude_list) 270 | self.metadata['time_intervals'] = time_intervals 271 | else: 272 | LOG.info('SPT file not found, place it in the data directory') 273 | # If the SPT file for some reason doesn't exist, save NaNs 274 | self.metadata['altitude'] = np.nan 275 | self.metadata['latitude'] = np.nan 276 | self.metadata['longitude'] = np.nan 277 | self.metadata['time_intervals'] = np.nan 278 | -------------------------------------------------------------------------------- /analyzing_cr_rejection/compare_results.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from collections import defaultdict 3 | import glob 4 | import json 5 | import logging 6 | import os 7 | _MOD_DIR = os.path.dirname(os.path.abspath(__file__)) 8 | _BASE = os.path.join('/', *_MOD_DIR.split('/')[:-1]) 9 | 10 | import shutil 11 | import sys 12 | sys.path.append(os.path.join(_BASE, 'pipeline')) 13 | 14 | import warnings 15 | warnings.simplefilter("ignore") 16 | 17 | 18 | from astropy.io import fits 19 | from astropy.table import Table 20 | from astropy.time import Time 21 | import matplotlib.pyplot as plt 22 | import matplotlib.dates as mdates 23 | import matplotlib.cbook as cbook 24 | from matplotlib.ticker import (MultipleLocator, FormatStrFormatter, 25 | AutoMinorLocator) 26 | plt.style.use('ggplot') 27 | import numpy as np 28 | import pandas as pd 29 | import scipy.ndimage as ndimage 30 | from utils import initialize 31 | from utils import datahandler as dh 32 | 33 | _PLOT_DIR = os.path.join(_BASE, 'analyzing_cr_rejection', 'plots') 34 | _RESULTS_DIR = os.path.join(_BASE, 35 | 'analyzing_cr_rejection', 36 | 'results', 37 | 'STIS' 38 | ) 39 | 40 | logging.basicConfig(format='%(levelname)-4s ' 41 | '[%(module)s.%(funcName)s:%(lineno)d]' 42 | ' %(message)s', 43 | ) 44 | LOG = logging.getLogger('compare_results') 45 | LOG.setLevel(logging.INFO) 46 | 47 | 48 | def create_data_objects(flist): 49 | """ 50 | Parameters 51 | ---------- 52 | flist : TYPE 53 | Description 54 | 55 | Returns 56 | ------- 57 | TYPE 58 | Description 59 | """ 60 | obj = dh.DataReader(instr='stis_ccd', statistic='incident_cr_rate') 61 | obj.hdf5_files = flist 62 | obj.read_cr_rate() 63 | return obj 64 | 65 | 66 | def make_MEF(fname, hdf5file1=None, hdf5file2=None, params1=None, params2=None): 67 | """ 68 | Parameters 69 | ---------- 70 | fname : TYPE 71 | Description 72 | hdf5file1 : None, optional 73 | Description 74 | hdf5file2 : None, optional 75 | Description 76 | """ 77 | LOG.info(f'Creating a MEF for {fname}') 78 | dset = os.path.basename(fname) 79 | hdu_list = fits.HDUList() 80 | with fits.open(fname) as hdu: 81 | prhdr = hdu[0].header 82 | scihdr = hdu[1].header 83 | sci = hdu[1].data 84 | dqhdr = hdu[3].header 85 | dq = hdu[3].data 86 | prhdu = fits.PrimaryHDU(header=prhdr) 87 | hdu_list.append(fits.ImageHDU(header=scihdr, data=sci)) 88 | hdu_list.append(fits.ImageHDU(header=dqhdr, data=dq)) 89 | if hdf5file1 is not None: 90 | label1, metadata1 = label_from_file( 91 | hdf5file=hdf5file1, 92 | dset_name=dset, 93 | shape=sci.shape 94 | ) 95 | hdr1 = fits.Header(cards=[], copy=False) 96 | params = '_'.join(os.path.basename(hdf5file1).split('.hdf5')[0].split('_')[-3:]) 97 | hdr1.fromkeys(metadata1) 98 | hdr1['EXTNAME'] = 'CRLABEL' 99 | hdr1['PARAMS'] = params1 100 | hdu_list.append(fits.ImageHDU(header=hdr1, data=label1)) 101 | if hdf5file2 is not None: 102 | label2, metadata2 = label_from_file( 103 | hdf5file=hdf5file2, 104 | dset_name=dset, 105 | shape=sci.shape 106 | ) 107 | 108 | hdr2 = fits.Header(cards=[], copy=False) 109 | params = '_'.join(os.path.basename(hdf5file2).split('.hdf5')[0].split('_')[-3:]) 110 | hdr2.fromkeys(metadata2) 111 | hdr2['EXTNAME'] = 'CRLABEL' 112 | hdr2['PARAMS'] = params2 113 | hdu_list.append(fits.ImageHDU(header=hdr2, data=label2)) 114 | LOG.info(f"{hdu_list.info()}") 115 | hdu_list.writeto(f"{fname.replace('_flt.fits', '_all.fits')}", overwrite=True) 116 | 117 | 118 | def label_from_file(hdf5file, dset_name, shape=None): 119 | """ 120 | Parameters 121 | ---------- 122 | hdf5file : TYPE 123 | Description 124 | dset_name : TYPE 125 | Description 126 | shape : None, optional 127 | Description 128 | 129 | Returns 130 | ------- 131 | TYPE 132 | Description 133 | """ 134 | dh1 = dh.DataReader(instr='stis_ccd', statistic='cr_affected_pixels') 135 | cr_affected_pixels, metadata = dh1.read_single_dst(hdf5file, dset_name) 136 | template = np.zeros(shape) 137 | for (y,x) in cr_affected_pixels: 138 | template[int(y)][int(x)] +=1 139 | label, num_feat = ndimage.label(template, 140 | structure=np.ones((3,3))) 141 | return label, metadata 142 | 143 | 144 | def examine_label(dirname=_RESULTS_DIR, exptime=60.0): 145 | """ 146 | Parameters 147 | ---------- 148 | dirname : TYPE, optional 149 | Description 150 | exptime : float, optional 151 | Description 152 | """ 153 | flist = glob.glob(f"{dirname}/stis*cr_affected_pixels*hdf5") 154 | file1 = display_menu(flist) 155 | params1 = '_'.join(os.path.basename(file1).split('.hdf5')[0].split('_')[-2:]) 156 | dir1 = os.path.join( 157 | _BASE,'analyzing_cr_rejection', 158 | f"{exptime}_{params1}" 159 | ) 160 | print(dir1) 161 | dataset1 = glob.glob(dir1+'/*flt.fits') 162 | file2 = display_menu(flist) 163 | params2 = '_'.join(os.path.basename(file2).split('.hdf5')[0].split('_')[-2:]) 164 | dir2 = os.path.join( 165 | _BASE, 'analyzing_cr_rejection', f"{exptime}_{params2}") 166 | print(dir2) 167 | dataset2 = glob.glob(dir2+'/*flt.fits') 168 | print(len(dataset1), len(dataset2)) 169 | 170 | for f1, f2 in zip(dataset1, dataset2): 171 | make_MEF(fname=f1, hdf5file1=file1, params1=params1) 172 | make_MEF(fname=f2, hdf5file2=file2, params2=params2) 173 | 174 | 175 | def exptime_summary(dh, title=''): 176 | """ 177 | Parameters 178 | ---------- 179 | dh : TYPE 180 | Description 181 | title : str, optional 182 | Description 183 | """ 184 | counts = dh.data_df.integration_time.value_counts() 185 | fig, ax = plt.subplots(nrows=1, ncols=1) 186 | counts.plot.barh(ax=ax) 187 | ax.set_title(title) 188 | ax.set_ylabel('Integration Time [seconds]') 189 | plt.show() 190 | 191 | 192 | def compare_by_exptime(dh, title=''): 193 | """ 194 | Parameters 195 | ---------- 196 | dh : TYPE 197 | Description 198 | title : str, optional 199 | Description 200 | """ 201 | longexp = dh.data_df.integration_time.gt(1000) 202 | shortexp = dh.data_df.integration_time.lt(100) 203 | 204 | fig, ax = plt.subplots(nrows=1, ncols=1) 205 | fig.autofmt_xdate() 206 | ax.scatter(dh.data_df[longexp].index, 207 | dh.data_df[longexp].incident_cr_rate, label='longexp') 208 | ax.scatter(dh.data_df[shortexp].index, 209 | dh.data_df[shortexp].incident_cr_rate, label='shortexp') 210 | ax.legend(loc='best') 211 | ax.set_title(title) 212 | plt.show() 213 | 214 | 215 | def compare_by_rej_params( 216 | dh1, 217 | dh2, 218 | label1='', 219 | label2='', 220 | title='', 221 | fout=None, 222 | figsize=(6,5) 223 | ): 224 | """ 225 | Parameters 226 | ---------- 227 | dh1 : TYPE 228 | Description 229 | dh2 : TYPE 230 | Description 231 | label1 : str, optional 232 | Description 233 | label2 : str, optional 234 | Description 235 | title : str, optional 236 | Description 237 | fout : None, optional 238 | Description 239 | figsize : tuple, optional 240 | Description 241 | """ 242 | 243 | years = mdates.YearLocator() # every year 244 | months = mdates.MonthLocator() # every month 245 | days_major = mdates.DayLocator(interval=5) 246 | days_minor = mdates.DayLocator(interval=1) 247 | years_fmt = mdates.DateFormatter('%Y-%m-%d') 248 | 249 | 250 | expcut = dh1.data_df.integration_time.gt(1000) 251 | shortexp = dh1.data_df.integration_time.lt(100) 252 | 253 | fig, (ax1, ax2) = plt.subplots( 254 | nrows=1, 255 | ncols=2, 256 | figsize=figsize, 257 | sharex=True 258 | ) 259 | 260 | diff = dh2.data_df[expcut].incident_cr_rate - \ 261 | dh1.data_df[expcut].incident_cr_rate 262 | dates = dh1.data_df[expcut].index.values 263 | rate1 = dh1.data_df[expcut].incident_cr_rate 264 | rate2 = dh2.data_df[expcut].incident_cr_rate 265 | fig.autofmt_xdate() 266 | # ax1.xaxis.set_major_locator(plt.MaxNLocator(10)) 267 | # ax2.xaxis.set_major_locator(plt.MaxNLocator(10)) 268 | ax1.scatter(dh1.data_df[expcut].index.values, 269 | rate1, label=label1) 270 | ax1.scatter(dh2.data_df[expcut].index.values, 271 | rate2, label=label2) 272 | ax1.set_ylabel('CR Rate [CR/cm$^2$/second]') 273 | ax1.legend(loc='best') 274 | ax2.scatter(diff.index.values, diff, c='k') 275 | ax2.set_title(f'{label2} - {label1}') 276 | ax1.set_xlim( 277 | (Time('2019-05-25', format='iso').to_datetime(), 278 | Time('2019-07-01', format='iso').to_datetime()) 279 | ) 280 | # ax1.fmt_xdata = mdates.DateFormatter('%m-%d') 281 | # ax2.fmt_xdata = mdates.DateFormatter('%m-%d') 282 | ax1.xaxis.set_major_locator(days_major) 283 | ax1.xaxis.set_major_formatter(mdates.DateFormatter('%b %d')) 284 | ax1.yaxis.set_minor_locator(AutoMinorLocator(5)) 285 | ax1.xaxis.set_minor_locator(days_minor) 286 | 287 | ax2.xaxis.set_major_locator(days_major) 288 | ax2.yaxis.set_minor_locator(AutoMinorLocator(5)) 289 | ax2.xaxis.set_major_formatter(mdates.DateFormatter('%b %d')) 290 | ax2.xaxis.set_minor_locator(days_minor) 291 | 292 | fig.suptitle(title) 293 | if fout is not None: 294 | fout = os.path.join(_PLOT_DIR, fout) 295 | fig.savefig(fout, format='png', dpi=300, bbox_inches='tight') 296 | plt.show() 297 | 298 | 299 | def get_default_parameters(dh): 300 | """ 301 | Parameters 302 | ---------- 303 | dh : TYPE 304 | Description 305 | """ 306 | expsum = dh.data_df.integration_time.sum() 307 | n_images = len(dh.data_df) 308 | tb = Table.read('/Users/nmiles/hst_cosmic_rays/j3m1403io_crr.fits') 309 | 310 | 311 | def display_menu(flist): 312 | # Generate a list of options for the user to choose from 313 | out_str = 'Choose a dataset to analyze:\n' 314 | for i, f in enumerate(flist): 315 | out_str += f"{i}) {os.path.basename(f)}\n" 316 | LOG.info(f"{out_str}\n{'-'*79}") 317 | idx = int(input('Enter selection: ')) 318 | file = flist[idx] 319 | LOG.info(f"Selected option: {file}") 320 | return file 321 | 322 | 323 | def run_comparison(fout=None): 324 | """ 325 | Parameters 326 | ---------- 327 | fout : None, optional 328 | Description 329 | """ 330 | results_dir = os.path.join(_BASE, 331 | 'analyzing_cr_rejection', 332 | 'results', 333 | 'STIS' 334 | ) 335 | # Get a list of all the files generated with median combination 336 | medflist = glob.glob( 337 | os.path.join(_RESULTS_DIR,'stis*cr_rate*med*hdf5') 338 | ) 339 | # Get a list of all the files generated with minimum combination 340 | minflist = glob.glob( 341 | os.path.join(_RESULTS_DIR, 'stis*cr_rate*min*hdf5') 342 | ) 343 | flist = medflist + minflist 344 | 345 | file1 = display_menu(flist) 346 | file2 = display_menu(flist) 347 | 348 | # Extract the CR parameters from the filename 349 | file1_rej_param = file1.replace('.hdf5','') 350 | file1_rej_param = ' '.join(file1_rej_param.split('_')[-2:]) 351 | 352 | # Extract the CR parameters from the filename 353 | file2_rej_param = file2.replace('.hdf5','') 354 | file2_rej_param = ' '.join(file2_rej_param.split('_')[-2:]) 355 | 356 | 357 | dh_1 = create_data_objects([file1]) 358 | dh_2 = create_data_objects([file2]) 359 | 360 | # exptime_summary(dh_min, title='Exposure Times') 361 | # compare_by_exptime(dh_min, title=min_rej_param) 362 | # compare_by_exptime(dh_med, title=med_rej_param) 363 | 364 | compare_by_rej_params(dh_1, dh_2, 365 | label1=file1_rej_param, label2=file2_rej_param, 366 | title=f'{file1_rej_param} vs. {file2_rej_param}', 367 | fout=f'{file1_rej_param}_{file2_rej_param}.png', figsize=(6.5,4.5)) 368 | 369 | examine_label() 370 | 371 | 372 | 373 | 374 | 375 | 376 | -------------------------------------------------------------------------------- /pipeline/utils/initialize.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | This module contains a class :py:class:`~initialize.initialize.Initializer` 4 | that contains a series of methods for performing basic initialization of the 5 | pipeline. In short it will perform the following: 6 | 7 | * Generate a series of empty HDF5 files for each statistic recorded. 8 | * Generate a list of the previously processed date ranges. 9 | * Generate a list of date intervals spanning one month periods to be used 10 | in the downloading process. 11 | 12 | """ 13 | 14 | from collections import defaultdict 15 | import logging 16 | import os 17 | import warnings 18 | import yaml 19 | 20 | from astropy.time import Time 21 | import h5py 22 | from numpy import array 23 | from pandas import date_range 24 | 25 | 26 | logging.basicConfig(format='%(levelname)-4s ' 27 | '[%(module)s:%(funcName)s:%(lineno)d]' 28 | ' %(message)s') 29 | LOG = logging.getLogger() 30 | LOG.setLevel(logging.INFO) 31 | 32 | 33 | class Initializer(object): 34 | """Class for initializing common things used by the entire pipeline 35 | 36 | Parameters 37 | ---------- 38 | instr : str 39 | Instrument to process 40 | 41 | cfg : dict 42 | Pipeline configuration object 43 | 44 | """ 45 | def __init__(self, instr, cfg=None, instr_cfg=None): 46 | 47 | self._instr = instr 48 | self._mod_dir = os.path.dirname(os.path.abspath(__file__)) 49 | self._base = os.path.join('/', 50 | *self._mod_dir.split('/')[:-2]) 51 | if cfg is None: 52 | cfg_file = os.path.join(self._base, 53 | 'CONFIG', 54 | 'pipeline_config.yaml') 55 | 56 | with open(cfg_file, 'r') as fobj: 57 | cfg = yaml.load(fobj) 58 | self._cfg = cfg 59 | else: 60 | self._cfg = cfg 61 | 62 | if instr_cfg is None: 63 | self._instr_cfg = cfg[instr] 64 | else: 65 | self._instr_cfg = instr_cfg 66 | 67 | self._dates = None 68 | self._inactive_range = { 69 | 'ACS_WFC': [ 70 | Time('2007-01-27', format='iso'), 71 | Time('2009-05-01', format='iso') 72 | ], 73 | 'ACS_HRC':[ 74 | Time('2007-01-27', format='iso'), 75 | Time.now() 76 | ], 77 | 'STIS_CCD': [ 78 | Time('2004-08-03', format='iso'), 79 | Time('2009-05-01', format='iso') 80 | ] 81 | } 82 | self._previously_analyzed = None 83 | self._start_date = None 84 | self._stop_date = None 85 | 86 | @property 87 | def base(self): 88 | return self._base 89 | 90 | @base.getter 91 | def base(self): 92 | """Base path of the pipleine repository `~/hst_cosmic_rays/`""" 93 | return self._base 94 | 95 | @property 96 | def cfg(self): 97 | return self._cfg 98 | 99 | @cfg.getter 100 | def cfg(self): 101 | """Configuration object 102 | 103 | Corresponds to the configuration object stored in the 104 | :py:attr:`~pipeline_updated.CosmicRayPipeline.cfg` attribute 105 | 106 | """ 107 | return self._cfg 108 | 109 | @property 110 | def instr_cfg(self): 111 | return self._instr_cfg 112 | 113 | @instr_cfg.getter 114 | def instr_cfg(self): 115 | """Instrument specific configuration object 116 | 117 | Corresponds to one of the `dict` configuration object stored in the 118 | :py:attr:`~pipeline_updated.CosmicRayPipeline.cfg` attribute 119 | 120 | """ 121 | return self._instr_cfg 122 | 123 | @property 124 | def dates(self): 125 | return self._dates 126 | 127 | @dates.getter 128 | def dates(self): 129 | """A list of one month date intervals""" 130 | return self._dates 131 | 132 | @dates.setter 133 | def dates(self, value): 134 | self._dates = value 135 | 136 | @property 137 | def inactive_range(self): 138 | return self._inactive_range 139 | 140 | @inactive_range.getter 141 | def inactive_range(self): 142 | """Periods of inactivity for each instrument if they exists""" 143 | return self._inactive_range 144 | 145 | @property 146 | def instr(self): 147 | return self._instr 148 | 149 | @instr.getter 150 | def instr(self): 151 | """Name of the instrument that is going to be analyzed""" 152 | return self._instr 153 | 154 | @property 155 | def previously_analyzed(self): 156 | return self._previously_analyzed 157 | 158 | @previously_analyzed.getter 159 | def previously_analyzed(self): 160 | """A list of previously analyzed date ranges""" 161 | return self._previously_analyzed 162 | 163 | @previously_analyzed.setter 164 | def previously_analyzed(self, value): 165 | self._previously_analyzed = value 166 | 167 | @property 168 | def start_date(self): 169 | return self._start_date 170 | 171 | @start_date.getter 172 | def start_date(self): 173 | """Earliest possible date for any observations taken by instrument""" 174 | return self._start_date 175 | 176 | @start_date.setter 177 | def start_date(self, value): 178 | self._start_date = value 179 | 180 | @property 181 | def stop_date(self): 182 | return self._stop_date 183 | 184 | @stop_date.getter 185 | def stop_date(self): 186 | """Latest possible date for any observations taken by instrument""" 187 | return self._stop_date 188 | 189 | @stop_date.setter 190 | def stop_date(self, value): 191 | self._stop_date = value 192 | 193 | def initialize_dates(self): 194 | """Determine the start and stop dates of all observations 195 | 196 | The ranges of dates we wish to analyze will depend on whether or not 197 | the instrument being analyzed is an active or legacy instrument. 198 | 199 | - Active instruments have no defined end date, so we set this to 200 | the current date using :py:meth:`astropy.time.Time.now` 201 | method. 202 | 203 | - Legacy instruments have a defined end date that corresponds to 204 | a date when the instrument either failed or was shutdown. 205 | 206 | Initializes the :py:attr:`start_date` and 207 | :py:attr:`stop_date` attributes 208 | """ 209 | cfg_dates = self.instr_cfg['astroquery']['date_range'] 210 | if isinstance(cfg_dates, list): 211 | # Inactive instrument have defined date ranges 212 | self.start_date = Time(cfg_dates[0], format='iso') 213 | self.stop_date = Time(cfg_dates[1], format='iso') 214 | else: 215 | self.start_date = Time(cfg_dates, format='iso') 216 | self.stop_date = Time.now() 217 | 218 | 219 | def get_date_ranges(self): 220 | """ Generate a list of tuples containing one month intervals 221 | 222 | For instruments that experienced failures, intervals that fall in a 223 | period of inactivity will be automatically removed. 224 | """ 225 | self.initialize_dates() 226 | pd_range = date_range(start=self.start_date.iso, 227 | end=self.stop_date.iso, 228 | freq='1MS') 229 | dates = [Time(date.date().isoformat(), format='iso') 230 | for date in pd_range] 231 | date_ranges_even = list(zip(dates[::2], dates[1::2])) 232 | date_ranges_odd = list(zip(dates[1::2], dates[2:-2:2])) 233 | 234 | 235 | date_ranges = sorted(date_ranges_even + date_ranges_odd, 236 | key=lambda x: x[0]) 237 | 238 | # Check if the instrument had any failures 239 | # instr = self.instr.split('/')[0] 240 | if self.instr in self.inactive_range.keys(): 241 | start_failure = self.inactive_range[self.instr][0] 242 | stop_failure = self.inactive_range[self.instr][1] 243 | keep = [] 244 | # Remove dates falling in the period of inactivity 245 | for range in date_ranges: 246 | 247 | if range[0] >= start_failure and range[1] <= stop_failure: 248 | continue 249 | keep.append(range) 250 | 251 | # Update the list of date intervals 252 | self.dates = array(keep) 253 | else: 254 | self.dates = array(date_ranges) 255 | 256 | def initialize_HDF5(self, chunks=4): 257 | """ Create the required hdf5 files that we will write to 258 | 259 | Each statistics stored in the HDF5 files will vary in size image to 260 | image. This makes it impossible to assign a single structure and 261 | utilize a chunked-dataset approach when writing to an HDF5 file. 262 | Additionally, because of the large number of datasets we require many 263 | open/close operations on each HDF5 to prevent loss of data. 264 | For example, if we were to keep our HDF5 file open during the entire 265 | pipeline and it faile 266 | 267 | """ 268 | hdf5_files = self.instr_cfg['hdf5_files'] 269 | new_flist = defaultdict(list) 270 | for key in hdf5_files.keys(): 271 | rel_path = hdf5_files[key] 272 | full_path = os.path.join(self.base, *rel_path.split('/')) 273 | if isinstance(chunks, str): 274 | fnew = full_path.replace('.hdf5', '_{}.hdf5'.format(chunks)) 275 | new_flist[key].append(fnew) 276 | continue 277 | i = 0 278 | while i < chunks: 279 | fnew = full_path.replace('.hdf5', '_{}.hdf5'.format(i + 1)) 280 | i += 1 281 | new_flist[key].append(fnew) 282 | 283 | for key in new_flist.keys(): 284 | for f in new_flist[key]: 285 | if not os.path.isdir(os.path.dirname(f)): 286 | os.makedirs(os.path.dirname(f), exist_ok=True) 287 | 288 | LOG.info( 289 | 'File structure: /{}'.format(self.cfg['grp_names'][key]) 290 | ) 291 | with h5py.File(f, 'w') as fobj: 292 | grp = fobj.create_group(self.cfg['grp_names'][key]) 293 | 294 | def get_processed_ranges(self): 295 | """ Get the previously processed date ranges 296 | 297 | This will be used to check if the date range in question has already 298 | been analyzed, if it has then it will be skipped and the next 299 | range will be tried. 300 | 301 | Parameters 302 | ---------- 303 | instr 304 | 305 | Returns 306 | ------- 307 | 308 | """ 309 | fname = os.path.join(self.base,'CONFIG', 310 | 'processed_dates_{}.txt'.format(self.instr)) 311 | try: 312 | with open(fname, 'r') as fobj: 313 | lines = fobj.readlines() 314 | dates = [line.strip('\n') for line in lines] 315 | except FileNotFoundError as e: 316 | LOG.warning("Found 0 previously processed date ranges") 317 | self.previously_analyzed = [] 318 | else: 319 | self.previously_analyzed = dates 320 | LOG.info( 321 | 'Found {} previously processed' 322 | ' date ranges'.format(len(self.previously_analyzed)) 323 | ) 324 | -------------------------------------------------------------------------------- /pipeline/prototyping_wfpc2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "%load_ext autoreload\n", 10 | "%autoreload 2\n", 11 | "%matplotlib notebook\n", 12 | "import glob\n", 13 | "from astropy.io import fits\n", 14 | "from astropy.convolution import Gaussian2DKernel\n", 15 | "from astropy.stats import gaussian_fwhm_to_sigma\n", 16 | "from astropy.stats import sigma_clipped_stats\n", 17 | "from astropy.visualization import ImageNormalize, LogStretch, ZScaleInterval\n", 18 | "import numpy as np\n", 19 | "\n", 20 | "\n", 21 | "from ComputeStats import ComputeStats\n", 22 | "from CosmicRayLabel import CosmicRayLabel\n", 23 | "import matplotlib.pyplot as plt\n", 24 | "\n", 25 | "from scipy import ndimage\n", 26 | "from matplotlib import colors\n", 27 | "from matplotlib.patches import Rectangle, Circle\n", 28 | "from photutils import detect_sources, detect_threshold\n", 29 | "from photutils import detection\n", 30 | "from photutils import source_properties, deblend_sources" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": null, 36 | "metadata": {}, 37 | "outputs": [], 38 | "source": [ 39 | " def get_data(fname):\n", 40 | " \"\"\" Grab the SCI extensions from fits file\n", 41 | " \"\"\"\n", 42 | " pc = ('sci', 1) # Chip 2\n", 43 | " wf2 = ('sci', 2) # Chip 1\n", 44 | " wf3 = ('sci', 3)\n", 45 | " wf4 = ('sci', 4)\n", 46 | " detector_data = []\n", 47 | " with fits.open(fname) as hdu:\n", 48 | " for ext in [pc, wf2, wf3, wf4]:\n", 49 | " try:\n", 50 | " ext = hdu.index_of(ext)\n", 51 | " ext_data = hdu[ext].data\n", 52 | " except KeyError:\n", 53 | " print('{1} is missing for {0}'.format(fname, ext))\n", 54 | " ext1 = None\n", 55 | " else:\n", 56 | " detector_data.append(ext_data)\n", 57 | " # If second ext is missing, only work with the first\n", 58 | " # Otherwise combine each DQ ext to make full-frame\n", 59 | "\n", 60 | " return detector_data\n", 61 | " " 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "metadata": {}, 68 | "outputs": [], 69 | "source": [ 70 | "def find_sources(fname, deblend=False):\n", 71 | " \"\"\" \n", 72 | " Generate segmentation map\n", 73 | " \"\"\"\n", 74 | " sci_data = get_data(fname)\n", 75 | " \n", 76 | " # Generate a detection threshold for the segmentation mapping, we want everything with SNR higher than 5\n", 77 | " threshold = detect_threshold(sci_data[0], snr=5.)\n", 78 | " \n", 79 | " # Generate a kernel for use in the segmentation mapping, normalize it's value to 1\n", 80 | " sigma = 1 * gaussian_fwhm_to_sigma # convert FWHM of 2.75 pix to sigma\n", 81 | " kernel = Gaussian2DKernel(sigma, x_size=3, y_size=3)\n", 82 | " kernel.normalize()\n", 83 | " data_list, segm_list = [], []\n", 84 | " for data in sci_data:\n", 85 | " # Find sources\n", 86 | " mean, median, std = sigma_clipped_stats(data, sigma_lower = 3, sigma_upper= 2)\n", 87 | " print('mean: {}, median: {}, std: {}'.format(mean, median, std))\n", 88 | " segm = detect_sources(data-median,\n", 89 | " threshold=median+5*std,\n", 90 | " npixels=4,\n", 91 | " filter_kernel=kernel,\n", 92 | " connectivity=8)\n", 93 | " \n", 94 | " \n", 95 | " if deblend:\n", 96 | " # Deblened sources\n", 97 | " print('deblending')\n", 98 | " try:\n", 99 | " segm_deblend = deblend_sources(sci_data[0],\n", 100 | " segm.data,\n", 101 | " npixels=4,\n", 102 | " nlevels=32,\n", 103 | " filter_kernel=kernel,\n", 104 | " contrast=0.1,\n", 105 | " connectivity=8,\n", 106 | " )\n", 107 | " except ValueError:\n", 108 | " pass\n", 109 | " else:\n", 110 | " segm = segm_deblend\n", 111 | "# return sci_data[0], segm_deblend\n", 112 | " segm_list.append(segm)\n", 113 | " data_list.append(data)\n", 114 | " return data_list, segm_list\n", 115 | "\n" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": null, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [ 124 | "flist = glob.glob('../crrejtab/WFPC2/mastDownload/HST/*/*c0m.fits')" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": null, 130 | "metadata": {}, 131 | "outputs": [], 132 | "source": [ 133 | "\n", 134 | "f_1000 = []\n", 135 | "t_exptime = []\n", 136 | "for f in flist:\n", 137 | " t = fits.getval(f, 'exptime')\n", 138 | " if t > 1000:\n", 139 | " f_1000.append(f)\n", 140 | " t_exptime.append(t)" 141 | ] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": null, 146 | "metadata": {}, 147 | "outputs": [], 148 | "source": [ 149 | "fname1 = f_1000[0]\n", 150 | "fname2 = f_1000[1]" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": null, 156 | "metadata": {}, 157 | "outputs": [], 158 | "source": [ 159 | "data1_list, segm1_list = find_sources(fname1, deblend=False)" 160 | ] 161 | }, 162 | { 163 | "cell_type": "code", 164 | "execution_count": null, 165 | "metadata": {}, 166 | "outputs": [], 167 | "source": [ 168 | "data2_list, segm2_list = find_sources(fname2, deblend=False)" 169 | ] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "execution_count": null, 174 | "metadata": {}, 175 | "outputs": [], 176 | "source": [ 177 | "high_val = np.where(data1_list[0] > 4.160646625864344 + 5*1.1595283054779255, 1, 0)" 178 | ] 179 | }, 180 | { 181 | "cell_type": "code", 182 | "execution_count": null, 183 | "metadata": {}, 184 | "outputs": [], 185 | "source": [ 186 | "high_val.shape" 187 | ] 188 | }, 189 | { 190 | "cell_type": "code", 191 | "execution_count": null, 192 | "metadata": {}, 193 | "outputs": [], 194 | "source": [ 195 | "cmap = colors.ListedColormap(['black', 'white'])" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": null, 201 | "metadata": {}, 202 | "outputs": [], 203 | "source": [ 204 | "threshold=2" 205 | ] 206 | }, 207 | { 208 | "cell_type": "code", 209 | "execution_count": null, 210 | "metadata": {}, 211 | "outputs": [], 212 | "source": [ 213 | "label, num_feat = ndimage.label(high_val,structure=np.ones((3,3)))\n", 214 | "print('Found {} objects'.format(num_feat))\n", 215 | "cr_labels = label.ravel() # Returns a flattened label\n", 216 | "# Count up the number of pixels associated with each unique label\n", 217 | "sizes = np.bincount(cr_labels)\n", 218 | "arg_max = np.argmax(sizes)\n", 219 | "sizes[arg_max] = 0\n", 220 | "large_CRs = sizes > threshold\n", 221 | "\n", 222 | "# Create a 2-D mask from the 1-D array of large cosmic rays, and set all\n", 223 | "# labels of cosmic rays smaller than threshold to 0 so they are ignored.\n", 224 | "label_mask = large_CRs[label]\n", 225 | "high_val[~label_mask] = 0\n", 226 | "label, num_feat = ndimage.label(high_val,\n", 227 | " structure=np.ones((3,3)))\n", 228 | "print('After thresholding there are {} objects'.format(num_feat))" 229 | ] 230 | }, 231 | { 232 | "cell_type": "code", 233 | "execution_count": null, 234 | "metadata": {}, 235 | "outputs": [], 236 | "source": [ 237 | "fig, axes = plt.subplots(nrows=2, ncols=2, sharex=True, sharey=True, figsize=(6,6))\n", 238 | "norm = ImageNormalize(data1_list[0], stretch=LogStretch(), vmin=0, vmax=500)\n", 239 | "\n", 240 | "axes[0, 0].imshow(data1_list[0], norm=norm, cmap='gray', origin='lower')\n", 241 | "# axes[0, 1].imshow(segm1_list[0].data, origin='lower', cmap=segm1_list[0].cmap(random_state=1245))\n", 242 | "axes[0, 1].imshow(data1_list[1], norm=norm, cmap='gray', origin='lower')\n", 243 | "\n", 244 | "axes[1, 0].imshow(data1_list[2], norm=norm, cmap='gray', origin='lower')\n", 245 | "# axes[1, 1].imshow(high_val, origin='lower', cmap=cmap)\n", 246 | "axes[1, 1].imshow(data1_list[3], norm=norm, cmap='gray', origin='lower')\n", 247 | "axes[0, 0].set_xlim(460, 470)\n", 248 | "axes[0, 0].set_ylim(415, 430)" 249 | ] 250 | }, 251 | { 252 | "cell_type": "code", 253 | "execution_count": null, 254 | "metadata": {}, 255 | "outputs": [], 256 | "source": [ 257 | "len(np.unique(segm1_list[0].data))/t_exptime[0]" 258 | ] 259 | }, 260 | { 261 | "cell_type": "code", 262 | "execution_count": null, 263 | "metadata": {}, 264 | "outputs": [], 265 | "source": [ 266 | "len(np.unique(label))/t_exptime[0]" 267 | ] 268 | }, 269 | { 270 | "cell_type": "code", 271 | "execution_count": null, 272 | "metadata": {}, 273 | "outputs": [], 274 | "source": [ 275 | "from ComputeStats import ComputeStats\n", 276 | "from CosmicRayLabel import CosmicRayLabel" 277 | ] 278 | }, 279 | { 280 | "cell_type": "code", 281 | "execution_count": null, 282 | "metadata": {}, 283 | "outputs": [], 284 | "source": [ 285 | "label_obj = CosmicRayLabel(fname1)\n", 286 | "label_obj.label_wfpc2_data()" 287 | ] 288 | }, 289 | { 290 | "cell_type": "code", 291 | "execution_count": null, 292 | "metadata": {}, 293 | "outputs": [], 294 | "source": [ 295 | "stats_obj = ComputeStats(fname1, label, label_obj.sci[1], label_obj.integration_time)" 296 | ] 297 | }, 298 | { 299 | "cell_type": "code", 300 | "execution_count": null, 301 | "metadata": {}, 302 | "outputs": [], 303 | "source": [ 304 | "affected_tmp, rate_tmp, sizes_tmp, shapes_tmp, deposition_tmp = stats_obj.compute_stats(threshold=True)" 305 | ] 306 | }, 307 | { 308 | "cell_type": "code", 309 | "execution_count": null, 310 | "metadata": {}, 311 | "outputs": [], 312 | "source": [ 313 | "fig, axes = plt.subplots(nrows=2, ncols=2, sharex=True, sharey=True, figsize=(6,6))\n", 314 | "norm = ImageNormalize(label_obj.sci[1], stretch=LogStretch(), vmin=0, vmax=500)\n", 315 | "\n", 316 | "axes[0, 0].imshow(label_obj.sci[1], norm=norm, cmap='gray', origin='lower')\n", 317 | "axes[0, 1].imshow(label, origin='lower', cmap=segm1_list[1].cmap(random_state=1245))\n", 318 | "\n", 319 | "axes[1, 0].imshow(data1_list[1], norm=norm, cmap='gray', origin='lower')\n", 320 | "axes[1, 1].imshow(segm1_list[1].data, origin='lower', cmap=segm1_list[1].cmap(random_state=1245))" 321 | ] 322 | }, 323 | { 324 | "cell_type": "code", 325 | "execution_count": null, 326 | "metadata": {}, 327 | "outputs": [], 328 | "source": [] 329 | } 330 | ], 331 | "metadata": { 332 | "kernelspec": { 333 | "display_name": "Python 3", 334 | "language": "python", 335 | "name": "python3" 336 | }, 337 | "language_info": { 338 | "codemirror_mode": { 339 | "name": "ipython", 340 | "version": 3 341 | }, 342 | "file_extension": ".py", 343 | "mimetype": "text/x-python", 344 | "name": "python", 345 | "nbconvert_exporter": "python", 346 | "pygments_lexer": "ipython3", 347 | "version": "3.6.6" 348 | } 349 | }, 350 | "nbformat": 4, 351 | "nbformat_minor": 2 352 | } 353 | -------------------------------------------------------------------------------- /pipeline/utils/compute_directionality.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Objectives for this script: 4 | 5 | 1) Generate labels for cosmic rays 6 | 2) Compute the centroids of the cosmic rays 7 | 3) Compute the centroids of the labels 8 | 4) Generate 20 x 20 pixel cutouts showing the labels, the SCI data, 9 | and the two centroids 10 | 11 | """ 12 | import os 13 | import sys 14 | sys.path.append('/Users/nmiles/hst_cosmic_rays/pipeline') 15 | 16 | from astropy.table import Table 17 | 18 | from astropy.visualization import ImageNormalize, LinearStretch, \ 19 | ZScaleInterval, SqrtStretch, LogStretch 20 | 21 | 22 | import matplotlib as mpl 23 | # mpl.use('Qt5Agg') 24 | from matplotlib import colors 25 | import matplotlib.pyplot as plt 26 | import matplotlib.patches as patch 27 | 28 | from matplotlib.backends.backend_pdf import PdfPages 29 | from mpl_toolkits.axes_grid1 import make_axes_locatable 30 | import matplotlib.gridspec as gridspec 31 | 32 | 33 | from label import labeler 34 | from stat_utils import statshandler 35 | 36 | import numpy as np 37 | import scipy.ndimage as ndimage 38 | 39 | 40 | def save_to_pdf(data1, sources1, data2, sources2, dirname=None): 41 | """ Save the results of the matched stars to a multipage pdf 42 | 43 | Using the common sources, save 40 by 40 pixel cutouts of each matched 44 | source. There will be a total of 16 cutouts per page, that are split 45 | into two groups of 8. Within each group of 8, the matching sources will 46 | be displayed column-wise. The top row of 4 stars will all be distinct 47 | and their matches will be displayed in the row immediately below. 48 | 49 | Parameters 50 | ---------- 51 | data1 : numpy.ndarray 52 | FITS data from image 1 53 | sources1 : astropy.table.Table 54 | Catalog of matched sources found in image 1 55 | data2 : numpy.ndarray 56 | Label for all cosmic rays in data1 57 | sources2 : astropy.table.Table 58 | Catalog of matches sources found in image 2 59 | dirname : 60 | Path to the directory containing the images that were analyzed 61 | 62 | Returns 63 | ------- 64 | None 65 | This function will save a pdf to the results directory containing 66 | the matched stars used in the analysis 67 | """ 68 | 69 | ncolors = np.max(data2) + 1 70 | prng = np.random.RandomState(1234) 71 | h = prng.uniform(low=0.0, high=1.0, size=ncolors) 72 | s = prng.uniform(low=0.2, high=0.7, size=ncolors) 73 | v = prng.uniform(low=0.5, high=1.0, size=ncolors) 74 | hsv = np.dstack((h, s, v)) 75 | 76 | rgb = np.squeeze(colors.hsv_to_rgb(hsv)) 77 | rgb[0] = (0, 0, 0) 78 | cmap = colors.ListedColormap(rgb) 79 | 80 | mk_patch = lambda xy, r, c, lw, fill: patch.Circle(xy=xy, 81 | radius=r, 82 | color=c, 83 | fill=fill, 84 | lw=lw) 85 | norm = ImageNormalize(data1, 86 | stretch=LogStretch(), 87 | vmin=0, 88 | vmax=1000) 89 | # interval=ZScaleInterval()) 90 | 91 | outname = 'centroid_comparison.pdf' 92 | print('Total number of sources {}'.format(len(sources1))) 93 | with PdfPages(outname) as pdf: 94 | num_pages = int(np.ceil(len(sources1) / 16)) 95 | # num_pages = 20 96 | start_idx = 0 97 | for i in range(num_pages): 98 | start_idx += 8 99 | sources1_to_plot = sources1[start_idx: start_idx + 8] 100 | sources2_to_plot = sources2[start_idx: start_idx + 8] 101 | # Initalize the plot, each axes list will contain 8 plots 102 | # The plots should be organize by columns, i.e. two plots in 103 | # the same column correspond to the same image 104 | fig, axes00, axes10 = mk_grid() 105 | 106 | 107 | 108 | # i will run from 0 to 15 (i.e. 16 elements) 109 | for j in range(len(sources1_to_plot)): 110 | cr1 = sources1_to_plot[j] 111 | cr2 = sources2_to_plot[j] 112 | # limits for the first star 113 | # print(j) 114 | if j < 4: 115 | # print(j%4) 116 | ax1 = axes00[j] 117 | else: 118 | # print(j%4) 119 | ax1 = axes10[j % 4] 120 | 121 | cutout_size = 40 122 | # Limits for the first plot 123 | xlimits1 = cr1['xcenter'] - cutout_size / 2, \ 124 | cr1['xcenter'] + cutout_size / 2 125 | ylimits1 = cr1['ycenter'] - cutout_size / 2, \ 126 | cr1['ycenter'] + cutout_size / 2 127 | 128 | if j < 4: 129 | # print(j%4 + 4) 130 | ax2 = axes00[j % 4 + 4] 131 | else: 132 | # print(j%4 + 4) 133 | ax2 = axes10[j % 4 + 4] 134 | 135 | # Limits for the second plot 136 | xlimits2 = cr2['xcenter'] - cutout_size/2, \ 137 | cr2['xcenter'] + cutout_size/2 138 | ylimits2 = cr2['ycenter'] - cutout_size/2, \ 139 | cr2['ycenter'] + cutout_size/2 140 | 141 | 142 | 143 | im1 = ax1.imshow(data1, norm=norm, cmap='gray', origin='lower') 144 | im2 = ax2.imshow(data2, cmap=cmap, origin='lower') 145 | 146 | # Add axes for color bar to show scale 147 | divider = make_axes_locatable(ax1) 148 | cax1 = divider.append_axes("right", size="8%", pad=0.05) 149 | 150 | 151 | cbar = fig.colorbar(im1, cax=cax1) 152 | n = len(cbar.ax.get_yticklabels()) 153 | 154 | labels_to_hide = [n-3, n - 2] 155 | loop = zip(cbar.ax.get_yticklabels(), 156 | cbar.ax.yaxis.get_major_ticks()) 157 | for i, (label, tick) in enumerate(loop): 158 | if i in labels_to_hide: 159 | label.set_visible(False) 160 | tick.set_visible(False) 161 | 162 | cbar.ax.set_yticklabels(cbar.ax.get_yticklabels(), 163 | rotation=10, 164 | horizontalalignment='left', 165 | verticalalignment='center', 166 | fontsize=5 167 | ) 168 | cbar.update_ticks() 169 | 170 | 171 | 172 | flux_max_patch = mk_patch((cr1['xmax'], cr1['ymax']), 173 | r=0.5, 174 | c='magenta', 175 | lw=1, 176 | fill=True) 177 | ax1.add_patch(flux_max_patch) 178 | 179 | flux_center_patch = mk_patch((cr1['xcenter'], cr1['ycenter']), 180 | r=0.5, 181 | c='red', 182 | lw=1., 183 | fill=True) 184 | ax1.add_patch(flux_center_patch) 185 | geo_center_patch = mk_patch((cr2['xcenter'], cr2['ycenter']), 186 | r=0.5, 187 | c='blue', 188 | lw=1., 189 | fill=True) 190 | 191 | ax1.add_patch(geo_center_patch) 192 | 193 | current_label_patch = mk_patch((cr2['xcenter'], cr2['ycenter']), 194 | r=4, 195 | c='white', 196 | lw=1.5, 197 | fill=False) 198 | 199 | 200 | ax2.add_patch(current_label_patch) 201 | 202 | ax1.set_title('Red: flux-weight\n ' 203 | 'Blue: uniform-weight \n' 204 | 'Magneta: max value', 205 | fontsize='medium') 206 | ax2.set_title('Label', fontsize='medium') 207 | 208 | # Set the plot limits for star 1 209 | ax1.set_xlim(xlimits1[0], xlimits1[1]) 210 | ax1.set_ylim(ylimits1[0], ylimits1[1]) 211 | 212 | # Set the plot limits for star 2 213 | ax2.set_xlim(xlimits2[0], xlimits2[1]) 214 | ax2.set_ylim(ylimits2[0], ylimits2[1]) 215 | 216 | ax1.grid(False) 217 | ax2.grid(False) 218 | # add colorbar to outer grid 219 | 220 | pdf.savefig(fig) 221 | plt.close() 222 | 223 | 224 | def mk_grid(): 225 | """Convenience function for making a grid of axes to plot 16 images cuts 226 | 227 | Returns 228 | ------- 229 | 230 | """ 231 | fig = plt.figure(figsize=(10, 8)) 232 | gs0 = gridspec.GridSpec(nrows=2, ncols=1, hspace=0.45) 233 | gs00 = gridspec.GridSpecFromSubplotSpec(nrows=2, 234 | ncols=4, 235 | subplot_spec=gs0[0], 236 | wspace=0.35, hspace=0.5) 237 | 238 | gs01 = gridspec.GridSpecFromSubplotSpec(nrows=2, 239 | ncols=4, subplot_spec=gs0[1], 240 | wspace=0.35, hspace=0.5) 241 | 242 | axes00 = [] 243 | axes10 = [] 244 | for i in range(2): 245 | for j in range(4): 246 | axes00.append(fig.add_subplot(gs00[i, j])) 247 | axes10.append(fig.add_subplot(gs01[i, j])) 248 | return fig, axes00, axes10 249 | 250 | 251 | def main(): 252 | fname = '/Users/nmiles/hst_cosmic_rays/crrejtab/ACS/mastDownload/HST/j8ba20foq/j8ba20foq_flt.fits' 253 | 254 | instr='ACS_WFC' 255 | cr_label = labeler.CosmicRayLabel(fname=fname) 256 | cr_label.get_data(ext='dq') 257 | cr_label.get_data(ext='sci') 258 | cr_label.get_label() 259 | fig, ax1, ax2 = cr_label.plot(show=False) 260 | 261 | stats_obj = statshandler.ComputeStats(fname=fname, 262 | instr=instr, 263 | label=cr_label.label, 264 | sci=cr_label.sci, 265 | integration_time=1000.0) 266 | 267 | 268 | 269 | # Get the sizes of each cosmic ray 270 | cr_sizes = stats_obj.compute_size() 271 | sizes_in_sig = list(cr_sizes.values()) 272 | sizes_in_sig = np.asarray(sizes_in_sig) 273 | large_crs_sig = np.where(sizes_in_sig > 0.75) 274 | 275 | # Get the size in pixels of each cosmic ray 276 | cr_label_ids = cr_label.label.ravel() # Returns a flattened label 277 | 278 | # get the unique labels 279 | cr_label_idx = np.unique(cr_label_ids)[1:] 280 | 281 | # Count up the number of pixels associated with each unique label 282 | sizes_in_pixels = np.bincount(cr_label_ids)[1:] 283 | # arg_max = np.argmax(sizes_in_pixels) 284 | # largest object is the background, set it to 0 285 | # sizes_in_pixels[arg_max] = 0 286 | large_crs_pix = sizes_in_pixels > 30 287 | 288 | # Compute the position where the most energy is deposited 289 | maxima = ndimage.maximum_position(cr_label.sci, 290 | labels=cr_label.label, 291 | index=cr_label_idx) 292 | maxima = np.asarray(maxima) 293 | 294 | # for idx in cr_label_idx: 295 | # maxima.append(ndimage.maximum_position(cr_label.sci, 296 | # labels=cr_label.label, 297 | # index=idx)) 298 | 299 | 300 | # Compute the flux weighted centroids, 301 | # Format is ([col1, row1], ...,[col2, row2]) 302 | flux_weighted_centroid = stats_obj.compute_first_moment() 303 | 304 | flux_weighted_centroid = flux_weighted_centroid[large_crs_pix] 305 | maxima = maxima[large_crs_pix] 306 | 307 | 308 | 309 | flux_weight_table = Table([flux_weighted_centroid[:, 1], 310 | flux_weighted_centroid[:, 0], 311 | maxima[:, 1], 312 | maxima[:, 0]], 313 | names=['xcenter','ycenter', 'xmax','ymax']) 314 | 315 | 316 | # Compute the uniformly weighted centroids (i.e. geometric centers) 317 | uniform_weight = np.ones(shape=cr_label.sci.shape) 318 | 319 | geometric_centroid = stats_obj.compute_first_moment(sci=uniform_weight) 320 | geometric_centroid = geometric_centroid[large_crs_pix] 321 | 322 | uniform_weight_table = Table([geometric_centroid[:, 1], 323 | geometric_centroid[:, 0]], 324 | names=['xcenter', 'ycenter']) 325 | 326 | save_to_pdf(cr_label.sci, flux_weight_table, 327 | cr_label.label, uniform_weight_table) 328 | 329 | 330 | 331 | 332 | 333 | 334 | 335 | 336 | if __name__ == '__main__': 337 | main() -------------------------------------------------------------------------------- /notebooks/analyzing_cr_rejection.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 3, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "%matplotlib widget\n", 10 | "import glob\n", 11 | "import os\n", 12 | "from mpl_toolkits.axes_grid1 import make_axes_locatable\n", 13 | "\n", 14 | "from astropy.io import fits\n", 15 | "from astropy.stats import sigma_clipped_stats\n", 16 | "from astropy.visualization import ImageNormalize, SqrtStretch, LogStretch, LinearStretch, ZScaleInterval, ManualInterval\n", 17 | "import ipywidgets as widgets\n", 18 | "from ipywidgets import interact, fixed, interactive, VBox, HBox\n", 19 | "import matplotlib.colors as colors\n", 20 | "import matplotlib.pyplot as plt\n", 21 | "import matplotlib.patches as patches\n", 22 | "from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,\n", 23 | " AutoMinorLocator)\n", 24 | "from matplotlib import ticker\n", 25 | "# plt.style.use('dark_background')\n", 26 | "plt.style.use('ggplot')\n", 27 | "\n", 28 | "import numpy as np" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 4, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "_BASE = '/Users/nmiles/hst_cosmic_rays/analyzing_cr_rejection/'" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": 5, 43 | "metadata": {}, 44 | "outputs": [], 45 | "source": [ 46 | "img_to_compare = 'odvbbpa5q_all.fits' # random 60 seconds exposure\n", 47 | "exptime=60.0\n", 48 | "\n", 49 | "# img_to_compare = 'odvbbozpq_all.fits' # random 1100 second exposure\n", 50 | "# exptime=1100.0" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": 6, 56 | "metadata": {}, 57 | "outputs": [], 58 | "source": [ 59 | "param1 = 'med_4'\n", 60 | "img1 = os.path.join(_BASE,f'{exptime}_{param1}', img_to_compare)\n", 61 | "\n", 62 | "param2 = 'med_6.5,5.5,4.5'\n", 63 | "img2 = os.path.join(_BASE,f'{exptime}_{param2}', img_to_compare)\n", 64 | "\n", 65 | "param3 = 'min_6.5,5.5,4.5'\n", 66 | "img3 = os.path.join(_BASE,f'{exptime}_{param3}', img_to_compare)" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": 7, 72 | "metadata": {}, 73 | "outputs": [], 74 | "source": [ 75 | "def read_image(img):\n", 76 | " data = {}\n", 77 | " with fits.open(img) as hdu:\n", 78 | " data['sci'] = hdu[0].data\n", 79 | " data['dq'] = hdu[1].data\n", 80 | " data['crlabel'] = hdu[2].data\n", 81 | " return data" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": 8, 87 | "metadata": {}, 88 | "outputs": [], 89 | "source": [ 90 | "data1 = read_image(img1)\n", 91 | "data2 = read_image(img2)\n", 92 | "data3 = read_image(img3)\n", 93 | "mean, med, std = sigma_clipped_stats(data1['sci'], sigma=5)" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": 9, 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [ 102 | "norm = ImageNormalize(data1['sci'], stretch=SqrtStretch(), interval=ManualInterval(vmin=0, vmax=med+10*std))" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": 10, 108 | "metadata": {}, 109 | "outputs": [], 110 | "source": [ 111 | "def generate_cmap(ncolors):\n", 112 | "# ncolors = np.max(label) + 1\n", 113 | "# print('Generating colormap for label')\n", 114 | " prng = np.random.RandomState(1234)\n", 115 | " h = prng.uniform(low=0.0, high=1.0, size=ncolors)\n", 116 | " s = prng.uniform(low=0.2, high=0.7, size=ncolors)\n", 117 | " v = prng.uniform(low=0.5, high=1.0, size=ncolors)\n", 118 | " hsv = np.dstack((h, s, v))\n", 119 | "\n", 120 | " rgb = np.squeeze(colors.hsv_to_rgb(hsv))\n", 121 | " rgb[0] = (0,0,0)\n", 122 | " cmap = colors.ListedColormap(rgb)\n", 123 | " return cmap" 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": 11, 129 | "metadata": {}, 130 | "outputs": [], 131 | "source": [ 132 | "xmin_slider = widgets.IntText(\n", 133 | " options=[i for i in range(1,1025)],\n", 134 | " value=500,\n", 135 | " description='xmin',\n", 136 | " disabled=False,\n", 137 | " continuous_update=True,\n", 138 | " orientation='horizontal',\n", 139 | " readout=True \n", 140 | ")\n", 141 | "xmax_slider = widgets.IntText(\n", 142 | " options=[i for i in range(1,1025)],\n", 143 | " value= 540,\n", 144 | " description='xmax',\n", 145 | " disabled=False,\n", 146 | " continuous_update=True,\n", 147 | " orientation='horizontal',\n", 148 | " readout=True \n", 149 | ")\n", 150 | "\n", 151 | "ymin_slider = widgets.IntText(\n", 152 | " options=[i for i in range(1,1025)],\n", 153 | " value=500,\n", 154 | " description='ymin',\n", 155 | " disabled=False,\n", 156 | " continuous_update=True,\n", 157 | " orientation='horizontal',\n", 158 | " readout=True \n", 159 | ")\n", 160 | "ymax_slider = widgets.IntText(\n", 161 | " options=[i for i in range(1,1025)],\n", 162 | " value= 540,\n", 163 | " description='ymax',\n", 164 | " disabled=False,\n", 165 | " continuous_update=True,\n", 166 | " orientation='horizontal',\n", 167 | " readout=True \n", 168 | ")\n", 169 | "l1 = widgets.link((xmin_slider, 'value'), (ymin_slider, 'value'))\n", 170 | "l2 = widgets.link((xmax_slider, 'value'), (ymax_slider, 'value'))" 171 | ] 172 | }, 173 | { 174 | "cell_type": "code", 175 | "execution_count": 12, 176 | "metadata": {}, 177 | "outputs": [], 178 | "source": [ 179 | "xmin_slider = widgets.IntSlider(\n", 180 | " min=0,\n", 181 | " max=1024,\n", 182 | " step=10,\n", 183 | " value=500,\n", 184 | " description='xmin',\n", 185 | " disabled=False,\n", 186 | " continuous_update=True,\n", 187 | " orientation='horizontal',\n", 188 | " readout=True \n", 189 | ")\n", 190 | "xmax_slider = widgets.IntSlider(\n", 191 | " min=0,\n", 192 | " max=1024,\n", 193 | " step=10,\n", 194 | " value=540,\n", 195 | " description='xmax',\n", 196 | " disabled=False,\n", 197 | " continuous_update=True,\n", 198 | " orientation='horizontal',\n", 199 | " readout=True \n", 200 | ")\n", 201 | "\n", 202 | "ymin_slider = widgets.IntSlider(\n", 203 | " min=0,\n", 204 | " max=1024,\n", 205 | " step=10,\n", 206 | " value=500,\n", 207 | " description='ymin',\n", 208 | " disabled=False,\n", 209 | " continuous_update=True,\n", 210 | " orientation='horizontal',\n", 211 | " readout=True \n", 212 | ")\n", 213 | "ymax_slider = widgets.IntSlider(\n", 214 | " min=0,\n", 215 | " max=1024,\n", 216 | " step=10,\n", 217 | " value=540,\n", 218 | " description='ymax',\n", 219 | " disabled=False,\n", 220 | " continuous_update=True,\n", 221 | " orientation='horizontal',\n", 222 | " readout=True \n", 223 | ")\n", 224 | "\n", 225 | "l1 = widgets.link((xmin_slider, 'value'), (ymin_slider, 'value'))\n", 226 | "l2 = widgets.link((xmax_slider, 'value'), (ymax_slider, 'value'))" 227 | ] 228 | }, 229 | { 230 | "cell_type": "code", 231 | "execution_count": 13, 232 | "metadata": {}, 233 | "outputs": [], 234 | "source": [ 235 | "def plot_results(\n", 236 | " axes=None,\n", 237 | " data1=None, \n", 238 | " data2=None, \n", 239 | " data3=None,\n", 240 | " figsize=(10,6),\n", 241 | " norm=None,\n", 242 | " param1=None,\n", 243 | " param2=None,\n", 244 | " param3=None,\n", 245 | " xmin=None,\n", 246 | " xmax=None,\n", 247 | " ymin=None,\n", 248 | " ymax=None\n", 249 | "):\n", 250 | "# fig, axes = plt.subplots(nrows=3, ncols=3, sharex=True, sharey=True, gridspec_kw={'wspace':0.01, 'hspace':0.15})\n", 251 | " first_col = axes[:, 0]\n", 252 | " second_col = axes[:, 1]\n", 253 | " third_col = axes[:, 2]\n", 254 | " cbar_bounds = [0, 8, 16, 32, 64, 256, 1024, 4096, 8192]\n", 255 | " dq_cmap = plt.cm.inferno\n", 256 | " dq_norm = colors.BoundaryNorm(boundaries=cbar_bounds,\n", 257 | " ncolors=dq_cmap.N)\n", 258 | " fig.canvas.layout.width = f'{figsize[0]}in'\n", 259 | " fig.canvas.layout.height= f'{figsize[1]}in'\n", 260 | " for ax, key in zip(first_col, data1.keys()):\n", 261 | " ax.grid(False)\n", 262 | " ax.set_xlim((xmin, xmax))\n", 263 | " ax.set_ylim((ymin, ymax))\n", 264 | " if key == 'sci':\n", 265 | " ax.set_title(f'SCI {param1}')\n", 266 | " ax.imshow(data1[key], norm=norm, cmap='gray', origin='lower')\n", 267 | " elif key == 'dq':\n", 268 | " ax.set_title('DQ')\n", 269 | " im = ax.imshow(data1[key], cmap='inferno', norm=dq_norm, origin='lower')\n", 270 | " elif key == 'crlabel':\n", 271 | " ax.set_title('CR LABEL')\n", 272 | " cmap_label1 = generate_cmap(np.max(data1[key]) + 1)\n", 273 | " ax.imshow(data1[key], cmap=cmap_label1, origin='lower')\n", 274 | " \n", 275 | " for ax, key in zip(second_col, data2.keys()):\n", 276 | " ax.grid(False)\n", 277 | " if key == 'sci':\n", 278 | " ax.set_title(f'SCI {param2}')\n", 279 | " ax.imshow(data2[key], norm=norm, cmap='gray', origin='lower')\n", 280 | " elif key == 'dq':\n", 281 | " ax.set_title('DQ')\n", 282 | " ax.imshow(data2[key], cmap='inferno', norm=dq_norm, origin='lower')\n", 283 | " \n", 284 | " elif key == 'crlabel':\n", 285 | " ax.set_title('CR LABEL')\n", 286 | " cmap_label2 = generate_cmap(np.max(data2[key]) + 1)\n", 287 | " ax.imshow(data2[key], cmap=cmap_label2, origin='lower')\n", 288 | " \n", 289 | " for ax, key in zip(third_col, data3.keys()):\n", 290 | " ax.grid(False)\n", 291 | " if key == 'sci':\n", 292 | " ax.set_title(f'SCI {param3}')\n", 293 | " sci_im = ax.imshow(data3[key], norm=norm, cmap='gray', origin='lower')\n", 294 | " sci_divider = make_axes_locatable(ax)\n", 295 | " sci_cax = sci_divider.append_axes(\"right\", size=\"5%\", pad=0.05)\n", 296 | " sci_cbar = fig.colorbar(sci_im, cax=sci_cax)\n", 297 | " sci_cbar.set_label(f\"SCI Value (COUNTS)\")\n", 298 | " elif key == 'dq':\n", 299 | " ax.set_title('DQ')\n", 300 | " im = ax.imshow(data3[key], cmap='inferno', norm=dq_norm, origin='lower')\n", 301 | " divider = make_axes_locatable(ax)\n", 302 | " cax = divider.append_axes(\"right\", size=\"5%\", pad=0.05)\n", 303 | " cbar = fig.colorbar(im, cax=cax)\n", 304 | " cbar.set_label('DQ Value')\n", 305 | " elif key == 'crlabel':\n", 306 | " ax.set_title('CR LABEL')\n", 307 | " cmap_label3 = generate_cmap(np.max(data3[key]) + 1)\n", 308 | " ax.imshow(data3[key], cmap=cmap_label3, origin='lower')\n", 309 | " \n", 310 | " \n", 311 | " \n", 312 | "# divider = make_axes_locatable(ax)\n", 313 | "# cax = divider.append_axes(\"right\", size=\"5%\", pad=0.05)\n", 314 | "# cbar = fig.colorbar(im, cax=cax)\n", 315 | "# cbar.set_label(f\"{units}\")\n", 316 | "# ax.set_xlim(xlim)\n", 317 | "# ax.set_ylim(ylim)\n", 318 | "# ax.set_title(title)\n", 319 | " plt.show()" 320 | ] 321 | }, 322 | { 323 | "cell_type": "code", 324 | "execution_count": 14, 325 | "metadata": {}, 326 | "outputs": [], 327 | "source": [ 328 | "out = widgets.Output(layout={'border': '1px solid black'})" 329 | ] 330 | }, 331 | { 332 | "cell_type": "code", 333 | "execution_count": 15, 334 | "metadata": {}, 335 | "outputs": [], 336 | "source": [ 337 | "with out:\n", 338 | " fig, axes = plt.subplots(nrows=3, ncols=3, sharex=True, sharey=True, gridspec_kw={'wspace':0.01, 'hspace':0.15})\n", 339 | " w = interactive(plot_results,axes=fixed(axes),\n", 340 | " data1=fixed(data1), param1=fixed(param1),\n", 341 | " data2=fixed(data2), param2=fixed(param2), \n", 342 | " data3=fixed(data3), param3=fixed(param3),\n", 343 | " norm=fixed(norm), figsize=fixed((12,10)), xmin=xmin_slider, xmax=xmax_slider, ymin=ymin_slider, ymax=ymax_slider)\n", 344 | " display(w)" 345 | ] 346 | }, 347 | { 348 | "cell_type": "code", 349 | "execution_count": 16, 350 | "metadata": {}, 351 | "outputs": [ 352 | { 353 | "data": { 354 | "application/vnd.jupyter.widget-view+json": { 355 | "model_id": "f5342439b5914364affde045a48e5c4f", 356 | "version_major": 2, 357 | "version_minor": 0 358 | }, 359 | "text/plain": [ 360 | "Output(layout=Layout(border='1px solid black'), outputs=({'output_type': 'display_data', 'data': {'text/plain'…" 361 | ] 362 | }, 363 | "metadata": {}, 364 | "output_type": "display_data" 365 | } 366 | ], 367 | "source": [ 368 | "out" 369 | ] 370 | }, 371 | { 372 | "cell_type": "code", 373 | "execution_count": 15, 374 | "metadata": {}, 375 | "outputs": [], 376 | "source": [ 377 | "out.clear_output()\n", 378 | "plt.close('all')" 379 | ] 380 | }, 381 | { 382 | "cell_type": "code", 383 | "execution_count": null, 384 | "metadata": {}, 385 | "outputs": [], 386 | "source": [] 387 | }, 388 | { 389 | "cell_type": "code", 390 | "execution_count": null, 391 | "metadata": {}, 392 | "outputs": [], 393 | "source": [] 394 | } 395 | ], 396 | "metadata": { 397 | "kernelspec": { 398 | "display_name": "Python 3", 399 | "language": "python", 400 | "name": "python3" 401 | }, 402 | "language_info": { 403 | "codemirror_mode": { 404 | "name": "ipython", 405 | "version": 3 406 | }, 407 | "file_extension": ".py", 408 | "mimetype": "text/x-python", 409 | "name": "python", 410 | "nbconvert_exporter": "python", 411 | "pygments_lexer": "ipython3", 412 | "version": "3.6.10" 413 | } 414 | }, 415 | "nbformat": 4, 416 | "nbformat_minor": 4 417 | } 418 | -------------------------------------------------------------------------------- /pipeline/utils/agu_plots.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import glob 4 | from astropy.time import Time 5 | from astropy.io import fits 6 | from astropy.stats import sigma_clipped_stats 7 | from astropy.convolution import convolve, convolve_fft, Box2DKernel 8 | from astropy.visualization import LinearStretch, ZScaleInterval,\ 9 | AsinhStretch, SqrtStretch, ImageNormalize 10 | import costools 11 | from itertools import chain 12 | import numpy as np 13 | 14 | from matplotlib import rc 15 | # rc('font',**{'family':'sans-serif','sans-serif':['Helvetica']}) 16 | ## for Palatino and other serif fonts use: 17 | #rc('font',**{'family':'serif','serif':['Palatino']}) 18 | # rc('text', usetex=True) 19 | import matplotlib.pyplot as plt 20 | from mpl_toolkits.axes_grid1 import make_axes_locatable 21 | from mpl_toolkits.basemap import Basemap 22 | import pandas as pd 23 | import plot_stats as ps 24 | from PIL import Image 25 | 26 | from sunpy.timeseries import TimeSeries 27 | from scipy.ndimage import median_filter, gaussian_filter 28 | 29 | 30 | 31 | def read_cr_data(): 32 | flist_wfc = glob.glob('./../data/ACS/acs_cr_rate_?.hdf5') 33 | flist_hrc = glob.glob('./../data/ACS/acs_hrc_cr_rate_?.hdf5') 34 | flist_wfpc2 = glob.glob('./../data/WFPC2/wfpc2_cr_rate_?.hdf5') 35 | flist_wfc3 = glob.glob('./../data/WFC3/wfc3_cr_rate_?.hdf5') 36 | flist_stis = glob.glob('./../data/STIS/stis_cr_rate.hdf5') 37 | 38 | ccd_data_in = {'ACS_WFC':flist_wfc, 39 | 'ACS_HRC':flist_hrc, 40 | 'WFPC2':flist_wfpc2, 41 | 'WFC3_UVIS': flist_wfc3, 42 | 'STIS_CCD':flist_stis 43 | } 44 | data_obj = {} 45 | for key in ccd_data_in.keys(): 46 | print(key) 47 | data_obj[key] = ps.PlotData(fname=None, 48 | flist=ccd_data_in[key], 49 | instr=key, 50 | subgrp='incident_cr_rate') 51 | data_obj[key].read_rate() 52 | return data_obj 53 | 54 | 55 | def exptime_comparison(instrument_data): 56 | exptime_data = {} 57 | for key in instrument_data.keys(): 58 | exptime_data[key] = instrument_data[key].data_df['exptime'] 59 | df = pd.DataFrame(exptime_data) 60 | sampled = df.resample(rule='120D').sum() 61 | fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(9,7)) 62 | 63 | CB_color_cycle = ['#377eb8', '#ff7f00', '#4daf4a', 64 | '#f781bf', '#a65628', '#984ea3', 65 | '#999999', '#e41a1c', '#dede00'] 66 | 67 | for i, col in enumerate(sampled.columns): 68 | no_nan = sampled[col].dropna() 69 | 70 | ax.semilogy(no_nan.index.values, 71 | no_nan,drawstyle='steps-mid', 72 | label='{}'.format(col.replace('_','/')), 73 | color=CB_color_cycle[i], fillstyle='full') 74 | print(col) 75 | ax.fill_between(no_nan.index.values, 76 | no_nan, 77 | 0, 78 | step='mid', 79 | color=CB_color_cycle[i]) 80 | ax.legend(loc='best') 81 | 82 | ax.set_ylabel('Cumulative Exposure Time [s]') 83 | ax.set_title('Temporal Coverage of Dark Exposures') 84 | ax.set_ylim(1e3, 1e7) 85 | fig.savefig('cumulative_exptime.png', format='png', dpi=350) 86 | plt.show() 87 | 88 | return no_nan 89 | 90 | 91 | def ccd_substrate_model(): 92 | fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(4, 8)) 93 | ax.set_facecolor('white') 94 | ax.get_xaxis().set_visible(False) 95 | ax.get_yaxis().set_visible(False) 96 | 97 | ax.text(1, 17,'ACS/WFC CCD Substrate Layers', fontsize='x-large') 98 | 99 | 100 | 101 | # top layer 102 | text_x = 8 103 | ax.text(6, 12, 'CCD Housing Environment', fontsize='large') 104 | 105 | arrow_w=0.05 106 | 107 | ax.text(0, 15.2,'Incoming Light') 108 | ax.arrow(2, 15, 0.55, -4.75, 109 | color='k',shape='full', width=arrow_w) 110 | # reflected 111 | ax.arrow(2+0.7, 15-4.95, 0.55, 3, 112 | color='k', linestyle='dashed', width=arrow_w) 113 | ax.plot([2.65,2.65],[10,15], ls='--', color='k') 114 | ax.axhline(10, xmin=0.1) 115 | 116 | # Si 117 | ax.arrow(2.65, 10., 0.53, -3.75, 118 | color='k', shape='full', width=arrow_w) 119 | # reflected 120 | ax.arrow(2.65 + 0.6, 10 - 3.95, 0.45, 1.75, 121 | color='k', linestyle='dashed', 122 | width=arrow_w) 123 | 124 | ax.plot([3.25, 3.25], [6, 9], ls='--', color='k') 125 | ax.text(text_x, 7.7, '$Si$', fontsize='large') 126 | ax.axhline(6, xmin=0.1) 127 | 128 | # SiO2 129 | ax.arrow(3.25, 6., 0.35, -1.75, color='k', shape='full', width=arrow_w) 130 | 131 | #reflected 132 | ax.arrow(3.24 + 0.45, 6 - 1.95, 0.3, 1., 133 | color='k', linestyle='dashed', 134 | width=arrow_w) 135 | ax.plot([3.68, 3.68], [4, 5.5], ls='--', color='k') 136 | ax.text(text_x, 4.8,r'$SiO_2$', fontsize='large') 137 | ax.axhline(4, xmin=0.1) 138 | 139 | # Si3N4 140 | ax.text(text_x, 2.8, r'$Si_{3}N_4$', fontsize='large') 141 | ax.arrow(3.7, 4., 0.45, -1.8, color='k', shape='full', width=arrow_w) 142 | # reflected 143 | ax.arrow(3.75 + 0.5, 4 - 1.95, 0.25, 1., 144 | color='k', linestyle='dashed', 145 | width=arrow_w) 146 | ax.plot([4.22, 4.22], [2, 3.5], ls='--', color='k') 147 | ax.axhline(2, xmin=0.1) 148 | 149 | # Si 150 | ax.text(text_x, 0.85, r'$Si$', fontsize='large') 151 | ax.arrow(4.25, 2, 0.55, -1.8, color='k', shape='full', width=arrow_w) 152 | # reflected 153 | ax.arrow(4.25 + 0.65, 2 - 1.9, 0.3, 1., 154 | color='k', linestyle='dashed', 155 | width=arrow_w) 156 | 157 | ax.plot([4.87, 4.87], [0, 1.5], ls='--', color='k') 158 | 159 | # Substrate 160 | ax.axhline(0, xmin=0.1) 161 | ax.text(text_x, -1, r'$Substrate$', fontsize='large') 162 | 163 | ax.axhline(-1.75, xmin=0.1) 164 | 165 | ax.grid(False) 166 | ax.set_xlim(-1,12) 167 | ax.set_ylim(-2,15.5) 168 | fig.savefig('ccd_substrate_example.png', 169 | format='png', 170 | dpi=350, 171 | bbox_inches='tight', 172 | transparent=False, 173 | frameon=False) 174 | # plt.show() 175 | # rc(useTex=False) 176 | 177 | 178 | 179 | def thickness_plot(fname=None, fname_comp=None, fout=None, instr=None): 180 | """ 181 | 182 | Parameters 183 | ---------- 184 | fname 185 | fname_comp 186 | 187 | Returns 188 | ------- 189 | 190 | """ 191 | rc('text', usetex=True) 192 | if 'fits' in fname_comp: 193 | comp_data = fits.getdata(fname_comp) 194 | astrofits=True 195 | else: 196 | comp_data = Image.open(fname_comp) 197 | astrofits=False 198 | 199 | with fits.open(fname) as hdu: 200 | data = hdu[0].data 201 | 202 | # mean, median, std = sigma_clipped_stats(data, sigma=3.0) 203 | # smoothed = median_filter(data, size=3) 204 | smoothed = gaussian_filter(data, sigma=2) 205 | # norm = ImageNormalize(data, 206 | # stretch=LinearStretch(), 207 | # vmin=np.min(data), vmax=np.max(data)) 208 | 209 | uvis = (140, 240) 210 | wfc = (50, 110) 211 | hrc = (130,200) 212 | norm = ImageNormalize(data, 213 | stretch=LinearStretch(), 214 | vmin=wfc[0], vmax=wfc[1]) 215 | fig1, ax1 = plt.subplots(nrows=1, ncols=1, figsize=(5,4)) 216 | fig2, ax2 = plt.subplots(nrows=1, ncols=1, figsize=(5,4)) 217 | im1 = ax1.imshow(smoothed, norm=norm, cmap='plasma', origin='lower') 218 | 219 | # Add a colorbar to show the image scaling 220 | divider1 = make_axes_locatable(ax1) 221 | cax1 = divider1.append_axes('bottom', size='5%', pad=0.1) 222 | cbar1 = fig1.colorbar(im1, cax=cax1, orientation='horizontal') 223 | cbar1.ax.set_xticklabels(cbar1.ax.get_xticklabels(), rotation=45) 224 | cbar1.set_label('Cosmic Ray Strikes') 225 | if not astrofits: 226 | norm1 = ImageNormalize(comp_data, 227 | stretch=LinearStretch(), 228 | interval=ZScaleInterval()) 229 | im2 = ax2.imshow(comp_data, norm=norm1, cmap='plasma') 230 | else: 231 | norm1 = ImageNormalize(comp_data, 232 | stretch=LinearStretch(), 233 | vmin=12.5, vmax=16) 234 | im2 = ax2.imshow(comp_data, cmap='plasma', norm=norm1)#, origin='lower') 235 | # Add a colorbar to show the image scaling 236 | divider2 = make_axes_locatable(ax2) 237 | cax2 = divider2.append_axes('bottom', size='5%', pad=0.1) 238 | cbar2 = fig2.colorbar(im2, cax=cax2, orientation='horizontal') 239 | cbar2.ax.set_xticklabels(cbar2.ax.get_xticklabels(), rotation=45) 240 | cbar2.set_label(r'Thickness $[\mu m]$') 241 | ax1.grid(False) 242 | ax2.grid(False) 243 | ax1.set_title('WFC Cosmic Ray Incidence Heat Map') 244 | ax2.set_title('WFC Fringing Thickness Map') 245 | ax1.get_xaxis().set_visible(False) 246 | ax1.get_yaxis().set_visible(False) 247 | ax2.get_xaxis().set_visible(False) 248 | ax2.get_yaxis().set_visible(False) 249 | # fig.suptitle(instr, 250 | # x=0.5, y=0.9, 251 | # horizontalalignment='center', 252 | # size=16, weight='bold') 253 | fig1.savefig('cr_heat_map_WFC.png', 254 | transparent=True, format='png', dpi=350, bbox_inches='tight') 255 | fig2.savefig('thickness_heat_map_WFC.png', transparent=True,format='png', dpi=350, bbox_inches='tight') 256 | plt.show() 257 | 258 | def read_solar_data(): 259 | noaa = TimeSeries('./../data/RecentIndices.txt', source='NOAAIndices') 260 | df = noaa.to_dataframe() 261 | return df 262 | 263 | 264 | def stis_saa_plot(data_df=None, i=5): 265 | 266 | 267 | stis = data_df['1997-02-01':'1997-02-28'] 268 | 269 | saa_eastern = (39.0, -30.0) # lon/lat 270 | saa_western = (267.0, -20.0) 271 | saa_northern = (312.0, 1.0) 272 | saa_southern = (300.0,-60.0) 273 | 274 | mask = (stis['latitude'] < saa_northern[1]) #& (stis['incident_cr_rate'] < 20) 275 | 276 | stis_saa_cut = stis[mask] 277 | fig, (ax1, ax2) = plt.subplots(nrows=2, ncols=1, figsize=(9,8)) 278 | 279 | # Create the lat/lon map 280 | m = Basemap(projection='cyl',llcrnrlon=-120, 281 | llcrnrlat= -60, 282 | urcrnrlon= 60, 283 | urcrnrlat= 10, 284 | ax=ax1) 285 | 286 | m.shadedrelief(scale=0.2) 287 | 288 | # lats and longs are returned as a dictionary 289 | lats = m.drawparallels(np.linspace(-90, 90, 13), 290 | labels=[True, True, False, False]) 291 | 292 | lons = m.drawmeridians(np.linspace(-180, 180, 13), 293 | labels=[False, False, False, True]) 294 | 295 | # keys contain the plt.Line2D instances 296 | lat_lines = chain(*(tup[1][0] for tup in lats.items())) 297 | lon_lines = chain(*(tup[1][0] for tup in lons.items())) 298 | all_lines = chain(lat_lines, lon_lines) 299 | # cycle through these lines and set the desired style 300 | for line in all_lines: 301 | line.set(linestyle='-', alpha=0.3, color='w') 302 | 303 | saa = [list(t) for t in zip(*costools.saamodel.saaModel(i))] 304 | # Ensure the polygon representing the SAA is a closed curve by adding 305 | # the starting points to the end of the list of lat/lon coords 306 | saa[0].append(saa[0][0]) 307 | saa[1].append(saa[1][0]) 308 | m.plot(saa[1], saa[0], 309 | c='r', 310 | latlon=True, 311 | label='SAA contour {}'.format(i)) 312 | 313 | hst_lon, hst_lat = stis_saa_cut['longitude'], stis_saa_cut['latitude'] 314 | shifted_lon = [] 315 | for i, lon in enumerate(hst_lon): 316 | if lon > 180.0: 317 | shifted_lon.append(lon - 360.0) 318 | else: 319 | shifted_lon.append(lon) 320 | x_coord, y_coord = m(shifted_lon, hst_lat) 321 | labels = [k for k in range(len(stis_saa_cut))] 322 | indices = [] 323 | for j, (lon, lat, label) in enumerate(zip(hst_lon, hst_lat, labels)): 324 | if j >= 4: 325 | indices.append(j) 326 | m.scatter(lon, lat, 327 | marker='o', s=10,c='r', 328 | latlon=True) 329 | 330 | ax1.annotate('{}'.format(j-4), 331 | xy=(x_coord[j], y_coord[j]), 332 | xycoords='data') 333 | 334 | ax2.scatter([k - 4 for k in indices], 335 | stis_saa_cut['incident_cr_rate'][indices]) 336 | ax1.legend(loc='best') 337 | ax2.set_xticks([k - 4 for k in indices]) 338 | ax2.set_ylabel('Cosmic Ray Flux [CR/s/cm^2]') 339 | ax2.set_xlabel('Observation Number') 340 | fig.savefig('stis_saa_crossing.png', format='png', dpi=350) 341 | plt.show() 342 | # with open('stis_saa_darks.txt', 'a') as fobj: 343 | # for indx in indices: 344 | # fobj.write('{}\n'.format(stis_saa_cut['date'][indx])) 345 | 346 | 347 | 348 | def get_solar_min_and_max(noaa_data): 349 | solar_cycle = {'Cycle 23': None, 'Cycle 24':None} 350 | min_1996 = noaa_data['1993-01-01':'1997-01-01'].idxmin() 351 | max_2000 = noaa_data['1998-01-01':'2004-01-01'].idxmax() 352 | min_2009 = noaa_data['2008-01-01':'2011-01-01'].idxmin() 353 | max_2014 = noaa_data['2011-01-01':'2017-01-01'].idxmax() 354 | 355 | solar_cycle['Cycle 23'] = (min_1996['sunspot RI smooth'], 356 | max_2000['sunspot RI smooth']) 357 | 358 | solar_cycle['Cycle 24'] = (min_2009['sunspot RI smooth'], 359 | max_2014['sunspot RI smooth']) 360 | return solar_cycle 361 | 362 | 363 | def plot(cr_data, noaa_data): 364 | grid = plt.GridSpec(2, 1, wspace=0.1, hspace=0.25) 365 | fig = plt.figure(figsize=(9, 10)) 366 | ax1 = fig.add_subplot(grid[0, 0]) 367 | ax2 = fig.add_subplot(grid[1, 0], sharex=ax1) 368 | 369 | for i, key in enumerate(cr_data.keys()): 370 | # if key == 'WFPC2': 371 | # continue 372 | mask = cr_data[key].perform_SAA_cut() 373 | cr_data[key].plot_rate_vs_time(window='120D', 374 | min_periods=100, 375 | ax=ax1, 376 | i=i, saa_exclude=False) 377 | ax2.plot(noaa_data.index.values, noaa_data['sunspot RI'], label='Daily RI Sunspot Number') 378 | ax2.plot(noaa_data.index.values, 379 | noaa_data['sunspot RI smooth'], 380 | label='Smoothed RI Sunspot Number') 381 | 382 | solar_cycle = get_solar_min_and_max(noaa_data) 383 | print(solar_cycle) 384 | ax1_legend = ax1.legend(loc='best', 385 | ncol=3, 386 | labelspacing=0.2, 387 | columnspacing=0.5) 388 | for i in range(len(ax1_legend.legendHandles)): 389 | ax1_legend.legendHandles[i]._sizes = [30] 390 | 391 | for cycle in solar_cycle.keys(): 392 | # Min 393 | ax1.axvline(solar_cycle[cycle][0], ls='--', color='k') 394 | ax2.axvline(solar_cycle[cycle][0],ls='--', color='k') 395 | 396 | # Max 397 | ax1.axvline(solar_cycle[cycle][1], ls='--', color='k') 398 | ax2.axvline(solar_cycle[cycle][1], ls='--', color='k') 399 | 400 | ax2_legend = ax2.legend(loc='best') 401 | date_min = Time('1991-12-01', format='iso') 402 | date_max = Time('2019-01-01', format='iso') 403 | ax1.set_xlim((date_min.to_datetime(), date_max.to_datetime())) 404 | ax2.set_title('International Sunspot Number') 405 | ax2.set_ylabel('Number of Sunspots') 406 | fig.savefig('cr_rate_vs_time.png',format='png',dpi=350) 407 | plt.show() 408 | 409 | def main(): 410 | data = read_cr_data() 411 | noaa_data = read_solar_data() 412 | plot(data, noaa_data) 413 | 414 | if __name__ == '__main__': 415 | main() --------------------------------------------------------------------------------