├── Code ├── __init__.py ├── DIA_CPU_header.py ├── DIA_GPU_header.py ├── Makefile ├── io_functions.py ├── KMT.py ├── data_structures.py ├── calibration_functions.py ├── cuda_interface_functions.py ├── image_functions.py ├── photometry_functions.py ├── detect.py ├── analysis_functions.py ├── DIA_common.py └── c_interface_functions.py ├── _config.yml ├── Documentation └── pyDIA-documentation.docx ├── README.md └── LICENSE /Code/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-cayman -------------------------------------------------------------------------------- /Code/DIA_CPU_header.py: -------------------------------------------------------------------------------- 1 | import c_interface_functions as CI 2 | -------------------------------------------------------------------------------- /Code/DIA_GPU_header.py: -------------------------------------------------------------------------------- 1 | import cuda_interface_functions as CI 2 | -------------------------------------------------------------------------------- /Documentation/pyDIA-documentation.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MichaelDAlbrow/pyDIA/HEAD/Documentation/pyDIA-documentation.docx -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pyDIA 2 | pyDIA is a modular python package for performing star detection, difference imaging and photometry on astronomical images. 3 | 4 | If you use this code, please cite 5 | 6 | DOI 7 | -------------------------------------------------------------------------------- /Code/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Makefile for pyDIA 3 | # 4 | 5 | targets : CPU GPU_single GPU_double 6 | 7 | CPU : DIA_CPU.py c_functions_dp.so 8 | 9 | GPU_double : DIA_GPU.py c_functions_dp.so 10 | rm -f cuda_functions.py 11 | ln -s cuda_functions_dp.py cuda_functions.py 12 | 13 | GPU_single : DIA_GPU.py c_functions_dp.so 14 | rm -f cuda_functions.py 15 | ln -s cuda_functions_sp.py cuda_functions.py 16 | 17 | DIA_CPU.py : DIA_common.py DIA_CPU_header.py 18 | cat DIA_CPU_header.py DIA_common.py > DIA_CPU.py 19 | 20 | DIA_GPU.py : DIA_common.py DIA_GPU_header.py 21 | cat DIA_GPU_header.py DIA_common.py > DIA_GPU.py 22 | 23 | c_functions_dp.so : c_functions_dp.c 24 | rm -f c_functions_dp.so 25 | gcc -fPIC -shared -o c_functions_dp.so c_functions_dp.c 26 | rm -f c_functions.so 27 | ln -s c_functions_dp.so c_functions.so 28 | 29 | clean : 30 | rm DIA_CPU.py DIA_GPU.py cuda_functions.py c_functions.so c_functions_sp.so c_functions_dp.so *.pyc 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 MichaelDAlbrow 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Code/io_functions.py: -------------------------------------------------------------------------------- 1 | import os 2 | from astropy.io import fits 3 | import numpy as np 4 | 5 | def get_date(file,key='JD'): 6 | target = file 7 | hdulist = fits.open(target) 8 | try: 9 | date = hdulist[0].header[key] 10 | except KeyError: 11 | print 'Error: No date found for file',file,'using FITS header keyword',key 12 | date = 2459999.0 13 | return date 14 | 15 | 16 | def read_fits_file(file,slice=None): 17 | if slice is not None: 18 | f = fits.open(file,memmap=True) 19 | data = f[0].section[slice[2]:slice[3],slice[0]:slice[1]] 20 | hdr = f[0].header 21 | else: 22 | data, hdr = fits.getdata(file, header=True) 23 | return np.float64(data), hdr 24 | 25 | 26 | def write_image(image,file): 27 | hdu = fits.PrimaryHDU(image.astype(np.float32)) 28 | try: 29 | hdu.writeto(file,overwrite=True) 30 | except IOError: 31 | print 'Warning - io_functions.write_image: could not write file',file 32 | pass 33 | 34 | def write_kernel_table(file,kernel_index,extended_basis,coeffs,params): 35 | if os.path.exists(file): 36 | os.remove(file) 37 | table1 = fits.TableHDU.from_columns([ \ 38 | fits.Column(name='x',format='I',array=kernel_index[:,0]), \ 39 | fits.Column(name='y',format='I',array=kernel_index[:,1]), \ 40 | fits.Column(name='extended',format='I5',array=extended_basis)]) 41 | table2 = fits.TableHDU.from_columns([ \ 42 | fits.Column(name='Spatial type',format='A4',array=np.array(['PDEG','SDEG','BDEG'])), \ 43 | fits.Column(name='degree',format='I',array=np.array([params.pdeg,params.sdeg,params.bdeg]))]) 44 | table3 = fits.TableHDU.from_columns( \ 45 | [fits.Column(name='Coefficients',format='E',array=coeffs)]) 46 | hdu = fits.PrimaryHDU() 47 | hdulist = fits.HDUList([hdu,table1,table2,table3]) 48 | hdulist.writeto(file) 49 | 50 | 51 | def read_kernel_table(file,params): 52 | hdulist = fits.open(file) 53 | t = hdulist[1].data 54 | k1 = t.field('x') 55 | k2 = t.field('y') 56 | extended_basis = t.field('extended') 57 | kernel_index = np.array([k1,k2]).T 58 | t = hdulist[2].data 59 | deg = t.field('degree') 60 | t = hdulist[3].data 61 | coeffs = t.field('Coefficients') 62 | if ((params.pdeg != deg[0]) or (params.sdeg != deg[1]) or 63 | (params.bdeg != deg[2])): 64 | print 'Warning: kernel degrees in',file,'do not match current parameters' 65 | params.pdeg = deg[0] 66 | params.sdeg = deg[1] 67 | params.bdeg = deg[2] 68 | return kernel_index, extended_basis, coeffs, params 69 | 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /Code/KMT.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import numpy as np 4 | 5 | import DIA_GPU as DIA 6 | import calibration_functions as cal 7 | import c_interface_functions as CF 8 | from astropy.io import fits 9 | 10 | def process_KMT_patch(site,coords=None,quality_max=1.25,mag_err_max=0.3,seeing_max=7.0,sky_max=10000,signal_min=125,name_pattern_has_site=True,date_header='MIDHJD', 11 | parameters=None,q_sigma_threshold=1.0,locate_date_range=None,locate_half_width=None,loc_data='RAW',RC_limit=16.0): 12 | 13 | params = DIA.DS.Parameters() 14 | params.gain = 1.5 15 | params.readnoise = 15.0 16 | params.pixel_max = 57000 17 | params.datekey = date_header 18 | params.use_GPU = True 19 | params.n_parallel = 1 20 | params.pdeg = 1 21 | params.sdeg = 1 22 | params.bdeg = 1 23 | params.reference_seeing_factor = 1.01 24 | params.reference_min_seeing = 1.0 25 | params.loc_data = loc_data 26 | params.fwhm_mult = 10 27 | params.iterations = 5 28 | 29 | params.bleed_mask_multiplier_above = 0 30 | params.bleed_mask_multiplier_below = 2.0 31 | 32 | 33 | if parameters is not None: 34 | for par, value in parameters.iteritems(): 35 | try: 36 | exec('params.'+par+' = '+str(value)) 37 | except NameError: 38 | exec('params.'+par+' = \''+str(value)+'\'') 39 | 40 | lightcurve_header=' Date Delta_Flux Err_Delta_Flux Mag Err_Mag Q FWHM Roundness Sky Signal' 41 | 42 | min_ref_images = {'I':20,'V':3} 43 | 44 | median_mag = {} 45 | median_mag_err = {} 46 | 47 | if os.path.exists(site+'-reg.fits'): 48 | params.registration_image = site+'-reg.fits' 49 | 50 | 51 | for band in ['I','V']: 52 | 53 | prefix = site+band 54 | params.loc_output = prefix 55 | 56 | params.ref_include_file = prefix+'-ref.list' 57 | 58 | if not(os.path.exists(params.ref_include_file)): 59 | params.ref_include_file = False 60 | 61 | photometric_zeropoint = 28.0 62 | if band == 'V': 63 | params.star_file = site+'I/ref.mags' 64 | params.star_reference_image = site+'I/ref.fits' 65 | params.registration_image = site+'I/ref.fits' 66 | photometric_zeropoint = 28.65 67 | 68 | if name_pattern_has_site: 69 | params.name_pattern = site+'*'+band+'*.fits' 70 | else: 71 | params.name_pattern = '*'+band+'*.fits' 72 | 73 | params.min_ref_images = min_ref_images[band] 74 | 75 | print params.loc_output 76 | print params.loc_data 77 | print params.name_pattern 78 | if not(os.path.exists(params.loc_output)): 79 | 80 | # Read the header of the first image to determine which direction the 81 | # detector has been read out. 82 | all_files = os.listdir(params.loc_data) 83 | hdr = fits.getheader(params.loc_data+'/'+all_files[0]) 84 | try: 85 | if hdr['ampname'][0] in ['K','N']: 86 | params.bleed_mask_multiplier_above = 0 87 | params.bleed_mask_multiplier_below = 2.0 88 | else: 89 | params.bleed_mask_multiplier_above = 2.0 90 | params.bleed_mask_multiplier_below = 0 91 | except: 92 | pass 93 | 94 | print 'starting imsub' 95 | DIA.imsub_all_fits(params) 96 | 97 | if not(os.path.exists(params.loc_output+'/calibration.png')): 98 | cal.calibrate(params.loc_output,ZP=photometric_zeropoint) 99 | 100 | if coords is not None: 101 | 102 | if band == 'I': 103 | 104 | x0, y0 = coords 105 | print 'Starting photometry for',prefix,'at', (x0,y0) 106 | dates, seeing, roundness, bgnd, signal, flux, dflux, quality, x0, y0 = \ 107 | CF.photom_variable_star(x0,y0,params,save_stamps=True,patch_half_width=20,locate_date_range=locate_date_range, 108 | locate_half_width=locate_half_width) 109 | print 'Converged to', (x0,y0) 110 | 111 | else: 112 | 113 | Imags = np.loadtxt(site+'I/ref.mags') 114 | Vmags = np.loadtxt(site+'V/ref.mags') 115 | x0 += np.median(Vmags[:,1]-Imags[:,1]) 116 | y0 += np.median(Vmags[:,2]-Imags[:,2]) 117 | print 'Starting photometry for',prefix,'at', (x0,y0) 118 | dates, seeing, roundness, bgnd, signal, flux, dflux, quality, x0, y0 = \ 119 | CF.photom_variable_star(x0,y0,params,save_stamps=True,patch_half_width=20,converge=False) 120 | print 'Converged to', (x0,y0) 121 | 122 | print 'Photometry for', site, band, 'at', x0, y0 123 | 124 | refmags = np.loadtxt(params.loc_output+'/ref.mags.calibrated') 125 | refflux = np.loadtxt(params.loc_output+'/ref.flux.calibrated') 126 | 127 | star_dist2 = (refmags[:,1]-x0)**2 + (refmags[:,2]-y0)**2 128 | star_num = np.argmin(star_dist2) 129 | 130 | print 'x0 y0:', x0, y0 131 | print 'Nearest star', star_num, 'located at', refmags[star_num,1], refmags[star_num,2] 132 | print 'Reference flux', refflux[star_num,:] 133 | print 'Reference mag', refmags[star_num,:] 134 | 135 | mag = photometric_zeropoint - 2.5*np.log10(refflux[star_num,0] + flux) 136 | mag_err = photometric_zeropoint - 2.5*np.log10(refflux[star_num,0] + flux - dflux) - mag 137 | 138 | np.savetxt(prefix+'-lightcurve.dat',np.vstack((dates,flux,dflux,mag,mag_err,quality,seeing,roundness,bgnd,signal)).T, \ 139 | fmt='%12.5f %12.4f %12.4f %7.4f % 7.4f %6.2f %6.2f %5.2f %10.2f %8.2f', \ 140 | header=lightcurve_header) 141 | 142 | if band == 'V': 143 | signal_min = 0.0 144 | 145 | q = np.where( (quality < quality_max) & (mag_err < mag_err_max) & (seeing < seeing_max) & (bgnd < sky_max) & (signal > signal_min) & \ 146 | (np.abs(flux) > 1.0e-6) )[0] 147 | 148 | for i in range(3): 149 | ldf = np.log10(dflux[q]) 150 | q = q[ldf > np.mean(ldf) - 4*np.std(ldf)] 151 | 152 | median_mag[band] = np.nanmedian(mag[q]) 153 | 154 | 155 | np.savetxt(prefix+'-lightcurve-filtered.dat',np.vstack((dates[q],flux[q],dflux[q], \ 156 | mag[q],mag_err[q],quality[q],seeing[q],roundness[q],bgnd[q],signal[q])).T, \ 157 | fmt='%12.5f %12.4f %12.4f %7.4f % 7.4f %6.2f %6.2f %5.2f %10.2f %8.2f', \ 158 | header=lightcurve_header) 159 | 160 | cal.plot_lightcurve(prefix+'-lightcurve.dat',plotfile=prefix+'-lightcurve.png') 161 | cal.plot_lightcurve(prefix+'-lightcurve-filtered.dat',plotfile=prefix+'-lightcurve-filtered.png') 162 | 163 | if site in ['A','SSOre']: 164 | return 165 | 166 | #RC = cal.makeCMD(site+'I',site+'V',RC_limit=16.0) 167 | 168 | VI, VI_err = cal.source_colour(site+'I-lightcurve-filtered.dat',site+'V-lightcurve-filtered.dat',plotfile=site+'-source-colour.png',VIoffset=0.65) 169 | 170 | cal.makeCMD(site+'I',site+'V',plot_density=True,IV=(median_mag['I'],median_mag['V']),source_colour=(VI[0],VI_err[0]),RC_limit=RC_limit) 171 | 172 | 173 | 174 | -------------------------------------------------------------------------------- /Code/data_structures.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import image_functions as IM 4 | import io_functions as IO 5 | 6 | # 7 | # Fundamental data structures 8 | # 9 | 10 | class EmptyBase(object): pass 11 | 12 | 13 | class Observation(object): 14 | """Container for all observation attributes""" 15 | 16 | def get_data(self): 17 | if not(isinstance(self._data,np.ndarray)): 18 | self._data, _ = IO.read_fits_file(self.fullname) 19 | if self._preconvolve_images: 20 | self._data = IM.convolve_gauss(self._data,self._preconvolve_FWHM) 21 | self.data_median = np.median(self._data) 22 | self.shape = self._data.shape 23 | return self._data 24 | 25 | def set_data(self,value): 26 | self._data = value 27 | 28 | def del_data(self): 29 | self._data = None 30 | 31 | data = property(get_data,set_data,del_data) 32 | 33 | 34 | def get_image(self): 35 | if not(isinstance(self._image,np.ndarray)): 36 | image_name = os.path.join(self.output_dir,'r_'+self.name) 37 | self._image, _ = IO.read_fits_file(image_name) 38 | return self._image 39 | 40 | def set_image(self,value): 41 | self._image = value 42 | image_name = os.path.join(self.output_dir,'r_'+self.name) 43 | IO.write_image(self._image,image_name) 44 | 45 | def del_image(self): 46 | self._image = None 47 | 48 | image = property(get_image,set_image,del_image) 49 | 50 | 51 | def get_mask(self): 52 | if not(isinstance(self._mask,np.ndarray)): 53 | mask_name = os.path.join(self.output_dir,'sm_'+self.name) 54 | self._mask, _ = IO.read_fits_file(mask_name) 55 | return self._mask 56 | 57 | def set_mask(self,value): 58 | self._mask = value 59 | mask_name = os.path.join(self.output_dir,'sm_'+self.name) 60 | IO.write_image(self._mask,mask_name) 61 | 62 | def del_mask(self): 63 | self._mask = None 64 | 65 | mask = property(get_mask,set_mask,del_mask) 66 | 67 | 68 | def get_inv_variance(self): 69 | if not(isinstance(self._inv_variance,np.ndarray)): 70 | inv_variance_name = os.path.join(self.output_dir,'iv_'+self.name) 71 | self._inv_variance, _ = IO.read_fits_file(inv_variance_name) 72 | return self._inv_variance 73 | 74 | def set_inv_variance(self,value): 75 | self._inv_variance = value 76 | inv_variance_name = os.path.join(self.output_dir,'iv_'+self.name) 77 | IO.write_image(self._inv_variance,inv_variance_name) 78 | 79 | def del_inv_variance(self): 80 | self._inv_variance = None 81 | 82 | inv_variance = property(get_inv_variance,set_inv_variance,del_inv_variance) 83 | 84 | 85 | def __init__(self,filename,params): 86 | self.fullname = filename 87 | self.name = os.path.basename(filename) 88 | self.output_dir = params.loc_output 89 | self._data = None 90 | self._image = None 91 | self._mask = None 92 | self._preconvolve_images = False 93 | if params.preconvolve_images: 94 | self._preconvolve_images = True 95 | self._preconvolve_FWHM = params.preconvolve_FWHM 96 | self.mask = IM.compute_saturated_pixel_mask(self.data,params) * \ 97 | IM.compute_bleed_mask2(self.data,params) 98 | if params.pixel_saturation_kernel is not None: 99 | self.mask *= IM.compute_kernel_saturation_mask(self.data,params) 100 | if params.error_image_prefix is None: 101 | self.inv_variance = 1.0/(self.data/params.gain + 102 | (params.readnoise/params.gain)**2) + self.mask 103 | else: 104 | d, _ = IO.read_fits_file(os.path.dirname(self.fullname)+'/'+params.error_image_prefix+self.name) 105 | self.inv_variance = 1.0/d**2 106 | if params.subtract_sky: 107 | self.data = IM.subtract_sky(self.data,params) 108 | self.fw, self.roundness, self.sky, self.signal = -1.0, -1.0, -1.0, -1.0 109 | if params.pixel_min < self.data_median < 0.5*params.pixel_max: 110 | self.fw, self.roundness, self.sky, self.signal = IM.compute_fwhm(self,params, 111 | seeing_file=params.loc_output+os.path.sep+'seeing') 112 | self.mask = 1-np.logical_or(1-self.mask,IM.convolve_disk(1-self.mask,3*self.fw)) 113 | del self.mask 114 | del self.inv_variance 115 | 116 | def register(self,reg,params): 117 | print self.name 118 | self._image, self._mask, self._inv_variance = IM.register(reg,self, 119 | params) 120 | if self._image is None: 121 | return False 122 | 123 | print 'registered', self.name 124 | 125 | rf = os.path.join(self.output_dir,'r_'+self.name) 126 | IO.write_image(self._image,rf) 127 | rf = os.path.join(self.output_dir,'sm_'+self.name) 128 | IO.write_image(self._mask,rf) 129 | rf = os.path.join(self.output_dir,'iv_'+self.name) 130 | IO.write_image(self._inv_variance,rf) 131 | del self.mask 132 | del self.data 133 | del self.inv_variance 134 | 135 | return True 136 | 137 | 138 | class Parameters: 139 | """Container for parameters""" 140 | def __init__(self): 141 | self.bdeg = 0 142 | self.bleed_mask_multiplier_above = 0 143 | self.bleed_mask_multiplier_below = 0 144 | self.ccd_group_size = 100 145 | self.cluster_mask_radius = 50 146 | self.datekey = 'MJD-OBS' 147 | self.detect_threshold = 4.0 148 | self.diff_std_threshold = 10.0 149 | self.do_photometry = True 150 | self.error_image_prefix = None 151 | self.fft_kernel_threshold = 3.0 152 | self.fwhm_mult = 6.5 153 | self.fwhm_section = None 154 | self.gain = 1.0 155 | self.image_list_file = 'images' 156 | self.iterations = 1 157 | self.kernel_maximum_radius = 20.0 158 | self.kernel_minimum_radius = 5.0 159 | self.loc_data = '.' 160 | self.loc_output = '.' 161 | self.make_difference_images = True 162 | self.mask_cluster = False 163 | self.mask_radius = 5 164 | self.max_ref_images = 100 165 | self.min_ref_images = 3 166 | self.n_parallel = 1 167 | self.name_pattern = '*.fits' 168 | self.nstamps = 200 169 | self.pdeg = 0 170 | self.pixel_max = 50000 171 | self.pixel_min = 0.0 172 | self.pixel_rejection_threshold = 3.0 173 | self.pixel_saturation_kernel = None 174 | self.pixel_saturation_kernel_max = 50000 175 | self.preconvolve_images = False 176 | self.preconvolve_FWHM = 1.5 177 | self.psf_fit_radius = 3.0 178 | self.psf_max_radius = 10 179 | self.psf_profile_type = 'gaussian' 180 | self.readnoise = 1.0 181 | self.ref_image_list = 'ref.images' 182 | self.ref_include_file = None 183 | self.ref_exclude_file = None 184 | self.reference_min_seeing = 1.0 185 | self.reference_max_roundness = 1.3 186 | self.reference_seeing_factor = 1.01 187 | self.reference_sky_factor = 2.0 188 | self.register_using_masks = False 189 | self.registration_image = None 190 | self.registration_max_background = 2000 191 | self.sdeg = 0 192 | self.sky_degree = 0 193 | self.sky_subtract_mode = 'percent' 194 | self.sky_subtract_percent = 0.01 195 | self.stamp_edge_distance = 40 196 | self.stamp_half_width = 20 197 | self.star_detect_sigma = 12 198 | self.star_file = None 199 | self.star_file_has_magnitudes = False 200 | self.star_file_is_one_based = True 201 | self.star_file_number_match = 10000 202 | self.star_file_transform_degree = 2 203 | self.star_reference_image = None 204 | self.subtract_sky = False 205 | self.use_fft_kernel_pixels = False 206 | self.use_GPU = True 207 | self.use_stamps = False 208 | 209 | 210 | -------------------------------------------------------------------------------- /Code/calibration_functions.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import matplotlib 4 | import fnmatch 5 | matplotlib.use("Agg") 6 | import matplotlib.pyplot as plt 7 | import numpy as np 8 | from sklearn.neighbors import KernelDensity 9 | from sklearn.model_selection import GridSearchCV 10 | from sklearn.model_selection import LeaveOneOut 11 | from scipy.ndimage.filters import maximum_filter 12 | from skimage.feature import peak_local_max 13 | from scipy.odr import ODR, Model, RealData 14 | 15 | 16 | def locate_intercept(x,y,x_range): 17 | print 'locating offset' 18 | ix = np.arange(len(x)) 19 | p = np.where(x[ix]>2)[0] 20 | ix = ix[p] 21 | p = np.where(np.logical_and(x[ix] < np.percentile(x[ix],x_range[1]),x[ix] > np.percentile(x[ix],x_range[0])))[0] 22 | ix = ix[p] 23 | for n in range(10): 24 | a = np.nanmedian(y[ix]-x[ix]) 25 | print a, x.shape 26 | p = np.where(np.abs(y[ix]-a-x[ix])<2.5*np.nanstd(y[ix]-a-x[ix]))[0] 27 | ix = ix[p] 28 | return a, ix 29 | 30 | 31 | def calibrate(dir,plotfile='calibration.png',magnitude_range_fraction=(0.1,8),sky_flux_cutoff_percent=0.1,ZP=25): 32 | 33 | magfile = os.path.join(dir,'ref.mags') 34 | fluxfile = os.path.join(dir,'ref.flux') 35 | 36 | mag = np.loadtxt(magfile) 37 | flux = np.loadtxt(fluxfile) 38 | 39 | p = np.where((mag[:,3] > 0) & (flux[:,0] > 0))[0] 40 | 41 | #sky_max_flux = np.percentile(flux[p,0],sky_flux_cutoff_percent) 42 | #q = np.where(flux[p,0] < sky_max_flux)[0] 43 | #sky_flux = 0.9*np.mean(flux[p[q],0]) 44 | 45 | #q = np.where(mag[p,3] > np.percentile(mag[p,3],99.9))[0] 46 | #sky_flux = 0.95*np.median(flux[p[q],0]) 47 | #flux[:,0] -= sky_flux 48 | 49 | x = np.linspace(np.min(mag[p,3]),np.max(mag[p,3]),3) 50 | offset, stars = locate_intercept(mag[p,3],ZP-2.5*np.log10(flux[p,0]),magnitude_range_fraction) 51 | 52 | # Axes definitions 53 | nullfmt = plt.NullFormatter() 54 | rect_scatter = [0.15, 0.15, 0.7, 0.7] 55 | rect_histx = [0.15, 0.85, 0.7, 0.1] 56 | rect_histy = [0.85, 0.15, 0.1, 0.7] 57 | 58 | binsize = 0.5 59 | bandwidth = 0.25 60 | 61 | ymin, ymax = (offset-1,offset+1) 62 | binsize_y = 0.05 63 | 64 | fig = plt.figure() 65 | ax1 = fig.add_subplot(223, position=rect_scatter) 66 | ax1.scatter(mag[p,3],ZP-2.5*np.log10(flux[p,0])-mag[p,3],s=5,color='k') 67 | ax1.scatter(mag[p[stars],3],ZP-2.5*np.log10(flux[p[stars],0])-mag[p[stars],3],s=5,color='c') 68 | #ax.plot(x,x,'r--') 69 | ax1.plot(x,x*0.0+offset,'r',label=r'$\Delta mag = %4.2f$'%offset) 70 | ax1.grid() 71 | ax1.legend(loc='upper left') 72 | ax1.set_xlabel(r'$M_{\rm DAO}$',fontsize='14') 73 | ax1.set_ylabel(r'$%5.2f-2.5*\log_{10} F_{\rm P}-M_{\rm DAO}$'%ZP,fontsize='14') 74 | xmin, xmax = plt.xlim() 75 | xx = np.linspace(xmin,xmax,1000) 76 | 77 | ax2 = fig.add_subplot(221, position=rect_histx) 78 | hval, bins, _ = ax2.hist(mag[p,3],range=(xmin,xmax),bins=int((xmax-xmin)/binsize+1), 79 | normed=True,alpha=0.3) 80 | kde_skl = KernelDensity(kernel='epanechnikov',bandwidth=bandwidth) 81 | sample = mag[p,3] 82 | kde_skl.fit(sample[:, np.newaxis]) 83 | log_pdf = kde_skl.score_samples(xx[:, np.newaxis]) 84 | ax2.plot(xx,np.exp(log_pdf),'r') 85 | ax2.xaxis.set_major_formatter(nullfmt) 86 | ax2.yaxis.set_major_formatter(nullfmt) 87 | ax2.set_title(dir) 88 | 89 | ax3 = fig.add_subplot(221, position=rect_histy) 90 | ax3.hist(ZP-2.5*np.log10(flux[p,0])-mag[p,3],range=(ymin,ymax),bins=int((ymax-ymin)/binsize_y+1), 91 | orientation='horizontal',normed=True,alpha=0.3) 92 | ax3.xaxis.set_major_formatter(nullfmt) 93 | ax3.yaxis.set_major_formatter(nullfmt) 94 | 95 | ax1.set_ylim((ymin,ymax)) 96 | ax3.set_ylim((ymin,ymax)) 97 | plt.savefig(os.path.join(dir,plotfile)) 98 | 99 | ax1.set_ylim((offset-0.1,offset+0.1)) 100 | ax3.set_ylim((offset-0.1,offset+0.1)) 101 | plt.savefig(os.path.join(dir,'zoom-'+plotfile)) 102 | 103 | 104 | mag[:,3] += offset 105 | if mag.shape[1] == 4: 106 | np.savetxt(os.path.join(dir,'ref.mags.calibrated'),mag,fmt='%5d %8.3f %8.3f %7.4f') 107 | else: 108 | np.savetxt(os.path.join(dir,'ref.mags.calibrated'),mag,fmt='%5d %8.3f %8.3f %7.4f %7.4f %7.3f %7.3f %7.3f') 109 | 110 | cflux = 10.0**(0.4*(ZP-mag[:,3])) 111 | if mag.shape[1] == 4: 112 | cfluxerr = 0.0*cflux 113 | else: 114 | cfluxerr = cflux - 10.0**(0.4*(ZP-mag[:,3]-mag[:,4])) 115 | np.savetxt(os.path.join(dir,'ref.flux.calibrated'),np.vstack((cflux,cfluxerr)).T,fmt='%12.4f %12.4f') 116 | 117 | 118 | 119 | def makeCMD(dirI,dirV,bandwidth = 0.2,ifile=None,vfile=None,plot_density=True,RC=None,source_colour=None,xlabel=None,ylabel=None, 120 | IV=None,legend=True,title=True,plotfile=None,density_plotfile=None,I_limit=21,V_limit=21,RC_limit=18,optimize_bandwidth=False): 121 | 122 | if ifile is None: 123 | ifile = os.path.join(dirI,'ref.mags.calibrated') 124 | 125 | if vfile is None: 126 | vfile = os.path.join(dirV,'ref.mags.calibrated') 127 | 128 | if xlabel is None: 129 | xlabel = r'$(V-I)_{\rm P}$' 130 | if ylabel is None: 131 | ylabel = r'$I_{\rm p}$' 132 | 133 | 134 | im = np.loadtxt(ifile) 135 | vm = np.loadtxt(vfile) 136 | p = np.where((im[:,4] > 0) & (im[:,3] < I_limit) & (vm[:,4] > 0) & (vm[:,3] < V_limit) )[0] 137 | 138 | plt.figure() 139 | plt.scatter(vm[p,3]-im[p,3],im[p,3],s=3,c='k') 140 | plt.grid() 141 | plt.gca().invert_yaxis() 142 | plt.xlabel(xlabel,fontsize='14') 143 | plt.ylabel(ylabel,fontsize='14') 144 | if title: 145 | plt.title(dirI+' '+dirV) 146 | 147 | if RC is not None: 148 | plt.scatter(RC[0],RC[1],color='r',s=60,label='Red Clump (%6.3f,%6.3f)'%RC) 149 | 150 | # if source_pos is not None: 151 | # plt.errorbar(source_pos[0],source_pos[1],xerr=source_pos[2],yerr=source_pos[3],color='c') 152 | # plt.scatter(source_pos[0],source_pos[1],marker='o',s=80,facecolors='none', edgecolors='c') 153 | 154 | if IV is not None: 155 | plt.scatter(IV[1]-IV[0],IV[0],color='m',marker='.',s=40,label='Blended Source (%6.3f,%6.3f)'%(IV[1]-IV[0],IV[0])) 156 | 157 | if source_colour is not None: 158 | plt.scatter(np.nan,np.nan,color='w',marker='.',s=40,label='$(V-I)_S$ = %6.3f +/- %5.3f'%(source_colour[0],source_colour[1])) 159 | 160 | if source_colour is not None and RC is not None: 161 | #plt.scatter(np.nan,np.nan,color='w',marker='.',s=40,label='$\Delta (V-I)_S$ = %6.3f +/- %5.3f'%(source_colour[0]-RC[0],source_colour[1])) 162 | plt.scatter(np.nan,np.nan,color='w',marker='.',s=40,label='$(V-I)_{S,0}$ = %6.3f +/- %5.3f'%(source_colour[0]-RC[0]+1.06,source_colour[1])) 163 | 164 | if legend: 165 | plt.legend(loc='upper left') 166 | 167 | xmin, xmax = plt.xlim() 168 | ymax, ymin = plt.ylim() 169 | if plotfile is None: 170 | plotfile = dirI+'-'+dirV+'-CMD.png' 171 | plt.savefig(plotfile) 172 | plt.close() 173 | 174 | print xmin, xmax, ymin, ymax 175 | 176 | np.savetxt(dirI+'-'+dirV+'-CMDdata', 177 | np.vstack((im[p,0],im[p,1],im[p,2],vm[p,3],vm[p,4],im[p,3],im[p,4])).T, 178 | fmt='%6d %9.3f %9.3f %7.4f %7.4f %7.4f %7.4f', 179 | header='ID xpos ypos V V_err I I_err') 180 | 181 | red_clump = None 182 | 183 | if plot_density: 184 | 185 | prob = np.ones(im.shape[0]) 186 | Zmax = 1.0 187 | p = np.where((RC_limit+2 > im[:,3]) & (im[:,3] >= RC_limit))[0] 188 | VI_main_sequence = np.nanmedian(vm[p,3]-im[p,3]) 189 | p = np.where(vm[:,3]-im[:,3] < VI_main_sequence)[0] 190 | prob[p] = 0.0 191 | 192 | 193 | for iteration in range(10): 194 | 195 | p = np.where((im[:,4] > 0) & (vm[:,4] > 0) & (im[:,3] < RC_limit) & (vm[:,3] < V_limit) & (vm[:,3]-im[:,3] > VI_main_sequence) & (prob > 0.2*Zmax))[0] 196 | samples = np.vstack([vm[p,3]-im[p,3],im[p,3]]).T 197 | 198 | if optimize_bandwidth: 199 | bandwidths = np.linspace(0.1,0.6,101) 200 | grid = GridSearchCV(KernelDensity(kernel='gaussian'), 201 | {'bandwidth': bandwidths}, 202 | cv=LeaveOneOut()) 203 | grid.fit(samples) 204 | bandwidth = grid.best_params_['bandwidth'] 205 | print 'optimal bandwidth =',bandwidth 206 | 207 | kde_skl = KernelDensity(kernel='gaussian',bandwidth=bandwidth) 208 | kde_skl.fit(samples) 209 | # score_samples() returns the log-likelihood of the samples 210 | prob = np.exp(kde_skl.score_samples(np.vstack([vm[:,3]-im[:,3],im[:,3]]).T)) 211 | xvi = np.linspace(xmin,xmax,int(40*(xmax-xmin)+1)) 212 | xi = np.linspace(ymin,ymax,int(40*(ymax-ymin)+1)) 213 | Y, X = np.meshgrid(xvi, xi[::-1]) 214 | xy = np.vstack([Y.ravel(), X.ravel()]).T 215 | Z = np.exp(kde_skl.score_samples(xy)) 216 | Z = Z.reshape(X.shape) 217 | levels = np.linspace(0, Z.max(), 25) 218 | Zmax = np.max(np.max(Z)) 219 | mx = maximum_filter(Z,size=20) 220 | #lm = (Z == mx) * (Z > 0.01*Zmax) * (Z < 0.99*Zmax) 221 | lm = (Z == mx) * (Z > 0.01*Zmax) 222 | if np.sum(lm) > 0: 223 | nlm = np.nonzero(lm) 224 | max_nlm = np.argmax(Z[nlm]) 225 | local_maxima = np.nonzero(lm) 226 | i0 = local_maxima[0][max_nlm] 227 | i1 = local_maxima[1][max_nlm] 228 | red_clump = (float(Y[i0,i1]),float(X[i0,i1])) 229 | print Z[local_maxima]/Zmax 230 | print 'Red clump detected at',red_clump 231 | else: 232 | print 'Error detecting red clump' 233 | red_clump = None 234 | 235 | plt.figure() 236 | plt.contourf(Y, X, Z, levels=levels, cmap=plt.cm.Reds) 237 | if np.sum(lm) > 0: 238 | plt.scatter(vm[:,3]-im[:,3],im[:,3],s=3,c='k') 239 | plt.scatter(vm[p,3]-im[p,3],im[p,3],s=3,c='b') 240 | plt.scatter(red_clump[0],red_clump[1],color='c',marker='+',s=60,label='Red Clump (%6.3f,%6.3f)'%red_clump) 241 | if source_colour is not None: 242 | plt.scatter(np.nan,np.nan,color='w',marker='.',s=40,label='$(V-I)_S$ = %6.3f +/- %5.3f'%(source_colour[0],source_colour[1])) 243 | plt.scatter(np.nan,np.nan,color='w',marker='.',s=40,\ 244 | label='$(V-I)_{S,0}$ = %6.3f +/- %5.3f'%(source_colour[0]-red_clump[0]+1.06,source_colour[1])) 245 | if legend: 246 | plt.legend(loc='upper left') 247 | plt.grid() 248 | plt.gca().invert_yaxis() 249 | plt.legend(loc='upper left') 250 | plt.xlabel(xlabel,fontsize='14') 251 | plt.ylabel(ylabel,fontsize='14') 252 | plt.title(dirI+' '+dirV) 253 | plt.xlim((xmin,xmax)) 254 | plt.ylim((ymax,ymin)) 255 | if density_plotfile is None: 256 | density_plotfile = dirI+'-'+dirV+'-CMD-density.png' 257 | plt.savefig(density_plotfile) 258 | plt.close() 259 | 260 | return red_clump 261 | 262 | 263 | def source_colour(ifile,vfile,plotfile='source_colour.png',VIoffset=0.0): 264 | 265 | # Define a function (quadratic in our case) to fit the data with. 266 | def linear_func1(p, x): 267 | m, c = p 268 | return m*x + c 269 | 270 | Idata = np.loadtxt(ifile) 271 | Vdata = np.loadtxt(vfile) 272 | qI = np.where(Idata[:,5] < 1.0) 273 | qV = np.where(Vdata[:,5] < 1.0) 274 | 275 | Idata = Idata[qI] 276 | Vdata = Vdata[qV] 277 | 278 | intervals=[0.025,0.05,0.1,0.2] 279 | colour = [] 280 | delta_colour = [] 281 | 282 | plt.figure(figsize=(12,12)) 283 | 284 | for inter,interval in enumerate(intervals): 285 | 286 | start = np.floor(np.min(Idata[:,0])) 287 | end = np.ceil(np.max(Idata[:,0])) 288 | 289 | time = np.arange(start,end,interval) 290 | 291 | flux1 = np.zeros_like(time) + np.nan 292 | flux2 = np.zeros_like(time) + np.nan 293 | flux1_err = np.zeros_like(time) + np.nan 294 | flux2_err = np.zeros_like(time) + np.nan 295 | 296 | for i in range(len(time)): 297 | q = np.where(np.abs(Idata[:,0] - time[i]) < interval/2.0)[0] 298 | if q.any(): 299 | flux1[i] = np.sum(Idata[q,1]/Idata[q,2]**2) / np.sum(1.0/Idata[q,2]**2) 300 | flux1_err[i] = np.sqrt(1.0 / np.sum(1.0/Idata[q,2]**2)) 301 | 302 | p = np.where(np.abs(Vdata[:,0] - time[i]) < interval/2.0)[0] 303 | if p.any(): 304 | flux2[i] = np.sum(Vdata[p,1]/Vdata[p,2]**2) / np.sum(1.0/Vdata[p,2]**2) 305 | flux2_err[i] = np.sqrt(1.0 / np.sum(1.0/Vdata[p,2]**2)) 306 | 307 | plt.subplot(2,2,inter+1) 308 | plt.errorbar(flux1/1000.0,flux2/1000.0,xerr=flux1_err/1000.0,yerr=flux2_err/1000.0,fmt='.') 309 | plt.xlabel(r'$\delta F_I (000)$') 310 | plt.ylabel(r'$\delta F_V (000)$') 311 | plt.grid() 312 | 313 | 314 | # Create a model for fitting. 315 | linear_model = Model(linear_func1) 316 | 317 | good_data = np.where(np.logical_and(np.isfinite(flux1),np.isfinite(flux2)))[0] 318 | offset = np.mean(flux2[good_data]-flux1[good_data]) 319 | 320 | # Create a RealData object using our initiated data from above. 321 | data = RealData(flux1[good_data], flux2[good_data], sx=flux1_err[good_data], sy=flux2_err[good_data]) 322 | 323 | # Set up ODR with the model and data. 324 | odr = ODR(data, linear_model, beta0=[1.0, offset]) 325 | 326 | # Run the regression. 327 | out = odr.run() 328 | 329 | # Use the in-built pprint method to give us results. 330 | out.pprint() 331 | 332 | x1, x2 = plt.gca().get_xlim() 333 | x_fit = np.linspace(x1*1000,x2*1000, 1000) 334 | y_fit = linear_func1(out.beta, x_fit) 335 | 336 | plt.plot(x_fit/1000.0,y_fit/1000.0,'r-',label=r"$\delta F_V = %5.3f \delta F_I + %5.3f$"%(out.beta[0],out.beta[1])) 337 | 338 | colour.append(VIoffset-2.5*np.log10(out.beta[0])) 339 | 340 | delta_colour.append(VIoffset-2.5*np.log10(out.beta[0]-out.sd_beta[0]) - colour[inter]) 341 | 342 | plt.title(r'$\Delta t = %5.3f \quad (V-I)_S = %8.3f \pm %8.3f$'%(interval,colour[inter],delta_colour[inter])) 343 | plt.legend() 344 | 345 | plt.savefig(plotfile) 346 | 347 | return colour, delta_colour 348 | 349 | def plot_lightcurve(file, columns=(0,3,4),plotfile='lightcurve.png',grid_on=True): 350 | 351 | data = np.loadtxt(file) 352 | 353 | plt.figure(figsize=(8,5)) 354 | plt.errorbar(data[:,columns[0]],data[:,columns[1]],data[:,columns[2]],fmt='.') 355 | plt.gca().invert_yaxis() 356 | plt.xlabel(r'$HJD - 2450000$') 357 | plt.ylabel(r'$Magnitude$') 358 | if grid_on: 359 | plt.grid() 360 | plt.savefig(plotfile) 361 | 362 | -------------------------------------------------------------------------------- /Code/cuda_interface_functions.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pycuda.driver as cuda 3 | import pycuda.autoinit 4 | from cuda_functions import cu_matrix_kernel 5 | from image_functions import convolve_undersample 6 | import sys 7 | 8 | 9 | def numpy3d_to_array(np_array, allow_surface_bind=False, layered=True): 10 | 11 | d, h, w = np_array.shape 12 | 13 | descr = cuda.ArrayDescriptor3D() 14 | descr.width = w 15 | descr.height = h 16 | descr.depth = d 17 | descr.format = cuda.dtype_to_array_format(np_array.dtype) 18 | descr.num_channels = 1 19 | descr.flags = 0 20 | 21 | if allow_surface_bind: 22 | descr.flags = cuda.array3d_flags.SURFACE_LDST 23 | 24 | if layered: 25 | descr.flags = cuda.array3d_flags.ARRAY3D_LAYERED 26 | 27 | device_array = cuda.Array(descr) 28 | 29 | copy = cuda.Memcpy3D() 30 | copy.set_src_host(np_array) 31 | copy.set_dst_array(device_array) 32 | copy.width_in_bytes = copy.src_pitch = np_array.strides[1] 33 | copy.src_height = copy.height = h 34 | copy.depth = d 35 | 36 | copy() 37 | 38 | return device_array 39 | 40 | 41 | def array_to_numpy3d(cuda_array): 42 | 43 | descriptor = cuda_array.get_descriptor_3d() 44 | 45 | w = descriptor.width 46 | h = descriptor.height 47 | d = descriptor.depth 48 | 49 | shape = d, h, w 50 | 51 | dtype = array_format_to_dtype(descriptor.format) 52 | 53 | numpy_array=np.zeros(shape, dtype) 54 | 55 | copy = cuda.Memcpy3D() 56 | copy.set_src_array(cuda_array) 57 | copy.set_dst_host(numpy_array) 58 | 59 | itemsize = numpy_array.dtype.itemsize 60 | 61 | copy.width_in_bytes = copy.src_pitch = w*itemsize 62 | copy.src_height = copy.height = h 63 | copy.depth = d 64 | 65 | copy() 66 | 67 | return numpy_array 68 | 69 | 70 | def compute_matrix_and_vector_cuda(R,RB,T,Vinv,mask,kernelIndex,extendedBasis, 71 | kernelRadius,params,stamp_positions=None): 72 | 73 | # Import CUDA function to compute the matrix 74 | cu_compute_matrix = cu_matrix_kernel.get_function('cu_compute_matrix') 75 | cu_compute_vector = cu_matrix_kernel.get_function('cu_compute_vector') 76 | cu_compute_matrix_stamps = cu_matrix_kernel.get_function('cu_compute_matrix_stamps') 77 | cu_compute_vector_stamps = cu_matrix_kernel.get_function('cu_compute_vector_stamps') 78 | 79 | # Copy the reference, target and inverse variance images to 80 | # GPU texture memory 81 | RTV = np.array([R,RB,T,Vinv,mask]).astype(np.float32).copy() 82 | RTV_cuda = numpy3d_to_array(RTV) 83 | texref = cu_matrix_kernel.get_texref("tex") 84 | texref.set_array(RTV_cuda) 85 | texref.set_filter_mode(cuda.filter_mode.POINT) 86 | 87 | # Create a numpy array for matrix H 88 | dp = (params.pdeg+1)*(params.pdeg+2)/2 89 | ds = (params.sdeg+1)*(params.sdeg+2)/2 90 | db = (params.bdeg+1)*(params.bdeg+2)/2 91 | hs = (kernelIndex.shape[0]-1)*ds+dp+db 92 | 93 | H = np.zeros([hs,hs]).astype(np.float32).copy() 94 | V = np.zeros(hs).astype(np.float32).copy() 95 | 96 | # Fill the elements of H 97 | print hs,' * ',hs,' elements' 98 | blockDim = (256,1,1) 99 | gridDim = (hs,hs,1) 100 | k0 = kernelIndex[:,0].astype(np.int32).copy() 101 | k1 = kernelIndex[:,1].astype(np.int32).copy() 102 | if params.use_stamps: 103 | posx = np.float32(stamp_positions[:params.nstamps,0].copy()-1.0) 104 | posy = np.float32(stamp_positions[:params.nstamps,1].copy()-1.0) 105 | cu_compute_matrix_stamps(np.int32(params.pdeg), 106 | np.int32(params.sdeg), 107 | np.int32(params.bdeg), 108 | np.int32(R.shape[1]), 109 | np.int32(R.shape[0]), 110 | np.int32(params.nstamps), 111 | np.int32(params.stamp_half_width), 112 | cuda.In(posx), 113 | cuda.In(posy), 114 | cuda.In(k0), 115 | cuda.In(k1), 116 | cuda.In(extendedBasis), 117 | np.int32(kernelIndex.shape[0]), 118 | np.int32(kernelRadius), 119 | cuda.Out(H), 120 | block=blockDim,grid=gridDim,texrefs=[texref]) 121 | else: 122 | cu_compute_matrix(np.int32(params.pdeg),np.int32(params.sdeg), 123 | np.int32(params.bdeg), 124 | np.int32(R.shape[1]),np.int32(R.shape[0]), 125 | cuda.In(k0), 126 | cuda.In(k1), 127 | cuda.In(extendedBasis), 128 | np.int32(kernelIndex.shape[0]),np.int32(kernelRadius), 129 | cuda.Out(H),block=blockDim,grid=gridDim, 130 | texrefs=[texref]) 131 | 132 | # Fill the elements of V 133 | blockDim = (256,1,1) 134 | gridDim = (hs,1,1) 135 | if params.use_stamps: 136 | cu_compute_vector_stamps(np.int32(params.pdeg),np.int32(params.sdeg), 137 | np.int32(params.bdeg), 138 | np.int32(R.shape[1]),np.int32(R.shape[0]),np.int32(params.nstamps), 139 | np.int32(params.stamp_half_width), 140 | cuda.In(posx), 141 | cuda.In(posy), 142 | cuda.In(k0), 143 | cuda.In(k1), 144 | cuda.In(extendedBasis), 145 | np.int32(kernelIndex.shape[0]),np.int32(kernelRadius), 146 | cuda.Out(V),block=blockDim,grid=gridDim, 147 | texrefs=[texref]) 148 | else: 149 | cu_compute_vector(np.int32(params.pdeg),np.int32(params.sdeg), 150 | np.int32(params.bdeg), 151 | np.int32(R.shape[1]),np.int32(R.shape[0]), 152 | cuda.In(k0), 153 | cuda.In(k1), 154 | cuda.In(extendedBasis), 155 | np.int32(kernelIndex.shape[0]),np.int32(kernelRadius), 156 | cuda.Out(V),block=blockDim,grid=gridDim, 157 | texrefs=[texref]) 158 | return H, V, texref 159 | 160 | 161 | def compute_model_cuda(image_size,texref,c,kernelIndex,extendedBasis,params): 162 | 163 | # Import CUDA function to perform the convolution 164 | cu_compute_model = cu_matrix_kernel.get_function('cu_compute_model') 165 | 166 | # Create a numpy array for the model M 167 | M = np.zeros(image_size).astype(np.float32).copy() 168 | 169 | # Call the cuda function to perform the convolution 170 | blockDim = (256,1,1) 171 | gridDim = (image_size[1],image_size[0])+(1,) 172 | k0 = kernelIndex[:,0].astype(np.int32).copy() 173 | k1 = kernelIndex[:,1].astype(np.int32).copy() 174 | cu_compute_model(np.int32(params.pdeg),np.int32(params.sdeg), 175 | np.int32(params.bdeg),cuda.In(k0), 176 | cuda.In(k1), cuda.In(extendedBasis), 177 | np.int32(kernelIndex.shape[0]), 178 | cuda.In(c),cuda.Out(M), 179 | block=blockDim,grid=gridDim,texrefs=[texref]) 180 | return M 181 | 182 | 183 | def photom_all_stars(diff,inv_variance,positions,psf_image,c,kernelIndex, 184 | extendedBasis,kernelRadius,params, 185 | star_group_boundaries=None, 186 | detector_mean_positions_x=None,detector_mean_positions_y=None): 187 | 188 | from astropy.io import fits 189 | # Read the PSF 190 | psf,psf_hdr = fits.getdata(psf_image,0,header='true') 191 | print 'CIF psf_shape',psf.shape 192 | print 'CIF psf_sum = ',np.sum(psf) 193 | psf_height = psf_hdr['PSFHEIGH'] 194 | psf_x = psf_hdr['PSFX'] 195 | psf_y = psf_hdr['PSFY'] 196 | psf_size = psf.shape[1] 197 | psf_fit_rad = params.psf_fit_radius 198 | if params.psf_profile_type == 'gaussian': 199 | psf_sigma_x = psf_hdr['PAR1']*0.8493218 200 | psf_sigma_y = psf_hdr['PAR2']*0.8493218 201 | psf_parameters = np.array([psf_size,psf_height,psf_sigma_x,psf_sigma_y,psf_x, 202 | psf_y,psf_fit_rad,params.gain]).astype(np.float32) 203 | profile_type = 0 204 | elif params.psf_profile_type == 'moffat25': 205 | print 'params.psf_profile_type moffat25 not working yet. Exiting.' 206 | sys.exit(0) 207 | psf_sigma_x = psf_hdr['PAR1'] 208 | psf_sigma_y = psf_hdr['PAR2'] 209 | psf_sigma_xy = psf_hdr['PAR3'] 210 | psf_parameters = np.array([psf_size,psf_height,psf_sigma_x,psf_sigma_y,psf_x, 211 | psf_y, 212 | psf_fit_rad,params.gain,psf_sigma_xy]).astype(np.float32) 213 | print 'psf_parameters',psf_parameters 214 | profile_type = 1 215 | else: 216 | print 'params.psf_profile_type undefined' 217 | sys.exit(0) 218 | 219 | # Copy the difference and inverse variance images into GPU texture memory 220 | RR = np.array([diff,inv_variance]).astype(np.float32).copy() 221 | diff_cuda = numpy3d_to_array(RR) 222 | texref = cu_matrix_kernel.get_texref("tex") 223 | texref.set_array(diff_cuda) 224 | texref.set_filter_mode(cuda.filter_mode.POINT) 225 | 226 | # Call the CUDA function to perform the photometry. 227 | # Each block is one star. 228 | # Each thread is one column of the PSF, but 32 threads per warp 229 | nstars = positions.shape[0] 230 | gridDim = (int(nstars),1,1) 231 | blockDim = (16,16,1) 232 | 233 | k0 = kernelIndex[:,0].astype(np.int32).copy() 234 | k1 = kernelIndex[:,1].astype(np.int32).copy() 235 | positions = positions.reshape(-1,2) 236 | if params.star_file_is_one_based: 237 | posx = np.float32(positions[:,0].copy()-1.0) 238 | posy = np.float32(positions[:,1].copy()-1.0) 239 | else: 240 | posx = np.float32(positions[:,0].copy()) 241 | posy = np.float32(positions[:,1].copy()) 242 | 243 | 244 | #psf_0 = convolve_undersample(psf[0]).astype(np.float32).copy() 245 | #psf_xd = convolve_undersample(psf[1]).astype(np.float32).copy()*0.0 246 | #psf_yd = convolve_undersample(psf[2]).astype(np.float32).copy()*0.0 247 | #psf_0 = psf[0].astype(np.float32).copy() 248 | #psf_xd = psf[1].astype(np.float32).copy()*0.0 249 | #psf_yd = psf[2].astype(np.float32).copy()*0.0 250 | psf_0 = psf.astype(np.float32).copy() 251 | psf_xd = psf.astype(np.float32).copy()*0.0 252 | psf_yd = psf.astype(np.float32).copy()*0.0 253 | flux = np.float32(posy.copy() * 0.0); 254 | dflux = np.float32(posy.copy() * 0.0); 255 | 256 | coeffs = c.astype(np.float32).copy() 257 | 258 | cu_photom = cu_matrix_kernel.get_function('cu_photom') 259 | 260 | try: 261 | cu_photom(np.int32(profile_type),np.int32(diff.shape[0]), np.int32(diff.shape[1]), 262 | np.int32(params.pdeg), 263 | np.int32(params.sdeg),np.int32(c.shape[0]),np.int32(kernelIndex.shape[0]), 264 | np.int32(kernelRadius),cuda.In(k0), 265 | cuda.In(k1),cuda.In(extendedBasis), 266 | cuda.In(psf_parameters),cuda.In(psf_0),cuda.In(psf_xd),cuda.In(psf_yd), 267 | cuda.In(posx),cuda.In(posy),cuda.In(coeffs),cuda.Out(flux),cuda.Out(dflux), 268 | block=blockDim,grid=gridDim, 269 | texrefs=[texref]) 270 | except: 271 | print 'Call to cu_photom failed.' 272 | print 'psf_parameters', psf_parameters 273 | print 'size of posx, posy:', posx.shape, posy.shape 274 | print 'Parameters:' 275 | for par in dir(params): 276 | print par, getattr(params, par) 277 | print 278 | 279 | return flux, dflux 280 | 281 | 282 | 283 | def convolve_image_with_psf(psf_image,image1,image2,c,kernelIndex,extendedBasis, 284 | kernelRadius,params): 285 | 286 | from astropy.io import fits 287 | 288 | # Read the PSF 289 | psf,psf_hdr = fits.getdata(psf_image,0,header='true') 290 | psf_height = psf_hdr['PSFHEIGH'] 291 | psf_x = psf_hdr['PSFX'] 292 | psf_y = psf_hdr['PSFY'] 293 | psf_size = psf.shape[1] 294 | psf_fit_rad = params.psf_fit_radius 295 | if params.psf_profile_type == 'gaussian': 296 | psf_sigma_x = psf_hdr['PAR1']*0.8493218 297 | psf_sigma_y = psf_hdr['PAR2']*0.8493218 298 | psf_parameters = np.array([psf_size,psf_height,psf_sigma_x,psf_sigma_y,psf_x, 299 | psf_y,psf_fit_rad,params.gain]).astype(np.float32) 300 | profile_type = 0 301 | elif params.psf_profile_type == 'moffat25': 302 | print 'params.psf_profile_type moffat25 not working yet. Exiting.' 303 | sys.exit(0) 304 | psf_sigma_x = psf_hdr['PAR1'] 305 | psf_sigma_y = psf_hdr['PAR2'] 306 | psf_sigma_xy = psf_hdr['PAR3'] 307 | psf_parameters = np.array([psf_size,psf_height,psf_sigma_x,psf_sigma_y,psf_x, 308 | psf_y, 309 | psf_fit_rad,params.gain,psf_sigma_xy]).astype(np.float32) 310 | profile_type = 1 311 | else: 312 | print 'params.psf_profile_type undefined' 313 | sys.exit(0) 314 | 315 | # Copy the images into GPU texture memory 316 | nx, ny = image1.shape 317 | RR = np.array([image1,image2]).astype(np.float32).copy() 318 | image_cuda = numpy3d_to_array(RR) 319 | texref = cu_matrix_kernel.get_texref("tex") 320 | texref.set_array(image_cuda) 321 | texref.set_filter_mode(cuda.filter_mode.POINT) 322 | 323 | # Call the CUDA function to perform the double convolution. 324 | # Each block is one image section. 325 | # Each thread is one pixel of the PSF, but 32 threads per warp 326 | 327 | cu_convolve = cu_matrix_kernel.get_function('convolve_image_psf') 328 | 329 | k0 = kernelIndex[:,0].astype(np.int32).copy() 330 | k1 = kernelIndex[:,1].astype(np.int32).copy() 331 | #psf_0 = convolve_undersample(psf[0]).astype(np.float32).copy() 332 | #psf_xd = convolve_undersample(psf[1]).astype(np.float32).copy()*0.0 333 | #psf_yd = convolve_undersample(psf[2]).astype(np.float32).copy()*0.0 334 | psf_0 = psf.astype(np.float32).copy() 335 | psf_xd = psf.astype(np.float32).copy()*0.0 336 | psf_yd = psf.astype(np.float32).copy()*0.0 337 | 338 | image_section_size = 32 339 | convolved_image1 = (0.0*image1).astype(np.float32) 340 | convolved_image2 = (0.0*image1).astype(np.float32) 341 | gridDim = (int((nx-1)/image_section_size+1),int((ny-1)/image_section_size+1),1) 342 | blockDim = (16,16,1) 343 | 344 | cu_convolve(np.int32(profile_type),np.int32(nx), np.int32(ny), 345 | np.int32(image_section_size), 346 | np.int32(image_section_size), np.int32(params.pdeg), 347 | np.int32(params.sdeg),np.int32(c.shape[0]), 348 | np.int32(kernelIndex.shape[0]),np.int32(kernelRadius),cuda.In(k0), 349 | cuda.In(k1),cuda.In(extendedBasis),cuda.In(psf_parameters),cuda.In(psf_0), 350 | cuda.In(psf_xd),cuda.In(psf_yd),cuda.In(c),cuda.Out(convolved_image1), 351 | cuda.Out(convolved_image2),block=blockDim,grid=gridDim,texrefs=[texref]) 352 | 353 | return convolved_image1, convolved_image2 354 | 355 | 356 | 357 | 358 | 359 | -------------------------------------------------------------------------------- /Code/image_functions.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | from scipy import ndimage 4 | from scipy.signal import fftconvolve, convolve2d 5 | from astropy.modeling import models, fitting 6 | 7 | 8 | def positional_shift(R,T): 9 | Rc = R[10:-10,10:-10] 10 | Tc = T[10:-10,10:-10] 11 | c = fftconvolve(Rc, Tc[::-1, ::-1]) 12 | cind = np.where(c == np.max(c)) 13 | print cind 14 | csmall = c[cind[0][0]-10:cind[0][0]+10,cind[1][0]-10:cind[1][0]+10] 15 | csmall = c[cind[0][0]-6:cind[0][0]+7,cind[1][0]-6:cind[1][0]+7] 16 | X,Y = np.indices(csmall.shape) 17 | total =csmall.sum() 18 | dx = (X*csmall).sum()/total - 6 + cind[0][0] - c.shape[0]/2.0 19 | dy = (Y*csmall).sum()/total - 6 + cind[1][0] - c.shape[1]/2.0 20 | return dx, dy 21 | 22 | 23 | 24 | 25 | 26 | def register(R,T,params): 27 | #Tc = T.data*T.mask 28 | #Rc = R.data*R.mask 29 | Tc_data = T.data.copy() 30 | Rc_data = R.data.copy() 31 | 32 | if params.register_using_masks: 33 | Tc_data[T.mask==0] = np.median(Tc_data[T.mask==1]) 34 | Rc_data[R.mask==0] = np.median(Rc_data[R.mask==1]) 35 | 36 | if isinstance(params.fwhm_section,np.ndarray): 37 | w = params.fwhm_section 38 | Tc = Tc_data[w[2]:w[3],w[0]:w[1]].copy() 39 | Rc = Rc_data[w[2]:w[3],w[0]:w[1]].copy() 40 | else: 41 | Tc = Tc_data 42 | Rc = Rc_data 43 | 44 | #nx, ny = R.shape 45 | #z = np.arange(-3,4) 46 | #saturated_pixels = np.where(R.mask==0) 47 | #for k in range(saturated_pixels[0].size): 48 | # p = np.array([z+saturated_pixels[0][k],z+saturated_pixels[1][k]]) 49 | # px, py = np.meshgrid(p[0],p[1]) 50 | # q = np.where((px>=0) & (px=0) & (py=0) & (px=0) & (py1.1*params.pixel_max) 111 | zp0 = z[p[0]] 112 | zp1 = z[p[1]] 113 | sp0 = bad_pixels[0][:,np.newaxis] 114 | sp1 = bad_pixels[1][:,np.newaxis] 115 | q0 = zp0 + sp0 116 | q1 = zp1 + sp1 117 | q0 = q0.flatten() 118 | q1 = q1.flatten() 119 | s = np.asarray(np.where((q0>=0) & (q0=0) & (q1params.pixel_max) 138 | for q in range(len(sx)): 139 | ymin = max(0,int(sy[q]-params.bleed_mask_multiplier_below*kernel_len)) 140 | ymax = min(d.shape[1],int(sy[q]+params.bleed_mask_multiplier_above*kernel_len)) 141 | mask[ymin:ymax,sx[q]] = 0 142 | return mask 143 | 144 | 145 | def compute_saturated_pixel_mask(im,params): 146 | radius = params.mask_radius 147 | rad2 = radius*radius 148 | rad = int(np.ceil(radius)) 149 | z = np.arange(2*rad+1)-rad 150 | x,y = np.meshgrid(z,z) 151 | p = np.array(np.where(x**2 + y**2 < rad2)) 152 | mask = np.ones(im.shape,dtype=bool) 153 | saturated_pixels = np.where((im > params.pixel_max) | 154 | (im <= params.pixel_min)) 155 | zp0 = z[p[0]] 156 | zp1 = z[p[1]] 157 | sp0 = saturated_pixels[0][:,np.newaxis] 158 | sp1 = saturated_pixels[1][:,np.newaxis] 159 | q0 = zp0 + sp0 160 | q1 = zp1 + sp1 161 | q0 = q0.flatten() 162 | q1 = q1.flatten() 163 | # q = np.array([[],[]]) 164 | # for k in range(saturated_pixels[0].size): 165 | # q = np.column_stack([q,np.array([zp0+saturated_pixels[0][k], 166 | # zp1+saturated_pixels[1][k]])]) 167 | # q1.append([r for r in zp0+saturated_pixels[0][k]]) 168 | # q2.append([r for r in zp1+saturated_pixels[1][k]]) 169 | # q = np.array([np.array(q1).flatten(),np.array(q2).flatten()]) 170 | s = np.asarray(np.where((q0>=0) & (q0=0) & (q1 params.pixel_max) | 183 | (im1 <= params.pixel_min) | 184 | (im2 > params.pixel_max) | 185 | (im2 <= params.pixel_min)) 186 | for k in range(saturated_pixels[0].size): 187 | q = np.array([z[p[0]]+saturated_pixels[0][k],z[p[1]]+saturated_pixels[1][k]]) 188 | s = np.asarray(np.where((q[0]>=0) & (q[0]=0) & (q[1] params.pixel_saturation_kernel_max) 203 | for k in range(saturated_pixels[0].size): 204 | q = np.array([z[p[0]]+saturated_pixels[0][k],z[p[1]]+saturated_pixels[1][k]]) 205 | s = np.asarray(np.where((q[0]>=0) & (q[0]=0) & (q[1]0.0001) 228 | sp = np.std(norm[p]) 229 | t = np.where(np.abs(norm) > threshold*sp) 230 | if t: 231 | print 'Rejecting',t[0].shape[0],'pixels' 232 | bmask[t] = 0 233 | count += 1 234 | else: 235 | not_finished = False 236 | return bmask 237 | 238 | 239 | 240 | def boxcar_blur(im): 241 | d = np.zeros(im.shape) 242 | m1 = im.shape[0] - 2 243 | m2 = im.shape[1] - 2 244 | for i in range(3): 245 | for j in range(3): 246 | d[1:m1+1,1:m2+1] += im[i:i+m1,j:j+m2] 247 | d /= 9.0 248 | return d 249 | 250 | 251 | def convolve_undersample(im): 252 | from scipy.ndimage.filters import convolve 253 | x = np.arange(3)-1 254 | xx,yy = np.meshgrid(x,x) 255 | kernel = 0.25*(np.ones([3,3])-abs(xx*0.5))*(np.ones([3,3])-abs(yy*0.5)) 256 | c = convolve(im,kernel) 257 | return c 258 | 259 | 260 | def convolve_gauss(im,fwhm): 261 | from scipy.ndimage.filters import convolve 262 | sigma = fwhm/(2*np.sqrt(2*np.log(2.0))) 263 | nk = 1 + 2*int(4*sigma) 264 | x = np.arange(nk)-nk/2 265 | xx,yy = np.meshgrid(x,x) 266 | kernel = np.exp(-(xx**2 + yy**2)/(2*sigma**2)) 267 | kernel /= np.sum(kernel) 268 | c = convolve(im,kernel) 269 | return c 270 | 271 | 272 | def convolve_disk(im,radius): 273 | from scipy.ndimage.filters import convolve 274 | radius = int(radius) 275 | diameter = 2*radius + 1 276 | x = np.arange(diameter)-radius 277 | xx,yy = np.meshgrid(x,x) 278 | kernel = np.zeros((diameter,diameter)) 279 | kernel[xx**2+yy**2<=radius**2] = 1.0 280 | kernel /= np.sum(kernel) 281 | fp_im = im*1.0 282 | c = convolve(fp_im,kernel) 283 | return c 284 | 285 | 286 | def apply_photometric_scale(d,c,pdeg): 287 | p = np.zeros(d.shape) 288 | (m,n) = d.shape 289 | eta = (range(n)-0.5*(n-1)*np.ones(n))/(n-1) 290 | xi = (range(m)-0.5*(m-1)*np.ones(m))/(m-1) 291 | x,y = np.meshgrid(eta,xi) 292 | i = 0 293 | for l in range(pdeg+1): 294 | for m in range(pdeg-l+1): 295 | t = (x**l)*(y**m) 296 | p += c[i]*t 297 | i += 1 298 | q = d/p 299 | return q 300 | 301 | 302 | def undo_photometric_scale(d,c,pdeg,size=None,position=(0,0)): 303 | md,nd = d.shape 304 | if size: 305 | (m,n) = size 306 | else: 307 | (m,n) = d.shape 308 | p = np.zeros([md,nd]) 309 | eta = (range(n)-0.5*(n-1)*np.ones(n))/(n-1) 310 | xi = (range(m)-0.5*(m-1)*np.ones(m))/(m-1) 311 | x0,y0 = np.meshgrid(eta,xi) 312 | x = x0[position[0]:position[0]+md,position[1]:position[1]+nd] 313 | y = y0[position[0]:position[0]+md,position[1]:position[1]+nd] 314 | i = 0 315 | for l in range(pdeg+1): 316 | for m in range(pdeg-l+1): 317 | t = (x**l)*(y**m) 318 | p += c[i]*t 319 | i += 1 320 | q = d*p 321 | return q 322 | 323 | 324 | 325 | def compute_fwhm(f,params,width=20,seeing_file='seeing',image_name=False): 326 | 327 | 328 | 329 | g_width = None 330 | 331 | if image_name: 332 | fname = f 333 | else: 334 | fname = f.name 335 | 336 | if os.path.exists(params.loc_output+os.path.sep+seeing_file): 337 | for line in open(params.loc_output+os.path.sep+seeing_file,'r'): 338 | sline = line.split() 339 | if sline[0] == fname: 340 | g_width = float(sline[1]) 341 | g_roundness = float(sline[2]) 342 | bgnd = float(sline[3]) 343 | signal = float(sline[4]) 344 | break 345 | 346 | if g_width is None: 347 | if isinstance(params.fwhm_section,np.ndarray): 348 | w = params.fwhm_section 349 | image = f.data[w[2]:w[3],w[0]:w[1]].copy() 350 | mask = f.mask[w[2]:w[3],w[0]:w[1]].copy() 351 | else: 352 | image = f.data.copy() 353 | mask = f.mask.copy() 354 | print image.shape 355 | print mask.shape 356 | bgnd = np.percentile(image[mask==1],30) 357 | image[mask==0] = bgnd 358 | image -= bgnd 359 | signal = image.sum()/image.size 360 | c = fftconvolve(image, image[::-1, ::-1]) 361 | xcen = c.shape[0]/2 362 | ycen = c.shape[1]/2 363 | c_small = c[xcen-20:xcen+20,ycen-20:ycen+20] 364 | c_small -= np.min(c_small) 365 | xsize, ysize = c_small.shape 366 | xcen = c_small.shape[0]/2 367 | ycen = c_small.shape[1]/2 368 | y, x = np.mgrid[:xsize, :ysize] 369 | g_init = models.Gaussian2D(amplitude=c_small[xcen,ycen],x_stddev=1,y_stddev=1,x_mean=xcen,y_mean=ycen) 370 | fit_g = fitting.LevMarLSQFitter() 371 | g=fit_g(g_init,x,y,c_small) 372 | gx = g.x_stddev.value 373 | gy = g.y_stddev.value 374 | g_width = np.mean((gx,gy))/np.sqrt(2.0) 375 | g_roundness = np.max((gx,gy))/np.min((gx,gy)) 376 | #x1 = int(round(c.shape[0]*0.5)) 377 | #x2 = int(round(c.shape[0]*0.5+width)) 378 | #y1 = int(round(c.shape[1]*0.5)) 379 | #xx = np.arange(x2-x1+1) 380 | #xnew = np.linspace(0,x2-x1,1000) 381 | #fint = interp1d(xx,c[x1:x2+1,y1]-np.min(c[x1:x2+1,y1]),kind='cubic') 382 | #ynew = fint(xnew) 383 | #ymax = max(ynew) 384 | #for i,y in enumerate(ynew): 385 | # if y 0.1: 403 | p = np.where(image2 > params.pixel_min) 404 | const = np.percentile(image2[p],params.sky_subtract_percent) 405 | else: 406 | const = np.percentile(image2,params.sky_subtract_percent) 407 | image2 -= const 408 | print 'subtracting sky, constant =',const 409 | return image2 410 | else: 411 | degree = params.sky_degree 412 | (ni,mi) = image.shape 413 | sxlen = image.shape[0]/5.0 414 | sylen = image.shape[1]/5.0 415 | x = np.zeros(25) 416 | y = np.zeros(25) 417 | z = np.zeros(25) 418 | k = 0 419 | for i in range(5): 420 | for j in range(5): 421 | section = image[int(i*sxlen):int((i+1)*sxlen), 422 | int(j*sylen):int((j+1)*sylen)].ravel() 423 | z[k] = np.min(section[section>params.pixel_min]) 424 | x[k] = ((i+0.5)*sxlen-0.5*(ni-1))/(ni-1) 425 | y[k] = ((j+0.5)*sylen-0.5*(mi-1))/(mi-1) 426 | print x[k],y[k],z[k] 427 | k += 1 428 | ncoeffs = (degree+1)*(degree+2)/2 429 | bf = np.zeros([ncoeffs,k]) 430 | m = 0 431 | for i in range(degree+1): 432 | for j in range(degree+1-i): 433 | bf[m,:] = (x[:k]**i) * (y[:k]**j) 434 | m += 1 435 | alpha = np.zeros([ncoeffs,ncoeffs]) 436 | beta = np.zeros(ncoeffs) 437 | for i in range(ncoeffs): 438 | for j in range(ncoeffs): 439 | alpha[i,j] = np.sum(bf[i,:]*bf[j,:]) 440 | beta[i] = np.sum(z[:k]*bf[i,:]) 441 | try: 442 | lu, piv = lu_factor(alpha) 443 | except LinAlgError: 444 | print 'LU decomposition failed in subtract_sky' 445 | return image 446 | c = lu_solve((lu,piv),beta).astype(np.float32).copy() 447 | x = (range(ni)-0.5*(ni-1)*np.ones(ni))/(ni-1) 448 | y = (range(mi)-0.5*(mi-1)*np.ones(mi))/(mi-1) 449 | xx, yy = np.meshgrid(y,x) 450 | m = 0 451 | sky_image = np.zeros_like(image) 452 | print 'coeffs = ',c 453 | print 'range x y:',np.min(x),np.max(x),np.min(y),np.max(y) 454 | for i in range(degree+1): 455 | for j in range(degree+1-i): 456 | sky_image += c[m] * (xx**i) * (yy**j) 457 | m += 1 458 | sky_image[sky_image<0.0] = 0.0 459 | image2 = image - sky_image 460 | return image2 461 | 462 | 463 | def mask_cluster(im,mask,params): 464 | cim = convolve_gauss(im,20) 465 | p = np.where(cim == np.max(cim)) 466 | xmax = p[0][0] 467 | ymax = p[1][0] 468 | x = np.arange(im.shape[0]) 469 | y = np.arange(im.shape[1]) 470 | xx,yy = np.meshgrid(x,y) 471 | rad2 = params.cluster_mask_radius**2 472 | q = np.where((xx-xmax)**2 + (yy-ymax)**2 < rad2) 473 | mask[q] = 0 474 | return mask 475 | 476 | 477 | 478 | def define_kernel_pixels_fft(ref,target,rad,INNER_RADIUS=7,threshold=3.0): 479 | from numpy.fft import fft2, ifft2 480 | from astropy.stats import mad_std 481 | nx, ny = ref.image.shape 482 | x = np.concatenate((np.arange(nx/2),np.arange(-nx/2,0))) 483 | y = np.concatenate((np.arange(ny/2),np.arange(-ny/2,0))) 484 | fr = fft2(ref.image) 485 | ft = fft2(target.image) 486 | fk = ft/fr 487 | k = ifft2(fk) 488 | nk = k/k.max() 489 | std_nk = mad_std(nk) 490 | kp = np.where(np.abs(nk)>threshold) 491 | print 'kernel radius',rad 492 | crad = int(np.ceil(rad)) 493 | rad2 = rad*rad 494 | inner_rad2 = INNER_RADIUS*INNER_RADIUS 495 | kCount = 1 496 | for p in range(kp[0].shape[0]): 497 | i = x[kp[0][p]] 498 | j = y[kp[1][p]] 499 | r2 = i*i + j*j 500 | if (r2 < rad2) and ((i,j) != (0,0)): 501 | if (r2 < inner_rad2): 502 | kCount += 1 503 | else: 504 | if (i/3 == i/3.0) and (j/3 == j/3.0): 505 | kCount += 1 506 | kInd = np.zeros([kCount,2],dtype=np.int32) 507 | kExtended = np.zeros(kCount,dtype=np.int32) 508 | kInd[0] = [0,0] 509 | k = 1 510 | for p in range(kp[0].shape[0]): 511 | i = x[kp[0][p]] 512 | j = y[kp[1][p]] 513 | r2 = i*i + j*j 514 | if (r2 < rad2) and ((i,j) != (0,0)): 515 | if (r2 < inner_rad2): 516 | kInd[k] = [i,j] 517 | k += 1 518 | else: 519 | if (i/3 == i/3.0) and (j/3 == j/3.0): 520 | kInd[k] = [i,j] 521 | kExtended[k] = 1 522 | k += 1 523 | n_extend = np.sum(kExtended) 524 | print kCount-n_extend,'modified delta basis functions' 525 | print n_extend,'extended basis functions' 526 | for k in range(kCount): 527 | print kInd[k] 528 | return kInd, kExtended 529 | 530 | 531 | def define_kernel_pixels(rad,INNER_RADIUS=7): 532 | print 'kernel radius',rad 533 | crad = int(np.ceil(rad)) 534 | rad2 = rad*rad 535 | inner_rad2 = INNER_RADIUS*INNER_RADIUS 536 | kCount = 0 537 | for i in range(-crad,crad): 538 | for j in range(-crad,crad): 539 | r2 = i*i + j*j 540 | if (r2 < rad2): 541 | if (r2 < inner_rad2): 542 | kCount += 1 543 | else: 544 | if (i/3 == i/3.0) and (j/3 == j/3.0): 545 | kCount += 1 546 | kInd = np.zeros([kCount,2],dtype=np.int32) 547 | kExtended = np.zeros(kCount,dtype=np.int32) 548 | kInd[0] = [0,0] 549 | k = 1 550 | for i in range(-crad,crad): 551 | for j in range(-crad,crad): 552 | r2 = i*i + j*j 553 | if (r2 < rad2) and ((i,j) != (0,0)): 554 | if (r2 < inner_rad2): 555 | kInd[k] = [i,j] 556 | k += 1 557 | else: 558 | if (i/3 == i/3.0) and (j/3 == j/3.0): 559 | kInd[k] = [i,j] 560 | kExtended[k] = 1 561 | k += 1 562 | n_extend = np.sum(kExtended) 563 | print kCount-n_extend,'modified delta basis functions' 564 | print n_extend,'extended basis functions' 565 | return kInd, kExtended 566 | 567 | -------------------------------------------------------------------------------- /Code/photometry_functions.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import numpy as np 4 | from astropy.io import fits 5 | from pyraf import iraf 6 | 7 | from io_functions import read_fits_file, write_image 8 | from image_functions import compute_saturated_pixel_mask, subtract_sky 9 | 10 | 11 | def transform_coeffs(deg,dx,xx,yy): 12 | a = np.zeros((deg+1,deg+1)) 13 | nterms = (deg+1)*(deg+2)/2 14 | M = np.zeros((nterms,nterms)) 15 | v = np.zeros(nterms) 16 | i = 0 17 | for m in range(deg+1): 18 | for n in range(deg+1-m): 19 | v[i] = np.sum(dx* xx**m * yy**n) 20 | j = 0 21 | for p in range(deg+1): 22 | for q in range(deg+1-p): 23 | M[i,j] = np.sum(xx**(m+p) * yy**(n+q)) 24 | j += 1 25 | i += 1 26 | c = np.linalg.solve(M,v) 27 | i = 0 28 | for m in range(deg+1): 29 | for n in range(deg+1-m): 30 | a[m,n] = c[i] 31 | i += 1 32 | return a 33 | 34 | 35 | def compute_xy_shift(pos1,pos2,threshold,dx=0.0,dy=0.0,degree=0): 36 | 37 | x1 = pos1[:,0] 38 | y1 = pos1[:,1] 39 | x2 = pos2[:,0] 40 | y2 = pos2[:,1] 41 | xx = (x1 - np.mean(x1))/np.mean(x1) 42 | yy = (y1 - np.mean(y1))/np.mean(y1) 43 | print 'Matching positions for',len(x1),'stars' 44 | 45 | match = np.zeros_like(x1,dtype=np.int32) 46 | deltax = np.zeros_like(x1) 47 | deltay = np.zeros_like(x1) 48 | 49 | for deg in range(degree+1): 50 | 51 | a = np.zeros((deg+1,deg+1)) 52 | b = np.zeros((deg+1,deg+1)) 53 | 54 | if deg == 0: 55 | a[0,0] = dx 56 | b[0,0] = dy 57 | else: 58 | for m in range(deg): 59 | for n in range(deg-m): 60 | a[m,n] = a_prev[m,n] 61 | b[m,n] = b_prev[m,n] 62 | 63 | for scale in range(21,0,-4): 64 | 65 | xoffset = np.zeros_like(x1) 66 | yoffset = np.zeros_like(x1) 67 | for m in range(deg+1): 68 | for n in range(deg+1-m): 69 | xoffset += a[m,n]*(xx**m)*(yy**n) 70 | yoffset += b[m,n]*(xx**m)*(yy**n) 71 | 72 | for j1 in range(len(x1)): 73 | r2 = (x1[j1]-x2-xoffset[j1])**2 + (y1[j1]-y2-yoffset[j1])**2 74 | mm = np.where(r2 == np.min(r2)) 75 | try: 76 | match[j1] = np.where(r2 == np.min(r2))[0][0] 77 | except: 78 | print r2 79 | print np.min(np.sqrt(r2)) 80 | print mm 81 | print mm[0] 82 | sys.exit(0) 83 | deltax[j1] = x1[j1] - x2[match[j1]] - xoffset[j1] 84 | deltay[j1] = y1[j1] - y2[match[j1]] - yoffset[j1] 85 | 86 | deltar = np.sqrt(deltax**2 + deltay**2) 87 | good = np.where(deltar md) & (stars[:,0] < xmax-md) & 166 | (stars[:,1] > md) & (stars[:,1] < ymax-md)) 167 | if len(q[0]) >= params.nstamps: 168 | gstars = stars[q] 169 | else: 170 | print 'Warning: using stamps close to edge of detector' 171 | gstars = stars 172 | md = int(params.stamp_half_width) 173 | i = 0 174 | while (n_good < params.nstamps) & (i md) & (gstars[i,0] < xmax-md) & (gstars[i,1] > md) & 176 | (gstars[i,1] < ymax-md)): 177 | mstamp = mask[int(gstars[i,0]+0.5)-md:int(gstars[i,0]+0.5)+md,int(gstars[i,1]+0.5)-md:int(gstars[i,1]+0.5)+md] 178 | q = np.where(mstamp<1) 179 | if len(q[0]) == 0: 180 | snum[n_good] = i 181 | n_good += 1 182 | i += 1 183 | if n_good < params.nstamps: 184 | print 'Warning: stamps may contain saturated pixels' 185 | stamps = gstars[:params.nstamps,:] 186 | else: 187 | stamps = gstars[snum] 188 | return stamps 189 | 190 | 191 | def rewrite_psg(file1,file2): 192 | min_separation = 100.0 193 | q = open(file2,'w') 194 | lastgroup = -1 195 | for line in open(file1,'r'): 196 | if line[0] == '#': 197 | q.write(line) 198 | else: 199 | group = int(line.split()[1]) 200 | if group > lastgroup: 201 | lastgroup = group 202 | x0 = float(line.split()[2]) 203 | y0 = float(line.split()[2]) 204 | else: 205 | x = float(line.split()[2]) 206 | y = float(line.split()[2]) 207 | separation = np.sqrt((x-x0)**2 + (y-y0)**2) 208 | if separation < min_separation: 209 | min_separation = separation 210 | q.write(line) 211 | q.close() 212 | return int(min_separation) 213 | 214 | 215 | 216 | 217 | def compute_psf_image(params,g,psf_deg=1,psf_rad=8, 218 | star_file='phot.mags',psf_image='psf.fits',edge_dist=5): 219 | iraf.digiphot() 220 | iraf.daophot() 221 | fp = params.loc_output+os.path.sep 222 | 223 | f_im = g.image*g.mask 224 | f = fp+'temp.ref.fits' 225 | write_image(f_im,f) 226 | 227 | g.fw = np.max([1.5,g.fw]) 228 | g.fw = np.min([0.5*params.psf_max_radius,g.fw]) 229 | 230 | logfile = fp+'psf.log' 231 | 232 | fd = fits.getdata(f) 233 | xmax = fd.shape[0] - edge_dist 234 | ymax = fd.shape[1] - edge_dist 235 | 236 | 237 | for d in ['temp.stars','temp.phot','temp.phot1','temp.phot2','temp.pst', 238 | 'temp.opst','temp.opst2', 239 | 'temp.psf.fits','temp.psf1.fits','temp.psf2.fits','temp.psg', 240 | 'temp.psg2','temp.psg3','temp.psg5','temp.rej','temp.rej2', 241 | 'temp.sub.fits','temp.sub1.fits', 242 | 'temp.sub2.fits','temp.opst1','temp.opst3','temp.rej3', 243 | 'temp.nst','temp.stars1','ref.mags',psf_image,'temp.als', 244 | 'temp.als2']: 245 | if os.path.exists(fp+d): 246 | os.remove(fp+d) 247 | 248 | 249 | # locate stars 250 | iraf.daofind(image=f,output=fp+'temp.stars',interactive='no',verify='no', 251 | threshold=3,sigma=params.star_detect_sigma,fwhmpsf=g.fw, 252 | datamin=1,datamax=params.pixel_max, 253 | epadu=params.gain,readnoise=params.readnoise, 254 | noise='poisson') 255 | 256 | if params.star_file: 257 | als_recenter = 'no' 258 | all_template_stars = np.genfromtxt(params.star_file) 259 | all_new_stars = np.genfromtxt(fp+'temp.stars') 260 | 261 | if all_new_stars.shape[0] > params.star_file_number_match: 262 | new_stars = all_new_stars[all_new_stars[:,2].argsort()][:params.star_file_number_match] 263 | else: 264 | new_stars = all_new_stars 265 | 266 | if all_template_stars.shape[0] > params.star_file_number_match: 267 | template_stars = all_template_stars[all_template_stars[:,3].argsort()][:params.star_file_number_match] 268 | else: 269 | template_stars = all_template_stars 270 | 271 | tx, ty = compute_xy_shift(new_stars,template_stars[:,1:3],0.5, 272 | degree=params.star_file_transform_degree) 273 | 274 | if params.star_file_has_magnitudes: 275 | star_positions = all_template_stars[:,1:4] 276 | xx = (star_positions[:,0]-np.mean(new_stars[:,0]))/np.mean(new_stars[:,0]) 277 | yy = (star_positions[:,1]-np.mean(new_stars[:,1]))/np.mean(new_stars[:,1]) 278 | for m in range(params.star_file_transform_degree+1): 279 | for n in range(params.star_file_transform_degree+1-m): 280 | star_positions[:,0] += tx[m,n]* xx**m * yy**n 281 | star_positions[:,1] += ty[m,n]* xx**m * yy**n 282 | np.savetxt(fp+'temp.stars.1',star_positions,fmt='%10.3f %10.3f %10.3f') 283 | else: 284 | star_positions = all_template_stars[:,1:3] 285 | xx = (star_positions[:,0]-np.mean(new_stars[:,0]))/np.mean(new_stars[:,0]) 286 | yy = (star_positions[:,1]-np.mean(new_stars[:,1]))/np.mean(new_stars[:,1]) 287 | for m in range(params.star_file_transform_degree+1): 288 | for n in range(params.star_file_transform_degree+1-m): 289 | star_positions[:,0] += tx[m,n]* xx**m * yy**n 290 | star_positions[:,1] += ty[m,n]* xx**m * yy**n 291 | np.savetxt(fp+'temp.stars.1',star_positions,fmt='%10.3f %10.3f') 292 | all_template_stars[:,1] = star_positions[:,0] 293 | all_template_stars[:,2] = star_positions[:,1] 294 | 295 | else: 296 | 297 | als_recenter = 'yes' 298 | star_positions = np.genfromtxt(fp+'temp.stars') 299 | np.savetxt(fp+'temp.stars.1',star_positions[:,:2],fmt='%10.3f %10.3f') 300 | 301 | iraf.phot(image=f,output=fp+'temp.phot',coords=fp+'temp.stars.1',interactive='no', 302 | verify='no', 303 | sigma=params.star_detect_sigma,fwhmpsf=g.fw,apertures=g.fw, 304 | datamin=1, 305 | datamax=2*params.pixel_max,epadu=params.gain,annulus=3*g.fw, 306 | dannulus=3.0, 307 | readnoise=params.readnoise,noise='poisson') 308 | 309 | print 'fw = ',g.fw 310 | #fw = np.max([4.0,fw]) 311 | #print 'fw = ',fw 312 | 313 | 314 | # select PSF stars 315 | iraf.pstselect(image=f,photfile=fp+'temp.phot',pstfile=fp+'temp.pst',maxnpsf=40, 316 | interactive='no',verify='no',datamin=1,fitrad=2.0, 317 | datamax=params.pixel_max,epadu=params.gain,psfrad=np.max([3.0,g.fw]), 318 | readnoise=params.readnoise,noise='poisson') 319 | 320 | if params.star_file and params.star_file_has_magnitudes: 321 | 322 | # We don't need to do the photometry - only make the PSF 323 | 324 | # Initial PSF estimate to generate PSF groups 325 | #psfrad=3*np.max([g.fw,1.8]) 326 | iraf.psf(image=f,photfile=fp+'temp.phot',pstfile=fp+'temp.pst',psfimage=fp+'temp.psf', 327 | function=params.psf_profile_type,opstfile=fp+'temp.opst', 328 | groupfile=fp+'temp.psg', 329 | interactive='no', 330 | verify='no',varorder=0 ,psfrad=2*np.max([g.fw,1.8]), 331 | datamin=-10000,datamax=0.95*params.pixel_max, 332 | scale=1.0) 333 | 334 | # construct a file of the psf neighbour stars 335 | slist = [] 336 | psf_stars = np.loadtxt(fp+'temp.opst',usecols=(0,1,2)) 337 | 338 | for star in range(psf_stars.shape[0]): 339 | 340 | xp = psf_stars[star,1] 341 | yp = psf_stars[star,2] 342 | xmin = np.max([np.int(xp-10*g.fw),0]) 343 | xmax = np.min([np.int(xp+10*g.fw),f_im.shape[0]]) 344 | ymin = np.max([np.int(yp-10*g.fw),0]) 345 | ymax = np.min([np.int(yp+10*g.fw),f_im.shape[1]]) 346 | 347 | p = star_positions[np.logical_and(np.logical_and(star_positions[:,0]>xmin, 348 | star_positions[:,0]ymin, 350 | star_positions[:,1]= '+str(edge_dist)+' && YCE <= '+str(ymax)+' && MAG != INDEF' 493 | iraf.pselect(infiles=fp+'temp.als2',outfiles=fp+'temp.als3',expr=selection) 494 | iraf.psort(infiles=fp+'temp.als3',field='MAG') 495 | iraf.prenumber(infile=fp+'temp.als3') 496 | 497 | s = iraf.pdump(infiles=fp+'temp.als3',Stdout=1, 498 | fields='ID,XCENTER,YCENTER,MAG,MERR,MSKY,SHARPNESS,CHI',expr='yes') 499 | sf = [k.replace('INDEF','22.00') for k in s] 500 | stars = np.zeros([len(sf),5]) 501 | for i, line in enumerate(sf): 502 | stars[i,:] = np.array(map(float,sf[i].split()[1:6])) 503 | 504 | s = iraf.pdump(infiles=fp+'temp.als3',Stdout=1, 505 | fields='ID,XCENTER,YCENTER,MAG,MERR,SHARPNESS,CHI,MSKY',expr='yes') 506 | sf = [k.replace('INDEF','22.00') for k in s] 507 | with open(fp+'ref.mags','w') as fid: 508 | for s in sf: 509 | fid.write(s+'\n') 510 | 511 | return stars 512 | 513 | 514 | 515 | def group_stars_ccd(params,star_positions,reference): 516 | print 'grouping stars' 517 | d, h = read_fits_file(reference) 518 | ccd_size = d.shape 519 | print d.shape 520 | xpos = np.abs(star_positions[:,0]) 521 | ypos = np.abs(star_positions[:,1]) 522 | g_size = params.ccd_group_size 523 | n_groups_x = (ccd_size[1]-1)/g_size + 1 524 | n_groups_y = (ccd_size[0]-1)/g_size + 1 525 | print np.min(xpos), np.min(ypos) 526 | print np.max(xpos), np.max(ypos) 527 | print n_groups_x, n_groups_y 528 | indx = (xpos*0).astype(np.int) 529 | c = 0 530 | k = 0 531 | mposx = np.zeros(n_groups_x*n_groups_y) 532 | mposy = np.zeros(n_groups_x*n_groups_y) 533 | g_bound = np.zeros(n_groups_x*n_groups_y).astype(np.int) 534 | for i in range(n_groups_x): 535 | for j in range(n_groups_y): 536 | print 'group',i,j,i*g_size,(i+1)*g_size,j*g_size,(j+1)*g_size 537 | mposx[k] = (i+0.5)*g_size 538 | mposy[k] = (j+0.5)*g_size 539 | p = np.where((xpos>=i*g_size) & (xpos<(i+1)*g_size) & 540 | (ypos>=j*g_size) & (ypos<(j+1)*g_size))[0] 541 | if p.shape[0]: 542 | pn = p.shape[0] 543 | indx[c:c+pn] = p 544 | c += pn 545 | print k, pn, c 546 | g_bound[k] = c 547 | k += 1 548 | return indx,g_bound,mposx,mposy 549 | 550 | 551 | -------------------------------------------------------------------------------- /Code/detect.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from astropy.io import fits 4 | import fnmatch 5 | import numpy as np 6 | from scipy.ndimage import label, center_of_mass, fourier_ellipsoid, fourier_gaussian, gaussian_filter 7 | from io_functions import * 8 | from image_functions import * 9 | from photometry_functions import * 10 | import c_interface_functions as ci 11 | from data_structures import Observation 12 | 13 | def gauss(x, p): 14 | A, x0, sigma = p 15 | return A*numpy.exp(-(x-x0)**2/(2.*sigma**2)) 16 | 17 | 18 | def final_variable_photometry(files,params,coords=None, 19 | coord_file=None,psf_file=None): 20 | 21 | if params.use_GPU: 22 | from cuda_interface_functions import * 23 | else: 24 | from c_interface_functions import * 25 | 26 | if not(psf_file): 27 | psf_file = params.loc_output+os.path.sep+'psf.fits' 28 | if not(os.path.exists(psf_file)): 29 | print 'Error: PSF file',psf_file,'not found' 30 | sys.exit(1) 31 | 32 | if coords==None: 33 | if coord_file: 34 | coords = np.loadtxt(coord_file) 35 | else: 36 | print 'Error in final_variable_photometry: no coordinates provided.' 37 | sys.exit(1) 38 | 39 | flux = np.zeros([len(files),coords.shape[0]])*np.nan 40 | dflux = np.zeros([len(files),coords.shape[0]])*np.nan 41 | 42 | star_group_boundaries = None 43 | detector_mean_positions_x = None 44 | detector_mean_positions_y = None 45 | star_unsort_index = None 46 | 47 | if not(params.use_GPU): 48 | 49 | star_sort_index,star_group_boundaries,detector_mean_positions_x,detector_mean_positions_y = group_stars_ccd(params,coords,params.loc_output+os.path.sep+'ref.fits') 50 | scoords = coords[star_sort_index] 51 | star_unsort_index = np.argsort(star_sort_index) 52 | 53 | for i,f in enumerate(files): 54 | print f 55 | basename = os.path.basename(f) 56 | dfile = params.loc_output+os.path.sep+'d_'+basename 57 | nfile = params.loc_output+os.path.sep+'n_'+basename 58 | mfile = params.loc_output+os.path.sep+'z_'+basename 59 | ktable = params.loc_output+os.path.sep+'k_'+basename 60 | if os.path.exists(dfile) and os.path.exists(nfile) and os.path.exists(ktable): 61 | kernelIndex, extendedBasis, c, params = \ 62 | read_kernel_table(ktable,params) 63 | kernelRadius = np.max(kernelIndex[:,0])+1 64 | if np.sum(extendedBasis) > 0: 65 | kernelRadius += 1 66 | diff, h = read_fits_file(dfile) 67 | norm, h = read_fits_file(nfile) 68 | mask, h = read_fits_file(mfile) 69 | inv_var = (norm/diff)**2 + (1-mask) 70 | diff = undo_photometric_scale(diff,c,params.pdeg) 71 | 72 | sflux, sdflux = photom_all_stars(diff,inv_var,coords, 73 | psf_file,c,kernelIndex, 74 | extendedBasis, 75 | kernelRadius,params, 76 | star_group_boundaries, 77 | detector_mean_positions_x, 78 | detector_mean_positions_y) 79 | 80 | flux[i,:] = sflux[star_unsort_index].copy() 81 | dflux[i,:] = sdflux[star_unsort_index].copy() 82 | 83 | else: 84 | 85 | for i,f in enumerate(files): 86 | print f 87 | basename = os.path.basename(f) 88 | dfile = params.loc_output+os.path.sep+'d_'+basename 89 | nfile = params.loc_output+os.path.sep+'n_'+basename 90 | mfile = params.loc_output+os.path.sep+'z_'+basename 91 | ktable = params.loc_output+os.path.sep+'k_'+basename 92 | if os.path.exists(dfile) and os.path.exists(nfile) and os.path.exists(ktable): 93 | kernelIndex, extendedBasis, c, params = \ 94 | read_kernel_table(ktable,params) 95 | kernelRadius = np.max(kernelIndex[:,0])+1 96 | if np.sum(extendedBasis) > 0: 97 | kernelRadius += 1 98 | diff, h = read_fits_file(dfile) 99 | norm, h = read_fits_file(nfile) 100 | mask, h = read_fits_file(mfile) 101 | inv_var = (norm/diff)**2 + (1-mask) 102 | diff = undo_photometric_scale(diff,c,params.pdeg) 103 | 104 | flux[i,:], dflux[i,:] = photom_all_stars(diff,inv_var,coords, 105 | psf_file,c,kernelIndex, 106 | extendedBasis, 107 | kernelRadius,params) 108 | 109 | 110 | 111 | return flux, dflux 112 | 113 | 114 | def detect_variables(params,psf_file=None,ccoords=None,time_sigma=4.0,star_coords=None): 115 | 116 | if params.use_GPU: 117 | from cuda_interface_functions import * 118 | else: 119 | from c_interface_functions import * 120 | 121 | # 122 | # Check that the PSF exists 123 | # 124 | if not(psf_file): 125 | psf_file = params.loc_output+os.path.sep+'psf.fits' 126 | if not(os.path.exists(psf_file)): 127 | print 'Error: PSF file',psf_file,'not found' 128 | sys.exit(1) 129 | 130 | # 131 | # Read the PSF 132 | # 133 | psf,psf_hdr = fits.getdata(psf_file,0,header='true') 134 | psf_height = psf_hdr['PSFHEIGH'] 135 | psf_sigma_x = psf_hdr['PAR1']*0.8493218 136 | psf_sigma_y = psf_hdr['PAR2']*0.8493218 137 | psf_x = psf_hdr['PSFX'] 138 | psf_y = psf_hdr['PSFY'] 139 | psf_size = psf.shape[1] 140 | 141 | # 142 | # Determine our list of files 143 | # 144 | all_files = os.listdir(params.loc_data) 145 | all_files.sort() 146 | filenames = [] 147 | nfiles = 0 148 | for f in all_files: 149 | if fnmatch.fnmatch(f,params.name_pattern): 150 | basename = os.path.basename(f) 151 | dfile = params.loc_output+os.path.sep+'d_'+basename 152 | nfile = params.loc_output+os.path.sep+'n_'+basename 153 | ktable = params.loc_output+os.path.sep+'k_'+basename 154 | if os.path.exists(dfile) and os.path.exists(nfile) and os.path.exists(ktable): 155 | filenames.append(f) 156 | nfiles += 1 157 | 158 | if (ccoords==None): 159 | 160 | # 161 | # 3-d array for image stack 162 | # 163 | d = fits.getdata(params.loc_output+os.path.sep+'d_'+filenames[0], 164 | header=False) 165 | xs, ys = d.shape 166 | 167 | good_centroids = [] 168 | 169 | j = 0 170 | 171 | while j < len(filenames): 172 | 173 | stack_size = np.min([len(filenames)-j,200]) 174 | stackd = np.zeros([stack_size,xs,ys]) 175 | stackn = np.zeros([stack_size,xs,ys]) 176 | print 'len_filenames, j, stack_size', len(filenames), j, stack_size 177 | 178 | nstack = 0 179 | while (j 0: 202 | kernelRadius += 1 203 | 204 | # 205 | # Convolve reference PSF with kernel 206 | # Convolve difference image with convolved PSF 207 | # 208 | stackd[nstack,:,:], stackn[nstack,:,:] = convolve_image_with_psf(\ 209 | psf_file,d,n,c,kernelIndex,extendedBasis,kernelRadius, 210 | params) 211 | nstack += 1 212 | 213 | 214 | # 215 | # Filter by time 216 | # 217 | stackn = gaussian_filter(stackn[:nstack,:,:],sigma=(time_sigma,0.0,0.0)) 218 | stackn[np.isnan(stackn)] = 0.0 219 | 220 | 221 | # 222 | # Detect positive and negative peaks in stack of noise-normalised 223 | # difference images 224 | # 225 | stackn_thresh = stackn[:nstack,:,:].copy() 226 | n_std = np.std(stackn_thresh) 227 | stackn_thresh = np.abs(stackn_thresh) 228 | print 'Standard deviation of norm stack = ',n_std 229 | stackn_thresh[stackn_threshdr) & (xdr) & (ydr) & (xdr) & (y 0: 299 | 300 | # 301 | # Do photometry at centroid positions 302 | # 303 | print 'Doing photometry at centroid positions' 304 | flux = np.zeros([len(filenames),len(final_centroids)])*np.nan 305 | dflux = np.zeros([len(filenames),len(final_centroids)])*np.nan 306 | 307 | star_group_boundaries = None 308 | detector_mean_positions_x = None 309 | detector_mean_positions_y = None 310 | star_unsort_index = None 311 | 312 | if not(params.use_GPU): 313 | star_sort_index,star_group_boundaries,detector_mean_positions_x,detector_mean_positions_y = group_stars_ccd(params,coords,params.loc_output+os.path.sep+'ref.fits') 314 | scoords = coords[star_sort_index] 315 | star_unsort_index = np.argsort(star_sort_index) 316 | 317 | for i,f in enumerate(filenames): 318 | basename = os.path.basename(f) 319 | dfile = params.loc_output+os.path.sep+'d_'+basename 320 | nfile = params.loc_output+os.path.sep+'n_'+basename 321 | mfile = params.loc_output+os.path.sep+'z_'+basename 322 | ktable = params.loc_output+os.path.sep+'k_'+basename 323 | if os.path.exists(dfile) and os.path.exists(nfile) and \ 324 | os.path.exists(ktable): 325 | diff, h = read_fits_file(dfile) 326 | norm, h = read_fits_file(nfile) 327 | mask, h = read_fits_file(mfile) 328 | inv_var = (norm/diff)**2 + (1-mask) 329 | diff = undo_photometric_scale(diff,c,params.pdeg) 330 | 331 | sflux, sdflux = photom_all_stars(diff,inv_var,scoords, 332 | psf_file,c,kernelIndex, 333 | extendedBasis, 334 | kernelRadius, 335 | params, 336 | star_group_boundaries, 337 | detector_mean_positions_x, 338 | detector_mean_positions_y) 339 | 340 | 341 | flux[i,:] = sflux[star_unsort_index].copy() 342 | dflux[i,:] = sdflux[star_unsort_index].copy() 343 | 344 | else: 345 | 346 | for i,f in enumerate(filenames): 347 | basename = os.path.basename(f) 348 | dfile = params.loc_output+os.path.sep+'d_'+basename 349 | nfile = params.loc_output+os.path.sep+'n_'+basename 350 | mfile = params.loc_output+os.path.sep+'z_'+basename 351 | ktable = params.loc_output+os.path.sep+'k_'+basename 352 | if os.path.exists(dfile) and os.path.exists(nfile) and \ 353 | os.path.exists(ktable): 354 | diff, h = read_fits_file(dfile) 355 | norm, h = read_fits_file(nfile) 356 | mask, h = read_fits_file(mfile) 357 | inv_var = (norm/diff)**2 + (1-mask) 358 | diff = undo_photometric_scale(diff,c,params.pdeg) 359 | 360 | flux[i,:], dflux[i,:] = photom_all_stars(diff,inv_var,coords, 361 | psf_file,c,kernelIndex, 362 | extendedBasis, 363 | kernelRadius, 364 | params) 365 | 366 | for i in range(len(final_centroids)): 367 | np.savetxt(params.loc_output+os.path.sep+'var%03d.flux'%i, 368 | np.vstack((flux[:,i],dflux[:,i])).T) 369 | 370 | # 371 | # Photometry with coordinate refinement 372 | # 373 | print 'Refining coordinates' 374 | ccoords = coords.copy() 375 | for i in range(len(final_centroids)): 376 | ccoords[i,:], flux, dflux = do_photometry_variables(filenames,params, 377 | coords[i,:], 378 | psf_file=psf_file) 379 | np.savetxt(params.loc_output+os.path.sep+'cvar%03d.flux'%i, 380 | np.vstack((flux,dflux)).T) 381 | 382 | np.savetxt(params.loc_output+os.path.sep+'variables.ccoords',ccoords) 383 | 384 | # 385 | # Final photometry using DIA routines 386 | # 387 | print 'Final photometry' 388 | flux, dflux = final_variable_photometry(filenames,params, 389 | coords=ccoords, 390 | psf_file=psf_file) 391 | 392 | for i in range(ccoords.shape[0]): 393 | np.savetxt(params.loc_output+os.path.sep+'fvar%03d.flux'%i, 394 | np.vstack((flux[:,i],dflux[:,i])).T) 395 | 396 | # 397 | # Final photometry of reference image 398 | # 399 | 400 | # 401 | # Do photometry of the reference image 402 | # 403 | frad = params.psf_fit_radius 404 | params.psf_fit_radius = 2.1 405 | ssmode = params.sky_subtract_mode 406 | params.sky_subtract_mode = 'percent' 407 | reference_image = 'ref.fits' 408 | ref = Observation(params.loc_output+os.path.sep+reference_image,params) 409 | ref.image = ref.data 410 | ref.inv_variance += 1 - ref.mask 411 | ktable = params.loc_output+os.path.sep+'k_'+os.path.basename(reference_image) 412 | kernelIndex, extendedBasis, c, params = read_kernel_table(ktable,params) 413 | kernelRadius = np.max(kernelIndex[:,0])+1 414 | if np.sum(extendedBasis) > 0: 415 | kernelRadius += 1 416 | print 'kernelIndex', kernelIndex 417 | print 'extendedBasis',extendedBasis 418 | print 'coeffs', c 419 | print 'kernelRadius',kernelRadius 420 | flux, dflux = photom_all_stars(ref.image,ref.inv_variance,ccoords, 421 | psf_file,c,kernelIndex,extendedBasis, 422 | kernelRadius,params) 423 | 424 | for i in range(ccoords.shape[0]): 425 | np.savetxt(params.loc_output+os.path.sep+'fvar%03d.ref.flux'%i, 426 | np.array([flux[i],dflux[i]])) 427 | params.psf_fit_radius = frad 428 | params.sky_subtract_mode = ssmode 429 | 430 | 431 | def do_photometry_variables(files,params,position,extname='vflux', 432 | psf_file=None,reference_image='ref.fits',iterations=10): 433 | 434 | # 435 | # Get image size 436 | # 437 | dtarget = params.loc_output+os.path.sep+'d_'+ \ 438 | os.path.basename(files[0]) 439 | d, h = read_fits_file(dtarget) 440 | imsize = d.shape 441 | 442 | # 443 | # Check that the PSF exists 444 | # 445 | if not(psf_file): 446 | psf_file = params.loc_output+os.path.sep+'psf.fits' 447 | if not(os.path.exists(psf_file)): 448 | print 'Error: PSF file',psf_file,'not found' 449 | sys.exit(1) 450 | 451 | #position[0] += 2.0 452 | 453 | 454 | # 455 | # Read and store difference images 456 | # 457 | nfiles = len(files) 458 | xpos = int(np.floor(position[0]+0.5)-16) 459 | ypos = int(np.floor(position[1]+0.5)-16) 460 | dd = np.zeros([nfiles,32,32]) 461 | vv = np.zeros([nfiles,32,32]) 462 | nn = np.zeros([nfiles,32,32]) 463 | good = np.zeros(nfiles, dtype=bool) 464 | slice = (xpos,xpos+32,ypos,ypos+32) 465 | print 'slice',slice 466 | 467 | for j,f in enumerate(files): 468 | 469 | print 'Reading',f 470 | target = f 471 | dtarget = params.loc_output+os.path.sep+'d_'+ \ 472 | os.path.basename(target) 473 | ntarget = params.loc_output+os.path.sep+'n_'+ \ 474 | os.path.basename(target) 475 | ztarget = params.loc_output+os.path.sep+'z_'+ \ 476 | os.path.basename(target) 477 | ktable = params.loc_output+os.path.sep+'k_'+ \ 478 | os.path.basename(target) 479 | 480 | if os.path.exists(dtarget) and os.path.exists(ntarget) and \ 481 | os.path.exists(ktable): 482 | norm, h = read_fits_file(ntarget,slice=slice) 483 | diff, h = read_fits_file(dtarget,slice=slice) 484 | mask, h = read_fits_file(ztarget,slice=slice) 485 | kernelIndex, extendedBasis, c, params = \ 486 | read_kernel_table(ktable,params) 487 | diff[abs(diff) > 1.2*params.pixel_max] = np.nan 488 | inv_var = (norm/diff)**2 + (1-mask) 489 | diff[np.isnan(diff)] = 0.0 490 | diff = undo_photometric_scale(diff,c,params.pdeg, 491 | size=imsize,position=(xpos,ypos)) 492 | 493 | dd[j,:,:] = diff 494 | nn[j,:,:] = norm 495 | vv[j,:,:] = inv_var 496 | 497 | 498 | 499 | ngood = 0 500 | good_threshold = 6.0 501 | while (ngood < 12) and (good_threshold > 3.0): 502 | ngood = 0 503 | print 'good_threshold =',good_threshold 504 | for j,f in enumerate(files): 505 | snorm = np.std(nn[j,:,:]) 506 | print 'j,snorm, max = ',j,snorm, np.max(np.abs(nn[j,14:17,14:17])) 507 | if not(np.isinf(vv[j,:,:]).any() or np.isnan(vv[j,:,:]).any() or np.isnan(dd[j,:,:]).any()) and (np.nanstd(nn[j,:,:]) < params.diff_std_threshold) and (np.max(np.abs(nn[j,14:17,14:17])) > good_threshold*snorm): 508 | good[j] = True 509 | ngood += 1 510 | good_threshold *= 0.9 511 | 512 | if ngood < 6: 513 | print 'Convergence failed' 514 | return position, np.zeros(nfiles), np.zeros(nfiles) 515 | 516 | # 517 | # Iterate 518 | # 519 | position0 = position.copy() 520 | position1 = position.copy() 521 | position = np.array([position[0]-int(np.floor(position[0]+0.5))+16, 522 | position[1]-int(np.floor(position[1]+0.5))+16]) 523 | for i in range(iterations): 524 | 525 | x1 = 0.0 526 | x2 = 0.0 527 | y1 = 0.0 528 | y2 = 0.0 529 | flux = np.zeros(nfiles) 530 | dflux = np.zeros(nfiles) 531 | 532 | # 533 | # Process difference images 534 | # 535 | for j,f in enumerate(files): 536 | 537 | print i,j,f 538 | ktable = params.loc_output+os.path.sep+'k_'+ \ 539 | os.path.basename(f) 540 | 541 | if os.path.exists(ktable) and good[j]: 542 | 543 | kernelIndex, extendedBasis, c, params = \ 544 | read_kernel_table(ktable,params) 545 | kernelRadius = np.max(kernelIndex[:,0])+1 546 | if np.sum(extendedBasis) > 0: 547 | kernelRadius += 1 548 | 549 | flux[j], dflux[j], sjx1, sjx2, sjy1, sjy2 = \ 550 | ci.photom_variable_star(dd[j,:,:],vv[j,:,:],position, 551 | position0,psf_file,c,kernelIndex, 552 | extendedBasis,kernelRadius, 553 | params) 554 | 555 | 556 | #print sjx1/sjx2, sjy1/sjy2, sjx1,sjx2,sjy1,sjy2 557 | #if good[j] & (abs(sjx2)>10) & (abs(sjy2)>10) & (sjx1/sjx2 < 4) & (sjy1/sjy2 < 4): 558 | if (sjx1/sjx2 < 2) & (sjy1/sjy2 < 2): 559 | x1 += sjx1 560 | x2 += sjx2 561 | y1 += sjy1 562 | y2 += sjy2 563 | 564 | if (abs(x2) > 1.e-6) and (abs(y2) > 1.e-6): 565 | dx = -x1/x2 566 | dy = -y1/y2 567 | else: 568 | dx = 0.0 569 | dy = 0.0 570 | print x1,x2,dx 571 | print y1,y2,dy 572 | if (abs(dx) > 5.0) | (abs(dy) > 5.0) | np.isnan(dx) | np.isnan(dy): 573 | break 574 | 575 | x0 = position0[0]+dx 576 | y0 = position0[1]+dy 577 | x = position[0]+dx 578 | y = position[1]+dy 579 | 580 | print position,' + ',dx,',',dy,' = ',x,y 581 | print position0,' + ',dx,',',dy,' = ',x0,y0 582 | 583 | position0[0] = x0 584 | position0[1] = y0 585 | position[0] = x 586 | position[1] = y 587 | 588 | if (position1[0]-position0[0])**2 + (position1[1]-position0[1])**2 > 25.0: 589 | position0 = position1 590 | print 'Convergence failed' 591 | 592 | 593 | return position0, flux, dflux 594 | -------------------------------------------------------------------------------- /Code/analysis_functions.py: -------------------------------------------------------------------------------- 1 | import os 2 | import fnmatch 3 | import cPickle as Pickle 4 | #from matplotlib.backends.backend_pdf import PdfPages 5 | import matplotlib 6 | matplotlib.use("Agg") 7 | import matplotlib.pyplot as plt 8 | 9 | import numpy as np 10 | from scipy.signal import spectral 11 | from scipy import sparse 12 | 13 | from data_structures import Observation 14 | from io_functions import * 15 | from photometry_functions import * 16 | from astropy.stats import mad_std 17 | 18 | from pyraf import iraf 19 | 20 | 21 | 22 | def find_star_at(xpos,ypos,folder): 23 | ref = np.loadtxt(os.path.join(folder,'ref.mags')) 24 | dist2 = (xpos-ref[:,1])**2 + (ypos-ref[:,2])**2 25 | return np.where(dist2 == np.min(dist2))[0][0] 26 | 27 | 28 | def readflux(folder,images_file='images',seeing_file='seeing',ZP=25.0): 29 | fnames = [] 30 | datelist = [] 31 | fwhmlist = [] 32 | bgndlist = [] 33 | siglist = [] 34 | try: 35 | with open(os.path.join(folder,'lightcurve_data.pkl'),'rb') as pkl_file: 36 | dates, flux, eflux, mag, emag, fwhm, bgnd, signal = Pickle.load(pkl_file) 37 | except: 38 | with open(os.path.join(folder,images_file),'r') as fid: 39 | file = fid.read() 40 | for line in file.split('\n'): 41 | if len(line) > 0: 42 | f, d = line.split() 43 | fnames.append(f) 44 | datelist.append(d) 45 | with open(os.path.join(folder,seeing_file),'r') as fid: 46 | file = fid.read() 47 | for i, line in enumerate(file.split('\n')): 48 | if len(line) > 0 and i < len(fnames): 49 | n, f, b, s = line.split() 50 | if n == fnames[i]: 51 | fwhmlist.append(f) 52 | bgndlist.append(b) 53 | siglist.append(s) 54 | else: 55 | print 'Mismatch between images and seeing files.' 56 | sys.exit(0) 57 | dates = np.asarray(datelist,dtype=np.float64) 58 | fwhm = np.asarray(fwhmlist,dtype=np.float64) 59 | bgnd = np.asarray(bgndlist,dtype=np.float64) 60 | signal = np.asarray(siglist,dtype=np.float64) 61 | rf = np.loadtxt(os.path.join(folder,'ref.flux.calibrated')) 62 | nstars = rf.shape[0] 63 | nfiles = len(fnames) 64 | flux = np.zeros([nfiles,nstars]) 65 | eflux = np.zeros([nfiles,nstars]) 66 | mag = np.zeros([nfiles,nstars]) 67 | emag = np.zeros([nfiles,nstars]) 68 | for i, f in enumerate(fnames): 69 | fl = np.loadtxt(os.path.join(folder,f+'.flux')) 70 | flux[i,:] = fl[:,0] 71 | eflux[i,:] = fl[:,1] 72 | mag[i,:] = ZP - 2.5*np.log10(rf[:,0]+fl[:,0]) 73 | emag[i,:] = mag[i,:] - ZP + 2.5*np.log10(rf[:,0]+fl[:,0]+fl[:,1]) 74 | with open(os.path.join(folder,'lightcurve_data.pkl'),'wb') as pkl_file: 75 | Pickle.dump((dates, flux, eflux, mag, emag, fwhm, bgnd, signal),pkl_file) 76 | return dates, flux, eflux, mag, emag, fwhm, bgnd, signal 77 | 78 | 79 | def index_good_data(mag,threshold=2.5,fraction=0.3): 80 | """Returns indices of good data epochs.""" 81 | nepoch, nstars = mag.shape 82 | nmag = np.zeros_like(mag) 83 | mag = np.nan_to_num(mag) 84 | for i in range(nstars): 85 | nmag[:,i] = np.abs((mag[:,i]-np.median(mag[:,i])))/mad_std(mag[:,i]) 86 | nmag[nmag <= threshold] = 0.0 87 | nmag[nmag > 0.001] = 1.0 88 | ind = np.ones(nepoch,dtype=bool) 89 | nmag = np.nan_to_num(nmag) 90 | ftest = fraction*nstars 91 | for i in range(nepoch): 92 | if np.sum(nmag[i,:]) > ftest: 93 | ind[i] = False 94 | print np.sum(ind),'epochs of data out of',nepoch,'flagged as good' 95 | return ind 96 | 97 | 98 | def plot_lightcurve_at(x,y,folder=None,params=None,error_limit=0.1,tmin=None,tmax=None): 99 | if not(folder): 100 | folder = params.loc_data 101 | star = find_star_at(x,y,folder) 102 | dates, flux, eflux, mag, emag, fwhm, bgnd, signal = readflux(folder) 103 | q = index_good_data(mag) 104 | q[np.isnan(emag[:,star])] = False 105 | q[emag[:,star]>error_limit] = False 106 | q[emag[:,star]>np.median(emag[q,star])+3*np.std(emag[q,star])] = False 107 | plt.figure() 108 | plt.errorbar(dates[q],mag[q,star],emag[q,star],fmt='.',ls='None',ms=2,c='k') 109 | if tmin and tmax: 110 | plt.xlim((tmin,tmax)) 111 | plt.gca().invert_yaxis() 112 | plt.xlabel('HJD - 2450000',fontsize='14') 113 | plt.ylabel('Mag',fontsize='14') 114 | plt.title(folder+'-S%05d'%(star+1),fontsize='14') 115 | figname = folder+'-S%05d.png'%(star+1) 116 | plt.savefig(folder+'-S%05d.png'%(star+1)) 117 | 118 | np.savetxt(folder+'-S%05d.mags'%(star+1),np.vstack((dates[q],mag[q,star],emag[q,star],flux[q,star],eflux[q,star],fwhm[q],bgnd[q],signal[q])).T,fmt='%10.5f %7.4f %7.4f %12.3f %12.3f %6.2f %9.2f %9.2f') 119 | np.savetxt(folder+'-S%05d.all'%(star+1),np.vstack((dates,mag[:,star],emag[:,star],flux[:,star],eflux[:,star],fwhm,bgnd,signal)).T,fmt='%10.5f %7.4f %7.4f %12.3f %12.3f %6.2f %9.2f %9.2f') 120 | 121 | return figname 122 | 123 | 124 | 125 | def plotlc(files,ref,n): 126 | d = [] 127 | flux = [] 128 | dflux = [] 129 | mag = [] 130 | dmag = [] 131 | for f in files: 132 | if isinstance(f.flux,np.ndarray): 133 | d.append(f.date) 134 | flux.append(f.flux[n-1]) 135 | dflux.append(f.dflux[n-1]) 136 | mag.append(30-2.5*np.log10(f.flux[n-1]+ref.flux[n-1])) 137 | dmag.append(f.dflux[n-1]*1.08574/(f.flux[n-1]+ref.flux[n-1])) 138 | plt.errorbar(d,mag,yerr=dmag,fmt='ro') 139 | ax=plt.gca() 140 | ax.set_ylim(ax.get_ylim()[::-1]) 141 | 142 | 143 | 144 | 145 | def trend_filter(y,n,iterations=1,star_number=None,mode='mean',period=None,dates=None,plotfile=None): 146 | 147 | def run_filter(s,templates,n,g,g1,g_inv_full,y,yt,iterations,debug=False,phase=None,n_phase_bins=25): 148 | if s in templates: 149 | p = np.where(templates == s) 150 | m = p[0][0] 151 | g1[:m,:m] = g[:m,:m] 152 | g1[m:,:m] = g[m+1:,:m] 153 | g1[:m,m:] = g[:m,m+1:] 154 | g1[m:,m:] = g[m+1:,m+1:] 155 | g_inv = np.linalg.inv(g1) 156 | temp = np.hstack((np.arange(m),np.arange(m+1,n))) 157 | ntemp = n-1 158 | else: 159 | g_inv = g_inv_full 160 | temp = np.arange(n) 161 | ntemp = n 162 | yy = y[s,:].copy() 163 | ya = yy*0.0 164 | for i in range(iterations): 165 | if isinstance(phase,np.ndarray): 166 | for p in range(n_phase_bins): 167 | q = np.where((phase > p/(1.0*n_phase_bins)) & (phase <= (p+1)/(1.0*n_phase_bins))) 168 | ya[q] = np.mean(yy[q]) 169 | h = np.zeros(ntemp) 170 | yy = y[s,:].copy() 171 | for j in range(ntemp): 172 | h[j] = np.sum((yy-ya)*yt[temp[j],:]) 173 | c = np.dot(g_inv,h) 174 | for j in range(ntemp): 175 | yy -= c[j]*yt[temp[j],:] 176 | if debug: 177 | print 'templates\\n',templates 178 | print 's\\n', s 179 | print 'c\\n', c 180 | return yy 181 | 182 | nstars = y.shape[0] 183 | templates = np.random.choice(nstars/5,n,replace=False) 184 | 185 | g = np.zeros([n,n]) 186 | g1 = np.zeros([n-1,n-1]) 187 | 188 | yc = y.copy() 189 | ymean = np.mean(yc,1).reshape(nstars,1) 190 | if star_number: 191 | if mode == 'minimum': 192 | ymean[star_number] = np.min(yc,1).reshape(nstars,1)[star_number] 193 | elif mode == 'maximum': 194 | ymean[star_number] = np.max(yc,1).reshape(nstars,1)[star_number] 195 | elif mode == 'median': 196 | ymean[star_number] = np.median(yc,1).reshape(nstars,1)[star_number] 197 | yc -= ymean 198 | 199 | yt = yc[templates,:].copy() 200 | 201 | for j in range(n): 202 | for k in range(j+1): 203 | g[j,k] = np.sum(yt[j,:]*yt[k,:]) 204 | g[k,j] = g[j,k] 205 | g_inv_full = np.linalg.inv(g) 206 | 207 | if star_number: 208 | phase = None 209 | if period: 210 | cycle = dates/period 211 | phase = cycle - np.floor(cycle) 212 | yf = run_filter(star_number,templates,n,g,g1,g_inv_full,yc,yt,iterations,phase=phase,debug=False) + ymean[star_number] 213 | if plotfile: 214 | plt.figure() 215 | plt.plot(phase,yf,'b.',phase+1,yf,'b.') 216 | ax=plt.gca() 217 | ax.set_ylim(ax.get_ylim()[::-1]) 218 | plt.title(str(star_number)+' P = '+"%6.3f"%period+' d') 219 | plt.xlabel('PHASE') 220 | plt.ylabel('Magnitude') 221 | plt.savefig(plotfile) 222 | plt.close() 223 | else: 224 | yf = y*0.0; 225 | for s in range(nstars): 226 | yf[s,:] = run_filter(s,templates,n,g,g1,g_inv_full,yc,yt,iterations) 227 | yf += ymean 228 | return yf 229 | 230 | 231 | def detrend_v2(data,err_data,v1,v2): 232 | nstars, ndates = data.shape 233 | if len(v1) != ndates: 234 | print 'Error: length of vector 1 is not equal to the number of data rows' 235 | return data 236 | if len(v2) != ndates: 237 | print 'Error: length of vector 2 is not equal to the number of data rows' 238 | return data 239 | for star in range(nstars): 240 | bf = np.zeros([3,ndates]) 241 | bf[0,:] = np.ones(ndates) 242 | bf[1,:] = v1 243 | bf[2,:] = v2 244 | alpha = np.zeros([3,3]) 245 | beta = np.zeros(3) 246 | for k in range(3): 247 | for j in range(3): 248 | alpha[k,j] = np.sum(bf[j,:]*bf[k,:]/err_data[star,:]**2) 249 | beta[k] = np.sum(data[star,:]*bf[k,:]/err_data[star,:]**2) 250 | c = np.linalg.inv(alpha) 251 | a = np.dot(c,beta) 252 | data[star,:] -= a[0] + a[1]*v1 + a[2]*v2 253 | return data 254 | 255 | 256 | def detrend_pca(data,err,npc=10): 257 | from sklearn.decomposition import PCA 258 | from scipy import linalg 259 | (nlc,npts) = data.shape 260 | p = np.where(err<1.e-4) 261 | if p[0].any(): 262 | err[p] = 1.e-4 263 | mdata = data - np.mean(data, axis=1)[:,None] 264 | ddata = data 265 | print 'Computing principal components' 266 | eigen = PCA(n_components = npc).fit(mdata.T).transform(mdata.T) 267 | print 'Finished computing principal components' 268 | for k in np.arange(nlc): 269 | A = eigen/err[k,:].reshape(npts,1) 270 | Cd = mdata[k,:]/err[k,:] 271 | try: 272 | M = linalg.inv(np.dot(A.T,A)) 273 | w = np.dot(M,np.dot(A.T,Cd)) 274 | ddata[k,:] -= np.dot(eigen,w.T).T 275 | except ValueError: 276 | pass 277 | return ddata 278 | 279 | 280 | 281 | def lomb_search(dates,y,pmin,pmax,nfreq=1000,fap_threshold=1e-7,maxvars=100,plotfile='/dev/null'): 282 | nstars = y.shape[0] 283 | npoints = y.shape[1] 284 | logp = np.linspace(np.log10(pmin), np.log10(pmax), nfreq) 285 | freq = 2*np.pi/(10.0**logp) 286 | variables = [] 287 | periods = [] 288 | faps = [] 289 | nvars = 0 290 | for star in range(nstars): 291 | if nvars < maxvars: 292 | qq = np.where(np.isfinite(y[star,:]))[0] 293 | if len(qq)>100: 294 | print star 295 | print dates[qq] 296 | print y[star,qq] 297 | lnp = spectral.lombscargle(dates[qq], (y[star,qq]-np.mean(y[star,qq]))/np.std(y[star,qq]), freq) 298 | lnpmax = np.max(lnp) 299 | p = 2*np.pi/freq[np.where(lnp == lnpmax)[0][0]] 300 | # fap = 1.0 - (1.0 - (1.0-2.0*lnpmax/npoints)**( (npoints-3)/2.0 ) )**nfreq 301 | fap = ((npoints-3)/2.0)*np.exp(-lnpmax) 302 | if fap < fap_threshold and not(np.abs(p-1.0) < 0.01) and not(np.abs(p-0.5) < 0.01) and not(np.abs(p-0.33333) < 0.01): 303 | print star,p,np.max(lnp),fap 304 | variables.append(star) 305 | periods.append(p) 306 | faps.append(fap) 307 | plt.figure() 308 | plt.subplot(3,1,1) 309 | plt.plot(dates[qq],y[star,qq],'b.') 310 | ymax = np.percentile(y[star,qq],99) 311 | ymin = np.percentile(y[star,qq],1) 312 | plt.ylim([ymin,ymax]) 313 | ax=plt.gca() 314 | ax.set_ylim(ax.get_ylim()[::-1]) 315 | lfap = np.log10(fap) 316 | plt.title(str(star)+' P = '+"%6.3f"%p+' d $log_{10}$ FAP = '+"%4.1f"%lfap) 317 | plt.xlabel('Date') 318 | plt.ylabel('Magnitude') 319 | plt.subplot(3,1,2) 320 | cycle = dates[qq]/p 321 | phase = cycle - np.floor(cycle) 322 | plt.plot(phase,y[star,qq],'b.',phase+1,y[star,qq],'b.') 323 | plt.ylim([ymin,ymax]) 324 | ax=plt.gca() 325 | ax.set_ylim(ax.get_ylim()[::-1]) 326 | plt.xlabel('PHASE') 327 | plt.ylabel('Magnitude') 328 | plt.subplot(3,1,3) 329 | plt.plot(logp,lnp,'r-') 330 | plt.xlabel('$log_{10}$ Period (d)') 331 | plt.ylabel('LNP') 332 | plt.savefig('var%(star)05d.png'%vars(),orientation='portrait',papertype='a4') 333 | plt.close() 334 | nvars += 1 335 | return variables, periods, faps 336 | 337 | 338 | 339 | def transit_search(dates,data,err,pmin,pmax,nfreq,min_intransit_points=4,max_intransit_points=20,threshold=30,maxvars=100,plotfile='/dev/null',renormalize=True,true_period =None): 340 | nstars = data.shape[0] 341 | ndates = data.shape[1] 342 | xphase = np.linspace(0,1,1000) 343 | vars = [] 344 | periods = [] 345 | n_transit_points = max_intransit_points-min_intransit_points+1 346 | C = sparse.lil_matrix((n_transit_points*ndates,ndates)) 347 | Nout = np.zeros(n_transit_points*ndates) 348 | for i in range(n_transit_points): 349 | for j in range(ndates): 350 | Nout[i*ndates+j] = ndates - i - min_intransit_points 351 | C[i*ndates+j,j:np.min([j+i+min_intransit_points,ndates])] = 1 352 | if j+i+min_intransit_points > ndates: 353 | C[i*ndates+j,:(j+i+min_intransit_points)%ndates] = 1 354 | renorm = 1.0 355 | RNout = 1.0/Nout 356 | S = C.tocsr() 357 | logp = np.linspace(np.log10(pmin), np.log10(pmax), nfreq) 358 | trial_periods = 10**logp 359 | nvars = 0 360 | # pdf = PdfPages(plotfile) 361 | for star in range(nstars): 362 | if nvars < maxvars: 363 | w = 1.0/err[star,:]**2 364 | # m = sum(w*data[star,:])/sum(w) 365 | m = np.median(data[star,:]) 366 | npts = ndates 367 | # d = (data[star,:]-m)/err[star,:] 368 | # md = np.std(d) 369 | # p = np.where(d < 3*md) 370 | # if p[0].shape[0]: 371 | # npts = p[0].shape[0] 372 | # m = sum(w[p]*data[star,p].flatten())/sum(w[p]) 373 | # else: 374 | # npts = ndates 375 | if renormalize: 376 | renorm = sum(w*(data[star,:].flatten()-m)**2)/(npts-1) 377 | t = np.zeros(nfreq) 378 | for i in range(nfreq): 379 | cycle = dates/trial_periods[i] 380 | phase = cycle - np.floor(cycle) 381 | ind = np.argsort(phase) 382 | d = data[star,ind] 383 | w = 1.0/(err[star,ind]**2) 384 | n = (d-m) * w 385 | sw = S.dot(w) 386 | sn = S.dot(n) 387 | t[i] = 0.5 * np.max(sn**2/sw) / renorm 388 | tmax = np.max(t) 389 | pw = np.where(t == tmax) 390 | if pw[0].shape[0] and (tmax > threshold): 391 | p = trial_periods[pw[0][0]] 392 | print star,p,tmax,np.std(t),np.mean(t),(tmax-np.mean(t))/np.std(t) 393 | vars.append(star) 394 | periods.append(p) 395 | plt.figure() 396 | plt.subplot(2,1,1) 397 | if true_period: 398 | p = true_period 399 | cycle = dates/p 400 | phase = cycle - np.floor(cycle) 401 | plt.plot(phase,data[star,:],'b.',phase+1,data[star,:],'b.') 402 | ax=plt.gca() 403 | ax.set_ylim(ax.get_ylim()[::-1]) 404 | plt.title(str(star)+' P = '+"%6.3f"%p+' d') 405 | plt.xlabel('PHASE') 406 | plt.ylabel('Magnitude') 407 | plt.subplot(2,1,2) 408 | plt.plot(logp,t,'r-') 409 | plt.xlabel('$log_{10}$ Period (d)') 410 | plt.ylabel('T') 411 | plt.savefig('tran_'+str(star)+'.png') 412 | # pdf.savefig() 413 | plt.close() 414 | nvars += 1 415 | # pdf.close() 416 | return vars, periods 417 | 418 | 419 | 420 | def insert_transit(dates,mags,period,phase_start,phase_duration,amplitude): 421 | cycle = dates/period 422 | phase = cycle - np.floor(cycle) 423 | p = np.where((phase > phase_start) & (phase < phase_start+phase_duration)) 424 | new_mags = mags 425 | if p[0].shape[0]: 426 | new_mags[p] += amplitude 427 | return new_mags 428 | 429 | 430 | 431 | def plotddp(k,dates,logp,sn,lnprob,mags,mi,eigen,ev): 432 | 433 | ph = np.linspace(0,2,500) 434 | cycle = dates/10**logp[mi] 435 | phase = cycle - np.floor(cycle) 436 | 437 | tt = np.linspace(dates[0],dates[-1:],5000) 438 | ttp = 2*np.pi*tt/10**logp[mi]; 439 | 440 | plt.subplot(5,1,1) 441 | plt.plot(logp,np.exp(lnprob-np.max(lnprob)),'r-') 442 | plt.title(str(k)+' P = '+str(10**logp[mi])+' S/N = '+str(sn)) 443 | plt.ylabel('Prob(P)') 444 | plt.xlabel('log P') 445 | 446 | plt.subplot(5,1,2) 447 | q1 = np.where(abs(mags-np.mean(mags))<3*np.std(mags))[0] 448 | npts = q1.shape[0] 449 | colour1 = np.arange(npts)/np.float(npts) 450 | plt.scatter(phase[q1],mags[q1],s=2,edgecolor='',c=colour1,cmap=mpl.cm.winter) 451 | plt.scatter(phase[q1]+1,mags[q1],s=2,edgecolor='',c=colour1,cmap=mpl.cm.winter) 452 | print ph.shape 453 | ddd = np.mean(mags)+ev[mi,-2]*np.sin(2*np.pi*ph)+ev[mi,-1]*np.cos(2*np.pi*ph) 454 | plt.plot(ph,np.mean(mags)+ev[mi,-2]*np.sin(2*np.pi*ph)+ev[mi,-1]*np.cos(2*np.pi*ph),'r-') 455 | plt.ylabel('Mag') 456 | plt.xlabel('Phase') 457 | ax=plt.gca() 458 | ax.set_xlim([0,2]) 459 | ymax = np.percentile(mags[q1],99) 460 | ymin = np.percentile(mags[q1],1) 461 | plt.ylim([ymin,ymax]) 462 | ax.set_ylim(ax.get_ylim()[::-1]) 463 | y_formatter = mpl.ticker.ScalarFormatter(useOffset=False) 464 | ax.yaxis.set_major_formatter(y_formatter) 465 | 466 | plt.subplot(5,1,3) 467 | dd = mags-np.dot(eigen[:,:-2],ev[mi,:-2].T).T 468 | # dd = d.reshape(d.shape[1],) 469 | q2 = np.where(abs(dd-np.mean(dd))<3*np.std(dd))[0] 470 | npts = q2.shape[0] 471 | colour2 = np.arange(npts)/np.float(npts) 472 | plt.scatter(phase[q2],dd[q2],s=2,edgecolor='',c=colour2,cmap=mpl.cm.winter) 473 | plt.scatter(phase[q2]+1,dd[q2],s=2,edgecolor='',c=colour2,cmap=mpl.cm.winter) 474 | plt.plot(ph,np.mean(mags)+ev[mi,-2]*np.sin(2*np.pi*ph)+ev[mi,-1]*np.cos(2*np.pi*ph),'r-') 475 | plt.ylabel('Detrended Mag') 476 | plt.xlabel('Phase') 477 | ax=plt.gca() 478 | ax.set_xlim([0,2]) 479 | ymaxd = np.percentile(mags[q1],99) 480 | ymind = np.percentile(mags[q1],1) 481 | plt.ylim([ymind,ymaxd]) 482 | ax.set_ylim(ax.get_ylim()[::-1]) 483 | y_formatter = mpl.ticker.ScalarFormatter(useOffset=False) 484 | ax.yaxis.set_major_formatter(y_formatter) 485 | 486 | plt.subplot(5,1,4) 487 | plt.scatter(dates[q1],mags[q1],s=2,edgecolor='',c=colour1,cmap=mpl.cm.winter) 488 | plt.plot(tt,np.mean(mags)+ev[mi,-2]*np.sin(ttp)+ev[mi,-1]*np.cos(ttp),'r-') 489 | plt.ylabel('Mag') 490 | plt.xlabel('Date') 491 | ax=plt.gca() 492 | plt.ylim([ymin,ymax]) 493 | ax.set_ylim(ax.get_ylim()[::-1]) 494 | y_formatter = mpl.ticker.ScalarFormatter(useOffset=False) 495 | ax.yaxis.set_major_formatter(y_formatter) 496 | 497 | plt.subplot(5,1,5) 498 | plt.scatter(dates[q2],dd[q2],s=2,edgecolor='',c=colour2,cmap=mpl.cm.winter) 499 | plt.plot(tt,np.mean(mags)+ev[mi,-2]*np.sin(ttp)+ev[mi,-1]*np.cos(ttp),'r-') 500 | plt.ylabel('Detrended Mag') 501 | plt.xlabel('Date') 502 | ax=plt.gca() 503 | plt.ylim([ymind,ymaxd]) 504 | ax.set_ylim(ax.get_ylim()[::-1]) 505 | y_formatter = mpl.ticker.ScalarFormatter(useOffset=False) 506 | ax.yaxis.set_major_formatter(y_formatter) 507 | 508 | 509 | 510 | def ddp_search(dates,data,err,pmin,pmax,nfreq,npc=10,nlc=None,BIC_threshold=7,recompute_eigen=True): 511 | from sklearn.decomposition import PCA 512 | from scipy import linalg 513 | if nlc: 514 | npts = data.shape[1] 515 | else: 516 | nlc, npts = data.shape 517 | print 'npts =',npts 518 | p = np.where(err<1.e-4) 519 | if p[0].any(): 520 | err[p] = 1.e-4 521 | mdata = data - np.mean(data, axis=1)[:,None] 522 | ndata = mdata / np.std(mdata, axis=1)[:,None] 523 | logp = np.linspace(np.log10(pmin), np.log10(pmax), nfreq) 524 | p = 10.0**logp 525 | q = np.where(np.abs(p-1.0)>0.02)[0] 526 | logp = logp[q] 527 | freq = 1/(10.0**logp) 528 | nfreq = freq.shape[0] 529 | amp = np.zeros(nfreq) 530 | phase = np.zeros(nfreq) 531 | err_amp = np.zeros(nfreq) 532 | lnprob = np.zeros(nfreq) 533 | ev = np.zeros((nfreq,npc+2)) 534 | print 'Computing principal components' 535 | eigen = PCA(n_components = npc).fit(mdata.T).transform(mdata.T) 536 | print 'Finished computing principal components' 537 | for k in np.arange(nlc): 538 | if recompute_eigen: 539 | odata = np.vstack((mdata[:k,:],mdata[k+1:,:])) 540 | eigen = PCA(n_components = npc).fit(odata.T).transform(odata.T) 541 | eigen_e = eigen/err[k,:].reshape(npts,1) 542 | C = np.diag(err[k,:]**2) 543 | Cd = mdata[k,:]/err[k,:] 544 | for i,f in enumerate(freq): 545 | sb = np.sin(2*np.pi*f*dates)/err[k,:] 546 | cb = np.cos(2*np.pi*f*dates)/err[k,:] 547 | # A = np.hstack((eigen,sb.reshape(npts,1),cb.reshape(npts,1)))/err[k,:].reshape(npts,1) 548 | A = np.hstack((eigen_e,sb.reshape(npts,1),cb.reshape(npts,1))) 549 | 550 | try: 551 | M = linalg.inv(np.dot(A.T,A)) 552 | w = np.dot(M,np.dot(A.T,Cd)) 553 | chi2 = np.sum((Cd-np.dot(A,w))**2) 554 | W0 = M[:-2,:-2] 555 | U0 = M[-2:,-2:] 556 | V = M[:-2,-2:] 557 | U = U0 - np.dot(V.T,linalg.solve(W0,V)) 558 | lnprob[i] = np.log(2*np.pi)-0.5*linalg.det(W0)-0.5*np.dot(w[-2:].T,np.dot(U,w[-2:]))-0.5*chi2 559 | # vs = U[0,0] 560 | # vc = U[1,1] 561 | # vsc =U[0,1] 562 | # cs = w[-2] 563 | # cc = w[-1] 564 | # amp2 = cs**2+cc**2 565 | # amp[i] = np.sqrt(amp2) 566 | # phase[i] = np.arctan2(cc,cs) 567 | # err_amp[i] = np.sqrt(cs**2*vs/amp2+cc**2*vc/amp2+2*cs*cc*vsc/amp2) 568 | ev[i,:] = w 569 | except ValueError: 570 | amp[i] = 0 571 | err_amp[i] = 1 572 | phase[i] = 0 573 | lnprob[i] = -1e6 574 | break 575 | p = np.where(lnprob <= -1e5) 576 | q = np.where(lnprob > -1e5) 577 | if p[0].any() and q[0].any(): 578 | lnprob[p] = np.min(lnprob[q]) 579 | mb = np.max(lnprob) 580 | p = np.arange(nfreq) 581 | for i in range(5): 582 | p = np.where(abs(lnprob[p]-np.mean(lnprob[p]))<3*np.std(lnprob[p])) 583 | sn = (mb-np.mean(lnprob[p]))/np.std(lnprob[p]) 584 | print 'star',k,' lnProb =',mb,' S/N =',sn 585 | if sn > BIC_threshold: 586 | p = np.where(lnprob == mb) 587 | print 'Detected period',10.0**logp[p],'for star',k,'with lnProb',mb 588 | # print 'Coefficients',w 589 | plt.figure(figsize=(8.27, 11.69), dpi=100) 590 | plotddp(k,dates,logp,sn,lnprob,data[k,:],p[0][0],A*err[k,:].reshape(npts,1),ev) 591 | plt.savefig('vard%(k)05d.pdf'%vars(),orientation='portrait',papertype='a4') 592 | plt.close() 593 | 594 | 595 | 596 | def slot_acf(t,x,x_err,delta_tau,n): 597 | """ Compute the slot autocorrelation function for timeseries x(t) at lags 598 | k*delta_tau for k in 0..n""" 599 | 600 | nx = len(x) 601 | 602 | # Renormalise uncertainties 603 | x_var = x_err**2 604 | xmean = np.sum(x/x_var)/np.sum(1.0/x_var) 605 | x -= xmean 606 | chi2 = np.sum(x**2/x_var) 607 | scale = np.sqrt(chi2/nx) 608 | print 'scale:',scale 609 | x_err *= scale 610 | x_err_inv = 1.0/x_err 611 | x_norm = x*x_err_inv 612 | 613 | Ti, Tj = np.meshgrid(t,t) 614 | Tau = Tj - Ti 615 | 616 | Xi, Xj = np.meshgrid(x_norm,x_norm) 617 | Xj = np.tril(Xj,k=-1) 618 | 619 | Xi_err_inv, Xj_err_inv = np.meshgrid(x_err_inv,x_err_inv) 620 | Xj_err_inv = np.tril(Xj_err_inv,k=-1) 621 | 622 | acf = np.zeros(n) 623 | 624 | Taud = Tau/delta_tau 625 | 626 | for k in range(n): 627 | b = np.zeros((nx,nx),dtype=bool) 628 | b[np.abs(Taud-k)<0.5] = 1 629 | b = np.tril(b,k=-1) 630 | Xjb = Xj*b 631 | Xj_err_inv_b = Xj_err_inv*b 632 | #acf[k] = np.sum(np.diag(np.dot(Xi.T,Xj*b)))/np.sum(b) 633 | #acf[k] = np.sum([np.dot(x[m]*np.ones(nx),Xjb[:,m]) for m in range(nx)]) / np.sum([np.dot(x_err_inv[m]*np.ones(nx),Xj_err_inv_b[:,m]) for m in range(nx)]) 634 | acf[k] = np.sum([np.dot(x_norm[m]*np.ones(nx),Xjb[:,m]) for m in range(nx)]) / np.sum(b) 635 | 636 | return acf 637 | 638 | 639 | def gaussian_kernel_acf(t,x,x_err,delta_tau,n): 640 | """ Compute the autocorrelation function for timeseries x(t) at lags 641 | k*delta_tau for k in 0..n using a gaussian kernel.""" 642 | 643 | for i in range(5): 644 | x = (x-np.median(x))/mad_std(x) 645 | q = np.where(np.abs(x)<3.5) 646 | x = x[q] 647 | t = t[q] 648 | 649 | nx = len(x) 650 | 651 | x = (x-np.mean(x))/np.std(x) 652 | 653 | Ti, Tj = np.meshgrid(t,t) 654 | Tau = Tj - Ti 655 | 656 | Xi, Xj = np.meshgrid(x,x) 657 | Xj = np.tril(Xj,k=-1) 658 | 659 | acf = np.zeros(n) 660 | 661 | for k in range(n): 662 | h = k*delta_tau 663 | b = np.exp(-Tau**2/(2*h**2))/np.sqrt(2*np.pi*h) 664 | b = np.tril(b,k=-1) 665 | Xjb = Xj*b 666 | acf[k] = np.sum([np.dot(x[m]*np.ones(nx),Xjb[:,m]) for m in range(nx)])/np.sum(b) 667 | 668 | return acf 669 | 670 | -------------------------------------------------------------------------------- /Code/DIA_common.py: -------------------------------------------------------------------------------- 1 | # pyDIA 2 | # 3 | # This software implements the difference-imaging algorithm of Bramich et al. (2010) 4 | # with mixed-resolution delta basis functions. It uses an NVIDIA GPU to do the heavy 5 | # processing. 6 | # 7 | # Subroutines deconvolve3_rows, deconvolve3_columns, resolve_coeffs_2d and 8 | # interpolate_2d are taken from the Gwiddion software for scanning probe 9 | # microscopy (http://gwyddion.net/), which is distributed under the GNU General 10 | # Public License. 11 | # 12 | # All remaining code is Copyright (C) 2014, 2015 Michael Albrow 13 | # 14 | # This program is free software: you can redistribute it and/or modify 15 | # it under the terms of the GNU General Public License as published by 16 | # the Free Software Foundation, either version 3 of the License, or 17 | # (at your option) any later version. 18 | # 19 | # This program is distributed in the hope that it will be useful, 20 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 21 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22 | # GNU General Public License for more details. 23 | # 24 | # You should have received a copy of the GNU General Public License 25 | # along with this program. If not, see . 26 | # 27 | 28 | import sys 29 | import os 30 | import time 31 | import fnmatch 32 | import itertools 33 | from multiprocessing import Pool 34 | 35 | import numpy as np 36 | import data_structures as DS 37 | import io_functions as IO 38 | import image_functions as IM 39 | import photometry_functions as PH 40 | 41 | import c_interface_functions as CIF 42 | 43 | from skimage.feature import register_translation 44 | 45 | 46 | def difference_image(ref,target,params,stamp_positions=None, 47 | psf_image=None,star_positions=None, 48 | star_group_boundaries=None,detector_mean_positions_x=None, 49 | detector_mean_positions_y=None,star_sky=None,kernelRadius=None,kernel_inner_rad=7): 50 | 51 | from scipy.linalg import lu_solve, lu_factor, LinAlgError 52 | 53 | start = time.time() 54 | print 'difference_image', ref.name, target.name 55 | 56 | # 57 | # Set the kernel size based on the difference in seeing from the reference 58 | # 59 | #kernelRadius = min(params.kernel_maximum_radius, 60 | # max(params.kernel_minimum_radius, 61 | # np.abs(target.fw-ref.fw)*params.fwhm_mult)) 62 | if kernelRadius is None: 63 | kernelRadius = min(params.kernel_maximum_radius,max(params.kernel_minimum_radius, 64 | np.sqrt(np.abs(target.fw**2-ref.fw**2))*params.fwhm_mult)) 65 | 66 | # 67 | # Mask saturated pixels 68 | # 69 | #print 'Masking ',target.name,time.time()-start 70 | #smask = compute_saturated_pixel_mask(target.image,kernelRadius,params) 71 | 72 | 73 | # 74 | # Define the kernel basis functions 75 | # 76 | print 'Defining kernel pixels',time.time()-start 77 | if params.use_fft_kernel_pixels: 78 | kernelIndex, extendedBasis = IM.define_kernel_pixels_fft(ref,target,kernelRadius+2,INNER_RADIUS=20,threshold=params.fft_kernel_threshold) 79 | else: 80 | kernelIndex, extendedBasis = IM.define_kernel_pixels(kernelRadius,INNER_RADIUS=kernel_inner_rad) 81 | nKernel = kernelIndex.shape[0] 82 | 83 | # 84 | # We dont want to use bad pixels in either the target or reference image 85 | # 86 | smask = target.mask * ref.mask 87 | bmask = np.ones(smask.shape,dtype=bool) 88 | 89 | g = DS.EmptyBase() 90 | 91 | for iteration in range(params.iterations): 92 | 93 | print 'Computing matrix',time.time()-start 94 | 95 | tmask = bmask * smask 96 | 97 | # 98 | # Compute the matrix and vector 99 | # 100 | H, V, texref = CI.compute_matrix_and_vector_cuda(ref.image,ref.blur, 101 | target.image,target.inv_variance, 102 | tmask,kernelIndex, 103 | extendedBasis, 104 | kernelRadius, 105 | params, 106 | stamp_positions=stamp_positions) 107 | 108 | 109 | # 110 | # Solve the matrix equation to find the kernel coefficients 111 | # 112 | print 'Solving matrix equation', time.time()-start 113 | try: 114 | lu, piv = lu_factor(H) 115 | c = lu_solve((lu,piv),V).astype(np.float32).copy() 116 | except (LinAlgError,ValueError): 117 | print 'LU decomposition failed' 118 | g.model = None 119 | g.flux = None 120 | g.diff = None 121 | print 'H' 122 | print H 123 | sys.stdout.flush() 124 | return g 125 | 126 | # 127 | # Compute the model image 128 | # 129 | print 'Computing model',time.time()-start 130 | g.model = CI.compute_model_cuda(ref.image.shape,texref,c,kernelIndex,extendedBasis,params) 131 | edges = np.where(ref.image < 1.0) 132 | g.model[edges] = 0.0 133 | 134 | 135 | # 136 | # Compute the difference image 137 | # 138 | difference = (target.image - g.model) 139 | g.norm = difference*np.sqrt(target.inv_variance) 140 | 141 | # 142 | # Recompute the variance image from the model 143 | # 144 | #target.inv_variance = 1.0/(g.model/params.gain + 145 | # (params.readnoise/params.gain)**2) + (1-smask) 146 | mp = np.where(tmask == 0) 147 | if len(mp[0]) > 0: 148 | target.inv_variance[mp] = 1.e-12 149 | 150 | # 151 | # Mask pixels that disagree with the model 152 | # 153 | if iteration > 2: 154 | bmask = IM.kappa_clip(smask,g.norm,params.pixel_rejection_threshold) 155 | 156 | print 'Iteration',iteration,'completed',time.time()-start 157 | 158 | # 159 | # Delete the target image array to save memory 160 | # 161 | del target.image 162 | 163 | # 164 | # Save the kernel coefficients to a file 165 | # 166 | if params.do_photometry and psf_image: 167 | kf = params.loc_output+os.path.sep+'k_'+os.path.basename(target.name) 168 | IO.write_kernel_table(kf,kernelIndex,extendedBasis,c,params) 169 | 170 | print 'coeffs', c 171 | 172 | 173 | g.norm = difference*np.sqrt(target.inv_variance) 174 | g.variance = 1.0/target.inv_variance 175 | g.mask = tmask 176 | 177 | # 178 | # Do the photometry if requested 179 | # 180 | g.flux = None 181 | if params.do_photometry and psf_image: 182 | print 'star_positions', star_positions.shape 183 | print 'star_group_boundaries', star_group_boundaries 184 | if ref.name == target.name: 185 | sky_image, _ = IO.read_fits_file(params.loc_output+os.path.sep+'temp.sub2.fits') 186 | phot_target = ref.image - sky_image 187 | IO.write_image(phot_target,params.loc_output+os.path.sep+'clean_'+ref.name) 188 | g.flux, g.dflux = CIF.photom_all_stars_simultaneous(phot_target,target.inv_variance,star_positions, 189 | psf_image,c,kernelIndex,extendedBasis,kernelRadius,params, 190 | star_group_boundaries, 191 | detector_mean_positions_x,detector_mean_positions_y) 192 | else: 193 | phot_target = difference 194 | g.flux, g.dflux = CI.photom_all_stars(phot_target,target.inv_variance,star_positions, 195 | psf_image,c,kernelIndex,extendedBasis, 196 | kernelRadius,params, 197 | star_group_boundaries, 198 | detector_mean_positions_x, 199 | detector_mean_positions_y) 200 | 201 | print 'Photometry completed',time.time()-start 202 | 203 | # 204 | # Apply the photometric scale factor to the difference image. 205 | # We don't do this prior to the photometry because the PSF is 206 | # being convolved by the kernel, which already includes the 207 | # photometric scale factor. 208 | # 209 | g.diff = IM.apply_photometric_scale(difference,c,params.pdeg) 210 | sys.stdout.flush() 211 | return g 212 | 213 | 214 | 215 | def process_reference_image(f,args): 216 | best_seeing_ref, params, stamp_positions = args 217 | result = difference_image(f,best_seeing_ref,params, 218 | stamp_positions=stamp_positions) 219 | del f.image 220 | del f.mask 221 | del f.inv_variance 222 | return result 223 | 224 | 225 | def process_reference_image_helper(args): 226 | return process_reference_image(*args) 227 | 228 | 229 | 230 | def make_reference(files,reg,params,reference_image='ref.fits'): 231 | seeing = {} 232 | sky = {} 233 | ref_seeing = 1000 234 | 235 | # 236 | # Have we specified the files to make the reference with? 237 | # 238 | if params.ref_include_file: 239 | 240 | ref_list = [] 241 | for line in open(params.ref_include_file,'r'): 242 | for f in files: 243 | if f.name == line.split()[0]: 244 | ref_list.append(f) 245 | print f.name, f.fw, f.signal 246 | if f.fw < ref_seeing: 247 | ref_sky = f.sky 248 | ref_seeing = f.fw 249 | best_seeing_ref = f 250 | 251 | else: 252 | 253 | # 254 | # We try to choose the best images 255 | # 256 | reference_exclude = [] 257 | if params.ref_exclude_file: 258 | for line in open(params.ref_exclude_file,'r'): 259 | reference_exclude.append(line.split()[0]) 260 | 261 | sig = [] 262 | for f in files: 263 | sig.append(f.signal) 264 | 265 | sig = np.asarray(sig) 266 | sigcut = np.mean(sig) - 2.0*np.std(sig) 267 | print 'signal: mean, std, cut = ',np.mean(sig),np.std(sig),sigcut 268 | 269 | 270 | print 'Searching for lowest-background image' 271 | ref_sky = 1.e6 272 | for f in files: 273 | print f.name, f.fw, f.sky, f.signal 274 | if (f.sky < ref_sky) and (f.fw > params.reference_min_seeing) and \ 275 | (f.roundness < params.reference_max_roundness) and (f.signal > sigcut) and not(f.name in reference_exclude): 276 | ref_sky = f.sky 277 | ref_sky = np.max([ref_sky,params.pixel_max/1.e5]) 278 | 279 | print 'Searching for best-seeing image' 280 | ref_seeing = 100.0 281 | for f in files: 282 | print f.name, f.fw, f.sky, f.signal 283 | if (f.fw < ref_seeing) and (f.fw > params.reference_min_seeing) and \ 284 | (f.roundness < params.reference_max_roundness) and (f.signal > sigcut) and not(f.name in reference_exclude): 285 | ref_seeing = f.fw 286 | best_seeing_ref = f 287 | 288 | 289 | ref_list = [] 290 | while len(ref_list) < params.min_ref_images: 291 | ref_list = [] 292 | print 'Reference FWHM = ',ref_seeing 293 | print 'Cutoff FWHM for reference = ',params.reference_seeing_factor*ref_seeing 294 | print 'Cutoff background for reference = ',params.reference_sky_factor*ref_sky 295 | print 'Combining for reference:' 296 | for f in files: 297 | if (f.fw < params.reference_seeing_factor*ref_seeing) and \ 298 | (f.roundness < params.reference_max_roundness) and (f.sky < params.reference_sky_factor*ref_sky) and \ 299 | (f.fw > params.reference_min_seeing) and (f.signal > sigcut) and not(f.name in reference_exclude): 300 | ref_list.append(f) 301 | print f.name, f.fw, f.sky, f.signal 302 | params.reference_seeing_factor *= 1.02 303 | params.reference_sky_factor *= 1.02 304 | 305 | if len(ref_list) > params.max_ref_images: 306 | ref_list = ref_list[:params.max_ref_images] 307 | 308 | sig = [] 309 | for f in ref_list: 310 | sig.append(f.signal) 311 | sig = np.asarray(sig) 312 | sigcut = np.mean(sig) - 2*np.std(sig) 313 | print 'signal: mean, std, cut = ',np.mean(sig),np.std(sig),sigcut 314 | 315 | ref_seeing = 1000 316 | ref_roundness = 2.0 317 | for f in ref_list: 318 | if (f.fw < ref_seeing) and (f.signal > sigcut): 319 | ref_sky = f.sky 320 | ref_seeing = f.fw 321 | ref_roundness = f.roundness 322 | best_seeing_ref = f 323 | 324 | 325 | # 326 | # Which ref image has the worst seeing? 327 | # 328 | worst_seeing = 0.0 329 | for f in ref_list: 330 | if f.fw > worst_seeing: 331 | worst_seeing = f.fw 332 | worst_seeing_ref = f 333 | 334 | 335 | if params.ref_image_list: 336 | with open(params.loc_output+os.path.sep+params.ref_image_list,'w') as fid: 337 | for f in ref_list: 338 | fid.write(f.name+' '+str(f.fw)+' '+str(f.sky)+' '+str(f.signal)+'\n') 339 | 340 | # 341 | # Find the locations of the brightest stars to use as stamp positions 342 | # if required 343 | # 344 | stamp_positions = None 345 | if params.use_stamps: 346 | stars = PH.choose_stamps(best_seeing_ref,params) 347 | stamp_positions = stars[:,0:2] 348 | 349 | 350 | # 351 | # Construct the reference image. 352 | # 353 | ref = np.zeros([1,1]) 354 | sum1 = 0 355 | sum2 = 0 356 | 357 | good_ref_list = [] 358 | 359 | for f in ref_list: 360 | f.blur = IM.boxcar_blur(f.image) 361 | good_ref_list.append(f) 362 | print 'difference_image:',f.name,best_seeing_ref.name 363 | 364 | 365 | 366 | if not(params.use_GPU) and (params.n_parallel > 1): 367 | 368 | # 369 | # Use ParallelProcessing to process images in the reference list 370 | # 371 | 372 | pool = Pool(params.n_parallel) 373 | results = pool.map(process_reference_image_helper, 374 | itertools.izip(ref_list, 375 | itertools.repeat((best_seeing_ref, 376 | params, 377 | stamp_positions)))) 378 | 379 | for i, f in enumerate(ref_list): 380 | f.result = results[i] 381 | 382 | else: 383 | 384 | for f in ref_list: 385 | f.result = process_reference_image(f,(best_seeing_ref,params, 386 | stamp_positions)) 387 | 388 | # 389 | # Remove bad reference models 390 | # 391 | 392 | rlist = [g for g in good_ref_list] 393 | for g in rlist: 394 | if not(isinstance(g.result.diff,np.ndarray)): 395 | print 'removing',g.name 396 | good_ref_list.remove(g) 397 | 398 | print 'good reference list:' 399 | for g in good_ref_list: 400 | print g.name 401 | 402 | print 'kappa-clipping reference list' 403 | for iterations in range(5): 404 | if len(good_ref_list) < 4: 405 | break 406 | sd = np.zeros(len(good_ref_list)) 407 | for i, g in enumerate(good_ref_list): 408 | print g.name, g.result.diff 409 | sd[i] = np.std(g.result.diff) 410 | sds = sd.std() 411 | sdm = sd.mean() 412 | rlist = [g for g in good_ref_list] 413 | for g in rlist: 414 | if np.std(g.result.diff) > (sdm + 2.5*sds): 415 | print 'removing',g.name 416 | good_ref_list.remove(g) 417 | 418 | # 419 | # Combine the good reference models 420 | # 421 | g = good_ref_list[0] 422 | gstack = np.zeros([len(good_ref_list),g.result.model.shape[0],g.result.model.shape[1]]) 423 | var_ref = np.zeros_like(g.result.model) 424 | mask = np.ones_like(g.result.model) 425 | print 'final reference list' 426 | for i,g in enumerate(good_ref_list): 427 | if isinstance(g.result.model,np.ndarray): 428 | print g.name, np.std(g.result.diff), np.median(g.result.model) 429 | IO.write_image(g.result.model,params.loc_output+os.path.sep+'mr_'+g.name) 430 | gstack[i,:,:] = g.result.model 431 | var_ref += g.result.model/params.gain + (params.readnoise/params.gain)**2 432 | mask *= g.mask 433 | rr = np.median(gstack,axis=0) 434 | IO.write_image(rr,params.loc_output+os.path.sep+reference_image) 435 | IO.write_image(mask,params.loc_output+os.path.sep+'mask_'+reference_image) 436 | var_ref /= np.float(len(good_ref_list)) 437 | IO.write_image(var_ref,params.loc_output+os.path.sep+'var_'+reference_image) 438 | if params.error_image_prefix is not None: 439 | IO.write_image(np.sqrt(var_ref),params.loc_output+os.path.sep+params.error_image_prefix+reference_image) 440 | 441 | for f in ref_list: 442 | f.result = None 443 | 444 | return stamp_positions 445 | 446 | 447 | 448 | def process_image(f,args): 449 | ref,params,stamp_positions,star_positions,star_group_boundaries,star_unsort_index, detector_mean_positions_x,detector_mean_positions_y = args 450 | dtarget = params.loc_output+os.path.sep+'d_'+f.name 451 | if not(os.path.exists(dtarget)): 452 | 453 | # 454 | # Compute difference image 455 | # 456 | result = difference_image(ref,f, 457 | params, 458 | stamp_positions=stamp_positions, 459 | psf_image=params.loc_output+os.path.sep+ 460 | 'psf.fits', 461 | star_positions=star_positions, 462 | star_group_boundaries=star_group_boundaries, 463 | detector_mean_positions_x=detector_mean_positions_x, 464 | detector_mean_positions_y=detector_mean_positions_y) 465 | del f.image 466 | del f.mask 467 | del f.inv_variance 468 | 469 | 470 | # 471 | # Save photometry to a file 472 | # 473 | if isinstance(result.flux,np.ndarray): 474 | #if not(params.use_GPU): 475 | print 'ungrouping fluxes' 476 | result.flux = result.flux[star_unsort_index].copy() 477 | result.dflux = result.dflux[star_unsort_index].copy() 478 | np.savetxt(params.loc_output+os.path.sep+f.name+'.flux', 479 | np.vstack((result.flux,result.dflux)).T) 480 | f.flux = result.flux.copy() 481 | f.dflux = result.dflux.copy() 482 | 483 | # 484 | # Save output images to files 485 | # 486 | if isinstance(result.diff,np.ndarray): 487 | IO.write_image(result.diff,params.loc_output+os.path.sep+'d_'+f.name) 488 | IO.write_image(result.model,params.loc_output+os.path.sep+'m_'+f.name) 489 | IO.write_image(result.norm,params.loc_output+os.path.sep+'n_'+f.name) 490 | IO.write_image(result.mask,params.loc_output+os.path.sep+'z_'+f.name) 491 | return 0 492 | 493 | 494 | def process_image_helper(args): 495 | return process_image(*args) 496 | 497 | 498 | 499 | 500 | def imsub_all_fits(params,reference='ref.fits'): 501 | 502 | # 503 | # Create the output directory if it doesn't exist 504 | # 505 | if not (os.path.exists(params.loc_output)): 506 | os.mkdir(params.loc_output) 507 | 508 | # 509 | # The degree of spatial shape changes has to be at least as 510 | # high as the degree of spatial photometric scale 511 | # 512 | if (params.sdeg < params.pdeg): 513 | print 'Increasing params.sdeg to ',params.pdeg 514 | params.sdeg = params.pdeg 515 | 516 | # 517 | # Print out the parameters for this run. 518 | # 519 | print 'Parameters:' 520 | for par in dir(params): 521 | print par, getattr(params, par) 522 | print 523 | 524 | 525 | 526 | # 527 | # Determine our list of images 528 | # 529 | all_files = os.listdir(params.loc_data) 530 | all_files.sort() 531 | files = [] 532 | for f in all_files: 533 | print 'file', f 534 | if fnmatch.fnmatch(f,params.name_pattern): 535 | g = DS.Observation(params.loc_data+os.path.sep+f,params) 536 | del g.data 537 | del g.mask 538 | print 'fw', g.fw 539 | if g.fw > 0.0: 540 | files.append(g) 541 | print g.name, 'accepted' 542 | 543 | if len(files) < 3: 544 | print 'Only',len(files),'files found matching',params.name_pattern 545 | print 'Exiting' 546 | sys.exit(0) 547 | 548 | # 549 | # Have we specified a registration template? 550 | # 551 | if params.registration_image: 552 | reg = DS.Observation(params.registration_image,params) 553 | else: 554 | reg = DS.EmptyBase() 555 | reg.fw = 999.0 556 | for f in files: 557 | if (f.fw < reg.fw) and (f.fw > params.reference_min_seeing) and (f.sky < params.registration_max_background): 558 | reg = f 559 | 560 | print 'Registration image:',reg.name 561 | 562 | 563 | # 564 | # Register images 565 | # 566 | print 'Registering images' 567 | files_copy = [f for f in files] 568 | for f in files: 569 | print f.name 570 | if f == reg: 571 | f.image = f.data 572 | rf = params.loc_output+os.path.sep+'r_'+f.name 573 | IO.write_image(f.image,rf) 574 | else: 575 | if not f.register(reg,params): 576 | files_copy.remove(f) 577 | # delete image arrays to save memory 578 | del f.image 579 | del f.mask 580 | del f.inv_variance 581 | del reg.data 582 | del reg.image 583 | del reg.mask 584 | del reg.inv_variance 585 | files = files_copy 586 | 587 | # 588 | # Write image names and dates to a file 589 | # 590 | if params.image_list_file: 591 | try: 592 | with open(params.loc_output+os.path.sep+params.image_list_file,'w') as fid: 593 | for f in files: 594 | date = None 595 | if params.datekey: 596 | date = IO.get_date(params.loc_data+os.path.sep+f.name, 597 | key=params.datekey)-2450000 598 | if date: 599 | fid.write(f.name+' %10.5f\n'%date) 600 | else: 601 | fid.write(f.name) 602 | except: 603 | raise 604 | 605 | # 606 | # Make the photometric reference image if we don't have it. 607 | # Find stamp positions if required. 608 | # 609 | if not(os.path.exists(params.loc_output+os.path.sep+reference)): 610 | print 'Reg = ',reg.name 611 | stamp_positions = make_reference(files,reg,params,reference_image=reference) 612 | ref = DS.Observation(params.loc_output+os.path.sep+reference,params) 613 | mask, _ = IO.read_fits_file(params.loc_output+os.path.sep+'mask_'+reference) 614 | variance, _ = IO.read_fits_file(params.loc_output+os.path.sep+'var_'+reference) 615 | ref.mask = mask 616 | ref.inv_variance = 1.0/variance 617 | ref.register(reg,params) 618 | else: 619 | ref = DS.Observation(params.loc_output+os.path.sep+reference,params) 620 | if os.path.exists(params.loc_output+os.path.sep+'mask_'+reference): 621 | mask, _ = IO.read_fits_file(params.loc_output+os.path.sep+'mask_'+reference) 622 | else: 623 | mask = np.ones_like(ref.data) 624 | ref.mask = mask 625 | ref.register(reg,params) 626 | stamp_positions = None 627 | if params.use_stamps: 628 | stamp_file = params.loc_output+os.path.sep+'stamp_positions' 629 | if os.path.exists(stamp_file): 630 | stamp_positions = np.genfromtxt(stamp_file) 631 | else: 632 | stars = PF.choose_stamps(ref,params) 633 | stamp_positions = stars[:,0:2] 634 | np.savetxt(stamp_file,stamp_positions) 635 | 636 | pm = params.pixel_max 637 | params.pixel_max *= 0.9 638 | ref.mask *= IM.compute_saturated_pixel_mask(ref.image,params) 639 | params.pixel_max = pm 640 | ref.blur = IM.boxcar_blur(ref.image) 641 | if params.mask_cluster: 642 | ref.mask *= IM.mask_cluster(ref.image,ref.mask,params) 643 | 644 | # 645 | # Detect stars and compute the PSF if we are doing photometry 646 | # 647 | star_positions = None 648 | sky = 0.0 649 | if params.do_photometry: 650 | star_file = params.loc_output+os.path.sep+'star_positions' 651 | psf_file = params.loc_output+os.path.sep+'psf.fits' 652 | if not(os.path.exists(psf_file)) or not(os.path.exists(star_file)): 653 | stars = PH.compute_psf_image(params,ref,psf_image=psf_file) 654 | star_positions = stars[:,0:2] 655 | star_sky = stars[:,4] 656 | if os.path.exists(star_file): 657 | star_positions = np.genfromtxt(star_file) 658 | star_sky = star_positions[:,0]*0.0; 659 | else: 660 | np.savetxt(star_file,star_positions) 661 | 662 | print 'sky =', sky 663 | 664 | # 665 | # If we have pre-determined star positions 666 | # 667 | if params.star_file: 668 | stars = np.genfromtxt(params.star_file) 669 | star_positions = stars[:,1:3] 670 | if params.star_reference_image: 671 | star_ref, h = IO.read_fits_file(params.star_reference_image) 672 | offset, _, _ = register_translation(star_ref, ref.image, 1000) 673 | dy, dx = offset 674 | #dy, dx = IM.positional_shift(ref.image,star_ref) 675 | print 'position shift =',dx,dy 676 | star_positions[:,0] -= dx 677 | star_positions[:,1] -= dy 678 | np.savetxt(star_file,star_positions) 679 | 680 | # 681 | # If we are using a CPU, group the stars by location 682 | # 683 | print 'Group_check' 684 | print 'params.do_photometry',params.do_photometry 685 | print 'params.use_GPU',params.use_GPU 686 | if params.do_photometry: 687 | star_group_boundaries = None 688 | detector_mean_positions_x = None 689 | detector_mean_positions_y = None 690 | star_unsort_index = None 691 | star_sort_index,star_group_boundaries,detector_mean_positions_x,detector_mean_positions_y = \ 692 | PH.group_stars_ccd(params,star_positions,params.loc_output+os.path.sep+reference) 693 | star_positions = star_positions[star_sort_index] 694 | star_sky = star_sky[star_sort_index] 695 | star_unsort_index = np.argsort(star_sort_index) 696 | 697 | 698 | # 699 | # Do photometry of the reference image 700 | # 701 | if params.do_photometry: 702 | ref_flux_file = params.loc_output+os.path.sep+'ref.flux' 703 | if not(os.path.exists(ref_flux_file)): 704 | result = difference_image(ref,ref,params, 705 | stamp_positions=stamp_positions, 706 | psf_image=psf_file, 707 | star_positions=star_positions, 708 | star_group_boundaries=star_group_boundaries, 709 | detector_mean_positions_x=detector_mean_positions_x, 710 | detector_mean_positions_y=detector_mean_positions_y, 711 | star_sky=star_sky) 712 | if isinstance(result.flux,np.ndarray): 713 | print 'ungrouping fluxes' 714 | result.flux = result.flux[star_unsort_index].copy() 715 | result.dflux = result.dflux[star_unsort_index].copy() 716 | np.savetxt(ref_flux_file, 717 | np.vstack((result.flux,result.dflux)).T) 718 | 719 | # 720 | # Process images 721 | # 722 | 723 | if params.make_difference_images: 724 | 725 | if not(params.use_GPU) and (params.n_parallel > 1): 726 | 727 | pool = Pool(params.n_parallel) 728 | pool.map(process_image_helper, itertools.izip(files, itertools.repeat( 729 | (ref,params,stamp_positions,star_positions,star_group_boundaries, 730 | star_unsort_index,detector_mean_positions_x,detector_mean_positions_y)))) 731 | 732 | else: 733 | 734 | for f in files: 735 | process_image(f,(ref,params,stamp_positions,star_positions, 736 | star_group_boundaries,star_unsort_index, 737 | detector_mean_positions_x,detector_mean_positions_y)) 738 | 739 | return files 740 | 741 | 742 | def do_photometry(params,extname='newflux',star_file='star_positions', 743 | psf_file='psf.fits',star_positions=None,reference_image='ref.fits'): 744 | 745 | # 746 | # Determine our list of files 747 | # 748 | all_files = os.listdir(params.loc_data) 749 | all_files.sort() 750 | files = [] 751 | for f in all_files: 752 | if fnmatch.fnmatch(f,params.name_pattern): 753 | g = DS.Observation(params.loc_data+os.path.sep+f,params) 754 | if g.fw > 0.0: 755 | files.append(g) 756 | 757 | ref = DS.Observation(params.loc_output+os.path.sep+reference_image,params) 758 | ref.register(ref,params) 759 | 760 | # 761 | # Detect stars and compute the PSF if necessary 762 | # 763 | 764 | psf_file = params.loc_output+os.path.sep+psf_file 765 | star_file = params.loc_output+os.path.sep+star_file 766 | 767 | print psf_file 768 | print os.path.exists(psf_file) 769 | print star_file 770 | print os.path.exists(star_file) 771 | 772 | if not(os.path.exists(psf_file)) or not(os.path.exists(star_file)): 773 | stars = PH.compute_psf_image(params,ref,psf_image=psf_file) 774 | 775 | if star_positions is None: 776 | 777 | if os.path.exists(star_file): 778 | star_positions = np.genfromtxt(star_file)[:,:2] 779 | else: 780 | star_positions = stars[:,0:2] 781 | 782 | 783 | # 784 | # Group the stars by location 785 | # 786 | star_group_boundaries = None 787 | detector_mean_positions_x = None 788 | detector_mean_positions_y = None 789 | star_sort_index,star_group_boundaries,detector_mean_positions_x,detector_mean_positions_y = \ 790 | PH.group_stars_ccd(params,star_positions,params.loc_output+os.path.sep+reference_image) 791 | star_positions = star_positions[star_sort_index] 792 | star_unsort_index = np.argsort(star_sort_index) 793 | 794 | # 795 | # Process the reference image 796 | # 797 | print 'Processing',reference_image 798 | ref = DS.Observation(params.loc_output+os.path.sep+reference_image,params) 799 | #reg = Observation(params.loc_data+os.path.sep+ 800 | # params.registration_image,params) 801 | mask, _ = IO.read_fits_file(params.loc_output+os.path.sep+'mask_'+reference_image) 802 | variance, _ = IO.read_fits_file(params.loc_output+os.path.sep+'var_'+reference_image) 803 | ref.mask = mask 804 | ref.inv_variance = 1.0/variance + (1-mask) 805 | ref.register(ref,params) 806 | smask = IM.compute_saturated_pixel_mask(ref.image,params) 807 | ref.inv_variance += (1 - (smask*mask))*1.e-12 808 | ktable = params.loc_output+os.path.sep+'k_'+os.path.basename(reference_image) 809 | kernelIndex, extendedBasis, c, params = IO.read_kernel_table(ktable,params) 810 | kernelRadius = np.max(kernelIndex[:,0])+1 811 | if np.sum(extendedBasis) > 0: 812 | kernelRadius += 1 813 | print 'kernelIndex', kernelIndex 814 | print 'extendedBasis',extendedBasis 815 | print 'coeffs', c 816 | print 'kernelRadius',kernelRadius 817 | print 'star_positions', star_positions.shape 818 | phot_target, _ = IO.read_fits_file(params.loc_output+os.path.sep+'clean_'+reference_image) 819 | ref.flux, ref.dflux = CIF.photom_all_stars_simultaneous(phot_target,ref.inv_variance,star_positions, 820 | psf_file,c,kernelIndex,extendedBasis,kernelRadius,params, 821 | star_group_boundaries, 822 | detector_mean_positions_x,detector_mean_positions_y) 823 | 824 | if isinstance(ref.flux,np.ndarray): 825 | if not(params.use_GPU): 826 | print 'ungrouping fluxes' 827 | ref.flux = ref.flux[star_unsort_index].copy() 828 | ref.dflux = ref.dflux[star_unsort_index].copy() 829 | print ref.flux.shape, star_positions.shape 830 | np.savetxt(params.loc_output+os.path.sep+reference_image+'.'+extname, 831 | np.vstack((ref.flux,ref.dflux))) 832 | 833 | # 834 | # Process difference images 835 | # 836 | for f in files: 837 | 838 | if not(os.path.exists(params.loc_output+os.path.sep+f.name+'.'+extname)): 839 | 840 | print 'Processing',f.name 841 | target = f.name 842 | dtarget = params.loc_output+os.path.sep+'d_'+os.path.basename(target) 843 | ntarget = params.loc_output+os.path.sep+'n_'+os.path.basename(target) 844 | ztarget = params.loc_output+os.path.sep+'z_'+os.path.basename(target) 845 | ktable = params.loc_output+os.path.sep+'k_'+os.path.basename(target) 846 | 847 | if os.path.exists(dtarget) and os.path.exists(ntarget) and os.path.exists(ktable): 848 | 849 | norm, h = IO.read_fits_file(ntarget) 850 | diff, h = IO.read_fits_file(dtarget) 851 | mask, h = IO.read_fits_file(ztarget) 852 | inv_var = (norm/diff)**2 + (1-mask) 853 | 854 | kernelIndex, extendedBasis, c, params = IO.read_kernel_table(ktable,params) 855 | kernelRadius = np.max(kernelIndex[:,0])+1 856 | if np.sum(extendedBasis) > 0: 857 | kernelRadius += 1 858 | 859 | print 'kernelIndex', kernelIndex 860 | print 'extendedBasis',extendedBasis 861 | print 'coeffs', c 862 | print 'kernelRadius',kernelRadius 863 | 864 | IO.write_image(diff,params.loc_output+os.path.sep+'diff1.fits') 865 | diff = IM.undo_photometric_scale(diff,c,params.pdeg) 866 | IO.write_image(diff,params.loc_output+os.path.sep+'diff2.fits') 867 | IO.write_image(inv_var,params.loc_output+os.path.sep+'inv_var.fits') 868 | IO.write_kernel_table(params.loc_output+os.path.sep+'ktable.fits',kernelIndex,extendedBasis,c,params) 869 | 870 | 871 | flux, dflux = CI.photom_all_stars(diff,inv_var,star_positions, 872 | psf_file,c,kernelIndex,extendedBasis, 873 | kernelRadius,params, 874 | star_group_boundaries, 875 | detector_mean_positions_x, 876 | detector_mean_positions_y) 877 | 878 | 879 | 880 | print 'flux[100:110]:' 881 | print flux[100:110] 882 | if isinstance(flux,np.ndarray): 883 | if not(params.use_GPU): 884 | print 'ungrouping fluxes' 885 | flux = flux[star_unsort_index].copy() 886 | dflux = dflux[star_unsort_index].copy() 887 | print 'unsort flux[100:110]:' 888 | print flux[100:110] 889 | np.savetxt(params.loc_output+os.path.sep+f.name+'.'+extname, 890 | np.vstack((flux,dflux)).T) 891 | 892 | sys.exit(0) 893 | -------------------------------------------------------------------------------- /Code/c_interface_functions.py: -------------------------------------------------------------------------------- 1 | 2 | import numpy as np 3 | import fnmatch 4 | import io_functions as IO 5 | import image_functions as IM 6 | 7 | from scipy.sparse import csr_matrix, csc_matrix 8 | from scipy.sparse import linalg as sp_linalg 9 | 10 | 11 | # 12 | # Load the C library versions of the CUDA functions 13 | # 14 | import os 15 | import ctypes 16 | from numpy.ctypeslib import ndpointer 17 | dllabspath = os.path.dirname(os.path.abspath(__file__)) + os.path.sep + 'c_functions.so' 18 | lib = ctypes.cdll.LoadLibrary(dllabspath) 19 | 20 | cu_convolve_image_psf = lib.cu_convolve_image_psf 21 | cu_photom = lib.cu_photom 22 | cu_make_residual = lib.cu_make_residual 23 | cu_multi_photom = lib.cu_multi_photom 24 | cu_photom_converge = lib.cu_photom_converge 25 | cu_compute_model = lib.cu_compute_model 26 | cu_compute_vector = lib.cu_compute_vector 27 | cu_compute_matrix = lib.cu_compute_matrix 28 | cu_compute_vector_stamps = lib.cu_compute_vector_stamps 29 | cu_compute_matrix_stamps = lib.cu_compute_matrix_stamps 30 | 31 | 32 | # 33 | # Specify the ctypes data types for the C function calls 34 | # 35 | 36 | cu_convolve_image_psf.restype = None 37 | cu_convolve_image_psf.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int, 38 | ctypes.c_int, ctypes.c_int, 39 | ctypes.c_int, ctypes.c_int, 40 | ctypes.c_int, ctypes.c_int, 41 | ctypes.c_int, 42 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 43 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 44 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 45 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 46 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 47 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 48 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 49 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 50 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 51 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 52 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 53 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS")] 54 | 55 | cu_photom.restype = None 56 | cu_photom.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, 57 | ctypes.c_int, ctypes.c_int, ctypes.c_int, 58 | ctypes.c_int, 59 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 60 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 61 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 62 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 63 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 64 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 65 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 66 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 67 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 68 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 69 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 70 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 71 | ctypes.c_long, ctypes.c_int, ctypes.c_int, 72 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 73 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 74 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 75 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 76 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 77 | ctypes.c_int] 78 | 79 | cu_make_residual.restype = None 80 | cu_make_residual.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, 81 | ctypes.c_int, ctypes.c_int, ctypes.c_int, 82 | ctypes.c_int, 83 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 84 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 85 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 86 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 87 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 88 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 89 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 90 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 91 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 92 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 93 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 94 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 95 | ctypes.c_long, ctypes.c_int, ctypes.c_int, 96 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 97 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 98 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 99 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 100 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 101 | ctypes.c_int] 102 | 103 | 104 | cu_multi_photom.restype = None 105 | cu_multi_photom.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, 106 | ctypes.c_int, ctypes.c_int, ctypes.c_int, 107 | ctypes.c_int, 108 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 109 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 110 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 111 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 112 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 113 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 114 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 115 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 116 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 117 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 118 | ctypes.c_long, ctypes.c_int, ctypes.c_int, 119 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 120 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 121 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 122 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 123 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 124 | ctypes.c_int, 125 | ctypes.POINTER(ctypes.c_int), 126 | ctypes.POINTER(ctypes.POINTER(ctypes.c_int)), 127 | ctypes.POINTER(ctypes.POINTER(ctypes.c_int)), 128 | ctypes.POINTER(ctypes.POINTER(ctypes.c_double)), 129 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 130 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 131 | ctypes.c_int] 132 | 133 | 134 | cu_photom_converge.restype = None 135 | cu_photom_converge.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int, 136 | ctypes.c_int, ctypes.c_int, 137 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 138 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 139 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 140 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 141 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 142 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 143 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 144 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 145 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 146 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 147 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 148 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 149 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 150 | ctypes.c_double, 151 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 152 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 153 | ctypes.c_double,ctypes.c_double, 154 | ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, 155 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 156 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 157 | ctypes.c_double,ctypes.c_int,ctypes.c_double] 158 | 159 | 160 | cu_compute_model.restype = None 161 | cu_compute_model.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int, 162 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 163 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 164 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 165 | ctypes.c_int, 166 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 167 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 168 | ctypes.c_int, ctypes.c_int, 169 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 170 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS")] 171 | 172 | 173 | cu_compute_vector.restype = None 174 | cu_compute_vector.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int, 175 | ctypes.c_int, ctypes.c_int, 176 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 177 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 178 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 179 | ctypes.c_int, ctypes.c_int, 180 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 181 | ctypes.c_int, ctypes.c_int, 182 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 183 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 184 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 185 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 186 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS")] 187 | 188 | 189 | cu_compute_matrix.restype = None 190 | cu_compute_matrix.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int, 191 | ctypes.c_int, ctypes.c_int, 192 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 193 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 194 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 195 | ctypes.c_int, ctypes.c_int, 196 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 197 | ctypes.c_int, ctypes.c_int, ctypes.c_int, 198 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 199 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 200 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 201 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS")] 202 | 203 | 204 | cu_compute_vector_stamps.restype = None 205 | cu_compute_vector_stamps.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int, 206 | ctypes.c_int, ctypes.c_int, ctypes.c_int, 207 | ctypes.c_int, 208 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 209 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 210 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 211 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 212 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 213 | ctypes.c_int, ctypes.c_int, 214 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 215 | ctypes.c_int, ctypes.c_int, 216 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 217 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 218 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 219 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 220 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS")] 221 | 222 | 223 | cu_compute_matrix_stamps.restype = None 224 | cu_compute_matrix_stamps.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int, 225 | ctypes.c_int, ctypes.c_int, ctypes.c_int, 226 | ctypes.c_int, 227 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 228 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 229 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 230 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 231 | ndpointer(ctypes.c_int, flags="C_CONTIGUOUS"), 232 | ctypes.c_int, ctypes.c_int, 233 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 234 | ctypes.c_int, ctypes.c_int, ctypes.c_int, 235 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 236 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 237 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS"), 238 | ndpointer(ctypes.c_double, flags="C_CONTIGUOUS")] 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | def compute_matrix_and_vector_cuda(R,RB,T,Vinv,mask,kernelIndex,extendedBasis, 247 | kernelRadius,params,stamp_positions=None): 248 | 249 | # Create a numpy array for matrix H 250 | dp = (params.pdeg+1)*(params.pdeg+2)/2 251 | ds = (params.sdeg+1)*(params.sdeg+2)/2 252 | db = (params.bdeg+1)*(params.bdeg+2)/2 253 | hs = (kernelIndex.shape[0]-1)*ds+dp+db 254 | 255 | H = np.zeros([hs,hs]).astype(np.float64).copy() 256 | V = np.zeros(hs).astype(np.float64).copy() 257 | 258 | # Fill the elements of H 259 | print hs,' * ',hs,' elements' 260 | k0 = kernelIndex[:,0].astype(np.int32).copy() 261 | k1 = kernelIndex[:,1].astype(np.int32).copy() 262 | if params.use_stamps: 263 | posx = np.float64(stamp_positions[:params.nstamps,0].copy()-1.0) 264 | posy = np.float64(stamp_positions[:params.nstamps,1].copy()-1.0) 265 | cu_compute_matrix_stamps(params.pdeg, 266 | params.sdeg, 267 | params.bdeg, 268 | R.shape[1], 269 | R.shape[0], 270 | params.nstamps, 271 | params.stamp_half_width, 272 | posx, 273 | posy, 274 | k0, 275 | k1, 276 | extendedBasis, 277 | kernelIndex.shape[0], 278 | np.int(kernelRadius), 279 | H, 256, hs, hs, 280 | np.float64(R), np.float64(RB), np.float64(Vinv), 281 | np.float64(mask)) 282 | else: 283 | cu_compute_matrix(params.pdeg, params.sdeg, 284 | params.bdeg, 285 | R.shape[1], R.shape[0], 286 | k0, 287 | k1, 288 | extendedBasis, 289 | kernelIndex.shape[0], np.int(kernelRadius), 290 | H, 256, hs, hs, 291 | np.float64(R),np.float64(RB),np.float64(Vinv),np.float64(mask)) 292 | 293 | # Fill the elements of V 294 | if params.use_stamps: 295 | cu_compute_vector_stamps(params.pdeg, params.sdeg, 296 | params.bdeg, 297 | R.shape[1], R.shape[0], params.nstamps, 298 | params.stamp_half_width, 299 | posx, 300 | posy, 301 | k0, 302 | k1, 303 | extendedBasis, 304 | kernelIndex.shape[0], np.int(kernelRadius), 305 | V, 256, hs, np.float64(R), np.float64(RB), 306 | np.float64(T), np.float64(Vinv), np.float64(mask)) 307 | else: 308 | cu_compute_vector(params.pdeg, params.sdeg, 309 | params.bdeg, 310 | R.shape[1], R.shape[0], 311 | k0, 312 | k1, 313 | extendedBasis, 314 | kernelIndex.shape[0], np.int(kernelRadius), 315 | V, 256, hs, np.float64(R), np.float64(RB), np.float64(T), 316 | np.float64(Vinv), np.float64(mask)) 317 | return H, V, (R, RB) 318 | 319 | 320 | def compute_model_cuda(image_size,(R,RB),c,kernelIndex,extendedBasis,params): 321 | 322 | 323 | # Create a numpy array for the model M 324 | M = np.zeros(image_size).astype(np.float64).copy() 325 | 326 | # Call the cuda function to perform the convolution 327 | blockDim = (256,1,1) 328 | gridDim = (image_size[1],image_size[0])+(1,) 329 | k0 = kernelIndex[:,0].astype(np.int32).copy() 330 | k1 = kernelIndex[:,1].astype(np.int32).copy() 331 | c64 = c.astype(np.float64).copy() 332 | cu_compute_model(params.pdeg, params.sdeg, 333 | params.bdeg, k0, 334 | k1, extendedBasis, 335 | kernelIndex.shape[0], 336 | c64, M, image_size[1], image_size[0], np.float64(R), np.float64(RB)) 337 | return M 338 | 339 | 340 | 341 | 342 | def photom_all_stars(diff,inv_variance,positions,psf_image,c,kernelIndex, 343 | extendedBasis,kernelRadius,params, 344 | star_group_boundaries, 345 | detector_mean_positions_x,detector_mean_positions_y): 346 | 347 | from astropy.io import fits 348 | # Read the PSF 349 | psf,psf_hdr = fits.getdata(psf_image,0,header='true') 350 | psf_height = psf_hdr['PSFHEIGH'] 351 | psf_x = psf_hdr['PSFX'] 352 | psf_y = psf_hdr['PSFY'] 353 | psf_size = psf.shape[1] 354 | psf_fit_rad = params.psf_fit_radius 355 | if params.psf_profile_type == 'gaussian': 356 | psf_sigma_x = psf_hdr['PAR1']*0.8493218 357 | psf_sigma_y = psf_hdr['PAR2']*0.8493218 358 | psf_parameters = np.array([psf_size,psf_height,psf_sigma_x,psf_sigma_y,psf_x, 359 | psf_y,psf_fit_rad,params.gain]).astype(np.float64) 360 | print 'psf_parameters',psf_parameters 361 | profile_type = 0 362 | elif params.psf_profile_type == 'moffat25': 363 | print 'params.psf_profile_type moffat25 not working yet. Exiting.' 364 | sys.exit(0) 365 | psf_sigma_x = psf_hdr['PAR1'] 366 | psf_sigma_y = psf_hdr['PAR2'] 367 | psf_sigma_xy = psf_hdr['PAR3'] 368 | psf_parameters = np.array([psf_size,psf_height,psf_sigma_x,psf_sigma_y,psf_x, 369 | psf_y, 370 | psf_fit_rad,params.gain,psf_sigma_xy]).astype(np.float64) 371 | print 'psf_parameters',psf_parameters 372 | profile_type = 1 373 | else: 374 | print 'params.psf_profile_type undefined' 375 | sys.exit(0) 376 | 377 | 378 | k0 = kernelIndex[:,0].astype(np.int32).copy() 379 | k1 = kernelIndex[:,1].astype(np.int32).copy() 380 | if params.star_file_is_one_based: 381 | posx = np.float64(positions[:,0]-1.0) 382 | posy = np.float64(positions[:,1]-1.0) 383 | else: 384 | posx = np.float64(positions[:,0]) 385 | posy = np.float64(positions[:,1]) 386 | 387 | #psf_0 = convolve_undersample(psf[0]).astype(np.float64).copy() 388 | #psf_xd = convolve_undersample(psf[1]).astype(np.float64).copy()*0.0 389 | #psf_yd = convolve_undersample(psf[2]).astype(np.float64).copy()*0.0 390 | #psf_0 = psf[0].astype(np.float64).copy() 391 | #psf_xd = psf[1].astype(np.float64).copy()*0.0 392 | #psf_yd = psf[2].astype(np.float64).copy()*0.0 393 | psf_0 = psf.astype(np.float64) 394 | psf_xd = np.zeros_like(psf_0,dtype=np.float64) 395 | psf_yd = np.zeros_like(psf_0,dtype=np.float64) 396 | nstars = positions.shape[0] 397 | flux = np.zeros(nstars,dtype=np.float64) 398 | dflux = np.zeros(nstars,dtype=np.float64) 399 | c64 = c.astype(np.float64).copy() 400 | 401 | print 'nstars', nstars 402 | print 'flux', flux.shape 403 | print 'dflux', dflux.shape 404 | 405 | cu_photom(np.int(profile_type), diff.shape[1], diff.shape[0], params.pdeg, 406 | params.sdeg, c.shape[0], kernelIndex.shape[0], 407 | np.int(kernelRadius), k0, 408 | k1, extendedBasis, 409 | psf_parameters, psf_0, psf_xd, psf_yd, 410 | posx, posy, c64, flux, dflux, long(nstars), 16, 16, np.float64(diff), 411 | np.float64(inv_variance),np.int32(star_group_boundaries), 412 | np.float64(detector_mean_positions_x), 413 | np.float64(detector_mean_positions_y),star_group_boundaries.shape[0]) 414 | 415 | return flux, dflux 416 | 417 | 418 | 419 | 420 | 421 | def photom_all_stars_simultaneous(diff,inv_variance,positions,psf_image,c,kernelIndex, 422 | extendedBasis,kernelRadius,params, 423 | star_group_boundaries, 424 | detector_mean_positions_x,detector_mean_positions_y): 425 | 426 | from astropy.io import fits 427 | # Read the PSF 428 | psf,psf_hdr = fits.getdata(psf_image,0,header='true') 429 | psf_height = psf_hdr['PSFHEIGH'] 430 | psf_x = psf_hdr['PSFX'] 431 | psf_y = psf_hdr['PSFY'] 432 | psf_size = psf.shape[1] 433 | psf_fit_rad = params.psf_fit_radius 434 | #psf_fit_rad = 3.1 435 | if params.psf_profile_type == 'gaussian': 436 | psf_sigma_x = psf_hdr['PAR1']*0.8493218 437 | psf_sigma_y = psf_hdr['PAR2']*0.8493218 438 | psf_parameters = np.array([psf_size,psf_height,psf_sigma_x,psf_sigma_y,psf_x, 439 | psf_y,psf_fit_rad,params.gain]).astype(np.float64) 440 | print 'psf_parameters',psf_parameters 441 | profile_type = 0 442 | elif params.psf_profile_type == 'moffat25': 443 | print 'params.psf_profile_type moffat25 not working yet. Exiting.' 444 | sys.exit(0) 445 | psf_sigma_x = psf_hdr['PAR1'] 446 | psf_sigma_y = psf_hdr['PAR2'] 447 | psf_sigma_xy = psf_hdr['PAR3'] 448 | psf_parameters = np.array([psf_size,psf_height,psf_sigma_x,psf_sigma_y,psf_x, 449 | psf_y, 450 | psf_fit_rad,params.gain,psf_sigma_xy]).astype(np.float64) 451 | print 'psf_parameters',psf_parameters 452 | profile_type = 1 453 | else: 454 | print 'params.psf_profile_type undefined' 455 | sys.exit(0) 456 | 457 | 458 | k0 = kernelIndex[:,0].astype(np.int32).copy() 459 | k1 = kernelIndex[:,1].astype(np.int32).copy() 460 | if params.star_file_is_one_based: 461 | posx = np.float64(positions[:,0]-1.0) 462 | posy = np.float64(positions[:,1]-1.0) 463 | else: 464 | posx = np.float64(positions[:,0]-0.0) 465 | posy = np.float64(positions[:,1]-0.0) 466 | 467 | psf_0 = psf.astype(np.float64) 468 | psf_xd = np.zeros_like(psf_0,dtype=np.float64) 469 | psf_yd = np.zeros_like(psf_0,dtype=np.float64) 470 | nstars = positions.shape[0] 471 | flux = np.zeros(nstars+1,dtype=np.float64) 472 | dflux = np.zeros(nstars+1,dtype=np.float64) 473 | c64 = c.astype(np.float64).copy() 474 | 475 | print 'nstars', nstars 476 | print 'flux', flux.shape 477 | print 'dflux', dflux.shape 478 | 479 | i_index = ctypes.POINTER(ctypes.c_int)() 480 | j_index = ctypes.POINTER(ctypes.c_int)() 481 | value = ctypes.POINTER(ctypes.c_double)() 482 | n_entries = ctypes.c_int() 483 | 484 | rvec = np.zeros(nstars).astype(np.float64).copy() 485 | 486 | for iteration in range(1): 487 | 488 | cu_multi_photom(np.int(profile_type), diff.shape[1], diff.shape[0], params.pdeg, 489 | params.sdeg, c.shape[0], kernelIndex.shape[0], 490 | np.int(kernelRadius), k0, 491 | k1, extendedBasis, 492 | psf_parameters, psf_0, psf_xd, psf_yd, 493 | posx, posy, c64, long(nstars), 16, 16, np.float64(diff), 494 | np.float64(inv_variance),np.int32(star_group_boundaries), 495 | np.float64(detector_mean_positions_x), 496 | np.float64(detector_mean_positions_y),star_group_boundaries.shape[0],ctypes.byref(n_entries), 497 | ctypes.byref(i_index), ctypes.byref(j_index), ctypes.byref(value), rvec, flux[:nstars], iteration) 498 | 499 | 500 | n_e = np.int32(n_entries) 501 | 502 | buf_from_mem = ctypes.pythonapi.PyBuffer_FromMemory 503 | buf_from_mem.restype = ctypes.py_object 504 | 505 | buffer = buf_from_mem(i_index, n_e*np.dtype(np.int32).itemsize) 506 | i_ind = np.frombuffer(buffer, np.int32) 507 | 508 | buffer = buf_from_mem(j_index, n_e*np.dtype(np.int32).itemsize) 509 | j_ind = np.frombuffer(buffer, np.int32) 510 | 511 | buffer = buf_from_mem(value, n_e*np.dtype(np.float64).itemsize) 512 | val = np.frombuffer(buffer, np.float64) 513 | 514 | 515 | #for row in range(20): 516 | # print 'Row', row 517 | # q = np.where(i_ind == row) 518 | # print q 519 | # for qq in q[0]: 520 | # print j_ind[qq], val[qq] 521 | # print rvec[row] 522 | 523 | 524 | A = csc_matrix((val,(i_ind,j_ind)),shape=(nstars, nstars)) 525 | 526 | flux = np.float64(sp_linalg.spsolve(A, rvec)) 527 | dflux = np.sqrt(sp_linalg.spsolve(A, np.ones_like(rvec))) 528 | 529 | print 'flux =', flux 530 | print 'dflux =', dflux 531 | 532 | cdiff = np.float64(diff).copy() 533 | 534 | cu_make_residual(np.int(profile_type), diff.shape[1], diff.shape[0], params.pdeg, 535 | params.sdeg, c.shape[0], kernelIndex.shape[0], 536 | np.int(kernelRadius), k0, 537 | k1, extendedBasis, 538 | psf_parameters, psf_0, psf_xd, psf_yd, 539 | posx, posy, c64, flux, dflux, long(nstars), 16, 16, cdiff, 540 | np.float64(inv_variance),np.int32(star_group_boundaries), 541 | np.float64(detector_mean_positions_x), 542 | np.float64(detector_mean_positions_y),star_group_boundaries.shape[0]) 543 | 544 | IO.write_image(cdiff,params.loc_output+os.path.sep+'p_clean_ref.fits') 545 | 546 | return flux, dflux 547 | 548 | 549 | def convolve_image_with_psf(psf_image,image1,image2,c,kernelIndex,extendedBasis, 550 | kernelRadius,params): 551 | 552 | from astropy.io import fits 553 | 554 | # Read the PSF 555 | psf,psf_hdr = fits.getdata(psf_image,0,header='true') 556 | psf_height = psf_hdr['PSFHEIGH'] 557 | psf_x = psf_hdr['PSFX'] 558 | psf_y = psf_hdr['PSFY'] 559 | psf_size = psf.shape[1] 560 | psf_fit_rad = params.psf_fit_radius 561 | if params.psf_profile_type == 'gaussian': 562 | psf_sigma_x = psf_hdr['PAR1']*0.8493218 563 | psf_sigma_y = psf_hdr['PAR2']*0.8493218 564 | psf_parameters = np.array([psf_size,psf_height,psf_sigma_x,psf_sigma_y,psf_x, 565 | psf_y,psf_fit_rad,params.gain]).astype(np.float64) 566 | profile_type = 0 567 | elif params.psf_profile_type == 'moffat25': 568 | print 'params.psf_profile_type moffat25 not working yet. Exiting.' 569 | sys.exit(0) 570 | psf_sigma_x = psf_hdr['PAR1'] 571 | psf_sigma_y = psf_hdr['PAR2'] 572 | psf_sigma_xy = psf_hdr['PAR3'] 573 | psf_parameters = np.array([psf_size,psf_height,psf_sigma_x,psf_sigma_y,psf_x, 574 | psf_y, 575 | psf_fit_rad,params.gain,psf_sigma_xy]).astype(np.float64) 576 | profile_type = 1 577 | else: 578 | print 'params.psf_profile_type undefined' 579 | sys.exit(0) 580 | 581 | 582 | k0 = kernelIndex[:,0].astype(np.int32).copy() 583 | k1 = kernelIndex[:,1].astype(np.int32).copy() 584 | #psf_0 = convolve_undersample(psf[0]).astype(np.float64).copy() 585 | #psf_xd = convolve_undersample(psf[1]).astype(np.float64).copy()*0.0 586 | #psf_yd = convolve_undersample(psf[2]).astype(np.float64).copy()*0.0 587 | psf_0 = psf.astype(np.float64).copy() 588 | 589 | c64 = c.astype(np.float64).copy() 590 | 591 | 592 | psf_xd = psf.astype(np.float64).copy()*0.0 593 | psf_yd = psf.astype(np.float64).copy()*0.0 594 | 595 | image_section_size = 32 596 | convolved_image1 = (0.0*image1).astype(np.float64) 597 | convolved_image2 = (0.0*image1).astype(np.float64) 598 | 599 | cu_convolve_image_psf(np.int(profile_type),image1.shape[1], image1.shape[0], 600 | np.int(image_section_size), 601 | np.int(image_section_size), params.pdeg, 602 | params.sdeg,c.shape[0], 603 | kernelIndex.shape[0],np.int(kernelRadius),k0, 604 | k1,extendedBasis,psf_parameters,psf_0, 605 | psf_xd,psf_yd,c64,convolved_image1, 606 | convolved_image2, np.float64(image1), 607 | np.float64(image2)) 608 | 609 | return convolved_image1, convolved_image2 610 | 611 | 612 | 613 | 614 | 615 | def photom_variable_star(x0,y0,params,patch_half_width=15,converge=True,save_stamps=False,stamp_prefix='mosaic',locate=True,locate_iterations=2, 616 | locate_half_width=14,q_sigma_threshold=1.0,locate_date_range=None): 617 | 618 | from astropy.io import fits 619 | from scipy.ndimage.filters import median_filter 620 | 621 | outer_radius = 15 622 | inner_radius = 12 623 | diameter = 2*outer_radius + 1 624 | x = np.arange(diameter)-outer_radius 625 | xx,yy = np.meshgrid(x,x) 626 | filter_kernel = np.zeros((diameter,diameter)) 627 | filter_kernel[xx**2+yy**2<=outer_radius**2] = 1 628 | filter_kernel[xx**2+yy**2<=inner_radius**2] = 0 629 | 630 | 631 | def save_mosaic(stack,nfiles,patch_size,name,diff_std,threshold): 632 | stamps_per_row = int(np.sqrt(nfiles)) 633 | nrows = (nfiles-1)/stamps_per_row+1; 634 | mx = stamps_per_row*(patch_size+1)+1 635 | my = nrows*(patch_size+1)+1 636 | mosaic = np.ones((my,mx))*1000.0 637 | for i in range(nfiles): 638 | mosaic[(i/stamps_per_row)*(patch_size+1)+1:(i/stamps_per_row+1)*(patch_size+1), \ 639 | (i%stamps_per_row)*(patch_size+1)+1:(i%stamps_per_row+1)*(patch_size+1)] \ 640 | = stack[i,:,:] 641 | if diff_std[i] > threshold: 642 | mosaic[(i/stamps_per_row)*(patch_size+1)+1:(i/stamps_per_row+1)*(patch_size+1), \ 643 | (i%stamps_per_row)*(patch_size+1)+1] = -1000.0 644 | mosaic[(i/stamps_per_row)*(patch_size+1)+1:(i/stamps_per_row+1)*(patch_size+1), \ 645 | (i%stamps_per_row+1)*(patch_size+1)-1] = -1000.0 646 | mosaic[(i/stamps_per_row)*(patch_size+1)+1, \ 647 | (i%stamps_per_row)*(patch_size+1)+1:(i%stamps_per_row+1)*(patch_size+1)] = -1000.0 648 | mosaic[(i/stamps_per_row+1)*(patch_size+1)-1, \ 649 | (i%stamps_per_row)*(patch_size+1)+1:(i%stamps_per_row+1)*(patch_size+1)] = -1000.0 650 | IO.write_image(mosaic,name) 651 | 652 | # Obtain a list of files 653 | 654 | all_files = os.listdir(params.loc_data) 655 | all_files.sort() 656 | filenames = [] 657 | nfiles = 0 658 | 659 | print 'Searching in', params.loc_output, 'for', params.name_pattern 660 | 661 | for f in all_files: 662 | 663 | if fnmatch.fnmatch(f,params.name_pattern): 664 | 665 | basename = os.path.basename(f) 666 | dfile = params.loc_output+os.path.sep+'d_'+basename 667 | ktable = params.loc_output+os.path.sep+'k_'+basename 668 | 669 | if os.path.exists(dfile) and os.path.exists(ktable): 670 | 671 | nfiles += 1 672 | filenames.append(f) 673 | 674 | # Load the kernel tables 675 | # Load the difference images into a data cube 676 | 677 | print len(filenames), 'files found' 678 | 679 | 680 | dates = np.zeros(nfiles) 681 | seeing = np.zeros(nfiles) 682 | roundness = np.zeros(nfiles) 683 | bgnd = np.zeros(nfiles) 684 | signal = np.zeros(nfiles) 685 | norm_std = np.zeros(nfiles,dtype=np.float64) 686 | diff_std = np.zeros(nfiles,dtype=np.float64) 687 | n_kernel = np.zeros(nfiles,dtype=np.int32) 688 | n_coeffs = np.zeros(nfiles,dtype=np.int32) 689 | kindex_x = np.arange(0,dtype=np.int32) 690 | kindex_y = np.arange(0,dtype=np.int32) 691 | kindex_ext = np.arange(0,dtype=np.int32) 692 | coeffs = np.arange(0,dtype=np.float64) 693 | 694 | filenames.sort() 695 | 696 | if not converge: 697 | locate_iterations = 1 698 | 699 | threshold = -10 700 | for iteration in range(np.max([1,locate_iterations])): 701 | 702 | #ix0 = np.int32(x0+0.5) 703 | #iy0 = np.int32(y0+0.5) 704 | ix0 = np.int32(x0) 705 | iy0 = np.int32(y0) 706 | 707 | x_patch = x0 - ix0 + patch_half_width 708 | y_patch = y0 - iy0 + patch_half_width 709 | 710 | patch_size = 2*patch_half_width+1 711 | patch_slice = np.array([ix0-patch_half_width, ix0+patch_half_width+1, iy0-patch_half_width, iy0+patch_half_width+1]) 712 | print 'patch_slice:', patch_slice 713 | 714 | # check that patch doesn't overlap the edge of the image 715 | f = filenames[0] 716 | diff, _ = IO.read_fits_file(params.loc_output+os.path.sep+'d_'+os.path.basename(f)) 717 | nx = diff.shape[1] 718 | ny = diff.shape[0] 719 | delta_patch_x = 0 720 | delta_patch_y = 0 721 | if patch_slice[0] < 0: 722 | delta_patch_x = -patch_slice[0] 723 | elif patch_slice[1] >= nx: 724 | delta_patch_x = nx - patch_slice[1] - 1 725 | if patch_slice[2] < 0: 726 | delta_patch_y = -patch_slice[2] 727 | elif patch_slice[3] >= ny: 728 | delta_patch_y = ny - patch_slice[3] - 1 729 | 730 | print 'delta_patch_x, delta_patch_y:', delta_patch_x, delta_patch_y 731 | 732 | patch_slice += np.array([delta_patch_x,delta_patch_x,delta_patch_y,delta_patch_y]) 733 | print 'patch_slice:', patch_slice 734 | 735 | x_patch -= delta_patch_x 736 | y_patch -= delta_patch_y 737 | 738 | d_image_stack = np.zeros((nfiles,patch_size,patch_size),dtype=np.float64) 739 | inv_var_image_stack = np.zeros((nfiles,patch_size,patch_size),dtype=np.float64) 740 | 741 | dmask = np.ones([patch_size,patch_size],dtype=np.bool) 742 | dmask_rad = 8.0 743 | dmix = np.linspace(-patch_half_width,patch_half_width,patch_size) - delta_patch_x 744 | dmiy = np.linspace(-patch_half_width,patch_half_width,patch_size) - delta_patch_y 745 | dmx, dmy = np.meshgrid(dmix,dmiy,indexing='ij') 746 | dmask[dmx**2 + dmy**2 < dmask_rad**2] = False 747 | 748 | for i, f in enumerate(filenames): 749 | 750 | basename = os.path.basename(f) 751 | ktable = params.loc_output+os.path.sep+'k_'+basename 752 | kernelIndex, extendedBasis, c, params = IO.read_kernel_table(ktable,params) 753 | coeffs = np.hstack((coeffs,c)) 754 | kindex_x = np.hstack((kindex_x,kernelIndex[:,0].T)) 755 | kindex_y = np.hstack((kindex_y,kernelIndex[:,1].T)) 756 | kindex_ext = np.hstack((kindex_ext,extendedBasis)) 757 | n_kernel[i] = kernelIndex.shape[0] 758 | n_coeffs[i] = c.shape[0] 759 | dates[i] = IO.get_date(params.loc_data+os.path.sep+basename,key=params.datekey) 760 | if dates[i] > 2450000: 761 | dates[i] -= 2450000 762 | seeing[i], roundness[i], bgnd[i], signal[i] = IM.compute_fwhm(f,params,width=20,image_name=True) 763 | 764 | dfile = params.loc_output+os.path.sep+'d_'+basename 765 | nfile = params.loc_output+os.path.sep+'n_'+basename 766 | zfile = params.loc_output+os.path.sep+'sm_'+basename 767 | ivfile = params.loc_output+os.path.sep+'iv_'+basename 768 | diff, _ = IO.read_fits_file(dfile) 769 | mask, _ = IO.read_fits_file(zfile) 770 | iv, _ = IO.read_fits_file(ivfile) 771 | diff_sc = IM.undo_photometric_scale(diff,c,params.pdeg) 772 | #diff_sc = diff 773 | #diff_sc -= median_filter(diff_sc,footprint=filter_kernel) 774 | diff_sc *= mask 775 | d_image_stack[i,:,:] = diff_sc[patch_slice[2]:patch_slice[3],patch_slice[0]:patch_slice[1]] 776 | inv_var_image_stack[i,:,:], _ = IO.read_fits_file(ivfile,slice=patch_slice) 777 | #inv_var_image_stack[i,:,:] = (norm / d_image_stack[i,:,:])**2 778 | #diff_std[i] = np.std(diff) 779 | diff_std[i] = np.std(d_image_stack[i,:,:][dmask]) 780 | d_image_stack[i,:,:] -= np.median(d_image_stack[i,:,:]) 781 | 782 | print 'kappa-clipping' 783 | qd = np.arange(len(filenames)) 784 | qd1 = np.where(np.isfinite(diff_std))[0] 785 | for iter in range(10): 786 | qd = np.where(diff_std[qd1]np.mean(diff_std[qd1])+1*np.std(diff_std[qd1])) 797 | print '2-sig diff reject:',np.where(diff_std>np.mean(diff_std[qd1])+2*np.std(diff_std[qd1])) 798 | print '3-sig diff reject:',np.where(diff_std>np.mean(diff_std[qd1])+3*np.std(diff_std[qd1])) 799 | 800 | threshold = np.mean(diff_std[qd1])+q_sigma_threshold*np.std(diff_std[qd1]) 801 | threshold2 = np.mean(diff_std[qd1])+2*np.std(diff_std[qd1]) 802 | threshold3 = np.mean(diff_std[qd1])+3*np.std(diff_std[qd1]) 803 | 804 | if locate_date_range is not None: 805 | diff_std_copy = diff_std.copy() 806 | diff_std = diff_std*0.0 + 100.0*threshold 807 | pp = np.where((dates>locate_date_range[0]) & (dates 5: 820 | diff_std[i] = 100.0*threshold 821 | inv_var_image_stack[i,:,:] = inv_var_image_stack[i,:,:]*0.0 822 | 823 | 824 | if save_stamps: 825 | save_mosaic(d_image_stack,nfiles,patch_size,params.loc_output+os.path.sep+stamp_prefix+'.fits',diff_std,threshold) 826 | 827 | dsum = np.zeros((patch_size,patch_size),dtype=np.float64) 828 | for i in range(nfiles): 829 | if diff_std[i] < threshold3: 830 | dsum += d_image_stack[i,:,:] 831 | IO.write_image(dsum,params.loc_output+os.path.sep+'dsum%d.fits'%iteration) 832 | dr = patch_half_width-int(locate_half_width) 833 | print 'dr:', dr 834 | dsum[:dr-delta_patch_y,:] = 0.0 835 | dsum[-dr-delta_patch_y:,:] = 0.0 836 | dsum[:,:dr-delta_patch_x] = 0.0 837 | dsum[:,-dr-delta_patch_x:] = 0.0 838 | IO.write_image(dsum,params.loc_output+os.path.sep+'dsum_m%d.fits'%iteration) 839 | ind_dsum_max = np.unravel_index(dsum.argmax(),dsum.shape) 840 | print 'Iteration',iteration,': dsum maximum located at ',ind_dsum_max 841 | 842 | if locate and converge: 843 | y0 += ind_dsum_max[0] - patch_half_width + delta_patch_y 844 | x0 += ind_dsum_max[1] - patch_half_width + delta_patch_x 845 | 846 | 847 | # Read the PSF 848 | 849 | psf_image = params.loc_output+os.path.sep+'psf.fits' 850 | psf,psf_hdr = fits.getdata(psf_image,0,header='true') 851 | psf_height = psf_hdr['PSFHEIGH'] 852 | psf_sigma_x = psf_hdr['PAR1']*0.8493218 853 | psf_sigma_y = psf_hdr['PAR2']*0.8493218 854 | psf_x = psf_hdr['PSFX'] 855 | psf_y = psf_hdr['PSFY'] 856 | psf_size = psf.shape[1] 857 | psf_fit_rad = params.psf_fit_radius 858 | psf_parameters = np.array([psf_size,psf_height,psf_sigma_x,psf_sigma_y,psf_x, 859 | psf_y,psf_fit_rad,params.gain]).astype(np.float64) 860 | 861 | if params.psf_profile_type == 'gaussian': 862 | psf_sigma_x = psf_hdr['PAR1']*0.8493218 863 | psf_sigma_y = psf_hdr['PAR2']*0.8493218 864 | psf_parameters = np.array([psf_size,psf_height,psf_sigma_x,psf_sigma_y,psf_x, 865 | psf_y,psf_fit_rad,params.gain]).astype(np.float64) 866 | profile_type = 0 867 | elif params.psf_profile_type == 'moffat25': 868 | print 'params.psf_profile_type moffat25 not working yet. Exiting.' 869 | sys.exit(0) 870 | psf_sigma_x = psf_hdr['PAR1'] 871 | psf_sigma_y = psf_hdr['PAR2'] 872 | psf_sigma_xy = psf_hdr['PAR3'] 873 | psf_parameters = np.array([psf_size,psf_height,psf_sigma_x,psf_sigma_y,psf_x, 874 | psf_y, 875 | psf_fit_rad,params.gain,psf_sigma_xy]).astype(np.float64) 876 | profile_type = 1 877 | else: 878 | print 'params.psf_profile_type undefined' 879 | sys.exit(0) 880 | 881 | psf_0 = psf.astype(np.float64).copy() 882 | psf_xd = psf.astype(np.float64).copy()*0.0 883 | psf_yd = psf.astype(np.float64).copy()*0.0 884 | flux = np.zeros(nfiles,dtype=np.float64) 885 | dflux = np.zeros(nfiles,dtype=np.float64) 886 | 887 | x0_arr = np.atleast_1d(np.array([x0],dtype=np.float64)) 888 | y0_arr = np.atleast_1d(np.array([y0],dtype=np.float64)) 889 | 890 | print 'Converging photometry' 891 | print 'x0, y0:', x0, y0 892 | print 'x_patch, y_patch:', x_patch, y_patch 893 | good_images = np.where(diff_std < threshold)[0] 894 | print 'using images', good_images 895 | print 'threshold', threshold 896 | for i, f in enumerate(filenames): 897 | if i in good_images: 898 | print i, 'd_'+f, diff_std[i] 899 | 900 | cu_photom_converge(profile_type, patch_half_width, params.pdeg, params.sdeg, nfiles, 901 | n_kernel, kindex_x, kindex_y, kindex_ext, n_coeffs, coeffs.astype(np.float64), 902 | psf_parameters, psf_0, psf_xd, psf_yd, 903 | np.float64(d_image_stack.ravel()), np.float64(inv_var_image_stack.ravel()), diff_std, np.float64(threshold), 904 | x0_arr, y0_arr, x_patch, y_patch, diff.shape[1], diff.shape[0], 16, 16, flux, dflux, 905 | np.float64(params.gain),np.int32(converge),np.float64(2.5)) 906 | 907 | if save_stamps: 908 | save_mosaic(d_image_stack,nfiles,patch_size,params.loc_output+os.path.sep+'p'+stamp_prefix+'.fits',diff_std,threshold) 909 | 910 | if locate_date_range is not None: 911 | diff_std = diff_std_copy 912 | 913 | return dates, seeing, roundness, bgnd, signal, flux, dflux, diff_std/threshold, x0_arr[0], y0_arr[0] 914 | 915 | --------------------------------------------------------------------------------