├── oxkat ├── __init__.py ├── 1GC_05_tricolour_autoflag_cals.py ├── 1GC_01_casa_rephase_primary_calibrator.py ├── FLAG_casa_autoflag_targets_CORRECTED_DATA.py ├── 1GC_09_casa_split_targets.py ├── 1GC_05_casa_autoflag_cals_DATA.py ├── 1GC_06_casa_split_calibrators.py ├── FLAG_casa_restore_flag_table.py ├── FLAG_casa_backup_flag_table.py ├── PLOT_killMS_sols.py ├── PRE_casa_average_to_1k_add_wtspec.py ├── FLAG_casa_manual_flags.py ├── 1GC_04_casa_setjy.py ├── PLOT_gaintables.py ├── 1GC_casa_UHF_basic_flags.py ├── 2GC_casa_selfcal_target_phases.py ├── 1GC_10_plot_visibilities.py ├── 1GC_02_casa_basic_flags.py ├── user_field_handler.py ├── casa_read_project_info.py ├── 2GC_casa_selfcal_target_amp_phases.py ├── 1GC_03_primary_cal_field_sources.py ├── 1GC_casa_LINE_refcal_primary.py └── 3GC_split_model_images.py ├── tools ├── swap_partitions.sh ├── cal_table_flags.py ├── pull_containers.sh ├── dilate_FITS_mask.py ├── image_maths.py ├── add_MS_column.py ├── fix_nan_models.py ├── image_logic.py ├── make_threshold_mask.py ├── zoom_fits.py ├── merge_FITS_masks.py ├── scrub_sols.py ├── render_pngs.py ├── azimuthal_average.py ├── merge_CHPC_jobs.py ├── copy_MS_column.py ├── casa_find_source-cal_pairs.py ├── make_movie.py ├── reg2npy.py ├── pbcor.py ├── sum_MS_columns.py ├── find_sun.py ├── scan_times.py ├── fft_amplitudes.py ├── restore_model.py ├── pyMakeMask.py ├── identify_dE_directions.py └── mask_FITS_with_region.py ├── data ├── calmodels │ └── B1934-638.tar.gz ├── peeling │ ├── 3C283_peel.reg │ ├── J0213-0457_XMMLSS_peel.reg │ ├── J0219-0539_XMMLSS_peel.reg │ ├── J0227-0621_XMMLSS_peel.reg │ ├── J0228-0337_XMMLSS_peel.reg │ ├── PKSB1934-638_peel.reg │ ├── J1003+0121_COSMOS_peel.reg │ ├── J1229+0203_peel.reg │ ├── J0221-0413_XMMLSS_peel.reg │ ├── J0042-4413_ELAIS_S1_peel.reg │ ├── J0229-0442_XMMLSS_peel.reg │ ├── PKS0326-288_CDFS_peel.reg │ ├── J2214-1701_peel.reg │ └── J0627-0553_V723Mon_peel.reg ├── tricolour │ ├── target_flagging_1_narrow.yaml │ └── target_flagging_1.yaml └── cubical │ ├── 2GC_delaycal.parset │ ├── 2GC_delaycal_solve_only.parset │ ├── 2GC_phasecal.parset │ ├── 2GC_apcal.parset │ ├── 2GC_apcal_robust.parset │ ├── apply_peelsols.parset │ ├── 3GC_peel.parset │ └── 3GC_peel_2dirs.parset ├── waterhole ├── README.md ├── setup_uvsub.py ├── setup_chgcentre.py ├── setup_split_by_scan.py ├── setup_movies.py ├── setup_restore.py ├── setup_intervals.py ├── setup_get_info.py ├── setup_image_secondaries.py └── setup_FLAG_only.py ├── setups ├── 0_GET_INFO.py └── 1GC.py └── README.md /oxkat/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /oxkat/1GC_05_tricolour_autoflag_cals.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tools/swap_partitions.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | find $PWD -name slurm\*sh -exec sed -i "s/Main/HighMem/g" {} \; 3 | -------------------------------------------------------------------------------- /data/calmodels/B1934-638.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IanHeywood/oxkat/HEAD/data/calmodels/B1934-638.tar.gz -------------------------------------------------------------------------------- /waterhole/README.md: -------------------------------------------------------------------------------- 1 | # waterhole 2 | 3 | This location may contain: 4 | 5 | * Half-baked ideas 6 | * Modified, cut-down, or older versions of setups 7 | * Project-specific variants of other scripts 8 | * Obsolete stuff that hasn't been disposed of yet 9 | 10 | -------------------------------------------------------------------------------- /data/peeling/3C283_peel.reg: -------------------------------------------------------------------------------- 1 | # Region file format: DS9 version 4.1 2 | global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1 3 | fk5 4 | circle(13:11:39.2515,-22:16:42.101,5.605") 5 | -------------------------------------------------------------------------------- /data/peeling/J0213-0457_XMMLSS_peel.reg: -------------------------------------------------------------------------------- 1 | # Region file format: DS9 version 4.1 2 | global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1 3 | fk5 4 | circle(2:13:12.2160,-4:57:19.082,7.578") 5 | -------------------------------------------------------------------------------- /data/peeling/J0219-0539_XMMLSS_peel.reg: -------------------------------------------------------------------------------- 1 | # Region file format: DS9 version 4.1 2 | global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1 3 | fk5 4 | circle(2:19:28.2886,-5:39:49.592,7.832") 5 | -------------------------------------------------------------------------------- /data/peeling/J0227-0621_XMMLSS_peel.reg: -------------------------------------------------------------------------------- 1 | # Region file format: DS9 version 4.1 2 | global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1 3 | fk5 4 | circle(2:27:44.4327,-6:21:07.439,6.419") 5 | -------------------------------------------------------------------------------- /data/peeling/J0228-0337_XMMLSS_peel.reg: -------------------------------------------------------------------------------- 1 | # Region file format: DS9 version 4.1 2 | global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1 3 | fk5 4 | circle(2:28:53.1635,-3:37:39.107,5.867") 5 | -------------------------------------------------------------------------------- /data/peeling/PKSB1934-638_peel.reg: -------------------------------------------------------------------------------- 1 | # Region file format: DS9 version 4.1 2 | global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1 3 | fk5 4 | circle(19:39:24.9879,-63:42:45.368,7.258") 5 | -------------------------------------------------------------------------------- /data/peeling/J1003+0121_COSMOS_peel.reg: -------------------------------------------------------------------------------- 1 | # Region file format: DS9 version 4.1 2 | global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1 3 | fk5 4 | circle(10:03:14.8085,+1:21:21.692,7.518") 5 | -------------------------------------------------------------------------------- /data/peeling/J1229+0203_peel.reg: -------------------------------------------------------------------------------- 1 | # Region file format: DS9 version 4.1 2 | global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1 3 | fk5 4 | circle(12:29:06.7424,+2:03:08.016,6.020") 5 | circle(12:29:05.7845,+2:02:53.009,5.898") 6 | -------------------------------------------------------------------------------- /data/peeling/J0221-0413_XMMLSS_peel.reg: -------------------------------------------------------------------------------- 1 | # Region file format: DS9 version 4.1 2 | global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1 3 | fk5 4 | circle(2:21:43.1413,-4:13:46.587,3.236") 5 | circle(2:21:42.9189,-4:13:43.262,3.409") 6 | -------------------------------------------------------------------------------- /data/peeling/J0042-4413_ELAIS_S1_peel.reg: -------------------------------------------------------------------------------- 1 | # Region file format: DS9 version 4.1 2 | global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1 3 | fk5 4 | circle(0:42:09.1161,-44:14:00.219,4.630") 5 | circle(0:42:08.6512,-44:13:59.063,4.769") 6 | -------------------------------------------------------------------------------- /data/peeling/J0229-0442_XMMLSS_peel.reg: -------------------------------------------------------------------------------- 1 | # Region file format: DS9 version 4.1 2 | global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1 3 | fk5 4 | circle(2:29:15.7445,-4:42:18.243,5.992") 5 | circle(2:29:16.2126,-4:42:13.451,5.284") 6 | circle(2:29:17.2723,-4:42:17.497,0.000") 7 | -------------------------------------------------------------------------------- /data/peeling/PKS0326-288_CDFS_peel.reg: -------------------------------------------------------------------------------- 1 | # Region file format: DS9 version 4.1 2 | global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1 3 | fk5 4 | circle(3:28:36.5742,-28:41:45.713,7.183") 5 | circle(3:28:36.4614,-28:42:02.710,6.125") 6 | circle(3:28:36.5176,-28:41:55.297,5.049") 7 | -------------------------------------------------------------------------------- /tools/cal_table_flags.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | import glob 6 | import numpy 7 | import sys 8 | from pyrap.tables import table 9 | 10 | 11 | def main(): 12 | 13 | pat = sys.argv[1] 14 | 15 | tabs = glob.glob('*'+pat+'*') 16 | 17 | for tab in tabs: 18 | 19 | tt = table(tab) 20 | 21 | flags = tt.getcol('FLAG') 22 | 23 | flag_pc = 100.0 * round(flags.sum() / float(flags.size),2) 24 | 25 | print tab,flag_pc 26 | 27 | 28 | if __name__ == "__main__": 29 | 30 | 31 | main() -------------------------------------------------------------------------------- /tools/pull_containers.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | singularity pull docker://stimela/casa:1.7.0 4 | singularity pull docker://stimela/cubical:1.6.7 5 | singularity pull docker://bhugo/ddfacet:0.6.0 6 | singularity pull docker://bhugo/killms:301 7 | singularity pull docker://stimela/owlcat:1.6.6 8 | singularity pull docker://stimela/pybdsf:1.6.6 9 | singularity pull docker://stimela/ragavi:1.7.3 10 | singularity pull docker://stimela/shadems:1.7.1 11 | singularity pull docker://stimela/tricolour:1.5.0 12 | singularity pull docker://stimela/wsclean:1.6.3 13 | #singularity pull docker://stimela/codex-africanus:dev 14 | -------------------------------------------------------------------------------- /data/peeling/J2214-1701_peel.reg: -------------------------------------------------------------------------------- 1 | # Region file format: DS9 version 4.1 2 | global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1 3 | fk5 4 | circle(22:14:26.1809,-17:02:22.577,22.877") 5 | circle(22:14:24.8012,-17:01:00.156,20.303") 6 | circle(22:14:25.8171,-17:01:05.047,11.857") 7 | circle(22:14:25.2973,-17:01:23.452,8.239") 8 | circle(22:14:25.5627,-17:01:39.066,10.751") 9 | circle(22:14:26.0964,-17:01:49.316,10.572") 10 | circle(22:14:26.6480,-17:01:59.054,8.925") 11 | circle(22:14:25.7195,-17:02:05.934,11.152") 12 | circle(22:14:25.4536,-17:01:53.902,0.000") 13 | circle(22:14:26.7092,-17:01:03.283,0.000") 14 | -------------------------------------------------------------------------------- /oxkat/1GC_01_casa_rephase_primary_calibrator.py: -------------------------------------------------------------------------------- 1 | # ian.heywood@physics.ox.ac.uk 2 | 3 | 4 | execfile('oxkat/casa_read_project_info.py') 5 | 6 | 7 | args = sys.argv 8 | for item in sys.argv: 9 | parts = item.split('=') 10 | if parts[0] == 'myms': 11 | myms = parts[1] 12 | 13 | 14 | if primary_tag == '0408': 15 | newphasecentre = 'J2000 04h08m20.3782s -65d45m09.080s' 16 | dorephase = True 17 | elif primary_tag == '1934': 18 | newphasecentre = 'J2000 19h39m25.0264s -63d42m45.624s' 19 | dorephase = True 20 | else: 21 | dorephase = False 22 | 23 | if dorephase: 24 | fixvis(vis = myms, 25 | field = bpcal, 26 | phasecenter = newphasecentre, 27 | refcode = 'J2000', 28 | datacolumn = 'DATA') 29 | 30 | -------------------------------------------------------------------------------- /data/peeling/J0627-0553_V723Mon_peel.reg: -------------------------------------------------------------------------------- 1 | # Region file format: DS9 version 4.1 2 | global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1 3 | fk5 4 | circle(6:27:10.0674,-5:53:04.744,8.849") 5 | circle(6:27:09.6898,-5:53:12.670,7.724") 6 | circle(6:27:08.5391,-5:53:45.658,7.276") 7 | circle(6:27:08.1957,-5:53:53.073,8.501") 8 | circle(6:27:08.1098,-5:53:56.142,8.517") 9 | circle(6:27:08.9168,-5:53:36.965,6.456") 10 | circle(6:27:09.7578,-5:53:22.393,4.447") 11 | circle(6:27:11.2156,-5:53:15.247,3.865") 12 | circle(6:27:10.8381,-5:53:20.871,4.152") 13 | circle(6:27:10.6835,-5:53:25.218,3.312") 14 | circle(6:27:10.7355,-5:53:15.753,5.473") 15 | circle(6:27:10.2897,-5:53:15.747,3.831") 16 | -------------------------------------------------------------------------------- /oxkat/FLAG_casa_autoflag_targets_CORRECTED_DATA.py: -------------------------------------------------------------------------------- 1 | # ian.heywood@physics.ox.ac.uk 2 | 3 | 4 | import json 5 | 6 | 7 | with open('project_info.json') as f: 8 | project_info = json.load(f) 9 | 10 | 11 | myms = project_info['master_ms'] 12 | targets = project_info['target_list'] 13 | 14 | 15 | clearstat() 16 | clearstat() 17 | 18 | 19 | for target in targets: 20 | flagdata(vis=myms,mode='rflag',datacolumn='corrected',field=target[1]) 21 | flagdata(vis=myms,mode='tfcrop',datacolumn='corrected',field=target[1]) 22 | flagdata(vis=myms,mode='extend',growtime=90.0,growfreq=90.0,growaround=True,flagneartime=True,flagnearfreq=True,field=target[1]) 23 | 24 | 25 | if SAVE_FLAGS: 26 | flagmanager(vis=myms,mode='save',versionname='tfcrop_targets') 27 | 28 | 29 | clearstat() 30 | clearstat() 31 | -------------------------------------------------------------------------------- /oxkat/1GC_09_casa_split_targets.py: -------------------------------------------------------------------------------- 1 | # ian.heywood@physics.ox.ac.uk 2 | 3 | 4 | execfile('oxkat/casa_read_project_info.py') 5 | execfile('oxkat/config.py') 6 | 7 | if PRE_FIELDS != '': 8 | target_names = user_targets 9 | 10 | for target in target_names: 11 | 12 | opms = '' 13 | 14 | for mm in target_ms: 15 | if target.replace(' ','_') in mm: 16 | opms = mm 17 | 18 | if opms != '': 19 | 20 | mstransform(vis=myms, 21 | outputvis=opms, 22 | field=target, 23 | usewtspectrum=True, 24 | realmodelcol=True, 25 | datacolumn='corrected') 26 | 27 | if SAVE_FLAGS: 28 | flagmanager(vis=opms, 29 | mode='save', 30 | versionname='post-1GC') 31 | 32 | else: 33 | 34 | print('Target/MS mismatch in project info for '+target+', please check.') -------------------------------------------------------------------------------- /oxkat/1GC_05_casa_autoflag_cals_DATA.py: -------------------------------------------------------------------------------- 1 | # ian.heywood@physics.ox.ac.uk 2 | 3 | 4 | execfile('oxkat/casa_read_project_info.py') 5 | execfile('oxkat/config.py') 6 | 7 | if PRE_FIELDS != '': 8 | pcals = user_pcals 9 | 10 | clearstat() 11 | clearstat() 12 | 13 | 14 | flagdata(vis=myms,mode='rflag',datacolumn='data',field=bpcal_name) 15 | flagdata(vis=myms,mode='tfcrop',datacolumn='data',field=bpcal_name) 16 | flagdata(vis=myms,mode='extend',growtime=90.0,growfreq=90.0,growaround=True,flagneartime=True,flagnearfreq=True,field=bpcal_name) 17 | 18 | 19 | for pcal in pcals: 20 | flagdata(vis=myms,mode='rflag',datacolumn='data',field=pcal) 21 | flagdata(vis=myms,mode='tfcrop',datacolumn='data',field=pcal) 22 | flagdata(vis=myms,mode='extend',growtime=90.0,growfreq=90.0,growaround=True,flagneartime=True,flagnearfreq=True,field=pcal) 23 | 24 | 25 | if SAVE_FLAGS: 26 | flagmanager(vis=myms,mode='save',versionname='autoflag_cals_data') 27 | 28 | 29 | clearstat() 30 | clearstat() 31 | -------------------------------------------------------------------------------- /oxkat/1GC_06_casa_split_calibrators.py: -------------------------------------------------------------------------------- 1 | # ian.heywood@physics.ox.ac.uk 2 | 3 | 4 | execfile('oxkat/casa_read_project_info.py') 5 | execfile('oxkat/config.py') 6 | 7 | myoutputchans = int(PRE_NCHANS) 8 | 9 | code = myms.split('/')[-1].split('_')[0] 10 | 11 | opms = code+'_calibrators.ms' 12 | 13 | 14 | field_selection = [bpcal] 15 | for pcal in pcals: 16 | field_selection.append(pcal) 17 | 18 | field_selection = ','.join(sorted(field_selection)) 19 | 20 | 21 | tb.open(myms+'/SPECTRAL_WINDOW') 22 | nchan = tb.getcol('NUM_CHAN')[0] 23 | tb.done() 24 | 25 | 26 | mychanbin = int(nchan/myoutputchans) 27 | if mychanbin == 1: 28 | mychanave = False 29 | else: 30 | mychanave = True 31 | 32 | 33 | mstransform(vis=myms, 34 | outputvis=opms, 35 | field=field_selection, 36 | datacolumn='all', 37 | chanaverage=mychanave, 38 | chanbin=mychanbin, 39 | regridms=True, 40 | mode='channel_b', 41 | nspw=8, 42 | timeaverage=True, 43 | timebin='8s', 44 | realmodelcol=True, 45 | usewtspectrum=True) 46 | -------------------------------------------------------------------------------- /oxkat/FLAG_casa_restore_flag_table.py: -------------------------------------------------------------------------------- 1 | # ian.heywood@physics.ox.ac.uk 2 | # 3 | # set versionname=... on command line call to CASA 4 | # can also specify csv mslist=...,... otherwise project_info.p will be 5 | # used and the operation will proceed on all available target Measurement Sets 6 | # 7 | 8 | 9 | 10 | import os 11 | import sys 12 | 13 | 14 | execfile('oxkat/casa_read_project_info.py') 15 | 16 | 17 | mslist = False 18 | 19 | args = sys.argv 20 | for item in sys.argv: 21 | parts = item.split('=') 22 | if parts[0] == 'versionname': 23 | versionname = parts[1] 24 | if parts[0] == 'mslist': 25 | mslist = parts[1].split(',') 26 | 27 | 28 | if not mslist: 29 | mslist = [] 30 | for targ in target_ms: 31 | mslist.append(targ) 32 | 33 | 34 | for myms in mslist: 35 | if os.path.isdir(myms): 36 | flagmanager(vis=myms, 37 | mode='restore', 38 | versionname=versionname) 39 | else: 40 | print(myms+' not found') 41 | -------------------------------------------------------------------------------- /oxkat/FLAG_casa_backup_flag_table.py: -------------------------------------------------------------------------------- 1 | # ian.heywood@physics.ox.ac.uk 2 | # 3 | # set versionname=... on command line call to CASA 4 | # can also specify csv mslist=...,... otherwise project_info.p will be 5 | # used and the operation will proceed on all available target Measurement Sets. 6 | # 7 | # versionname must be supplied 8 | # 9 | 10 | import os 11 | import sys 12 | 13 | 14 | execfile('oxkat/casa_read_project_info.py') 15 | 16 | 17 | mslist = False 18 | 19 | args = sys.argv 20 | for item in sys.argv: 21 | parts = item.split('=') 22 | if parts[0] == 'versionname': 23 | versionname = parts[1] 24 | if parts[0] == 'mslist': 25 | mslist = parts[1].split(',') 26 | 27 | 28 | if not mslist: 29 | mslist = [] 30 | for targ in target_ms: 31 | mslist.append(targ) 32 | 33 | 34 | for myms in mslist: 35 | if os.path.isdir(myms): 36 | flagmanager(vis=myms, 37 | mode='save', 38 | versionname=versionname) 39 | else: 40 | print(myms+' not found') 41 | -------------------------------------------------------------------------------- /tools/dilate_FITS_mask.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import sys 3 | from astropy.io import fits 4 | from scipy.ndimage.morphology import binary_dilation 5 | 6 | 7 | def getImage(fitsfile): 8 | input_hdu = fits.open(fitsfile)[0] 9 | if len(input_hdu.data.shape) == 2: 10 | image = numpy.array(input_hdu.data[:,:]) 11 | elif len(input_hdu.data.shape) == 3: 12 | image = numpy.array(input_hdu.data[0,:,:]) 13 | else: 14 | image = numpy.array(input_hdu.data[0,0,:,:]) 15 | return image 16 | 17 | 18 | def flushFits(newimage,fitsfile): 19 | f = fits.open(fitsfile,mode='update') 20 | input_hdu = f[0] 21 | if len(input_hdu.data.shape) == 2: 22 | input_hdu.data[:,:] = newimage 23 | elif len(input_hdu.data.shape) == 3: 24 | input_hdu.data[0,:,:] = newimage 25 | else: 26 | input_hdu.data[0,0,:,:] = newimage 27 | f.flush() 28 | 29 | 30 | infits = sys.argv[1] 31 | niter = int(sys.argv[2]) 32 | 33 | maskimage = getImage(infits) 34 | dilated = binary_dilation(input=maskimage,iterations=niter) 35 | flushFits(dilated,infits) -------------------------------------------------------------------------------- /oxkat/PLOT_killMS_sols.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | import glob 6 | import os 7 | import os.path as o 8 | import subprocess 9 | import sys 10 | sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) 11 | 12 | 13 | from oxkat import generate_jobs as gen 14 | from oxkat import config as cfg 15 | 16 | 17 | 18 | def main(): 19 | 20 | myms = sys.argv[1] 21 | solname = sys.argv[2] 22 | 23 | plotargs = ['--corr0 0 --corr1 0 --iterfreq', 24 | '--corr0 1 --corr1 1 --iterfreq', 25 | '--corr0 0 --corr1 0 --iterdir', 26 | '--corr0 1 --corr1 1 --iterdir',] 27 | 28 | TOOLS = cfg.TOOLS 29 | GAINPLOTS = cfg.GAINPLOTS 30 | gen.setup_dir(GAINPLOTS) 31 | 32 | kmstab = myms.rstrip('/')+'/killMS.'+solname+'.sols.npz' 33 | 34 | for plotarg in plotargs: 35 | syscall = 'python3 '+TOOLS+'/plot_killMS.py '+plotarg+' --outdir '+GAINPLOTS+' '+kmstab 36 | subprocess.run([syscall],shell=True) 37 | 38 | 39 | if __name__ == "__main__": 40 | 41 | 42 | main() -------------------------------------------------------------------------------- /oxkat/PRE_casa_average_to_1k_add_wtspec.py: -------------------------------------------------------------------------------- 1 | # ian.heywood@physics.ox.ac.uk 2 | 3 | import glob 4 | 5 | 6 | execfile('oxkat/config.py') 7 | execfile('oxkat/casa_read_project_info.py') 8 | 9 | myfields = PRE_FIELDS 10 | myscans = PRE_SCANS 11 | myoutputchans = int(PRE_NCHANS) 12 | mytimebins = PRE_TIMEBIN 13 | 14 | 15 | master_ms = glob.glob('*.ms')[0] 16 | opms = master_ms.replace('.ms','_'+str(myoutputchans)+'ch.ms') 17 | 18 | 19 | tb.open(master_ms+'/SPECTRAL_WINDOW') 20 | nchan = tb.getcol('NUM_CHAN')[0] 21 | tb.done() 22 | 23 | 24 | mychanbin = int(nchan/myoutputchans) 25 | if mychanbin <= 1: 26 | mychanave = False 27 | else: 28 | mychanave = True 29 | 30 | 31 | mstransform(vis = master_ms, 32 | outputvis = opms, 33 | field = myfields, 34 | scan = myscans, 35 | datacolumn = 'data', 36 | chanaverage = mychanave, 37 | chanbin = mychanbin, 38 | # timeaverage = True, 39 | # timebin = '8s', 40 | realmodelcol = True, 41 | usewtspectrum = True) 42 | 43 | if SAVE_FLAGS: 44 | flagmanager(vis = opms, mode = 'save', versionname = 'observatory') 45 | 46 | clearcal(vis = opms, addmodel = True) 47 | 48 | 49 | clearstat() 50 | clearstat() -------------------------------------------------------------------------------- /oxkat/FLAG_casa_manual_flags.py: -------------------------------------------------------------------------------- 1 | # ian.heywood@physics.ox.ac.uk 2 | 3 | 4 | import glob 5 | import json 6 | import sys 7 | 8 | 9 | with open('project_info.json') as f: 10 | project_info = json.load(f) 11 | 12 | 13 | flag_file = glob.glob('*manualflags.txt') 14 | 15 | 16 | if len(flag_file) == 1: 17 | flag_file = flag_file[0] 18 | else: 19 | print('No or multiple manual flag files found.') 20 | sys.exit() 21 | 22 | 23 | myms = project_info['master_ms'] 24 | 25 | 26 | clearstat() 27 | clearstat() 28 | 29 | 30 | f = open(flag_file,'r') 31 | line = f.readline().rstrip('\n') 32 | while line: 33 | if line[0] != '#': 34 | cols = line.split(':') 35 | if len(cols) == 1: 36 | ant = cols[0] 37 | scans = '' 38 | elif len(cols) == 2: 39 | ant = cols[0] 40 | scans = cols[1] 41 | else: 42 | print('Check manual flag instruction:') 43 | print(line) 44 | continue 45 | flagdata(vis=myms, 46 | mode = 'manual', 47 | antenna = ant, 48 | scan = scans) 49 | line = f.readline().rstrip('\n') 50 | f.close() 51 | 52 | 53 | if SAVE_FLAGS: 54 | flagmanager(vis=myms,mode='save',versionname='manual_flags') 55 | 56 | 57 | clearstat() 58 | clearstat() 59 | -------------------------------------------------------------------------------- /oxkat/1GC_04_casa_setjy.py: -------------------------------------------------------------------------------- 1 | # ian.heywood@physics.ox.ac.uk 2 | 3 | 4 | import glob 5 | import shutil 6 | import time 7 | 8 | 9 | execfile('oxkat/casa_read_project_info.py') 10 | execfile('oxkat/config.py') 11 | 12 | 13 | args = sys.argv 14 | for item in sys.argv: 15 | parts = item.split('=') 16 | if parts[0] == 'myms': 17 | myms = parts[1] 18 | 19 | 20 | tb.open(myms) 21 | colnames = tb.colnames() 22 | tb.done() 23 | 24 | 25 | 26 | # ------- Setjy models 27 | 28 | 29 | if primary_tag == '1934': 30 | setjy(vis=myms, 31 | field=bpcal_name, 32 | standard='Stevens-Reynolds 2016', 33 | scalebychan=True, 34 | usescratch=True) 35 | 36 | 37 | elif primary_tag == '0408': 38 | bpcal_mod = CAL_1GC_0408_MODEL 39 | setjy(vis=myms, 40 | field=bpcal_name, 41 | standard='manual', 42 | fluxdensity=bpcal_mod[0], 43 | spix=bpcal_mod[1], 44 | reffreq=bpcal_mod[2], 45 | scalebychan=True, 46 | usescratch=True) 47 | 48 | 49 | elif primary_tag == 'other': 50 | setjy(vis=myms, 51 | field=bpcal_name, 52 | standard='Perley-Butler 2013', 53 | scalebychan=True, 54 | usescratch=True) 55 | -------------------------------------------------------------------------------- /tools/image_maths.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import shutil 3 | import sys 4 | from astropy.io import fits 5 | 6 | 7 | def getImage(fitsfile): 8 | input_hdu = fits.open(fitsfile)[0] 9 | if len(input_hdu.data.shape) == 2: 10 | image = numpy.array(input_hdu.data[:,:]) 11 | elif len(input_hdu.data.shape) == 3: 12 | image = numpy.array(input_hdu.data[0,:,:]) 13 | else: 14 | image = numpy.array(input_hdu.data[0,0,:,:]) 15 | return image 16 | 17 | 18 | def flushFits(newimage,fitsfile): 19 | f = fits.open(fitsfile,mode='update') 20 | input_hdu = f[0] 21 | if len(input_hdu.data.shape) == 2: 22 | input_hdu.data[:,:] = newimage 23 | elif len(input_hdu.data.shape) == 3: 24 | input_hdu.data[0,:,:] = newimage 25 | else: 26 | input_hdu.data[0,0,:,:] = newimage 27 | f.flush() 28 | 29 | 30 | im1 = sys.argv[1] 31 | operator = sys.argv[2] 32 | im2 = sys.argv[3] 33 | out = sys.argv[4] 34 | 35 | shutil.copyfile(im1,out) 36 | 37 | data1 = getImage(im1) 38 | data2 = getImage(im2) 39 | 40 | if operator == 'plus': 41 | dataout = data1+data2 42 | elif operator == 'minus': 43 | dataout = data1-data2 44 | elif operator == 'times': 45 | dataout = data1*data2 46 | elif operator == 'over': 47 | dataout = data1/data2 48 | 49 | flushFits(dataout,out) 50 | -------------------------------------------------------------------------------- /tools/add_MS_column.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | import sys 5 | from optparse import OptionParser 6 | from pyrap.tables import table 7 | 8 | 9 | def add_data_col(msname,colname): 10 | tt = table(msname,readonly=False) 11 | colnames = tt.colnames() 12 | if colname in colnames: 13 | print(colname+' already exists, will not be created') 14 | else: 15 | print('Adding '+colname+' to '+msname) 16 | desc = tt.getcoldesc('DATA') 17 | desc['name'] = colname 18 | desc['comment'] = desc['comment'].replace(' ','_') 19 | tt.addcols(desc) 20 | tt.done() 21 | 22 | 23 | def main(): 24 | 25 | parser = OptionParser(usage = '%prog [options] msname') 26 | parser.add_option('--colname', dest = 'colname', default = 'DIR1_DATA', help = 'Name (or comma-separated list) of new data column(s) (default = DIR1_DATA)') 27 | (options,args) = parser.parse_args() 28 | colname = options.colname 29 | 30 | if len(args) != 1: 31 | print('Please specify a Measurement Set') 32 | sys.exit() 33 | else: 34 | msname = args[0].rstrip('/') 35 | 36 | for col in colname.split(','): 37 | add_data_col(msname,col) 38 | 39 | 40 | if __name__ == '__main__': 41 | 42 | main() 43 | -------------------------------------------------------------------------------- /tools/fix_nan_models.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import glob 3 | import numpy 4 | from astropy.io import fits 5 | 6 | 7 | def get_image(fitsfile): 8 | input_hdu = fits.open(fitsfile)[0] 9 | if len(input_hdu.data.shape) == 2: 10 | image = numpy.array(input_hdu.data[:,:]) 11 | elif len(input_hdu.data.shape) == 3: 12 | image = numpy.array(input_hdu.data[0,:,:]) 13 | else: 14 | image = numpy.array(input_hdu.data[0,0,:,:]) 15 | return image 16 | 17 | 18 | def flush_fits(newimage,fitsfile): 19 | f = fits.open(fitsfile,mode='update') 20 | input_hdu = f[0] 21 | if len(input_hdu.data.shape) == 2: 22 | input_hdu.data[:,:] = newimage 23 | elif len(input_hdu.data.shape) == 3: 24 | input_hdu.data[0,:,:] = newimage 25 | else: 26 | input_hdu.data[0,0,:,:] = newimage 27 | f.flush() 28 | 29 | 30 | pattern = sys.argv[1] 31 | 32 | 33 | fitslist = sorted(glob.glob(pattern+'*model.fits')) 34 | for fitsfile in fitslist: 35 | img = get_image(fitsfile) 36 | maxval = numpy.max(img) 37 | if numpy.isnan(maxval): 38 | new_img = numpy.zeros((img.shape[0],img.shape[1])) 39 | print(fitsfile,maxval,'zeroing NaN model') 40 | flush_fits(new_img,fitsfile) 41 | else: 42 | print(fitsfile,maxval) -------------------------------------------------------------------------------- /tools/image_logic.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import shutil 3 | import sys 4 | from astropy.io import fits 5 | 6 | 7 | def getImage(fitsfile): 8 | input_hdu = fits.open(fitsfile)[0] 9 | if len(input_hdu.data.shape) == 2: 10 | image = numpy.array(input_hdu.data[:,:]) 11 | elif len(input_hdu.data.shape) == 3: 12 | image = numpy.array(input_hdu.data[0,:,:]) 13 | else: 14 | image = numpy.array(input_hdu.data[0,0,:,:]) 15 | return image 16 | 17 | 18 | def flushFits(newimage,fitsfile): 19 | f = fits.open(fitsfile,mode='update') 20 | input_hdu = f[0] 21 | if len(input_hdu.data.shape) == 2: 22 | input_hdu.data[:,:] = newimage 23 | elif len(input_hdu.data.shape) == 3: 24 | input_hdu.data[0,:,:] = newimage 25 | else: 26 | input_hdu.data[0,0,:,:] = newimage 27 | f.flush() 28 | 29 | 30 | im1 = sys.argv[1] 31 | operator = sys.argv[2].upper() 32 | im2 = sys.argv[3] 33 | out = sys.argv[4] 34 | 35 | shutil.copyfile(im1,out) 36 | 37 | data1 = getImage(im1) 38 | data2 = getImage(im2) 39 | 40 | if operator == 'OR': 41 | dataout = numpy.logical_or(data1,data2) 42 | elif operator == 'AND': 43 | dataout = numpy.logical_and(data1,data2) 44 | elif operator == 'XOR': 45 | dataout = numpy.logical_xor(data1,data2) 46 | else: 47 | print('Operator not recognised, please use OR, AND or XOR') 48 | sys.exit() 49 | 50 | flushFits(dataout,out) 51 | -------------------------------------------------------------------------------- /tools/make_threshold_mask.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ianh@astro.ox.ac.uk 3 | 4 | 5 | from astropy.io import fits 6 | from scipy.ndimage.morphology import binary_dilation 7 | from shutil import copyfile 8 | import numpy 9 | import sys 10 | 11 | 12 | def getImage(fitsfile): 13 | input_hdu = fits.open(fitsfile)[0] 14 | if len(input_hdu.data.shape) == 2: 15 | image = numpy.array(input_hdu.data[:,:]) 16 | elif len(input_hdu.data.shape) == 3: 17 | image = numpy.array(input_hdu.data[0,:,:]) 18 | else: 19 | image = numpy.array(input_hdu.data[0,0,:,:]) 20 | return image 21 | 22 | 23 | def flushFits(newimage,fitsfile): 24 | f = fits.open(fitsfile,mode='update') 25 | input_hdu = f[0] 26 | if len(input_hdu.data.shape) == 2: 27 | input_hdu.data[:,:] = newimage 28 | elif len(input_hdu.data.shape) == 3: 29 | input_hdu.data[0,:,:] = newimage 30 | else: 31 | input_hdu.data[0,0,:,:] = newimage 32 | f.flush() 33 | 34 | 35 | def main(): 36 | 37 | infits = sys.argv[1] 38 | thresh = float(sys.argv[2]) 39 | 40 | thresh_str = '.thresh'+str(thresh).replace('.','p')+'.mask.fits' 41 | opfits = infits.replace('.fits',thresh_str) 42 | 43 | copyfile(infits,opfits) 44 | 45 | img = getImage(infits) 46 | maskimg = img * 0.0 47 | mask = img > thresh 48 | maskimg[mask] = 1.0 49 | 50 | maskimg = binary_dilation(maskimg,iterations=2) 51 | 52 | flushFits(maskimg,opfits) 53 | 54 | 55 | if __name__ == "__main__": 56 | 57 | main() 58 | -------------------------------------------------------------------------------- /oxkat/PLOT_gaintables.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | import glob 6 | import os 7 | import os.path as o 8 | import subprocess 9 | import sys 10 | sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) 11 | 12 | 13 | from oxkat import generate_jobs as gen 14 | from oxkat import config as cfg 15 | 16 | 17 | def main(): 18 | 19 | 20 | GAINPLOTS = cfg.GAINPLOTS 21 | GAINTABLES = cfg.GAINTABLES 22 | gen.setup_dir(GAINPLOTS) 23 | 24 | 25 | include = sys.argv[1] 26 | if len(sys.argv) > 2: 27 | exclude = sys.argv[2] 28 | else: 29 | exclude = '' 30 | 31 | caltabs = sorted([item for item in glob.glob(GAINTABLES+'/'+include) if not os.path.basename(item).endswith('flagversions')]) 32 | if exclude != '': 33 | exclude = glob.glob(GAINTABLES+'/'+exclude) 34 | 35 | for caltab in caltabs: 36 | if caltab not in exclude: 37 | gaintype = caltab.split('.')[-1][0].upper() 38 | htmlname = GAINPLOTS+'/'+caltab.split('/')[-1]+'.html' 39 | plotname = GAINPLOTS+'/'+caltab.split('/')[-1]+'.png' 40 | if not os.path.isfile(htmlname): 41 | syscall = 'ragavi-gains -g '+gaintype+' -t '+caltab+' --htmlname='+htmlname+' --plotname='+plotname 42 | subprocess.run([syscall],shell=True) 43 | else: 44 | print(htmlname+' exists, skipping') 45 | 46 | if __name__ == "__main__": 47 | 48 | 49 | main() -------------------------------------------------------------------------------- /oxkat/1GC_casa_UHF_basic_flags.py: -------------------------------------------------------------------------------- 1 | # ian.heywood@physics.ox.ac.uk 2 | # UHF calibration is experimental 3 | 4 | 5 | import numpy 6 | 7 | 8 | execfile('oxkat/casa_read_project_info.py') 9 | 10 | 11 | args = sys.argv 12 | for item in sys.argv: 13 | parts = item.split('=') 14 | if parts[0] == 'myms': 15 | myms = parts[1] 16 | 17 | 18 | clearstat() 19 | clearstat() 20 | 21 | 22 | # ------------------------------------------------------------------------ 23 | # Band edges 24 | 25 | badfreqs = ['540~570MHz', '1010~1150MHz'] 26 | 27 | myspw = '' 28 | for badfreq in badfreqs: 29 | myspw += '*:'+badfreq+',' 30 | myspw = myspw.rstrip(',') 31 | 32 | flagdata(vis = myms, 33 | mode = 'manual', 34 | spw = myspw) 35 | 36 | 37 | 38 | # ------------------------------------------------------------------------ 39 | # Clipping, quacking, zeros, autos 40 | # Note that clip will always flag NaN/Inf values even with a range 41 | 42 | #flagdata(vis = myms, 43 | # mode = 'quack', 44 | # quackinterval = 8.0, 45 | # quackmode = 'beg') 46 | 47 | flagdata(vis = myms, 48 | mode = 'manual', 49 | autocorr = True) 50 | 51 | flagdata(vis = myms, 52 | mode = 'clip', 53 | clipzeros = True) 54 | 55 | flagdata(vis = myms, 56 | mode = 'clip', 57 | clipminmax = [0.0,200.0]) 58 | 59 | # ------------------------------------------------------------------------ 60 | # Save the flags 61 | 62 | flagmanager(vis = myms, 63 | mode = 'save', 64 | versionname = 'basic') 65 | 66 | 67 | clearstat() 68 | clearstat() 69 | -------------------------------------------------------------------------------- /tools/zoom_fits.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import glob 3 | 4 | 5 | def write_slurm(opfile, 6 | jobname, 7 | logfile, 8 | container, 9 | syscall, 10 | ntasks='1', 11 | nodes='1', 12 | cpus='8', 13 | mem='8GB'): 14 | 15 | f = open(opfile,'w') 16 | f.writelines(['#!/bin/bash\n', 17 | '#file: '+opfile+':\n', 18 | '#SBATCH --job-name='+jobname+'\n', 19 | '#SBATCH --time=01:00:00\n', 20 | '#SBATCH --ntasks='+ntasks+'\n', 21 | '#SBATCH --nodes='+nodes+'\n', 22 | '#SBATCH --cpus-per-task='+cpus+'\n', 23 | '#SBATCH --mem='+mem+'\n', 24 | '#SBATCH --output='+logfile+'\n', 25 | syscall+'\n', 26 | 'sleep 10\n']) 27 | f.close() 28 | 29 | 30 | 31 | def main(): 32 | 33 | 34 | MEQTREES_CONTAINER = '/idia/software/containers/STIMELA_IMAGES/stimela_meqtrees_1.2.4.sif' 35 | 36 | pattern = sys.argv[1] 37 | 38 | 39 | fitslist = glob.glob(pattern) 40 | 41 | slurmfile = 'slurm_zoom_fits.sh' 42 | logfile = 'slurm_zoom_fits.log' 43 | 44 | syscall = '' 45 | 46 | print(fitslist) 47 | 48 | for infits in fitslist: 49 | print(infits) 50 | syscall += 'singularity exec '+MEQTREES_CONTAINER+' fitstool.py -z 5450 '+infits+'\n' 51 | 52 | write_slurm(opfile=slurmfile, 53 | jobname='zoomfits', 54 | logfile=logfile, 55 | container=MEQTREES_CONTAINER, 56 | syscall=syscall) 57 | 58 | print('sbatch '+slurmfile) 59 | 60 | 61 | if __name__ == "__main__": 62 | 63 | 64 | main() -------------------------------------------------------------------------------- /oxkat/2GC_casa_selfcal_target_phases.py: -------------------------------------------------------------------------------- 1 | # ian.heywood@physics.ox.ac.uk 2 | 3 | 4 | import pickle 5 | import sys 6 | import time 7 | 8 | 9 | execfile('oxkat/config.py') 10 | 11 | 12 | def stamp(): 13 | now = str(datetime.datetime.now()).replace(' ','-').replace(':','-').split('.')[0] 14 | return now 15 | 16 | 17 | myuvrange = '' 18 | 19 | args = sys.argv 20 | for item in sys.argv: 21 | parts = item.split('=') 22 | if parts[0] == 'mslist': 23 | mslist = parts[1].split(',') 24 | if parts[0] == 'uvmin': 25 | myuvrange = '>'+parts[1] 26 | 27 | 28 | if myuvrange == '': 29 | myuvrange = '>150m' 30 | 31 | with open('project_info.json') as f: 32 | project_info = json.load(f) 33 | 34 | ref_ant = project_info['ref_ant'] 35 | 36 | 37 | for myms in mslist: 38 | 39 | 40 | clearstat() 41 | clearstat() 42 | 43 | 44 | gtab = GAINTABLES+'/cal_'+myms+'_'+stamp()+'.GP0' 45 | 46 | 47 | gaincal(vis=myms, 48 | field='0', 49 | uvrange=myuvrange, 50 | caltable=gtab, 51 | refant = str(ref_ant), 52 | solint='64s', 53 | solnorm=False, 54 | combine='', 55 | minsnr=3, 56 | calmode='p', 57 | parang=False, 58 | gaintable=[], 59 | gainfield=[], 60 | interp=[], 61 | append=False) 62 | 63 | 64 | applycal(vis=myms, 65 | gaintable=[gtab], 66 | field='0', 67 | calwt=False, 68 | parang=False, 69 | applymode='calonly', 70 | gainfield='0', 71 | interp = ['nearest']) 72 | 73 | 74 | # statwt(vis=myms, 75 | # field='0') 76 | 77 | 78 | clearstat() 79 | clearstat() 80 | 81 | 82 | -------------------------------------------------------------------------------- /data/tricolour/target_flagging_1_narrow.yaml: -------------------------------------------------------------------------------- 1 | # Narrow sumthreshol 2 | strategies: 3 | - 4 | name: final_st_broad 5 | task: sum_threshold 6 | kwargs: 7 | outlier_nsigma: 10 8 | windows_time: [1, 2, 4, 8] 9 | windows_freq: [1, 2, 4, 8] 10 | background_reject: 2.0 11 | background_iterations: 5 12 | spike_width_time: 6.5 13 | spike_width_freq: 10.0 14 | time_extend: 3 15 | freq_extend: 3 16 | freq_chunks: 10 17 | average_freq: 1 18 | flag_all_time_frac: 0.6 19 | flag_all_freq_frac: 0.8 20 | rho: 1.3 21 | num_major_iterations: 1 22 | - 23 | name: final_st_narrow 24 | task: sum_threshold 25 | kwargs: 26 | outlier_nsigma: 10 27 | windows_time: [1, 2, 4, 8] 28 | windows_freq: [1, 2, 4, 8] 29 | background_reject: 2.0 30 | background_iterations: 5 31 | spike_width_time: 2 32 | spike_width_freq: 10.0 33 | time_extend: 3 34 | freq_extend: 3 35 | freq_chunks: 10 36 | average_freq: 1 37 | flag_all_time_frac: 0.6 38 | flag_all_freq_frac: 0.8 39 | rho: 1.3 40 | num_major_iterations: 1 41 | - 42 | name: residual_flag_final 43 | task: uvcontsub_flagger 44 | kwargs: 45 | major_cycles: 10 46 | or_original_from_cycle: 0 47 | taylor_degrees: 25 48 | sigma: 13.0 49 | - 50 | name: combine_with_input_flags 51 | task: combine_with_input_flags 52 | 53 | -------------------------------------------------------------------------------- /tools/merge_FITS_masks.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | from astropy.io import fits 6 | import numpy 7 | import random 8 | from scipy.ndimage.morphology import binary_dilation 9 | import shutil 10 | import sys 11 | 12 | 13 | def genhex(): 14 | ran = random.randrange(10**80) 15 | myhex = "%064x" % ran 16 | myhex = myhex[:32] 17 | return myhex 18 | 19 | 20 | def getImage(fitsfile): 21 | input_hdu = fits.open(fitsfile)[0] 22 | if len(input_hdu.data.shape) == 2: 23 | image = numpy.array(input_hdu.data[:,:]) 24 | elif len(input_hdu.data.shape) == 3: 25 | image = numpy.array(input_hdu.data[0,:,:]) 26 | else: 27 | image = numpy.array(input_hdu.data[0,0,:,:]) 28 | return image 29 | 30 | 31 | def flushFits(newimage,fitsfile): 32 | f = fits.open(fitsfile,mode='update') 33 | input_hdu = f[0] 34 | if len(input_hdu.data.shape) == 2: 35 | input_hdu.data[:,:] = newimage 36 | elif len(input_hdu.data.shape) == 3: 37 | input_hdu.data[0,:,:] = newimage 38 | else: 39 | input_hdu.data[0,0,:,:] = newimage 40 | f.flush() 41 | 42 | 43 | def main(): 44 | 45 | prefix = sys.argv[1] 46 | opfits = sys.argv[2] 47 | 48 | modelfits = prefix+'-MFS-model.fits' 49 | makemaskfits = prefix+'-MFS-image.fits.mask.fits' 50 | 51 | shutil.copyfile(modelfits,opfits) 52 | 53 | modeldata = getImage(modelfits) 54 | makemaskdata = getImage(makemaskfits) 55 | 56 | finalmaskdata = modeldata+makemaskdata 57 | 58 | finalmaskdata = binary_dilation(finalmaskdata,iterations=4) 59 | 60 | flushFits(finalmaskdata,opfits) 61 | 62 | 63 | 64 | if __name__ == "__main__": 65 | 66 | 67 | main() 68 | -------------------------------------------------------------------------------- /waterhole/setup_uvsub.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import glob 3 | import os.path as o 4 | 5 | sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) 6 | 7 | from oxkat import generate_jobs as gen 8 | from oxkat import config as cfg 9 | 10 | 11 | def write_slurm(opfile,jobname,logfile,syscall): 12 | 13 | f = open(opfile,'w') 14 | f.writelines(['#!/bin/bash\n', 15 | '#file: '+opfile+':\n', 16 | '#SBATCH --job-name='+jobname+'\n', 17 | '#SBATCH --time=06:00:00\n', 18 | '#SBATCH --partition=Main\n' 19 | '#SBATCH --ntasks=1\n', 20 | '#SBATCH --nodes=1\n', 21 | '#SBATCH --cpus-per-task=4\n', 22 | '#SBATCH --mem=64GB\n', 23 | '#SBATCH --account=b24-thunderkat-ag\n', 24 | '#SBATCH --output='+logfile+'\n', 25 | syscall+'\n', 26 | 'sleep 10\n']) 27 | f.close() 28 | 29 | 30 | def main(): 31 | 32 | INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(('','idia')) 33 | ASTROPY_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.ASTROPY_PATTERN,True) 34 | 35 | pattern = sys.argv[1] 36 | mslist = glob.glob(pattern) 37 | submit_file = 'submit_uvsub_jobs.sh' 38 | 39 | f = open(submit_file,'w') 40 | 41 | for myms in mslist: 42 | code = myms.split('_')[-1].rstrip('.ms').replace('scan','uvsub') 43 | syscall = 'singularity exec '+ASTROPY_CONTAINER+' ' 44 | syscall += 'python3 tools/sum_MS_columns.py --src=MODEL_DATA --dest=CORRECTED_DATA --subtract '+myms 45 | 46 | slurm_file = 'SCRIPTS/slurm_uvsub_'+myms+'.sh' 47 | log_file = 'LOGS/slurm_uvsub_'+myms+'.log' 48 | 49 | write_slurm(opfile=slurm_file,jobname=code,logfile=log_file,syscall=syscall ) 50 | f.writelines(['sbatch '+slurm_file+'\n']) 51 | 52 | f.close() 53 | gen.make_executable(submit_file) 54 | print('Wrote '+submit_file) 55 | 56 | 57 | if __name__ == "__main__": 58 | 59 | 60 | main() -------------------------------------------------------------------------------- /oxkat/1GC_10_plot_visibilities.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | import json 6 | import os.path as o 7 | import subprocess 8 | import sys 9 | sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) 10 | 11 | 12 | from oxkat import generate_jobs as gen 13 | from oxkat import config as cfg 14 | 15 | 16 | 17 | def main(): 18 | 19 | 20 | VISPLOTS = cfg.VISPLOTS 21 | gen.setup_dir(VISPLOTS) 22 | 23 | 24 | with open('project_info.json') as f: 25 | project_info = json.load(f) 26 | 27 | 28 | myms = project_info['working_ms'] 29 | bpcal = project_info['primary_name'] 30 | pcals = project_info['secondary_ids'] 31 | targets = project_info['target_ids'] 32 | 33 | if cfg.PRE_FIELDS != '': 34 | from oxkat import user_field_handler as ufh 35 | pcals = ufh.user_pcals 36 | 37 | fields = [bpcal] 38 | for pcal in pcals: 39 | fields.append(pcal) 40 | 41 | plots = ['--xaxis CORRECTED_DATA:real:XX,CORRECTED_DATA:real:YY --yaxis CORRECTED_DATA:imag:XX,CORRECTED_DATA:imag:YY', 42 | '--xaxis FREQ,FREQ --yaxis CORRECTED_DATA:amp:XX,CORRECTED_DATA:amp:YY', 43 | '--xaxis FREQ,FREQ --yaxis CORRECTED_DATA:phase:XX,CORRECTED_DATA:phase:YY', 44 | '--xaxis UV,UV,UV,UV --yaxis CORRECTED_DATA:amp:XX,CORRECTED_DATA:amp:YY,CORRECTED_DATA:phase:XX,CORRECTED_DATA:phase:YY'] 45 | 46 | colour_by = ['--colour-by ANTENNA1 --cnum 64', 47 | '--colour-by SCAN_NUMBER --cnum 48 '] 48 | 49 | # shadems_base = 'shadems --profile --dir '+VISPLOTS+' ' 50 | shadems_base = 'shadems --dir '+VISPLOTS+' ' 51 | 52 | for field in fields: 53 | for plot in plots: 54 | for col in colour_by: 55 | syscall = shadems_base+' '+plot+' '+col+' --field '+str(field)+' '+myms 56 | subprocess.run([syscall],shell=True) 57 | 58 | 59 | if __name__ == "__main__": 60 | 61 | 62 | main() 63 | -------------------------------------------------------------------------------- /tools/scrub_sols.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | # Clip killMS solutions based on the first differential of the amplitudes 6 | 7 | 8 | import numpy 9 | import sys 10 | 11 | 12 | def nan_helper(y): 13 | # https://stackoverflow.com/questions/6518811/interpolate-nan-values-in-a-numpy-array 14 | return numpy.isnan(y), lambda z: z.nonzero()[0] 15 | 16 | 17 | gaintab = sys.argv[1] 18 | optab = gaintab.replace('.npz','_scrubbed.npz') 19 | thresh = 5.0 20 | 21 | 22 | tab = dict(numpy.load(gaintab)) 23 | 24 | 25 | clusters = tab['ClusterCat'] 26 | freqs = tab['FreqDomains'] 27 | ant_names = tab['StationNames'] 28 | t0 = tab['Sols']['t0'] 29 | t1 = tab['Sols']['t1'] 30 | dt = t1 - t0 31 | t = t0 + (dt/2) 32 | 33 | 34 | gains = (tab['Sols']) 35 | gains = gains.view(numpy.recarray) 36 | 37 | 38 | gshape = gains['G'].shape 39 | nt = gshape[0] 40 | nfreq = gshape[1] 41 | nant = gshape[2] 42 | ndir = gshape[3] 43 | ncorr1 = gshape[4] 44 | ncorr2 = gshape[5] 45 | 46 | 47 | print (gaintab,'properties:') 48 | print ('') 49 | print ('Time slots: '+str(nt)) 50 | print ('Frequency chunks: '+str(nfreq)) 51 | print ('Antennas: '+str(nant)) 52 | print ('Directions: '+str(ndir)) 53 | print ('Correlations: '+str(ncorr1*ncorr2)) 54 | 55 | 56 | for ant in range(0,nant): 57 | 58 | for mydir in range(0,ndir): 59 | 60 | for corr1,corr2 in [(0,0),(1,1)]: 61 | 62 | g0 = (gains['G'][:,0,ant,mydir,corr1,corr2]) 63 | 64 | amp = numpy.abs(g0) 65 | phase = numpy.angle(g0) 66 | 67 | fd = numpy.gradient(amp) 68 | fd_mean = numpy.mean(fd) 69 | fd_std = numpy.std(fd) 70 | 71 | mask = abs(fd) > 0.1 72 | 73 | amp[mask] = numpy.nan 74 | 75 | nans, x= nan_helper(amp) 76 | amp[nans]= numpy.interp(x(nans), x(~nans), amp[~nans]) 77 | 78 | g1 = amp * numpy.exp(1j*phase) 79 | 80 | gains['G'][:,0,ant,mydir,corr1,corr2] = g1 81 | 82 | 83 | tab['Sols'] = gains 84 | numpy.savez(optab,**tab) 85 | 86 | -------------------------------------------------------------------------------- /tools/render_pngs.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import glob 3 | 4 | 5 | def write_slurm(opfile, 6 | jobname, 7 | logfile, 8 | container, 9 | syscall, 10 | ntasks='1', 11 | nodes='1', 12 | cpus='8', 13 | mem='32GB'): 14 | 15 | f = open(opfile,'w') 16 | f.writelines(['#!/bin/bash\n', 17 | '#file: '+opfile+':\n', 18 | '#SBATCH --job-name='+jobname+'\n', 19 | '#SBATCH --ntasks='+ntasks+'\n', 20 | '#SBATCH --nodes='+nodes+'\n', 21 | '#SBATCH --cpus-per-task='+cpus+'\n', 22 | '#SBATCH --mem='+mem+'\n', 23 | '#SBATCH --output='+logfile+'\n', 24 | syscall+'\n', 25 | 'sleep 10\n']) 26 | f.close() 27 | 28 | 29 | def generate_syscall_mviewer(infits): 30 | outpng = infits+'.png' 31 | syscall = 'mViewer ' 32 | syscall += '-color yellow ' 33 | syscall += '-grid Equatorial J2000 ' 34 | syscall += '-ct 0 ' 35 | syscall += '-gray '+infits+' ' 36 | syscall += '-2s max gaussian-log ' 37 | # syscall += '-2s 5e-5 ' 38 | syscall += '-png '+outpng+' ' 39 | return syscall 40 | 41 | 42 | def main(): 43 | 44 | 45 | KERN_CONTAINER = '/data/exp_soft/containers/kern5.simg' 46 | 47 | 48 | pattern = sys.argv[1] 49 | 50 | 51 | print(pattern) 52 | 53 | fitslist = glob.glob(pattern) 54 | 55 | slurmfile = 'slurm_render_pngs.sh' 56 | logfile = 'slurm_render_pngs.log' 57 | 58 | syscall = '' 59 | 60 | print(fitslist) 61 | 62 | for infits in fitslist: 63 | print(infits) 64 | syscall += 'singularity exec '+KERN_CONTAINER+' '+generate_syscall_mviewer(infits)+'\n' 65 | 66 | write_slurm(opfile=slurmfile, 67 | jobname='makepngs', 68 | logfile=logfile, 69 | container=KERN_CONTAINER, 70 | syscall=syscall) 71 | 72 | print('sbatch '+slurmfile) 73 | 74 | 75 | if __name__ == "__main__": 76 | 77 | 78 | main() -------------------------------------------------------------------------------- /waterhole/setup_chgcentre.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import os 3 | import sys 4 | 5 | 6 | def write_slurm(opfile,jobname,logfile,syscall): 7 | 8 | f = open(opfile,'w') 9 | f.writelines(['#!/bin/bash\n', 10 | '#file: '+opfile+':\n', 11 | '#SBATCH --job-name='+jobname+'\n', 12 | '#SBATCH --time=01:00:00\n', 13 | '#SBATCH --partition=Main\n' 14 | '#SBATCH --ntasks=1\n', 15 | '#SBATCH --nodes=1\n', 16 | '#SBATCH --cpus-per-task=8\n', 17 | '#SBATCH --mem=32GB\n', 18 | '#SBATCH --account=b24-thunderkat-ag\n', 19 | '#SBATCH --output='+logfile+'\n', 20 | syscall+'\n', 21 | 'sleep 10\n']) 22 | f.close() 23 | 24 | 25 | def main(): 26 | 27 | 28 | CHGCENTRE_CONTAINER = '/idia/software/containers/STIMELA_IMAGES/stimela_chgcentre_1.2.0.sif' 29 | 30 | runfile = 'submit_chgcentre_jobs.sh' 31 | 32 | op = open(runfile,'w') 33 | op.writelines(['#!/bin/bash\n']) 34 | 35 | sunfile = sys.argv[1] 36 | f = open(sunfile) 37 | line = f.readline() 38 | while line: 39 | cols = line.split() 40 | if len(cols) == 18: 41 | scan = cols[5] 42 | field = cols[7] 43 | ra = cols[10] 44 | dec = cols[11] 45 | myms = glob.glob('*_'+field+'_scan'+scan+'.ms') 46 | 47 | if len(myms) == 1: 48 | 49 | myms = myms[0] 50 | code = 'chgcen'+scan 51 | syscall = 'singularity exec '+CHGCENTRE_CONTAINER+' ' 52 | syscall += 'chgcentre '+myms+' '+ra+' '+dec 53 | 54 | slurm_file = 'SCRIPTS/slurm_'+code+'.sh' 55 | log_file = 'LOGS/slurm_'+code+'.log' 56 | 57 | write_slurm(opfile=slurm_file,jobname=code,logfile=log_file,syscall=syscall) 58 | 59 | op.writelines(['sbatch '+slurm_file+'\n']) 60 | 61 | line = f.readline() 62 | f.close() 63 | 64 | op.close() 65 | 66 | print('Wrote '+runfile+' script') 67 | 68 | if __name__ == "__main__": 69 | 70 | 71 | main() -------------------------------------------------------------------------------- /oxkat/1GC_02_casa_basic_flags.py: -------------------------------------------------------------------------------- 1 | # ian.heywood@physics.ox.ac.uk 2 | 3 | 4 | import numpy 5 | 6 | 7 | execfile('oxkat/casa_read_project_info.py') 8 | execfile('oxkat/config.py') 9 | 10 | 11 | args = sys.argv 12 | for item in sys.argv: 13 | parts = item.split('=') 14 | if parts[0] == 'myms': 15 | myms = parts[1] 16 | 17 | 18 | clearstat() 19 | clearstat() 20 | 21 | 22 | # ------------------------------------------------------------------------ 23 | # Frequency ranges to flag over all baselines 24 | 25 | if CAL_1GC_BAD_FREQS != []: 26 | 27 | myspw = ','.join(CAL_1GC_BAD_FREQS) 28 | 29 | # myspw = '' 30 | # for badfreq in CAL_1GC_BAD_FREQS: 31 | # myspw += '*:'+badfreq+',' 32 | # myspw = myspw.rstrip(',') 33 | 34 | flagdata(vis = myms, 35 | mode = 'manual', 36 | spw = myspw) 37 | 38 | 39 | # ------------------------------------------------------------------------ 40 | # Frequency ranges to flag over a subset of baselines 41 | 42 | if CAL_1GC_BL_FREQS != []: 43 | 44 | myspw = ','.join(CAL_1GC_BL_FREQS) 45 | 46 | # myspw = '' 47 | # for badfreq in CAL_1GC_BL_FREQS: 48 | # myspw += '*:'+badfreq+',' 49 | # myspw = myspw.rstrip(',') 50 | 51 | flagdata(vis = myms, 52 | mode = 'manual', 53 | spw = myspw, 54 | uvrange = CAL_1GC_BL_FLAG_UVRANGE) 55 | 56 | 57 | # ------------------------------------------------------------------------ 58 | # Clipping, quacking, zeros, autos 59 | # Note that clip will always flag NaN/Inf values even with a range 60 | 61 | #flagdata(vis = myms, 62 | # mode = 'quack', 63 | # quackinterval = 8.0, 64 | # quackmode = 'beg') 65 | 66 | flagdata(vis = myms, 67 | mode = 'manual', 68 | autocorr = True) 69 | 70 | flagdata(vis = myms, 71 | mode = 'clip', 72 | clipzeros = True) 73 | 74 | flagdata(vis = myms, 75 | mode = 'clip', 76 | clipminmax = [0.0,100.0]) 77 | 78 | # ------------------------------------------------------------------------ 79 | # Save the flags 80 | 81 | if SAVE_FLAGS: 82 | 83 | flagmanager(vis = myms, 84 | mode = 'save', 85 | versionname = 'basic') 86 | 87 | 88 | clearstat() 89 | clearstat() 90 | -------------------------------------------------------------------------------- /waterhole/setup_split_by_scan.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path as o 3 | import sys 4 | sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) 5 | 6 | 7 | from oxkat import generate_jobs as gen 8 | from oxkat import config as cfg 9 | 10 | 11 | def write_slurm(opfile,jobname,logfile,syscall): 12 | 13 | f = open(opfile,'w') 14 | f.writelines(['#!/bin/bash\n', 15 | '#file: '+opfile+':\n', 16 | '#SBATCH --job-name='+jobname+'\n', 17 | '#SBATCH --time=03:00:00\n', 18 | '#SBATCH --partition=Main\n' 19 | '#SBATCH --ntasks=1\n', 20 | '#SBATCH --nodes=1\n', 21 | '#SBATCH --cpus-per-task=8\n', 22 | '#SBATCH --mem=32GB\n', 23 | '#SBATCH --account=b24-thunderkat-ag\n', 24 | '#SBATCH --output='+logfile+'\n', 25 | syscall+'\n']) 26 | f.close() 27 | 28 | 29 | def main(): 30 | 31 | if len(sys.argv) == 1: 32 | print('Please specify a Measurement Set to split') 33 | sys.exit() 34 | else: 35 | myms = sys.argv[1] 36 | 37 | INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(('','idia')) 38 | CASA_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.CASA_PATTERN,True) 39 | 40 | casa_file = 'casa_split_by_scan.py' 41 | slurm_file = 'slurm_split_by_scan.sh' 42 | log_file = slurm_file.replace('.sh','.log') 43 | 44 | f = open(casa_file,'w') 45 | f.writelines(['tb.open("'+myms+'")\n', 46 | 'scans = sorted(set(tb.getcol("SCAN_NUMBER")))\n', 47 | 'tb.done\n', 48 | 'for scan in scans:\n', 49 | ' scan = str(scan)\n', 50 | ' opms = "'+myms+'".replace(".ms","_scan"+scan+".ms")\n', 51 | ' split(vis="'+myms+'",outputvis=opms,scan=scan,datacolumn="all")\n']) 52 | f.close() 53 | 54 | 55 | syscall = 'singularity exec '+CASA_CONTAINER+' casa -c '+casa_file+' --nologger --log2term --nogui' 56 | 57 | write_slurm(opfile=slurm_file,jobname='split',logfile=log_file,syscall=syscall ) 58 | 59 | print('Run "sbatch '+slurm_file+'" to submit') 60 | 61 | 62 | if __name__ == "__main__": 63 | 64 | main() 65 | -------------------------------------------------------------------------------- /oxkat/user_field_handler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | import json 6 | import os.path as o 7 | import sys 8 | sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) 9 | 10 | from oxkat import generate_jobs as gen 11 | from oxkat import config as cfg 12 | 13 | PRE_FIELDS = cfg.PRE_FIELDS 14 | 15 | 16 | with open('project_info.json') as f: 17 | project_info = json.load(f) 18 | 19 | 20 | bpcal_name = project_info['primary_name'] 21 | target_names = project_info['target_names'] 22 | pcal_names = project_info['secondary_names'] 23 | bpcal = project_info['primary_id'] 24 | targets = project_info['target_ids'] 25 | pcals = project_info['secondary_ids'] 26 | target_cal_map = project_info['target_cal_map'] 27 | 28 | 29 | 30 | if PRE_FIELDS != '': 31 | 32 | pre_field_list = PRE_FIELDS.split(',') 33 | 34 | user_targets = [] 35 | user_pcals = [] 36 | user_pcal_ids = [] 37 | user_cal_map = [] 38 | 39 | names = False 40 | for src in [bpcal_name]+target_names+pcal_names: 41 | if src in pre_field_list: 42 | names = True 43 | 44 | if names: 45 | if bpcal_name not in pre_field_list: 46 | print('Pre-field selection does not include a primary calibrator') 47 | sys.exit() 48 | for src in target_names: 49 | if src in pre_field_list: 50 | user_targets.append(src) 51 | for src in pcal_names: 52 | if src in pre_field_list: 53 | user_pcals.append(src) 54 | 55 | if not names: 56 | if bpcal not in pre_field_list: 57 | print('Pre-field selection does not include a primary calibrator') 58 | sys.exit() 59 | for src in targets: 60 | idx = targets.index(src) 61 | if src in pre_field_list: 62 | user_targets.append(target_names[idx]) 63 | for src in pcals: 64 | idx = pcals.index(src) 65 | if src in pre_field_list: 66 | user_pcals.append(pcal_names[idx]) 67 | 68 | for src in user_targets: 69 | idx = target_names.index(src) 70 | user_cal_map.append(target_cal_map[idx]) 71 | 72 | # for src in user_pcals: 73 | # idx = pcal_names.index(src) 74 | # user_pcal_ids.append(pcals[idx]) 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | -------------------------------------------------------------------------------- /tools/azimuthal_average.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | # Requires: pip install scikit-ued 6 | 7 | 8 | import numpy 9 | import shutil 10 | import sys 11 | from astropy.io import fits 12 | from skued import azimuthal_average as aa 13 | 14 | 15 | def getImage(fitsfile): 16 | input_hdu = fits.open(fitsfile)[0] 17 | if len(input_hdu.data.shape) == 2: 18 | image = numpy.array(input_hdu.data[:,:]) 19 | elif len(input_hdu.data.shape) == 3: 20 | image = numpy.array(input_hdu.data[0,:,:]) 21 | elif len(input_hdu.data.shape) == 4: 22 | image = numpy.array(input_hdu.data[0,0,:,:]) 23 | elif len(input_hdu.data.shape) == 5: 24 | image = numpy.array(input_hdu.data[0,0,0,:,:]) 25 | return image 26 | 27 | 28 | def flushFits(newimage,fitsfile): 29 | f = fits.open(fitsfile,mode='update') 30 | input_hdu = f[0] 31 | if len(input_hdu.data.shape) == 2: 32 | input_hdu.data[:,:] = newimage 33 | elif len(input_hdu.data.shape) == 3: 34 | input_hdu.data[0,:,:] = newimage 35 | elif len(input_hdu.data.shape) == 4: 36 | input_hdu.data[0,0,:,:] = newimage 37 | elif len(input_hdu.data.shape) == 5: 38 | input_hdu.data[0,0,0,:,:] = newimage 39 | f.flush() 40 | 41 | 42 | def main(): 43 | 44 | input_fits = sys.argv[1] 45 | 46 | output_fits = input_fits.replace('.fits','_azavg.fits') 47 | 48 | shutil.copyfile(input_fits,output_fits) 49 | 50 | input_img = getImage(input_fits) 51 | output_img = input_img * 0.0 52 | ny,nx = input_img.shape 53 | x0 = int(nx/2) 54 | y0 = int(ny/2) 55 | 56 | radius,average = aa(input_img,center=(x0,y0)) 57 | 58 | for y in range(0,ny): 59 | for x in range(0,nx): 60 | val = (((float(y)-y0)**2.0)+((float(x)-x0)**2.0))**0.5 61 | output_img[y][x] = average[int(val)] 62 | 63 | flushFits(output_img,output_fits) 64 | 65 | 66 | 67 | if __name__ == "__main__": 68 | 69 | 70 | main() -------------------------------------------------------------------------------- /waterhole/setup_movies.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import os 3 | import os.path as o 4 | import sys 5 | sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) 6 | 7 | 8 | from oxkat import generate_jobs as gen 9 | from oxkat import config as cfg 10 | 11 | 12 | def write_slurm(opfile,jobname,logfile,syscall): 13 | 14 | f = open(opfile,'w') 15 | f.writelines(['#!/bin/bash\n', 16 | '#file: '+opfile+':\n', 17 | '#SBATCH --job-name='+jobname+'\n', 18 | '#SBATCH --time=02:00:00\n', 19 | '#SBATCH --partition=Main\n' 20 | '#SBATCH --ntasks=1\n', 21 | '#SBATCH --nodes=1\n', 22 | '#SBATCH --cpus-per-task=4\n', 23 | '#SBATCH --mem=16GB\n', 24 | '#SBATCH --account=b24-thunderkat-ag\n', 25 | '#SBATCH --output='+logfile+'\n', 26 | syscall+'\n']) 27 | f.close() 28 | 29 | 30 | def main(): 31 | 32 | 33 | INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(('','idia')) 34 | ASTROPY_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.ASTROPY_PATTERN,True) 35 | 36 | intervals = sorted(glob.glob('INTERVALS/*scan*')) 37 | rootdir = os.getcwd() 38 | 39 | runfile = 'submit_movie_jobs.sh' 40 | 41 | f = open(runfile,'w') 42 | f.writelines(['#!/bin/bash\n']) 43 | 44 | for mydir in intervals: 45 | 46 | os.chdir(mydir) 47 | code = os.getcwd().split('/')[-1].split('_')[-1].replace('scan','movie') 48 | syscall = 'singularity exec '+ASTROPY_CONTAINER+' ' 49 | syscall += 'python3 '+rootdir+'/tools/make_movie.py' 50 | 51 | slurm_file = 'slurm_'+code+'.sh' 52 | log_file = 'slurm_'+code+'.log' 53 | 54 | write_slurm(opfile=slurm_file,jobname=code,logfile=log_file,syscall=syscall ) 55 | os.chdir('../../') 56 | 57 | # print('cd '+mydir) 58 | # print('sbatch '+slurm_file) 59 | # print('cd ../../') 60 | 61 | f.writelines(['cd '+mydir+'\n', 62 | 'sbatch '+slurm_file+'\n', 63 | 'cd ../../\n']) 64 | 65 | f.close() 66 | gen.make_executable(runfile) 67 | print('Wrote '+runfile+' script') 68 | 69 | if __name__ == "__main__": 70 | 71 | main() -------------------------------------------------------------------------------- /tools/merge_CHPC_jobs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | # Usage: python tools/merge_CHPC_1GC_jobs.py 6 | # Will merge all the sequential jobs in the submit_jobs.sh script into a single submission 7 | # in an attempt to get around the haunted job queue. 8 | 9 | 10 | import sys 11 | import os.path as o 12 | sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) 13 | from oxkat import config as cfg 14 | 15 | 16 | submit_file = sys.argv[1] 17 | 18 | jobname = submit_file.split('_')[1] 19 | 20 | pbs_config = cfg.PBS_WSCLEAN 21 | pbs_program = pbs_config['PROGRAM'] 22 | pbs_walltime = '48:00:00' 23 | pbs_queue = pbs_config['QUEUE'] 24 | pbs_nodes = pbs_config['NODES'] 25 | pbs_ppn = pbs_config['PPN'] 26 | pbs_mem = pbs_config['MEM'] 27 | 28 | 29 | outfile = submit_file.replace('submit','pbs_merged') 30 | pbs_logfile = cfg.LOGS+'/'+outfile.replace('.sh','.log') 31 | pbs_errfile = pbs_logfile.replace('.log','.err') 32 | 33 | 34 | 35 | pbs_list = [] 36 | f = open(submit_file,'r') 37 | line = f.readline() 38 | while line: 39 | if line[0] != '#': 40 | cols = line.split() 41 | for col in cols: 42 | if '/pbs_' in col: 43 | pbs_list.append(col) 44 | line = f.readline() 45 | f.close() 46 | 47 | 48 | singularity_calls = [] 49 | for pbs_file in pbs_list: 50 | f = open(pbs_file,'r') 51 | line = f.readline() 52 | while line: 53 | cols = line.split() 54 | for col in cols: 55 | if 'exec' in col: 56 | singularity_calls.append(line) 57 | line = f.readline() 58 | f.close() 59 | 60 | 61 | f = open(outfile,'w') 62 | f.writelines(['#!/bin/bash\n', 63 | '#PBS -N '+jobname+' \n', 64 | '#PBS -P '+pbs_program+'\n', 65 | '#PBS -l walltime='+pbs_walltime+'\n', 66 | '#PBS -l nodes='+pbs_nodes+':ppn='+pbs_ppn+',mem='+pbs_mem+'\n', 67 | '#PBS -q '+pbs_queue+'\n', 68 | '#PBS -o '+pbs_logfile+'\n', 69 | '#PBS -e '+pbs_errfile+'\n', 70 | 'module load chpc/singularity\n', 71 | 'cd '+cfg.CWD+'\n']) 72 | for ii in singularity_calls: 73 | f.writelines([ii]) 74 | f.writelines(['sleep 10\n']) 75 | 76 | 77 | print('qsub '+outfile) 78 | -------------------------------------------------------------------------------- /waterhole/setup_restore.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import os 3 | import os.path as o 4 | import sys 5 | sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) 6 | 7 | 8 | from oxkat import generate_jobs as gen 9 | from oxkat import config as cfg 10 | 11 | 12 | def write_slurm(opfile,jobname,logfile,syscall): 13 | 14 | f = open(opfile,'w') 15 | f.writelines(['#!/bin/bash\n', 16 | '#file: '+opfile+':\n', 17 | '#SBATCH --job-name='+jobname+'\n', 18 | '#SBATCH --time=03:00:00\n', 19 | '#SBATCH --partition=Main\n' 20 | '#SBATCH --ntasks=1\n', 21 | '#SBATCH --nodes=1\n', 22 | '#SBATCH --cpus-per-task=4\n', 23 | '#SBATCH --mem=16GB\n', 24 | '#SBATCH --account=b24-thunderkat-ag\n', 25 | '#SBATCH --output='+logfile+'\n', 26 | syscall+'\n']) 27 | f.close() 28 | 29 | 30 | def main(): 31 | 32 | if len(sys.argv) == 1: 33 | print('Please specify the full path to the model image') 34 | sys.exit() 35 | else: 36 | model_fits = sys.argv[1] 37 | 38 | INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(('','idia')) 39 | ASTROPY_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.ASTROPY_PATTERN,True) 40 | 41 | intervals = sorted(glob.glob('INTERVALS/*scan*')) 42 | rootdir = os.getcwd() 43 | 44 | runfile = 'submit_restore_jobs.sh' 45 | 46 | f = open(runfile,'w') 47 | f.writelines(['#!/bin/bash\n']) 48 | 49 | for mydir in intervals: 50 | 51 | os.chdir(mydir) 52 | code = os.getcwd().split('/')[-1].split('_')[-1].replace('scan','rstor') 53 | syscall = 'singularity exec '+ASTROPY_CONTAINER+' ' 54 | syscall += 'python3 '+rootdir+'/tools/restore_model.py '+model_fits 55 | 56 | slurm_file = 'slurm_'+code+'.sh' 57 | log_file = 'slurm_'+code+'.log' 58 | 59 | write_slurm(opfile=slurm_file,jobname=code,logfile=log_file,syscall=syscall ) 60 | os.chdir('../../') 61 | 62 | f.writelines(['cd '+mydir+'\n', 63 | 'sbatch '+slurm_file+'\n', 64 | 'cd ../../\n']) 65 | 66 | f.close() 67 | gen.make_executable(runfile) 68 | print('\nWrote '+runfile+' script') 69 | 70 | if __name__ == "__main__": 71 | 72 | 73 | main() 74 | -------------------------------------------------------------------------------- /oxkat/casa_read_project_info.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | import json 6 | import sys 7 | execfile('oxkat/config.py') 8 | 9 | 10 | def str_iterator(inlist): 11 | xx = [] 12 | for yy in inlist: 13 | xx.append(str(yy)) 14 | return xx 15 | 16 | 17 | with open('project_info.json') as f: 18 | project_info = json.load(f) 19 | 20 | master_ms = str(project_info['master_ms']) 21 | myms = str(project_info['working_ms']) 22 | band = str(project_info['band']) 23 | nchan = int(project_info['nchan']) 24 | ref_ant = str(project_info['ref_ant']) 25 | bpcal = str(project_info['primary_id']) 26 | bpcal_name = str(project_info['primary_name']) 27 | primary_tag = str(project_info['primary_tag']) 28 | pcal_names = str_iterator(project_info['secondary_names']) 29 | pcals = str_iterator(project_info['secondary_ids']) 30 | pcal_dirs = project_info['secondary_dirs'] 31 | target_names = str_iterator(project_info['target_names']) 32 | targets = str_iterator(project_info['target_ids']) 33 | target_dirs = project_info['target_dirs'] 34 | target_cal_map = str_iterator(project_info['target_cal_map']) 35 | target_ms = str_iterator(project_info['target_ms']) 36 | 37 | if PRE_FIELDS != '': 38 | 39 | pre_field_list = PRE_FIELDS.split(',') 40 | 41 | user_targets = [] 42 | user_pcals = [] 43 | user_cal_map = [] 44 | 45 | names = False 46 | for src in [bpcal_name]+target_names+pcal_names: 47 | if src in pre_field_list: 48 | names = True 49 | 50 | if names: 51 | if bpcal_name not in pre_field_list: 52 | print('Pre-field selection does not include a primary calibrator') 53 | sys.exit() 54 | for src in target_names: 55 | if src in pre_field_list: 56 | user_targets.append(src) 57 | for src in pcal_names: 58 | if src in pre_field_list: 59 | user_pcals.append(src) 60 | 61 | if not names: 62 | if bpcal not in pre_field_list: 63 | print('Pre-field selection does not include a primary calibrator') 64 | sys.exit() 65 | for src in targets: 66 | idx = targets.index(src) 67 | if src in pre_field_list: 68 | user_targets.append(target_names[idx]) 69 | for src in pcals: 70 | idx = pcals.index(src) 71 | if src in pre_field_list: 72 | user_pcals.append(pcal_names[idx]) 73 | 74 | for src in user_targets: 75 | idx = target_names.index(src) 76 | user_cal_map.append(target_cal_map[idx]) 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | -------------------------------------------------------------------------------- /tools/copy_MS_column.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | import numpy 6 | import sys 7 | from optparse import OptionParser 8 | from pyrap.tables import table 9 | 10 | 11 | def copycol(msname,fromcol,tocol,field,rowchunk): 12 | 13 | if field == '': 14 | tt = table(msname,readonly=False) 15 | else: 16 | print('Selecting FIELD_ID '+str(field)) 17 | t0 = table(msname,readonly=False) 18 | tt = t0.query(query='FIELD_ID=='+str(field)) 19 | 20 | colnames = tt.colnames() 21 | if fromcol not in colnames or tocol not in colnames: 22 | print('One or more requested columns not present in MS') 23 | sys.exit() 24 | 25 | spws = numpy.unique(tt.getcol('DATA_DESC_ID')) 26 | print('Spectral windows: '+str(spws)) 27 | 28 | for spw in spws: 29 | spw_tab = tt.query(query='DATA_DESC_ID=='+str(spw)) 30 | 31 | nrows = spw_tab.nrows() 32 | for start_row in range(0,nrows,rowchunk): 33 | nr = min(rowchunk,nrows-start_row) 34 | print('Processing rows: '+str(start_row)+' to '+str(start_row+nr)+' for SPW '+str(spw)) 35 | spw_tab.putcol(tocol,spw_tab.getcol(fromcol,start_row,nr),start_row,nr) 36 | spw_tab.done() 37 | tt.done() 38 | 39 | if field != '': t0.done() 40 | 41 | 42 | def main(): 43 | 44 | 45 | parser = OptionParser(usage = '%prog [options] msname') 46 | parser.add_option('--fromcol', dest = 'fromcol', default = 'MODEL_DATA', help = 'Name of source column (default = MODEL_DATA') 47 | parser.add_option('--tocol', dest = 'tocol', default = 'DIR1_DATA', help = 'Name of destination column (default = DIR1_DATA') 48 | parser.add_option('--field', dest = 'field', default = '', help = 'Field selection (default = all fields)') 49 | parser.add_option('--rowchunk', dest = 'rowchunk', default = 500000, help = 'Number of chunks to process at once (default = 500000)') 50 | (options,args) = parser.parse_args() 51 | fromcol = options.fromcol 52 | tocol = options.tocol 53 | field = options.field 54 | rowchunk = int(options.rowchunk) 55 | 56 | 57 | if len(args) != 1: 58 | print('Please specify a Measurement Set') 59 | sys.exit() 60 | else: 61 | msname = args[0].rstrip('/') 62 | 63 | 64 | copycol(msname,fromcol,tocol,field,rowchunk) 65 | 66 | 67 | 68 | if __name__ == '__main__': 69 | 70 | main() -------------------------------------------------------------------------------- /tools/casa_find_source-cal_pairs.py: -------------------------------------------------------------------------------- 1 | # Find primary cal 2 | # Find nearest secondary cal to each target 3 | # Split a new MS containing primary and secondary-target pairs 4 | # ian.heywood@physics.ox.ac.uk 5 | 6 | 7 | import numpy 8 | import glob 9 | import sys 10 | 11 | 12 | myms = glob.glob('*.ms')[0] 13 | 14 | 15 | tb.open(myms+'/STATE') 16 | modes = tb.getcol('OBS_MODE') 17 | tb.close() 18 | 19 | 20 | for i in range(0,len(modes)): 21 | if modes[i] == 'TARGET': 22 | target_state = i 23 | if 'BANDPASS' in modes[i]: 24 | primary_state = i 25 | if 'PHASE' in modes[i]: 26 | secondary_state = i 27 | 28 | 29 | tb.open(myms+'/FIELD') 30 | fld_names = tb.getcol('NAME') 31 | dirs = tb.getcol('REFERENCE_DIR') 32 | tb.close() 33 | 34 | 35 | targets = [] 36 | pcals = [] 37 | bpcal = '' 38 | 39 | 40 | tb.open(myms) 41 | for i in range(0,len(fld_names)): 42 | sub_tab = tb.query(query='FIELD_ID=='+str(i)) 43 | state = numpy.unique(sub_tab.getcol('STATE_ID')) 44 | if state == target_state: 45 | opms = myms.rstrip('/').replace('.ms',fld_names[i]+'.ms') 46 | targets.append((str(i),fld_names[i])) 47 | elif state == primary_state: 48 | bpcal = str(i) 49 | elif state == secondary_state: 50 | pcals.append((str(i),fld_names[i])) 51 | tb.done() 52 | 53 | 54 | field_selections = [] 55 | 56 | 57 | for pcal in pcals: 58 | 59 | seps = [] 60 | pcal_idx = pcal[0] 61 | 62 | ra_pcal = str(180.0*dirs[0][0][int(pcal_idx)]/numpy.pi)+'deg' 63 | dec_pcal = str(180.0*dirs[1][0][int(pcal_idx)]/numpy.pi)+'deg' 64 | 65 | 66 | dir_pcal = me.direction('J2000',ra_pcal,dec_pcal) 67 | 68 | 69 | for target in targets: 70 | targ_idx = target[0] 71 | ra_target = str(180.0*dirs[0][0][int(targ_idx)]/numpy.pi)+'deg' 72 | dec_target = str(180.0*dirs[1][0][int(targ_idx)]/numpy.pi)+'deg' 73 | 74 | dir_target = me.direction('J2000',ra_target,dec_target) 75 | 76 | seps.append(me.separation(dir_target,dir_pcal)['value']) 77 | 78 | seps = numpy.array(seps) 79 | 80 | targ_match = numpy.where((seps == numpy.min(seps))==True)[0][0] 81 | 82 | field_selections.append(bpcal+','+pcal_idx+','+str(targets[targ_match][0])) 83 | 84 | 85 | 86 | for i in range(0,len(field_selections)): 87 | 88 | opms = myms.replace('.ms','_BLOCK'+str(i)+'.ms') 89 | 90 | mstransform(vis=myms,outputvis=opms,field=field_selections[i],usewtspectrum=True,datacolumn='data',realmodelcol=True,chanaverage=True,chanbin=4,timeaverage=False) 91 | -------------------------------------------------------------------------------- /oxkat/2GC_casa_selfcal_target_amp_phases.py: -------------------------------------------------------------------------------- 1 | # ian.heywood@physics.ox.ac.uk 2 | 3 | 4 | import sys 5 | import time 6 | 7 | 8 | execfile('oxkat/config.py') 9 | execfile('oxkat/casa_read_project_info.py') 10 | 11 | 12 | def stamp(): 13 | now = str(datetime.datetime.now()).replace(' ','-').replace(':','-').split('.')[0] 14 | return now 15 | 16 | 17 | myuvrange = '' 18 | psolint = '' 19 | apsolint = '' 20 | 21 | args = sys.argv 22 | for item in sys.argv: 23 | parts = item.split('=') 24 | if parts[0] == 'mslist': 25 | mslist = parts[1].split(',') 26 | if parts[0] == 'uvmin': 27 | myuvrange = '>'+parts[1] 28 | if parts[0] == 'psolint': 29 | psolint = parts[1] 30 | if parts[0] == 'apsolint': 31 | apsolint = parts[1] 32 | 33 | if myuvrange == '': 34 | myuvrange = CAL_2GC_UVRANGE 35 | if psolint == '': 36 | psolint = CAL_2GC_PSOLINT 37 | if apsolint == '': 38 | apsolint = CAL_2GC_APSOLINT 39 | 40 | 41 | 42 | for myms in mslist: 43 | 44 | clearstat() 45 | clearstat() 46 | 47 | 48 | gptab = GAINTABLES+'/cal_2GC_'+myms+'_'+stamp()+'.GP0' 49 | gatab = GAINTABLES+'/cal_2GC_'+myms+'_'+stamp()+'.GA0' 50 | 51 | 52 | gaincal(vis=myms, 53 | field='0', 54 | uvrange=myuvrange, 55 | caltable=gptab, 56 | refant = str(ref_ant), 57 | solint='64s', 58 | solnorm=False, 59 | combine='', 60 | minsnr=3, 61 | calmode='p', 62 | parang=False, 63 | gaintable=[], 64 | gainfield=[], 65 | interp=[], 66 | append=False) 67 | 68 | 69 | gaincal(vis=myms, 70 | field='0', 71 | uvrange=myuvrange, 72 | caltable=gatab, 73 | refant = str(ref_ant), 74 | solint='inf', 75 | solnorm=False, 76 | combine='', 77 | minsnr=3, 78 | calmode='ap', 79 | parang=False, 80 | gaintable=[gptab], 81 | gainfield=[''], 82 | interp=[''], 83 | append=False) 84 | 85 | 86 | 87 | applycal(vis=myms, 88 | gaintable=[gptab,gatab], 89 | field='0', 90 | calwt=False, 91 | parang=False, 92 | applymode='calonly', 93 | gainfield=['',''], 94 | interp = ['nearest','linear']) 95 | 96 | 97 | # statwt(vis=myms, 98 | # field='0') 99 | 100 | 101 | clearstat() 102 | clearstat() 103 | 104 | 105 | -------------------------------------------------------------------------------- /oxkat/1GC_03_primary_cal_field_sources.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | import glob 6 | import json 7 | import os 8 | import os.path as o 9 | import subprocess 10 | import sys 11 | import tarfile 12 | sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) 13 | 14 | 15 | from oxkat import generate_jobs as gen 16 | from oxkat import config as cfg 17 | 18 | 19 | 20 | 21 | 22 | 23 | def main(): 24 | 25 | 26 | with open('project_info.json') as f: 27 | project_info = json.load(f) 28 | 29 | 30 | remove_models = True 31 | 32 | 33 | DATA = cfg.DATA 34 | CALMODELPATH = DATA+'/calmodels/' 35 | 36 | 37 | myms = project_info['master_ms'] 38 | primary_id = project_info['primary_id'] 39 | primary_name = project_info['primary_name'] 40 | primary_tag = project_info['primary_tag'] 41 | 42 | 43 | if cfg.CAL_1GC_PRIMARY_MODEL == 'auto': 44 | caltar = glob.glob(CALMODELPATH+'*'+primary_tag+'*.tar.gz') 45 | if len(caltar) == 1: 46 | caltar = caltar[0] 47 | print('Found '+caltar+', untarring...') 48 | tf = tarfile.open(caltar) 49 | tf.extractall(path=CALMODELPATH) 50 | fitslist = sorted(glob.glob(CALMODELPATH+'*'+primary_tag+'*.fits')) 51 | nchan = len(fitslist) 52 | prefix = fitslist[0].split('-00')[0] 53 | print('Prefix '+prefix+' has '+str(nchan)+' frequency planes') 54 | else: 55 | print('No model images found for '+primary_name) 56 | elif cfg.CAL_1GC_PRIMARY_MODEL == 'setjy': 57 | print('Component model for setjy requested, no additional image-based model prediction will be done.') 58 | prefix = '' 59 | else: 60 | prefix = cfg.CAL_1GC_PRIMARY_MODEL 61 | fitslist = sorted(glob.glob(prefix+'*.fits')) 62 | if len(fitslist) == 0: 63 | print('No FITS images found matching '+prefix+'*.fits') 64 | prefix = '' 65 | else: 66 | nchan = len(fitslist) 67 | print('Prefix '+prefix+' has '+str(nchan)+' frequency planes') 68 | 69 | if prefix != '': 70 | syscall = gen.generate_syscall_predict(msname=myms,imgbase=prefix,field=primary_id,chanout=nchan) 71 | print(syscall) 72 | os.system(syscall) 73 | if remove_models: 74 | for item in fitslist: 75 | print('Removing '+item) 76 | os.remove(item) 77 | 78 | 79 | if __name__ == "__main__": 80 | 81 | 82 | main() 83 | -------------------------------------------------------------------------------- /tools/make_movie.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | import glob 6 | import logging 7 | import os 8 | import random 9 | import numpy 10 | import string 11 | 12 | from astropy.io import fits 13 | from astropy.time import Time 14 | from multiprocessing import Pool 15 | from PIL import Image,ImageDraw,ImageFont 16 | 17 | fontPath = '/usr/share/fonts/truetype/dejavu/DejaVuSansMono.ttf' 18 | sans30 = ImageFont.truetype ( fontPath, 30 ) 19 | 20 | 21 | def generate_temp(k=16): 22 | tmpfits = 'temp_'+''.join(random.choices(string.ascii_uppercase + string.digits, k=k))+'.fits' 23 | return tmpfits 24 | 25 | 26 | def make_png(ff,i): 27 | 28 | tmpfits = generate_temp() 29 | 30 | logging.info(' | File '+str(i)+' | Input image '+ff) 31 | logging.info(' | File '+str(i)+' | Temp image '+tmpfits) 32 | 33 | os.system('mShrink '+ff+' '+tmpfits+' 2') 34 | 35 | input_hdu = fits.open(ff)[0] 36 | hdr = input_hdu.header 37 | map_date = hdr.get('DATE-OBS') 38 | t_mjd = Time(map_date, format='isot', scale='utc').mjd 39 | tt = map_date+' | '+str(t_mjd) 40 | # pp = str(i).zfill(4)+'_'+ff.replace('.fits','.png') 41 | pp = 'pic_'+str(i).zfill(4)+'.png' 42 | # syscall = 'mViewer -ct 0 -gray '+ff+' -0.0004 0.0008 -out '+pp 43 | logging.info(' | File '+str(i)+' | PNG '+pp) 44 | syscall = 'mViewer -ct 0 -gray '+tmpfits+' -0.0008 0.0018 -out '+pp 45 | os.system(syscall) 46 | logging.info(' | File '+str(i)+' | Time '+tt) 47 | img = Image.open(pp) 48 | xx,yy = img.size 49 | draw = ImageDraw.Draw(img) 50 | draw.text((0.03*xx,0.90*yy),'Frame : '+str(i).zfill(len(str(nframes)))+' / '+str(nframes),fill=('white'),font=sans30) 51 | draw.text((0.03*xx,0.93*yy),'Time : '+tt,fill=('white'),font=sans30) 52 | draw.text((0.03*xx,0.96*yy),'Image : '+ff,fill=('white'),font=sans30) 53 | img.save(pp) 54 | os.system('rm '+tmpfits) 55 | logging.info(' | File '+str(i)+' | Done') 56 | 57 | if __name__ == '__main__': 58 | 59 | logfile = 'make_movie.log' 60 | logging.basicConfig(filename=logfile, level=logging.DEBUG, format='%(asctime)s | %(message)s', datefmt='%d/%m/%Y %H:%M:%S ') 61 | 62 | 63 | fitslist = sorted(glob.glob('*-t*-image-restored.fits')) 64 | ids = numpy.arange(0,len(fitslist)) 65 | nframes = len(fitslist) 66 | j = 8 67 | 68 | pool = Pool(processes=j) 69 | # pool.map(make_png,fitslist) 70 | pool.starmap(make_png,zip(fitslist,ids)) 71 | 72 | frame = '2340x2340' 73 | fps = 10 74 | opmovie = fitslist[0].split('-t')[0]+'.mp4' 75 | os.system('ffmpeg -r '+str(fps)+' -f image2 -s '+frame+' -i pic_%04d.png -vcodec libx264 -crf 25 -pix_fmt yuv420p '+opmovie) 76 | -------------------------------------------------------------------------------- /tools/reg2npy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | # Convert a DS9 (circles only) region file to a ClusterCat.npy file 5 | # for use with killMS. This is a pure Python script to avoid having 6 | # to invoke MakeModel.py for this simple purpose. 7 | 8 | 9 | import numpy 10 | import sys 11 | from optparse import OptionParser 12 | 13 | 14 | def deg2rad(xx): 15 | return xx*numpy.pi/180.0 16 | 17 | def hms2deg(ra,delimiter=':'): 18 | hms = numpy.fromstring(ra,dtype='float',sep=delimiter) 19 | radeg = 15.0*((hms[0])+(hms[1]/60.0)+(hms[2]/3600.0)) 20 | return radeg 21 | 22 | def dms2deg(dec,delimiter=':'): 23 | if dec[0] == '+': 24 | sign = 1.0 25 | dec = dec.lstrip('+') 26 | elif dec[0] == '-': 27 | sign = -1.0 28 | dec = dec.lstrip('-') 29 | else: 30 | sign = 1.0 31 | dms = numpy.fromstring(dec,dtype='float',sep=delimiter) 32 | dec = sign * ((dms[0])+(dms[1]/60.0)+(dms[2]/3600.0)) 33 | return dec 34 | 35 | 36 | def main(): 37 | 38 | parser = OptionParser(usage = '%prog [options] regionfile') 39 | parser.add_option('--outname', dest = 'outname', default = '', help = 'Name of output npy file (default = append .npy)') 40 | (options,args) = parser.parse_args() 41 | outname = options.outname 42 | 43 | 44 | if len(args) != 1: 45 | print('Please specify a DS9 region file') 46 | sys.exit() 47 | else: 48 | regfile = args[0] 49 | 50 | if outname == '': 51 | npyfile = regfile+'.npy' 52 | else: 53 | npyfile = outname 54 | 55 | 56 | print('Reading '+regfile) 57 | 58 | centres = [] 59 | f = open(regfile,'r') 60 | line = f.readline() 61 | while line: 62 | if line[0:6] == 'circle': 63 | coords = line.split('(')[-1].split(')')[0] 64 | x,y,rad = coords.split(',') 65 | if ':' in line: 66 | ra = hms2deg(x) 67 | dec = dms2deg(y) 68 | else: 69 | ra = float(x) 70 | dec = float(y) 71 | centres.append((ra,dec)) 72 | line = f.readline() 73 | f.close() 74 | 75 | 76 | ClusterCat=numpy.zeros((len(centres),),dtype=[('Name','|S200'),('ra',float),('dec',float),('SumI',float),("Cluster",int)]) 77 | 78 | 79 | for i in range(0,len(centres)): 80 | ClusterCat[i]['Name'] = str(i) 81 | ClusterCat[i]['ra'] = deg2rad(centres[i][0]) 82 | ClusterCat[i]['dec'] = deg2rad(centres[i][1]) 83 | ClusterCat[i]['SumI'] = 1.0 84 | ClusterCat[i]['Cluster'] = i 85 | 86 | print('Writing '+npyfile) 87 | 88 | numpy.save(npyfile,ClusterCat) 89 | 90 | 91 | 92 | if __name__ == '__main__': 93 | 94 | main() -------------------------------------------------------------------------------- /tools/pbcor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | import glob 6 | import numpy 7 | import os 8 | import shutil 9 | import sys 10 | from astropy.io import fits 11 | from optparse import OptionParser 12 | 13 | 14 | 15 | def getImage(fitsfile): 16 | input_hdu = fits.open(fitsfile)[0] 17 | if len(input_hdu.data.shape) == 2: 18 | image = numpy.array(input_hdu.data[:,:]) 19 | elif len(input_hdu.data.shape) == 3: 20 | image = numpy.array(input_hdu.data[0,:,:]) 21 | elif len(input_hdu.data.shape) == 4: 22 | image = numpy.array(input_hdu.data[0,0,:,:]) 23 | else: 24 | image = numpy.array(input_hdu.data[0,0,0,:,:]) 25 | return image 26 | 27 | def flushFits(newimage,fitsfile): 28 | f = fits.open(fitsfile,mode='update') 29 | input_hdu = f[0] 30 | if len(input_hdu.data.shape) == 2: 31 | input_hdu.data[:,:] = newimage 32 | elif len(input_hdu.data.shape) == 3: 33 | input_hdu.data[0,:,:] = newimage 34 | elif len(input_hdu.data.shape) == 4: 35 | input_hdu.data[0,0,:,:] = newimage 36 | else: 37 | input_hdu.data[0,0,0,:,:] = newimage 38 | f.flush() 39 | 40 | 41 | 42 | def main(): 43 | 44 | parser = OptionParser(usage = '%prog [options]') 45 | parser.add_option('--pbfits', dest = 'pbfits', help = 'Primary beam FITS image') 46 | parser.add_option('--pattern', dest = 'pattern', help = 'Pattern for images to correct') 47 | parser.add_option('--threshold', dest = 'threshold', help = 'Primary beam cutoff (default = 0.3)', default = 0.3) 48 | parser.add_option('--doweight', dest = 'doweight', help = 'Save a weight image for mosaicking', action = 'store_true', default = False) 49 | 50 | (options,args) = parser.parse_args() 51 | pbfits = options.pbfits 52 | pattern = options.pattern 53 | threshold = float(options.threshold) 54 | doweight = options.doweight 55 | 56 | pbimg = getImage(pbfits) 57 | mask = pbimg < threshold 58 | pbimg[mask] = numpy.nan 59 | 60 | fitslist = glob.glob(pattern) 61 | 62 | for infits in fitslist: 63 | 64 | pbcorfits = infits.replace('.fits','_pbcor.fits') 65 | 66 | if os.path.isfile(pbcorfits): 67 | print(pbcorfits,'exists, skipping') 68 | else: 69 | print('Correcting',infits) 70 | shutil.copy(infits,pbcorfits) 71 | inimg = getImage(infits) 72 | pbcorimg = inimg / pbimg 73 | flushFits(pbcorimg,pbcorfits) 74 | if doweight: 75 | wtfits = infits.replace('.fits','_wt.fits') 76 | shutil.copy(infits,wtfits) 77 | flushFits(pbimg**2.0,wtfits) 78 | 79 | 80 | if __name__ == "__main__": 81 | 82 | 83 | main() 84 | -------------------------------------------------------------------------------- /tools/sum_MS_columns.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | import numpy 6 | import sys 7 | from optparse import OptionParser 8 | from pyrap.tables import table 9 | 10 | 11 | def sumcol(msname,src,dest,field,subtract,rowchunk): 12 | 13 | if field == '': 14 | tt = table(msname,readonly=False) 15 | else: 16 | print('Selecting FIELD_ID '+str(field)) 17 | t0 = table(msname,readonly=False) 18 | tt = t0.query(query='FIELD_ID=='+str(field)) 19 | 20 | colnames = tt.colnames() 21 | if src not in colnames or dest not in colnames: 22 | print('One or more requested columns not present in MS') 23 | sys.exit() 24 | 25 | spws = numpy.unique(tt.getcol('DATA_DESC_ID')) 26 | print('Spectral windows: '+str(spws)) 27 | 28 | print(msname) 29 | if subtract: 30 | print('Subtracting '+src+' from '+dest) 31 | else: 32 | print('Adding '+src+' to '+dest) 33 | 34 | for spw in spws: 35 | spw_tab = tt.query(query='DATA_DESC_ID=='+str(spw)) 36 | 37 | nrows = spw_tab.nrows() 38 | for start_row in range(0,nrows,rowchunk): 39 | nr = min(rowchunk,nrows-start_row) 40 | print('Processing rows: '+str(start_row)+' to '+str(start_row+nr)+' for SPW '+str(spw)) 41 | src_data = spw_tab.getcol(src,start_row,nr) 42 | dest_data = spw_tab.getcol(dest,start_row,nr) 43 | if subtract: 44 | dest_data = dest_data - src_data 45 | else: 46 | dest_data += src_data 47 | spw_tab.putcol(dest,dest_data,start_row,nr) 48 | spw_tab.done() 49 | tt.done() 50 | 51 | if field != '': t0.done() 52 | 53 | 54 | def main(): 55 | 56 | 57 | parser = OptionParser(usage = '%prog [options] msname') 58 | parser.add_option('--src', dest = 'src', help = 'Name of source column.') 59 | parser.add_option('--dest', dest = 'dest', help = 'Name of destination column to which source column will be added.') 60 | parser.add_option('--field', dest = 'field', default = '', help = 'Field selection (default = all fields)') 61 | parser.add_option('--subtract', dest = 'subtract', default = False, help = 'Enable to subtract source column from destination column.', action = 'store_true') 62 | parser.add_option('--rowchunk', dest = 'rowchunk', default = 500000, help = 'Number of chunks to process at once (default = 500000)') 63 | (options,args) = parser.parse_args() 64 | src = options.src 65 | dest = options.dest 66 | field = options.field 67 | subtract = options.subtract 68 | rowchunk = int(options.rowchunk) 69 | 70 | 71 | if len(args) != 1: 72 | print('Please specify a Measurement Set') 73 | sys.exit() 74 | else: 75 | msname = args[0].rstrip('/') 76 | 77 | 78 | sumcol(msname,src,dest,field,subtract,rowchunk) 79 | 80 | 81 | 82 | if __name__ == '__main__': 83 | 84 | main() -------------------------------------------------------------------------------- /data/tricolour/target_flagging_1.yaml: -------------------------------------------------------------------------------- 1 | 2 | # Modified from Ben Hugo's strategy to remove steps 3 | # already taken by basic_flagging 4 | # List of strategies to apply in order 5 | strategies: 6 | # only enable me if you really want to start from scratch 7 | # - 8 | # name: reset_flags: 9 | # task: unflag 10 | - 11 | name: background_flags 12 | task: sum_threshold 13 | kwargs: 14 | outlier_nsigma: 10 15 | windows_time: [1, 2, 4, 8] 16 | windows_freq: [1, 2, 4, 8] 17 | background_reject: 2.0 18 | background_iterations: 5 19 | spike_width_time: 12.5 20 | spike_width_freq: 10.0 21 | time_extend: 3 22 | freq_extend: 3 23 | freq_chunks: 10 24 | average_freq: 1 25 | flag_all_time_frac: 0.6 26 | flag_all_freq_frac: 0.8 27 | rho: 1.3 28 | num_major_iterations: 5 29 | - 30 | name: residual_flag_initial 31 | task: uvcontsub_flagger 32 | kwargs: 33 | major_cycles: 7 34 | or_original_from_cycle: 1 35 | taylor_degrees: 20 36 | sigma: 15.0 37 | # flags are discarded at this point since we or from cycle 1 38 | - 39 | name: final_st_very_broad 40 | task: sum_threshold 41 | kwargs: 42 | outlier_nsigma: 10 43 | windows_time: [1, 2, 4, 8] 44 | windows_freq: [32, 48, 64, 128] 45 | background_reject: 2.0 46 | background_iterations: 5 47 | spike_width_time: 6.5 48 | spike_width_freq: 64.0 49 | time_extend: 3 50 | freq_extend: 3 51 | freq_chunks: 10 52 | average_freq: 1 53 | flag_all_time_frac: 0.6 54 | flag_all_freq_frac: 0.8 55 | rho: 1.3 56 | num_major_iterations: 1 57 | - 58 | name: final_st_broad 59 | task: sum_threshold 60 | kwargs: 61 | outlier_nsigma: 10 62 | windows_time: [1, 2, 4, 8] 63 | windows_freq: [1, 2, 4, 8] 64 | background_reject: 2.0 65 | background_iterations: 5 66 | spike_width_time: 6.5 67 | spike_width_freq: 10.0 68 | time_extend: 3 69 | freq_extend: 3 70 | freq_chunks: 10 71 | average_freq: 1 72 | flag_all_time_frac: 0.6 73 | flag_all_freq_frac: 0.8 74 | rho: 1.3 75 | num_major_iterations: 1 76 | - 77 | name: final_st_narrow 78 | task: sum_threshold 79 | kwargs: 80 | outlier_nsigma: 10 81 | windows_time: [1, 2, 4, 8] 82 | windows_freq: [1, 2, 4, 8] 83 | background_reject: 2.0 84 | background_iterations: 5 85 | spike_width_time: 2 86 | spike_width_freq: 10.0 87 | time_extend: 3 88 | freq_extend: 3 89 | freq_chunks: 10 90 | average_freq: 1 91 | flag_all_time_frac: 0.6 92 | flag_all_freq_frac: 0.8 93 | rho: 1.3 94 | num_major_iterations: 1 95 | - 96 | name: residual_flag_final 97 | task: uvcontsub_flagger 98 | kwargs: 99 | major_cycles: 10 100 | or_original_from_cycle: 0 101 | taylor_degrees: 25 102 | sigma: 13.0 103 | - 104 | name: combine_with_input_flags 105 | task: combine_with_input_flags 106 | 107 | -------------------------------------------------------------------------------- /waterhole/setup_intervals.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | import glob 6 | import numpy 7 | import os 8 | import pickle 9 | import sys 10 | import os.path as o 11 | 12 | sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) 13 | 14 | from oxkat import generate_jobs as gen 15 | from oxkat import config as cfg 16 | 17 | 18 | def write_slurm(opfile,jobname,logfile,syscall): 19 | 20 | f = open(opfile,'w') 21 | f.writelines(['#!/bin/bash\n', 22 | '#file: '+opfile+':\n', 23 | '#SBATCH --job-name='+jobname+'\n', 24 | '#SBATCH --time=07:00:00\n', 25 | '#SBATCH --partition=Main\n' 26 | '#SBATCH --ntasks=1\n', 27 | '#SBATCH --nodes=1\n', 28 | '#SBATCH --cpus-per-task=16\n', 29 | '#SBATCH --mem=115GB\n', 30 | '#SBATCH --account=b24-thunderkat-ag\n', 31 | '#SBATCH --output='+logfile+'\n', 32 | syscall+'\n']) 33 | f.close() 34 | 35 | 36 | def get_scan_times(scanpickle): 37 | scan_times = [] 38 | ss = pickle.load(open(scanpickle,'rb')) 39 | fields = [] 40 | for ii in ss: 41 | fields.append(ii[1]) 42 | fields = numpy.unique(fields).tolist() 43 | for field in fields: 44 | scans = [] 45 | intervals = [] 46 | for ii in ss: 47 | if ii[1] == field: 48 | scans.append(ii[0]) 49 | intervals.append(ii[5]) 50 | scan_times.append((field,scans,intervals)) 51 | return scan_times 52 | 53 | 54 | def main(): 55 | 56 | INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(('','idia')) 57 | WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.WSCLEAN_PATTERN,True) 58 | 59 | 60 | scan_pickle = sys.argv[1] 61 | scan_times = get_scan_times(scan_pickle) 62 | 63 | if not os.path.isdir('INTERVALS'): 64 | os.mkdir('INTERVALS') 65 | 66 | 67 | runfile = 'submit_interval_jobs.sh' 68 | 69 | f = open(runfile,'w') 70 | f.writelines(['#!/bin/bash\n']) 71 | 72 | for ss in scan_times: 73 | targetname = ss[0] 74 | scans = ss[1] 75 | intervals = ss[2] 76 | print('Target: '+targetname) 77 | print('Scans: '+str(scans)) 78 | print('Intervals: '+str(intervals)) 79 | for i in range(0,len(scans)): 80 | myms = glob.glob('*'+targetname+'*scan'+str(scans[i])+'.ms') 81 | if len(myms) == 1: 82 | myms = myms[0] 83 | if os.path.isdir(myms): 84 | opdir = 'INTERVALS/'+targetname+'_scan'+str(scans[i]) 85 | if not os.path.isdir(opdir): 86 | os.mkdir(opdir) 87 | 88 | imgname = opdir+'/img_'+myms+'_modelsub' 89 | code = 'intrvl'+str(scans[i]) 90 | 91 | syscall = 'singularity exec '+WSCLEAN_CONTAINER+' ' 92 | syscall += 'wsclean -intervals-out '+str(intervals[i])+' -interval 0 '+str(intervals[i])+' ' 93 | syscall += '-log-time -field 0 -no-dirty -make-psf -size 4680 4680 -scale 1.1asec -baseline-averaging 10 -no-update-model-required ' 94 | syscall += '-nwlayers 1 -niter 0 -name '+imgname+' ' 95 | syscall += '-weight briggs -0.3 -data-column CORRECTED_DATA -padding 1.2 -absmem 110 '+myms 96 | 97 | slurm_file = 'SCRIPTS/slurm_intervals_'+code+'.sh' 98 | log_file = 'LOGS/slurm_intervals_'+code+'.log' 99 | 100 | write_slurm(opfile=slurm_file,jobname=code,logfile=log_file,syscall=syscall) 101 | 102 | f.writelines(['sbatch '+slurm_file+'\n']) 103 | 104 | f.close() 105 | gen.make_executable(runfile) 106 | print('Wrote '+runfile+' script') 107 | 108 | 109 | if __name__ == "__main__": 110 | 111 | 112 | main() 113 | -------------------------------------------------------------------------------- /waterhole/setup_get_info.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | import glob 6 | import os.path as o 7 | import pickle 8 | import sys 9 | sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) 10 | 11 | 12 | from oxkat import generate_jobs as gen 13 | from oxkat import config as cfg 14 | 15 | 16 | def main(): 17 | 18 | USE_SINGULARITY = cfg.USE_SINGULARITY 19 | 20 | gen.preamble() 21 | print(gen.col()+'Get MS info') 22 | gen.print_spacer() 23 | 24 | # ------------------------------------------------------------------------------ 25 | # 26 | # Setup paths, required containers, infrastructure 27 | # 28 | # ------------------------------------------------------------------------------ 29 | 30 | 31 | gen.setup_dir(cfg.LOGS) 32 | gen.setup_dir(cfg.SCRIPTS) 33 | 34 | 35 | INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) 36 | if CONTAINER_PATH is not None: 37 | CONTAINER_RUNNER='singularity exec ' 38 | else: 39 | CONTAINER_RUNNER='' 40 | 41 | 42 | MAKEMASK_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.MAKEMASK_PATTERN,USE_SINGULARITY) 43 | 44 | 45 | # ------------------------------------------------------------------------------ 46 | # 47 | # MS info recipe definition 48 | # 49 | # ------------------------------------------------------------------------------ 50 | 51 | 52 | original_ms = glob.glob('*.ms')[0] 53 | code = gen.get_code(original_ms) 54 | 55 | steps = [] 56 | 57 | 58 | step = {} 59 | step['step'] = 0 60 | step['comment'] = 'Split and average master MS' 61 | step['dependency'] = None 62 | step['id'] = 'INFO_'+code 63 | syscall = CONTAINER_RUNNER+MAKEMASK_CONTAINER+' ' if USE_SINGULARITY else '' 64 | syscall += 'bash -c "' 65 | syscall += 'python '+cfg.TOOLS+'/ms_info.py '+original_ms+' && ' 66 | syscall += 'python '+cfg.TOOLS+'/find_sun.py '+original_ms 67 | syscall += '"' 68 | step['syscall'] = syscall 69 | steps.append(step) 70 | 71 | 72 | 73 | # ------------------------------------------------------------------------------ 74 | # 75 | # Write the run file and kill file based on the recipe 76 | # 77 | # ------------------------------------------------------------------------------ 78 | 79 | 80 | submit_file = 'submit_msinfo_job.sh' 81 | 82 | f = open(submit_file,'w') 83 | f.write('#!/usr/bin/env bash\n') 84 | f.write('export SINGULARITY_BINDPATH='+cfg.BINDPATH+'\n') 85 | 86 | id_list = [] 87 | 88 | for step in steps: 89 | 90 | step_id = step['id'] 91 | id_list.append(step_id) 92 | if step['dependency'] is not None: 93 | dependency = steps[step['dependency']]['id'] 94 | else: 95 | dependency = None 96 | syscall = step['syscall'] 97 | if 'slurm_config' in step.keys(): 98 | slurm_config = step['slurm_config'] 99 | else: 100 | slurm_config = cfg.SLURM_DEFAULTS 101 | if 'pbs_config' in step.keys(): 102 | pbs_config = step['pbs_config'] 103 | else: 104 | pbs_config = cfg.PBS_DEFAULTS 105 | comment = step['comment'] 106 | 107 | run_command = gen.job_handler(syscall = syscall, 108 | jobname = step_id, 109 | infrastructure = INFRASTRUCTURE, 110 | dependency = dependency, 111 | slurm_config = slurm_config, 112 | pbs_config = pbs_config) 113 | 114 | 115 | f.write('\n# '+comment+'\n') 116 | f.write(run_command) 117 | 118 | 119 | f.close() 120 | 121 | gen.make_executable(submit_file) 122 | 123 | gen.print_spacer() 124 | print(gen.col('Run file')+submit_file) 125 | gen.print_spacer() 126 | 127 | # ------------------------------------------------------------------------------ 128 | 129 | 130 | 131 | if __name__ == "__main__": 132 | 133 | 134 | main() -------------------------------------------------------------------------------- /tools/find_sun.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | import logging 6 | import numpy 7 | import sys 8 | from pyrap.tables import table 9 | from astropy.time import Time 10 | from astropy import units as u 11 | from astropy.coordinates import SkyCoord 12 | from astropy.coordinates import solar_system_ephemeris, EarthLocation, AltAz 13 | from astropy.coordinates import get_body_barycentric, get_body, get_moon 14 | 15 | 16 | def rad2deg(xx): 17 | return 180.0*xx/numpy.pi 18 | 19 | 20 | def get_fields(myms): 21 | fieldtab = table(myms+'/FIELD',ack=False) 22 | ids = fieldtab.getcol("SOURCE_ID") 23 | names = fieldtab.getcol("NAME") 24 | dirs = fieldtab.getcol("PHASE_DIR") 25 | return ids,names,dirs 26 | 27 | 28 | def match_field(ids,names,dirs,i): 29 | idx = numpy.where(ids == i)[0] 30 | radec = rad2deg(dirs[idx,0,:])[0] 31 | return names[i],radec[0],radec[1] 32 | 33 | 34 | def calcsep(ra0,dec0,ra1,dec1): 35 | c0 = SkyCoord(ra0*u.deg,dec0*u.deg,frame='fk5') 36 | c1 = SkyCoord(ra1*u.deg,dec1*u.deg,frame='fk5') 37 | sep = round(c0.separation(c1).deg,4) 38 | return sep 39 | 40 | 41 | def format_coords(ra0,dec0): 42 | c = SkyCoord(ra0*u.deg,dec0*u.deg,frame='fk5') 43 | hms = str(c.ra.to_string(u.hour)) 44 | dms = str(c.dec) 45 | return hms,dms 46 | 47 | 48 | def main(): 49 | 50 | # MeerKAT 51 | obs_lat = -30.71323598930457 52 | obs_lon = 21.443001467965008 53 | loc = EarthLocation.from_geodetic(obs_lat,obs_lon) #,obs_height,ellipsoid) 54 | 55 | 56 | myms = sys.argv[1].rstrip('/') 57 | maintab = table(myms,ack=False) 58 | scans = list(numpy.unique(maintab.getcol('SCAN_NUMBER'))) 59 | ids,names,dirs = get_fields(myms) 60 | 61 | logfile = 'sun_'+myms+'.log' 62 | logging.basicConfig(filename=logfile, level=logging.DEBUG, format='%(asctime)s | %(message)s', datefmt='%d/%m/%Y %H:%M:%S ') 63 | 64 | 65 | logging.info(myms+' | '+str(len(ids))+' fields | '+str(len(scans))+' scans') 66 | #header = 'Scan Field ID t[iso] t[s] t0[s] t1[s] int0 int1 Duration[m] N_int' 67 | header = '# t[iso] Scan Field Name SunRA[deg] SunDec[deg] SunRA[hms] SunDec[dms] SunSep[deg] SunAlt[deg] MoonRA[deg] MoonDec[deg] MoonRA[hms] MoonDec[dms] MoonSep[deg] MoonAlt[deg]' 68 | logging.info('-'*len(header)) 69 | logging.info(header) 70 | logging.info('-'*len(header)) 71 | for scan in scans: 72 | subtab = maintab.query(query='SCAN_NUMBER=='+str(scan)) 73 | field = numpy.unique(subtab.getcol('FIELD_ID'))[0] 74 | name,field_ra,field_dec = match_field(ids,names,dirs,field) 75 | field_hms,field_dms = format_coords(field_ra,field_dec) 76 | t_scan = numpy.mean(subtab.getcol('TIME')) 77 | t = Time(t_scan/86400.0,format='mjd') 78 | with solar_system_ephemeris.set('builtin'): 79 | sun = get_body('Sun', t, loc) 80 | moon = get_body('Moon', t, loc) 81 | sun_ra = sun.ra.value 82 | sun_dec = sun.dec.value 83 | sun_altaz = sun.transform_to(AltAz(obstime=t,location=loc)) 84 | sun_alt = sun_altaz.alt.value 85 | moon_ra = moon.ra.value 86 | moon_dec = moon.dec.value 87 | moon_altaz = moon.transform_to(AltAz(obstime=t,location=loc)) 88 | moon_alt = moon_altaz.alt.value 89 | sun_hms,sun_dms = format_coords(sun_ra,sun_dec) 90 | moon_hms,moon_dms = format_coords(moon_ra,moon_dec) 91 | delta_ra_sun = field_ra - sun_ra 92 | delta_dec_sun = field_dec - sun_dec 93 | delta_ra_moon = field_ra - moon_ra 94 | delta_dec_moon = field_dec - moon_dec 95 | sun_sep = calcsep(field_ra,field_dec,sun_ra,sun_dec) 96 | moon_sep = calcsep(field_ra,field_dec,moon_ra,moon_dec) 97 | # print field,name,sun_sep 98 | logging.info('%-28s %-5i %-5i %-12s %-12f %-12f %-16s %-16s %-12f %-12f %-12f %-12f %-16s %-16s %-12f %-12f' % 99 | (t.iso,scan,field,name,sun_ra,sun_dec,sun_hms,sun_dms,sun_sep,sun_alt,moon_ra,moon_dec,moon_hms,moon_dms,moon_sep,moon_alt)) 100 | 101 | logging.info('-'*len(header)) 102 | 103 | 104 | if __name__ == "__main__": 105 | 106 | main() -------------------------------------------------------------------------------- /tools/scan_times.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | from astropy.time import Time 6 | from pyrap.tables import table 7 | import logging 8 | import numpy 9 | import pickle 10 | import sys 11 | 12 | 13 | def main(): 14 | 15 | if len(sys.argv) == 1: 16 | print('Please specify a Measurement Set') 17 | sys.exit() 18 | elif len(sys.argv) > 1: 19 | myms = sys.argv[1].rstrip('/') 20 | logfile = 'scantimes_'+myms+'.log' 21 | 22 | if len(sys.argv) > 2: 23 | myscan = sys.argv[2] 24 | logfile = 'scantimes_'+myms+'_scan'+myscan+'.log' 25 | else: 26 | myscan = '' 27 | 28 | logging.basicConfig(filename=logfile, level=logging.DEBUG, format='%(asctime)s | %(message)s', datefmt='%d/%m/%Y %H:%M:%S ') 29 | stream = logging.StreamHandler() 30 | stream.setLevel(logging.DEBUG) 31 | streamformat = logging.Formatter('%(asctime)s | %(message)s', datefmt='%d/%m/%Y %H:%M:%S ') 32 | stream.setFormatter(streamformat) 33 | mylogger = logging.getLogger(__name__) 34 | mylogger.setLevel(logging.DEBUG) 35 | mylogger.addHandler(stream) 36 | 37 | tt = table(myms,ack=False) 38 | all_times = list(numpy.unique(tt.getcol('TIME'))) 39 | track_length = round(((all_times[-1] - all_times[0]) / 3600.0),3) 40 | scan_numbers = list(set(tt.getcol('SCAN_NUMBER'))) 41 | n_scans = len(scan_numbers) 42 | exposure = round(numpy.mean(tt.getcol('EXPOSURE')),4) 43 | field_tab = table(myms+'/FIELD',ack=False) 44 | field_names = field_tab.getcol('NAME') 45 | n_fields = len(field_names) 46 | field_tab.done() 47 | 48 | scan_list = [] 49 | pickle_name = 'scantimes_'+myms+'.p' 50 | 51 | mylogger.info(myms+' | '+str(n_fields)+' fields | '+str(n_scans)+' scans | track = '+str(track_length)+' h | t_int = '+str(exposure)+' s') 52 | 53 | if myscan == '': 54 | header = 'Scan Field ID t[iso] t[s] t0[s] t1[s] int0 int1 Duration[m] N_int' 55 | mylogger.info('-'*len(header)) 56 | mylogger.info(header) 57 | mylogger.info('-'*len(header)) 58 | for scan in scan_numbers: 59 | subtab = tt.query(query='SCAN_NUMBER=='+str(scan)) 60 | times = subtab.getcol('TIME') 61 | field_id = list(set(subtab.getcol("FIELD_ID")))[0] 62 | field_name = field_names[field_id] 63 | subtab.done() 64 | 65 | t0 = times[0] # start time for this scan 66 | t1 = times[-1] # end time for this scan 67 | int0 = all_times.index(t0) # start interval number in the full MS 68 | int1 = all_times.index(t1) # end interval number in the full MS 69 | dt = (t1-t0) # duration of this scan 70 | duration = round((dt/60.0),2) # duration in minutes 71 | n_int = int(dt / exposure) # number of integration times in this scan 72 | tc = t0+(dt/2.0) # central time of this scan 73 | t_iso = Time(tc/86400.0,format='mjd').iso # central time of this scan in ISO format 74 | 75 | mylogger.info('%-5i %-12s %-5s %-25s %-20f %-20f %-20f %-7s %-7s %-12s %-5i' % 76 | (scan,field_name,field_id,t_iso,tc,t0,t1,int0,int1,duration,n_int)) 77 | 78 | scan_list.append((scan,field_name,field_id,int0,int1,n_int)) 79 | 80 | mylogger.info('-'*len(header)) 81 | 82 | pickle.dump(scan_list,open(pickle_name,'wb')) 83 | 84 | else: 85 | 86 | subtab = tt.query(query='SCAN_NUMBER=='+str(myscan)) 87 | times = numpy.unique(subtab.getcol('TIME')) 88 | subtab.done() 89 | 90 | mylogger.info('Per-integration time details for scan '+myscan) 91 | header = 't[iso] t[s] int' 92 | mylogger.info('-'*len(header)) 93 | mylogger.info(header) 94 | mylogger.info('-'*len(header)) 95 | for t_i in times: 96 | t_iso = Time(t_i/86400.0,format='mjd').iso 97 | int_i = all_times.index(t_i) 98 | mylogger.info('%-25s %-20f %-7s' % 99 | (t_iso,t_i,int_i)) 100 | mylogger.info('-'*len(header)) 101 | 102 | tt.done() 103 | 104 | 105 | if __name__ == "__main__": 106 | 107 | main() 108 | -------------------------------------------------------------------------------- /tools/fft_amplitudes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | import imageio 6 | import numpy 7 | from argparse import ArgumentParser 8 | from astropy.io import fits 9 | import shutil 10 | 11 | 12 | def get_image(fitsfile): 13 | input_hdu = fits.open(fitsfile)[0] 14 | if len(input_hdu.data.shape) == 2: 15 | image = numpy.array(input_hdu.data[:,:]) 16 | elif len(input_hdu.data.shape) == 3: 17 | image = numpy.array(input_hdu.data[0,:,:]) 18 | else: 19 | image = numpy.array(input_hdu.data[0,0,:,:]) 20 | return image 21 | 22 | 23 | def flush_fits(image,fits_file): 24 | f = fits.open(fits_file,mode='update') 25 | input_hdu = f[0] 26 | if len(input_hdu.data.shape) == 2: 27 | input_hdu.data[:,:] =image 28 | elif len(input_hdu.data.shape) == 3: 29 | input_hdu.data[0,:,:] = image 30 | else: 31 | input_hdu.data[0,0,:,:] = image 32 | f.flush() 33 | 34 | 35 | def fft_image(image): 36 | fft_image = numpy.fft.fftshift(numpy.fft.fft2(image)) 37 | amplitudes = numpy.abs(fft_image) 38 | return amplitudes 39 | 40 | 41 | def hist_eq(image,nbins): 42 | hist,bins = numpy.histogram(image.flatten(), nbins, density = True) 43 | cdf = hist.cumsum() 44 | cdf = nbins * cdf / cdf[-1] 45 | image_eq = numpy.interp(image.flatten(), bins[:-1], cdf) 46 | image_eq = image_eq.reshape(image.shape) 47 | return image_eq 48 | 49 | 50 | def to_uint16(img): 51 | import numpy as np 52 | # replace NaNs and Infs with 0 53 | img = np.nan_to_num(img, nan=0.0, posinf=0.0, neginf=0.0) 54 | lo, hi = np.min(img), np.max(img) 55 | if hi > lo: 56 | img = (img - lo) / (hi - lo) 57 | else: 58 | img = np.zeros_like(img) 59 | return (img * 65535).astype(np.uint16) 60 | 61 | 62 | def apply_hanning(data): 63 | # Apply Hanning filter to reduce edge effects 64 | window = numpy.outer(numpy.hanning(data.shape[0]), numpy.hanning(data.shape[1])) 65 | windowed_data = data * window 66 | return windowed_data 67 | 68 | 69 | 70 | def main(): 71 | 72 | parser = ArgumentParser(description='FFT a FITS images and render the amplitudes to a PNG for inspection') 73 | 74 | parser.add_argument('infits', 75 | help = 'Input FITS image') 76 | parser.add_argument('-o','--output', dest = 'pngname', 77 | help = 'Output PNG name (default is based on input FITS name)', default = '') 78 | parser.add_argument('--noeq', dest = 'noeq', 79 | help = 'Disable histogram equalisation', default = False, action = 'store_true') 80 | parser.add_argument('-n','--nbins', dest = 'nbins', 81 | help = 'Number of bins for histogram equalisation (default = 1024)', default = 1024) 82 | parser.add_argument('--nofits', dest = 'nofits', 83 | help = 'Do not save amplitudes as FITS image', default = False, action = 'store_true') 84 | parser.add_argument('--nohanning', dest = 'nohanning', 85 | help = 'Do not apply Hanning filter', default = False, action = 'store_true') 86 | 87 | 88 | options = parser.parse_args() 89 | infits = options.infits 90 | pngname = options.pngname 91 | noeq = options.noeq 92 | nbins = int(options.nbins) 93 | nofits = options.nofits 94 | nohanning = options.nohanning 95 | 96 | img = get_image(infits) 97 | if not nohanning: 98 | img = apply_hanning(img) 99 | fftimg = fft_image(img) 100 | 101 | if not nofits: 102 | fftfits = infits.replace('.fits','_FFT_amplitudes.fits') 103 | shutil.copyfile(infits,fftfits) 104 | flush_fits(fftimg,fftfits) 105 | 106 | # if not noeq: 107 | # fftimg = hist_eq(fftimg,nbins) 108 | 109 | # if pngname == '': 110 | # pngname = infits+'_FFT_amplitudes.png' 111 | 112 | # imageio.imwrite(pngname,fftimg) 113 | 114 | 115 | # Optional histogram equalization (still float) 116 | if not noeq: 117 | fftimg = hist_eq(fftimg, nbins) # returns floats 118 | 119 | # Nice for dynamic range in FFTs 120 | viz = numpy.log1p(fftimg) # still float 121 | 122 | # Map to integer type PNG understands 123 | viz16 = to_uint16(viz) # or to_uint8(viz) 124 | 125 | if pngname == '': 126 | pngname = infits + '_FFT_amplitudes.png' 127 | 128 | imageio.imwrite(pngname, viz16) # now OK: uint16 grayscale PNG 129 | 130 | 131 | if __name__ == "__main__": 132 | 133 | 134 | main() 135 | -------------------------------------------------------------------------------- /data/cubical/2GC_delaycal.parset: -------------------------------------------------------------------------------- 1 | [data] 2 | _Help = Visibility data options 3 | ms = 4 | column = DATA 5 | time-chunk = 36 6 | freq-chunk = 1024 7 | rebin-time = 1 8 | rebin-freq = 1 9 | chunk-by = SCAN_NUMBER 10 | chunk-by-jump = 0 11 | single-chunk = 12 | single-tile = -1 13 | normalize = 0 14 | 15 | [sel] 16 | _Help = Data selection options 17 | field = 0 18 | ddid = None 19 | taql = 20 | chan = 21 | diag = False 22 | 23 | [out] 24 | _Help = Options for output products 25 | dir = delaycal 26 | name = delaycal.cc-out/delaycal 27 | overwrite = False 28 | backup = 1 29 | mode = sc 30 | apply-solver-flags = True 31 | column = CORRECTED_DATA 32 | derotate = None 33 | model-column = 34 | weight-column = 35 | reinit-column = False 36 | subtract-model = 0 37 | subtract-dirs = 0 38 | correct-dir = 0 39 | plots = 1 40 | casa-gaintables = True 41 | 42 | [model] 43 | _Help = Calibration model options 44 | list = MODEL_DATA 45 | ddes = auto 46 | beam-pattern = None 47 | beam-l-axis = None 48 | beam-m-axis = None 49 | feed-rotate = auto 50 | pa-rotate = False 51 | 52 | [montblanc] 53 | _Help = Montblanc simulation options 54 | device-type = CPU 55 | dtype = double 56 | mem-budget = 1024 57 | verbosity = WARNING 58 | threads = 0 59 | pa-rotate = None 60 | 61 | [weight] 62 | _Help = Weighting options 63 | column = WEIGHT_SPECTRUM 64 | fill-offdiag = False 65 | legacy-v1-2 = False 66 | 67 | [flags] 68 | _Help = General flagging options 69 | apply = -cubical 70 | auto-init = legacy 71 | save = cubical 72 | save-legacy = auto 73 | reinit-bitflags = False 74 | warn-thr = 0.3 75 | see-no-evil = 0 76 | 77 | [degridding] 78 | _Help = Options for the degridder. Only in use when predicting from DicoModels using DDFacet 79 | OverS = 11 80 | Support = 7 81 | Nw = 100 82 | wmax = 0.0 83 | Padding = 1.7 84 | NDegridBand = 16 85 | MaxFacetSize = 0.25 86 | MinNFacetPerAxis = 1 87 | NProcess = 8 88 | BeamModel = None 89 | NBand = 0 90 | FITSFile = beam_$(corr)_$(reim).fits 91 | FITSFeed = None 92 | FITSFeedSwap = False 93 | DtBeamMin = 5.0 94 | FITSParAngleIncDeg = 5.0 95 | FITSLAxis = -X 96 | FITSMAxis = Y 97 | FITSVerbosity = 0 98 | FeedAngle = 0.0 99 | FlipVisibilityHands = 0 100 | 101 | [postmortem] 102 | _Help = Options for "postmortem" flagging based on solution statistics 103 | enable = False 104 | tf-chisq-median = 1.2 105 | tf-np-median = 0.5 106 | time-density = 0.5 107 | chan-density = 0.5 108 | ddid-density = 0.5 109 | 110 | [madmax] 111 | _Help = Options for the "Mad Max" flagger 112 | enable = 0 113 | residuals = 0 114 | estimate = corr 115 | diag = True 116 | offdiag = True 117 | threshold = 10 118 | global-threshold = 12 119 | plot = 1 120 | plot-frac-above = 0.01 121 | plot-bl = 122 | flag-ant = 0 123 | flag-ant-thr = 5 124 | 125 | [sol] 126 | _Help = Solution options which apply at the solver level 127 | jones = K 128 | precision = 32 129 | delta-g = 1e-06 130 | delta-chi = 1e-06 131 | chi-int = 5 132 | last-rites = True 133 | stall-quorum = 0.99 134 | term-iters = 50,50 135 | flag-divergence = 0 136 | min-bl = 100.0 137 | max-bl = 0 138 | subset = 139 | 140 | [bbc] 141 | _Help = Options for baseline-based corrections (a.k.a. BBCs, a.k.a. interferometer gains). 142 | load-from = 143 | compute-2x2 = False 144 | apply-2x2 = False 145 | save-to = {out[name]}-BBC-field_{sel[field]}-ddid_{sel[ddid]}.parmdb 146 | per-chan = True 147 | plot = True 148 | 149 | [dist] 150 | _Help = Parallelization and distribution options 151 | ncpu = 6 152 | nworker = 0 153 | nthread = 0 154 | max-chunks = 2 155 | min-chunks = 0 156 | pin = 0 157 | pin-io = False 158 | pin-main = io 159 | safe = True 160 | 161 | [log] 162 | _Help = Options related to logging 163 | memory = True 164 | stats = chi2:.3f 165 | stats-warn = chi2:10 166 | boring = True 167 | append = False 168 | verbose = 0 169 | file-verbose = None 170 | 171 | [debug] 172 | _Help = Debugging options for the discerning masochist 173 | pdb = False 174 | panic-amplitude = 0.0 175 | stop-before-solver = False 176 | escalate-warnings = 0 177 | 178 | [misc] 179 | _Help = Miscellaneous options 180 | random-seed = None 181 | parset-version = 0.1 182 | 183 | [k] 184 | _Help = Options for K-Jones term 185 | label = K 186 | solvable = 1 187 | type = f-slope 188 | load-from = 189 | xfer-from = 190 | save-to = delaycal.parmdb 191 | dd-term = 0 192 | fix-dirs = 193 | update-type = full 194 | time-int = 4 195 | freq-int = 1024 196 | max-prior-error = 0.0 197 | max-post-error = 0.0 198 | clip-low = 0.1 199 | clip-high = 10 200 | clip-after = 5 201 | max-iter = 50 202 | epsilon = 1e-06 203 | delta-chi = 1e-06 204 | conv-quorum = 0.99 205 | ref-ant = None 206 | prop-flags = default 207 | estimate-pzd = False 208 | diag-only = 1 209 | offdiag-only = 0 210 | robust-cov = compute 211 | robust-npol = 2 212 | robust-int = 1 213 | robust-save-weights = 0 214 | _Templated = True 215 | 216 | 217 | -------------------------------------------------------------------------------- /data/cubical/2GC_delaycal_solve_only.parset: -------------------------------------------------------------------------------- 1 | [data] 2 | _Help = Visibility data options 3 | ms = 4 | column = DATA 5 | time-chunk = 36 6 | freq-chunk = 1024 7 | rebin-time = 1 8 | rebin-freq = 1 9 | chunk-by = SCAN_NUMBER 10 | chunk-by-jump = 0 11 | single-chunk = 12 | single-tile = -1 13 | normalize = 0 14 | 15 | [sel] 16 | _Help = Data selection options 17 | field = 0 18 | ddid = None 19 | taql = 20 | chan = 21 | diag = False 22 | 23 | [out] 24 | _Help = Options for output products 25 | dir = delaycal 26 | name = delaycal.cc-out/delaycal 27 | overwrite = False 28 | backup = 1 29 | mode = so 30 | apply-solver-flags = True 31 | column = CORRECTED_DATA 32 | derotate = None 33 | model-column = 34 | weight-column = 35 | reinit-column = False 36 | subtract-model = 0 37 | subtract-dirs = 0 38 | correct-dir = 0 39 | plots = 1 40 | casa-gaintables = True 41 | 42 | [model] 43 | _Help = Calibration model options 44 | list = MODEL_DATA 45 | ddes = auto 46 | beam-pattern = None 47 | beam-l-axis = None 48 | beam-m-axis = None 49 | feed-rotate = auto 50 | pa-rotate = False 51 | 52 | [montblanc] 53 | _Help = Montblanc simulation options 54 | device-type = CPU 55 | dtype = double 56 | mem-budget = 1024 57 | verbosity = WARNING 58 | threads = 0 59 | pa-rotate = None 60 | 61 | [weight] 62 | _Help = Weighting options 63 | column = WEIGHT_SPECTRUM 64 | fill-offdiag = False 65 | legacy-v1-2 = False 66 | 67 | [flags] 68 | _Help = General flagging options 69 | apply = -cubical 70 | auto-init = legacy 71 | save = cubical 72 | save-legacy = auto 73 | reinit-bitflags = False 74 | warn-thr = 0.3 75 | see-no-evil = 0 76 | 77 | [degridding] 78 | _Help = Options for the degridder. Only in use when predicting from DicoModels using DDFacet 79 | OverS = 11 80 | Support = 7 81 | Nw = 100 82 | wmax = 0.0 83 | Padding = 1.7 84 | NDegridBand = 16 85 | MaxFacetSize = 0.25 86 | MinNFacetPerAxis = 1 87 | NProcess = 8 88 | BeamModel = None 89 | NBand = 0 90 | FITSFile = beam_$(corr)_$(reim).fits 91 | FITSFeed = None 92 | FITSFeedSwap = False 93 | DtBeamMin = 5.0 94 | FITSParAngleIncDeg = 5.0 95 | FITSLAxis = -X 96 | FITSMAxis = Y 97 | FITSVerbosity = 0 98 | FeedAngle = 0.0 99 | FlipVisibilityHands = 0 100 | 101 | [postmortem] 102 | _Help = Options for "postmortem" flagging based on solution statistics 103 | enable = False 104 | tf-chisq-median = 1.2 105 | tf-np-median = 0.5 106 | time-density = 0.5 107 | chan-density = 0.5 108 | ddid-density = 0.5 109 | 110 | [madmax] 111 | _Help = Options for the "Mad Max" flagger 112 | enable = 0 113 | residuals = 0 114 | estimate = corr 115 | diag = True 116 | offdiag = True 117 | threshold = 10 118 | global-threshold = 12 119 | plot = 1 120 | plot-frac-above = 0.01 121 | plot-bl = 122 | flag-ant = 0 123 | flag-ant-thr = 5 124 | 125 | [sol] 126 | _Help = Solution options which apply at the solver level 127 | jones = K 128 | precision = 32 129 | delta-g = 1e-06 130 | delta-chi = 1e-06 131 | chi-int = 5 132 | last-rites = True 133 | stall-quorum = 0.99 134 | term-iters = 50,50 135 | flag-divergence = 0 136 | min-bl = 100.0 137 | max-bl = 0 138 | subset = 139 | 140 | [bbc] 141 | _Help = Options for baseline-based corrections (a.k.a. BBCs, a.k.a. interferometer gains). 142 | load-from = 143 | compute-2x2 = False 144 | apply-2x2 = False 145 | save-to = {out[name]}-BBC-field_{sel[field]}-ddid_{sel[ddid]}.parmdb 146 | per-chan = True 147 | plot = True 148 | 149 | [dist] 150 | _Help = Parallelization and distribution options 151 | ncpu = 6 152 | nworker = 0 153 | nthread = 0 154 | max-chunks = 2 155 | min-chunks = 0 156 | pin = 0 157 | pin-io = False 158 | pin-main = io 159 | safe = True 160 | 161 | [log] 162 | _Help = Options related to logging 163 | memory = True 164 | stats = chi2:.3f 165 | stats-warn = chi2:10 166 | boring = True 167 | append = False 168 | verbose = 0 169 | file-verbose = None 170 | 171 | [debug] 172 | _Help = Debugging options for the discerning masochist 173 | pdb = False 174 | panic-amplitude = 0.0 175 | stop-before-solver = False 176 | escalate-warnings = 0 177 | 178 | [misc] 179 | _Help = Miscellaneous options 180 | random-seed = None 181 | parset-version = 0.1 182 | 183 | [k] 184 | _Help = Options for K-Jones term 185 | label = K 186 | solvable = 1 187 | type = f-slope 188 | load-from = 189 | xfer-from = 190 | save-to = delaycal.parmdb 191 | dd-term = 0 192 | fix-dirs = 193 | update-type = full 194 | time-int = 4 195 | freq-int = 1024 196 | max-prior-error = 0.0 197 | max-post-error = 0.0 198 | clip-low = 0.1 199 | clip-high = 10 200 | clip-after = 5 201 | max-iter = 50 202 | epsilon = 1e-06 203 | delta-chi = 1e-06 204 | conv-quorum = 0.99 205 | ref-ant = None 206 | prop-flags = default 207 | estimate-pzd = False 208 | diag-only = 1 209 | offdiag-only = 0 210 | robust-cov = compute 211 | robust-npol = 2 212 | robust-int = 1 213 | robust-save-weights = 0 214 | _Templated = True 215 | 216 | 217 | -------------------------------------------------------------------------------- /data/cubical/2GC_phasecal.parset: -------------------------------------------------------------------------------- 1 | [data] 2 | _Help = Visibility data options 3 | ms = 4 | column = DATA 5 | time-chunk = 96 6 | freq-chunk = 0 7 | rebin-time = 1 8 | rebin-freq = 1 9 | chunk-by = SCAN_NUMBER 10 | chunk-by-jump = 0 11 | single-chunk = 12 | single-tile = -1 13 | normalize = 0 14 | 15 | [sel] 16 | _Help = Data selection options 17 | field = 0 18 | ddid = None 19 | taql = 20 | chan = 21 | diag = False 22 | 23 | [out] 24 | _Help = Options for output products 25 | dir = selfcal 26 | name = selfcal.cc-out/selfcal 27 | overwrite = False 28 | backup = 1 29 | mode = sc 30 | apply-solver-flags = True 31 | column = CORRECTED_DATA 32 | derotate = None 33 | model-column = 34 | weight-column = 35 | reinit-column = False 36 | subtract-model = 0 37 | subtract-dirs = 0 38 | correct-dir = 0 39 | plots = 1 40 | casa-gaintables = True 41 | 42 | [model] 43 | _Help = Calibration model options 44 | list = MODEL_DATA 45 | ddes = auto 46 | beam-pattern = None 47 | beam-l-axis = None 48 | beam-m-axis = None 49 | feed-rotate = auto 50 | pa-rotate = False 51 | 52 | [montblanc] 53 | _Help = Montblanc simulation options 54 | device-type = CPU 55 | dtype = double 56 | mem-budget = 1024 57 | verbosity = WARNING 58 | threads = 0 59 | pa-rotate = None 60 | 61 | [weight] 62 | _Help = Weighting options 63 | column = WEIGHT_SPECTRUM 64 | fill-offdiag = False 65 | legacy-v1-2 = False 66 | 67 | [flags] 68 | _Help = General flagging options 69 | apply = -cubical 70 | auto-init = legacy 71 | save = cubical 72 | save-legacy = auto 73 | reinit-bitflags = False 74 | warn-thr = 0.3 75 | see-no-evil = 0 76 | 77 | [degridding] 78 | _Help = Options for the degridder. Only in use when predicting from DicoModels using DDFacet 79 | OverS = 11 80 | Support = 7 81 | Nw = 100 82 | wmax = 0.0 83 | Padding = 1.7 84 | NDegridBand = 16 85 | MaxFacetSize = 0.25 86 | MinNFacetPerAxis = 1 87 | NProcess = 8 88 | BeamModel = None 89 | NBand = 0 90 | FITSFile = beam_$(corr)_$(reim).fits 91 | FITSFeed = None 92 | FITSFeedSwap = False 93 | DtBeamMin = 5.0 94 | FITSParAngleIncDeg = 5.0 95 | FITSLAxis = -X 96 | FITSMAxis = Y 97 | FITSVerbosity = 0 98 | FeedAngle = 0.0 99 | FlipVisibilityHands = 0 100 | 101 | [postmortem] 102 | _Help = Options for "postmortem" flagging based on solution statistics 103 | enable = False 104 | tf-chisq-median = 1.2 105 | tf-np-median = 0.5 106 | time-density = 0.5 107 | chan-density = 0.5 108 | ddid-density = 0.5 109 | 110 | [madmax] 111 | _Help = Options for the "Mad Max" flagger 112 | enable = 0 113 | residuals = 0 114 | estimate = corr 115 | diag = True 116 | offdiag = True 117 | threshold = 10 118 | global-threshold = 12 119 | plot = 1 120 | plot-frac-above = 0.01 121 | plot-bl = 122 | flag-ant = 0 123 | flag-ant-thr = 5 124 | 125 | [sol] 126 | _Help = Solution options which apply at the solver level 127 | jones = K,G 128 | precision = 32 129 | delta-g = 1e-06 130 | delta-chi = 1e-06 131 | chi-int = 5 132 | last-rites = True 133 | stall-quorum = 0.99 134 | term-iters = 50,50 135 | flag-divergence = 0 136 | min-bl = 100.0 137 | max-bl = 0 138 | subset = 139 | 140 | [bbc] 141 | _Help = Options for baseline-based corrections (a.k.a. BBCs, a.k.a. interferometer gains). 142 | load-from = 143 | compute-2x2 = False 144 | apply-2x2 = False 145 | save-to = {out[name]}-BBC-field_{sel[field]}-ddid_{sel[ddid]}.parmdb 146 | per-chan = True 147 | plot = True 148 | 149 | [dist] 150 | _Help = Parallelization and distribution options 151 | ncpu = 8 152 | nworker = 0 153 | nthread = 0 154 | max-chunks = 2 155 | min-chunks = 0 156 | pin = 0 157 | pin-io = False 158 | pin-main = io 159 | safe = True 160 | 161 | [log] 162 | _Help = Options related to logging 163 | memory = True 164 | stats = chi2:.3f 165 | stats-warn = chi2:10 166 | boring = True 167 | append = False 168 | verbose = 0 169 | file-verbose = None 170 | 171 | [debug] 172 | _Help = Debugging options for the discerning masochist 173 | pdb = False 174 | panic-amplitude = 0.0 175 | stop-before-solver = False 176 | escalate-warnings = 0 177 | 178 | [misc] 179 | _Help = Miscellaneous options 180 | random-seed = None 181 | parset-version = 0.1 182 | 183 | 184 | [g] 185 | _Help = Options for G-Jones term 186 | label = G 187 | solvable = 1 188 | type = complex-2x2 189 | delay-estimate-pad-factor = 8 190 | load-from = 191 | xfer-from = 192 | save-to = {out[name]}-{JONES}-field_{sel[field]}-ddid_{sel[ddid]}.parmdb 193 | dd-term = 0 194 | fix-dirs = 195 | update-type = full 196 | estimate-pzd = False 197 | time-int = 16 198 | freq-int = 0 199 | max-prior-error = 0.3 200 | max-post-error = 0.3 201 | low-snr-warn = 75 202 | high-gain-var-warn = 30 203 | clip-low = 0.1 204 | clip-high = 10000 205 | clip-after = 5 206 | max-iter = 50 207 | pin-slope-iters = 0 208 | epsilon = 1e-06 209 | delta-chi = 1e-06 210 | conv-quorum = 0.99 211 | ref-ant = None 212 | prop-flags = default 213 | diag-only = 0 214 | offdiag-only = False 215 | robust-cov = compute 216 | robust-scale = 1 217 | robust-npol = 2 218 | robust-int = 1 219 | robust-save-weights = 0 220 | estimate-delays = False 221 | _Templated = True 222 | -------------------------------------------------------------------------------- /setups/0_GET_INFO.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | import glob 6 | import os.path as o 7 | import sys 8 | sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) 9 | 10 | 11 | from oxkat import generate_jobs as gen 12 | from oxkat import config as cfg 13 | 14 | def main(): 15 | 16 | USE_SINGULARITY = cfg.USE_SINGULARITY 17 | 18 | gen.preamble() 19 | print(gen.col()+'Setting up job to examine master MS') 20 | gen.print_spacer() 21 | 22 | # ------------------------------------------------------------------------------ 23 | # 24 | # Setup paths, required containers, infrastructure 25 | # 26 | # ------------------------------------------------------------------------------ 27 | 28 | 29 | gen.setup_dir(cfg.LOGS) 30 | gen.setup_dir(cfg.SCRIPTS) 31 | gen.setup_dir(cfg.GAINTABLES) 32 | 33 | 34 | INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) 35 | if CONTAINER_PATH is not None: 36 | CONTAINER_RUNNER='singularity exec ' 37 | else: 38 | CONTAINER_RUNNER='' 39 | 40 | 41 | ASTROPY_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.ASTROPY_PATTERN,USE_SINGULARITY) 42 | OWLCAT_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.OWLCAT_PATTERN,USE_SINGULARITY) 43 | 44 | 45 | # ------------------------------------------------------------------------------ 46 | # 47 | # 1GC recipe definition 48 | # 49 | # ------------------------------------------------------------------------------ 50 | 51 | 52 | myms = glob.glob('*.ms') 53 | if len(myms) == 0: 54 | print(gen.col()+'No MS found in current directory, please check') 55 | gen.print_spacer() 56 | sys.exit() 57 | 58 | myms = myms[0] 59 | code = gen.get_code(myms) 60 | 61 | steps = [] 62 | 63 | 64 | step = {} 65 | step['step'] = 0 66 | step['comment'] = 'Run setup script to generate project_info json file' 67 | step['dependency'] = None 68 | step['id'] = 'INFO_'+code 69 | syscall = CONTAINER_RUNNER+OWLCAT_CONTAINER+' ' if USE_SINGULARITY else '' 70 | syscall += ' python3 '+cfg.TOOLS+'/ms_info.py '+myms+'\n' 71 | syscall += CONTAINER_RUNNER+OWLCAT_CONTAINER+' ' if USE_SINGULARITY else '' 72 | syscall += ' python3 '+cfg.TOOLS+'/scan_times.py '+myms+'\n' 73 | syscall += CONTAINER_RUNNER+ASTROPY_CONTAINER+' ' if USE_SINGULARITY else '' 74 | syscall += ' python3 '+cfg.TOOLS+'/find_sun.py '+myms+'\n' 75 | syscall += CONTAINER_RUNNER+OWLCAT_CONTAINER+' ' if USE_SINGULARITY else '' 76 | syscall += ' python3 '+cfg.OXKAT+'/1GC_00_setup.py '+myms 77 | step['syscall'] = syscall 78 | steps.append(step) 79 | 80 | 81 | 82 | # ------------------------------------------------------------------------------ 83 | # 84 | # Write the run file and kill file based on the recipe 85 | # 86 | # ------------------------------------------------------------------------------ 87 | 88 | 89 | submit_file = 'submit_info_job.sh' 90 | kill_file = cfg.SCRIPTS+'/kill_info_job.sh' 91 | 92 | f = open(submit_file,'w') 93 | f.write('#!/usr/bin/env bash\n') 94 | f.write('export SINGULARITY_BINDPATH='+cfg.BINDPATH+'\n') 95 | 96 | id_list = [] 97 | 98 | for step in steps: 99 | 100 | step_id = step['id'] 101 | id_list.append(step_id) 102 | if step['dependency'] is not None: 103 | dependency = steps[step['dependency']]['id'] 104 | else: 105 | dependency = None 106 | syscall = step['syscall'] 107 | if 'slurm_config' in step.keys(): 108 | slurm_config = step['slurm_config'] 109 | else: 110 | slurm_config = cfg.SLURM_DEFAULTS 111 | if 'pbs_config' in step.keys(): 112 | pbs_config = step['pbs_config'] 113 | else: 114 | pbs_config = cfg.PBS_DEFAULTS 115 | comment = step['comment'] 116 | 117 | run_command = gen.job_handler(syscall = syscall, 118 | jobname = step_id, 119 | infrastructure = INFRASTRUCTURE, 120 | dependency = dependency, 121 | slurm_config = slurm_config, 122 | pbs_config = pbs_config) 123 | 124 | 125 | f.write('\n# '+comment+'\n') 126 | f.write(run_command) 127 | 128 | 129 | if INFRASTRUCTURE == 'idia' or INFRASTRUCTURE == 'hippo': 130 | kill = '\necho "scancel "$'+'" "$'.join(id_list)+' > '+kill_file+'\n' 131 | f.write(kill) 132 | elif INFRASTRUCTURE == 'chpc': 133 | kill = '\necho "qdel "$'+'" "$'.join(id_list)+' > '+kill_file+'\n' 134 | f.write(kill) 135 | 136 | f.close() 137 | 138 | gen.make_executable(submit_file) 139 | 140 | gen.print_spacer() 141 | print(gen.col('Run file')+submit_file) 142 | gen.print_spacer() 143 | 144 | # ------------------------------------------------------------------------------ 145 | 146 | 147 | 148 | if __name__ == "__main__": 149 | 150 | 151 | main() -------------------------------------------------------------------------------- /tools/restore_model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | import glob 6 | import logging 7 | import numpy 8 | import os 9 | import random 10 | import scipy.signal 11 | import shutil 12 | import string 13 | import sys 14 | 15 | from astropy.io import fits 16 | from astropy.convolution import convolve,Gaussian2DKernel 17 | from itertools import repeat 18 | from multiprocessing import Pool 19 | 20 | 21 | def get_image(fitsfile): 22 | input_hdu = fits.open(fitsfile)[0] 23 | if len(input_hdu.data.shape) == 2: 24 | image = numpy.array(input_hdu.data[:,:]) 25 | elif len(input_hdu.data.shape) == 3: 26 | image = numpy.array(input_hdu.data[0,:,:]) 27 | elif len(input_hdu.data.shape) == 4: 28 | image = numpy.array(input_hdu.data[0,0,:,:]) 29 | else: 30 | image = numpy.array(input_hdu.data[0,0,0,:,:]) 31 | return image 32 | 33 | 34 | def flush_fits(newimage,fitsfile): 35 | f = fits.open(fitsfile,mode='update') 36 | input_hdu = f[0] 37 | if len(input_hdu.data.shape) == 2: 38 | input_hdu.data[:,:] = newimage 39 | elif len(input_hdu.data.shape) == 3: 40 | input_hdu.data[0,:,:] = newimage_ 41 | elif len(input_hdu.data.shape) == 4: 42 | input_hdu.data[0,0,:,:] = newimage 43 | else: 44 | input_hdu.data[0,0,0,:,:] = newimage 45 | f.flush() 46 | 47 | 48 | def deg2rad(xx): 49 | return numpy.pi*xx/180.0 50 | 51 | 52 | def get_header(fitsfile): 53 | input_hdu = fits.open(fitsfile)[0] 54 | hdr = input_hdu.header 55 | bmaj = hdr.get('BMAJ') 56 | bmin = hdr.get('BMIN') 57 | bpa = hdr.get('BPA') 58 | pixscale = hdr.get('CDELT2') 59 | return bmaj,bmin,bpa,pixscale 60 | 61 | 62 | def beam_header(fitsfile,bmaj,bmin,bpa): 63 | outhdu = fits.open(fitsfile,mode='update') 64 | outhdr = outhdu[0].header 65 | outhdr.set('BMAJ',bmaj,after='BUNIT') 66 | outhdr.set('BMIN',bmin,after='BMAJ') 67 | outhdr.set('BPA',bpa,after='BMIN') 68 | outhdr.remove('HISTORY') 69 | outhdu.flush() 70 | 71 | 72 | def convolve_fits(residual_fits,model_image,proc_id): 73 | 74 | proc_id = str(proc_id) 75 | 76 | # Set up FITS files 77 | psf_fits = residual_fits.replace('image','psf') 78 | restored_fits = residual_fits.replace('image','image-restored') 79 | shutil.copyfile(residual_fits,restored_fits) 80 | 81 | 82 | # Get the fitted beam 83 | bmaj,bmin,bpa,pixscale = get_header(psf_fits) 84 | 85 | logging.info('(File '+proc_id+') Residual : '+residual_fits) 86 | logging.info('(File '+proc_id+') PSF : '+psf_fits) 87 | logging.info('(File '+proc_id+') Restored : '+restored_fits) 88 | logging.info('(File '+proc_id+') Fitted bmaj : '+str(bmaj*3600)) 89 | logging.info('(File '+proc_id+') Fitted bmin : '+str(bmin*3600)) 90 | logging.info('(File '+proc_id+') Fitted bpa : '+str(bpa)) 91 | 92 | # Create restoring beam image 93 | xstd = bmin/(2.3548*pixscale) 94 | ystd = bmaj/(2.3548*pixscale) 95 | theta = deg2rad(bpa) 96 | restoring = Gaussian2DKernel(x_stddev=xstd,y_stddev=ystd,theta=theta,x_size=cropsize,y_size=cropsize,mode='center') 97 | restoring_beam_image = restoring.array 98 | restoring_beam_image = restoring_beam_image / numpy.max(restoring_beam_image) 99 | 100 | # Convolve model with restoring beam 101 | model_conv_image = scipy.signal.fftconvolve(model_image, restoring_beam_image, mode='same') 102 | 103 | # Open residual image and add convolved model 104 | residual_image = get_image(residual_fits) 105 | restored_image = residual_image + model_conv_image 106 | 107 | # Flush restored FITS file and fix the header 108 | flush_fits(restored_image,restored_fits) 109 | beam_header(restored_fits,bmaj,bmin,bpa) 110 | 111 | 112 | if __name__ == '__main__': 113 | 114 | if len(sys.argv) == 1: 115 | print('Please specify model FITS image') 116 | sys.exit() 117 | 118 | logfile = 'restore_model.log' 119 | logging.basicConfig(filename=logfile, level=logging.DEBUG, format='%(asctime)s | %(message)s', datefmt='%d/%m/%Y %H:%M:%S ') 120 | 121 | model_fits = sys.argv[1] 122 | 123 | j = 1 # Number of parallel convolutions 124 | cropsize = 51 # Size of kernel thumbnail 125 | fits_list = sorted(glob.glob('*image.fits')) # List of residuals 126 | ids = numpy.arange(0,len(fits_list)) 127 | 128 | # Get the image size from first image in list and create matched model image 129 | img0 = get_image(fits_list[0]) 130 | nx,ny = numpy.shape(img0) 131 | if nx != ny: 132 | print('Only square images are supported at present') 133 | sys.exit() 134 | tmpmodel_fits = 'temp_model_'+''.join(random.choices(string.ascii_uppercase + string.digits, k=16))+'.fits' 135 | os.system('fitstool.py -z '+str(nx)+' -o '+tmpmodel_fits+' '+model_fits) 136 | model_image = get_image(tmpmodel_fits) 137 | 138 | for i in range(0,len(fits_list)): 139 | convolve_fits(fits_list[i],model_image,ids[i]) 140 | 141 | # pool = Pool(processes=j) 142 | # pool.starmap(convolve_fits,zip(fits_list,repeat(model_image),ids)) 143 | 144 | -------------------------------------------------------------------------------- /data/cubical/2GC_apcal.parset: -------------------------------------------------------------------------------- 1 | [data] 2 | _Help = Visibility data options 3 | ms = 4 | column = DATA 5 | time-chunk = 48 6 | freq-chunk = 1024 7 | rebin-time = 1 8 | rebin-freq = 1 9 | chunk-by = SCAN_NUMBER 10 | chunk-by-jump = 0 11 | single-chunk = 12 | single-tile = -1 13 | normalize = 0 14 | 15 | [sel] 16 | _Help = Data selection options 17 | field = 0 18 | ddid = None 19 | taql = 20 | chan = 21 | diag = False 22 | 23 | [out] 24 | _Help = Options for output products 25 | dir = selfcal 26 | name = selfcal.cc-out/selfcal 27 | overwrite = False 28 | backup = 1 29 | mode = sc 30 | apply-solver-flags = True 31 | column = CORRECTED_DATA 32 | derotate = None 33 | model-column = 34 | weight-column = 35 | reinit-column = False 36 | subtract-model = 0 37 | subtract-dirs = 0 38 | correct-dir = 0 39 | plots = 1 40 | casa-gaintables = True 41 | 42 | [model] 43 | _Help = Calibration model options 44 | list = MODEL_DATA 45 | ddes = auto 46 | beam-pattern = None 47 | beam-l-axis = None 48 | beam-m-axis = None 49 | feed-rotate = auto 50 | pa-rotate = False 51 | 52 | [montblanc] 53 | _Help = Montblanc simulation options 54 | device-type = CPU 55 | dtype = double 56 | mem-budget = 1024 57 | verbosity = WARNING 58 | threads = 0 59 | pa-rotate = None 60 | 61 | [weight] 62 | _Help = Weighting options 63 | column = WEIGHT_SPECTRUM 64 | fill-offdiag = False 65 | legacy-v1-2 = False 66 | 67 | [flags] 68 | _Help = General flagging options 69 | apply = -cubical 70 | auto-init = legacy 71 | save = cubical 72 | save-legacy = auto 73 | reinit-bitflags = False 74 | warn-thr = 0.3 75 | see-no-evil = 0 76 | 77 | [degridding] 78 | _Help = Options for the degridder. Only in use when predicting from DicoModels using DDFacet 79 | OverS = 11 80 | Support = 7 81 | Nw = 100 82 | wmax = 0.0 83 | Padding = 1.7 84 | NDegridBand = 16 85 | MaxFacetSize = 0.25 86 | MinNFacetPerAxis = 1 87 | NProcess = 8 88 | BeamModel = None 89 | NBand = 0 90 | FITSFile = beam_$(corr)_$(reim).fits 91 | FITSFeed = None 92 | FITSFeedSwap = False 93 | DtBeamMin = 5.0 94 | FITSParAngleIncDeg = 5.0 95 | FITSLAxis = -X 96 | FITSMAxis = Y 97 | FITSVerbosity = 0 98 | FeedAngle = 0.0 99 | FlipVisibilityHands = 0 100 | 101 | [postmortem] 102 | _Help = Options for "postmortem" flagging based on solution statistics 103 | enable = False 104 | tf-chisq-median = 1.2 105 | tf-np-median = 0.5 106 | time-density = 0.5 107 | chan-density = 0.5 108 | ddid-density = 0.5 109 | 110 | [madmax] 111 | _Help = Options for the "Mad Max" flagger 112 | enable = 0 113 | residuals = 0 114 | estimate = corr 115 | diag = True 116 | offdiag = True 117 | threshold = 10 118 | global-threshold = 12 119 | plot = 1 120 | plot-frac-above = 0.01 121 | plot-bl = 122 | flag-ant = 0 123 | flag-ant-thr = 5 124 | 125 | [sol] 126 | _Help = Solution options which apply at the solver level 127 | jones = K,G 128 | precision = 32 129 | delta-g = 1e-06 130 | delta-chi = 1e-06 131 | chi-int = 5 132 | last-rites = True 133 | stall-quorum = 0.99 134 | term-iters = 50,50 135 | flag-divergence = 0 136 | min-bl = 100.0 137 | max-bl = 0 138 | subset = 139 | 140 | [bbc] 141 | _Help = Options for baseline-based corrections (a.k.a. BBCs, a.k.a. interferometer gains). 142 | load-from = 143 | compute-2x2 = False 144 | apply-2x2 = False 145 | save-to = {out[name]}-BBC-field_{sel[field]}-ddid_{sel[ddid]}.parmdb 146 | per-chan = True 147 | plot = True 148 | 149 | [dist] 150 | _Help = Parallelization and distribution options 151 | ncpu = 8 152 | nworker = 0 153 | nthread = 0 154 | max-chunks = 2 155 | min-chunks = 0 156 | pin = 0 157 | pin-io = False 158 | pin-main = io 159 | safe = True 160 | 161 | [log] 162 | _Help = Options related to logging 163 | memory = True 164 | stats = chi2:.3f 165 | stats-warn = chi2:10 166 | boring = True 167 | append = False 168 | verbose = 0 169 | file-verbose = None 170 | 171 | [debug] 172 | _Help = Debugging options for the discerning masochist 173 | pdb = False 174 | panic-amplitude = 0.0 175 | stop-before-solver = False 176 | escalate-warnings = 0 177 | 178 | [misc] 179 | _Help = Miscellaneous options 180 | random-seed = None 181 | parset-version = 0.1 182 | 183 | [k] 184 | _Help = Options for K-Jones term 185 | label = K 186 | solvable = 0 187 | type = f-slope 188 | load-from = {out[name]}-{JONES}-field_{sel[field]}-ddid_{sel[ddid]}.parmdb 189 | xfer-from = 190 | save-to = 191 | dd-term = 0 192 | fix-dirs = 193 | update-type = full 194 | time-int = 4 195 | freq-int = 1024 196 | max-prior-error = 0.0 197 | max-post-error = 0.0 198 | clip-low = 0.1 199 | clip-high = 10 200 | clip-after = 5 201 | max-iter = 50 202 | epsilon = 1e-06 203 | delta-chi = 1e-06 204 | conv-quorum = 0.99 205 | ref-ant = None 206 | prop-flags = default 207 | estimate-pzd = False 208 | diag-only = 1 209 | offdiag-only = 0 210 | robust-cov = compute 211 | robust-npol = 2 212 | robust-int = 1 213 | robust-save-weights = 0 214 | _Templated = True 215 | 216 | [g] 217 | _Help = Options for G-Jones term 218 | label = G 219 | solvable = 1 220 | type = complex-2x2 221 | delay-estimate-pad-factor = 8 222 | load-from = 223 | xfer-from = 224 | save-to = {out[name]}-{JONES}-field_{sel[field]}-ddid_{sel[ddid]}.parmdb 225 | dd-term = 0 226 | fix-dirs = 227 | update-type = full 228 | estimate-pzd = False 229 | time-int = 48 230 | freq-int = 1024 231 | max-prior-error = 0.3 232 | max-post-error = 0.3 233 | low-snr-warn = 75 234 | high-gain-var-warn = 30 235 | clip-low = 0.1 236 | clip-high = 10000 237 | clip-after = 5 238 | max-iter = 50 239 | pin-slope-iters = 0 240 | epsilon = 1e-06 241 | delta-chi = 1e-06 242 | conv-quorum = 0.99 243 | ref-ant = None 244 | prop-flags = default 245 | diag-only = 0 246 | offdiag-only = False 247 | robust-cov = compute 248 | robust-scale = 1 249 | robust-npol = 2 250 | robust-int = 1 251 | robust-save-weights = 0 252 | estimate-delays = False 253 | _Templated = True 254 | -------------------------------------------------------------------------------- /data/cubical/2GC_apcal_robust.parset: -------------------------------------------------------------------------------- 1 | [data] 2 | _Help = Visibility data options 3 | ms = 4 | column = DATA 5 | time-chunk = 48 6 | freq-chunk = 1024 7 | rebin-time = 1 8 | rebin-freq = 1 9 | chunk-by = SCAN_NUMBER 10 | chunk-by-jump = 0 11 | single-chunk = 12 | single-tile = -1 13 | normalize = 0 14 | 15 | [sel] 16 | _Help = Data selection options 17 | field = 0 18 | ddid = None 19 | taql = 20 | chan = 21 | diag = False 22 | 23 | [out] 24 | _Help = Options for output products 25 | dir = selfcal 26 | name = selfcal.cc-out/selfcal 27 | overwrite = False 28 | backup = 1 29 | mode = sc 30 | apply-solver-flags = True 31 | column = CORRECTED_DATA 32 | derotate = None 33 | model-column = 34 | weight-column = 35 | reinit-column = False 36 | subtract-model = 0 37 | subtract-dirs = 0 38 | correct-dir = 0 39 | plots = 1 40 | casa-gaintables = True 41 | 42 | [model] 43 | _Help = Calibration model options 44 | list = MODEL_DATA 45 | ddes = auto 46 | beam-pattern = None 47 | beam-l-axis = None 48 | beam-m-axis = None 49 | feed-rotate = auto 50 | pa-rotate = False 51 | 52 | [montblanc] 53 | _Help = Montblanc simulation options 54 | device-type = CPU 55 | dtype = double 56 | mem-budget = 1024 57 | verbosity = WARNING 58 | threads = 0 59 | pa-rotate = None 60 | 61 | [weight] 62 | _Help = Weighting options 63 | column = WEIGHT_SPECTRUM 64 | fill-offdiag = False 65 | legacy-v1-2 = False 66 | 67 | [flags] 68 | _Help = General flagging options 69 | apply = -cubical 70 | auto-init = legacy 71 | save = cubical 72 | save-legacy = auto 73 | reinit-bitflags = False 74 | warn-thr = 0.3 75 | see-no-evil = 0 76 | 77 | [degridding] 78 | _Help = Options for the degridder. Only in use when predicting from DicoModels using DDFacet 79 | OverS = 11 80 | Support = 7 81 | Nw = 100 82 | wmax = 0.0 83 | Padding = 1.7 84 | NDegridBand = 16 85 | MaxFacetSize = 0.25 86 | MinNFacetPerAxis = 1 87 | NProcess = 8 88 | BeamModel = None 89 | NBand = 0 90 | FITSFile = beam_$(corr)_$(reim).fits 91 | FITSFeed = None 92 | FITSFeedSwap = False 93 | DtBeamMin = 5.0 94 | FITSParAngleIncDeg = 5.0 95 | FITSLAxis = -X 96 | FITSMAxis = Y 97 | FITSVerbosity = 0 98 | FeedAngle = 0.0 99 | FlipVisibilityHands = 0 100 | 101 | [postmortem] 102 | _Help = Options for "postmortem" flagging based on solution statistics 103 | enable = False 104 | tf-chisq-median = 1.2 105 | tf-np-median = 0.5 106 | time-density = 0.5 107 | chan-density = 0.5 108 | ddid-density = 0.5 109 | 110 | [madmax] 111 | _Help = Options for the "Mad Max" flagger 112 | enable = 0 113 | residuals = 0 114 | estimate = corr 115 | diag = True 116 | offdiag = True 117 | threshold = 10 118 | global-threshold = 12 119 | plot = 1 120 | plot-frac-above = 0.01 121 | plot-bl = 122 | flag-ant = 0 123 | flag-ant-thr = 5 124 | 125 | [sol] 126 | _Help = Solution options which apply at the solver level 127 | jones = K,G 128 | precision = 32 129 | delta-g = 1e-06 130 | delta-chi = 1e-06 131 | chi-int = 5 132 | last-rites = True 133 | stall-quorum = 0.99 134 | term-iters = 50,50 135 | flag-divergence = 0 136 | min-bl = 100.0 137 | max-bl = 0 138 | subset = 139 | 140 | [bbc] 141 | _Help = Options for baseline-based corrections (a.k.a. BBCs, a.k.a. interferometer gains). 142 | load-from = 143 | compute-2x2 = False 144 | apply-2x2 = False 145 | save-to = {out[name]}-BBC-field_{sel[field]}-ddid_{sel[ddid]}.parmdb 146 | per-chan = True 147 | plot = True 148 | 149 | [dist] 150 | _Help = Parallelization and distribution options 151 | ncpu = 8 152 | nworker = 0 153 | nthread = 0 154 | max-chunks = 2 155 | min-chunks = 0 156 | pin = 0 157 | pin-io = False 158 | pin-main = io 159 | safe = True 160 | 161 | [log] 162 | _Help = Options related to logging 163 | memory = True 164 | stats = chi2:.3f 165 | stats-warn = chi2:10 166 | boring = True 167 | append = False 168 | verbose = 0 169 | file-verbose = None 170 | 171 | [debug] 172 | _Help = Debugging options for the discerning masochist 173 | pdb = False 174 | panic-amplitude = 0.0 175 | stop-before-solver = False 176 | escalate-warnings = 0 177 | 178 | [misc] 179 | _Help = Miscellaneous options 180 | random-seed = None 181 | parset-version = 0.1 182 | 183 | [k] 184 | _Help = Options for K-Jones term 185 | label = K 186 | solvable = 0 187 | type = f-slope 188 | load-from = {out[name]}-{JONES}-field_{sel[field]}-ddid_{sel[ddid]}.parmdb 189 | xfer-from = 190 | save-to = 191 | dd-term = 0 192 | fix-dirs = 193 | update-type = full 194 | time-int = 4 195 | freq-int = 1024 196 | max-prior-error = 0.0 197 | max-post-error = 0.0 198 | clip-low = 0.1 199 | clip-high = 10 200 | clip-after = 5 201 | max-iter = 50 202 | epsilon = 1e-06 203 | delta-chi = 1e-06 204 | conv-quorum = 0.99 205 | ref-ant = None 206 | prop-flags = default 207 | estimate-pzd = False 208 | diag-only = 1 209 | offdiag-only = 0 210 | robust-cov = compute 211 | robust-npol = 2 212 | robust-int = 1 213 | robust-save-weights = 0 214 | _Templated = True 215 | 216 | [g] 217 | _Help = Options for G-Jones term 218 | label = G 219 | solvable = 1 220 | type = robust-2x2 221 | delay-estimate-pad-factor = 8 222 | load-from = 223 | xfer-from = 224 | save-to = {out[name]}-{JONES}-field_{sel[field]}-ddid_{sel[ddid]}.parmdb 225 | dd-term = 0 226 | fix-dirs = 227 | update-type = full 228 | estimate-pzd = False 229 | time-int = 48 230 | freq-int = 1024 231 | max-prior-error = 0.3 232 | max-post-error = 0.3 233 | low-snr-warn = 75 234 | high-gain-var-warn = 30 235 | clip-low = 0.1 236 | clip-high = 10000 237 | clip-after = 5 238 | max-iter = 50 239 | pin-slope-iters = 0 240 | epsilon = 1e-06 241 | delta-chi = 1e-06 242 | conv-quorum = 0.99 243 | ref-ant = None 244 | prop-flags = default 245 | diag-only = 0 246 | offdiag-only = False 247 | robust-cov = compute 248 | robust-scale = 1 249 | robust-npol = 2 250 | robust-int = 1 251 | robust-save-weights = 0 252 | estimate-delays = False 253 | _Templated = True 254 | -------------------------------------------------------------------------------- /oxkat/1GC_casa_LINE_refcal_primary.py: -------------------------------------------------------------------------------- 1 | # ian.heywood@physics.ox.ac.uk 2 | 3 | 4 | # Work in progress. 5 | # This is a straightforward copy of the stages for the primary calibrator for the normal 1GC script. 6 | # There is a fillgaps modifier for the high spectral resolution MMS. 7 | # MMS naming schemes etc. still need fixing, at present it globs for *.mms. 8 | # Consider off a KGB block with an iterator and moving the primary flagging to tricolour if CASA proves to be too slow. 9 | # Intermediate flag table states have been dropped for reasons of speed. 10 | 11 | 12 | import glob 13 | import shutil 14 | import time 15 | 16 | 17 | execfile('oxkat/casa_read_project_info.py') 18 | execfile('oxkat/config.py') 19 | 20 | 21 | 22 | def stamp(): 23 | now = str(datetime.datetime.now()).replace(' ','-').replace(':','-').split('.')[0] 24 | return now 25 | 26 | 27 | # ------- Parameters 28 | 29 | 30 | mymms = glob.glob('*.mms')[0] 31 | myuvrange = CAL_1GC_UVRANGE 32 | gapfill = CAL_1GC_LINE_FILLGAPS 33 | 34 | 35 | # ------- Setup names 36 | 37 | 38 | tt = stamp() 39 | 40 | 41 | ktab0 = GAINTABLES+'/cal_1GC_LINE_'+mymms+'_'+tt+'.K0' 42 | bptab0 = GAINTABLES+'/cal_1GC_LINE_'+mymms+'_'+tt+'.B0' 43 | gtab0 = GAINTABLES+'/cal_1GC_LINE_'+mymms+'_'+tt+'.G0' 44 | 45 | 46 | ktab1 = GAINTABLES+'/cal_1GC_LINE_'+mymms+'_'+tt+'.K1' 47 | bptab1 = GAINTABLES+'/cal_1GC_LINE_'+mymms+'_'+tt+'.B1' 48 | gtab1 = GAINTABLES+'/cal_1GC_LINE_'+mymms+'_'+tt+'.G1' 49 | 50 | 51 | ktab2 = GAINTABLES+'/cal_1GC_LINE_'+mymms+'_'+tt+'.K2' 52 | gtab2 = GAINTABLES+'/cal_1GC_LINE_'+mymms+'_'+tt+'.G2' 53 | ftab2 = GAINTABLES+'/cal_1GC_LINE_'+mymms+'_'+tt+'.flux2' 54 | 55 | 56 | ktab3 = GAINTABLES+'/cal_1GC_LINE_'+mymms+'_'+tt+'.K3' 57 | gtab3 = GAINTABLES+'/cal_1GC_LINE_'+mymms+'_'+tt+'.G3' 58 | ftab3 = GAINTABLES+'/cal_1GC_LINE_'+mymms+'_'+tt+'.flux3' 59 | 60 | 61 | 62 | # ------- K0 (primary) 63 | 64 | 65 | gaincal(vis=mymms, 66 | field=bpcal, 67 | #uvrange=myuvrange, 68 | caltable=ktab0, 69 | refant = str(ref_ant), 70 | gaintype = 'K', 71 | solint = 'inf', 72 | parang=False) 73 | 74 | 75 | # ------- G0 (primary; apply K0) 76 | 77 | 78 | gaincal(vis=mymms, 79 | field=bpcal, 80 | uvrange=myuvrange, 81 | caltable=gtab0, 82 | gaintype='G', 83 | solint='inf', 84 | calmode='p', 85 | minsnr=5, 86 | gainfield=[bpcal], 87 | interp = ['nearest'], 88 | gaintable=[ktab0]) 89 | 90 | 91 | # ------- B0 (primary; apply K0, G0) 92 | 93 | 94 | bandpass(vis=mymms, 95 | field=bpcal, 96 | uvrange=myuvrange, 97 | caltable=bptab0, 98 | refant = str(ref_ant), 99 | solint='inf', 100 | combine='', 101 | solnorm=False, 102 | minblperant=4, 103 | minsnr=3.0, 104 | bandtype='B', 105 | fillgaps=gapfill, 106 | parang=False, 107 | gainfield=[bpcal,bpcal], 108 | interp = ['nearest','nearest'], 109 | gaintable=[ktab0,gtab0]) 110 | 111 | 112 | flagdata(vis=bptab0,mode='tfcrop',datacolumn='CPARAM') 113 | flagdata(vis=bptab0,mode='rflag',datacolumn='CPARAM') 114 | 115 | 116 | # ------- Correct primary data with K0,B0,G0 117 | 118 | 119 | applycal(vis=mymms, 120 | gaintable=[ktab0,gtab0,bptab0], 121 | # applymode='calonly', 122 | field=bpcal, 123 | # calwt=False, 124 | parang=False, 125 | gainfield=[bpcal,bpcal,bpcal], 126 | interp = ['nearest','nearest','nearest']) 127 | 128 | 129 | # ------- Flag primary on CORRECTED_DATA - MODEL_DATA 130 | 131 | 132 | flagdata(vis=mymms, 133 | mode='rflag', 134 | datacolumn='residual', 135 | field=bpcal) 136 | 137 | 138 | flagdata(vis=mymms, 139 | mode='tfcrop', 140 | datacolumn='residual', 141 | field=bpcal) 142 | 143 | 144 | 145 | # --------------------------------------------------------------- # 146 | # --------------------------------------------------------------- # 147 | # --------------------------- STAGE 1 --------------------------- # 148 | # --------------------------------------------------------------- # 149 | # --------------------------------------------------------------- # 150 | 151 | 152 | # ------- K1 (primary; apply B0, G0) 153 | 154 | 155 | gaincal(vis=mymms, 156 | field=bpcal, 157 | #uvrange=myuvrange, 158 | caltable=ktab1, 159 | refant = str(ref_ant), 160 | gaintype = 'K', 161 | solint = 'inf', 162 | parang=False, 163 | gaintable=[bptab0,gtab0], 164 | gainfield=[bpcal,bpcal], 165 | interp=['nearest','nearest']) 166 | 167 | 168 | # ------- G1 (primary; apply K1,B0) 169 | 170 | 171 | gaincal(vis=mymms, 172 | field=bpcal, 173 | uvrange=myuvrange, 174 | caltable=gtab1, 175 | gaintype='G', 176 | solint='inf', 177 | calmode='p', 178 | minsnr=5, 179 | gainfield=[bpcal,bpcal], 180 | interp = ['nearest','nearest'], 181 | gaintable=[ktab1,bptab0]) 182 | 183 | 184 | # ------- B1 (primary; apply K1, G1) 185 | 186 | 187 | bandpass(vis=mymms, 188 | field=bpcal, 189 | uvrange=myuvrange, 190 | caltable=bptab1, 191 | refant = str(ref_ant), 192 | solint='inf', 193 | combine='', 194 | solnorm=False, 195 | minblperant=4, 196 | minsnr=3.0, 197 | bandtype='B', 198 | fillgaps=gapfill, 199 | parang=False, 200 | gainfield=[bpcal,bpcal], 201 | interp = ['nearest','nearest'], 202 | gaintable=[ktab1,gtab1]) 203 | 204 | 205 | flagdata(vis=bptab1,mode='tfcrop',datacolumn='CPARAM') 206 | flagdata(vis=bptab1,mode='rflag',datacolumn='CPARAM') 207 | 208 | 209 | # ------- Correct primary data with K1,G1,B1 210 | 211 | 212 | applycal(vis=mymms, 213 | gaintable=[ktab1,gtab1,bptab1], 214 | # applymode='calonly', 215 | field=bpcal, 216 | # calwt=False, 217 | parang=False, 218 | gainfield=[bpcal,bpcal,bpcal], 219 | interp = ['nearest','nearest','nearest']) 220 | 221 | 222 | if SAVE_FLAGS: 223 | flagmanager(vis=mymms,mode='save',versionname='primary-refcal') 224 | -------------------------------------------------------------------------------- /tools/pyMakeMask.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import scipy.ndimage 3 | import scipy.special 4 | import shutil 5 | import sys 6 | from astropy.io import fits 7 | from optparse import OptionParser 8 | from scipy.ndimage.morphology import binary_dilation 9 | from scipy.ndimage.measurements import label 10 | from scipy.ndimage import sum as ndi_sum 11 | 12 | 13 | def get_image(fitsfile): 14 | print('Reading '+fitsfile) 15 | input_hdu = fits.open(fitsfile)[0] 16 | if len(input_hdu.data.shape) == 2: 17 | image = numpy.array(input_hdu.data[:,:]) 18 | elif len(input_hdu.data.shape) == 3: 19 | image = numpy.array(input_hdu.data[0,:,:]) 20 | else: 21 | image = numpy.array(input_hdu.data[0,0,:,:]) 22 | return image 23 | 24 | 25 | def flush_fits(newimage,fitsfile): 26 | print('Writing '+fitsfile) 27 | f = fits.open(fitsfile,mode='update') 28 | input_hdu = f[0] 29 | if len(input_hdu.data.shape) == 2: 30 | input_hdu.data[:,:] = newimage 31 | elif len(input_hdu.data.shape) == 3: 32 | input_hdu.data[0,:,:] = newimage 33 | else: 34 | input_hdu.data[0,0,:,:] = newimage 35 | f.flush() 36 | 37 | 38 | def make_noise_map(restored_image,boxsize): 39 | # Cyril's magic minimum filter 40 | # Plundered from the depths of https://github.com/cyriltasse/DDFacet/blob/master/SkyModel/MakeMask.py 41 | print('Generating noise map') 42 | box = (boxsize,boxsize) 43 | n = boxsize**2.0 44 | x = numpy.linspace(-10,10,1000) 45 | f = 0.5 * (1.0 + scipy.special.erf(x / numpy.sqrt(2.0))) 46 | F = 1.0 - (1.0 - f)**n 47 | ratio = numpy.abs(numpy.interp(0.5, F, x)) 48 | noise = -scipy.ndimage.filters.minimum_filter(restored_image, box) / ratio 49 | negative_mask = noise < 0.0 50 | noise[negative_mask] = 1.0e-10 51 | median_noise = numpy.median(noise) 52 | median_mask = noise < median_noise 53 | noise[median_mask] = median_noise 54 | print('Median noise value is '+str(median_noise)) 55 | return noise 56 | 57 | 58 | def main(): 59 | 60 | parser = OptionParser(usage = '%prog [options] restored_image') 61 | parser.add_option('--threshold', dest = 'threshold', help = 'Sigma threshold for masking (default = 6.0)', default = 6.0) 62 | parser.add_option('--boxsize', dest = 'boxsize', help = 'Box size to use for fields with compact islands (default = 500)', default = 500) 63 | parser.add_option('--smallbox', dest = 'smallbox', help = 'Box size to switch to for fields with small islands (default = 50), set to zero to just use boxsize', default = 50) 64 | parser.add_option('--islandsize', dest = 'islandsize', help = 'Island size in pixels below which smallbox is used (default = 30000)', default = 30000) 65 | parser.add_option('--dilate', dest = 'dilate', help = 'Number of iterations of binary dilation (default = 3, set to 0 to disable)', default = 3) 66 | parser.add_option('--savenoise', dest = 'savenoise', help = 'Enable to export noise image as FITS file (default = do not save noise image', action = 'store_true', default = False) 67 | parser.add_option('--outfile', dest = 'outfile', help = 'Suffix for mask image (default = restored_image.replace(".fits",".mask.fits"))', default = '') 68 | (options,args) = parser.parse_args() 69 | threshold = float(options.threshold) 70 | boxsize = int(options.boxsize) 71 | smallbox = int(options.smallbox) 72 | islandsize = int(options.islandsize) 73 | dilate = int(options.dilate) 74 | savenoise = options.savenoise 75 | outfile = options.outfile 76 | 77 | if len(args) != 1: 78 | print('Please specify a FITS file') 79 | sys.exit() 80 | else: 81 | input_fits = args[0].rstrip('/') 82 | 83 | input_image = get_image(input_fits) 84 | 85 | print('Computing mask with box size: '+str(boxsize)+' pixels') 86 | print('Threshold: '+str(threshold)) 87 | noise_image = make_noise_map(input_image,boxsize) 88 | mask_image = input_image > threshold * noise_image 89 | 90 | if smallbox != 0: 91 | print('Counting islands...') 92 | sizes = [] 93 | mask_image = mask_image.byteswap().newbyteorder() 94 | labeled_mask_image, n_islands = label(mask_image) 95 | print('Found '+str(n_islands)) 96 | print('Sizing them up...') 97 | indices = numpy.arange(0,n_islands) 98 | sizes = ndi_sum(mask_image, labeled_mask_image, indices) 99 | sizes = numpy.sort(sizes)[::-1] 100 | print('Island size threshold: '+str(islandsize)) 101 | print('Top five island sizes:') 102 | print(sizes[0:5]) 103 | if sizes[0] < islandsize: 104 | print('Largest island has fewer than '+str(islandsize)+' pixels') 105 | print('Recomputing mask with box size: '+str(smallbox)+' pixels') 106 | noise_image = make_noise_map(input_image,smallbox) 107 | mask_image = input_image > threshold * noise_image 108 | else: 109 | print('Sticking with box size: '+str(boxsize)+' pixels') 110 | 111 | if savenoise: 112 | noise_fits = input_fits.replace('.fits', '.noise.fits') 113 | shutil.copyfile(input_fits, noise_fits) 114 | flush_fits(noise_image, noise_fits) 115 | 116 | mask_image[:,-1]=0 117 | mask_image[:,0]=0 118 | mask_image[0,:]=0 119 | mask_image[-1,:]=0 120 | 121 | if dilate != 0: 122 | print('Dilating mask, '+str(dilate)+' iteration(s)') 123 | dilated = binary_dilation(input = mask_image, iterations = dilate) 124 | mask_image = dilated 125 | 126 | if outfile == '': 127 | mask_fits = input_fits.replace('.fits', '.mask.fits') 128 | else: 129 | mask_fits = outfile 130 | shutil.copyfile(input_fits, mask_fits) 131 | 132 | flush_fits(mask_image,mask_fits) 133 | 134 | print('Done') 135 | 136 | 137 | if __name__ == '__main__': 138 | main() 139 | 140 | -------------------------------------------------------------------------------- /data/cubical/apply_peelsols.parset: -------------------------------------------------------------------------------- 1 | [data] 2 | _Help = Visibility data options 3 | ms = 4 | column = DATA 5 | time-chunk = 72 6 | freq-chunk = 256 7 | rebin-time = 1 8 | rebin-freq = 1 9 | chunk-by = SCAN_NUMBER 10 | chunk-by-jump = 1 11 | single-chunk = 12 | single-tile = -1 13 | normalize = 0 14 | 15 | [sel] 16 | _Help = Data selection options 17 | field = 0 18 | ddid = None 19 | taql = 20 | chan = 21 | diag = False 22 | 23 | [out] 24 | _Help = Options for output products 25 | dir = peeling 26 | name = peeling.cc-out/peeling 27 | overwrite = False 28 | backup = 1 29 | mode = ar 30 | apply-solver-flags = True 31 | column = CORRECTED_DATA 32 | derotate = None 33 | model-column = 34 | weight-column = 35 | reinit-column = False 36 | subtract-model = 0 37 | subtract-dirs = 1 38 | correct-dir = 0 39 | plots = 1 40 | casa-gaintables = True 41 | 42 | [model] 43 | _Help = Calibration model options 44 | list = MODEL_DATA+-DIR1_DATA:DIR1_DATA 45 | ddes = auto 46 | beam-pattern = None 47 | beam-l-axis = None 48 | beam-m-axis = None 49 | feed-rotate = auto 50 | pa-rotate = False 51 | 52 | [montblanc] 53 | _Help = Montblanc simulation options 54 | device-type = CPU 55 | dtype = double 56 | mem-budget = 1024 57 | verbosity = WARNING 58 | threads = 0 59 | pa-rotate = None 60 | 61 | [weight] 62 | _Help = Weighting options 63 | column = WEIGHT_SPECTRUM 64 | fill-offdiag = False 65 | legacy-v1-2 = False 66 | 67 | [flags] 68 | _Help = General flagging options 69 | apply = -cubical 70 | auto-init = legacy 71 | save = cubical 72 | save-legacy = auto 73 | reinit-bitflags = False 74 | warn-thr = 0.3 75 | see-no-evil = 0 76 | 77 | [degridding] 78 | _Help = Options for the degridder. Only in use when predicting from DicoModels using DDFacet 79 | OverS = 11 80 | Support = 7 81 | Nw = 100 82 | wmax = 0.0 83 | Padding = 1.7 84 | NDegridBand = 16 85 | MaxFacetSize = 0.25 86 | MinNFacetPerAxis = 1 87 | NProcess = 8 88 | BeamModel = None 89 | NBand = 0 90 | FITSFile = beam_$(corr)_$(reim).fits 91 | FITSFeed = None 92 | FITSFeedSwap = False 93 | DtBeamMin = 5.0 94 | FITSParAngleIncDeg = 5.0 95 | FITSLAxis = -X 96 | FITSMAxis = Y 97 | FITSVerbosity = 0 98 | FeedAngle = 0.0 99 | FlipVisibilityHands = 0 100 | 101 | [postmortem] 102 | _Help = Options for "postmortem" flagging based on solution statistics 103 | enable = False 104 | tf-chisq-median = 1.2 105 | tf-np-median = 0.5 106 | time-density = 0.5 107 | chan-density = 0.5 108 | ddid-density = 0.5 109 | 110 | [madmax] 111 | _Help = Options for the "Mad Max" flagger 112 | enable = 0 113 | residuals = 0 114 | estimate = corr 115 | diag = True 116 | offdiag = True 117 | threshold = 10 118 | global-threshold = 12 119 | plot = 1 120 | plot-frac-above = 0.01 121 | plot-bl = 122 | flag-ant = 0 123 | flag-ant-thr = 5 124 | 125 | [sol] 126 | _Help = Solution options which apply at the solver level 127 | jones = G,DE 128 | precision = 32 129 | delta-g = 1e-06 130 | delta-chi = 1e-06 131 | chi-int = 5 132 | last-rites = True 133 | stall-quorum = 0.99 134 | term-iters = 50,50 135 | flag-divergence = 0 136 | min-bl = 150.0 137 | max-bl = 0 138 | subset = 139 | 140 | [bbc] 141 | _Help = Options for baseline-based corrections (a.k.a. BBCs, a.k.a. interferometer gains). 142 | load-from = 143 | compute-2x2 = False 144 | apply-2x2 = False 145 | save-to = 146 | per-chan = True 147 | plot = True 148 | 149 | [dist] 150 | _Help = Parallelization and distribution options 151 | ncpu = 16 152 | nworker = 0 153 | nthread = 0 154 | max-chunks = 2 155 | min-chunks = 0 156 | pin = 0 157 | pin-io = False 158 | pin-main = io 159 | safe = True 160 | 161 | [log] 162 | _Help = Options related to logging 163 | memory = True 164 | stats = chi2:.3f 165 | stats-warn = chi2:10 166 | boring = True 167 | append = False 168 | verbose = 0 169 | file-verbose = None 170 | 171 | [debug] 172 | _Help = Debugging options for the discerning masochist 173 | pdb = False 174 | panic-amplitude = 0.0 175 | stop-before-solver = False 176 | escalate-warnings = 0 177 | 178 | [misc] 179 | _Help = Miscellaneous options 180 | random-seed = None 181 | parset-version = 0.1 182 | 183 | [g] 184 | _Help = Options for G-Jones term 185 | label = G 186 | solvable = 0 187 | type = complex-2x2 188 | delay-estimate-pad-factor = 8 189 | load-from = {out[name]}-{JONES}-field_{sel[field]}-ddid_{sel[ddid]}.parmdb 190 | xfer-from = 191 | save-to = 192 | dd-term = 0 193 | fix-dirs = 194 | update-type = full 195 | estimate-pzd = False 196 | time-int = 16 197 | freq-int = 256 198 | max-prior-error = 0.3 199 | max-post-error = 0.3 200 | low-snr-warn = 75 201 | high-gain-var-warn = 30 202 | clip-low = 0.1 203 | clip-high = 10 204 | clip-after = 5 205 | max-iter = 50 206 | pin-slope-iters = 0 207 | epsilon = 1e-06 208 | delta-chi = 1e-06 209 | conv-quorum = 0.99 210 | ref-ant = None 211 | prop-flags = default 212 | diag-only = 0 213 | offdiag-only = False 214 | robust-cov = compute 215 | robust-scale = 1 216 | robust-npol = 2 217 | robust-int = 1 218 | robust-save-weights = 0 219 | estimate-delays = False 220 | _Templated = True 221 | 222 | [de] 223 | _Help = Options for G-Jones term 224 | label = dE 225 | solvable = 0 226 | type = complex-2x2 227 | delay-estimate-pad-factor = 8 228 | load-from = {out[name]}-{JONES}-field_{sel[field]}-ddid_{sel[ddid]}.parmdb 229 | xfer-from = 230 | save-to = 231 | dd-term = 1 232 | fix-dirs = 233 | update-type = full 234 | estimate-pzd = False 235 | time-int = 72 236 | freq-int = 64 237 | max-prior-error = 5.0 238 | max-post-error = 5.0 239 | low-snr-warn = 75 240 | high-gain-var-warn = 30 241 | clip-low = 0.0 242 | clip-high = 1000000 243 | clip-after = 5 244 | max-iter = 50 245 | pin-slope-iters = 0 246 | epsilon = 1e-06 247 | delta-chi = 1e-06 248 | conv-quorum = 0.99 249 | ref-ant = None 250 | prop-flags = default 251 | diag-only = 0 252 | offdiag-only = False 253 | robust-cov = compute 254 | robust-scale = 1 255 | robust-npol = 2 256 | robust-int = 1 257 | robust-save-weights = 0 258 | estimate-delays = False 259 | _Templated = True 260 | 261 | -------------------------------------------------------------------------------- /data/cubical/3GC_peel.parset: -------------------------------------------------------------------------------- 1 | [data] 2 | _Help = Visibility data options 3 | ms = 4 | column = DATA 5 | time-chunk = 72 6 | freq-chunk = 256 7 | rebin-time = 1 8 | rebin-freq = 1 9 | chunk-by = SCAN_NUMBER 10 | chunk-by-jump = 1 11 | single-chunk = 12 | single-tile = -1 13 | normalize = 0 14 | 15 | [sel] 16 | _Help = Data selection options 17 | field = 0 18 | ddid = None 19 | taql = 20 | chan = 21 | diag = False 22 | 23 | [out] 24 | _Help = Options for output products 25 | dir = peeling 26 | name = peeling.cc-out/peeling 27 | overwrite = False 28 | backup = 1 29 | mode = sr 30 | apply-solver-flags = True 31 | column = CORRECTED_DATA 32 | derotate = None 33 | model-column = 34 | weight-column = 35 | reinit-column = False 36 | subtract-model = 0 37 | subtract-dirs = 1 38 | correct-dir = 0 39 | plots = 1 40 | casa-gaintables = True 41 | 42 | [model] 43 | _Help = Calibration model options 44 | list = MODEL_DATA+-DIR1_DATA:DIR1_DATA 45 | ddes = auto 46 | beam-pattern = None 47 | beam-l-axis = None 48 | beam-m-axis = None 49 | feed-rotate = auto 50 | pa-rotate = False 51 | 52 | [montblanc] 53 | _Help = Montblanc simulation options 54 | device-type = CPU 55 | dtype = double 56 | mem-budget = 1024 57 | verbosity = WARNING 58 | threads = 0 59 | pa-rotate = None 60 | 61 | [weight] 62 | _Help = Weighting options 63 | column = WEIGHT_SPECTRUM 64 | fill-offdiag = False 65 | legacy-v1-2 = False 66 | 67 | [flags] 68 | _Help = General flagging options 69 | apply = FLAG 70 | auto-init = None 71 | save = 0 72 | save-legacy = 1 73 | reinit-bitflags = False 74 | warn-thr = 0.3 75 | see-no-evil = 0 76 | 77 | [degridding] 78 | _Help = Options for the degridder. Only in use when predicting from DicoModels using DDFacet 79 | OverS = 11 80 | Support = 7 81 | Nw = 100 82 | wmax = 0.0 83 | Padding = 1.7 84 | NDegridBand = 16 85 | MaxFacetSize = 0.25 86 | MinNFacetPerAxis = 1 87 | NProcess = 8 88 | BeamModel = None 89 | NBand = 0 90 | FITSFile = beam_$(corr)_$(reim).fits 91 | FITSFeed = None 92 | FITSFeedSwap = False 93 | DtBeamMin = 5.0 94 | FITSParAngleIncDeg = 5.0 95 | FITSLAxis = -X 96 | FITSMAxis = Y 97 | FITSVerbosity = 0 98 | FeedAngle = 0.0 99 | FlipVisibilityHands = 0 100 | 101 | [postmortem] 102 | _Help = Options for "postmortem" flagging based on solution statistics 103 | enable = False 104 | tf-chisq-median = 1.2 105 | tf-np-median = 0.5 106 | time-density = 0.5 107 | chan-density = 0.5 108 | ddid-density = 0.5 109 | 110 | [madmax] 111 | _Help = Options for the "Mad Max" flagger 112 | enable = 0 113 | residuals = 0 114 | estimate = corr 115 | diag = True 116 | offdiag = True 117 | threshold = 10 118 | global-threshold = 12 119 | plot = 1 120 | plot-frac-above = 0.01 121 | plot-bl = 122 | flag-ant = 0 123 | flag-ant-thr = 5 124 | 125 | [sol] 126 | _Help = Solution options which apply at the solver level 127 | jones = G,DE 128 | precision = 32 129 | delta-g = 1e-06 130 | delta-chi = 1e-06 131 | chi-int = 5 132 | last-rites = True 133 | stall-quorum = 0.99 134 | term-iters = 50,50 135 | flag-divergence = 0 136 | min-bl = 150.0 137 | max-bl = 0 138 | subset = 139 | 140 | [bbc] 141 | _Help = Options for baseline-based corrections (a.k.a. BBCs, a.k.a. interferometer gains). 142 | load-from = 143 | compute-2x2 = False 144 | apply-2x2 = False 145 | save-to = {out[name]}-BBC-field_{sel[field]}-ddid_{sel[ddid]}.parmdb 146 | per-chan = True 147 | plot = True 148 | 149 | [dist] 150 | _Help = Parallelization and distribution options 151 | ncpu = 6 152 | nworker = 0 153 | nthread = 0 154 | max-chunks = 2 155 | min-chunks = 0 156 | pin = 0 157 | pin-io = False 158 | pin-main = io 159 | safe = True 160 | 161 | [log] 162 | _Help = Options related to logging 163 | memory = True 164 | stats = chi2:.3f 165 | stats-warn = chi2:10 166 | boring = True 167 | append = False 168 | verbose = 0 169 | file-verbose = None 170 | 171 | [debug] 172 | _Help = Debugging options for the discerning masochist 173 | pdb = False 174 | panic-amplitude = 0.0 175 | stop-before-solver = False 176 | escalate-warnings = 0 177 | 178 | [misc] 179 | _Help = Miscellaneous options 180 | random-seed = None 181 | parset-version = 0.1 182 | 183 | [g] 184 | _Help = Options for G-Jones term 185 | label = G 186 | solvable = 1 187 | type = complex-2x2 188 | delay-estimate-pad-factor = 8 189 | load-from = 190 | xfer-from = 191 | save-to = {out[name]}-{JONES}-field_{sel[field]}-ddid_{sel[ddid]}.parmdb 192 | dd-term = 0 193 | fix-dirs = 194 | update-type = full 195 | estimate-pzd = False 196 | time-int = 18 197 | freq-int = 256 198 | max-prior-error = 0.3 199 | max-post-error = 0.3 200 | low-snr-warn = 75 201 | high-gain-var-warn = 30 202 | clip-low = 0.1 203 | clip-high = 10 204 | clip-after = 5 205 | max-iter = 50 206 | pin-slope-iters = 0 207 | epsilon = 1e-06 208 | delta-chi = 1e-06 209 | conv-quorum = 0.99 210 | ref-ant = None 211 | prop-flags = default 212 | diag-only = 0 213 | offdiag-only = False 214 | robust-cov = compute 215 | robust-scale = 1 216 | robust-npol = 2 217 | robust-int = 1 218 | robust-save-weights = 0 219 | estimate-delays = False 220 | _Templated = True 221 | 222 | [de] 223 | _Help = Options for G-Jones term 224 | label = dE 225 | solvable = 1 226 | type = complex-2x2 227 | delay-estimate-pad-factor = 8 228 | load-from = 229 | xfer-from = 230 | save-to = {out[name]}-{JONES}-field_{sel[field]}-ddid_{sel[ddid]}.parmdb 231 | dd-term = 1 232 | fix-dirs = 0 233 | update-type = full 234 | estimate-pzd = False 235 | time-int = 72 236 | freq-int = 64 237 | max-prior-error = 5.0 238 | max-post-error = 5.0 239 | low-snr-warn = 75 240 | high-gain-var-warn = 30 241 | clip-low = 0.0 242 | clip-high = 1000000 243 | clip-after = 5 244 | max-iter = 50 245 | pin-slope-iters = 0 246 | epsilon = 1e-06 247 | delta-chi = 1e-06 248 | conv-quorum = 0.99 249 | ref-ant = None 250 | prop-flags = default 251 | diag-only = 0 252 | offdiag-only = False 253 | robust-cov = compute 254 | robust-scale = 1 255 | robust-npol = 2 256 | robust-int = 1 257 | robust-save-weights = 0 258 | estimate-delays = False 259 | _Templated = True 260 | 261 | -------------------------------------------------------------------------------- /data/cubical/3GC_peel_2dirs.parset: -------------------------------------------------------------------------------- 1 | [data] 2 | _Help = Visibility data options 3 | ms = 4 | column = DATA 5 | time-chunk = 72 6 | freq-chunk = 256 7 | rebin-time = 1 8 | rebin-freq = 1 9 | chunk-by = SCAN_NUMBER 10 | chunk-by-jump = 1 11 | single-chunk = 12 | single-tile = -1 13 | normalize = 0 14 | 15 | [sel] 16 | _Help = Data selection options 17 | field = 0 18 | ddid = None 19 | taql = 20 | chan = 21 | diag = False 22 | 23 | [out] 24 | _Help = Options for output products 25 | dir = peeling 26 | name = peeling.cc-out/peeling 27 | overwrite = False 28 | backup = 1 29 | mode = sr 30 | apply-solver-flags = True 31 | column = CORRECTED_DATA 32 | derotate = None 33 | model-column = 34 | weight-column = 35 | reinit-column = False 36 | subtract-model = 0 37 | subtract-dirs = 1,2 38 | correct-dir = 0 39 | plots = 1 40 | casa-gaintables = True 41 | 42 | [model] 43 | _Help = Calibration model options 44 | list = MODEL_DATA+-DIR1_DATA+-DIR2_DATA:DIR1_DATA:DIR2_DATA 45 | ddes = auto 46 | beam-pattern = None 47 | beam-l-axis = None 48 | beam-m-axis = None 49 | feed-rotate = auto 50 | pa-rotate = False 51 | 52 | [montblanc] 53 | _Help = Montblanc simulation options 54 | device-type = CPU 55 | dtype = double 56 | mem-budget = 1024 57 | verbosity = WARNING 58 | threads = 0 59 | pa-rotate = None 60 | 61 | [weight] 62 | _Help = Weighting options 63 | column = WEIGHT_SPECTRUM 64 | fill-offdiag = False 65 | legacy-v1-2 = False 66 | 67 | [flags] 68 | _Help = General flagging options 69 | apply = FLAG 70 | auto-init = None 71 | save = 0 72 | save-legacy = 1 73 | reinit-bitflags = False 74 | warn-thr = 0.3 75 | see-no-evil = 0 76 | 77 | [degridding] 78 | _Help = Options for the degridder. Only in use when predicting from DicoModels using DDFacet 79 | OverS = 11 80 | Support = 7 81 | Nw = 100 82 | wmax = 0.0 83 | Padding = 1.7 84 | NDegridBand = 16 85 | MaxFacetSize = 0.25 86 | MinNFacetPerAxis = 1 87 | NProcess = 8 88 | BeamModel = None 89 | NBand = 0 90 | FITSFile = beam_$(corr)_$(reim).fits 91 | FITSFeed = None 92 | FITSFeedSwap = False 93 | DtBeamMin = 5.0 94 | FITSParAngleIncDeg = 5.0 95 | FITSLAxis = -X 96 | FITSMAxis = Y 97 | FITSVerbosity = 0 98 | FeedAngle = 0.0 99 | FlipVisibilityHands = 0 100 | 101 | [postmortem] 102 | _Help = Options for "postmortem" flagging based on solution statistics 103 | enable = False 104 | tf-chisq-median = 1.2 105 | tf-np-median = 0.5 106 | time-density = 0.5 107 | chan-density = 0.5 108 | ddid-density = 0.5 109 | 110 | [madmax] 111 | _Help = Options for the "Mad Max" flagger 112 | enable = 0 113 | residuals = 0 114 | estimate = corr 115 | diag = True 116 | offdiag = True 117 | threshold = 10 118 | global-threshold = 12 119 | plot = 1 120 | plot-frac-above = 0.01 121 | plot-bl = 122 | flag-ant = 0 123 | flag-ant-thr = 5 124 | 125 | [sol] 126 | _Help = Solution options which apply at the solver level 127 | jones = G,DE 128 | precision = 32 129 | delta-g = 1e-06 130 | delta-chi = 1e-06 131 | chi-int = 5 132 | last-rites = True 133 | stall-quorum = 0.99 134 | term-iters = 50,50 135 | flag-divergence = 0 136 | min-bl = 150.0 137 | max-bl = 0 138 | subset = 139 | 140 | [bbc] 141 | _Help = Options for baseline-based corrections (a.k.a. BBCs, a.k.a. interferometer gains). 142 | load-from = 143 | compute-2x2 = False 144 | apply-2x2 = False 145 | save-to = {out[name]}-BBC-field_{sel[field]}-ddid_{sel[ddid]}.parmdb 146 | per-chan = True 147 | plot = True 148 | 149 | [dist] 150 | _Help = Parallelization and distribution options 151 | ncpu = 6 152 | nworker = 0 153 | nthread = 0 154 | max-chunks = 2 155 | min-chunks = 0 156 | pin = 0 157 | pin-io = False 158 | pin-main = io 159 | safe = True 160 | 161 | [log] 162 | _Help = Options related to logging 163 | memory = True 164 | stats = chi2:.3f 165 | stats-warn = chi2:10 166 | boring = True 167 | append = False 168 | verbose = 0 169 | file-verbose = None 170 | 171 | [debug] 172 | _Help = Debugging options for the discerning masochist 173 | pdb = False 174 | panic-amplitude = 0.0 175 | stop-before-solver = False 176 | escalate-warnings = 0 177 | 178 | [misc] 179 | _Help = Miscellaneous options 180 | random-seed = None 181 | parset-version = 0.1 182 | 183 | [g] 184 | _Help = Options for G-Jones term 185 | label = G 186 | solvable = 1 187 | type = complex-2x2 188 | delay-estimate-pad-factor = 8 189 | load-from = 190 | xfer-from = 191 | save-to = {out[name]}-{JONES}-field_{sel[field]}-ddid_{sel[ddid]}.parmdb 192 | dd-term = 0 193 | fix-dirs = 194 | update-type = full 195 | estimate-pzd = False 196 | time-int = 18 197 | freq-int = 256 198 | max-prior-error = 0.3 199 | max-post-error = 0.3 200 | low-snr-warn = 75 201 | high-gain-var-warn = 30 202 | clip-low = 0.1 203 | clip-high = 10 204 | clip-after = 5 205 | max-iter = 50 206 | pin-slope-iters = 0 207 | epsilon = 1e-06 208 | delta-chi = 1e-06 209 | conv-quorum = 0.99 210 | ref-ant = None 211 | prop-flags = default 212 | diag-only = 0 213 | offdiag-only = False 214 | robust-cov = compute 215 | robust-scale = 1 216 | robust-npol = 2 217 | robust-int = 1 218 | robust-save-weights = 0 219 | estimate-delays = False 220 | _Templated = True 221 | 222 | [de] 223 | _Help = Options for G-Jones term 224 | label = dE 225 | solvable = 1 226 | type = complex-2x2 227 | delay-estimate-pad-factor = 8 228 | load-from = 229 | xfer-from = 230 | save-to = {out[name]}-{JONES}-field_{sel[field]}-ddid_{sel[ddid]}.parmdb 231 | dd-term = 1 232 | fix-dirs = 0 233 | update-type = full 234 | estimate-pzd = False 235 | time-int = 72 236 | freq-int = 64 237 | max-prior-error = 5.0 238 | max-post-error = 5.0 239 | low-snr-warn = 75 240 | high-gain-var-warn = 30 241 | clip-low = 0.0 242 | clip-high = 1000000 243 | clip-after = 5 244 | max-iter = 50 245 | pin-slope-iters = 0 246 | epsilon = 1e-06 247 | delta-chi = 1e-06 248 | conv-quorum = 0.99 249 | ref-ant = None 250 | prop-flags = default 251 | diag-only = 0 252 | offdiag-only = False 253 | robust-cov = compute 254 | robust-scale = 1 255 | robust-npol = 2 256 | robust-int = 1 257 | robust-save-weights = 0 258 | estimate-delays = False 259 | _Templated = True 260 | 261 | -------------------------------------------------------------------------------- /waterhole/setup_image_secondaries.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | import glob 6 | import json 7 | import os.path as o 8 | import sys 9 | sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) 10 | 11 | 12 | from oxkat import generate_jobs as gen 13 | from oxkat import config as cfg 14 | 15 | 16 | def main(): 17 | 18 | USE_SINGULARITY = cfg.USE_SINGULARITY 19 | 20 | gen.preamble() 21 | print(gen.now()+'Imaging secondary calibrators') 22 | 23 | 24 | # ------------------------------------------------------------------------------ 25 | # 26 | # Setup paths, required containers, infrastructure 27 | # 28 | # ------------------------------------------------------------------------------ 29 | 30 | 31 | OXKAT = cfg.OXKAT 32 | IMAGES = cfg.IMAGES 33 | SCRIPTS = cfg.SCRIPTS 34 | 35 | gen.setup_dir(IMAGES) 36 | gen.setup_dir(cfg.LOGS) 37 | gen.setup_dir(cfg.SCRIPTS) 38 | 39 | 40 | INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) 41 | if CONTAINER_PATH is not None: 42 | CONTAINER_RUNNER='singularity exec ' 43 | else: 44 | CONTAINER_RUNNER='' 45 | 46 | 47 | WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.WSCLEAN_PATTERN,USE_SINGULARITY) 48 | 49 | 50 | # Get secondary calibrator information from project info json file 51 | 52 | with open('project_info.json') as f: 53 | project_info = json.load(f) 54 | 55 | myms = project_info['working_ms'] 56 | pcal_names = project_info['secondary_names'] 57 | pcal_ids = project_info['secondary_ids'] 58 | 59 | if cfg.PRE_FIELDS != '': 60 | from oxkat import user_field_handler as ufh 61 | pcal_names = ufh.user_pcals 62 | pcal_ids = ufh.user_pcal_ids 63 | 64 | # ------------------------------------------------------------------------------ 65 | # 66 | # Image calibrators -- recipe definition 67 | # 68 | # ------------------------------------------------------------------------------ 69 | 70 | 71 | cal_steps = [] 72 | codes = [] 73 | ii = 1 74 | 75 | # Loop over secondaries 76 | 77 | for i in range(0,len(pcal_ids)): 78 | 79 | field = pcal_ids[i] 80 | calname = pcal_names[i] 81 | 82 | 83 | steps = [] 84 | filename_calname = gen.scrub_target_name(calname) 85 | 86 | 87 | code = gen.get_target_code(calname) 88 | if code in codes: 89 | code += '_'+str(ii) 90 | ii += 1 91 | codes.append(code) 92 | 93 | 94 | gen.print_spacer() 95 | print(gen.now()+'Secondary | '+calname) 96 | print(gen.now()+'Code | '+code) 97 | 98 | 99 | # Image prefix 100 | img_prefix = IMAGES+'/img_'+myms+'_'+filename_calname+'_corrblind' 101 | 102 | 103 | step = {} 104 | step['step'] = i 105 | step['comment'] = 'Run wsclean, blind deconvolution of the CORRECTED_DATA for '+calname 106 | if i == 0: 107 | step['dependency'] = None 108 | else: 109 | step['dependency'] = i-1 110 | step['id'] = 'WSCMA'+code 111 | step['slurm_config'] = cfg.SLURM_WSCLEAN 112 | step['pbs_config'] = cfg.PBS_WSCLEAN 113 | syscall = CONTAINER_RUNNER+WSCLEAN_CONTAINER+' ' if USE_SINGULARITY else '' 114 | syscall = 'singularity exec '+WSCLEAN_CONTAINER+' ' 115 | syscall += gen.generate_syscall_wsclean(mslist = [myms], 116 | imgname = img_prefix, 117 | field = field, 118 | datacol = 'CORRECTED_DATA', 119 | imsize = 8192, 120 | niter = 40000, 121 | gain = 0.2, 122 | mgain = 0.9, 123 | bda = True, 124 | mask = 'none') 125 | step['syscall'] = syscall 126 | steps.append(step) 127 | 128 | 129 | 130 | # ------------------------------------------------------------------------------ 131 | # 132 | # Write the run file and kill file based on the recipe 133 | # 134 | # ------------------------------------------------------------------------------ 135 | 136 | 137 | submit_file = 'submit_image_secondary_jobs.sh' 138 | kill_file = cfg.SCRIPTS+'/kill_image_secondary_jobs.sh' 139 | 140 | f = open(submit_file,'w') 141 | f.write('#!/usr/bin/env bash\n') 142 | f.write('export SINGULARITY_BINDPATH='+cfg.BINDPATH+'\n') 143 | 144 | id_list = [] 145 | 146 | for step in steps: 147 | 148 | step_id = step['id'] 149 | id_list.append(step_id) 150 | if step['dependency'] is not None: 151 | dependency = steps[step['dependency']]['id'] 152 | else: 153 | dependency = None 154 | syscall = step['syscall'] 155 | if 'slurm_config' in step.keys(): 156 | slurm_config = step['slurm_config'] 157 | else: 158 | slurm_config = cfg.SLURM_DEFAULTS 159 | if 'pbs_config' in step.keys(): 160 | pbs_config = step['pbs_config'] 161 | else: 162 | pbs_config = cfg.PBS_DEFAULTS 163 | comment = step['comment'] 164 | 165 | run_command = gen.job_handler(syscall = syscall, 166 | jobname = step_id, 167 | infrastructure = INFRASTRUCTURE, 168 | dependency = dependency, 169 | slurm_config = slurm_config, 170 | pbs_config = pbs_config) 171 | 172 | 173 | f.write('\n# '+comment+'\n') 174 | f.write(run_command) 175 | 176 | 177 | if INFRASTRUCTURE == 'idia' or INFRASTRUCTURE == 'hippo': 178 | kill = '\necho "scancel "$'+'" "$'.join(id_list)+' > '+kill_file+'\n' 179 | f.write(kill) 180 | elif INFRASTRUCTURE == 'chpc': 181 | kill = '\necho "qdel "$'+'" "$'.join(id_list)+' > '+kill_file+'\n' 182 | f.write(kill) 183 | 184 | f.close() 185 | 186 | gen.make_executable(submit_file) 187 | 188 | gen.print_spacer() 189 | print(gen.now()+'Created '+submit_file) 190 | gen.print_spacer() 191 | 192 | # ------------------------------------------------------------------------------ 193 | 194 | 195 | 196 | if __name__ == "__main__": 197 | 198 | 199 | main() 200 | -------------------------------------------------------------------------------- /tools/identify_dE_directions.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | import glob 6 | import Tigger 7 | import numpy 8 | import os 9 | import sys 10 | from astLib import astCoords as ac 11 | 12 | 13 | def rad2deg(xx): 14 | return 180.0*xx/numpy.pi 15 | 16 | 17 | def tiggerConvert(gaul): 18 | 19 | # Adapted from code by @SpheMakh 20 | 21 | dict_gaul2lsm = {'Gaus_id':'name', 22 | 'Isl_id':'Isl_id', 23 | 'Source_id':'Source_id', 24 | 'Wave_id':'Wave_id', 25 | 'RA':'ra_d', 26 | 'E_RA':'E_RA', 27 | 'DEC':'dec_d', 28 | 'E_DEC':'E_DEC', 29 | 'Total_flux':'i', 30 | 'E_Total_flux':'E_Total_flux', 31 | 'Peak_flux':'Peak_flux', 32 | 'E_Peak_flux':'E_Peak_flux', 33 | 'Xposn':'Xposn', 34 | 'E_Xposn':'E_Xposn', 35 | 'Yposn':'Yposn', 36 | 'E_Yposn':'E_Yposn', 37 | 'Maj':'Maj', 38 | 'E_Maj':'E_Maj', 39 | 'Min':'Min', 40 | 'E_Min':'E_Min', 41 | 'PA':'PA', 42 | 'E_PA':'E_PA', 43 | 'Maj_img_plane':'Maj_img_plane', 44 | 'E_Maj_img_plane':'E_Maj_img_plane', 45 | 'Min_img_plane':'Min_img_plane', 46 | 'E_Min_img_plane':'E_Min_img_plane', 47 | 'PA_img_plane':'PA_img_plane', 48 | 'E_PA_img_plane':'E_PA_img_plane', 49 | 'DC_Maj':'emaj_d', 50 | 'E_DC_Maj':'E_DC_Maj', 51 | 'DC_Min':'emin_d', 52 | 'E_DC_Min':'E_DC_Min', 53 | 'DC_PA':'pa_d', 54 | 'E_DC_PA':'E_DC_PA', 55 | 'DC_Maj_img_plane':'DC_Maj_img_plane', 56 | 'E_DC_Maj_img_plane':'E_DC_Maj_img_plane', 57 | 'DC_Min_img_plane':'DC_Min_img_plane', 58 | 'E_DC_Min_img_plane':'E_DC_Min_img_plane', 59 | 'DC_PA_img_plane':'DC_PA_img_plane', 60 | 'E_DC_PA_img_plane':'E_DC_PA_img_plane', 61 | 'Isl_Total_flux':'Isl_Total_flux', 62 | 'E_Isl_Total_flux':'E_Isl_Total_flux', 63 | 'Isl_rms':'Isl_rms', 64 | 'Isl_mean':'Isl_mean', 65 | 'Resid_Isl_rms':'Resid_Isl_rms', 66 | 'Resid_Isl_mean':'Resid_Isl_mean', 67 | 'S_Code':'S_Code', 68 | 'Total_Q':'q', 69 | 'E_Total_Q':'E_Total_Q', 70 | 'Total_U':'u', 71 | 'E_Total_U':'E_Total_U', 72 | 'Total_V':'v', 73 | 'E_Total_V':'E_Total_V', 74 | 'Linear_Pol_frac':'Linear_Pol_frac', 75 | 'Elow_Linear_Pol_frac':'Elow_Linear_Pol_frac', 76 | 'Ehigh_Linear_Pol_frac':'Ehigh_Linear_Pol_frac', 77 | 'Circ_Pol_Frac':'Circ_Pol_Frac', 78 | 'Elow_Circ_Pol_Frac':'Elow_Circ_Pol_Frac', 79 | 'Ehigh_Circ_Pol_Frac':'Ehigh_Circ_Pol_Frac', 80 | 'Total_Pol_Frac':'Total_Pol_Frac', 81 | 'Elow_Total_Pol_Frac':'Elow_Total_Pol_Frac', 82 | 'Ehigh_Total_Pol_Frac':'Ehigh_Total_Pol_Frac', 83 | 'Linear_Pol_Ang':'Linear_Pol_Ang', 84 | 'E_Linear_Pol_Ang':'E_Linear_Pol_Ang'} 85 | 86 | 87 | dict_pol_flag = {'Gaus_id':0, 88 | 'Isl_id':0, 89 | 'Source_id':0, 90 | 'Wave_id':0, 91 | 'RA':0, 92 | 'E_RA':0, 93 | 'DEC':0, 94 | 'E_DEC':0, 95 | 'Total_flux':0, 96 | 'E_Total_flux':0, 97 | 'Peak_flux':0, 98 | 'E_Peak_flux':0, 99 | 'Xposn':0, 100 | 'E_Xposn':0, 101 | 'Yposn':0, 102 | 'E_Yposn':0, 103 | 'Maj':0, 104 | 'E_Maj':0, 105 | 'Min':0, 106 | 'E_Min':0, 107 | 'PA':0, 108 | 'E_PA':0, 109 | 'Maj_img_plane':0, 110 | 'E_Maj_img_plane':0, 111 | 'Min_img_plane':0, 112 | 'E_Min_img_plane':0, 113 | 'PA_img_plane':0, 114 | 'E_PA_img_plane':0, 115 | 'DC_Maj':0, 116 | 'E_DC_Maj':0, 117 | 'DC_Min':0, 118 | 'E_DC_Min':0, 119 | 'DC_PA':0, 120 | 'E_DC_PA':0, 121 | 'DC_Maj_img_plane':0, 122 | 'E_DC_Maj_img_plane':0, 123 | 'DC_Min_img_plane':0, 124 | 'E_DC_Min_img_plane':0, 125 | 'DC_PA_img_plane':0, 126 | 'E_DC_PA_img_plane':0, 127 | 'Isl_Total_flux':0, 128 | 'E_Isl_Total_flux':0, 129 | 'Isl_rms':0, 130 | 'Isl_mean':0, 131 | 'Resid_Isl_rms':0, 132 | 'Resid_Isl_mean':0, 133 | 'S_Code':0, 134 | 'Total_Q':1, 135 | 'E_Total_Q':1, 136 | 'Total_U':1, 137 | 'E_Total_U':1, 138 | 'Total_V':1, 139 | 'E_Total_V':1, 140 | 'Linear_Pol_frac':1, 141 | 'Elow_Linear_Pol_frac':1, 142 | 'Ehigh_Linear_Pol_frac':1, 143 | 'Circ_Pol_Frac':1, 144 | 'Elow_Circ_Pol_Frac':1, 145 | 'Ehigh_Circ_Pol_Frac':1, 146 | 'Total_Pol_Frac':1, 147 | 'Elow_Total_Pol_Frac':1, 148 | 'Ehigh_Total_Pol_Frac':1, 149 | 'Linear_Pol_Ang':1, 150 | 'E_Linear_Pol_Ang':1} 151 | 152 | 153 | lines = [line.strip() for line in open(gaul)] 154 | 155 | 156 | for line in range(len(lines)): 157 | if lines[line]: 158 | if lines[line].split()[0] is not '#': 159 | gaul_params = lines[line-1].split()[1:] 160 | break 161 | 162 | 163 | lsm_params_general = [] 164 | lsm_params_polarization = [] 165 | 166 | 167 | for param in gaul_params: 168 | if dict_pol_flag[param] is 0: 169 | lsm_params_general.append(dict_gaul2lsm[param]) 170 | if dict_pol_flag[param] is 1: 171 | lsm_params_polarization.append(dict_gaul2lsm[param]) 172 | 173 | 174 | general_params_string = ' '.join(lsm_params_general) 175 | pol_params_string = ' '.join(lsm_params_polarization) 176 | 177 | 178 | output = gaul.replace('.gaul','.lsm.html') 179 | 180 | 181 | cluster = 800.0 182 | 183 | 184 | syscall = 'tigger-convert -t ASCII --format "' 185 | syscall += general_params_string+'" ' 186 | syscall += '-f --rename --cluster-dist '+str(cluster)+' ' 187 | syscall += gaul 188 | 189 | os.system(syscall) 190 | 191 | return output 192 | 193 | 194 | def writeDS9(nodes,regionfile): 195 | f = open(regionfile,'w') 196 | f.write('# Region file format: DS9 version 4.1\n') 197 | f.write('global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1\n') 198 | f.write('fk5\n') 199 | for i in range(0,len(nodes)): 200 | ra = nodes[i][0] 201 | dec = nodes[i][1] 202 | ra_hms = ac.decimal2hms(ra,delimiter=':') 203 | dec_dms = ac.decimal2dms(dec,delimiter=':') 204 | f.write('circle('+ra_hms+','+dec_dms+',1.0)\n') 205 | f.close() 206 | 207 | 208 | def main(): 209 | 210 | 211 | infile = glob.glob('*.gaul')[0] 212 | 213 | # infile = sys.argv[1] 214 | 215 | regionfile = infile.replace('.gaul','.reg') 216 | 217 | 218 | tiggerConvert(infile) 219 | 220 | 221 | model = Tigger.load(infile.replace('gaul','lsm.html')) 222 | 223 | nodes = [] 224 | 225 | for src in model.sources: 226 | lead = src.getTag('cluster_lead') 227 | if lead: 228 | cluster_flux = src.getTag('cluster_flux') 229 | if cluster_flux > 0.08: 230 | ra = rad2deg(src.pos.ra) 231 | dec = rad2deg(src.pos.dec) 232 | nodes.append((ra,dec)) 233 | # print src.name,ra,dec,'1.0' 234 | 235 | writeDS9(nodes,regionfile) 236 | 237 | 238 | if __name__ == "__main__": 239 | 240 | 241 | main() 242 | 243 | 244 | -------------------------------------------------------------------------------- /oxkat/3GC_split_model_images.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | import glob 6 | import numpy 7 | import shutil 8 | from astropy import wcs 9 | from astropy.io import fits 10 | from optparse import OptionParser 11 | 12 | 13 | # --------------------------------------------------------------------------------------- 14 | 15 | 16 | def hms2deg(hms,delimiter=':'): 17 | 18 | """ 19 | Right ascention string in hms to float in decimal degrees 20 | """ 21 | 22 | h,m,s = hms.split(delimiter) 23 | h = float(h) 24 | m = float(m) 25 | s = float(s) 26 | deg = 15.0*(h+(m/60.0)+(s/3600.0)) 27 | return deg 28 | 29 | 30 | def dms2deg(dms,delimiter=':'): 31 | 32 | """ 33 | Declination string in dms to float in decimal degrees 34 | """ 35 | 36 | d,m,s = dms.split(delimiter) 37 | if d[0] == '-': 38 | decsign = -1.0 39 | d = float(d[1:]) 40 | elif d[0] == '+': 41 | decsign = 1.0 42 | d = float(d[1:]) 43 | else: 44 | decsign = 1.0 45 | d = float(d) 46 | m = float(m) 47 | s = float(s) 48 | deg = decsign*(d+(m/60.0)+(s/3600.0)) 49 | 50 | return deg 51 | 52 | 53 | def radius2deg(radius): 54 | 55 | """ 56 | String with arcsec or arcmin unit to decimal degrees 57 | """ 58 | 59 | if radius[-1] == '"': 60 | radius = float(radius[:-1])/3600.0 61 | elif radius[-1] == "'": 62 | radius = float(radius[:-1])/60.0 63 | else: 64 | radius = float(radius) 65 | 66 | return radius 67 | 68 | 69 | def process_region_file(region_file): 70 | 71 | """ 72 | Extract RA,dec,radius as floats in degrees 73 | from a DS9 region file containing circles 74 | """ 75 | 76 | circles = [] 77 | 78 | f = open(region_file,'r') 79 | line = f.readline() 80 | while line: 81 | if line[0:6] == 'circle': 82 | line = line.replace(' ','') 83 | line = line.rstrip('\n').replace('(',' ').replace(')',' ') 84 | ra,dec,radius = line.split()[1].split(',') 85 | if ':' in ra: 86 | ra = hms2deg(ra) 87 | else: 88 | ra = float(ra) 89 | if ':' in dec: 90 | dec = dms2deg(dec) 91 | else: 92 | dec = float(dec) 93 | radius = radius2deg(radius) 94 | circles.append((ra,dec,radius)) 95 | line = f.readline() 96 | f.close() 97 | 98 | return circles 99 | 100 | 101 | def get_image(fits_file): 102 | 103 | """ 104 | Get the image data from a FITS file 105 | """ 106 | 107 | input_hdu = fits.open(fits_file)[0] 108 | if len(input_hdu.data.shape) == 2: 109 | image = numpy.array(input_hdu.data[:,:]) 110 | elif len(input_hdu.data.shape) == 3: 111 | image = numpy.array(input_hdu.data[0,:,:]) 112 | else: 113 | image = numpy.array(input_hdu.data[0,0,:,:]) 114 | 115 | return image 116 | 117 | 118 | def flush_fits(image,fits_file): 119 | 120 | """ 121 | Write 2D numpy array image to fits_file 122 | """ 123 | 124 | f = fits.open(fits_file,mode='update') 125 | input_hdu = f[0] 126 | if len(input_hdu.data.shape) == 2: 127 | input_hdu.data[:,:] =image 128 | elif len(input_hdu.data.shape) == 3: 129 | input_hdu.data[0,:,:] = image 130 | else: 131 | input_hdu.data[0,0,:,:] = image 132 | f.flush() 133 | 134 | 135 | def apply_circle(image,xpix,ypix,rpix): 136 | 137 | """ 138 | Apply a circle with values of 1, of radius rpix to xpix,ypix in image array 139 | """ 140 | 141 | xg,yg = numpy.mgrid[int(xpix-rpix):int(xpix+rpix)+1,int(ypix-rpix):int(ypix+rpix)+1] 142 | xg = xg.ravel() 143 | yg = yg.ravel() 144 | 145 | for i,j in zip(xg,yg): 146 | sep = ((i-xpix)**2.0 + (j-ypix)**2.0)**0.5 147 | if sep < rpix: 148 | image[j,i] = 1.0 149 | 150 | return image 151 | 152 | 153 | def fmt(xx): 154 | return str(round(xx,5)) 155 | 156 | 157 | def spacer(): 158 | print('--------------|---------------------------------------------') 159 | 160 | 161 | # --------------------------------------------------------------------------------------- 162 | 163 | 164 | def main(): 165 | 166 | parser = OptionParser(usage = '%prog [options]') 167 | parser.add_option('--region', dest = 'region_file', help = 'DS9 region file') 168 | parser.add_option('--prefix', dest = 'model_pattern', help = 'wsclean image prefix') 169 | parser.add_option('--subtract', dest = 'subtract', help = 'Produce model image with components within region subtracted (default = False)', action = 'store_true', default = False) 170 | (options,args) = parser.parse_args() 171 | region_file = options.region_file 172 | model_pattern = options.model_pattern 173 | subtract = options.subtract 174 | 175 | circles = process_region_file(region_file) 176 | suffix = region_file.split('/')[-1].split('.')[0] 177 | 178 | spacer() 179 | print('DS9 region : '+region_file) 180 | print('Contains : '+str(len(circles))+' circles') 181 | print('Model suffix : '+suffix) 182 | spacer() 183 | 184 | model_list = sorted(glob.glob(model_pattern+'-0*model*fits')) 185 | 186 | for fits_file in model_list: 187 | 188 | print('Reading : '+fits_file) 189 | 190 | dir1_fits = fits_file.replace(model_pattern,model_pattern+'-'+suffix) 191 | 192 | if subtract: 193 | subtract_fits = fits_file.replace(model_pattern,model_pattern+'-'+suffix+'-subtracted') 194 | 195 | img = get_image(fits_file) 196 | mask = img*0.0 197 | 198 | hdulist = fits.open(fits_file) 199 | w = wcs.WCS(hdulist[0].header) 200 | ref_pix1 = hdulist[0].header['CRPIX1'] 201 | ref_pix2 = hdulist[0].header['CRPIX2'] 202 | pixscale = hdulist[0].header['CDELT2'] 203 | 204 | for circle in circles: 205 | ra = circle[0] 206 | dec = circle[1] 207 | coord = (ra,dec,0,0) 208 | pixels = w.wcs_world2pix([coord],0) 209 | radius = circle[2] 210 | xpix = pixels[0][0] 211 | ypix = pixels[0][1] 212 | rpix = radius/pixscale 213 | print('Masking : sky '+fmt(ra)+' '+fmt(dec)+' '+fmt(radius)) 214 | print(' : pixel '+fmt(xpix)+' '+fmt(ypix)+' '+fmt(rpix)) 215 | mask = apply_circle(mask,xpix,ypix,rpix) 216 | 217 | dir1 = img*mask 218 | 219 | print('Writing : '+dir1_fits) 220 | shutil.copyfile(fits_file,dir1_fits) 221 | flush_fits(dir1,dir1_fits) 222 | 223 | if subtract: 224 | subt = img*(1.0-mask) 225 | print('Writing : '+subtract_fits) 226 | shutil.copyfile(fits_file,subtract_fits) 227 | flush_fits(subt,subtract_fits) 228 | 229 | 230 | spacer() 231 | 232 | 233 | if __name__ == '__main__': 234 | 235 | main() 236 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![](https://i.imgur.com/u3bvu9n.png) 2 | # oxkat 3 | 4 | "Ceci n'est pas une pipe[line]"
5 | _The Treachery of Images_, René Magritte, 1929 6 | 7 | --- 8 | 9 | * [What is this?](README.md#what-is-this) 10 | * [Quick start](README.md#quick-start) 11 | * [Containers](README.md#containers) 12 | * [Software roll-call](README.md#software-package-roll-call) 13 | 14 | --- 15 | 16 | ## What is this? 17 | 18 | 19 | * A set of `Python` scripts with the aim of (semi-)automatically processing [MeerKAT](https://www.sarao.ac.za/science-engineering/meerkat/) data. 20 | 21 | 22 | * At the core is a set of functions that generate calls to [various pieces](README.md#software-package-roll-call) of radio astronomy software, a semi-modular bunch of CASA scripts for performing reference calibration, and a [fairly sizeable list](oxkat/config.py) of default parameters. The defaults at present cater for full-band Stokes-I continuum imaging, including the correction of direction-dependent effects. 23 | 24 | 25 | * Job script generation and dependency chains are automatically handled when running on the [ilifu/IDIA](https://www.idia.ac.za/) cluster, UKZN's [hippo](https://astro.ukzn.ac.za/~hippo/) cluster, or the [CHPC](https://www.chpc.ac.za/)'s [Lengau](https://www.chpc.ac.za/index.php/resources/lengau-cluster) cluster. 26 | 27 | 28 | * Setup scripts glue the above components together into a processing recipe. The default procedure is broken down into stages, after each of which it is advisable to pause and examine the state of the process before continuing. 29 | 30 | 31 | * The intention is that the bar to entry is low. If you have stock `Python` then nothing else needs installing apart from [`Singularity`](https://github.com/hpcng/singularity), which is available on both the ilifu/IDIA and CHPC clusters. All the underlying radio astronomy packages are containerised. The `Singularity` layer can also be disabled for running installations on your own machine, either directly, or inside a Python virtual environment. 32 | 33 | 34 | * If you publish results that have made use of `oxkat` then [please cite the ACSL entry](https://ui.adsabs.harvard.edu/abs/2020ascl.soft09003H/abstract), and (more importantly) the [underlying packages](README.md#software-package-roll-call) used. 35 | 36 | 37 | * Please file bugs, suggestions, questions, etc. as [issues](https://github.com/IanHeywood/oxkat/issues). 38 | 39 | 40 | --- 41 | 42 | ## Quick start 43 | 44 | 1. Once you have the [container(s) in place](README.md#containers) then log into your machine or cluster, e.g.: 45 | 46 | ``` 47 | $ ssh ianh@slurm.ilifu.ac.za 48 | ``` 49 | 50 | 2. Navigate to a working area / scratch space: 51 | 52 | ``` 53 | $ cd /scratch/users/ianh/XMM12 54 | ``` 55 | 56 | 3. Clone the root contents of this repo into it: 57 | 58 | ``` 59 | $ git clone https://github.com/IanHeywood/oxkat.git . 60 | ``` 61 | 62 | 4. Make a symlink to your MeerKAT Measurement Set (or place it in the working folder, it will not be modified at all): 63 | 64 | ``` 65 | $ ln -s /idia/projects/mightee/1538856059/1538856059_sdp_l0.full_1284.full_pol.ms . 66 | ``` 67 | 68 | 5. The first step is to run a script that gathers some required information about the observation: 69 | 70 | ``` 71 | $ python setups/0_GET_INFO.py idia 72 | $ ./submit_info_job.sh 73 | ``` 74 | 75 | 6. Once this is complete you can generate and submit the jobs required for the reference calibration (1GC): 76 | 77 | ``` 78 | $ python setups/1GC.py idia 79 | $ ./submit_1GC_jobs.sh 80 | ``` 81 | 82 | 7. If something goes wrong you can kill the running and queued jobs on a cluster with: 83 | 84 | ``` 85 | $ source SCRIPTS/kill_1GC_jobs.sh 86 | ``` 87 | 88 | 8. Once all the jobs have completed then you can examine the products, and move on to the setup for the next steps in the same fashion. 89 | 90 | Please see the [setups README](setups/README.md) for more details about the general workflow. Most of the settings can be tuned via the [`config.py`](oxkat/config.py) file. 91 | 92 | --- 93 | 94 | ## Containers 95 | 96 | There is a dedicated `Singularity` container (`oxkat-0.41.sif`) available that contains all the necessary packages and dependencies. This is available in the general container repository on the ilifu/IDIA cluster, and the default settings should pick it up automatically when that cluster is being used. For other systems the container will have to be downloaded (or copied over). The container can be downloaded [here](https://entangled.physics.ox.ac.uk/index.php/s/jmHRBQyyB6Zm2fj). 97 | 98 | String patterns for package-specific containers are specified in the [`config.py`](oxkat/config.py) file. The scripts will search for containers that match these patterns in the container paths, so it's simple to swap a particular package out for a different version as long as you have it containerised. 99 | 100 | --- 101 | 102 | ## Software package roll-call 103 | 104 | 105 | | Package | Stage | Purpose | Reference | 106 | | --- | --- | --- | --- | 107 | | [`astropy`](https://www.astropy.org/) | 1GC, 3GC | Coordinates, time standards, FITS file manipulation | [Astropy Collaboration, 2013](https://ui.adsabs.harvard.edu/abs/2013A%26A...558A..33A/abstract), [Astropy Collaboration, 2018](https://ui.adsabs.harvard.edu/abs/2018AJ....156..123A/abstract)| 108 | | [`CASA`](https://casa.nrao.edu/) | 1GC | Averaging, splitting, cross calibration, DI self-calibration, flagging | [McMullin et al., 2007](https://ui.adsabs.harvard.edu/abs/2007ASPC..376..127M/abstract)| 109 | | [`CubiCal`](https://github.com/ratt-ru/CubiCal) | 2GC, 3GC | DI / DD self-calibration | [Kenyon et al., 2018](https://ui.adsabs.harvard.edu/abs/2018MNRAS.478.2399K/abstract)| 110 | | [`DDFacet`](https://github.com/saopicc/DDFacet) | 3GC | Imaging with direction-dependent corrections | [Tasse et al., 2018](https://ui.adsabs.harvard.edu/abs/2018A%26A...611A..87T/abstract) | 111 | | [`killMS`](https://github.com/saopicc/killMS) | 3GC | DD self-calibration| [Tasse, 2014](https://ui.adsabs.harvard.edu/abs/2014A%26A...566A.127T/abstract); [Smirnov & Tasse, 2014](https://ui.adsabs.harvard.edu/abs/2015MNRAS.449.2668S/abstract) | 112 | | [`owlcat`](https://github.com/ska-sa/owlcat/) | 2GC, 3GC | FITS file manipulation | - | 113 | | [`ragavi`](https://github.com/ratt-ru/ragavi/) | 1GC, 2GC | Plotting gain solutions | - | 114 | | [`shadeMS`](https://github.com/ratt-ru/shadeMS/) | 1GC | Plotting visibilities | [Smirnov et al., 2022](https://ui.adsabs.harvard.edu/abs/2022ASPC..532..385S/abstract) | 115 | | [`Singularity`](https://github.com/hpcng/singularity) | 1GC, FLAG, 2GC, 3GC | Containerisation | [Kurtzer, Sochat & Bauer, 2017](https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0177459) | 116 | | [`tricolour`](https://github.com/ska-sa/tricolour) | FLAG | Flagging | [Hugo et al., 2022](https://ui.adsabs.harvard.edu/abs/2022arXiv220609179H/abstract) | 117 | | [`wsclean`](https://gitlab.com/aroffringa/wsclean) | FLAG, 2GC, 3GC | Imaging, model prediction | [Offringa et al., 2014](https://ui.adsabs.harvard.edu/abs/2014MNRAS.444..606O/abstract)| 118 | 119 | --- 120 | 121 | Thank you for visiting. 122 | 123 | 124 | -------------------------------------------------------------------------------- /tools/mask_FITS_with_region.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | import glob 6 | import numpy 7 | import shutil 8 | from astropy import wcs 9 | from astropy.io import fits 10 | from optparse import OptionParser 11 | 12 | 13 | # --------------------------------------------------------------------------------------- 14 | 15 | 16 | def hms2deg(hms,delimiter=':'): 17 | 18 | """ 19 | Right ascention string in hms to float in decimal degrees 20 | """ 21 | 22 | h,m,s = hms.split(delimiter) 23 | h = float(h) 24 | m = float(m) 25 | s = float(s) 26 | deg = 15.0*(h+(m/60.0)+(s/3600.0)) 27 | return deg 28 | 29 | 30 | def dms2deg(dms,delimiter=':'): 31 | 32 | """ 33 | Declination string in dms to float in decimal degrees 34 | """ 35 | 36 | d,m,s = dms.split(delimiter) 37 | if d[0] == '-': 38 | decsign = -1.0 39 | d = float(d[1:]) 40 | elif d[0] == '+': 41 | decsign = 1.0 42 | d = float(d[1:]) 43 | else: 44 | decsign = 1.0 45 | d = float(d) 46 | m = float(m) 47 | s = float(s) 48 | deg = decsign*(d+(m/60.0)+(s/3600.0)) 49 | 50 | return deg 51 | 52 | 53 | def radius2deg(radius): 54 | 55 | """ 56 | String with arcsec or arcmin unit to decimal degrees 57 | """ 58 | 59 | radius = radius.replace(' ','') 60 | if radius[-1] == '"': 61 | radius = float(radius[:-1])/3600.0 62 | elif radius[-1] == "'": 63 | radius = float(radius[:-1])/60.0 64 | else: 65 | radius = float(radius) 66 | 67 | return radius 68 | 69 | 70 | def process_region_file(region_file): 71 | 72 | """ 73 | Extract RA,dec,radius as floats in degrees 74 | from a DS9 region file containing circles 75 | """ 76 | 77 | circles = [] 78 | 79 | f = open(region_file,'r') 80 | line = f.readline() 81 | while line: 82 | if line[0:6] == 'circle': 83 | line = line.rstrip('\n').replace('(',' ').replace(')',' ').split('#')[0].replace('circle','') 84 | print(line) 85 | # ra,dec,radius = line.split()[1].split(',') 86 | ra,dec,radius = line.split(',') 87 | if ':' in ra: 88 | ra = hms2deg(ra) 89 | else: 90 | ra = float(ra) 91 | if ':' in dec: 92 | dec = dms2deg(dec) 93 | else: 94 | dec = float(dec) 95 | radius = radius2deg(radius) 96 | circles.append((ra,dec,radius)) 97 | line = f.readline() 98 | f.close() 99 | 100 | return circles 101 | 102 | 103 | def get_image(fits_file): 104 | 105 | """ 106 | Get the image data from a FITS file 107 | """ 108 | 109 | input_hdu = fits.open(fits_file)[0] 110 | if len(input_hdu.data.shape) == 2: 111 | image = numpy.array(input_hdu.data[:,:]) 112 | elif len(input_hdu.data.shape) == 3: 113 | image = numpy.array(input_hdu.data[0,:,:]) 114 | else: 115 | image = numpy.array(input_hdu.data[0,0,:,:]) 116 | 117 | return image 118 | 119 | 120 | def flush_fits(image,fits_file): 121 | 122 | """ 123 | Write 2D numpy array image to fits_file 124 | """ 125 | 126 | f = fits.open(fits_file,mode='update') 127 | input_hdu = f[0] 128 | if len(input_hdu.data.shape) == 2: 129 | input_hdu.data[:,:] =image 130 | elif len(input_hdu.data.shape) == 3: 131 | input_hdu.data[0,:,:] = image 132 | else: 133 | input_hdu.data[0,0,:,:] = image 134 | f.flush() 135 | 136 | 137 | def apply_circle(image,xpix,ypix,rpix,invert): 138 | 139 | """ 140 | Apply a circle with values of 1, of radius rpix to xpix,ypix in image array 141 | """ 142 | 143 | xg,yg = numpy.mgrid[int(xpix-rpix):int(xpix+rpix)+1,int(ypix-rpix):int(ypix+rpix)+1] 144 | xg = xg.ravel() 145 | yg = yg.ravel() 146 | 147 | for i,j in zip(xg,yg): 148 | sep = ((i-xpix)**2.0 + (j-ypix)**2.0)**0.5 149 | if sep < rpix: 150 | if invert: 151 | image[j,i] = 0.0 152 | else: 153 | image[j,i] = 1.0 154 | 155 | return image 156 | 157 | 158 | def fmt(xx): 159 | return str(round(xx,5)) 160 | 161 | 162 | def spacer(): 163 | print('--------------|---------------------------------------------') 164 | 165 | 166 | # --------------------------------------------------------------------------------------- 167 | 168 | 169 | def main(): 170 | 171 | parser = OptionParser(usage = '%prog [options]') 172 | parser.add_option('--region', dest = 'region_file', help = 'DS9 region file') 173 | parser.add_option('--fitsfile', dest = 'fits_file', help = 'FITS image') 174 | parser.add_option('--invert',dest = 'invert', help = 'Remove region instead of only keeping it', action = 'store_true', default = False) 175 | parser.add_option('--writemask', dest = 'writemask', help = 'Write out the mask as a FITS file', action = 'store_true', default = False) 176 | (options,args) = parser.parse_args() 177 | region_file = options.region_file 178 | fits_file = options.fits_file 179 | invert = options.invert 180 | writemask = options.writemask 181 | 182 | circles = process_region_file(region_file) 183 | suffix = region_file.split('/')[-1].split('.')[0] 184 | 185 | spacer() 186 | print('DS9 region : '+region_file) 187 | print('Contains : '+str(len(circles))+' circles') 188 | print('Model suffix : '+suffix) 189 | spacer() 190 | 191 | 192 | print('Reading : '+fits_file) 193 | 194 | masked_fits = fits_file.replace('.fits','-'+suffix+'.fits') 195 | mask_fits = masked_fits.replace('.fits','.mask.fits') 196 | 197 | img = get_image(fits_file) 198 | 199 | mask = img*0.0 200 | 201 | if invert: 202 | mask += 1 203 | 204 | hdulist = fits.open(fits_file) 205 | w = wcs.WCS(hdulist[0].header) 206 | ref_pix1 = hdulist[0].header['CRPIX1'] 207 | ref_pix2 = hdulist[0].header['CRPIX2'] 208 | pixscale = hdulist[0].header['CDELT2'] 209 | 210 | for circle in circles: 211 | ra = circle[0] 212 | dec = circle[1] 213 | coord = (ra,dec,0,0) 214 | pixels = w.wcs_world2pix([coord],0) 215 | radius = circle[2] 216 | xpix = pixels[0][0] 217 | ypix = pixels[0][1] 218 | rpix = radius/pixscale 219 | print('Masking : sky '+fmt(ra)+' '+fmt(dec)+' '+fmt(radius)) 220 | print(' : pixel '+fmt(xpix)+' '+fmt(ypix)+' '+fmt(rpix)) 221 | mask = apply_circle(mask,xpix,ypix,rpix,invert) 222 | 223 | masked_img = img*mask 224 | 225 | # if invert: 226 | # masked_img = numpy.logical_and(img,mask) 227 | # else: 228 | # masked_img = numpy.logical_or(img,mask) 229 | 230 | print('Writing : '+masked_fits) 231 | shutil.copyfile(fits_file,masked_fits) 232 | flush_fits(masked_img,masked_fits) 233 | 234 | if writemask: 235 | print('Writing : '+mask_fits) 236 | shutil.copyfile(fits_file,mask_fits) 237 | flush_fits(mask,mask_fits) 238 | 239 | 240 | spacer() 241 | 242 | 243 | if __name__ == '__main__': 244 | 245 | main() 246 | -------------------------------------------------------------------------------- /waterhole/setup_FLAG_only.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | import glob 6 | import json 7 | import os.path as o 8 | import sys 9 | sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) 10 | 11 | 12 | from oxkat import generate_jobs as gen 13 | from oxkat import config as cfg 14 | 15 | 16 | def main(): 17 | 18 | USE_SINGULARITY = cfg.USE_SINGULARITY 19 | 20 | gen.preamble() 21 | print(gen.col()+'FLAG (target flagging) setup') 22 | gen.print_spacer() 23 | 24 | # ------------------------------------------------------------------------------ 25 | # 26 | # Setup paths, required containers, infrastructure 27 | # 28 | # ------------------------------------------------------------------------------ 29 | 30 | 31 | OXKAT = cfg.OXKAT 32 | DATA = cfg.DATA 33 | SCRIPTS = cfg.SCRIPTS 34 | TOOLS = cfg.TOOLS 35 | 36 | 37 | gen.setup_dir(cfg.LOGS) 38 | gen.setup_dir(cfg.SCRIPTS) 39 | 40 | 41 | INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) 42 | if CONTAINER_PATH is not None: 43 | CONTAINER_RUNNER='singularity exec ' 44 | else: 45 | CONTAINER_RUNNER='' 46 | 47 | 48 | CASA_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.CASA_PATTERN,USE_SINGULARITY) 49 | TRICOLOUR_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.TRICOLOUR_PATTERN,USE_SINGULARITY) 50 | 51 | 52 | # Get target information from project info json file 53 | 54 | with open('project_info.json') as f: 55 | project_info = json.load(f) 56 | 57 | target_ids = project_info['target_ids'] 58 | target_names = project_info['target_names'] 59 | target_ms = project_info['target_ms'] 60 | 61 | 62 | # ------------------------------------------------------------------------------ 63 | # 64 | # FLAG recipe definition 65 | # 66 | # ------------------------------------------------------------------------------ 67 | 68 | 69 | target_steps = [] 70 | codes = [] 71 | ii = 1 72 | 73 | # Loop over targets 74 | 75 | for tt in range(0,len(target_ids)): 76 | 77 | targetname = target_names[tt] 78 | myms = target_ms[tt] 79 | 80 | if not o.isdir(myms): 81 | 82 | gen.print_spacer() 83 | print(gen.col('Target')+targetname) 84 | print(gen.col('MS')+'not found, skipping') 85 | 86 | else: 87 | 88 | steps = [] 89 | filename_targetname = gen.scrub_target_name(targetname) 90 | 91 | code = gen.get_target_code(targetname) 92 | if code in codes: 93 | code += '_'+str(ii) 94 | ii += 1 95 | codes.append(code) 96 | 97 | # Target-specific kill file 98 | kill_file = SCRIPTS+'/kill_flag_jobs_'+filename_targetname+'.sh' 99 | 100 | gen.print_spacer() 101 | print(gen.col('Target')+targetname) 102 | print(gen.col('Measurement Set')+myms) 103 | print(gen.col('Code')+code) 104 | 105 | 106 | step = {} 107 | step['step'] = 0 108 | step['comment'] = 'Run Tricolour on '+myms 109 | step['dependency'] = None 110 | step['id'] = 'TRIC0'+code 111 | step['slurm_config'] = cfg.SLURM_TRICOLOUR 112 | step['pbs_config'] = cfg.PBS_TRICOLOUR 113 | syscall = CONTAINER_RUNNER+TRICOLOUR_CONTAINER+' ' if USE_SINGULARITY else '' 114 | syscall += gen.generate_syscall_tricolour(myms = myms, 115 | config = DATA+'/tricolour/target_flagging_1_narrow.yaml', 116 | datacol = 'DATA', 117 | fields = '0', 118 | strategy = 'polarisation') 119 | step['syscall'] = syscall 120 | steps.append(step) 121 | 122 | 123 | step = {} 124 | step['step'] = 1 125 | step['comment'] = 'Backup flag table for '+myms 126 | step['dependency'] = 0 127 | step['id'] = 'SAVFG'+code 128 | syscall = CONTAINER_RUNNER+CASA_CONTAINER+' ' if USE_SINGULARITY else '' 129 | syscall += 'casa -c '+OXKAT+'/FLAG_casa_backup_flag_table.py --nologger --log2term --nogui ' 130 | syscall += 'versionname=tricolour1 mslist='+myms 131 | step['syscall'] = syscall 132 | steps.append(step) 133 | 134 | 135 | target_steps.append((steps,kill_file,targetname)) 136 | 137 | 138 | # ------------------------------------------------------------------------------ 139 | # 140 | # Write the run file and kill file based on the recipe 141 | # 142 | # ------------------------------------------------------------------------------ 143 | 144 | 145 | submit_file = 'submit_flag_jobs.sh' 146 | 147 | f = open(submit_file,'w') 148 | f.write('#!/usr/bin/env bash\n') 149 | f.write('export SINGULARITY_BINDPATH='+cfg.BINDPATH+'\n') 150 | 151 | for content in target_steps: 152 | steps = content[0] 153 | kill_file = content[1] 154 | targetname = content[2] 155 | id_list = [] 156 | 157 | f.write('\n#---------------------------------------\n') 158 | f.write('# '+targetname) 159 | f.write('\n#---------------------------------------\n') 160 | 161 | for step in steps: 162 | 163 | step_id = step['id'] 164 | id_list.append(step_id) 165 | if step['dependency'] is not None: 166 | dependency = steps[step['dependency']]['id'] 167 | else: 168 | dependency = None 169 | syscall = step['syscall'] 170 | if 'slurm_config' in step.keys(): 171 | slurm_config = step['slurm_config'] 172 | else: 173 | slurm_config = cfg.SLURM_DEFAULTS 174 | if 'pbs_config' in step.keys(): 175 | pbs_config = step['pbs_config'] 176 | else: 177 | pbs_config = cfg.PBS_DEFAULTS 178 | comment = step['comment'] 179 | 180 | run_command = gen.job_handler(syscall = syscall, 181 | jobname = step_id, 182 | infrastructure = INFRASTRUCTURE, 183 | dependency = dependency, 184 | slurm_config = slurm_config, 185 | pbs_config = pbs_config) 186 | 187 | 188 | f.write('\n# '+comment+'\n') 189 | f.write(run_command) 190 | 191 | if INFRASTRUCTURE != 'node': 192 | f.write('\n# Generate kill script for '+targetname+'\n') 193 | if INFRASTRUCTURE == 'idia' or INFRASTRUCTURE == 'hippo': 194 | kill = 'echo "scancel "$'+'" "$'.join(id_list)+' > '+kill_file+'\n' 195 | f.write(kill) 196 | elif INFRASTRUCTURE == 'chpc': 197 | kill = 'echo "qdel "$'+'" "$'.join(id_list)+' > '+kill_file+'\n' 198 | f.write(kill) 199 | 200 | 201 | f.close() 202 | 203 | gen.make_executable(submit_file) 204 | 205 | gen.print_spacer() 206 | print(gen.col('Run file')+submit_file) 207 | gen.print_spacer() 208 | 209 | # ------------------------------------------------------------------------------ 210 | 211 | 212 | 213 | if __name__ == "__main__": 214 | 215 | 216 | main() 217 | -------------------------------------------------------------------------------- /setups/1GC.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ian.heywood@physics.ox.ac.uk 3 | 4 | 5 | import glob 6 | import json 7 | import os.path as o 8 | import sys 9 | sys.path.append(o.abspath(o.join(o.dirname(sys.modules[__name__].__file__), ".."))) 10 | 11 | 12 | from oxkat import generate_jobs as gen 13 | from oxkat import config as cfg 14 | 15 | 16 | def main(): 17 | 18 | USE_SINGULARITY = cfg.USE_SINGULARITY 19 | 20 | gen.preamble() 21 | print(gen.col()+'1GC (referenced calibration) setup') 22 | gen.print_spacer() 23 | 24 | if cfg.PRE_FIELDS != '': 25 | print(gen.col('Field selection')+cfg.PRE_FIELDS) 26 | if cfg.PRE_SCANS != '': 27 | print(gen.col('Scan selection')+cfg.PRE_SCANS) 28 | gen.print_spacer() 29 | 30 | # ------------------------------------------------------------------------------ 31 | # 32 | # Setup paths, required containers, infrastructure 33 | # 34 | # ------------------------------------------------------------------------------ 35 | 36 | 37 | gen.setup_dir(cfg.LOGS) 38 | gen.setup_dir(cfg.SCRIPTS) 39 | gen.setup_dir(cfg.GAINTABLES) 40 | 41 | 42 | INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) 43 | if CONTAINER_PATH is not None: 44 | CONTAINER_RUNNER='singularity exec ' 45 | else: 46 | CONTAINER_RUNNER='' 47 | 48 | 49 | CASA_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.CASA_PATTERN,USE_SINGULARITY) 50 | OWLCAT_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.OWLCAT_PATTERN,USE_SINGULARITY) 51 | RAGAVI_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.RAGAVI_PATTERN,USE_SINGULARITY) 52 | SHADEMS_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.SHADEMS_PATTERN,USE_SINGULARITY) 53 | 54 | 55 | # ------------------------------------------------------------------------------ 56 | # 57 | # 1GC recipe definition 58 | # 59 | # ------------------------------------------------------------------------------ 60 | 61 | 62 | with open('project_info.json') as f: 63 | project_info = json.load(f) 64 | 65 | myms = project_info['master_ms'] 66 | code = gen.get_code(myms) 67 | 68 | steps = [] 69 | 70 | 71 | step = {} 72 | step['step'] = 0 73 | step['comment'] = 'Split and average master MS' 74 | step['dependency'] = None 75 | step['id'] = 'SPPRE'+code 76 | syscall = CONTAINER_RUNNER+CASA_CONTAINER+' ' if USE_SINGULARITY else '' 77 | syscall += gen.generate_syscall_casa(casascript=cfg.OXKAT+'/PRE_casa_average_to_1k_add_wtspec.py') 78 | step['syscall'] = syscall 79 | steps.append(step) 80 | 81 | 82 | step = {} 83 | step['step'] = 1 84 | step['comment'] = 'Apply basic flagging steps to all fields' 85 | step['dependency'] = 0 86 | step['id'] = 'FGBAS'+code 87 | syscall = CONTAINER_RUNNER+CASA_CONTAINER+' ' if USE_SINGULARITY else '' 88 | syscall += gen.generate_syscall_casa(casascript=cfg.OXKAT+'/1GC_02_casa_basic_flags.py') 89 | step['syscall'] = syscall 90 | steps.append(step) 91 | 92 | 93 | step = {} 94 | step['step'] = 2 95 | step['comment'] = 'Run auto-flaggers on calibrators' 96 | step['dependency'] = 1 97 | step['id'] = 'FGCAL'+code 98 | syscall = CONTAINER_RUNNER+CASA_CONTAINER+' ' if USE_SINGULARITY else '' 99 | syscall += gen.generate_syscall_casa(casascript=cfg.OXKAT+'/1GC_05_casa_autoflag_cals_DATA.py') 100 | step['syscall'] = syscall 101 | steps.append(step) 102 | 103 | 104 | step = {} 105 | step['step'] = 3 106 | step['comment'] = 'Generate reference calibration solutions and apply to target(s)' 107 | step['dependency'] = 2 108 | step['id'] = 'CL1GC'+code 109 | syscall = CONTAINER_RUNNER+CASA_CONTAINER+' ' if USE_SINGULARITY else '' 110 | syscall += gen.generate_syscall_casa(casascript=cfg.OXKAT+'/1GC_casa_refcal.py') 111 | step['syscall'] = syscall 112 | steps.append(step) 113 | 114 | 115 | step = {} 116 | step['step'] = 4 117 | step['comment'] = 'Plot the gain solutions' 118 | step['dependency'] = 3 119 | step['id'] = 'PLTAB'+code 120 | syscall = CONTAINER_RUNNER+RAGAVI_CONTAINER+' ' if USE_SINGULARITY else '' 121 | syscall += 'python3 '+cfg.OXKAT+'/PLOT_gaintables.py cal_1GC_*' 122 | step['syscall'] = syscall 123 | steps.append(step) 124 | 125 | 126 | step = {} 127 | step['step'] = 5 128 | step['comment'] = 'Split the corrected target data' 129 | step['dependency'] = 3 130 | step['id'] = 'SPTRG'+code 131 | syscall = CONTAINER_RUNNER+CASA_CONTAINER+' ' if USE_SINGULARITY else '' 132 | syscall += gen.generate_syscall_casa(casascript=cfg.OXKAT+'/1GC_09_casa_split_targets.py') 133 | step['syscall'] = syscall 134 | steps.append(step) 135 | 136 | 137 | step = {} 138 | step['step'] = 6 139 | step['comment'] = 'Plot the corrected calibrator visibilities' 140 | step['dependency'] = 5 141 | step['id'] = 'PLVIS'+code 142 | syscall = CONTAINER_RUNNER+SHADEMS_CONTAINER+' ' if USE_SINGULARITY else '' 143 | syscall += 'python3 '+cfg.OXKAT+'/1GC_10_plot_visibilities.py' 144 | step['syscall'] = syscall 145 | steps.append(step) 146 | 147 | 148 | # ------------------------------------------------------------------------------ 149 | # 150 | # Write the run file and kill file based on the recipe 151 | # 152 | # ------------------------------------------------------------------------------ 153 | 154 | 155 | submit_file = 'submit_1GC_jobs.sh' 156 | kill_file = cfg.SCRIPTS+'/kill_1GC_jobs.sh' 157 | 158 | f = open(submit_file,'w') 159 | f.write('#!/usr/bin/env bash\n') 160 | f.write('export SINGULARITY_BINDPATH='+cfg.BINDPATH+'\n') 161 | 162 | id_list = [] 163 | 164 | for step in steps: 165 | 166 | step_id = step['id'] 167 | id_list.append(step_id) 168 | if step['dependency'] is not None: 169 | dependency = steps[step['dependency']]['id'] 170 | else: 171 | dependency = None 172 | syscall = step['syscall'] 173 | if 'slurm_config' in step.keys(): 174 | slurm_config = step['slurm_config'] 175 | else: 176 | slurm_config = cfg.SLURM_DEFAULTS 177 | if 'pbs_config' in step.keys(): 178 | pbs_config = step['pbs_config'] 179 | else: 180 | pbs_config = cfg.PBS_DEFAULTS 181 | comment = step['comment'] 182 | 183 | run_command = gen.job_handler(syscall = syscall, 184 | jobname = step_id, 185 | infrastructure = INFRASTRUCTURE, 186 | dependency = dependency, 187 | slurm_config = slurm_config, 188 | pbs_config = pbs_config) 189 | 190 | 191 | f.write('\n# '+comment+'\n') 192 | f.write(run_command) 193 | 194 | 195 | if INFRASTRUCTURE == 'idia' or INFRASTRUCTURE == 'hippo': 196 | kill = '\necho "scancel "$'+'" "$'.join(id_list)+' > '+kill_file+'\n' 197 | f.write(kill) 198 | elif INFRASTRUCTURE == 'chpc': 199 | kill = '\necho "qdel "$'+'" "$'.join(id_list)+' > '+kill_file+'\n' 200 | f.write(kill) 201 | 202 | f.close() 203 | 204 | gen.make_executable(submit_file) 205 | 206 | gen.print_spacer() 207 | print(gen.col('Run file')+submit_file) 208 | gen.print_spacer() 209 | 210 | # ------------------------------------------------------------------------------ 211 | 212 | 213 | 214 | if __name__ == "__main__": 215 | 216 | 217 | main() --------------------------------------------------------------------------------