├── __init__.py ├── Sensitivity ├── __init__.py └── sensitivity.py ├── README.md ├── sacm ├── __init__.py ├── dicts.py ├── pgutils.py ├── utils.py └── geo_helper.py ├── schedulled.py ├── .gitignore ├── arcs.csv ├── sbExport.py ├── releaseList.txt ├── ict4456.py ├── checkUpdateSACM211.py ├── fieldvsource.py ├── sacm450.py ├── .ipynb_checkpoints └── Trending-checkpoint.ipynb ├── MC.py ├── Trending.ipynb ├── execFraction.py ├── list_csv2999.csv ├── CalculateFieldError.py ├── allList.txt ├── csv2999.csv ├── releaseImage.txt ├── MetaData.py ├── final.csv ├── sacm211.py ├── createReadme.py ├── fixSACM211.py └── checker.py /__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sagonzal' 2 | 3 | -------------------------------------------------------------------------------- /Sensitivity/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sagonzal' 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | sacm 2 | ==== 3 | 4 | ALMA Data management utilities 5 | -------------------------------------------------------------------------------- /sacm/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sdk' 2 | from utils import * 3 | from dicts import * 4 | from pgutils import * 5 | -------------------------------------------------------------------------------- /sacm/dicts.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | __author__ = 'sdk' 3 | 4 | FlagError = {'EmptyUID': False, 5 | 'NullStateID': False} 6 | 7 | -------------------------------------------------------------------------------- /sacm/pgutils.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sagonzal' 2 | from password import * 3 | import psycopg2 4 | 5 | try: 6 | pgconn = psycopg2.connect (host=pghost, database=pguser ,user=pguser, password=pgpassword) 7 | pgcursor = pgconn.cursor() 8 | except Exception as e: 9 | print e 10 | pass 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /schedulled.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sagonzal' 2 | from MetaData import * 3 | import cx_Oracle 4 | from sacm.password import databaseSCO as database 5 | results = list() 6 | sql = '''select EXECBLOCKUID,starttime from ALMA.AQUA_EXECBLOCK where OBSPROJECTCODE not like '%CSV' order by STARTTIME desc''' 7 | orcl = cx_Oracle.connect(database) 8 | cursor = orcl.cursor() 9 | cursor.execute(sql) 10 | 11 | tocheck = AsdmCheck() 12 | for uid in cursor: 13 | print uid[0],uid[1] 14 | try: 15 | tocheck.setUID(uid[0]) 16 | tocheck.doCheck() 17 | tocheck.save() 18 | except Exception as e: 19 | print e 20 | print "Error at ASDM:", tocheck.uid -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled source # 2 | ################### 3 | *.com 4 | *.class 5 | *.dll 6 | *.exe 7 | *.o 8 | *.so 9 | *.pyc 10 | 11 | # Packages # 12 | ############ 13 | # it's better to unpack these files and commit the raw source 14 | # git has its own built in compression methods 15 | *.7z 16 | *.dmg 17 | *.gz 18 | *.iso 19 | *.jar 20 | *.rar 21 | *.tar 22 | *.zip 23 | *.bin 24 | *.tgz 25 | 26 | # Logs and databases # 27 | ###################### 28 | *.log 29 | *.sql 30 | *.sqlite 31 | 32 | # OS generated files # 33 | ###################### 34 | .DS_Store 35 | .DS_Store? 36 | ._* 37 | .Spotlight-V100 38 | .Trashes 39 | ehthumbs.db 40 | Thumbs.db 41 | 42 | .idea/ 43 | *.xml 44 | *.pyc 45 | password* 46 | uid___* 47 | -------------------------------------------------------------------------------- /arcs.csv: -------------------------------------------------------------------------------- 1 | 2012.1.00060.S NA 2 | 2012.1.00080.S EA 3 | 2012.1.00261.S EU 4 | 2012.1.00323.S EU 5 | 2012.1.00333.S EU 6 | 2012.1.00335.S EA 7 | 2012.1.00385.S EU 8 | 2012.1.00394.S CL 9 | 2012.1.00501.S NA 10 | 2012.1.00542.S CL 11 | 2012.1.00554.S EA 12 | 2012.1.00603.S EA 13 | 2012.1.00683.S CL 14 | 2012.1.00762.S EA 15 | 2012.1.00781.S CL 16 | 2012.1.00844.S EU 17 | 2012.1.00932.S NA 18 | 2012.1.00940.S EA 19 | 2012.1.01123.S EA 20 | 2013.1.00214.S EA 21 | 2013.1.00269.S EU 22 | 2013.1.00351.S EU 23 | 2013.1.00617.S EU 24 | 2013.1.00839.S EA 25 | 2013.1.00861.S CL 26 | 2013.1.00914.S EU 27 | 2013.1.00993.S EA 28 | 2013.1.01014.S EU 29 | 2013.1.01029.S EU 30 | 2013.1.01204.S EU 31 | 2013.1.01242.S NA 32 | 2013.1.01391.S NA 33 | 2015.1.00357.S NA 34 | -------------------------------------------------------------------------------- /sbExport.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sagonzal' 2 | from sacm.password import * 3 | import sys 4 | import os 5 | import cx_Oracle 6 | import subprocess 7 | import time 8 | try: 9 | conn = cx_Oracle.connect(databaseSCO) 10 | cursor = conn.cursor() 11 | except Exception as e: 12 | print e 13 | sys.exit(1) 14 | 15 | sbUID = sys.argv[1] 16 | if 'uid___' in sbUID: 17 | sbUID = sbUID.replace('___','://').replace('_','_') 18 | 19 | sql = """select se_eb_uid from ALMA.SHIFTLOG_ENTRIES where SE_SB_ID = '%s' and SE_QA0FLAG = 'Pass'"""%sbUID 20 | #print sql 21 | 22 | rows = cursor.execute(sql) 23 | sbUID_norma = sbUID.replace('://','___').replace('/','_') 24 | 25 | if rows > 0: 26 | os.mkdir('SB_'+sbUID_norma) 27 | os.chdir('SB_'+sbUID_norma) 28 | for i in rows: 29 | subprocess.Popen(['asdmExport','-m',i[0]]) 30 | time.sleep(15) 31 | os.chdir('../') 32 | else: 33 | print "No EBs detected for this SB uid" 34 | 35 | sys.exit(0) 36 | 37 | 38 | -------------------------------------------------------------------------------- /releaseList.txt: -------------------------------------------------------------------------------- 1 | code 2 | 2012.1.00001.S 3 | 2012.1.00019.S 4 | 2012.1.00060.S 5 | 2012.1.00080.S 6 | 2012.1.00108.S 7 | 2012.1.00133.S 8 | 2012.1.00142.S 9 | 2012.1.00239.S 10 | 2012.1.00242.S 11 | 2012.1.00261.S 12 | 2012.1.00271.S 13 | 2012.1.00285.S 14 | 2012.1.00323.S 15 | 2012.1.00333.S 16 | 2012.1.00335.S 17 | 2012.1.00357.S 18 | 2012.1.00382.S 19 | 2012.1.00385.S 20 | 2012.1.00394.S 21 | 2012.1.00501.S 22 | 2012.1.00532.S 23 | 2012.1.00542.S 24 | 2012.1.00554.S 25 | 2012.1.00603.S 26 | 2012.1.00608.S 27 | 2012.1.00641.S 28 | 2012.1.00650.S 29 | 2012.1.00683.S 30 | 2012.1.00720.S 31 | 2012.1.00762.S 32 | 2012.1.00781.S 33 | 2012.1.00844.S 34 | 2012.1.00932.S 35 | 2012.1.00934.S 36 | 2012.1.00940.S 37 | 2012.1.00952.S 38 | 2012.1.01004.S 39 | 2012.1.01099.S 40 | 2012.1.01123.S 41 | 2013.1.00041.S 42 | 2013.1.00211.S 43 | 2013.1.00214.S 44 | 2013.1.00269.S 45 | 2013.1.00312.S 46 | 2013.1.00351.S 47 | 2013.1.00356.S 48 | 2013.1.00532.S 49 | 2013.1.00584.S 50 | 2013.1.00617.S 51 | 2013.1.00652.S 52 | 2013.1.00662.S 53 | 2013.1.00724.S 54 | 2013.1.00803.S 55 | 2013.1.00832.S 56 | 2013.1.00839.S 57 | 2013.1.00861.S 58 | 2013.1.00897.S 59 | 2013.1.00914.S 60 | 2013.1.00993.S 61 | 2013.1.00999.S 62 | 2013.1.01014.S 63 | 2013.1.01029.S 64 | 2013.1.01042.S 65 | 2013.1.01091.S 66 | 2013.1.01114.S 67 | 2013.1.01119.S 68 | 2013.1.01161.S 69 | 2013.1.01202.S 70 | 2013.1.01204.S 71 | 2013.1.01242.S 72 | 2013.1.01305.S 73 | 2013.1.01312.S 74 | 2013.1.01358.S 75 | 2013.1.01364.S 76 | 2013.1.01391.S 77 | 2015.1.00357.S 78 | -------------------------------------------------------------------------------- /ict4456.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sagonzal' 2 | from MetaData import * 3 | 4 | def checkOrder(sb_uid=None): 5 | a,b = getSB_spectralconf(sb_uid) 6 | baseband = a[[1,2]] 7 | baseband.columns = [['partId','BB']] 8 | spw = b[[0,3,4]] 9 | spw.columns = [['spwId','SB_UID','partId']] 10 | 11 | df = pd.merge(baseband,spw, left_on='partId', right_on='partId',copy=False ) 12 | 13 | c = df.groupby('spwId') 14 | 15 | for name,group in c: 16 | unordered = list() 17 | ordered = list() 18 | i = c.get_group(name) 19 | values = i['BB'].values 20 | for j in values: 21 | unordered.append(j) 22 | ordered.append(j) 23 | 24 | ordered.sort() 25 | #print ordered 26 | #print unordered 27 | 28 | if unordered != ordered: 29 | return False 30 | return True 31 | 32 | 33 | if __name__ == '__main__': 34 | 35 | orcl = cx_Oracle.connect(database) 36 | cursor = orcl.cursor() 37 | projects = list() 38 | 39 | sql = """select al1.DOMAIN_ENTITY_ID,al1.OBS_PROJECT_ID 40 | from ALMA.SCHED_BLOCK_STATUS al1, 41 | ALMA.BMMV_OBSPROJECT al2 42 | where 43 | (al2.code like '2011._.%._' or al2.code like '2012._.%._' or al2.code like '2013._.%._') 44 | and al1.OBS_PROJECT_ID = al2.PRJ_ARCHIVE_UID 45 | order by 2 desc""" 46 | 47 | cursor.execute(sql) 48 | 49 | for i,j in cursor: 50 | print i,j 51 | try: 52 | projects.append([i,j,checkOrder(i)]) 53 | except Exception as e: 54 | print 'SB with problem:', i 55 | pass 56 | df = pd.DataFrame(projects) 57 | 58 | ict = open('ict4456.csv','w') 59 | df.to_csv(ict,index=False,sep= ' ') 60 | ict.close() 61 | -------------------------------------------------------------------------------- /checkUpdateSACM211.py: -------------------------------------------------------------------------------- 1 | #!/bin/env python 2 | 3 | import pandas as pd 4 | import cx_Oracle 5 | from sacm.password import * 6 | import sys 7 | 8 | 9 | try: 10 | conn = cx_Oracle.connect(databaseSCO) 11 | cursor = conn.cursor() 12 | except Exception as e: 13 | print e 14 | sys.exit(1) 15 | 16 | 17 | 18 | def checkDB(uid=None,cursor=None): 19 | sql = "select * from alma.xml_updates where asdm_uid = '%s' and elaboration like 'SACM-211'"%uid 20 | #print sql 21 | try: 22 | cursor.execute(sql) 23 | except Exception as e: 24 | print e 25 | sys.exit(1) 26 | result = cursor.fetchall() 27 | if len(result) > 0: 28 | return True 29 | else: 30 | return False 31 | 32 | image = pd.read_csv('releaseImage.txt',sep='|') 33 | image['updated'] = image.apply(lambda x: checkDB(x['EB'], cursor) , axis = 1) 34 | 35 | 36 | 37 | arcs = pd.read_csv('arcs.csv',sep = ' ', header= None) 38 | arcs.columns = ['all'] 39 | arcs['code'],arcs['arc'] = zip(*arcs.apply(lambda x: x['all'].split('\t') , axis = 1 )) 40 | df = pd.merge(image,arcs,left_on='CODE',right_on='code',how='inner') 41 | df1 = df[['CODE','arc','SB','EB','updated']] 42 | to_html = open('sacm211.html','w') 43 | 44 | to_html.write(''+df1.to_html(index=False,escape=False)+'') 45 | to_html.close() 46 | 47 | f = open('point_zero_one.csv', 'r') 48 | g = list() 49 | foo = f.readlines() 50 | for idx, i in enumerate(foo): 51 | if 'uid' in i: 52 | g.append((foo[idx].replace('\n','').split(' ')[0], foo[idx].replace('\n','').split(' ')[1], foo[idx].replace('\n','').split(' ')[2], foo[idx+1].replace('Max.Offset error:','').replace('\n','').replace(' ',''))) 53 | df = pd.DataFrame(g, columns=['uid','sb','code','error']) 54 | df['updated'] = df.apply(lambda x: checkDB(x['uid'], cursor) , axis = 1) 55 | df = df.sort(['code']) 56 | df = df.reset_index(drop=True) 57 | df1 = df[['code','sb','uid','updated']] 58 | to_html = open('sacm211-all.html','w') 59 | to_html.write(''+df1.to_html(index=False,escape=False)+'') 60 | to_html.close() 61 | 62 | -------------------------------------------------------------------------------- /fieldvsource.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sagonzal' 2 | from MetaData import * 3 | import sys 4 | import pandas as pd 5 | import cx_Oracle 6 | import pylab as pl 7 | 8 | sql= """select SE_EB_UID, SE_SB_ID, SE_PROJECT_CODE from ALMA.SHIFTLOG_ENTRIES where SE_QA0FLAG = 'Pass' 9 | and (SE_PROJECT_CODE like '%.S' or SE_PROJECT_CODE like '%.T' or SE_PROJECT_CODE like '%.A') 10 | and SE_START >= to_date('2013-01-15 00:00:00','YYYY-MM-DD HH24:MI:SS') 11 | order by SE_START asc""" 12 | 13 | try: 14 | conn = cx_Oracle.connect(databaseSCO) 15 | cursor = conn.cursor() 16 | except Exception as e: 17 | print e 18 | sys.exit(1) 19 | 20 | asdm = AsdmCheck() 21 | 22 | rows = cursor.execute(sql) 23 | 24 | for i in rows: 25 | print "#"+i[0] 26 | try: 27 | asdm.setUID(i[0]) 28 | scan = getScan(asdm.asdmDict['Scan']) 29 | field = getField(asdm.asdmDict['Field']) 30 | source = getSource(asdm.asdmDict['Source']) 31 | scan['target'] = scan.apply(lambda x: True if str(x['scanIntent']).find('OBSERVE_TARGET') > 0 else False ,axis = 1) 32 | targets = map(unicode.strip,list(scan[scan['target'] == True].sourceName.values)) 33 | field['target'] = field.apply(lambda x: True if str(x['fieldName']).strip() in targets else False, axis = 1) 34 | source['target'] = source.apply(lambda x: True if str(x['sourceName']).strip() in targets else False, axis = 1) 35 | source['ra'], source['dec'] = zip(*source.apply(lambda x: arrayParser(x['direction'],1), axis = 1)) 36 | field['ra'],field['dec'] = zip(*field.apply(lambda x: arrayParser(x['referenceDir'],2)[0], axis = 1)) 37 | del source['spectralWindowId'] 38 | source = source.drop_duplicates() 39 | source2 = source[source['target'] == True] 40 | field2 = field[field['target'] == True] 41 | merge = pd.merge(field2, source2, left_on='sourceId',right_on='sourceId',how='inner') 42 | merge['ra_diff'] = merge.apply(lambda x: pl.absolute(float(x['ra_x'])-float(x['ra_y'])), axis = 1) 43 | merge['dec_diff'] = merge.apply(lambda x: pl.absolute(float(x['dec_x'])-float(x['dec_y'])), axis = 1) 44 | merge['total'] = merge.apply(lambda x: ((x['ra_diff']**2 + x['dec_diff']**2)**(0.5))*206264.80624709636, axis = 1) 45 | if merge.total.describe()[7] > 1: 46 | print i[0]+'|'+i[1]+'|'+i[2] 47 | except Exception as e: 48 | print "#Trouble UID" 49 | pass -------------------------------------------------------------------------------- /sacm450.py: -------------------------------------------------------------------------------- 1 | #!/bin/env python 2 | from MetaData import * 3 | import pandas as pd 4 | import cx_Oracle 5 | import sys 6 | 7 | try: 8 | orcl = cx_Oracle.connect(databaseSCO) 9 | cursor = orcl.cursor() 10 | except Exception as e: 11 | sys.exit(1) 12 | 13 | sql = """select SE_PROJECT_CODE,SE_EXECUTIVE,SE_SB_ID, SE_EB_UID from ALMA.SHIFTLOG_ENTRIES where se_sb_id in ('uid://A002/Xad2d3f/X3 ', 14 | 'uid://A001/X2fe/X267', 15 | 'uid://A001/X2fe/X268', 16 | 'uid://A001/X2fe/X26a', 17 | 'uid://A001/X2fe/X26b', 18 | 'uid://A001/X2fe/X26c', 19 | 'uid://A001/X2fe/X26e', 20 | 'uid://A001/X2fb/X638', 21 | 'uid://A001/X2d1/X25', 22 | 'uid://A002/Xaa5ac1/X36', 23 | 'uid://A002/Xb0eb5c/X6b', 24 | 'uid://A002/Xb0eb5c/X6d', 25 | 'uid://A002/Xb0eb5c/X6f', 26 | 'uid://A002/Xb0eb5c/X70', 27 | 'uid://A002/Xb0eb5c/X71', 28 | 'uid://A001/X2f6/X481', 29 | 'uid://A001/X2f6/X482', 30 | 'uid://A001/X2f6/X484', 31 | 'uid://A001/X2fb/X10e', 32 | 'uid://A001/X2de/Xa3', 33 | 'uid://A001/X2fb/X614', 34 | 'uid://A001/X2fb/X187', 35 | 'uid://A001/X2f6/Xf8', 36 | 'uid://A001/X2f6/Xf9', 37 | 'uid://A001/X2d8/X46') and SE_QA0FLAG = 'Pass' order by 2,1,3""" 38 | 39 | cursor.execute(sql) 40 | rows = cursor.fetchall() 41 | text = '' 42 | a = AsdmCheck() 43 | for i in rows: 44 | a.setUID(i[3]) 45 | text += '

'+i[0]+' ARC: '+str(i[1])+' SB: '+i[2]+' EB: '+i[3]+'

' 46 | spw = getSpectralWindow(a.asdmDict['SpectralWindow']) 47 | scan = getScan(a.asdmDict['Scan']) 48 | source = getSource(a.asdmDict['Source']) 49 | scan['target'] = scan.apply(lambda x: True if str(x['scanIntent']).find('OBSERVE_TARGET') > 0 else False ,axis = 1) 50 | targets = map(unicode.strip,list(scan[scan['target'] == True].sourceName.values)) 51 | source['target'] = source.apply(lambda x: True if str(x['sourceName']).strip() in targets else False, axis = 1) 52 | df = source[['spectralWindowId','target']].query('target == True') 53 | df1 = pd.merge(spw,df,left_on='spectralWindowId', right_on='spectralWindowId', how = 'inner') 54 | df1['freqStart'] = df1.apply(lambda x: float(x['chanFreqStart'])/1e9, axis = 1 ) 55 | df1['freqEnd'] = df1.apply(lambda x: (float(x['chanFreqStart']) + float(x['chanFreqStep'])*float(x['numChan']) )/1e9 , axis = 1 ) 56 | df1['centerFreq'] = df1.apply(lambda x: (x['freqStart'] + x['freqEnd'] )/2. , axis = 1 ) 57 | df1['referenceFreq'] = df1.apply(lambda x: float(x['refFreq'])/1e9 , axis = 1 ) 58 | text += df1[['spectralWindowId','basebandName','netSideband','numChan','referenceFreq','freqStart','freqEnd','centerFreq']].drop_duplicates().to_html() 59 | 60 | text +='' 61 | 62 | html = open('sacm450.html','w') 63 | html.write(text) 64 | html.close() -------------------------------------------------------------------------------- /.ipynb_checkpoints/Trending-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "name": "", 4 | "signature": "sha256:1bf6011c3bdac424c01c3da2f831619ad93d2c8db2536d760c469e7a4568dd46" 5 | }, 6 | "nbformat": 3, 7 | "nbformat_minor": 0, 8 | "worksheets": [ 9 | { 10 | "cells": [ 11 | { 12 | "cell_type": "code", 13 | "collapsed": false, 14 | "input": [ 15 | "from MetaData import *\n", 16 | "import matplotlib.pyplot as plt\n", 17 | "import numpy as np\n", 18 | "import plotly.plotly as py\n", 19 | "import plotly.tools as tls\n", 20 | "from plotly.graph_objs import *\n", 21 | "py.sign_in('sdk.cl','ddgrpr0imy')\n" 22 | ], 23 | "language": "python", 24 | "metadata": {}, 25 | "outputs": [], 26 | "prompt_number": 1 27 | }, 28 | { 29 | "cell_type": "code", 30 | "collapsed": false, 31 | "input": [ 32 | "sql = \"select * from metadata;\"\n", 33 | "pgcursor.execute(sql)\n", 34 | "df = pd.DataFrame(pgcursor.fetchall())\n" 35 | ], 36 | "language": "python", 37 | "metadata": {}, 38 | "outputs": [], 39 | "prompt_number": 2 40 | }, 41 | { 42 | "cell_type": "code", 43 | "collapsed": false, 44 | "input": [ 45 | "data = df[[1,2,3,4,5,6,9]]\n", 46 | "\n", 47 | "data.columns = ['check','syscal','validuid','csv2555','fixplanets','nullstate','date']\n", 48 | "data.set_index('date',inplace=True)\n", 49 | "data.index = data.index.to_datetime()\n", 50 | "data.sort_index(inplace=True)\n", 51 | "\n", 52 | "\n" 53 | ], 54 | "language": "python", 55 | "metadata": {}, 56 | "outputs": [], 57 | "prompt_number": 3 58 | }, 59 | { 60 | "cell_type": "code", 61 | "collapsed": false, 62 | "input": [ 63 | "#data['check_s'] = data.apply(lambda x: 1 if x['check'] == False else 0, axis = 1)\n", 64 | "data['syscal_s'] = data.apply(lambda x: 1 if x['syscal'] == False else 0, axis = 1)\n", 65 | "data['validuid_s'] = data.apply(lambda x: 1 if x['validuid'] == False else 0, axis = 1)\n", 66 | "data['csv2555_s'] = data.apply(lambda x: 1 if x['csv2555'] == False else 0, axis = 1)\n", 67 | "data['fixplanets_s'] = data.apply(lambda x: 1 if x['fixplanets'] == False else 0, axis = 1)\n", 68 | "data['nullstate_s'] = data.apply(lambda x: 1 if x['nullstate'] == False else 0, axis = 1)\n", 69 | "\n", 70 | "df2 = data[['syscal_s','validuid_s','csv2555_s','fixplanets_s','nullstate_s']].cumsum()" 71 | ], 72 | "language": "python", 73 | "metadata": {}, 74 | "outputs": [], 75 | "prompt_number": 6 76 | }, 77 | { 78 | "cell_type": "code", 79 | "collapsed": false, 80 | "input": [ 81 | "df2.plot()\n", 82 | "fig = plt.gcf()\n", 83 | "py.iplot_mpl(fig)" 84 | ], 85 | "language": "python", 86 | "metadata": {}, 87 | "outputs": [ 88 | { 89 | "html": [ 90 | "" 91 | ], 92 | "metadata": {}, 93 | "output_type": "pyout", 94 | "prompt_number": 7, 95 | "text": [ 96 | "" 97 | ] 98 | } 99 | ], 100 | "prompt_number": 7 101 | }, 102 | { 103 | "cell_type": "code", 104 | "collapsed": false, 105 | "input": [], 106 | "language": "python", 107 | "metadata": {}, 108 | "outputs": [] 109 | } 110 | ], 111 | "metadata": {} 112 | } 113 | ] 114 | } -------------------------------------------------------------------------------- /MC.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sagonzal' 2 | from MetaData import * 3 | import pandas as pd 4 | 5 | 6 | def process(): 7 | sb_list = list() 8 | band89 = pd.DataFrame() 9 | spectralScan = pd.DataFrame() 10 | 11 | projects = getProjectCodes(2) 12 | for prjuid, code in projects: 13 | schedblocks = getSBs(prjuid) 14 | for partid, sbuid in schedblocks: 15 | #print prjuid,code,sbuid, partid 16 | bb,specs,target,phase,science,field = getSBData (sbuid) 17 | if len(field[field[1] == 'Ephemeris']) > 0: 18 | ephemeris = True 19 | else: 20 | ephemeris = False 21 | if len(specs[specs[5] == 'XX,YY,XY,YX']) > 0: 22 | polarization = True 23 | else: 24 | polarization = False 25 | 26 | df1 = pd.merge(science, target, left_on= 0, right_on = 'ObsParameter', how = 'inner', copy= False) 27 | if len(phase) > 0: 28 | df2 = pd.merge(phase, target, left_on= 0, right_on = 'ObsParameter', how = 'inner', copy= False) 29 | df3 = pd.merge(df1,df2 , right_on = 'InstrumentSpec', left_on = 'InstrumentSpec' , how = 'inner', copy = False) 30 | if len(df3) > 0: 31 | sb_list.append((prjuid,code,sbuid, partid, 'Same PhaseCal Setup',ephemeris, polarization)) 32 | else: 33 | sb_list.append((prjuid,code,sbuid, partid, 'Different PhaseCal Setup',ephemeris, polarization)) 34 | else: 35 | sb_list.append((prjuid,code,sbuid, partid, 'No PhaseCal (TP?)',ephemeris, polarization)) 36 | 37 | band89 = pd.concat ([band89,pd.DataFrame(is_band89(prjuid))], ignore_index=True) 38 | spectralScan = pd.concat([spectralScan,pd.DataFrame(is_spectralscan(prjuid))],ignore_index=True) 39 | 40 | sbmous = pd.DataFrame(getSBMOUS()) 41 | sbnames = pd.DataFrame(getSBNames()) 42 | sbs = pd.DataFrame(sb_list) 43 | return (sbs,sbmous,band89,spectralScan,sbnames) 44 | 45 | 46 | def main(): 47 | sbs, sbmous, band89, spectralScan, sbnames = process() 48 | 49 | del spectralScan[0] 50 | del spectralScan[1] 51 | del band89[0] 52 | 53 | sbs = pd.merge(sbs, spectralScan , left_on = 3, right_on = 3, how='left', copy=False) 54 | sbs = pd.merge(sbs, band89, left_on = '2_x' ,right_on = 1, how = 'left', copy=False) 55 | sbmous.columns = [[0,'MOUS']] 56 | sbs = pd.merge(sbs,sbmous,left_on='2_x', right_on = 0, how = 'left', copy=False) 57 | sbnames.columns = [[0,'SB_NAME']] 58 | sbs = pd.merge(sbs,sbnames, left_on = '2_x', right_on = 0, how = 'left', copy=False) 59 | 60 | sbs = sbs.fillna('') 61 | sbs['Pipeline'] = sbs.apply(lambda x: True if x[4] == 'Same PhaseCal Setup' else False, axis = 1) 62 | sbs['Pipeline'] = sbs.apply(lambda x: False if 'RB_08' in x[2] else x['Pipeline'] , axis = 1) 63 | sbs['Pipeline'] = sbs.apply(lambda x: False if 'RB_09' in x[2] else x['Pipeline'] , axis = 1) 64 | sbs['Pipeline'] = sbs.apply(lambda x: False if x[5] == True else x['Pipeline'] , axis = 1) 65 | sbs['Pipeline'] = sbs.apply(lambda x: False if x[6] == True else x['Pipeline'] , axis = 1) 66 | sbs['SpectralScan'] = sbs.apply(lambda x: True if 'scan' in x['2_y'] else False , axis =1) 67 | final = sbs[['1_x','0_x','MOUS','SB_NAME','2_x','Pipeline',4,'SpectralScan',2,5,6]] 68 | final.columns = ['PRJ_CODE','PRJ_UID','MOUS','SB_NAME','SB_UID','Pipeline','PhaseSetup','SpectralScan','BAND','Ephemeris','Polarization'] 69 | 70 | 71 | 72 | to_html = open('pipeline.html','w') 73 | to_html.write(''+final.to_html()+'') 74 | to_html.close() 75 | 76 | to_csv = open('pipeline.csv','w') 77 | to_csv.write(final.to_csv(sep='\t')) 78 | to_csv.close() 79 | 80 | if __name__ == "__main__": 81 | main() 82 | 83 | -------------------------------------------------------------------------------- /Trending.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "name": "", 4 | "signature": "sha256:0530ed1b5e8801f3e544176d36e00fa779c2c98e9db7be9ff75b6dc6104fca4b" 5 | }, 6 | "nbformat": 3, 7 | "nbformat_minor": 0, 8 | "worksheets": [ 9 | { 10 | "cells": [ 11 | { 12 | "cell_type": "code", 13 | "collapsed": false, 14 | "input": [ 15 | "from MetaData import *\n", 16 | "import matplotlib.pyplot as plt\n", 17 | "import numpy as np\n", 18 | "import plotly.plotly as py\n", 19 | "import plotly.tools as tls\n", 20 | "from plotly.graph_objs import *\n", 21 | "py.sign_in('sdk.cl','ddgrpr0imy')\n" 22 | ], 23 | "language": "python", 24 | "metadata": {}, 25 | "outputs": [], 26 | "prompt_number": 1 27 | }, 28 | { 29 | "cell_type": "code", 30 | "collapsed": false, 31 | "input": [ 32 | "sql = \"select * from metadata;\"\n", 33 | "pgcursor.execute(sql)\n", 34 | "df = pd.DataFrame(pgcursor.fetchall())\n" 35 | ], 36 | "language": "python", 37 | "metadata": {}, 38 | "outputs": [], 39 | "prompt_number": 2 40 | }, 41 | { 42 | "cell_type": "code", 43 | "collapsed": false, 44 | "input": [ 45 | "data = df[[1,2,3,4,5,6,9]]\n", 46 | "\n", 47 | "data.columns = ['check','syscal','validuid','csv2555','fixplanets','nullstate','date']\n", 48 | "data.set_index('date',inplace=True)\n", 49 | "data.index = data.index.to_datetime()\n", 50 | "data.sort_index(inplace=True)\n", 51 | "\n", 52 | "\n" 53 | ], 54 | "language": "python", 55 | "metadata": {}, 56 | "outputs": [], 57 | "prompt_number": 3 58 | }, 59 | { 60 | "cell_type": "code", 61 | "collapsed": false, 62 | "input": [ 63 | "#data['check_s'] = data.apply(lambda x: 1 if x['check'] == False else 0, axis = 1)\n", 64 | "#data['syscal_s'] = data.apply(lambda x: 1 if x['syscal'] == False else 0, axis = 1)\n", 65 | "#data['validuid_s'] = data.apply(lambda x: 1 if x['validuid'] == False else 0, axis = 1)\n", 66 | "data['csv2555_s'] = data.apply(lambda x: 1 if x['csv2555'] == False else 0, axis = 1)\n", 67 | "#data['fixplanets_s'] = data.apply(lambda x: 1 if x['fixplanets'] == False else 0, axis = 1)\n", 68 | "#data['nullstate_s'] = data.apply(lambda x: 1 if x['nullstate'] == False else 0, axis = 1)\n", 69 | "\n", 70 | "#df2 = data[['syscal_s','validuid_s','csv2555_s','fixplanets_s','nullstate_s']].cumsum()\n", 71 | "df2 = data[['csv2555_s']].cumsum()\n" 72 | ], 73 | "language": "python", 74 | "metadata": {}, 75 | "outputs": [], 76 | "prompt_number": 4 77 | }, 78 | { 79 | "cell_type": "code", 80 | "collapsed": false, 81 | "input": [ 82 | "df2.plot()\n", 83 | "fig = plt.gcf()\n", 84 | "py.iplot_mpl(fig)" 85 | ], 86 | "language": "python", 87 | "metadata": {}, 88 | "outputs": [ 89 | { 90 | "output_type": "stream", 91 | "stream": "stderr", 92 | "text": [ 93 | "/home/sagonzal/anaconda/lib/python2.7/site-packages/plotly/matplotlylib/renderer.py:382: UserWarning:\n", 94 | "\n", 95 | "Bummer! Plotly can currently only draw Line2D objects from matplotlib that are in 'data' coordinates!\n", 96 | "\n", 97 | "/home/sagonzal/anaconda/lib/python2.7/site-packages/plotly/matplotlylib/renderer.py:479: UserWarning:\n", 98 | "\n", 99 | "I found a path object that I don't think is part of a bar chart. Ignoring.\n", 100 | "\n" 101 | ] 102 | }, 103 | { 104 | "html": [ 105 | "" 106 | ], 107 | "metadata": {}, 108 | "output_type": "pyout", 109 | "prompt_number": 5, 110 | "text": [ 111 | "" 112 | ] 113 | } 114 | ], 115 | "prompt_number": 5 116 | }, 117 | { 118 | "cell_type": "code", 119 | "collapsed": false, 120 | "input": [], 121 | "language": "python", 122 | "metadata": {}, 123 | "outputs": [] 124 | } 125 | ], 126 | "metadata": {} 127 | } 128 | ] 129 | } -------------------------------------------------------------------------------- /execFraction.py: -------------------------------------------------------------------------------- 1 | #!/opt/python2/bin/python 2 | __author__ = 'sagonzal' 3 | from MetaData import * 4 | import pandas as pd 5 | import sys 6 | import Sensitivity.sensitivity as sensitivity 7 | ############################## 8 | 9 | #configDir = '/home/sagonzal/druva/PycharmProjects/MetaData/Sensitivity' 10 | configDir = '/users/sagonzal/sagonzal/workspace/sacm/Sensitivity' 11 | 12 | try: 13 | asdmUID = sys.argv[1] 14 | asdm = AsdmCheck() 15 | asdm.setUID(asdmUID) 16 | sb = getSBSummary(asdm.asdmDict['SBSummary']) 17 | except Exception as e: 18 | print e 19 | sys.exit(1) 20 | 21 | c3 = datetime.datetime(2015,10,1,0,0,0) 22 | c2 = datetime.datetime(2014,6,3,0,0,0) 23 | c1 = datetime.datetime(2013,10,12,0,0,0) 24 | 25 | if asdm.toc >= c3: 26 | NAntOT = 36. 27 | cycle = 'Cycle 3' 28 | elif asdm.toc >= c2: 29 | NAntOT = 34. 30 | cycle = 'Cycle 2' 31 | elif asdm.toc >= c1: 32 | NAntOT = 32. 33 | cycle = 'Cycle 1' 34 | else: 35 | NAntOT = 14. 36 | cycle = 'Cycle 0' 37 | 38 | ############################ 39 | sbUID = sb.values[0][0] 40 | freq = float(sb.values[0][3])*1e9 41 | band = sb.values[0][4] 42 | 43 | for i in sb.values[0][6].split('"'): 44 | if i.find('maxPWVC') >= 0: maxPWV = float(i.split('=')[1].split(' ')[1]) 45 | 46 | maxPWV = returnMAXPWVC(maxPWV) 47 | 48 | 49 | print sbUID 50 | 51 | science = getSBScience(sbUID) 52 | field = getSBFields(sbUID) 53 | target = getSBTargets(sbUID) 54 | 55 | df1 = pd.merge(science,target, left_on='entityPartId',right_on='ObsParameter',how='inner') 56 | df2 = pd.merge(df1,field,left_on='FieldSource',right_on='entityPartId',how='inner') 57 | 58 | dec = float(df2['latitude'].values[0]) 59 | ToSOT = float(df2['integrationTime'].values[0]) 60 | ############################### 61 | 62 | ############################# 63 | 64 | s = sensitivity.SensitivityCalculator(config_dir=configDir) 65 | result = s.calcSensitivity(maxPWV,freq,dec=dec,latitude=-23.029, N=NAntOT, BW=7.5e9, mode='image', N_pol=2,returnFull=True) 66 | TsysOT = result['Tsys'] 67 | 68 | ################################ 69 | ant = getAntennas(asdm.asdmDict['Antenna']) 70 | NantEB = float(ant.antennaId.count()) 71 | 72 | 73 | ################################ 74 | scan = getScan(asdm.asdmDict['Scan']) 75 | scan['target'] = scan.apply(lambda x: True if str(x['scanIntent']).find('OBSERVE_TARGET') > 0 else False ,axis = 1) 76 | scan['delta'] = scan.apply(lambda x: (gtm2(x['endTime']) - gtm2(x['startTime'])).total_seconds() ,axis = 1) 77 | 78 | target = list(scan[scan['target'] == True]['sourceName'].unique()) 79 | scan['atm'] = scan.apply(lambda x: True if str(x['scanIntent']).find('CALIBRATE_ATMOSPHERE') > 0 and x['sourceName'] in target else False,axis =1 ) 80 | 81 | ToSEB = float(scan['delta'][scan['target'] == True].sum()) 82 | 83 | 84 | ################################ 85 | syscal = getSysCal (asdm.asdmDict['SysCal']) 86 | syscal['startTime'] = syscal.apply(lambda x: int(x['timeInterval'].split(' ')[1]) - int(x['timeInterval'].split(' ')[2])/2 ,axis=1 ) 87 | 88 | 89 | ################################### 90 | spw = getSpectralWindow(asdm.asdmDict['SpectralWindow']) 91 | spw['repWindow'] = spw.apply(lambda x: findChannel(float(x['chanFreqStart']),float(x['chanFreqStep']), freq, int(x['numChan'])), axis = 1) 92 | 93 | 94 | 95 | ################################ 96 | 97 | 98 | df1 = syscal[syscal['startTime'].isin(scan[scan['atm'] == True]['startTime'])] 99 | df2 = df1[df1['spectralWindowId'] == spw[spw['repWindow'] != 0]['spectralWindowId'].values[0].strip()] 100 | channel = int(spw[spw['repWindow'] != 0]['repWindow'].values[0]) 101 | 102 | if channel < 0: 103 | channel+=128 104 | 105 | data = df2.apply(lambda x: arrayParser(x['tsysSpectrum'],2), axis = 1) 106 | data2 = pd.DataFrame (data) 107 | data2.columns = ['hola'] 108 | x = pd.concat([pd.DataFrame(v,index=np.repeat(k,len(v))) for k,v in data2.hola.to_dict().items()]) 109 | x = x.convert_objects(convert_numeric=True) 110 | 111 | 112 | TsysEB = x[channel].median() 113 | 114 | result = {'TimeOnSource_EB':ToSEB, 115 | 'TimeOnSource_OT':ToSOT, 116 | 'Tsys_OT':TsysOT, 117 | 'Tsys_EB':TsysEB, 118 | 'NAntennas_EB':NantEB, 119 | 'NAntennas_OT': NAntOT, 120 | 'Channel': channel} 121 | 122 | execFrac = ((TsysOT/TsysEB)**2)*((NantEB*(NantEB-1.))/(NAntOT*(NAntOT-1.)))*(ToSEB/ToSOT) 123 | 124 | print 'Executional Fraction: ', execFrac 125 | print cycle 126 | print 'Details:' 127 | print result 128 | 129 | 130 | -------------------------------------------------------------------------------- /list_csv2999.csv: -------------------------------------------------------------------------------- 1 | uid___A002_X7143f6_X1369 2 | uid___A002_X7143f6_X15a 3 | uid___A002_X7143f6_X1779 4 | uid___A002_X7143f6_X6e5 5 | uid___A002_X7143f6_Xa90 6 | uid___A002_X7143f6_Xf73 7 | uid___A002_X715a3e_X13b 8 | uid___A002_X716e68_X21a 9 | uid___A002_X717cdc_X213 10 | uid___A002_X71895d_X127 11 | uid___A002_X71895d_X509 12 | uid___A002_X71895d_X88f 13 | uid___A002_X71895d_Xb83 14 | uid___A002_X71895d_Xf81 15 | uid___A002_X71a45c_X5c 16 | uid___A002_X71a45c_X668 17 | uid___A002_X71a45c_X9dc 18 | uid___A002_X71a45c_Xe44 19 | uid___A002_X71cec2_X535 20 | uid___A002_X71cec2_X936 21 | uid___A002_X71e4ae_X2c1 22 | uid___A002_X71e4ae_X6dc 23 | uid___A002_X71e4ae_Xb04 24 | uid___A002_X71e4ae_Xf2d 25 | uid___A002_X72bc38_X2d8 26 | uid___A002_X72bc38_X498 27 | uid___A002_X72bc38_X741 28 | uid___A002_X72bc38_X7a 29 | uid___A002_X72c4aa_X18e 30 | uid___A002_X72c4aa_X614 31 | uid___A002_X72d23d_X1d4 32 | uid___A002_X72d23d_X5d2 33 | uid___A002_X72d23d_X89e 34 | uid___A002_X72e960_X1219 35 | uid___A002_X72e960_X161c 36 | uid___A002_X72e960_X515 37 | uid___A002_X72e960_X913 38 | uid___A002_X72e960_X923 39 | uid___A002_X72e960_Xdcc 40 | uid___A002_X72e960_Xe7b 41 | uid___A002_X72e960_Xfd8 42 | uid___A002_X7310ce_X520 43 | uid___A002_X7310ce_X978 44 | uid___A002_X7330a2_X393 45 | uid___A002_X7330a2_X769 46 | uid___A002_X7330a2_X77 47 | uid___A002_X735201_X2c6 48 | uid___A002_X735201_X38a 49 | uid___A002_X735201_X4a3 50 | uid___A002_X7363c7_X1d5 51 | uid___A002_X7363c7_X431 52 | uid___A002_X736ee1_X1f3 53 | uid___A002_X736ee1_X456 54 | uid___A002_X736ee1_X69e 55 | uid___A002_X74821d_X109b 56 | uid___A002_X74821d_X1272 57 | uid___A002_X74821d_X16bf 58 | uid___A002_X74821d_X1702 59 | uid___A002_X74821d_Xcec 60 | uid___A002_X74821d_Xeb9 61 | uid___A002_X74a173_X11f3 62 | uid___A002_X74a173_X1536 63 | uid___A002_X74a173_X1a8f 64 | uid___A002_X74a173_X227c 65 | uid___A002_X74a173_X23b7 66 | uid___A002_X74a173_Xacf 67 | uid___A002_X74a173_Xeac 68 | uid___A002_X74d4df_X17c 69 | uid___A002_X74d4df_X4b9 70 | uid___A002_X74d4df_X834 71 | uid___A002_X74d4df_X913 72 | uid___A002_X74deb6_X1304 73 | uid___A002_X74deb6_X14df 74 | uid___A002_X74deb6_X1585 75 | uid___A002_X74deb6_X1787 76 | uid___A002_X74deb6_X292 77 | uid___A002_X74deb6_X44e 78 | uid___A002_X74deb6_X479 79 | uid___A002_X74deb6_X59 80 | uid___A002_X74deb6_X66f 81 | uid___A002_X74deb6_X84b 82 | uid___A002_X74deb6_Xbf5 83 | uid___A002_X74deb6_Xfb1 84 | uid___A002_X74eb2a_X1b7 85 | uid___A002_X74eb2a_X24b 86 | uid___A002_X74eb2a_X5c8 87 | uid___A002_X74eb2a_Xaa4 88 | uid___A002_X74fea5_X105d 89 | uid___A002_X74fea5_X10d0 90 | uid___A002_X74fea5_X1311 91 | uid___A002_X74fea5_X141 92 | uid___A002_X74fea5_X1558 93 | uid___A002_X74fea5_X348 94 | uid___A002_X74fea5_X4fa 95 | uid___A002_X74fea5_X5c 96 | uid___A002_X74fea5_X696 97 | uid___A002_X74fea5_X841 98 | uid___A002_X74fea5_X893 99 | uid___A002_X74fea5_Xa54 100 | uid___A002_X74fea5_Xc53 101 | uid___A002_X74fea5_Xdad 102 | uid___A002_X75ab74_X11f4 103 | uid___A002_X75ab74_X158e 104 | uid___A002_X75ab74_X2d4 105 | uid___A002_X75ab74_X311 106 | uid___A002_X75ab74_X5fd 107 | uid___A002_X75ab74_X84f 108 | uid___A002_X75ab74_Xc7e 109 | uid___A002_X75ab74_Xeb1 110 | uid___A002_X75bfbf_X101 111 | uid___A002_X75bfbf_X1015 112 | uid___A002_X75bfbf_X1130 113 | uid___A002_X75bfbf_X1430 114 | uid___A002_X75bfbf_X1728 115 | uid___A002_X75bfbf_X465 116 | uid___A002_X75bfbf_X466 117 | uid___A002_X75bfbf_X79 118 | uid___A002_X75bfbf_X848 119 | uid___A002_X75bfbf_X9a9 120 | uid___A002_X75bfbf_Xab8 121 | uid___A002_X75bfbf_Xace 122 | uid___A002_X75bfbf_Xce3 123 | uid___A002_X75bfbf_Xd9e 124 | uid___A002_X75d7ca_X117e 125 | uid___A002_X75d7ca_X118b 126 | uid___A002_X75d7ca_X1400 127 | uid___A002_X75d7ca_X166f 128 | uid___A002_X75d7ca_X1e8 129 | uid___A002_X75d7ca_X561 130 | uid___A002_X75d7ca_X7d0 131 | uid___A002_X75d7ca_Xa72 132 | uid___A002_X75d7ca_Xc97 133 | uid___A002_X75d7ca_Xe79 134 | uid___A002_X75d7ca_Xea4 135 | uid___A002_X75f169_X1003 136 | uid___A002_X75f169_X1106 137 | uid___A002_X75f169_X1348 138 | uid___A002_X75f169_X15f6 139 | uid___A002_X75f169_X2dd 140 | uid___A002_X75f169_X4c6 141 | uid___A002_X75f169_X4d9 142 | uid___A002_X75f169_X69 143 | uid___A002_X75f169_X6fc 144 | uid___A002_X75f169_X6fd 145 | uid___A002_X75f169_X8b4 146 | uid___A002_X75f169_X97b 147 | uid___A002_X75f169_Xbf6 148 | uid___A002_X75f169_Xe48 149 | uid___A002_X760eb8_X96 150 | uid___A002_X7116f1_Xed8 151 | uid___A002_X71a45c_X5c6 152 | uid___A002_X72bc38_X7a3 153 | uid___A002_X7330a2_X776 154 | uid___A002_X74deb6_X593 155 | uid___A002_X74fea5_X5c5 156 | uid___A002_X75bfbf_X794 157 | uid___A002_X75f169_X690 158 | uid___A002_X75f169_X693 159 | -------------------------------------------------------------------------------- /CalculateFieldError.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sagonzal' 2 | from MetaData import * 3 | import math 4 | import sacm.geo_helper as gh 5 | import pylab as pl 6 | from itertools import combinations 7 | import pandas as pd 8 | import sys 9 | def measureDistance(lat1, lon1, lat2, lon2): 10 | R = 6383.137 # Radius of earth at Chajnantor aprox. in KM 11 | dLat = (lat2 - lat1) * np.pi / 180. 12 | dLon = (lon2 - lon1) * np.pi / 180. 13 | a = pl.sin(dLat/2.) * pl.sin(dLat/2.) + pl.cos(lat1 * np.pi / 180.) * pl.cos(lat2 * np.pi / 180.) * pl.sin(dLon/2.) * pl.sin(dLon/2.) 14 | c = 2. * math.atan2(pl.sqrt(a), pl.sqrt(1-a)) 15 | d = R * c 16 | return d * 1000. # meters 17 | 18 | def rot(Pl, rLong, rLat): 19 | return [math.cos(rLong)*math.cos(rLat) * Pl[0] - math.sin(rLong) * Pl[1] - math.cos(rLong)*math.sin(rLat) * Pl[2], 20 | math.sin(rLong)*math.cos(rLat) * Pl[0] + math.cos(rLong) * Pl[1] - math.sin(rLong)*math.sin(rLat) * Pl[2], 21 | math.sin(rLat) * Pl[0] + math.cos(rLat) * Pl[2]] 22 | 23 | def posByRotation(refRA, refDE, xi, eta): 24 | Pl = [math.cos(xi)*math.cos(eta), math.sin(xi)*math.cos(eta), math.sin(eta)] 25 | Ps = rot(Pl, refRA, refDE) 26 | return (math.atan2(Ps[1], Ps[0]) % (2.*math.pi), math.asin(Ps[2])) 27 | 28 | def posErr(refRA, refDE, fieldRA, fieldDE): 29 | cosRefDE = math.cos(refDE) 30 | sinRefDE = math.sin(refDE) 31 | # get the intended offset, assuming simplistic formula was used 32 | xi = cosRefDE * (fieldRA - refRA) 33 | eta = fieldDE - refDE 34 | (offRA, offDE) = posByRotation(refRA, refDE, xi, eta) 35 | errHoriz = (offRA-refRA)*cosRefDE - xi 36 | errDE = (offDE-refDE) - eta 37 | errTot = math.sqrt(errHoriz*errHoriz + errDE*errDE) 38 | #print "xi, eta: (%15.11f, %15.11f) = (%9.3f, %9.3f) arcsec" % (xi, eta, 3600.0*math.degrees(xi), 3600.0*math.degrees(eta)) 39 | #print "reference RA,DE: (%15.11f, %15.11f)" % (refRA, refDE) 40 | #print "offset 'rotate' RA,DE: (%15.11f, %15.11f)" % (offRA, offDE) 41 | #print "offset 'simple' RA,DE: (%15.11f, %15.11f)" % (refRA + xi/cosRefDE, refDE + eta) 42 | #print "err Horiz, DE, total: %.2f, %.2f, %.2f milli-arcsec" % (3.6e6*math.degrees(errHoriz), 3.6e6*math.degrees(errDE), 3.6e6*math.degrees(errTot)) 43 | return 3.6e6*math.degrees(errTot) 44 | 45 | df = pd.read_csv('final.csv', sep='|', header=None) 46 | asdm = AsdmCheck() 47 | for row in df.values: 48 | uid = row[0] 49 | sb = row[1] 50 | project = row[2] 51 | asdm.setUID(uid) 52 | ################################################################ 53 | #Calculate the 10% of the Resolution 54 | antenna = getAntennas(asdm.asdmDict['Antenna']) 55 | station = getStation(asdm.asdmDict['Station']) 56 | geo = pd.merge(antenna,station, left_on='stationId', right_on = 'stationId', how = 'inner') 57 | geo['pos'] = geo.apply(lambda x: arrayParser(x['position'],1) , axis = 1 ) 58 | geo['lat'], geo['lon'], geo['alt'] = zip(*geo.apply(lambda x: gh.turn_xyz_into_llh(float(x.pos[0]),float(x.pos[1]),float(x.pos[2]), 'wgs84'),axis=1)) 59 | comb = combinations(geo[['lat','lon']].values, 2) 60 | combList = list() 61 | for i in comb: 62 | combList.append((i[0][0],i[0][1],i[1][0],i[1][1])) 63 | baseLines = pd.DataFrame(combList) 64 | baseLines['dist'] = baseLines.apply(lambda x: measureDistance(x[0],x[1],x[2],x[3]) , axis = 1) 65 | blMax = baseLines.dist.describe().values[7] 66 | sbsummary = getSBSummary(asdm.asdmDict['SBSummary']) 67 | restfreq = float(sbsummary.values[0][3])*1e9 68 | c = 299792458 69 | l = c / restfreq 70 | beam = l / blMax 71 | sbeam = beam * 206264.80624709636 / 10. 72 | ############################################################## 73 | #Calculate all the Offsets errors 74 | field = getField(asdm.asdmDict['Field']) 75 | source = getSource(asdm.asdmDict['Source']) 76 | scan = getScan(asdm.asdmDict['Scan']) 77 | 78 | scan['target'] = scan.apply(lambda x: True if str(x['scanIntent']).find('OBSERVE_TARGET') > 0 else False ,axis = 1) 79 | targets = map(unicode.strip,list(scan[scan['target'] == True].sourceName.values)) 80 | field['target'] = field.apply(lambda x: True if str(x['fieldName']).strip() in targets else False, axis = 1) 81 | source['target'] = source.apply(lambda x: True if str(x['sourceName']).strip() in targets else False, axis = 1) 82 | source['ra'], source['dec'] = zip(*source.apply(lambda x: arrayParser(x['direction'],1), axis = 1)) 83 | del source['spectralWindowId'] 84 | source = source.drop_duplicates() 85 | source2 = source[source['target'] == True] 86 | field['ra'],field['dec'] = zip(*field.apply(lambda x: arrayParser(x['referenceDir'],2)[0], axis = 1)) 87 | field2 = field[field['target'] == True] 88 | diff = pd.merge(field2, source2, left_on='sourceId',right_on='sourceId',how='inner') 89 | diff['total'] = diff.apply(lambda x: posErr(float(x['ra_y']),float(x['dec_y']),float(x['ra_x']),float(x['dec_x'])), axis = 1) 90 | maxOffset = diff.total.describe()[7]/1000. 91 | #Check if the SB needs Re-Image 92 | print '|',uid,'|',sb,'|',project,'|',str(maxOffset),'|',sbeam,'|' 93 | 94 | 95 | 96 | 97 | -------------------------------------------------------------------------------- /allList.txt: -------------------------------------------------------------------------------- 1 | ,,error,error 2 | ,,count,mean 3 | code,sb,, 4 | 2012.1.00060.S,uid://A002/X639a2a/X1,1,0.218652645262 5 | 2012.1.00080.S,uid://A002/X6444ba/X7,10,0.0669054153745 6 | 2012.1.00080.S,uid://A002/X6444ba/X8,22,234212.283253 7 | 2012.1.00261.S,uid://A002/X6444ba/X9e,1,0.0855921285605 8 | 2012.1.00261.S,uid://A002/X6444ba/X9f,2,0.0855921285605 9 | 2012.1.00323.S,uid://A002/X67ccb6/X9,1,0.524202032949 10 | 2012.1.00333.S,uid://A002/X6f9b0f/X106,1,0.725253073517 11 | 2012.1.00335.S,uid://A002/X75fbd6/X27,2,0.252851808197 12 | 2012.1.00335.S,uid://A002/X75fbd6/X28,6,0.259273499551 13 | 2012.1.00385.S,uid://A002/X6b0cc1/Xa0,2,1295920.61462 14 | 2012.1.00394.S,uid://A002/X639a2a/X3a,1,40.7996913997 15 | 2012.1.00394.S,uid://A002/X639a2a/X3c,2,40.7996913997 16 | 2012.1.00394.S,uid://A002/X639a2a/X3e,1,0.157410817168 17 | 2012.1.00394.S,uid://A002/X639a2a/X3f,4,0.0179379340385 18 | 2012.1.00394.S,uid://A002/X639a2a/X40,1,403.776318195 19 | 2012.1.00394.S,uid://A002/X639a2a/X41,8,0.0930490779259 20 | 2012.1.00501.S,uid://A002/X5d5f8a/X3,1,1097897.21299 21 | 2012.1.00542.S,uid://A002/X6444ba/X33,1,2.83098501364 22 | 2012.1.00542.S,uid://A002/X6444ba/X34,2,3.22937708025 23 | 2012.1.00554.S,uid://A002/X5d7935/X39e,2,0.0321198935977 24 | 2012.1.00554.S,uid://A002/X5d7935/X39f,3,0.0246841930904 25 | 2012.1.00554.S,uid://A002/X5d7935/X3a0,2,0.0227417295163 26 | 2012.1.00554.S,uid://A002/X5d7935/X3a1,4,0.0563884846055 27 | 2012.1.00554.S,uid://A002/X5d7935/X3a2,2,0.322795793625 28 | 2012.1.00554.S,uid://A002/X5d7935/X3a3,4,0.327330264639 29 | 2012.1.00554.S,uid://A002/X5d7935/X3a4,2,0.299879699998 30 | 2012.1.00554.S,uid://A002/X5d7935/X3a5,6,0.295965713264 31 | 2012.1.00603.S,uid://A002/X6f9b0f/Xbb,1,0.0742755136503 32 | 2012.1.00603.S,uid://A002/X6f9b0f/Xbc,4,0.0661886871339 33 | 2012.1.00603.S,uid://A002/X6f9b0f/Xbf,1,0.0432364618337 34 | 2012.1.00603.S,uid://A002/X6f9b0f/Xc0,5,0.0744969529509 35 | 2012.1.00603.S,uid://A002/X6f9b0f/Xc3,2,0.0925892526522 36 | 2012.1.00603.S,uid://A002/X6f9b0f/Xc4,6,0.0812592684423 37 | 2012.1.00603.S,uid://A002/X6f9b0f/Xc7,2,0.0929910376203 38 | 2012.1.00603.S,uid://A002/X6f9b0f/Xc8,8,0.0771629118868 39 | 2012.1.00603.S,uid://A002/X6f9b0f/Xcb,1,0.108307671875 40 | 2012.1.00603.S,uid://A002/X6f9b0f/Xcc,4,0.0286034926 41 | 2012.1.00683.S,uid://A002/X74fe5d/X2d,2,0.0953109664304 42 | 2012.1.00683.S,uid://A002/X74fe5d/X2e,6,0.109402146961 43 | 2012.1.00683.S,uid://A002/X74fe5d/X31,1,0.0537796287791 44 | 2012.1.00683.S,uid://A002/X74fe5d/X32,2,0.103003257617 45 | 2012.1.00762.S,uid://A002/X5a9a13/X684,5,257255.801711 46 | 2012.1.00762.S,uid://A002/X5a9a13/X685,15,257255.433726 47 | 2012.1.00781.S,uid://A002/X628157/X8,1,374312.032747 48 | 2012.1.00932.S,uid://A002/X5eed86/X48,2,0.0160047990968 49 | 2012.1.00940.S,uid://A002/X6444ba/X1b,1,1287291.56682 50 | 2013.1.00214.S,uid://A001/X145/X438,1,0.244677085536 51 | 2013.1.00214.S,uid://A001/X146/Xcb,2,0.252851801312 52 | 2013.1.00214.S,uid://A001/X146/Xcc,4,0.259273490612 53 | 2013.1.00269.S,uid://A001/X121/X4b2,4,0.0802846755656 54 | 2013.1.00269.S,uid://A001/X121/X4b3,4,0.0802846755656 55 | 2013.1.00269.S,uid://A001/X121/X4b4,4,0.0829723569348 56 | 2013.1.00351.S,uid://A001/X122/X4ff,2,0.0737373468181 57 | 2013.1.00351.S,uid://A001/X122/X500,7,0.0756302940259 58 | 2013.1.00351.S,uid://A001/X122/X501,2,0.383881766562 59 | 2013.1.00351.S,uid://A001/X122/X502,5,0.375183698424 60 | 2013.1.00617.S,uid://A001/X197/Xad,6,3.57471130081 61 | 2013.1.00617.S,uid://A001/X197/Xb1,3,2.21110632928 62 | 2013.1.00839.S,uid://A001/X133/X2,1,1.36297214414 63 | 2013.1.00839.S,uid://A001/X133/X3,4,1.35599281699 64 | 2013.1.00839.S,uid://A001/X133/X4,2,0.0914316693354 65 | 2013.1.00839.S,uid://A001/X133/X5,4,0.0883247719615 66 | 2013.1.00839.S,uid://A001/X133/X6,1,0.315132559954 67 | 2013.1.00839.S,uid://A001/X133/X7,2,0.295268798528 68 | 2013.1.00861.S,uid://A001/X11f/X2c,1,0.175370578914 69 | 2013.1.00914.S,uid://A001/X21f/Xcc,1,0.115207453178 70 | 2013.1.00993.S,uid://A002/X996c88/X2c,3,0.124680658309 71 | 2013.1.00993.S,uid://A002/X996c88/X2d,1,0.140803573223 72 | 2013.1.00993.S,uid://A002/X996c88/X2e,1,0.140151215919 73 | 2013.1.01014.S,uid://A001/X145/X3d2,4,3.08095000526 74 | 2013.1.01029.S,uid://A001/X145/X405,1,0.341427724042 75 | 2013.1.01029.S,uid://A001/X145/X406,1,0.321088823119 76 | 2013.1.01029.S,uid://A001/X145/X407,1,0.358096188211 77 | 2013.1.01029.S,uid://A001/X145/X408,1,0.3867823743 78 | 2013.1.01204.S,uid://A001/X197/X39,3,0.345154188383 79 | 2013.1.01204.S,uid://A001/X197/Xea,6,0.306851779519 80 | 2013.1.01242.S,uid://A001/X197/X20,1,0.0150494935533 81 | 2013.1.01242.S,uid://A001/X197/X21,1,0.0145566919713 82 | 2013.1.01391.S,uid://A001/X147/X261,2,0.217121765921 83 | 2013.1.01391.S,uid://A001/X147/X262,3,0.200864201049 84 | 2013.1.01391.S,uid://A001/X147/X265,2,0.216558755759 85 | 2013.1.01391.S,uid://A001/X147/X266,3,0.193657630859 86 | 2013.1.01391.S,uid://A001/X147/X269,2,0.120859411443 87 | 2013.1.01391.S,uid://A001/X147/X26a,3,0.108650951972 88 | 2013.1.01391.S,uid://A001/X147/X26d,2,0.109361661718 89 | 2013.1.01391.S,uid://A001/X147/X26e,3,0.106126062929 90 | 2013.1.01391.S,uid://A001/X147/X271,2,0.1964650045 91 | 2013.1.01391.S,uid://A001/X147/X272,3,0.21350530309 92 | 2013.1.01391.S,uid://A001/X147/X275,1,0.195959284859 93 | 2013.1.01391.S,uid://A001/X147/X276,5,0.208088214602 94 | 2015.1.00357.S,uid://A001/X2d8/X9d,4,0.099461770072 95 | 2015.1.00357.S,uid://A001/X2d8/X9e,1,0.0998010699117 96 | 2015.1.00357.S,uid://A001/X2d8/Xa1,4,0.161806292907 97 | 2015.1.00357.S,uid://A001/X2d8/Xa2,7,0.171759450848 98 | 2015.1.00357.S,uid://A001/X2d8/Xa5,2,0.21821605215 99 | 2015.1.00357.S,uid://A001/X2d8/Xa9,2,0.171103096826 100 | 2015.1.00357.S,uid://A001/X2d8/Xad,2,0.229753380716 101 | -------------------------------------------------------------------------------- /csv2999.csv: -------------------------------------------------------------------------------- 1 | uid___A002_X7143f6_X1369 2 | uid___A002_X7143f6_X15a 3 | uid___A002_X7143f6_X1779 4 | uid___A002_X7143f6_X6e5 5 | uid___A002_X7143f6_Xa90 6 | uid___A002_X7143f6_Xf73 7 | uid___A002_X715a3e_X13b 8 | uid___A002_X716e68_X21a 9 | uid___A002_X717cdc_X213 10 | uid___A002_X71895d_X127 11 | uid___A002_X71895d_X509 12 | uid___A002_X71895d_X88f 13 | uid___A002_X71895d_Xb83 14 | uid___A002_X71895d_Xf81 15 | uid___A002_X71a45c_X5c 16 | uid___A002_X71a45c_X668 17 | uid___A002_X71a45c_X9dc 18 | uid___A002_X71a45c_Xe44 19 | uid___A002_X71cec2_X535 20 | uid___A002_X71cec2_X936 21 | uid___A002_X71e4ae_X2c1 22 | uid___A002_X71e4ae_X6dc 23 | uid___A002_X71e4ae_Xb04 24 | uid___A002_X71e4ae_Xf2d 25 | uid___A002_X72bc38_X2d8 26 | uid___A002_X72bc38_X498 27 | uid___A002_X72bc38_X741 28 | uid___A002_X72bc38_X7a 29 | uid___A002_X72c4aa_X18e 30 | uid___A002_X72c4aa_X614 31 | uid___A002_X72d23d_X1d4 32 | uid___A002_X72d23d_X5d2 33 | uid___A002_X72d23d_X89e 34 | uid___A002_X72e960_X1219 35 | uid___A002_X72e960_X161c 36 | uid___A002_X72e960_X515 37 | uid___A002_X72e960_X913 38 | uid___A002_X72e960_X923 39 | uid___A002_X72e960_Xdcc 40 | uid___A002_X72e960_Xe7b 41 | uid___A002_X72e960_Xfd8 42 | uid___A002_X7310ce_X520 43 | uid___A002_X7310ce_X978 44 | uid___A002_X7330a2_X393 45 | uid___A002_X7330a2_X769 46 | uid___A002_X7330a2_X77 47 | uid___A002_X735201_X2c6 48 | uid___A002_X735201_X38a 49 | uid___A002_X735201_X4a3 50 | uid___A002_X7363c7_X1d5 51 | uid___A002_X7363c7_X431 52 | uid___A002_X736ee1_X1f3 53 | uid___A002_X736ee1_X456 54 | uid___A002_X736ee1_X69e 55 | uid___A002_X74821d_X109b 56 | uid___A002_X74821d_X1272 57 | uid___A002_X74821d_X16bf 58 | uid___A002_X74821d_X1702 59 | uid___A002_X74821d_Xcec 60 | uid___A002_X74821d_Xeb9 61 | uid___A002_X74a173_X11f3 62 | uid___A002_X74a173_X1536 63 | uid___A002_X74a173_X1a8f 64 | uid___A002_X74a173_X227c 65 | uid___A002_X74a173_X23b7 66 | uid___A002_X74a173_Xacf 67 | uid___A002_X74a173_Xeac 68 | uid___A002_X74d4df_X17c 69 | uid___A002_X74d4df_X4b9 70 | uid___A002_X74d4df_X834 71 | uid___A002_X74d4df_X913 72 | uid___A002_X74deb6_X1304 73 | uid___A002_X74deb6_X14df 74 | uid___A002_X74deb6_X1585 75 | uid___A002_X74deb6_X1787 76 | uid___A002_X74deb6_X292 77 | uid___A002_X74deb6_X44e 78 | uid___A002_X74deb6_X479 79 | uid___A002_X74deb6_X59 80 | uid___A002_X74deb6_X66f 81 | uid___A002_X74deb6_X84b 82 | uid___A002_X74deb6_Xbf5 83 | uid___A002_X74deb6_Xfb1 84 | uid___A002_X74eb2a_X1b7 85 | uid___A002_X74eb2a_X24b 86 | uid___A002_X74eb2a_X5c8 87 | uid___A002_X74eb2a_Xaa4 88 | uid___A002_X74fea5_X105d 89 | uid___A002_X74fea5_X10d0 90 | uid___A002_X74fea5_X1311 91 | uid___A002_X74fea5_X141 92 | uid___A002_X74fea5_X1558 93 | uid___A002_X74fea5_X348 94 | uid___A002_X74fea5_X4fa 95 | uid___A002_X74fea5_X5c 96 | uid___A002_X74fea5_X696 97 | uid___A002_X74fea5_X841 98 | uid___A002_X74fea5_X893 99 | uid___A002_X74fea5_Xa54 100 | uid___A002_X74fea5_Xc53 101 | uid___A002_X74fea5_Xdad 102 | uid___A002_X75ab74_X11f4 103 | uid___A002_X75ab74_X158e 104 | uid___A002_X75ab74_X2d4 105 | uid___A002_X75ab74_X311 106 | uid___A002_X75ab74_X5fd 107 | uid___A002_X75ab74_X84f 108 | uid___A002_X75ab74_Xc7e 109 | uid___A002_X75ab74_Xeb1 110 | uid___A002_X75bfbf_X101 111 | uid___A002_X75bfbf_X1015 112 | uid___A002_X75bfbf_X1130 113 | uid___A002_X75bfbf_X1430 114 | uid___A002_X75bfbf_X1728 115 | uid___A002_X75bfbf_X465 116 | uid___A002_X75bfbf_X466 117 | uid___A002_X75bfbf_X79 118 | uid___A002_X75bfbf_X848 119 | uid___A002_X75bfbf_X9a9 120 | uid___A002_X75bfbf_Xab8 121 | uid___A002_X75bfbf_Xace 122 | uid___A002_X75bfbf_Xce3 123 | uid___A002_X75bfbf_Xd9e 124 | uid___A002_X75d7ca_X117e 125 | uid___A002_X75d7ca_X118b 126 | uid___A002_X75d7ca_X1400 127 | uid___A002_X75d7ca_X166f 128 | uid___A002_X75d7ca_X1e8 129 | uid___A002_X75d7ca_X561 130 | uid___A002_X75d7ca_X7d0 131 | uid___A002_X75d7ca_Xa72 132 | uid___A002_X75d7ca_Xc97 133 | uid___A002_X75d7ca_Xe79 134 | uid___A002_X75d7ca_Xea4 135 | uid___A002_X75f169_X1003 136 | uid___A002_X75f169_X1106 137 | uid___A002_X75f169_X1348 138 | uid___A002_X75f169_X15f6 139 | uid___A002_X75f169_X2dd 140 | uid___A002_X75f169_X4c6 141 | uid___A002_X75f169_X4d9 142 | uid___A002_X75f169_X69 143 | uid___A002_X75f169_X6fc 144 | uid___A002_X75f169_X6fd 145 | uid___A002_X75f169_X8b4 146 | uid___A002_X75f169_X97b 147 | uid___A002_X75f169_Xbf6 148 | uid___A002_X75f169_Xe48 149 | uid___A002_X760eb8_X96 150 | uid___A002_X7116f1_Xed8 151 | uid___A002_X7143f6_X1369 152 | uid___A002_X7143f6_X1369 153 | uid___A002_X7143f6_X15a 154 | uid___A002_X7143f6_X15a 155 | uid___A002_X7143f6_X1779 156 | uid___A002_X7143f6_X6e5 157 | uid___A002_X7143f6_Xa90 158 | uid___A002_X7143f6_Xf73 159 | uid___A002_X716e68_X21a 160 | uid___A002_X717cdc_X213 161 | uid___A002_X71895d_X127 162 | uid___A002_X71895d_X509 163 | uid___A002_X71895d_X509 164 | uid___A002_X71895d_X88f 165 | uid___A002_X71895d_X88f 166 | uid___A002_X71895d_Xb83 167 | uid___A002_X71895d_Xf81 168 | uid___A002_X71895d_Xf81 169 | uid___A002_X71a45c_X5c6 170 | uid___A002_X71a45c_X668 171 | uid___A002_X71a45c_X9dc 172 | uid___A002_X71a45c_Xe44 173 | uid___A002_X71cec2_X535 174 | uid___A002_X71cec2_X936 175 | uid___A002_X71e4ae_X2c1 176 | uid___A002_X71e4ae_X6dc 177 | uid___A002_X71e4ae_Xb04 178 | uid___A002_X71e4ae_Xf2d 179 | uid___A002_X72bc38_X2d8 180 | uid___A002_X72bc38_X2d8 181 | uid___A002_X72bc38_X498 182 | uid___A002_X72bc38_X498 183 | uid___A002_X72bc38_X741 184 | uid___A002_X72bc38_X741 185 | uid___A002_X72bc38_X7a3 186 | uid___A002_X72c4aa_X18e 187 | uid___A002_X72c4aa_X614 188 | uid___A002_X72d23d_X1d4 189 | uid___A002_X72d23d_X5d2 190 | uid___A002_X72d23d_X89e 191 | uid___A002_X72e960_X1219 192 | uid___A002_X72e960_X161c 193 | uid___A002_X72e960_X161c 194 | uid___A002_X72e960_X515 195 | uid___A002_X72e960_X515 196 | uid___A002_X72e960_X913 197 | uid___A002_X72e960_X913 198 | uid___A002_X72e960_X923 199 | uid___A002_X72e960_X923 200 | uid___A002_X72e960_Xdcc 201 | uid___A002_X72e960_Xe7b 202 | uid___A002_X72e960_Xe7b 203 | uid___A002_X72e960_Xfd8 204 | uid___A002_X7310ce_X520 205 | uid___A002_X7310ce_X520 206 | uid___A002_X7310ce_X978 207 | uid___A002_X7310ce_X978 208 | uid___A002_X7330a2_X393 209 | uid___A002_X7330a2_X393 210 | uid___A002_X7330a2_X769 211 | uid___A002_X7330a2_X776 212 | uid___A002_X735201_X2c6 213 | uid___A002_X735201_X38a 214 | uid___A002_X735201_X4a3 215 | uid___A002_X7363c7_X1d5 216 | uid___A002_X7363c7_X431 217 | uid___A002_X736ee1_X1f3 218 | uid___A002_X736ee1_X456 219 | uid___A002_X736ee1_X69e 220 | uid___A002_X74821d_X109b 221 | uid___A002_X74821d_X1272 222 | uid___A002_X74821d_X16bf 223 | uid___A002_X74821d_X1702 224 | uid___A002_X74821d_X1702 225 | uid___A002_X74821d_Xcec 226 | uid___A002_X74821d_Xeb9 227 | uid___A002_X74a173_X11f3 228 | uid___A002_X74a173_X1536 229 | uid___A002_X74a173_X1536 230 | uid___A002_X74a173_X1a8f 231 | uid___A002_X74a173_X227c 232 | uid___A002_X74a173_X227c 233 | uid___A002_X74a173_X23b7 234 | uid___A002_X74a173_Xacf 235 | uid___A002_X74a173_Xacf 236 | uid___A002_X74a173_Xeac 237 | uid___A002_X74d4df_X17c 238 | uid___A002_X74d4df_X17c 239 | uid___A002_X74d4df_X4b9 240 | uid___A002_X74d4df_X834 241 | uid___A002_X74d4df_X834 242 | uid___A002_X74d4df_X913 243 | uid___A002_X74deb6_X1304 244 | uid___A002_X74deb6_X14df 245 | uid___A002_X74deb6_X1585 246 | uid___A002_X74deb6_X1787 247 | uid___A002_X74deb6_X292 248 | uid___A002_X74deb6_X44e 249 | uid___A002_X74deb6_X479 250 | uid___A002_X74deb6_X593 251 | uid___A002_X74deb6_X66f 252 | uid___A002_X74deb6_X84b 253 | uid___A002_X74deb6_X84b 254 | uid___A002_X74deb6_Xbf5 255 | uid___A002_X74deb6_Xfb1 256 | uid___A002_X74deb6_Xfb1 257 | uid___A002_X74eb2a_X1b7 258 | uid___A002_X74eb2a_X24b 259 | uid___A002_X74eb2a_X5c8 260 | uid___A002_X74eb2a_Xaa4 261 | uid___A002_X74fea5_X105d 262 | uid___A002_X74fea5_X10d0 263 | uid___A002_X74fea5_X1311 264 | uid___A002_X74fea5_X141 265 | uid___A002_X74fea5_X1558 266 | uid___A002_X74fea5_X348 267 | uid___A002_X74fea5_X4fa 268 | uid___A002_X74fea5_X5c5 269 | uid___A002_X74fea5_X696 270 | uid___A002_X74fea5_X841 271 | uid___A002_X74fea5_X893 272 | uid___A002_X74fea5_Xa54 273 | uid___A002_X74fea5_Xc53 274 | uid___A002_X74fea5_Xdad 275 | uid___A002_X75ab74_X11f4 276 | uid___A002_X75ab74_X158e 277 | uid___A002_X75ab74_X2d4 278 | uid___A002_X75ab74_X311 279 | uid___A002_X75ab74_X5fd 280 | uid___A002_X75ab74_X84f 281 | uid___A002_X75ab74_Xc7e 282 | uid___A002_X75ab74_Xeb1 283 | uid___A002_X75bfbf_X101 284 | uid___A002_X75bfbf_X1015 285 | uid___A002_X75bfbf_X1130 286 | uid___A002_X75bfbf_X1430 287 | uid___A002_X75bfbf_X1728 288 | uid___A002_X75bfbf_X465 289 | uid___A002_X75bfbf_X466 290 | uid___A002_X75bfbf_X794 291 | uid___A002_X75bfbf_X848 292 | uid___A002_X75bfbf_X9a9 293 | uid___A002_X75bfbf_Xab8 294 | uid___A002_X75bfbf_Xace 295 | uid___A002_X75bfbf_Xce3 296 | uid___A002_X75bfbf_Xd9e 297 | uid___A002_X75d7ca_X117e 298 | uid___A002_X75d7ca_X118b 299 | uid___A002_X75d7ca_X1400 300 | uid___A002_X75d7ca_X166f 301 | uid___A002_X75d7ca_X1e8 302 | uid___A002_X75d7ca_X561 303 | uid___A002_X75d7ca_X561 304 | uid___A002_X75d7ca_X7d0 305 | uid___A002_X75d7ca_Xa72 306 | uid___A002_X75d7ca_Xa72 307 | uid___A002_X75d7ca_Xc97 308 | uid___A002_X75d7ca_Xe79 309 | uid___A002_X75d7ca_Xea4 310 | uid___A002_X75f169_X1003 311 | uid___A002_X75f169_X1106 312 | uid___A002_X75f169_X1348 313 | uid___A002_X75f169_X15f6 314 | uid___A002_X75f169_X2dd 315 | uid___A002_X75f169_X4c6 316 | uid___A002_X75f169_X4d9 317 | uid___A002_X75f169_X690 318 | uid___A002_X75f169_X693 319 | uid___A002_X75f169_X6fc 320 | uid___A002_X75f169_X6fd 321 | uid___A002_X75f169_X8b4 322 | uid___A002_X75f169_X97b 323 | uid___A002_X75f169_Xbf6 324 | uid___A002_X75f169_Xe48 325 | uid___A002_X760eb8_X96 326 | -------------------------------------------------------------------------------- /releaseImage.txt: -------------------------------------------------------------------------------- 1 | |CODE|SB|EB|Beam.10|MaxOffErr 2 | |2012.1.00060.S|uid://A002/X639a2a/X1|uid://A002/X826a79/Xcdb|0.0957382951046|0.218652645262 3 | |2012.1.00080.S|uid://A002/X6444ba/X7|uid://A002/X651f57/Xadd|0.0490915825538|0.06690605946 4 | |2012.1.00080.S|uid://A002/X6444ba/X7|uid://A002/X680f8a/Xadc|0.0230738163701|0.0669060592574 5 | |2012.1.00080.S|uid://A002/X6444ba/X7|uid://A002/X6a533e/Xd66|0.0230738163701|0.0669060592574 6 | |2012.1.00261.S|uid://A002/X6444ba/X9e|uid://A002/X7c6ce6/Xa4|0.0634051978479|0.0855921285605 7 | |2012.1.00261.S|uid://A002/X6444ba/X9f|uid://A002/X7d44e7/X1bf|0.0604955016033|0.0855921285605 8 | |2012.1.00261.S|uid://A002/X6444ba/X9f|uid://A002/X7d44e7/X2dd|0.0604955016033|0.0855921285605 9 | |2012.1.00323.S|uid://A002/X67ccb6/X9|uid://A002/X7776f1/X1ba9|0.0454364680938|0.524202032949 10 | |2012.1.00333.S|uid://A002/X6f9b0f/X106|uid://A002/X71e4ae/X317|0.0208631734278|0.725253073517 11 | |2012.1.00335.S|uid://A002/X75fbd6/X27|uid://A002/X77da97/X43d|0.146355789431|0.252851808197 12 | |2012.1.00335.S|uid://A002/X75fbd6/X27|uid://A002/X77da97/X836|0.146355789431|0.252851808197 13 | |2012.1.00394.S|uid://A002/X639a2a/X3e|uid://A002/X74fea5/X1311|0.150215712506|0.157410817168 14 | |2012.1.00542.S|uid://A002/X6444ba/X33|uid://A002/X7c8369/X64e|0.0420403232|2.83098501364 15 | |2012.1.00542.S|uid://A002/X6444ba/X34|uid://A002/X7d76cc/X520|0.0406568958143|3.22937708025 16 | |2012.1.00542.S|uid://A002/X6444ba/X34|uid://A002/X95b353/Xb11|0.0456261091978|3.22937708025 17 | |2012.1.00554.S|uid://A002/X5d7935/X3a2|uid://A002/X74deb6/X14df|0.0604549750492|0.321383161871 18 | |2012.1.00554.S|uid://A002/X5d7935/X3a2|uid://A002/X7c18c1/X4c7|0.0660123971767|0.324208425379 19 | |2012.1.00554.S|uid://A002/X5d7935/X3a4|uid://A002/X74eb2a/X1b7|0.120907742586|0.299937242716 20 | |2012.1.00554.S|uid://A002/X5d7935/X3a4|uid://A002/X74eb2a/X24b|0.120907742586|0.299822157281 21 | |2012.1.00603.S|uid://A002/X6f9b0f/Xc3|uid://A002/X75f169/X4d9|0.0436798724377|0.113680929036 22 | |2012.1.00603.S|uid://A002/X6f9b0f/Xc7|uid://A002/X75f169/X6fc|0.0436798724377|0.11762937821 23 | |2012.1.00603.S|uid://A002/X6f9b0f/Xcb|uid://A002/X75f169/X8b4|0.0436798724377|0.108307671875 24 | |2012.1.00683.S|uid://A002/X74fe5d/X2d|uid://A002/X75ab74/X5fd|0.041753173045|0.134815248606 25 | |2012.1.00762.S|uid://A002/X5a9a13/X684|uid://A002/X5e6e29/X13|0.0539194241288|0.0979287715419 26 | |2012.1.00762.S|uid://A002/X5a9a13/X684|uid://A002/X680f8a/X6b5|0.0196466258543|0.0979287715419 27 | |2012.1.00762.S|uid://A002/X5a9a13/X684|uid://A002/X6a1245/X23c|0.041727098547|0.0979287715419 28 | |2012.1.00932.S|uid://A002/X5eed86/X48|uid://A002/X6a533e/X123c|0.0103448685257|0.0160047990968 29 | |2013.1.00214.S|uid://A001/X145/X438|uid://A002/X98ed3f/X1cf5|0.153291792381|0.244677085536 30 | |2013.1.00214.S|uid://A001/X146/Xcb|uid://A002/X98ed3f/X21cf|0.160344029134|0.252851801312 31 | |2013.1.00214.S|uid://A001/X146/Xcb|uid://A002/X9a24bb/X1460|0.160344029134|0.252851801312 32 | |2013.1.00269.S|uid://A001/X121/X4b2|uid://A002/Xa4ce71/X49f|0.0430518614977|0.0802846755656 33 | |2013.1.00269.S|uid://A001/X121/X4b2|uid://A002/Xa4dcb9/X8ac|0.0430518614977|0.0802846755656 34 | |2013.1.00269.S|uid://A001/X121/X4b2|uid://A002/Xa4dcb9/Xbad|0.0430518614977|0.0802846755656 35 | |2013.1.00269.S|uid://A001/X121/X4b2|uid://A002/Xa4dcb9/Xebe|0.0430518614977|0.0802846755656 36 | |2013.1.00351.S|uid://A001/X122/X501|uid://A002/X966cea/X88|0.153240802298|0.383881766562 37 | |2013.1.00351.S|uid://A001/X122/X501|uid://A002/X966cea/X35d|0.153240802298|0.383881766562 38 | |2013.1.00617.S|uid://A001/X197/Xad|uid://A002/Xa8666f/X1f3|0.501821918651|3.57471130081 39 | |2013.1.00617.S|uid://A001/X197/Xad|uid://A002/Xa8666f/X587|0.501821918651|3.57471130081 40 | |2013.1.00617.S|uid://A001/X197/Xad|uid://A002/Xa8666f/X8f4|0.501821918651|3.57471130081 41 | |2013.1.00617.S|uid://A001/X197/Xad|uid://A002/Xa9a44e/X132|0.501821918651|3.57471130081 42 | |2013.1.00617.S|uid://A001/X197/Xad|uid://A002/Xa9a44e/X4ae|0.501821918651|3.57471130081 43 | |2013.1.00617.S|uid://A001/X197/Xad|uid://A002/Xaa8932/X148f|0.501821918651|3.57471130081 44 | |2013.1.00617.S|uid://A001/X197/Xb1|uid://A002/Xa7f13b/X194|0.501821918651|2.21110632928 45 | |2013.1.00617.S|uid://A001/X197/Xb1|uid://A002/Xa81701/X1fc|0.501821918651|2.21110632928 46 | |2013.1.00617.S|uid://A001/X197/Xb1|uid://A002/Xa81701/X75b|0.501821918651|2.21110632928 47 | |2013.1.00839.S|uid://A001/X133/X2|uid://A002/X9f9284/X175e|0.0804026850404|1.36297214414 48 | |2013.1.00839.S|uid://A001/X133/X3|uid://A002/X892c75/X30f|0.572540818707|1.35599281699 49 | |2013.1.00839.S|uid://A001/X133/X3|uid://A002/X894188/X429|0.572540818707|1.35599281699 50 | |2013.1.00839.S|uid://A001/X133/X3|uid://A002/X895c89/X76c|0.572540818707|1.35599281699 51 | |2013.1.00839.S|uid://A001/X133/X3|uid://A002/X8970e9/X1f4e|0.572540818707|1.35599281699 52 | |2013.1.00839.S|uid://A001/X133/X4|uid://A002/X9f15bd/X17f|0.0804026850404|0.0914316693354 53 | |2013.1.00839.S|uid://A001/X133/X4|uid://A002/X9f15bd/X716|0.0804026850404|0.0914316693354 54 | |2013.1.00839.S|uid://A001/X133/X6|uid://A002/X9fddd8/X2172|0.0804026850404|0.315132559954 55 | |2013.1.00861.S|uid://A001/X11f/X2c|uid://A002/X804fed/Xa46|0.0481201114267|0.175370578914 56 | |2013.1.00914.S|uid://A001/X21f/Xcc|uid://A002/Xa9c359/X726|0.0355695209995|0.115207453178 57 | |2013.1.00993.S|uid://A002/X996c88/X2e|uid://A002/X9d9aa5/Xe7c|0.0766473603677|0.140151215919 58 | |2013.1.01014.S|uid://A001/X145/X3d2|uid://A002/X960614/X194c|0.197929719318|3.08095000526 59 | |2013.1.01014.S|uid://A001/X145/X3d2|uid://A002/X9896b4/X392c|0.197929453555|3.08095000526 60 | |2013.1.01014.S|uid://A001/X145/X3d2|uid://A002/X9f15bd/X10cf|0.197929453555|3.08095000526 61 | |2013.1.01014.S|uid://A001/X145/X3d2|uid://A002/X9f15bd/X153e|0.197929453555|3.08095000526 62 | |2013.1.01029.S|uid://A001/X145/X405|uid://A002/X99c183/X7b2e|0.203283166556|0.341427724042 63 | |2013.1.01029.S|uid://A001/X145/X406|uid://A002/X9a755e/X15e7|0.189439690189|0.321088823119 64 | |2013.1.01029.S|uid://A001/X145/X407|uid://A002/X9a5759/X28b8|0.203348133882|0.358096188211 65 | |2013.1.01029.S|uid://A001/X145/X408|uid://A002/X9a8f80/X398|0.189439690189|0.3867823743 66 | |2013.1.01204.S|uid://A001/X197/X39|uid://A002/Xa7dc73/X12aa|0.0421817888232|0.345154188383 67 | |2013.1.01204.S|uid://A001/X197/X39|uid://A002/Xa8df68/Xfc|0.0421817888232|0.345154188383 68 | |2013.1.01204.S|uid://A001/X197/X39|uid://A002/Xa916fc/X561e|0.0450364020443|0.345154188383 69 | |2013.1.01242.S|uid://A001/X197/X21|uid://A002/Xaaa05f/X37a|0.0124934818872|0.0145566919713 70 | |2013.1.01391.S|uid://A001/X147/X261|uid://A002/X9dcf39/X28ff|0.1991711688|0.217121765921 71 | |2013.1.01391.S|uid://A001/X147/X261|uid://A002/X9e0695/X388a|0.1991711688|0.217121765921 72 | |2013.1.01391.S|uid://A001/X147/X265|uid://A002/X9de499/X2f4a|0.199170371575|0.216558755759 73 | |2013.1.01391.S|uid://A001/X147/X265|uid://A002/X9e0695/X2b52|0.199170371575|0.216558755759 74 | |2013.1.01391.S|uid://A001/X147/X271|uid://A002/X9de499/X4652|0.189462434395|0.1964650045 75 | |2013.1.01391.S|uid://A001/X147/X271|uid://A002/X9e0695/X32ff|0.189462434395|0.1964650045 76 | |2013.1.01391.S|uid://A001/X147/X275|uid://A002/X9e0695/X15c1|0.189461676031|0.195959284859 77 | |2015.1.00357.S|uid://A001/X2d8/Xad|uid://A002/Xae75f3/Xf18|0.0857468191354|0.229753380716 78 | |2015.1.00357.S|uid://A001/X2d8/Xad|uid://A002/Xae75f3/X1211|0.0857468191354|0.229753380716 79 | |2012.1.00080.S|uid://A002/X6444ba/X8|uid://A002/X717cdc/X213|1.28296097738|1288167.24584 80 | |2012.1.00080.S|uid://A002/X6444ba/X8|uid://A002/X71a45c/X5c|1.28296097738|1288167.24365 81 | |2012.1.00080.S|uid://A002/X6444ba/X8|uid://A002/X7330a2/X77|1.28296097738|1288167.25293 82 | |2012.1.00080.S|uid://A002/X6444ba/X8|uid://A002/X734e5c/X7e|1.28296097738|1288167.24548 83 | |2012.1.00385.S|uid://A002/X6b0cc1/Xa0|uid://A002/X75bfbf/X1430|0.0413931240849|1295920.78126 84 | |2012.1.00385.S|uid://A002/X6b0cc1/Xa0|uid://A002/X75bfbf/X1728|0.0413931240849|1295920.44799 85 | |2012.1.00394.S|uid://A002/X639a2a/X3a|uid://A002/X775008/X3d9|0.0542708219873|40.7996913997 86 | |2012.1.00394.S|uid://A002/X639a2a/X3c|uid://A002/X7776f1/X60|0.0516253542969|40.7996913997 87 | |2012.1.00394.S|uid://A002/X639a2a/X3c|uid://A002/X7776f1/X4a4|0.0516253542969|40.7996913997 88 | |2012.1.00394.S|uid://A002/X639a2a/X40|uid://A002/X74fea5/X105d|0.142893346648|403.776318195 89 | |2012.1.00501.S|uid://A002/X5d5f8a/X3|uid://A002/X75f169/X1003|0.041986240541|1097897.21299 90 | |2012.1.00762.S|uid://A002/X5a9a13/X684|uid://A002/X74821d/X16bf|0.0417999421865|1286278.61684 91 | |2012.1.00762.S|uid://A002/X5a9a13/X685|uid://A002/X74deb6/X1304|1.09240075001|1286274.39354 92 | |2012.1.00762.S|uid://A002/X5a9a13/X685|uid://A002/X74fea5/X10d0|1.09240075001|1286276.77128 93 | |2012.1.00762.S|uid://A002/X5a9a13/X685|uid://A002/X74fea5/X1558|1.09240075001|1286279.20802 94 | |2012.1.00781.S|uid://A002/X628157/X8|uid://A002/X74deb6/X1787|0.366558167333|374312.032747 95 | |2012.1.00844.S|uid://A002/X5a9a13/X545|uid://A002/X69fe48/X18b|0.0564901316411|443987.760261 96 | |2012.1.00844.S|uid://A002/X5a9a13/X545|uid://A002/X75f169/X173|0.0565888080233|443987.760261 97 | |2012.1.00844.S|uid://A002/X5a9a13/X546|uid://A002/X775008/Xfd|0.0541976788999|483217.315729 98 | |2012.1.00844.S|uid://A002/X5a9a13/X547|uid://A002/X775008/X26f|0.0520004757013|437781.372679 99 | |2012.1.00844.S|uid://A002/X5a9a13/X548|uid://A002/X75d7ca/X361|0.143555180579|489087.540012 100 | |2012.1.00844.S|uid://A002/X5a9a13/X549|uid://A002/X75f169/X33f|0.0481004868198|489087.540012 101 | |2012.1.00940.S|uid://A002/X6444ba/X1b|uid://A002/X715a3e/X13b|0.54527149331|1287291.56682 102 | |2012.1.01123.S|uid://A002/X6802f4/Xc|uid://A002/Xa25bbf/X5cd|0.0128234876246|1296000.0 103 | |2012.1.01123.S|uid://A002/X6802f4/Xc|uid://A002/Xa2d681/X354|0.0128234876246|1296000.0 104 | -------------------------------------------------------------------------------- /MetaData.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sagonzal' 2 | from checker import * 3 | import pylab as mypl 4 | import math as mymath 5 | import itertools 6 | 7 | class AsdmCheck: 8 | """ 9 | 10 | """ 11 | uid = '' 12 | asdmDict = dict() 13 | check = dict() 14 | toc = '' 15 | 16 | 17 | 18 | 19 | def setUID(self,uid=None): 20 | self.uid = uid 21 | try: 22 | asdmList, asdm , self.toc = getASDM(self.uid) 23 | for i in asdmList: 24 | self.asdmDict[i[0].strip()] = i[3].strip() 25 | return True 26 | except Exception as e: 27 | print 'There is a problem with the uid: ', uid 28 | print e 29 | 30 | 31 | def isNullState(self): 32 | """ 33 | Checks the Main.xml table for "null" states 34 | Sets in the self.check dictionary the value 'NullState' with True or False 35 | True: There is no state with null values 36 | False: There is at least one state in the Main.xml table with null state. 37 | 38 | :return: 39 | """ 40 | try: 41 | main = getMain(self.asdmDict['Main']) 42 | main['null'] = main.apply(lambda x: True if 'null' in x['stateId'] else False, axis = 1) 43 | if len(main['null'].unique()) > 1: 44 | self.check['NullState'] = False 45 | else: 46 | self.check['NullState'] = True 47 | except Exception as e: 48 | print e 49 | return False 50 | return True 51 | 52 | 53 | def isValidUID(self): 54 | """ 55 | 56 | :return: 57 | """ 58 | import re 59 | regex = re.compile("^uid\:\/\/A00.\/X[a-zA-Z0-9]+\/X[a-zA-Z0-9]+") 60 | try: 61 | for k,v in self.asdmDict.iteritems(): 62 | if '/X0/X0/X0' in v: 63 | self.check['ValidUID'] = False 64 | return True 65 | if regex.match(v) is None: 66 | self.check['ValidUID'] = False 67 | return True 68 | self.check['ValidUID'] = True 69 | return True 70 | except Exception as e: 71 | print e 72 | return False 73 | 74 | def isSyscaltimestamp(self): 75 | try: 76 | syscal = getSysCal(self.asdmDict['SysCal']) 77 | dfa = syscal[['spectralWindowId','antennaId','timeInterval']] 78 | spw = dfa.groupby('spectralWindowId') 79 | for name,group in spw: 80 | df = group 81 | df['time'] = df.apply(lambda x: int(x['timeInterval'].strip().split(' ')[0]) / 1000000000.0, axis = 1 ) 82 | df['interval'] = df.apply(lambda x: int(x['timeInterval'].strip().split(' ')[1]) / 1000000000.0, axis = 1 ) 83 | df['timestamp'] = df.apply(lambda x: x.time - x.interval / 2, axis = 1) 84 | t0 = 86400.0 * mymath.floor(df.timestamp.min() / 86400.0) 85 | df['utimes'] = df.apply(lambda x: x['time'] - t0, axis =1) 86 | nT = df.utimes.nunique() 87 | df['utimestamp'] = df.apply(lambda x: mypl.floor(x['timestamp']) - t0 , axis =1) 88 | nTS = df.utimestamp.nunique() 89 | #print name,nT, nTS 90 | #print (group) 91 | if nT != nTS: 92 | self.check['SysCalTimes'] = True 93 | return True 94 | self.check['SysCalTimes'] = False 95 | return True 96 | except Exception as e: 97 | return False 98 | 99 | 100 | def iscsv2555(self): 101 | try: 102 | source = getSource(self.asdmDict['Source']) 103 | field = getField(self.asdmDict['Field']) 104 | src = source[['sourceId', 'sourceName']] 105 | src['sourceName1'] = src.apply(lambda x: x['sourceName'].strip(), axis = 1) 106 | src = src.drop_duplicates() 107 | fld = field[['sourceId', 'fieldName']] 108 | fld['fieldName1'] = fld.apply(lambda x: x['fieldName'].strip(), axis = 1) 109 | fld = fld.drop_duplicates() 110 | a = pd.merge(src,fld,left_on = 'sourceId',right_on='sourceId',how='outer') 111 | a['csv2555'] = a.apply(lambda x: True if x['sourceName1'] == x['fieldName1'] else False, axis = 1) 112 | if a['csv2555'].nunique() == 1 and a['csv2555'].unique()[0] is True: 113 | self.check['CSV2555'] = True 114 | else: 115 | self.check['CSV2555'] = False 116 | return True 117 | except Exception as e: 118 | return False 119 | 120 | 121 | def isfixplanets(self): 122 | try: 123 | source = getSource(self.asdmDict['Source']) 124 | df = source[['sourceName','direction']].drop_duplicates() 125 | df['coordinate'] = df.apply(lambda x: True if float(arrayParser(x['direction'].strip() , 1 )[0]) == 0.0 else False , axis = 1) 126 | df['coordinate2'] = df.apply(lambda x: True if float(arrayParser(x['direction'].strip() , 1 )[1]) == 0.0 else False , axis = 1) 127 | if df['coordinate'].unique()[0] is False and df['coordinate'].nunique() == 1 and df['coordinate2'].unique()[0] is False and df['coordinate2'].nunique() == 1: 128 | self.check['FixPlanets'] = True 129 | else: 130 | self.check['FixPlanets'] = False 131 | return True 132 | except Exception as e: 133 | return False 134 | 135 | def ict4871(self): 136 | try: 137 | ant = getAntennas(self.asdmDict['Antenna']) 138 | nant = ant.antennaId.nunique() 139 | syscal = getSysCal(self.asdmDict['SysCal']) 140 | sys_nant = syscal.antennaId.nunique() 141 | scan = getScan(self.asdmDict['Scan']) 142 | sc = scan[['scanNumber','startTime','scanIntent']] 143 | problem_list = list() 144 | if nant == sys_nant: 145 | df = syscal[['timeInterval','antennaId','spectralWindowId']] 146 | df['start'] = df.apply(lambda x: int(x['timeInterval'].strip().split(' ')[0]) - int(x['timeInterval'].strip().split(' ')[1])/2, axis = 1) 147 | df2 = pd.merge (df,sc, left_on='start',right_on='startTime',copy=False,how='inner') 148 | antlist = [x.strip(' ') for x in ant.antennaId.unique().tolist()] 149 | df3 = df2.groupby(['antennaId','spectralWindowId','scanNumber']) 150 | spw_list = syscal.spectralWindowId.unique().tolist() 151 | scan_list = df2.scanNumber.unique().tolist() 152 | fu = list(itertools.product(antlist,spw_list,scan_list)) 153 | for i in fu: 154 | try: 155 | df3.groups[i] 156 | except KeyError as k: 157 | problem_list.append(i) 158 | 159 | if len(problem_list) > 0: 160 | df = pd.DataFrame(problem_list, columns= ['antennaId','spectralWindowId','scanNumber']) 161 | popular_scan = df.scanNumber.mode().values[0] 162 | antennas = df.antennaId.nunique() 163 | self.check['MissingTsys'] = False 164 | self.check['MissingTsys_explain'] = 'Scan: '+str(popular_scan)+' Antennas Affected: '+str(antennas) 165 | else: 166 | self.check['MissingTsys'] = True 167 | 168 | else: 169 | self.check['AntennasMissing'] = "Number of antennas are diferent between Antenna.xml ("+nant+") and SysCal.xml ("+sys_nant+")" 170 | self.check['MissingTsys'] = False 171 | self.check['MissingTsys_explain'] = 'Some antenna is completly missing from Syscal.xml table' 172 | 173 | 174 | except Exception as e: 175 | print e 176 | return False 177 | 178 | def doCheck(self): 179 | self.iscsv2555() 180 | self.isSyscaltimestamp() 181 | self.isValidUID() 182 | self.isfixplanets() 183 | self.isNullState() 184 | self.ict4871() 185 | 186 | 187 | 188 | def save(self): 189 | if len(self.check) != 0: 190 | sql = "select * from public.metadata where asdm_uid ='"+self.uid+"'" 191 | pgcursor.execute(sql) 192 | row = pgcursor.fetchone() 193 | if self.check['SysCalTimes'] and self.check['ValidUID'] and self.check['CSV2555'] and self.check['FixPlanets'] and self.check['NullState']: 194 | overall = True 195 | else: 196 | overall = False 197 | #print row 198 | if row is not None: 199 | print "ASDM: %s exist. UPDATING" % self.uid 200 | try: 201 | pgcursor.execute("""UPDATE public.metadata set "check" = %s, 202 | syscal_time = %s, 203 | valid_uid = %s, 204 | csv2555 = %s, 205 | fixplanet = %s, 206 | null_state = %s, 207 | udate = %s 208 | where asdm_uid = %s;""", 209 | ( overall, self.check['SysCalTimes'], self.check['ValidUID'], self.check['CSV2555'], 210 | self.check['FixPlanets'],self.check['NullState'],datetime.datetime.now (),self.uid,)) 211 | pgconn.commit() 212 | except Exception as e: 213 | print e 214 | pass 215 | else: 216 | print "ASDM: %s exist. INSERTING" % self.uid 217 | try: 218 | pgcursor.execute("""INSERT INTO "public"."metadata" ("asdm_uid","check","syscal_time","valid_uid","csv2555","fixplanet","null_state","cdate","udate","time_of_creation") 219 | values(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s);""", 220 | (self.uid, overall, self.check['SysCalTimes'], self.check['ValidUID'], self.check['CSV2555'], 221 | self.check['FixPlanets'],self.check['NullState'],datetime.datetime.now (),datetime.datetime.now (),self.toc)) 222 | pgconn.commit() 223 | except Exception as e: 224 | print e 225 | pass 226 | else: 227 | print "Run docheck() before save" 228 | pass 229 | 230 | 231 | 232 | 233 | 234 | 235 | -------------------------------------------------------------------------------- /final.csv: -------------------------------------------------------------------------------- 1 | uid://A002/Xac2df7/X127c|uid://A001/X2d1/X7f|2015.1.00979.S 2 | uid://A002/Xac3e1a/X46ec|uid://A001/X2d1/X7f|2015.1.00979.S 3 | uid://A002/Xac3e1a/X43aa|uid://A001/X2d1/X7f|2015.1.00979.S 4 | uid://A002/Xac5575/X8ed9|uid://A001/X2d1/X91|2015.1.01512.S 5 | uid://A002/Xac9e3c/X8bb|uid://A001/X2d1/X91|2015.1.01512.S 6 | uid://A002/Xac0269/X1123|uid://A001/X2d1/Xc1|2015.1.00896.S 7 | uid://A002/Xac0269/X18dd|uid://A001/X2d1/Xc1|2015.1.00896.S 8 | uid://A002/Xabcd5b/X1097|uid://A001/X2d1/Xc1|2015.1.00896.S 9 | uid://A002/Xabc3f8/X3aa5|uid://A001/X2d1/Xf0|2015.1.00702.S 10 | uid://A002/Xabe765/X5bd|uid://A001/X2d1/Xf0|2015.1.00702.S 11 | uid://A002/Xac3e1a/X4a3b|uid://A001/X2d1/Xf0|2015.1.00702.S 12 | uid://A002/Xac0269/X756|uid://A001/X2d2/X65|2015.1.01163.S 13 | uid://A002/Xac5575/X7ee8|uid://A001/X2d2/X89|2015.1.00773.S 14 | uid://A002/Xac2df7/X1967|uid://A001/X2d6/X1c|2015.1.00113.S 15 | uid://A002/Xac2df7/X1b18|uid://A001/X2d6/X1d|2015.1.00113.S 16 | uid://A002/Xabf8b9/X1330|uid://A001/X2d6/X2ae|2015.1.01268.S 17 | uid://A002/Xac5575/X5f69|uid://A001/X2d6/X2ae|2015.1.01268.S 18 | uid://A002/Xac4b9f/X106f|uid://A001/X2d6/X2ae|2015.1.01268.S 19 | uid://A002/Xac5575/X1139|uid://A001/X2d6/X2ae|2015.1.01268.S 20 | uid://A002/Xac5575/X1756|uid://A001/X2d6/X2ae|2015.1.01268.S 21 | uid://A002/Xac5575/X1fd0|uid://A001/X2d6/X2ae|2015.1.01268.S 22 | uid://A002/Xac2df7/X160b|uid://A001/X2d6/X2b4|2015.1.01352.S 23 | uid://A002/Xac2df7/Xdf7|uid://A001/X2d6/X2c4|2015.1.00350.S 24 | uid://A002/Xac3e1a/X1aa2|uid://A001/X2d6/X2fa|2015.1.00341.S 25 | uid://A002/Xac3e1a/X269f|uid://A001/X2d6/X2fa|2015.1.00341.S 26 | uid://A002/Xad47ae/X1256|uid://A001/X2d6/X309|2015.1.00496.S 27 | uid://A002/Xad5116/Xea3|uid://A001/X2d6/X309|2015.1.00496.S 28 | uid://A002/Xac8406/X18bb|uid://A001/X2d6/X309|2015.1.00496.S 29 | uid://A002/Xac8406/X1bd5|uid://A001/X2d6/X309|2015.1.00496.S 30 | uid://A002/Xac0269/X22e0|uid://A001/X2d6/X310|2015.1.01518.S 31 | uid://A002/Xac0269/X25cd|uid://A001/X2d6/X310|2015.1.01518.S 32 | uid://A002/Xac5575/X3619|uid://A001/X2d6/X340|2015.1.01602.S 33 | uid://A002/Xac5575/X815|uid://A001/X2d6/X341|2015.1.01602.S 34 | uid://A002/Xac5575/X2938|uid://A001/X2d6/X342|2015.1.01602.S 35 | uid://A002/Xac5575/X340e|uid://A001/X2d6/X343|2015.1.01602.S 36 | uid://A002/Xac5575/X19e|uid://A001/X2d6/Xe|2015.1.01596.S 37 | uid://A002/Xac5575/X3bf|uid://A001/X2d6/Xe|2015.1.01596.S 38 | uid://A002/Xac5575/X43ef|uid://A001/X2d8/X329|2015.1.01384.S 39 | uid://A002/Xac5575/X49a6|uid://A001/X2d8/X329|2015.1.01384.S 40 | uid://A002/Xac0269/Xe15|uid://A001/X2d8/X380|2015.1.01290.S 41 | uid://A002/Xac5575/X8a5f|uid://A001/X2d8/X391|2015.1.00370.S 42 | uid://A002/Xac5575/X2c41|uid://A001/X2d8/X3b|2015.1.00078.S 43 | uid://A002/Xac3425/X29e2|uid://A001/X2d8/X3b|2015.1.00078.S 44 | uid://A002/Xac5575/X9911|uid://A001/X2d8/X40b|2015.1.01384.S 45 | uid://A002/Xac5575/X9bb6|uid://A001/X2d8/X40b|2015.1.01384.S 46 | uid://A002/Xacbbd0/X2017|uid://A001/X2d8/X40c|2015.1.01384.S 47 | uid://A002/Xacbbd0/X234f|uid://A001/X2d8/X40c|2015.1.01384.S 48 | uid://A002/Xacbbd0/X23d0|uid://A001/X2d8/X40c|2015.1.01384.S 49 | uid://A002/Xad2075/Xeb1|uid://A001/X2d8/X411|2015.1.01302.S 50 | uid://A002/Xad2439/Xee6|uid://A001/X2d8/X412|2015.1.01302.S 51 | uid://A002/Xad2439/X2bc7|uid://A001/X2d8/X413|2015.1.01302.S 52 | uid://A002/Xad2439/X2db5|uid://A001/X2d8/X413|2015.1.01302.S 53 | uid://A002/Xad565b/X1ae7|uid://A001/X2d8/X432|2015.1.00480.S 54 | uid://A002/Xad565b/X1da0|uid://A001/X2d8/X432|2015.1.00480.S 55 | uid://A002/Xad565b/X20c4|uid://A001/X2d8/X432|2015.1.00480.S 56 | uid://A002/Xac2df7/X1146|uid://A001/X2d8/X445|2015.1.01323.S 57 | uid://A002/Xac0269/X161a|uid://A001/X2d8/X46|2015.1.01415.S 58 | uid://A002/Xac4b9f/X9f2|uid://A001/X2d8/X46|2015.1.01415.S 59 | uid://A002/Xacdf75/X2815|uid://A001/X2d8/X66|2015.1.01302.S 60 | uid://A002/Xacc4e4/X219|uid://A002/Xaa5ac1/X1|2015.1.00502.S 61 | uid://A002/Xacc4e4/X346|uid://A002/Xaa5ac1/X1|2015.1.00502.S 62 | uid://A002/Xac5575/X944c|uid://A002/Xaa5ac1/X1|2015.1.00502.S 63 | uid://A002/Xad2439/X1366|uid://A001/X2c9/X19|2015.1.00597.S 64 | uid://A002/Xad057f/X1dc5|uid://A001/X2c9/X19|2015.1.00597.S 65 | uid://A002/Xac5575/X9ef|uid://A001/X2c9/X1a|2015.1.00597.S 66 | uid://A002/Xac5575/Xbf8|uid://A001/X2c9/X1a|2015.1.00597.S 67 | uid://A002/Xad6824/X396b|uid://A001/X2d2/X64|2015.1.01163.S 68 | uid://A002/Xad6824/X41ef|uid://A001/X2d2/X64|2015.1.01163.S 69 | uid://A002/Xac0269/X756|uid://A001/X2d2/X65|2015.1.01163.S 70 | uid://A002/Xac5575/X7ee8|uid://A001/X2d2/X89|2015.1.00773.S 71 | uid://A002/Xac5575/X6cf8|uid://A001/X2d2/Xaf|2015.1.00631.S 72 | uid://A002/Xac97ce/X1bd8|uid://A001/X2d2/Xaf|2015.1.00631.S 73 | uid://A002/Xac5575/Xb9e6|uid://A001/X2d2/Xb0|2015.1.00631.S 74 | uid://A002/Xac5575/Xb06f|uid://A001/X2d2/Xb0|2015.1.00631.S 75 | uid://A002/Xacca73/X2d12|uid://A001/X2d2/Xb3|2015.1.00631.S 76 | uid://A002/Xac5575/Xa01b|uid://A001/X2d2/Xb3|2015.1.00631.S 77 | uid://A002/Xac5575/X3946|uid://A001/X2d2/Xcb|2015.1.00761.S 78 | uid://A002/Xad2439/X1769|uid://A001/X2d2/Xcb|2015.1.00761.S 79 | uid://A002/Xad3217/X4037|uid://A001/X2d6/X14|2015.1.00686.S 80 | uid://A002/Xad0ecf/X21c3|uid://A001/X2d6/X14|2015.1.00686.S 81 | uid://A002/Xad47ae/Xa73|uid://A001/X2d6/X14|2015.1.00686.S 82 | uid://A002/Xac2df7/X1967|uid://A001/X2d6/X1c|2015.1.00113.S 83 | uid://A002/Xad47ae/Xdc1|uid://A001/X2d6/X293|2015.1.00555.S 84 | uid://A002/Xad47ae/Xf76|uid://A001/X2d6/X294|2015.1.00555.S 85 | uid://A002/Xac5575/X2f95|uid://A001/X2d6/X2b5|2015.1.01352.S 86 | uid://A002/Xad2439/X15f3|uid://A001/X2d6/X2b5|2015.1.01352.S 87 | uid://A002/Xac5575/X2e4f|uid://A001/X2d6/X2b6|2015.1.01352.S 88 | uid://A002/Xad2439/X3110|uid://A001/X2d6/X2b6|2015.1.01352.S 89 | uid://A002/Xad47ae/X1256|uid://A001/X2d6/X309|2015.1.00496.S 90 | uid://A002/Xad5116/Xea3|uid://A001/X2d6/X309|2015.1.00496.S 91 | uid://A002/Xac8406/X18bb|uid://A001/X2d6/X309|2015.1.00496.S 92 | uid://A002/Xac8406/X1bd5|uid://A001/X2d6/X309|2015.1.00496.S 93 | uid://A002/Xac0269/X22e0|uid://A001/X2d6/X310|2015.1.01518.S 94 | uid://A002/Xac0269/X25cd|uid://A001/X2d6/X310|2015.1.01518.S 95 | uid://A002/Xacdf75/X1d1f|uid://A001/X2d6/X311|2015.1.01518.S 96 | uid://A002/Xacdf75/X22b9|uid://A001/X2d6/X311|2015.1.01518.S 97 | uid://A002/Xacca73/X6c87|uid://A001/X2d6/X311|2015.1.01518.S 98 | uid://A002/Xac5575/X3dd3|uid://A001/X2d6/X31b|2015.1.01227.S 99 | uid://A002/Xacbbd0/Xef4|uid://A001/X2d6/X322|2015.1.00607.S 100 | uid://A002/Xad177d/X1b39|uid://A001/X2d6/X322|2015.1.00607.S 101 | uid://A002/Xad5f93/Xa0c|uid://A001/X2d6/X322|2015.1.00607.S 102 | uid://A002/Xad5f93/Xd05|uid://A001/X2d6/X322|2015.1.00607.S 103 | uid://A002/Xad565b/X76a|uid://A001/X2d6/X322|2015.1.00607.S 104 | uid://A002/Xad6824/X3541|uid://A001/X2d6/X32c|2015.1.00823.S 105 | uid://A002/Xac5575/X2938|uid://A001/X2d6/X342|2015.1.01602.S 106 | uid://A002/Xac5575/X340e|uid://A001/X2d6/X343|2015.1.01602.S 107 | uid://A002/Xac9e3c/Xb37|uid://A001/X2d8/X1|2015.1.00030.S 108 | uid://A002/Xac97ce/Xbae|uid://A001/X2d8/X1|2015.1.00030.S 109 | uid://A002/Xac77dc/X302|uid://A001/X2d8/X1|2015.1.00030.S 110 | uid://A002/Xac77dc/X682|uid://A001/X2d8/X1|2015.1.00030.S 111 | uid://A002/Xac5575/X4f49|uid://A001/X2d8/X381|2015.1.01290.S 112 | uid://A002/Xacc268/X6f3|uid://A001/X2d8/X382|2015.1.01290.S 113 | uid://A002/Xac5575/X2c41|uid://A001/X2d8/X3b|2015.1.00078.S 114 | uid://A002/Xac3425/X29e2|uid://A001/X2d8/X3b|2015.1.00078.S 115 | uid://A002/Xad2075/Xeb1|uid://A001/X2d8/X411|2015.1.01302.S 116 | uid://A002/Xad2439/Xee6|uid://A001/X2d8/X412|2015.1.01302.S 117 | uid://A002/Xad2439/X2bc7|uid://A001/X2d8/X413|2015.1.01302.S 118 | uid://A002/Xad2439/X2db5|uid://A001/X2d8/X413|2015.1.01302.S 119 | uid://A002/Xacca73/X678c|uid://A001/X2d8/X419|2015.1.00695.S 120 | uid://A002/Xacdf75/X186e|uid://A001/X2d8/X41a|2015.1.00695.S 121 | uid://A002/Xacca73/X3308|uid://A001/X2d8/X41b|2015.1.00695.S 122 | uid://A002/Xac97ce/X8e4|uid://A001/X2d8/X41c|2015.1.00695.S 123 | uid://A002/Xad565b/X1ae7|uid://A001/X2d8/X432|2015.1.00480.S 124 | uid://A002/Xad565b/X1da0|uid://A001/X2d8/X432|2015.1.00480.S 125 | uid://A002/Xad565b/X20c4|uid://A001/X2d8/X432|2015.1.00480.S 126 | uid://A002/Xacdf75/X2815|uid://A001/X2d8/X66|2015.1.01302.S 127 | uid://A002/Xacaec2/X119|uid://A001/X2de/X108|2015.1.00889.S 128 | uid://A002/Xac8406/X5ed|uid://A001/X2de/X108|2015.1.00889.S 129 | uid://A002/Xacdf75/X2601|uid://A001/X2de/X10f|2015.1.00466.S 130 | uid://A002/Xad2439/X2ff7|uid://A001/X2de/X10f|2015.1.00466.S 131 | uid://A002/Xacf797/X14d9|uid://A001/X2de/X10f|2015.1.00466.S 132 | uid://A002/Xacaa68/X217|uid://A001/X2de/X110|2015.1.00466.S 133 | uid://A002/Xad2439/Xfd2|uid://A001/X2de/X111|2015.1.00466.S 134 | uid://A002/Xacf797/X13d3|uid://A001/X2de/X111|2015.1.00466.S 135 | uid://A002/Xac97ce/X118f|uid://A001/X2de/X112|2015.1.00466.S 136 | uid://A002/Xad5116/Xa70|uid://A001/X2de/X150|2015.1.00806.S 137 | uid://A002/Xad7dc8/X16c|uid://A001/X2de/X151|2015.1.00806.S 138 | uid://A002/Xac5575/X7878|uid://A001/X2de/X24|2015.1.00415.S 139 | uid://A002/Xaca510/X997|uid://A001/X2de/X24|2015.1.00415.S 140 | uid://A002/Xad565b/X3af|uid://A001/X2de/X3d|2015.1.00024.S 141 | uid://A002/Xac8406/X877|uid://A001/X2de/X3d|2015.1.00024.S 142 | uid://A002/Xad74b0/X14ef|uid://A001/X2de/X3e|2015.1.00024.S 143 | uid://A002/Xad74b0/X1a94|uid://A001/X2de/X3e|2015.1.00024.S 144 | uid://A002/Xad6824/Xd30|uid://A001/X2de/X3e|2015.1.00024.S 145 | uid://A002/Xad5f93/X59c|uid://A001/X2de/X3e|2015.1.00024.S 146 | uid://A002/Xac8406/Xb41|uid://A001/X2de/Xed|2015.1.00141.S 147 | uid://A002/Xac8406/Xe70|uid://A001/X2de/Xed|2015.1.00141.S 148 | uid://A002/Xaca510/X78d|uid://A001/X2de/Xf3|2015.1.00206.S 149 | uid://A002/Xac97ce/X14e|uid://A001/X2de/Xf3|2015.1.00206.S 150 | uid://A002/Xac9223/X58|uid://A001/X2de/Xf3|2015.1.00206.S 151 | uid://A002/Xacca73/X5ddc|uid://A001/X2de/Xf9|2015.1.00986.S 152 | uid://A002/Xad6824/X1211|uid://A001/X2de/Xf9|2015.1.00986.S 153 | uid://A002/Xad6824/X675|uid://A001/X2de/Xf9|2015.1.00986.S 154 | uid://A002/Xad47ae/X659|uid://A001/X2de/Xf9|2015.1.00986.S 155 | uid://A002/Xacca73/X178|uid://A001/X2df/X131|2015.1.01170.S 156 | uid://A002/Xacb746/X8db|uid://A001/X2df/X131|2015.1.01170.S 157 | uid://A002/Xacbbd0/X132c|uid://A001/X2df/X131|2015.1.01170.S 158 | uid://A002/Xacbbd0/X15f2|uid://A001/X2df/X131|2015.1.01170.S 159 | uid://A002/Xad057f/Xd88|uid://A001/X2df/X13a|2015.1.00025.S 160 | uid://A002/Xac9e3c/X14d|uid://A001/X2df/X153|2015.1.01345.S 161 | uid://A002/Xac97ce/X4eb|uid://A001/X2df/X153|2015.1.01345.S 162 | uid://A002/Xac9e3c/X619|uid://A001/X2df/X154|2015.1.01345.S 163 | uid://A002/Xac9223/X483|uid://A001/X2df/X154|2015.1.01345.S 164 | uid://A002/Xaca510/X10a|uid://A001/X2df/X191|2015.1.00810.S 165 | uid://A002/Xacbbd0/X8e8|uid://A001/X2df/X19f|2015.1.00456.S 166 | uid://A002/Xaca510/X454|uid://A001/X2df/X1a0|2015.1.00456.S 167 | uid://A002/Xacc4e4/Xcd3|uid://A001/X2df/X1a1|2015.1.00456.S 168 | uid://A002/Xacc268/Xc3|uid://A001/X2df/X1de|2015.1.00810.S 169 | uid://A002/Xacdf75/Xcbd|uid://A001/X2df/X1df|2015.1.00810.S 170 | uid://A002/Xad5f93/X252|uid://A001/X2df/X1e0|2015.1.00810.S 171 | uid://A002/Xad3217/X4385|uid://A001/X2f7/X15a|2015.1.01446.S 172 | uid://A002/Xad6824/X2bfe|uid://A001/X2f7/X15a|2015.1.01446.S 173 | uid://A002/Xad565b/X1365|uid://A001/X2f7/X15a|2015.1.01446.S 174 | uid://A002/Xad5116/X72d|uid://A002/Xad2d3f/Xf|2015.A.00005.S -------------------------------------------------------------------------------- /sacm211.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sagonzal' 2 | from MetaData import * 3 | import ASDM 4 | from asdmTypes import * 5 | from ASDM import * 6 | from Pointing import * 7 | from ASDMParseOptions import * 8 | from sys import argv 9 | from sys import stdout 10 | import ephem 11 | import sacm.geo_helper as gh 12 | import matplotlib.pyplot as plt 13 | plt.style.use('ggplot') 14 | import dateutil.parser as prs 15 | import pylab as pl 16 | pd.options.display.width = 300 17 | from itertools import combinations 18 | from astropy import units as u 19 | from astropy.coordinates import SkyCoord 20 | from math import atan2 21 | J0 = ephem.julian_date(0) 22 | 23 | def azelToRaDec(az=None, el=None,lat=None,lon=None,alt=None, ut=None): 24 | global J0 25 | observer = ephem.Observer() 26 | observer.lat = str(lat) 27 | observer.lon = str(lon) 28 | observer.elevation = alt 29 | observer.date = ut - J0 30 | return observer.radec_of(az, el) 31 | 32 | def WriteNewField(fielduid = None, dir = None, df = None): 33 | fieldXML = GetXML(fielduid,'Field') 34 | if fieldXML is not False: 35 | f = minidom.parseString(fieldXML) 36 | r = f.getElementsByTagName('row') 37 | for i in r: 38 | fieldId = unicode(i.getElementsByTagName('fieldId')[0].firstChild.data) 39 | try: 40 | ra_new, dec_new = df[fieldId] 41 | text = ' 2 1 2 %s %s '%(ra_new,dec_new) 42 | i.getElementsByTagName('delayDir')[0].firstChild.replaceWholeText(text) 43 | i.getElementsByTagName('phaseDir')[0].firstChild.replaceWholeText(text) 44 | i.getElementsByTagName('referenceDir')[0].firstChild.replaceWholeText(text) 45 | except KeyError as e: 46 | pass 47 | 48 | open(dir+"Field.xml.new","wb").write(f.toxml()) 49 | 50 | def measureDistance(lat1, lon1, lat2, lon2): 51 | R = 6383.137 # Radius of earth at Chajnantor aprox. in KM 52 | dLat = (lat2 - lat1) * np.pi / 180. 53 | dLon = (lon2 - lon1) * np.pi / 180. 54 | a = pl.sin(dLat/2.) * pl.sin(dLat/2.) + pl.cos(lat1 * np.pi / 180.) * pl.cos(lat2 * np.pi / 180.) * pl.sin(dLon/2.) * pl.sin(dLon/2.) 55 | c = 2. * atan2(pl.sqrt(a), pl.sqrt(1-a)) 56 | d = R * c 57 | return d * 1000. # meters 58 | 59 | def rot(Pl, rLong, rLat): 60 | return [pl.cos(rLong)*pl.cos(rLat) * Pl[0] - pl.sin(rLong) * Pl[1] - pl.cos(rLong)*pl.sin(rLat) * Pl[2], 61 | pl.sin(rLong)*pl.cos(rLat) * Pl[0] + pl.cos(rLong) * Pl[1] - pl.sin(rLong)*pl.sin(rLat) * Pl[2], 62 | pl.sin(rLat) * Pl[0] + pl.cos(rLat) * Pl[2]] 63 | 64 | parser = ASDMParseOptions() 65 | parser.asALMA() 66 | parser.loadTablesOnDemand(True) 67 | # Read ASDM 68 | asdmtable = ASDM() 69 | if len(argv) == 1: 70 | asdmdir = 'uid___A002_X7c8369_X64e' 71 | else: 72 | asdmdir = argv[1] 73 | 74 | asdmtable.setFromFile(asdmdir, parser) 75 | uid = asdmtable.entity().toString().split('"')[1] 76 | asdm = AsdmCheck() 77 | asdm.setUID(uid) 78 | sb = getSBSummary(asdm.asdmDict['SBSummary']) 79 | 80 | #Get all the Parts that we need 81 | scan = getScan(asdm.asdmDict['Scan']) 82 | subscan = getSubScan(asdm.asdmDict['Subscan']) 83 | field = getField(asdm.asdmDict['Field']) 84 | antenna = getAntennas(asdm.asdmDict['Antenna']) 85 | station = getStation(asdm.asdmDict['Station']) 86 | source = getSource(asdm.asdmDict['Source']) 87 | sbUID = sb.values[0][0] 88 | sbfield = getSBFields(sbUID) 89 | main = getMain(asdm.asdmDict['Main']) 90 | 91 | 92 | rows = asdmtable.pointingTable().get() 93 | pointingList = list() 94 | for idx,row in enumerate(rows): 95 | pointingList.append((idx, str(row.antennaId()), float(row.numSample()), float(row.numTerm()),str(row.timeInterval()), str(row.timeOrigin()))) 96 | 97 | pointingAll = pd.DataFrame(pointingList,columns = ['rowNum','antennaId','samples','iter','duration','origin']) 98 | 99 | #TODO: Fix to match any antenna 100 | pointing = pointingAll[pointingAll['antennaId'] == 'Antenna_1'] 101 | 102 | #do some transformations for matching the data 103 | scan['target'] = scan.apply(lambda x: True if str(x['scanIntent']).find('OBSERVE_TARGET') > 0 else False ,axis = 1) 104 | tsysScans = list(set(scan.sourceName[scan['target'] == True].values)) 105 | scan['target'] = scan.apply(lambda x: True if str(x['sourceName']) in tsysScans else x['target'] ,axis = 1) 106 | targets = map(unicode.strip,list(scan[scan['target'] == True].sourceName.values)) 107 | 108 | source['target'] = source.apply(lambda x: True if str(x['sourceName']).strip() in targets else False, axis = 1) 109 | source['ra'], source['dec'] = zip(*source.apply(lambda x: arrayParser(x['direction'],1), axis = 1)) 110 | field['target'] = field.apply(lambda x: True if str(x['fieldName']).strip() in targets else False, axis = 1) 111 | 112 | foo = list(scan.scanNumber[scan['target'] == True]) 113 | bar = list(main.loc[main['scanNumber'].isin(foo) ]['fieldId'].unique()) 114 | 115 | pointing['go'] = False 116 | 117 | #horrible hack to match the pointing table timescale with the subscan table 118 | for i in subscan.loc[subscan['scanNumber'].isin(foo) ][['startTime','endTime']].values: 119 | pointing['go'] = pointing.apply(lambda x: True if prs.parse(sdmTimeString(i[0])) - datetime.timedelta(seconds=1) <= prs.parse(x['origin']) and prs.parse(sdmTimeString(i[1])) >= prs.parse(x['origin']) else x['go'], axis = 1) 120 | 121 | ra = float(source[source['target'] ==True]['ra'].unique()[0]) 122 | if ra < 0: 123 | ra = ra * -1. 124 | dec = float(source[source['target'] ==True]['dec'].unique()[0]) 125 | 126 | geo = pd.merge(antenna,station, left_on='stationId', right_on = 'stationId', how = 'inner') 127 | geo['pos'] = geo.apply(lambda x: arrayParser(x['position'],1) , axis = 1 ) 128 | geo['lat'], geo['lon'], geo['alt'] = zip(*geo.apply(lambda x: gh.turn_xyz_into_llh(float(x.pos[0]),float(x.pos[1]),float(x.pos[2]), 'wgs84'),axis=1)) 129 | field['ra'],field['dec'] = zip(*field.apply(lambda x: arrayParser(x['referenceDir'],2)[0], axis = 1)) 130 | 131 | 132 | correctedList = list() 133 | correctedList.append((ra,dec,0)) 134 | for i in pointing.query('go == True').rowNum.values: 135 | row = rows[i] 136 | dRA,dDec = [[float(str(p[0]).replace('rad','').replace(',','.')),float(str(p[1]).replace('rad','').replace(',','.'))] for p in row.sourceOffset() ][row.numSample()/2] 137 | Pl = [pl.cos(dRA)*pl.cos(dDec), pl.sin(dRA)*pl.cos(dDec), pl.sin(dDec)] 138 | Ps = rot(Pl, ra, dec) 139 | correctedList.append((pl.arctan2(Ps[1], Ps[0]) % (2.*pl.pi), pl.arcsin(Ps[2]), i)) 140 | 141 | correctedAll = pd.DataFrame(correctedList, columns=['ra','dec', 'row']) 142 | corrected = correctedAll[['ra','dec']] 143 | corrected['series'] = 'Corrected (Pointing)' 144 | 145 | observed = field[field['target'] == True][['fieldId','ra','dec']] 146 | observed.ra = observed.ra.astype(float) 147 | observed.dec = observed.dec.astype(float) 148 | observed = observed.loc[observed['fieldId'].isin(bar) ] 149 | observed = observed.reset_index(drop=True) 150 | corrected = corrected.drop_duplicates() 151 | corrected = corrected.reset_index(drop=True) 152 | cat = SkyCoord(observed.ra.values * u.rad, observed.dec.values * u.rad, frame='icrs') 153 | cat2 = SkyCoord(corrected.ra.values * u.rad, corrected.dec.values *u.rad, frame='icrs') 154 | match, separ, dist = cat2.match_to_catalog_sky(cat) 155 | 156 | observed['fieldId'] 157 | 158 | observed['series'] = 'Field.xml' 159 | observed['ra'] = observed.apply(lambda x: -1*float(x['ra']) if float(x['ra']) < 0 else float(x['ra']), axis = 1) 160 | 161 | 162 | #SB Queries and data manipulation 163 | sboffset = getSBOffsets(sbUID) 164 | sb = getSBSummary(asdm.asdmDict['SBSummary']) 165 | sbUID = sb.values[0][0] 166 | target = getSBTargets(sbUID) 167 | science = getSBScience(sbUID) 168 | partId = target[target['ObsParameter'] == science.entityPartId.values[0]].FieldSource.values[0] 169 | predicted = sboffset[sboffset['partId'] == partId][['latitude','longitude']] 170 | longitude, lat = sbfield[sbfield['entityPartId'] == partId][['longitude','latitude']].values[0] 171 | predicted[['raoff','decoff']] = predicted[['longitude','latitude']].astype(float) 172 | RA0 = float(longitude)*pl.pi/180. 173 | Dec0 = float(lat)*pl.pi/180. 174 | predicted['dRA'] = predicted.apply(lambda x: pl.radians(x['raoff']/3600.), axis = 1) 175 | predicted['dDec'] = predicted.apply(lambda x: pl.radians(x['decoff']/3600.), axis = 1) 176 | predicted['Pl'] = predicted.apply(lambda x: list((pl.cos(x['dRA'])*pl.cos(x['dDec']), pl.sin(x['dRA'])*pl.cos(x['dDec']), pl.sin(x['dDec']))) , axis = 1) 177 | predicted['Ps'] = predicted.apply(lambda x: rot(x['Pl'],RA0,Dec0), axis = 1) 178 | predicted['otcoor'] = predicted.apply(lambda x: list((pl.arctan2(x['Ps'][1], x['Ps'][0]) % (2.*pl.pi), pl.arcsin(x['Ps'][2]))), axis =1) 179 | predicted['otcoor_ra'] = predicted.apply(lambda x: x['otcoor'][0], axis =1 ) 180 | predicted['otcoor_dec'] = predicted.apply(lambda x: x['otcoor'][1], axis =1 ) 181 | predictedList = list() 182 | #predictedList.append((RA0,Dec0)) 183 | pred = pd.DataFrame(predictedList, columns = ['ra','dec']) 184 | ot = predicted[['otcoor_ra','otcoor_dec']] 185 | ot.columns= ['ra','dec'] 186 | pred = pd.concat([pred,ot]) 187 | pred['series'] = 'SchedBlock' 188 | 189 | comb = combinations(geo[['lat','lon']].values, 2) 190 | combList = list() 191 | for i in comb: 192 | combList.append((i[0][0],i[0][1],i[1][0],i[1][1])) 193 | baseLines = pd.DataFrame(combList) 194 | baseLines['dist'] = baseLines.apply(lambda x: measureDistance(x[0],x[1],x[2],x[3]) , axis = 1) 195 | 196 | blMax = baseLines.dist.describe().values[7] 197 | sbfreq = np.float(sb.frequency.values[0])*1e9 198 | c = 299792458 199 | l = c / sbfreq 200 | beam = l / blMax 201 | sbeam = beam * 206264.80624709636 202 | 203 | 204 | diff = pd.concat([observed.ix[match].reset_index(drop=True),corrected] , axis = 1) 205 | diff.columns = ['fieldId','ra_field','dec_field','field','ra_pointing','dec_pointing','pointing'] 206 | diff['ra_diff'] = diff.apply(lambda x: pl.absolute(x['ra_field'] - x['ra_pointing']), axis = 1) 207 | diff['dec_diff'] = diff.apply(lambda x: pl.absolute(x['dec_field'] - x['dec_pointing']), axis = 1) 208 | 209 | diff['total'] = diff.apply(lambda x: ((x['ra_diff']**2 + x['dec_diff']**2)**(0.5))*206264.80624709636, axis = 1) 210 | 211 | if diff.total.describe().values[7] >= sbeam /5.: 212 | print 'Needs New Field Table' 213 | print 'Mean Offset (arcsec) :',str(diff.total.describe().values[1]) 214 | print 'Max Offset (arcsec) :',str(diff.total.describe().values[7]) 215 | print 'Sbeam / 5:', str(sbeam /5.) 216 | else: 217 | print 'Does not need fix' 218 | print 'Mean Offset (arcsec) :',str(diff.total.describe().values[1]) 219 | print 'Max Offset (arcsec) :',str(diff.total.describe().values[7]) 220 | print 'Sbeam / 5:', str(sbeam /5.) 221 | 222 | #Plotting 223 | 224 | new = diff[['fieldId','ra_pointing','dec_pointing']] 225 | newDict = new.set_index('fieldId').T.to_dict('list') 226 | 227 | WriteNewField(asdm.asdmDict['Field'] ,asdmdir ,newDict) 228 | 229 | final = pd.concat([corrected,observed,pred]) 230 | final[['ra','dec']] = final[['ra','dec']].astype(float) 231 | groups = final.groupby('series') 232 | 233 | fig, ax = plt.subplots() 234 | ax.margins(0.05) 235 | marks = ['.','+','x'] 236 | colors = ['b','r','k'] 237 | 238 | for idx, x in enumerate(groups): 239 | ax.plot(x[1].ra, x[1].dec, marker=marks[idx], color=colors[idx],linestyle='', ms=12, label=x[0], alpha=0.6) 240 | 241 | ax.legend() 242 | plt.show() 243 | 244 | -------------------------------------------------------------------------------- /Sensitivity/sensitivity.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | 4 | """ 5 | Pseudo OT sensitivity calculator 6 | 2014/11/08 ahirota 7 | """ 8 | 9 | dtor = np.pi / 180. 10 | const_c = 2.99792458e8 11 | const_k = 1.38e-23 12 | const_h = 6.626e-34 13 | bandDefinitions = { 14 | 1: [31.3e9, 45e9], 15 | 2: [67.0e9, 84e9], 16 | 3: [84.0e9, 116e9], 17 | 4: [125.e9, 163e9], 18 | 5: [163.e9, 211e9], 19 | 6: [211.e9, 275e9], 20 | 7: [275.e9, 373e9], 21 | 8: [385.e9, 500e9], 22 | 9: [602.e9, 720e9], 23 | 10: [787e9, 950e9], 24 | } 25 | 26 | path = os.environ['HOME'] 27 | 28 | 29 | def planck(freq, T): 30 | """ 31 | Parameters 32 | ---------- 33 | freq : [Hz] 34 | T : [K] 35 | """ 36 | tmp = const_h * freq / const_k 37 | return tmp / (np.exp(tmp / T) - 1.0) 38 | 39 | 40 | def getReceiverBand(frequency): 41 | """ 42 | Parameters 43 | ---------- 44 | frequency : [Hz] 45 | """ 46 | for band in range(1, 11): 47 | freq0, freq1 = bandDefinitions[band] 48 | if frequency >= freq0 and frequency <= freq1: 49 | return band 50 | raise Exception("Invalid frequency [%s]" % (frequency)) 51 | 52 | 53 | def getReceiverTemperature(band): 54 | """ 55 | Based on ReceiverTemperature.java 56 | 57 | Parameters 58 | ---------- 59 | band : Integer 60 | 61 | """ 62 | d = dict() 63 | d[1] = 17. 64 | d[2] = 30. 65 | d[3] = 45. 66 | d[4] = 51. 67 | d[5] = 65. 68 | d[6] = 55. 69 | d[7] = 75. 70 | d[8] = 150. 71 | d[9] = 110. 72 | d[10] = 230. 73 | return d[band] 74 | 75 | 76 | def readATMFile(infile): 77 | infile_ = os.path.expanduser(infile) 78 | with file(infile_) as f: 79 | lines = [l.strip() for l in f.readlines()] 80 | values = [] 81 | for iLine, line in enumerate(lines): 82 | if line.startswith('#'): 83 | continue 84 | tokens = line.split(' ') 85 | if len(tokens) != 3: 86 | raise Exception('Unsupported file format [%s] [L%d:"%s"]' % (infile_, iLine, line)) 87 | values.append(tokens) 88 | values = np.array(values, dtype=float) 89 | return values 90 | 91 | 92 | class SensitivityCalculator: 93 | """ 94 | Simple copy of OT sensitivity calculator. 95 | 96 | Example 97 | ------- 98 | from os import path 99 | from AcsutilPy.FindFile import findFile 100 | configDir = path.split(findFile('config/SKY.SPE0001.trim')[0])[0] 101 | s = SensitivityCalculator(configDir=configDir) 102 | latitude = -23.029 103 | dec = -22. 104 | v = s.calcSensitivity(5.186, 115e9, dec=dec, latitude=latitude, N_pol=2, 105 | N=32., BW=1.e9, D=12.) 106 | print v * 1.0e3 107 | 0.50284 108 | """ 109 | 110 | def __init__(self, config_dir=".", load_all=False): 111 | """ 112 | Parameters 113 | ---------- 114 | config_dir : 115 | Directory which stores ATM parameter files. 116 | 117 | load_all : 118 | If specified (bool), load all the ATM parameter files 119 | while initializing the instance. Otherwise, loading 120 | will be delayed. 121 | 122 | """ 123 | self._configDir = config_dir 124 | self._atmospheresFileName = [ 125 | "SKY.SPE0001.trim", 126 | "SKY.SPE0002.trim", 127 | "SKY.SPE0003.trim", 128 | "SKY.SPE0004.trim", 129 | "SKY.SPE0005.trim", 130 | "SKY.SPE0006.trim", 131 | "SKY.SPE0007.trim", 132 | ] 133 | self.WV_MAP_VALUES = [ 134 | 0.472, 135 | 0.658, 136 | 0.913, 137 | 1.262, 138 | 1.796, 139 | 2.748, 140 | 5.186, 141 | ] 142 | self._atmdata = dict() 143 | if load_all: 144 | for pwv in self.WV_MAP_VALUES: 145 | self._load(pwv) 146 | 147 | def get_available_pwvs(self): 148 | return self.WV_MAP_VALUES 149 | 150 | def lookup(self, pwv, freq): 151 | if pwv not in self._atmdata: 152 | self._load(pwv) 153 | d = self._atmdata[pwv] 154 | inp_freq = np.argmin(np.abs(d['freq'] - freq / 1.0e9)) 155 | return d['tau0'][inp_freq], d['Tsky'][inp_freq] 156 | 157 | def _load(self, pwv): 158 | """ 159 | Load total zenith opacity and atmospheric temperature 160 | from ATM model file 161 | """ 162 | # select octile 163 | iPWV = self.WV_MAP_VALUES.index(pwv) 164 | fileName = self._atmospheresFileName[iPWV] 165 | fileName = os.path.join(self._configDir, fileName) 166 | d = readATMFile(fileName) 167 | freq, tau0, Tsky = d.transpose() 168 | self._atmdata[pwv] = dict(freq=freq, tau0=tau0, Tsky=Tsky) 169 | 170 | def get_tsys(self, pwv, freq, el=None, dec=None, latitude=None, 171 | verbose=False, return_full=False): 172 | """ 173 | dec [deg] 174 | latitude [deg] 175 | pwv [mm] 176 | freq [Hz] 177 | :param return_full: 178 | :param verbose: 179 | :param latitude: 180 | :param dec: 181 | :param el: 182 | :param freq: 183 | :param pwv: 184 | """ 185 | Tamb = 270 # Ambient temperature (260 - 280 K) 186 | eta_feed = 0.95 # Forward efficiency 187 | tau0, Tsky = self.lookup(pwv, freq) 188 | 189 | # airmass 190 | if el: 191 | airmass = 1. / np.sin(el * dtor) 192 | else: 193 | sinDec = np.sin(dec * dtor) 194 | cosDec = np.cos(dec * dtor) 195 | sinLat = np.sin(latitude * dtor) 196 | cosLat = np.cos(latitude * dtor) 197 | # HA=0. 198 | sinAltitude = sinDec * sinLat + cosDec * cosLat 199 | airmass = 1.0 / sinAltitude 200 | band = getReceiverBand(freq) 201 | # Receiver temperature 202 | Trx = getReceiverTemperature(band) 203 | # sideband gain ratio 204 | gr = 0. if freq < 600.e9 else 1. 205 | 206 | Tsky_atmp = planck(freq, Tsky) 207 | Tamb = planck(freq, Tamb) 208 | 209 | # Calculate Tsky along line of sight 210 | Tsky_z = Tsky_atmp * (1. - np.exp(-tau0 * airmass)) / \ 211 | (1. - np.exp(-tau0)) 212 | if verbose: 213 | for key in ['tau0', 'Tsky', 'Tamb', 'Trx']: 214 | v = eval(key) 215 | print "%-8s = %f" % (key, v) 216 | 217 | Tsys = Trx + Tsky_z * eta_feed + Tamb * (1. - eta_feed) 218 | Tsys = Tsys * np.exp(tau0 * airmass) / eta_feed * (1. + gr) 219 | 220 | if return_full: 221 | return dict(Tsys=Tsys, 222 | Trx=Trx, 223 | Tsky=Tsky, 224 | Tsky_z=Tsky_z, 225 | tau0=tau0, 226 | airmass=airmass, 227 | freq=freq, 228 | pwv=pwv, 229 | band=band, 230 | el=el, 231 | ) 232 | return Tsys 233 | 234 | def calcSensitivity(self, pwv, freq, 235 | el=None, 236 | dec=None, latitude=None, 237 | tint=60., BW=1.0e9, N=32., 238 | N_pol=1., D=12., 239 | mode="image", 240 | returnFull=False, 241 | isSingleDish=False): 242 | """ 243 | Calculate sensitivity 244 | 245 | Parameters 246 | ---------- 247 | el : Elevation of the target source in [deg]. 248 | dec : Dec of the target source in [deg]. 249 | Required, if 'el' is not specified. 250 | latitude : Latitude of the observatory in [deg]. 251 | Required, if 'el' is not specified. 252 | t_int : Integration time [sec] 253 | BW : Bandwidth [Hz] 254 | N : Number of antennas 255 | N_pol : Number of polarizations 256 | D : Antenna diameter [m] 257 | mode : image, baseline, or antenna 258 | 'image' - Imaging sensitivity 259 | 'baseline' - Fringe sensitivity 260 | 'antena' -> Antenna based gain solution sensitivity 261 | 262 | returnFull : If specified, returns a dictionary 263 | which stores all the relevant information. 264 | Otherwise, just returns sensitivity in [Jy]. 265 | """ 266 | 267 | if isSingleDish: 268 | print '[WARNING] [IsSingleDish] switch is deprecated' 269 | 270 | correlatorEfficiency = 0.88 * 0.96 271 | # atmosphericDecorrelationCoeff 272 | atmDecorrCoeff = 1. 273 | # instrumentalDecorrelationCoeff 274 | instDecorrCoeff = 1. 275 | 276 | # Tsys 277 | d = self.get_tsys(pwv, freq, el=el, dec=dec, latitude=latitude, 278 | return_full=True) 279 | Tsys = d['Tsys'] 280 | 281 | # Area 282 | A = np.pi * (D / 2.) ** 2 283 | # Surface accuracy (spec values) 284 | sigma = 25e-6 if D > 10 else 20e-6 285 | eta_A = 0.72 * np.exp(-16. * (np.pi * sigma / (const_c / freq)) ** 2) 286 | # Antenna efficiency 287 | rho_e = 2. * const_k / (eta_A * A) 288 | # [Jy] 289 | SEFD = Tsys * rho_e * 1.0e26 290 | 291 | efficiencies = atmDecorrCoeff * correlatorEfficiency * instDecorrCoeff 292 | 293 | # Single antenna sensitivity [Jy] 294 | s0 = SEFD / efficiencies / np.sqrt(N_pol * tint * BW) 295 | 296 | # Calculate sensitivity in Jy 297 | if mode == "sd_image": 298 | s = s0 / np.sqrt(N) 299 | elif mode == "image": 300 | if N >= 2: 301 | N_bl = N * (N - 1.) / 2. 302 | s = s0 / np.sqrt(2. * N_bl) 303 | else: 304 | s = s0 305 | elif mode == "baseline": 306 | # Sensitivity per a baseline 307 | s = s0 / np.sqrt(2.) 308 | elif mode == "antenna": 309 | # Antenna-based gain solution sensitivity 310 | # (Sensitivity which gives gain error of unity) 311 | if N >= 4: 312 | s = s0 / np.sqrt(2. * (N - 3.)) 313 | else: 314 | s = s0 # For 2-antenna pointing observation, we can divide it by sqrt(2). 315 | else: 316 | raise Exception('Invalid mode [%s] specified' % mode) 317 | 318 | if returnFull: 319 | d['eta_A'] = eta_A 320 | d['SEFD'] = SEFD 321 | d['sensitivity'] = s 322 | d['BW'] = BW 323 | d['tint'] = tint 324 | d['N_pol'] = N_pol 325 | d['N'] = N 326 | d['D'] = D 327 | d['mode'] = mode 328 | return d 329 | else: 330 | return s 331 | 332 | def selecticAutomaticPWV(self, freq, el, timeIncrease=1.5): 333 | tList = [] 334 | for iPWV, pwv in enumerate(self.WV_MAP_VALUES): 335 | s = self.calcSensitivity(pwv, freq, el=el, 336 | tint=1., BW=15.e9, 337 | N=50, D=12.) 338 | # Time required to achieve 1Jy sensitivity 339 | t = (s / 1.) ** 2 340 | tList.append(t) 341 | 342 | # Get the highest PWV which is within facor of 343 | # 'timeIncrease' in integration time 344 | w = np.where(tList < tList[0] * timeIncrease) 345 | iSelected = w[0][-1] 346 | pwv = self.WV_MAP_VALUES[iSelected] 347 | return pwv 348 | 349 | 350 | def _test(opts): 351 | # /ALMA-10_6_0-B/OBSPREP/ObservingTool/src/alma/observatorycharacteristics/site/ 352 | latitude = -23.029 353 | longitude = -67.754 354 | 355 | print "performing simple tests..." 356 | 357 | configDir = "." if opts.dir is None else opts.dir 358 | tsc = SensitivityCalculator(config_dir=configDir) 359 | 360 | dec = -22. 361 | print "[1] automatic PWV selection tests..." 362 | paramList = [ 363 | [115.e9, 90, 5.186], 364 | [115.e9, 30, 2.748], 365 | ] 366 | for freq, el, pwv_expected in paramList: 367 | pwv = tsc.selecticAutomaticPWV(freq, el=el) 368 | print "Freq=%7.2f GHz El=%5.1f [deg] -> PWV=%f [expected=%f]" % \ 369 | (freq / 1.0e9, el, pwv, pwv_expected) 370 | assert(pwv == pwv_expected) 371 | 372 | paramList = [ 373 | [5.186, 80.e9, 0.23393], 374 | [5.186, 115e9, 0.50284], 375 | [1.262, 230e9, 0.36854], 376 | [1.262, 251e9, 0.38670], 377 | [0.913, 345e9, 0.63490], 378 | [0.472, 492e9, 2.45195], 379 | [0.472, 720e9, 15.9761738724], 380 | ] 381 | print "[2] Sensitivity calculation tests.... [interferometer]" 382 | for pwv, freq, v_expected in paramList: 383 | v = tsc.calcSensitivity(pwv, freq, dec=dec, latitude=latitude, 384 | N_pol=2, N=32., BW=1.e9, D=12.) * 1.0e3 385 | print "v=%10.5f v_expected=%10.5f" % (v, v_expected) 386 | assert(np.abs(v / v_expected - 1.) < 1.0e-4) 387 | 388 | print "[3] Sensitivity calculation test... [single dish]" 389 | paramList = [ 390 | [5.186, 80.e9, 5.20998], 391 | [5.186, 115e9, 11.19874], 392 | [0.913, 345e9, 14.13992], 393 | [0.472, 720e9, 355.80512], 394 | ] 395 | for pwv, freq, v_expected in paramList: 396 | v = tsc.calcSensitivity(pwv, freq, dec=dec, latitude=latitude, 397 | N_pol=2, N=2., BW=1.e9, D=12.) * 1.0e3 398 | print "v=%10.5f v_expected=%10.5f" % (v, v_expected) 399 | assert(np.abs(v / v_expected - 1.) < 1.0e-4) 400 | 401 | if __name__ == '__main__': 402 | from optparse import OptionParser 403 | parser = OptionParser() 404 | parser.add_option('-d', '--dir', 405 | help="config directory which stores ATM data") 406 | parser.add_option('-t', '--test', action="store_true") 407 | opts, args = parser.parse_args() 408 | 409 | if opts.test: # For test 410 | _test(opts) 411 | -------------------------------------------------------------------------------- /createReadme.py: -------------------------------------------------------------------------------- 1 | #!/bin/env python 2 | import cx_Oracle 3 | import sys 4 | from MetaData import * 5 | 6 | mous = sys.argv[1] 7 | if '://' not in mous: 8 | mous = mous.replace('___','://').replace('_','/') 9 | print "MOUS: ", mous 10 | 11 | 12 | sql = '''select 13 | al1.DOMAIN_ENTITY_ID as sb_uid, 14 | al1.PARENT_OBS_UNIT_SET_STATUS_ID as mous_uid, 15 | al2.PROJECTUID, 16 | al2.PI_FULLNAME, 17 | al2.PI_USERID, 18 | al2.ASSOCIATEDEXEC, 19 | al4.title, 20 | al3.sb_name, 21 | al3.REQUESTEDARRAY, 22 | al3.MINACCEPTABLEANGRESOLUTION, 23 | al3.MAXACCEPTABLEANGRESOLUTION, 24 | al4.code 25 | from 26 | ALMA.SCHED_BLOCK_STATUS al1, 27 | ALMA.BMMV_OBSPROPOSAL al2, 28 | ALMA.BMMV_SCHEDBLOCK al3, 29 | ALMA.BMMV_OBSPROJECT al4 30 | where 31 | al1.OBS_PROJECT_ID = al2.PROJECTUID 32 | and al1.DOMAIN_ENTITY_ID = al3.ARCHIVE_UID 33 | and al2.PROJECTUID = al4.PRJ_ARCHIVE_UID 34 | and al1.PARENT_OBS_UNIT_SET_STATUS_ID = 'XXXXYYYY' 35 | ''' 36 | 37 | sql = sql.replace('XXXXYYYY',mous) 38 | 39 | try: 40 | conn = cx_Oracle.connect(databaseSCO) 41 | cursor = conn.cursor() 42 | cursor.execute(sql) 43 | row = cursor.fetchall() 44 | except Exception as e: 45 | print e 46 | sys.exit(1) 47 | 48 | sb_uid ,mous_uid ,prj_uid ,pi_fullname, pi_user, arc, title, sb_name, array, minAngRes, maxAngRes ,code = row[0] 49 | 50 | sql2 = '''select al1.archive_uid Project_UID, x.* 51 | from 52 | ALMA.XML_OBSPROJECT_ENTITIES al1 , 53 | XMLTable('/*:ObsProject/*:ObsProgram/*:ObsPlan/*:ObsUnitSet/*:ObsUnitSet/*:ObsUnitSet' 54 | PASSING al1 .XML COLUMNS 55 | angular varchar2(672) PATH '*:DataProcessingParameters/*:angularResolution', 56 | SB_UID VARCHAR2(50) PATH '*:SchedBlockRef/@entityId' 57 | ) x 58 | where al1.archive_uid = 'XXXXYYYY' 59 | and x.SB_UID = 'ZZZZVVVV' ''' 60 | 61 | sql2 = sql2.replace('XXXXYYYY',prj_uid).replace('ZZZZVVVV',sb_uid) 62 | 63 | try: 64 | cursor.execute(sql2) 65 | row = cursor.fetchall() 66 | except Exception as e: 67 | print e 68 | sys.exit(1) 69 | 70 | angular = row[0][1] 71 | 72 | sql3 = '''select 73 | extractvalue(al1.xml, '/*:SchedBlock/*:ScienceParameters/*:sensitivityGoal' ), 74 | extractvalue(al1.xml, '/*:SchedBlock/*:ScienceParameters/*:sensitivityGoal/@unit' ) 75 | from ALMA.XML_SCHEDBLOCK_ENTITIES al1 76 | where archive_uid = 'XXXXYYYY' ''' 77 | 78 | sql3 = sql3.replace('XXXXYYYY',sb_uid) 79 | 80 | try: 81 | cursor.execute(sql3) 82 | row = cursor.fetchall() 83 | except Exception as e: 84 | print e 85 | sys.exit(1) 86 | 87 | sens,unit = row[0] 88 | 89 | sql4 = '''with t1 as (select al1.PARENT_OBS_UNIT_SET_STATUS_ID as sg_uid 90 | from ALMA.OBS_UNIT_SET_STATUS al1 91 | where al1.STATUS_ENTITY_ID = 'ZZZZXXXXX') 92 | select count(sg_uid) 93 | from t1, ALMA.OBS_UNIT_SET_STATUS 94 | where PARENT_OBS_UNIT_SET_STATUS_ID = t1.sg_uid ''' 95 | 96 | sql4 = sql4.replace('ZZZZXXXXX',mous_uid) 97 | 98 | try: 99 | cursor.execute(sql4) 100 | row = cursor.fetchall() 101 | except Exception as e: 102 | print e 103 | sys.exit(1) 104 | 105 | count = row[0][0] 106 | text = '''Atacama Large Millimeter/submillimeter Array (ALMA) 107 | 108 | ##### 109 | 110 | Cycle: 3 111 | Project code: %s 112 | SB name: %s 113 | PI name: %s (Username: %s) 114 | Project title: %s 115 | Configuration: %s 116 | Proposed rms: %s %s 117 | Proposed beam size: %s arcsec 118 | CASA version used for reduction: 4.x 119 | QA2 Result: PASS/SEMIPASS 120 | Total Number of Member SBs in this OUS Group: %s 121 | Comments from Reducer: 122 | '''%(code,sb_name,pi_fullname,pi_user,title,array,sens,unit ,angular,count) 123 | 124 | text += '''##### 125 | 126 | This file describes the content of the tar file you have received. The 127 | full data structure is inserted below. 128 | 129 | At this stage, we are releasing data after completion of one SB (excuted 130 | multiple times if required), so you will find only one member_ouss_id 131 | directory. This directory contains this README file and the following 132 | directories:calibration, script, qa2, log, product. 133 | 134 | - 'calibration' contains the files needed for calibration starting from 135 | the initial ms to the fully calibrated data. 136 | - 'script' contains the reduction scripts used to process the initial ms 137 | to calibrated data, but also to obtain concatenated data (if more than 138 | one execution) and imaging products. There are usually several scripts 139 | dealing with different parts of the processing. 140 | In case the calibration was done by the automated pipeline, you will 141 | also see the Pipeline Processing Request File (PPR). 142 | The most important script for you is the "scriptForPI.py". See the 143 | section "How to obtain the calibrated MeasurementSet (MS) for your data" 144 | further below. 145 | - 'product' contains the fits files of the selected image products. 146 | These will not include all images of scientific value, but will indicate 147 | the quality of the calibration and images. 148 | - 'qa' contains the qa2 reports that show plots and text information 149 | needed to assess the quality of the processing. The resultant image 150 | rms, compared with that proposed, is given. In case the calibration was 151 | done by the automated pipeline, you will find the pipeline Weblog here. 152 | - 'log' contains the CASA log files. 153 | 154 | For more information see also the "ALMA QA2 Data Products" document 155 | which is available for download from the ALMA Science Portal at 156 | https://almascience.nrao.edu/documents-and-tools 157 | and the ALMA Knowledgebase article on the QA2 pass/fail criteria at 158 | https://help.almascience.org/index.php?/XX/Knowledgebase/Article/View/285 159 | (where the XX is to be replaced by ea, eu, or na depending on your location). 160 | 161 | ##### 162 | 163 | TOTAL POWER DATA 164 | 165 | This section only concerns ALMA total power (TP) data. For ALMA 12m 166 | and 7m interferometric data, please refer to the sections further below. 167 | 168 | The ALMA TP data is reduced manually, using standard scripts, or using 169 | the TP data reduction pipeline which performs calibration and imaging. 170 | You can determine whether your data was reduced by Pipeline by 171 | looking for a file "PPR...xml" in the "script" subdirectory (below 172 | the directory containing this README). If such a file is present, your 173 | data was pipeline-reduced. 174 | 175 | TP datasets will often consist of many execution blocks (EBs). Each of 176 | them was calibrated individually. Imaging is done on all calibrated data 177 | together. 178 | 179 | The images included in this delivery have a native frequency resolution 180 | and a cell size of one ninths of beam size. 181 | If you want to change them to your preferable frequency resolution and 182 | cell size, import the delivered FITS cubes to CASA and regrid it using 183 | the task imregrid. 184 | 185 | One important aspect of the calibration is the conversion of the data 186 | from K to Jy/beam. This is done per EB. For manually reduced data, the 187 | values that were used are available at the end of each calibration script. 188 | In case of pipeline-reduced data, the Jy per K conversion factors can be 189 | found via the pipeline weblog in the "qa" directory or in the file 190 | "jyperk.csv" under the "calibration" directory of this package. 191 | 192 | The conversion factors were derived from the analysis of the associated 193 | AMPCAL data from the same project. This analysis is done using standard 194 | scripts or Pipeline. We are not providing those data by default to the 195 | users, but in case you would be interested to have them, you are welcome 196 | to contact the helpdesk of your region. 197 | 198 | 199 | Pipeline-calibrated TP data 200 | 201 | The following two paragraphs only describe the procedure for pipeline- 202 | calibrated TP data. 203 | 204 | If your TP dataset was pipeline-calibrated and you would like to 205 | re-create the calibrated MeasurementSets, please read on in section 206 | "How to obtain the calibrated MeasurementSet (MS) for your data". 207 | Please note that processing may take a significant amount of time. 208 | To know how long it will take, please see the "Execution Duration" 209 | which is shown on the top-page of the weblog. 210 | 211 | If you want to perform baseline subtraction using your preferable 212 | mask range rather than the pipeline-decided range, we recommend to 213 | do it on the images using the task imcontsub or to do it using 214 | the task sdbaseline during your own manual calibration (see 215 | https://casaguides.nrao.edu/index.php?title=M100_Band3_SingleDish_4.3 ). 216 | 217 | 218 | Manually calibrated TP data 219 | 220 | The following three paragraphs only describe the procedure for manually 221 | calibrated TP data. 222 | 223 | At this stage, the QA report for manually reduced data is rudimentary. 224 | If your TP data was reduced manually, please see the official CASA guide, 225 | available at the following link: 226 | 227 | https://casaguides.nrao.edu/index.php?title=M100_Band3_SingleDish_4.3 228 | 229 | To regenerate the calibrated data, you will need to download the raw 230 | data (ASDMs) from the ALMA archive, remove the '.asdm.sdm' extensions 231 | and then run the scripts ('*.scriptForSDcalibration.py') in the scripts 232 | folder. We recommend that you create an individual folder for each ASDM, 233 | and calibrate each ASDM in its own folder. 234 | 235 | For the imaging, we recommend you use the provided script, as it 236 | contains certain values (e.g. beams) that must be set to a precise 237 | value. You could run the script step-by-step, as for the calibration 238 | script. Before doing that, you may need to update the paths to the 239 | calibrated data, in the msNames variable near the top of the script. 240 | 241 | 242 | ##### 243 | 244 | PRIMARY BEAM CORRECTION 245 | 246 | The images included in this delivery are corrected for the primary beam (PB), 247 | i.e. the dependence of the instruments sensitivity on direction within the FOV. 248 | 249 | For each image, two files are being delivered: 250 | a) the PB-corrected image (file name ending in ".pbcor.fits") 251 | b) the image of the PB which was used in the correction (ending in ".flux.fits") 252 | The image noise was measured in the uncorrected image. 253 | The corrected image (a) was then obtained by dividing the uncorrected image by 254 | the PB image (b). 255 | The uncorrected image can be recovered using the CASA task impbcor in mode "m": 256 | impbcor(imagename='image.pbcor.fits', pbimage='image.flux.fits', mode='m', 257 | outfile='image.recovered') 258 | 259 | 260 | ##### 261 | 262 | HOW TO OBTAIN THE CALIBRATED MEASUREMENTSET (MS) FOR YOUR DATA 263 | 264 | In case you want to re-reduce your data yourself, you will need to 265 | obtain the raw data in ASDM format from the request handler or 266 | other server where it is staged for you (see your notification 267 | email). 268 | 269 | If you downloaded and untarred all available files for this delivery 270 | as described in the notification email, then you will already see 271 | (in addition to the directories shown in the tree listing above) 272 | a directory "raw" containing your raw data in subdirectories 273 | named "uid*.asdm.sdm" and no further action is necessary. 274 | 275 | If you do not have a raw directory, yet, you will need to download 276 | and untar the tar balls of the raw data belonging to this delivery 277 | and make sure they are put into the "raw" directory in your 278 | "member_ouss_..." directory. 279 | 280 | Once the raw data is in place, cd into directory "script", start 281 | 282 | casapy --pipeline 283 | 284 | and type 285 | 286 | execfile('scriptForPI.py') 287 | 288 | (If you have not installed the CASA version with the ALMA pipeline 289 | included, the "--pipeline" switch is not available. 290 | Check in the "script" directory of your delivery package to see 291 | if it contains a file named "PPR*.xml". 292 | If there is no such file, you will be able to run the calibration 293 | without the pipeline. Otherwise, you will have to install 294 | the CASA version with pipeline.) 295 | 296 | For more information on the execution of the pipeline please refer to 297 | the ALMA Pipeline Quickstart Guide available at 298 | https://almascience.nrao.edu/documents-and-tools/alma-science-pipeline-quickstart-guide-for-casa-4.5 299 | 300 | Running the scriptForPI will execute the entire calibration procedure 301 | and result in an MS or a set of MSs ready for imaging. 302 | 303 | In case the data was processed using the automated pipeline, 304 | scriptForPI.py will produce the calibrated MS(s) by running the 305 | "casa_piperestorescript" which applies the packaged calibration 306 | and flagging tables (rather then regenerating them). 307 | 308 | If the casa_piperestorescript is not available (as can be the case 309 | for pipeline-calibrated TP datasets), the scriptForPI will instead 310 | run the entire calibration pipeline using the "casa_pipescript". 311 | 312 | You can force the execution of the casa_pipescript instead of 313 | the casa_piperestorescript by moving the casa_piperestorescript.py 314 | out of the script directory. Rerunning the pipeline can be useful 315 | if you want to tweak its parameters. Otherwise the restore is faster. 316 | 317 | The calibrated MS(s) can then be processed with "scriptForImaging.py". 318 | 319 | The "scriptForImaging.py" may partially be interactive (for masking) 320 | and should be executed by copy and paste. 321 | 322 | For TP MOUSs pipeline-calibrated, the scriptForImaging.py is NOT provided. 323 | You are able to find all used sdimaging task parameters in the CASA logs for stage 13 in the weblog. 324 | 325 | The scriptForPI offers a "SPACESAVING" option to limit the disk space 326 | usage during and after its execution. In order to make use of this, 327 | the Python global variable SPACESAVING needs to be set before starting 328 | the script, e.g. using 329 | 330 | SPACESAVING = N 331 | execfile('scriptForPI.py') 332 | 333 | where N is an integer from 0 to 3 with the following meaning: 334 | SPACESAVING = 0 same as not set (all intermediate MSs are kept) 335 | = 1 do not keep intermediate MSs named *.ms.split 336 | = 2 do not keep intermediate MSs named *.ms and *.ms.split 337 | = 3 do not keep intermediate MSs named *.ms, *.ms.split, 338 | and *.ms.split.cal (if possible) 339 | 340 | With SPACESAVING=0, the required additional diskspace is up to 14 times 341 | as large as the delivered data (products and rawdata) while with 342 | SPACESAVING=3 (maximum savings), it is up to 6 times as large. 343 | The script will estimate the required disk space and will not execute 344 | if there is not sufficient free space available. 345 | 346 | ##### 347 | 348 | ''' 349 | 350 | f = open('README','w') 351 | f.write(text) 352 | f.close() 353 | 354 | -------------------------------------------------------------------------------- /sacm/utils.py: -------------------------------------------------------------------------------- 1 | __author__ = 'sdk' 2 | from time import strftime, gmtime, mktime 3 | import datetime 4 | from xml.dom import minidom 5 | import numpy as np 6 | import cx_Oracle 7 | from password import databaseSCO as database 8 | import pandas as pd 9 | pd.options.mode.chained_assignment = None 10 | 11 | tables = {"ASDM": "XML_ASDM_ENTITIES", "Main": "XML_MAINTABLE_ENTITIES", 12 | "AlmaRadiometer": "XML_ALMARADIOMETERTAB_ENTITIES", "Antenna": "XML_ANTENNATABLE_ENTITIES", 13 | "CalAmpli": "XML_CALAMPLITABLE_ENTITIES", "CalAtmosphere": "XML_CALATMOSPHERETABL_ENTITIES", 14 | "CalCurve": "XML_CALCURVETABLE_ENTITIES", "CalSeeing": "XML_CALSEEINGTABLE_ENTITIES", 15 | "CalWVR": "XML_CALWVRTABLE_ENTITIES", "CalData": "XML_CALDATATABLE_ENTITIES", 16 | "CalDelay": "XML_CALDELAYTABLE_ENTITIES", "CalDevice": "XML_CALDEVICETABLE_ENTITIES", 17 | "CalFlux": "XML_CALFLUXTABLE_ENTITIES", "CalPhase": "XML_CALPHASETABLE_ENTITIES", 18 | "CalReduction": "XML_CALREDUCTIONTABLE_ENTITIES", "ConfigDescription": "XML_CONFIGDESCRIPTION_ENTITIES", 19 | "CorrelatorMode": "XML_CORRELATORMODETAB_ENTITIES", "DataDescription": "XML_DATADESCRIPTIONTA_ENTITIES", 20 | "ExecBlock": "XML_EXECBLOCKTABLE_ENTITIES", "Feed": "XML_FEEDTABLE_ENTITIES", 21 | "Annotation": "XML_ANNOTATIONTABLE_ENTITIES", "Ephemeris": "XML_EPHEMERISTABLE_ENTITIES", 22 | "Anotation": "XML_ANNOTATIONTABLE_ENTITIES", "CalBandpass": "XML_CALBANDPASSTABLE_ENTITIES", 23 | "CalPointing": "XML_CALPOINTINGTABLE_ENTITIES", "Field": "XML_FIELDTABLE_ENTITIES", 24 | "Flag": "XML_FLAGTABLE_ENTITIES", "Focus": "XML_FOCUSTABLE_ENTITIES", 25 | "FocusModel": "XML_FOCUSMODELTABLE_ENTITIES", "Pointing": "XML_POINTINGTABLE_ENTITIES", 26 | "PointingModel": "XML_POINTINGMODELTABL_ENTITIES", "Polarization": "XML_POLARIZATIONTABLE_ENTITIES", 27 | "Processor": "XML_PROCESSORTABLE_ENTITIES", "Receiver": "XML_RECEIVERTABLE_ENTITIES", 28 | "SBSummary": "XML_SBSUMMARYTABLE_ENTITIES", "Scan": "XML_SCANTABLE_ENTITIES", 29 | "Source": "XML_SOURCETABLE_ENTITIES", "SpectralWindow": "XML_SPECTRALWINDOWTAB_ENTITIES", 30 | "State": "XML_STATETABLE_ENTITIES", "Station": "XML_STATIONTABLE_ENTITIES", "Subscan": "XML_SUBSCANTABLE_ENTITIES", 31 | "SquareLawDetector": "XML_SQUARELAWDETECTOR_ENTITIES", "SwitchCycle": "XML_SWITCHCYCLETABLE_ENTITIES", 32 | "SysCal": "XML_SYSCALTABLE_ENTITIES", "Weather": "XML_WEATHERTABLE_ENTITIES", 33 | "SchedBlock":"XML_SCHEDBLOCK_ENTITIES", "ObsProject":"XML_OBSPROJECT_ENTITIES"} 34 | 35 | def sdmTimeString(number=None): 36 | """ 37 | Convert a time value (as used by ASDM, i.e. MJD in nanoseconds) into a FITS type string. 38 | :param number: 39 | """ 40 | st = number/1000000000L 41 | # decimal microseconds ... 42 | number = (number-st*1000000000L)/1000 43 | # number of seconds since 1970-01-01T00:00:00 44 | st = st-3506716800L 45 | return strftime("%Y-%m-%dT%H:%M:%S", gmtime(st))+(".%6.6d" % number) 46 | 47 | 48 | def gtm(t=None): 49 | """ 50 | Convert a time value (as used by ASDM, i.e. MJD in nanoseconds) into a FITS type string. 51 | :param t: 52 | """ 53 | st = t-3506716800000000000L 54 | return st/1000000000L 55 | 56 | 57 | def gtm2(number=None): 58 | """ 59 | Convert a time value (as used by ASDM, i.e. MJD in nanoseconds) into a FITS type string. 60 | :param number: 61 | """ 62 | st = number/1000000000L 63 | # decimal microseconds ... 64 | number = (number-st*1000000000L)/1000 65 | # number of seconds since 1970-01-01T00:00:00 66 | st = st-3506716800L 67 | return datetime.datetime.fromtimestamp(mktime(gmtime(st))).replace(microsecond=(number)) 68 | 69 | def returnMAXPWVC(pwv=None): 70 | if pwv <= 0.472: 71 | return 0.472 72 | elif pwv <= 0.658: 73 | return 0.658 74 | elif pwv <= 0.913: 75 | return 0.913 76 | elif pwv <= 1.262: 77 | return 1.262 78 | elif pwv <= 1.796: 79 | return 1.796 80 | elif pwv <= 2.748: 81 | return 2.748 82 | else: 83 | return 5.186 84 | 85 | 86 | def findChannel(start=None, width=None, repFreq=None, nchan=None): 87 | channel = 0 88 | if width < 0: 89 | for i in xrange(nchan): 90 | if start > repFreq: 91 | start = start + width 92 | else: 93 | channel = -1.*i 94 | break 95 | else: 96 | for i in xrange(nchan): 97 | if start < repFreq: 98 | start = start + width 99 | else: 100 | channel = i 101 | break 102 | 103 | return channel 104 | 105 | 106 | def RadianTo(num=None, unit=None): 107 | """ 108 | 109 | :param num: 110 | :param unit: 111 | :return: 112 | """ 113 | Deg = float(num)*180.0/np.pi 114 | if unit == 'dms': 115 | if Deg < 0: 116 | Deg = -Deg 117 | sign = '-' 118 | else: 119 | sign = '+' 120 | g = int(Deg) 121 | m = int((Deg-g)*60.) 122 | s = (Deg-g-m/60.)*3600. 123 | return sign+str(g)+":"+str(m)+":"+str('%5.5f' % s) 124 | if unit == 'hms': 125 | h = int(Deg/15.) 126 | m = int((Deg/15.-h)*60.) 127 | s = (Deg/15.-h-m/60.)*3600. 128 | return str(h)+":"+str(m)+":"+str('%5.5f' % s) 129 | 130 | 131 | def arrayParser(line=None, dimensions=None): 132 | """ 133 | 134 | :param line: String to be formated 135 | :param dimensions: dimensions of the array 136 | :return: a list, or a list of list 1D o 2D arrays, no support for 3D arrays yet 137 | """ 138 | result = list() 139 | line = line.strip() 140 | if dimensions == 1: 141 | elements = line.split(' ')[1] 142 | splits = line.split(' ')[2:] 143 | for i in splits: 144 | result.append(i) 145 | if int(elements) == len(result): 146 | return result 147 | else: 148 | return False 149 | 150 | if dimensions == 2: 151 | rows = int(line.split(' ')[1]) 152 | columns = int(line.split(' ')[2]) 153 | splits = line.split(' ')[3:] 154 | for j in range(0, rows): 155 | temp = list() 156 | for i in range(0, columns): 157 | temp.append(splits[i+(j*columns)]) 158 | result.append(temp) 159 | return result 160 | 161 | 162 | 163 | 164 | def GetXML(archiveUID=None,table=None): 165 | """ 166 | 167 | :param archiveUID: Archive UID 168 | :param table: Table 169 | :return: XML String 170 | """ 171 | sqlXML = "select XMLType.GetClobVal(xml) from ALMA.XXXXYYY where archive_uid='ZZZZCCCC' " 172 | sqlXML = sqlXML.replace('XXXXYYY',tables[table]).replace('ZZZZCCCC',archiveUID) 173 | try: 174 | orcl = cx_Oracle.connect(database) 175 | cursorXML = orcl.cursor() 176 | cursorXML.execute(sqlXML) 177 | XMLTable = cursorXML.fetchone() 178 | return XMLTable[0].read() 179 | except Exception as e: 180 | print e 181 | return False 182 | return False 183 | 184 | 185 | def getProjectUID(projectCode=None): 186 | """ 187 | 188 | :param projectCode: 189 | :return: 190 | """ 191 | sql = "select prj_archive_uid from ALMA.BMMV_OBSPROJECT where prj_code = 'XXXYYY'" 192 | sql = sql.replace('XXXYYY',projectCode) 193 | try: 194 | orcl = cx_Oracle.connect(database) 195 | cursor = orcl.cursor() 196 | cursor.execute(sql) 197 | data = cursor.fetchall() 198 | orcl.close() 199 | return data[0][0] 200 | 201 | except Exception as e: 202 | print e 203 | 204 | def getSBMOUS(): 205 | sql = "select DOMAIN_ENTITY_ID, PARENT_OBS_UNIT_SET_STATUS_ID from ALMA.SCHED_BLOCK_STATUS" 206 | try: 207 | orcl = cx_Oracle.connect(database) 208 | cursor = orcl.cursor() 209 | cursor.execute(sql) 210 | data = cursor.fetchall() 211 | status = list() 212 | for i in data: 213 | status.append((i[0],i[1])) 214 | orcl.close() 215 | return status 216 | except Exception as e: 217 | print e 218 | 219 | def getSBNames(): 220 | sql = "select archive_uid, sb_name from ALMA.BMMV_SCHEDBLOCK" 221 | try: 222 | orcl = cx_Oracle.connect(database) 223 | cursor = orcl.cursor() 224 | cursor.execute(sql) 225 | data = cursor.fetchall() 226 | sbnames = list() 227 | for i in data: 228 | sbnames.append((i[0],i[1])) 229 | orcl.close() 230 | return sbnames 231 | except Exception as e: 232 | print e 233 | 234 | def getProjectCodes(cycle=2): 235 | cycle_code = dict() 236 | cycle_code[0] = '2011._.%._' 237 | cycle_code[1] = '2012._.%._' 238 | cycle_code[2] = '2013._.%._' 239 | cycle_code[3] = '2015._.%._' 240 | 241 | sql = '''select al2.PRJ_ARCHIVE_UID, al2.code 242 | from ALMA.OBS_PROJECT_STATUS al1, 243 | ALMA.BMMV_OBSPROJECT al2 244 | where al1.obs_project_id in (select prj_archive_uid from ALMA.BMMV_OBSPROJECT where prj_code like 'XXXYYYZZZ') 245 | and al1.domain_entity_state in ('Ready', 'Canceled', 'InProgress', 'Broken','Completed', 'Repaired','Phase2Submitted') 246 | and al1.OBS_PROJECT_ID = al2.PRJ_ARCHIVE_UID ''' 247 | 248 | sql = sql.replace('XXXYYYZZZ',cycle_code[int(cycle)]) 249 | try: 250 | orcl = cx_Oracle.connect(database) 251 | cursor = orcl.cursor() 252 | cursor.execute(sql) 253 | data = cursor.fetchall() 254 | codes = list() 255 | for i in data: 256 | codes.append((i[0],i[1])) 257 | orcl.close() 258 | return codes 259 | except Exception as e: 260 | print e 261 | 262 | 263 | 264 | def getSBs(prj_uid=None): 265 | sql = '''with t1 as ( 266 | select status_entity_id as seid1 from ALMA.OBS_UNIT_SET_STATUS where OBS_PROJECT_ID = 'PPPRRRJJJ' and PARENT_OBS_UNIT_SET_STATUS_ID is null 267 | ), 268 | t2 as ( 269 | select status_entity_id as seid2, PARENT_OBS_UNIT_SET_STATUS_ID as paid2, domain_entity_id from ALMA.OBS_UNIT_SET_STATUS where OBS_PROJECT_ID = 'PPPRRRJJJ' 270 | ), 271 | t3 as ( 272 | select status_entity_id as seid3, PARENT_OBS_UNIT_SET_STATUS_ID as paid3 from ALMA.OBS_UNIT_SET_STATUS where OBS_PROJECT_ID = 'PPPRRRJJJ' 273 | ), 274 | t4 as ( 275 | select status_entity_id as seid4, PARENT_OBS_UNIT_SET_STATUS_ID as paid4 from ALMA.OBS_UNIT_SET_STATUS where OBS_PROJECT_ID = 'PPPRRRJJJ' 276 | ), 277 | t5 as ( 278 | select domain_entity_id as scheckblock_uid, PARENT_OBS_UNIT_SET_STATUS_ID as paid5 from ALMA.SCHED_BLOCK_STATUS where OBS_PROJECT_ID = 'PPPRRRJJJ' 279 | ) 280 | SELECT t2.domain_entity_id, t5.scheckblock_uid 281 | FROM t1, 282 | t2, 283 | t3, 284 | t4, 285 | t5 286 | WHERE t1.seid1 = t2.paid2 287 | AND t2.seid2 = t3.paid3 288 | AND t3.seid3 = t4.paid4 289 | AND t4.seid4 = t5.paid5 290 | ORDER BY 1 ASC''' 291 | sql = sql.replace('PPPRRRJJJ', prj_uid) 292 | try: 293 | orcl = cx_Oracle.connect(database) 294 | cursor = orcl.cursor() 295 | cursor.execute(sql) 296 | sb = cursor.fetchall() 297 | orcl.close() 298 | return sb 299 | except Exception as e: 300 | orcl.close() 301 | print e 302 | 303 | 304 | def spectrals_sb(prj_uid=None, partid=None): 305 | sql = '''with t1 as ( 306 | select status_entity_id as seid1 from ALMA.OBS_UNIT_SET_STATUS where OBS_PROJECT_ID = 'PPPRRRJJJ' and PARENT_OBS_UNIT_SET_STATUS_ID is null 307 | ), 308 | t2 as ( 309 | select status_entity_id as seid2, PARENT_OBS_UNIT_SET_STATUS_ID as paid2, domain_entity_id from ALMA.OBS_UNIT_SET_STATUS where OBS_PROJECT_ID = 'PPPRRRJJJ' 310 | ), 311 | t3 as ( 312 | select status_entity_id as seid3, PARENT_OBS_UNIT_SET_STATUS_ID as paid3 from ALMA.OBS_UNIT_SET_STATUS where OBS_PROJECT_ID = 'PPPRRRJJJ' 313 | ), 314 | t4 as ( 315 | select status_entity_id as seid4, PARENT_OBS_UNIT_SET_STATUS_ID as paid4 from ALMA.OBS_UNIT_SET_STATUS where OBS_PROJECT_ID = 'PPPRRRJJJ' 316 | ), 317 | t5 as ( 318 | select domain_entity_id as scheckblock_uid, PARENT_OBS_UNIT_SET_STATUS_ID as paid5 from ALMA.SCHED_BLOCK_STATUS where OBS_PROJECT_ID = 'PPPRRRJJJ' 319 | ) 320 | SELECT t2.domain_entity_id, t5.scheckblock_uid 321 | FROM t1, 322 | t2, 323 | t3, 324 | t4, 325 | t5 326 | WHERE t1.seid1 = t2.paid2 327 | AND t2.seid2 = t3.paid3 328 | AND t3.seid3 = t4.paid4 329 | AND t4.seid4 = t5.paid5 330 | AND t2.domain_entity_id = 'ZZZXXXYYY' 331 | ORDER BY 1 ASC''' 332 | sql = sql.replace('PPPRRRJJJ', prj_uid).replace('ZZZXXXYYY', partid) 333 | try: 334 | orcl = cx_Oracle.connect(database) 335 | cursor = orcl.cursor() 336 | cursor.execute(sql) 337 | sb = cursor.fetchall() 338 | specscan = list() 339 | for i in sb: 340 | specscan.append((prj_uid,i[1],'SpectralScan')) 341 | return specscan 342 | except Exception as e: 343 | print e 344 | 345 | 346 | def is_spectralscan(prj_uid=None): 347 | sql = '''select al1.archive_uid, x.* 348 | from 349 | ALMA.XML_OBSPROJECT_ENTITIES al1, 350 | XMLTable('for $first in /*:ObsProject/*:ObsProgram/*:ScienceGoal return element i { 351 | element pol { data($first/*:SpectralSetupParameters/@polarisation)}, 352 | element type { data($first/*:SpectralSetupParameters/@type)}, 353 | element partid { data($first/*:ObsUnitSetRef/@partId)} 354 | }' 355 | PASSING al1.XML COLUMNS 356 | pol varchar2(50) PATH 'pol', 357 | type varchar2(32) PATH 'type', 358 | partid varchar2(20) PATH 'partid' 359 | ) x 360 | where al1. archive_uid = 'XXXXYYYY' 361 | order by al1.timestamp desc''' 362 | sql = sql.replace('XXXXYYYY', prj_uid) 363 | try: 364 | orcl = cx_Oracle.connect(database) 365 | cursor = orcl.cursor() 366 | cursor.execute(sql) 367 | science_goals = cursor.fetchall() 368 | cursor.close() 369 | return science_goals 370 | except Exception as e: 371 | print e 372 | 373 | 374 | 375 | def is_band89(prj_uid=None): 376 | sql = '''with t1 as ( 377 | select status_entity_id as seid1 from ALMA.OBS_UNIT_SET_STATUS where OBS_PROJECT_ID = 'XXXXYYYYZZZZ' and PARENT_OBS_UNIT_SET_STATUS_ID is null 378 | ), 379 | t2 as ( 380 | select status_entity_id as seid2, PARENT_OBS_UNIT_SET_STATUS_ID as paid2, domain_entity_id from ALMA.OBS_UNIT_SET_STATUS where OBS_PROJECT_ID = 'XXXXYYYYZZZZ' 381 | ), 382 | t3 as ( 383 | select status_entity_id as seid3, PARENT_OBS_UNIT_SET_STATUS_ID as paid3 from ALMA.OBS_UNIT_SET_STATUS where OBS_PROJECT_ID = 'XXXXYYYYZZZZ' 384 | ), 385 | t4 as ( 386 | select status_entity_id as seid4, PARENT_OBS_UNIT_SET_STATUS_ID as paid4 from ALMA.OBS_UNIT_SET_STATUS where OBS_PROJECT_ID = 'XXXXYYYYZZZZ' 387 | ), 388 | t5 as ( 389 | select domain_entity_id as schedblock_uid, PARENT_OBS_UNIT_SET_STATUS_ID as paid5 from ALMA.SCHED_BLOCK_STATUS where OBS_PROJECT_ID = 'XXXXYYYYZZZZ' 390 | ), 391 | t6 as ( 392 | select archive_uid as sb_uid, receiver_band as band from ALMA.BMMV_SCHEDBLOCK where prj_ref = 'XXXXYYYYZZZZ' 393 | ) 394 | SELECT t2.domain_entity_id, t5.schedblock_uid,t6.band 395 | FROM t1, 396 | t2, 397 | t3, 398 | t4, 399 | t5, 400 | t6 401 | WHERE t1.seid1 = t2.paid2 402 | AND t2.seid2 = t3.paid3 403 | AND t3.seid3 = t4.paid4 404 | AND t4.seid4 = t5.paid5 405 | and t6.sb_uid = t5.schedblock_uid 406 | ORDER BY 1 ASC''' 407 | sql = sql.replace('XXXXYYYYZZZZ',prj_uid) 408 | try: 409 | orcl = cx_Oracle.connect(database) 410 | cursor = orcl.cursor() 411 | cursor.execute(sql) 412 | sb = cursor.fetchall() 413 | cursor.close() 414 | return sb 415 | except Exception as e: 416 | print e 417 | 418 | 419 | 420 | 421 | -------------------------------------------------------------------------------- /fixSACM211.py: -------------------------------------------------------------------------------- 1 | #!/bin/env python 2 | from MetaData import * 3 | import ASDM 4 | from asdmTypes import * 5 | from ASDM import * 6 | from Pointing import * 7 | from ASDMParseOptions import * 8 | import sys 9 | import sacm.geo_helper as gh 10 | import matplotlib.pyplot as plt 11 | plt.style.use('ggplot') 12 | import dateutil.parser as prs 13 | import pylab as pl 14 | pd.options.display.width = 300 15 | from itertools import combinations 16 | from astropy import units as u 17 | from astropy.coordinates import SkyCoord 18 | from math import atan2 19 | 20 | def getNewSource(f=None): 21 | sourceXML = open(f,'r') 22 | if sourceXML is not False: 23 | source = minidom.parseString(sourceXML.read()) 24 | sourceList = list() 25 | rows = source.getElementsByTagName('row') 26 | #there are missing fields in some rows for the Source table. 27 | for i in rows: 28 | sourceList.append((int(i.getElementsByTagName('sourceId')[0].firstChild.data), 29 | i.getElementsByTagName('timeInterval')[0].firstChild.data, 30 | i.getElementsByTagName('direction')[0].firstChild.data, 31 | i.getElementsByTagName('directionCode')[0].firstChild.data, 32 | i.getElementsByTagName('sourceName')[0].firstChild.data, 33 | i.getElementsByTagName('spectralWindowId')[0].firstChild.data)) 34 | sourceXML.close() 35 | return pd.DataFrame(sourceList,columns=['sourceId','timeInterval','direction','directionCode','sourceName', 36 | 'spectralWindowId']) 37 | else: 38 | return False 39 | 40 | def getNewField(f=None): 41 | fieldXML = open(f,'r') 42 | if fieldXML is not False: 43 | f = minidom.parseString(fieldXML.read()) 44 | fieldList = list() 45 | rows = f.getElementsByTagName('row') 46 | for i in rows: 47 | fieldList.append((i.getElementsByTagName('fieldId')[0].firstChild.data, 48 | i.getElementsByTagName('fieldName')[0].firstChild.data, 49 | i.getElementsByTagName('numPoly')[0].firstChild.data, 50 | #i.getElementsByTagName('delayDir')[0].firstChild.data, 51 | #i.getElementsByTagName('phaseDir')[0].firstChild.data, 52 | i.getElementsByTagName('referenceDir')[0].firstChild.data, 53 | int(i.getElementsByTagName('time')[0].firstChild.data), 54 | i.getElementsByTagName('code')[0].firstChild.data, 55 | i.getElementsByTagName('directionCode')[0].firstChild.data, 56 | int(i.getElementsByTagName('sourceId')[0].firstChild.data))) 57 | #return pd.DataFrame(fieldList, columns=['fieldId', 'fieldName', 'numPoly','delayDir','phaseDir','referenceDir', 'time', 'code', 'directionCode', 'sourceId']) 58 | fieldXML.close() 59 | return pd.DataFrame(fieldList, columns=['fieldId', 'fieldName', 'numPoly','referenceDir', 'time', 'code', 'directionCode', 'sourceId']) 60 | else: 61 | return False 62 | 63 | def WriteNewSource(sourceuid = None, dir = None, df = None): 64 | sourceXML = GetXML(sourceuid,'Source') 65 | if sourceXML is not False: 66 | f = minidom.parseString(sourceXML) 67 | r = f.getElementsByTagName('row') 68 | for i in r: 69 | sourceName = unicode(i.getElementsByTagName('sourceName')[0].firstChild.data).strip() 70 | direction = unicode(i.getElementsByTagName('direction')[0].firstChild.data).strip() 71 | try: 72 | text = str(df[(sourceName,direction)]) 73 | i.getElementsByTagName('sourceId')[0].firstChild.replaceWholeText(text) 74 | except KeyError as e: 75 | pass 76 | open(dir+"/Source.xml.new","wb").write(f.toxml()) 77 | 78 | def WriteNewField(fielduid = None, dir = None, fieldDict = None, sourceDict=None): 79 | fieldXML = GetXML(fielduid,'Field') 80 | if fieldXML is not False: 81 | f = minidom.parseString(fieldXML) 82 | r = f.getElementsByTagName('row') 83 | for i in r: 84 | fieldId = unicode(i.getElementsByTagName('fieldId')[0].firstChild.data) 85 | sourceName = unicode(i.getElementsByTagName('fieldName')[0].firstChild.data).strip() 86 | referenceDir = unicode(i.getElementsByTagName('referenceDir')[0].firstChild.data).strip() 87 | try: 88 | sourceId = str(sourceDict[(sourceName,referenceDir[2:])]) 89 | except KeyError as e: 90 | newDict = dict() 91 | for key, value in sourceDict.iteritems(): 92 | newDict[key[0]] = value 93 | sourceId = str(newDict[sourceName]) 94 | i.getElementsByTagName('sourceId')[0].firstChild.replaceWholeText(sourceId) 95 | try: 96 | ra_new, dec_new = fieldDict[fieldId] 97 | text = ' 2 1 2 %.16f %.16f '%(ra_new,dec_new) 98 | i.getElementsByTagName('delayDir')[0].firstChild.replaceWholeText(text) 99 | i.getElementsByTagName('phaseDir')[0].firstChild.replaceWholeText(text) 100 | i.getElementsByTagName('referenceDir')[0].firstChild.replaceWholeText(text) 101 | except KeyError as e: 102 | pass 103 | 104 | open(dir+"/Field.xml.new","wb").write(f.toxml()) 105 | 106 | def measureDistance(lat1, lon1, lat2, lon2): 107 | R = 6383.137 # Radius of earth at Chajnantor aprox. in KM 108 | dLat = (lat2 - lat1) * np.pi / 180. 109 | dLon = (lon2 - lon1) * np.pi / 180. 110 | a = pl.sin(dLat/2.) * pl.sin(dLat/2.) + pl.cos(lat1 * np.pi / 180.) * pl.cos(lat2 * np.pi / 180.) * pl.sin(dLon/2.) * pl.sin(dLon/2.) 111 | c = 2. * atan2(pl.sqrt(a), pl.sqrt(1-a)) 112 | d = R * c 113 | return d * 1000. # meters 114 | 115 | def rot(Pl, rLong, rLat): 116 | return [pl.cos(rLong)*pl.cos(rLat) * Pl[0] - pl.sin(rLong) * Pl[1] - pl.cos(rLong)*pl.sin(rLat) * Pl[2], 117 | pl.sin(rLong)*pl.cos(rLat) * Pl[0] + pl.cos(rLong) * Pl[1] - pl.sin(rLong)*pl.sin(rLat) * Pl[2], 118 | pl.sin(rLat) * Pl[0] + pl.cos(rLat) * Pl[2]] 119 | 120 | parser = ASDMParseOptions() 121 | parser.asALMA() 122 | parser.loadTablesOnDemand(True) 123 | # Read ASDM 124 | asdmtable = ASDM() 125 | if len(sys.argv[1]) >= 1: 126 | asdmdir = sys.argv[1] 127 | else: 128 | asdmdir = 'uid___A002_X826a79_Xcdb' 129 | 130 | try: 131 | if sys.argv[2] == 'silent': 132 | silent = True 133 | else: 134 | silent = False 135 | except IndexError as e: 136 | silent = False 137 | pass 138 | 139 | asdmtable.setFromFile(asdmdir, parser) 140 | uid = asdmtable.entity().toString().split('"')[1] 141 | asdm = AsdmCheck() 142 | asdm.setUID(uid) 143 | sb = getSBSummary(asdm.asdmDict['SBSummary']) 144 | #Get all the Parts that we need 145 | scan = getScan(asdm.asdmDict['Scan']) 146 | subscan = getSubScan(asdm.asdmDict['Subscan']) 147 | field = getField(asdm.asdmDict['Field']) 148 | antenna = getAntennas(asdm.asdmDict['Antenna']) 149 | station = getStation(asdm.asdmDict['Station']) 150 | source = getSource(asdm.asdmDict['Source']) 151 | sbUID = sb.values[0][0] 152 | main = getMain(asdm.asdmDict['Main']) 153 | 154 | rows = asdmtable.pointingTable().get() 155 | pointingList = list() 156 | for idx,row in enumerate(rows): 157 | pointingList.append((idx, str(row.antennaId()), float(row.numSample()), float(row.numTerm()),str(row.timeInterval()), str(row.timeOrigin()))) 158 | 159 | pointingAll = pd.DataFrame(pointingList,columns = ['rowNum','antennaId','samples','iter','duration','origin']) 160 | 161 | #TODO: Fix to match any antenna 162 | pointing = pointingAll[pointingAll['antennaId'] == 'Antenna_1'] 163 | #do some transformations for matching the data 164 | scan['target'] = scan.apply(lambda x: True if str(x['scanIntent']).find('OBSERVE_TARGET') > 0 else False ,axis = 1) 165 | tsysScans = list(set(scan.sourceName[scan['target'] == True].values)) 166 | scan['target'] = scan.apply(lambda x: True if str(x['sourceName']) in tsysScans else x['target'] ,axis = 1) 167 | targets = map(unicode.strip,list(scan[scan['target'] == True].sourceName.values)) 168 | source['target'] = source.apply(lambda x: True if str(x['sourceName']).strip() in targets else False, axis = 1) 169 | subscan['target'] = subscan.apply(lambda x: True if str(x['fieldName']).strip() in targets else False, axis = 1) 170 | source['ra'], source['dec'] = zip(*source.apply(lambda x: arrayParser(x['direction'],1), axis = 1)) 171 | field['target'] = field.apply(lambda x: True if str(x['fieldName']).strip() in targets else False, axis = 1) 172 | 173 | list_of_targets = source[source['target'] ==True][['sourceName','ra','dec']].drop_duplicates() 174 | g = pd.merge(subscan,list_of_targets, left_on = 'fieldName', right_on= 'sourceName', how = 'inner') 175 | pointing['go'] = False 176 | field['ra'],field['dec'] = zip(*field.apply(lambda x: arrayParser(x['referenceDir'],2)[0], axis = 1)) 177 | field['ra'] = field['ra'].astype(float) 178 | field['dec'] = field['dec'].astype(float) 179 | field['ra'] = field.apply(lambda x: 2*pl.pi + x['ra'] if x['ra'] < 0 else x['ra'], axis = 1) 180 | correctedList = list() 181 | fullbar = list() 182 | for name, groups in g.groupby('sourceName'): 183 | foo = list(groups.scanNumber[groups['target'] == True]) 184 | print foo 185 | bar = list(main.loc[main['scanNumber'].isin(foo) ]['fieldId'].unique()) 186 | for i in bar: 187 | fullbar.append(i) 188 | for i in subscan.loc[subscan['scanNumber'].isin(foo) ][['startTime','endTime']].values: 189 | pointing['go'] = pointing.apply(lambda x: name if prs.parse(sdmTimeString(i[0])) - datetime.timedelta(seconds=1) <= prs.parse(x['origin']) and prs.parse(sdmTimeString(i[1])) >= prs.parse(x['origin']) else x['go'], axis = 1) 190 | 191 | ra = float(groups[groups['target'] ==True]['ra'].unique()[0]) 192 | if ra < 0: 193 | ra = ra * -1. 194 | dec = float(groups[groups['target'] ==True]['dec'].unique()[0]) 195 | 196 | correctedList.append((ra,dec,0)) 197 | for i in pointing.query('go == "%s"'%name).rowNum.values: 198 | row = rows[i] 199 | dRA,dDec = [[p[0].get(),p[1].get()] for p in row.sourceOffset() ][row.numSample()/2] 200 | # if dRA == 0. or dDec == 0.: 201 | # print "no Offset positions in the Pointing Table" 202 | # print "Is this EB a MultiSource?" 203 | # sys.exit(1) 204 | Pl = [pl.cos(dRA)*pl.cos(dDec), pl.sin(dRA)*pl.cos(dDec), pl.sin(dDec)] 205 | Ps = rot(Pl, ra, dec) 206 | correctedList.append((pl.arctan2(Ps[1], Ps[0]) % (2.*pl.pi), pl.arcsin(Ps[2]), i)) 207 | 208 | correctedAll = pd.DataFrame(correctedList, columns=['ra','dec', 'row']) 209 | corrected = correctedAll[['ra','dec']] 210 | corrected['series'] = 'Corrected (Pointing.bin)' 211 | corrected = corrected.drop_duplicates() 212 | corrected = corrected.reset_index(drop=True) 213 | 214 | observed = field[field['target'] == True][['fieldId','ra','dec']] 215 | observed.ra = observed.ra.astype(float) 216 | observed.dec = observed.dec.astype(float) 217 | observed['ra'] = observed.apply(lambda x: x['ra']*-1. if x['ra'] < 0.0 else x['ra'], axis = 1) 218 | observed = observed.loc[observed['fieldId'].isin(fullbar) ] 219 | observed = observed.reset_index(drop=True) 220 | observed['series'] = 'Field.xml' 221 | 222 | 223 | cat = SkyCoord(observed.ra.values * u.rad, observed.dec.values * u.rad, frame='icrs') 224 | cat2 = SkyCoord(corrected.ra.values * u.rad, corrected.dec.values *u.rad, frame='icrs') 225 | match, separ, dist = cat2.match_to_catalog_sky(cat) 226 | 227 | geo = pd.merge(antenna,station, left_on='stationId', right_on = 'stationId', how = 'inner') 228 | geo['pos'] = geo.apply(lambda x: arrayParser(x['position'],1) , axis = 1 ) 229 | geo['lat'], geo['lon'], geo['alt'] = zip(*geo.apply(lambda x: gh.turn_xyz_into_llh(float(x.pos[0]),float(x.pos[1]),float(x.pos[2]), 'wgs84'),axis=1)) 230 | comb = combinations(geo[['lat','lon']].values, 2) 231 | combList = list() 232 | for i in comb: 233 | combList.append((i[0][0],i[0][1],i[1][0],i[1][1])) 234 | baseLines = pd.DataFrame(combList) 235 | baseLines['dist'] = baseLines.apply(lambda x: measureDistance(x[0],x[1],x[2],x[3]) , axis = 1) 236 | blMax = baseLines.dist.describe().values[7] 237 | sbfreq = np.float(sb.frequency.values[0])*1e9 238 | c = 299792458 239 | l = c / sbfreq 240 | beam = l / blMax 241 | sbeam = beam * 206264.80624709636 242 | 243 | 244 | diff = pd.concat([observed.ix[match].reset_index(drop=True),corrected] , axis = 1) 245 | diff.columns = ['fieldId','ra_field','dec_field','field','ra_pointing','dec_pointing','pointing'] 246 | diff['ra_diff'] = diff.apply(lambda x: pl.absolute(x['ra_field'] - x['ra_pointing'])*pl.cos(dec), axis = 1) 247 | diff['dec_diff'] = diff.apply(lambda x: pl.absolute(x['dec_field'] - x['dec_pointing']), axis = 1) 248 | diff['total'] = diff.apply(lambda x: ((x['ra_diff']**2 + x['dec_diff']**2)**(0.5))*206264.80624709636, axis = 1) 249 | 250 | 251 | print "###########################" 252 | print "###### FIX SACM 211 #######" 253 | print "###########################" 254 | print "Number of pointings in Fields.xml: " , str(observed.ra.describe().values[0]) 255 | print "Number of pointings in Pointing : " , str(corrected.ra.describe().values[0]) 256 | print 'Mean Offset (arcsec) :',str(diff.total.describe().values[1]) 257 | print 'Max Offset (arcsec) :',str(diff.total.describe().values[7]) 258 | print '10% Sbeam :', str(sbeam /10.) 259 | print "###########################" 260 | print "The Fields are matching the following Sources" 261 | print source[source['sourceId'].isin(field.query('target == True').sourceId.unique())][['sourceId','sourceName']].drop_duplicates() 262 | print "###########################" 263 | print source[['sourceId','sourceName','ra','dec']].drop_duplicates() 264 | print "###########################" 265 | print field 266 | 267 | 268 | #Field == Pointing, no fix in Main.xml 269 | #Field > Pointing, fix in Main.xml, replace the old fieldIds with new ones 270 | #Field < Pointing , re index new pointings and add them into the list of fields <<< Should NOT HAPPEND! 271 | 272 | #Normal Fix 273 | if not silent: 274 | doit = raw_input('\n Would you like to rebuild Source Table? (Y/n)') 275 | if 'Y' in doit or 'y' in doit or len(doit)==0: 276 | df = source[['sourceName','direction']].drop_duplicates().reset_index(drop=True) 277 | df['sourceName'] = df.apply(lambda x: unicode(x['sourceName']).strip(), axis = 1) 278 | df['direction'] = df.apply(lambda x: unicode(x['direction']).strip(), axis = 1) 279 | sourceDict = dict(zip(zip(df.sourceName.values, df.direction.values),df.index)) 280 | WriteNewSource(asdm.asdmDict['Source'],asdmdir,sourceDict) 281 | newSource = getNewSource(asdmdir+'/Source.xml.new') 282 | print "New Values for Source table" 283 | print newSource[['sourceId','sourceName','direction']].drop_duplicates() 284 | new = diff[['fieldId','ra_pointing','dec_pointing']] 285 | newDict = new.set_index('fieldId').T.to_dict('list') 286 | WriteNewField(asdm.asdmDict['Field'] ,asdmdir ,fieldDict=newDict, sourceDict=sourceDict) 287 | print "New Values for Fields Table" 288 | newField = getNewField(asdmdir+'/Field.xml.new') 289 | print newField 290 | else: 291 | df = source[['sourceName','direction']].drop_duplicates().reset_index(drop=True) 292 | df['sourceName'] = df.apply(lambda x: unicode(x['sourceName']).strip(), axis = 1) 293 | df['direction'] = df.apply(lambda x: unicode(x['direction']).strip(), axis = 1) 294 | sourceDict = dict(zip(zip(df.sourceName.values, df.direction.values),df.index)) 295 | new = diff[['fieldId','ra_pointing','dec_pointing']] 296 | newDict = new.set_index('fieldId').T.to_dict('list') 297 | WriteNewField(asdm.asdmDict['Field'] ,asdmdir ,fieldDict=newDict, sourceDict=sourceDict) 298 | print "New Values for Fields Table" 299 | newField = getNewField(asdmdir+'/Field.xml.new') 300 | print newField 301 | else: 302 | df = source[['sourceName','direction']].drop_duplicates().reset_index(drop=True) 303 | df['sourceName'] = df.apply(lambda x: unicode(x['sourceName']).strip(), axis = 1) 304 | df['direction'] = df.apply(lambda x: unicode(x['direction']).strip(), axis = 1) 305 | sourceDict = dict(zip(zip(df.sourceName.values, df.direction.values),df.index)) 306 | WriteNewSource(asdm.asdmDict['Source'],asdmdir,sourceDict) 307 | newSource = getNewSource(asdmdir+'/Source.xml.new') 308 | print "New Values for Source table" 309 | print newSource[['sourceId','sourceName','direction']].drop_duplicates() 310 | new = diff[['fieldId','ra_pointing','dec_pointing']] 311 | newDict = new.set_index('fieldId').T.to_dict('list') 312 | WriteNewField(asdm.asdmDict['Field'] ,asdmdir ,fieldDict=newDict, sourceDict=sourceDict) 313 | print "New Values for Fields Table" 314 | newField = getNewField(asdmdir+'/Field.xml.new') 315 | print newField 316 | 317 | 318 | 319 | 320 | final = pd.concat([corrected,observed]) 321 | final[['ra','dec']] = final[['ra','dec']].astype(float) 322 | groups = final.groupby('series') 323 | 324 | if silent: 325 | fig, ax = plt.subplots() 326 | ax.margins(0.05) 327 | marks = ['.','+','x'] 328 | colors = ['b','r','k'] 329 | for idx, x in enumerate(groups): 330 | ax.plot(x[1].ra, x[1].dec, marker=marks[idx], color=colors[idx],linestyle='', ms=12, label=x[0], alpha=0.6) 331 | ax.legend() 332 | plt.savefig(asdmdir+'/'+asdmdir+'.png', bbox_inches='tight') 333 | else: 334 | fig, ax = plt.subplots() 335 | ax.margins(0.05) 336 | marks = ['.','+','x'] 337 | colors = ['b','r','k'] 338 | for idx, x in enumerate(groups): 339 | ax.plot(x[1].ra, x[1].dec, marker=marks[idx], color=colors[idx],linestyle='', ms=12, label=x[0], alpha=0.6) 340 | ax.legend() 341 | plt.show() 342 | 343 | 344 | 345 | -------------------------------------------------------------------------------- /sacm/geo_helper.py: -------------------------------------------------------------------------------- 1 | # Geographical helper functions for nmea_info.py and friends 2 | # 3 | # Helps with geographic functions, including: 4 | # Lat+Long+Height -> XYZ 5 | # XYZ -> Lat+Long+Height 6 | # Lat+Long -> other Lat+Long (Helmert Transform) 7 | # Lat+Long -> easting/northing (OS GB+IE Only) 8 | # easting/northing -> Lat+Long (OS GB+IE Only) 9 | # OS easting/northing -> OS 6 figure ref 10 | # 11 | # See http://gagravarr.org/code/ for updates and information 12 | # 13 | # GPL 14 | # 15 | # Nick Burch - v0.06 (30/05/2007) 16 | 17 | import math 18 | 19 | # For each co-ordinate system we do, what are the A, B and E2 values? 20 | # List is A, B, E^2 (E^2 calculated after) 21 | abe_values = { 22 | 'wgs84': [ 6378137.0, 6356752.3141, -1 ], 23 | 'osgb' : [ 6377563.396, 6356256.91, -1 ], 24 | 'osie' : [ 6377340.189, 6356034.447, -1 ] 25 | } 26 | 27 | # The earth's radius, in meters, as taken from an average of the WGS84 28 | # a and b parameters (should be close enough) 29 | earths_radius = (abe_values['wgs84'][0] + abe_values['wgs84'][1]) / 2.0 30 | 31 | # Calculate the E2 values 32 | for system in abe_values.keys(): 33 | a = abe_values[system][0] 34 | b = abe_values[system][1] 35 | e2 = (a*a - b*b) / (a*a) 36 | abe_values[system][2] = e2 37 | 38 | # For each co-ordinate system we can translate between, what are 39 | # the tx, ty, tz, s, rx, ry and rz values? 40 | # List is tx, ty, tz, s, rx, ry, rz 41 | transform_values = { 42 | 'wgs84_to_osgb' : [ -446.448, 125.157, -542.060, 43 | 20.4894 / 1000.0 / 1000.0, # given as ppm 44 | -0.1502 / 206265.0, # given as seconds of arc 45 | -0.2470 / 206265.0, # given as seconds of arc 46 | -0.8421 / 206265.0 # given as seconds of arc 47 | ], 48 | 'wgs84_to_osie' : [ -482.530, 130.596, -564.557, 49 | -8.1500 / 1000.0 / 1000.0, # given as ppm 50 | -1.0420 / 206265.0, # given as seconds of arc 51 | -0.2140 / 206265.0, # given as seconds of arc 52 | -0.6310 / 206265.0 # given as seconds of arc 53 | ], 54 | 'itrs2000_to_etrs89' : [ 0.054, 0.051, -0.048, 0, 55 | 0.000081 / 206265.0, # given as seconds of arc 56 | 0.00049 / 206265.0, # given as seconds of arc 57 | 0.000792 / 206265.0 # given as seconds of arc 58 | ] 59 | } 60 | 61 | # Calculate reverse transforms 62 | for systems in [('wgs84','osgb'), ('wgs84','osie'), ('itrs2000','etrs89')]: 63 | fs = systems[0] + "_to_" + systems[1] 64 | rs = systems[1] + "_to_" + systems[0] 65 | ra = [] 66 | for val in transform_values[fs]: 67 | ra.append(-1.0 * val) 68 | transform_values[rs] = ra 69 | 70 | # Easting and Northin system values, for the systems we work with. 71 | # List is n0, e0, F0, theta0 and landa0 72 | en_values = { 73 | 'osgb' : [ -100000.0, 400000.0, 0.9996012717, 74 | 49.0 /360.0 *2.0*math.pi, 75 | -2.0 /360.0 *2.0*math.pi 76 | ], 77 | 'osie' : [ 250000.0, 200000.0, 1.000035, 78 | 53.5 /360.0 *2.0*math.pi, 79 | -8.0 /360.0 *2.0*math.pi 80 | ] 81 | } 82 | 83 | # Cassini Projection Origins 84 | # List is lat (rad), long (rad), false easting, false northing 85 | cassini_values = { 86 | 'osgb' : [ (53.0 + (13.0 / 60.0) + (17.274 / 3600.0)) /360.0 *2.0*math.pi, 87 | -(2.0 + (41.0 / 60.0) + (3.562 / 3600.0)) /360.0 *2.0*math.pi, 88 | 0, 0 ] 89 | } 90 | 91 | # How many feet to the meter 92 | feet_per_meter = 1.0 / 0.3048007491 # 3.28083 93 | 94 | ############################################################## 95 | # OS GB Specific Helpers for Generic Methods # 96 | ############################################################## 97 | 98 | def turn_wgs84_into_osgb36(lat_dec,long_dec,height): 99 | """See http://www.gps.gov.uk/guide6.asp#6.2 and http://www.gps.gov.uk/guide6.asp#6.6 for the calculations, and http://www.posc.org/Epicentre.2_2/DataModel/ExamplesofUsage/eu_cs34h.html for some background.""" 100 | 101 | wgs84_xyz = turn_llh_into_xyz(lat_dec,long_dec,height,'wgs84') 102 | 103 | osgb_xyz = turn_xyz_into_other_xyz( 104 | wgs84_xyz[0],wgs84_xyz[1],wgs84_xyz[2],'wgs84','osgb') 105 | 106 | osgb_latlong = turn_xyz_into_llh( 107 | osgb_xyz[0],osgb_xyz[1],osgb_xyz[2],'osgb') 108 | return osgb_latlong 109 | 110 | def turn_osgb36_into_wgs84(lat_dec,long_dec,height): 111 | """See http://www.gps.gov.uk/guide6.asp#6.2 and http://www.gps.gov.uk/guide6.asp#6.6 for the calculations, and http://www.posc.org/Epicentre.2_2/DataModel/ExamplesofUsage/eu_cs34h.html for some background.""" 112 | 113 | osgb_xyz = turn_llh_into_xyz(lat_dec,long_dec,height,'osgb') 114 | 115 | wgs84_xyz = turn_xyz_into_other_xyz( 116 | osgb_xyz[0],osgb_xyz[1],osgb_xyz[2],'osgb','wgs84') 117 | 118 | wgs84_latlong = turn_xyz_into_llh( 119 | wgs84_xyz[0],wgs84_xyz[1],wgs84_xyz[2],'wgs84') 120 | 121 | return wgs84_latlong 122 | 123 | def turn_osgb36_into_eastingnorthing(lat_dec,long_dec): 124 | """Turn OSGB36 (decimal) lat/long values into OS easting and northing values.""" 125 | return turn_latlong_into_eastingnorthing(lat_dec,long_dec,'osgb') 126 | 127 | def turn_eastingnorthing_into_osgb36(easting,northing): 128 | """Turn OSGB36 easting and northing values into (decimal) lat/long values inOSGB36.""" 129 | return turn_eastingnorthing_into_latlong(easting,northing,'osgb') 130 | 131 | ############################################################## 132 | # OS IE Specific Helpers for Generic Methods # 133 | ############################################################## 134 | 135 | def turn_wgs84_into_osie36(lat_dec,long_dec,height): 136 | """As per turn_wgs84_into_osgb36, but for Irish grid""" 137 | 138 | wgs84_xyz = turn_llh_into_xyz(lat_dec,long_dec,height,'wgs84') 139 | 140 | osie_xyz = turn_xyz_into_other_xyz( 141 | wgs84_xyz[0],wgs84_xyz[1],wgs84_xyz[2],'wgs84','osie') 142 | 143 | osie_latlong = turn_xyz_into_llh( 144 | osie_xyz[0],osie_xyz[1],osie_xyz[2],'osie') 145 | return osie_latlong 146 | 147 | def turn_osie36_into_wgs84(lat_dec,long_dec,height): 148 | """As per turn_osgb36_into_wgs84, but for Irish grid""" 149 | 150 | osie_xyz = turn_llh_into_xyz(lat_dec,long_dec,height,'osie') 151 | 152 | wgs84_xyz = turn_xyz_into_other_xyz( 153 | osie_xyz[0],osie_xyz[1],osie_xyz[2],'osie','wgs84') 154 | 155 | wgs84_latlong = turn_xyz_into_llh( 156 | wgs84_xyz[0],wgs84_xyz[1],wgs84_xyz[2],'wgs84') 157 | 158 | return wgs84_latlong 159 | 160 | def turn_osie36_into_eastingnorthing(lat_dec,long_dec): 161 | """Turn OSIE36 (decimal) lat/long values into OS IE easting and northing values.""" 162 | return turn_latlong_into_eastingnorthing(lat_dec,long_dec,'osie') 163 | 164 | def turn_eastingnorthing_into_osie36(easting,northing): 165 | """Turn OSIE36 easting and northing values into (decimal) lat/long values inOSIE36.""" 166 | return turn_eastingnorthing_into_latlong(easting,northing,'osie') 167 | 168 | ############################################################## 169 | # Generic Transform Functions # 170 | ############################################################## 171 | 172 | def turn_llh_into_xyz(lat_dec,long_dec,height,system): 173 | """Convert Lat, Long and Height into 3D Cartesian x,y,z 174 | See http://www.ordnancesurvey.co.uk/gps/docs/convertingcoordinates3D.pdf""" 175 | 176 | a = abe_values[system][0] 177 | b = abe_values[system][1] 178 | e2 = abe_values[system][2] 179 | 180 | theta = float(lat_dec) / 360.0 * 2.0 * math.pi 181 | landa = float(long_dec) / 360.0 * 2.0 * math.pi 182 | height = float(height) 183 | 184 | v = a / math.sqrt( 1.0 - e2 * (math.sin(theta) * math.sin(theta)) ) 185 | x = (v + height) * math.cos(theta) * math.cos(landa) 186 | y = (v + height) * math.cos(theta) * math.sin(landa) 187 | z = ( (1.0 - e2) * v + height ) * math.sin(theta) 188 | 189 | return [x,y,z] 190 | 191 | def turn_xyz_into_llh(x,y,z,system): 192 | """Convert 3D Cartesian x,y,z into Lat, Long and Height 193 | See http://www.ordnancesurvey.co.uk/gps/docs/convertingcoordinates3D.pdf""" 194 | 195 | a = abe_values[system][0] 196 | b = abe_values[system][1] 197 | e2 = abe_values[system][2] 198 | 199 | p = math.sqrt(x*x + y*y) 200 | 201 | long = math.atan(y/x) 202 | lat_init = math.atan( z / (p * (1.0 - e2)) ) 203 | v = a / math.sqrt( 1.0 - e2 * (math.sin(lat_init) * math.sin(lat_init)) ) 204 | lat = math.atan( (z + e2*v*math.sin(lat_init)) / p ) 205 | 206 | height = (p / math.cos(lat)) - v # Ignore if a bit out 207 | 208 | # Turn from radians back into degrees 209 | long = long / 2 / math.pi * 360 210 | lat = lat / 2 / math.pi * 360 211 | 212 | return [lat,long,height] 213 | 214 | def turn_xyz_into_other_xyz(old_x,old_y,old_z,from_scheme,to_scheme): 215 | """Helmert Transformation between one lat+long system and another 216 | See http://www.ordnancesurvey.co.uk/oswebsite/gps/information/coordinatesystemsinfo/guidecontents/guide6.html for the calculations, and http://www.movable-type.co.uk/scripts/LatLongConvertCoords.html for a friendlier version with examples""" 217 | 218 | transform = from_scheme + "_to_" + to_scheme 219 | tx = transform_values[transform][0] 220 | ty = transform_values[transform][1] 221 | tz = transform_values[transform][2] 222 | s = transform_values[transform][3] 223 | rx = transform_values[transform][4] 224 | ry = transform_values[transform][5] 225 | rz = transform_values[transform][6] 226 | 227 | # Do the transform 228 | new_x = tx + ((1.0+s) * old_x) + (-rz * old_y) + (ry * old_z) 229 | new_y = ty + (rz * old_x) + ((1.0+s) * old_y) + (-rx * old_z) 230 | new_z = tz + (-ry * old_x) + (rx * old_y) + ((1.0+s) * old_z) 231 | 232 | return [new_x,new_y,new_z] 233 | 234 | def calculate_distance_and_bearing(from_lat_dec,from_long_dec,to_lat_dec,to_long_dec): 235 | """Uses the spherical law of cosines to calculate the distance and bearing between two positions""" 236 | 237 | # Turn them all into radians 238 | from_theta = float(from_lat_dec) / 360.0 * 2.0 * math.pi 239 | from_landa = float(from_long_dec) / 360.0 * 2.0 * math.pi 240 | to_theta = float(to_lat_dec) / 360.0 * 2.0 * math.pi 241 | to_landa = float(to_long_dec) / 360.0 * 2.0 * math.pi 242 | 243 | d = math.acos( 244 | math.sin(from_theta) * math.sin(to_theta) + 245 | math.cos(from_theta) * math.cos(to_theta) * math.cos(to_landa-from_landa) 246 | ) * earths_radius 247 | 248 | bearing = math.atan2( 249 | math.sin(to_landa-from_landa) * math.cos(to_theta), 250 | math.cos(from_theta) * math.sin(to_theta) - 251 | math.sin(from_theta) * math.cos(to_theta) * math.cos(to_landa-from_landa) 252 | ) 253 | bearing = bearing / 2.0 / math.pi * 360.0 254 | 255 | return [d,bearing] 256 | 257 | ############################################################## 258 | # Easting/Northing Transform Methods # 259 | ############################################################## 260 | 261 | def turn_latlong_into_eastingnorthing(lat_dec,long_dec,scheme): 262 | """Turn OSGB36 or OSIE36 (decimal) lat/long values into OS easting and northing values. See http://www.ordnancesurvey.co.uk/oswebsite/gps/information/coordinatesystemsinfo/guidecontents/guide7.html for the calculations, and http://www.posc.org/Epicentre.2_2/DataModel/ExamplesofUsage/eu_cs34h.html for some background.""" 263 | 264 | n0 = en_values[scheme][0] 265 | e0 = en_values[scheme][1] 266 | f0 = en_values[scheme][2] 267 | 268 | theta0 = en_values[scheme][3] 269 | landa0 = en_values[scheme][4] 270 | 271 | a = abe_values[scheme][0] 272 | b = abe_values[scheme][1] 273 | e2 = abe_values[scheme][2] 274 | 275 | theta = float(lat_dec) /360.0 *2.0*math.pi 276 | landa = float(long_dec) /360.0 *2.0*math.pi 277 | 278 | n = (a-b) / (a+b) 279 | v = a * f0 * math.pow( (1 - e2 * math.sin(theta)*math.sin(theta)), -0.5 ) 280 | ro = a * f0 * (1 - e2) * math.pow( (1 - e2 * math.sin(theta)*math.sin(theta)), -1.5 ) 281 | nu2 = v/ro - 1 282 | 283 | M = b * f0 * ( \ 284 | (1.0 + n + 5.0/4.0 *n*n + 5.0/4.0 *n*n*n) * (theta-theta0) - \ 285 | (3.0*n + 3.0*n*n + 21.0/8.0 *n*n*n) *math.sin(theta-theta0) *math.cos(theta+theta0) + \ 286 | (15.0/8.0*n*n + 15.0/8.0*n*n*n) *math.sin(2.0*(theta-theta0)) *math.cos(2.0*(theta+theta0)) - \ 287 | 35.0/24.0*n*n*n *math.sin(3.0*(theta-theta0)) *math.cos(3.0*(theta+theta0)) \ 288 | ) 289 | 290 | I = M + n0 291 | II = v/2.0 * math.sin(theta) * math.cos(theta) 292 | III = v/24.0 * math.sin(theta) * math.pow( math.cos(theta),3 ) * \ 293 | (5.0 - math.pow(math.tan(theta),2) + 9.0*nu2) 294 | IIIa = v/720.0 * math.sin(theta) * math.pow( math.cos(theta),5 ) * \ 295 | ( 61.0 - 58.0 *math.pow(math.tan(theta),2) + math.pow(math.tan(theta),4) ) 296 | IV = v * math.cos(theta) 297 | V = v/6.0 * math.pow( math.cos(theta),3 ) * \ 298 | ( v/ro - math.pow(math.tan(theta),2) ) 299 | VI = v/120.0 * math.pow(math.cos(theta),5) * \ 300 | ( 5.0 - 18.0 *math.pow(math.tan(theta),2) + \ 301 | math.pow(math.tan(theta),4) + 14.0*nu2 - \ 302 | 58.0 * math.pow(math.tan(theta),2)*nu2 ) 303 | 304 | northing = I + II*math.pow(landa-landa0,2) + \ 305 | III*math.pow(landa-landa0,4) + \ 306 | IIIa*math.pow(landa-landa0,6) 307 | easting = e0 + IV*(landa-landa0) + V*math.pow(landa-landa0,3) + \ 308 | VI*math.pow(landa-landa0,5) 309 | 310 | return (easting,northing) 311 | 312 | def turn_eastingnorthing_into_latlong(easting,northing,scheme): 313 | """Turn OSGB36 or OSIE36 easting and northing values into (decimal) lat/long values in OSGB36 / OSIE36. See http://www.ordnancesurvey.co.uk/oswebsite/gps/information/coordinatesystemsinfo/guidecontents/guide7.html for the calculations, and http://www.posc.org/Epicentre.2_2/DataModel/ExamplesofUsage/eu_cs34h.html for some background.""" 314 | 315 | n0 = en_values[scheme][0] 316 | e0 = en_values[scheme][1] 317 | f0 = en_values[scheme][2] 318 | 319 | theta0 = en_values[scheme][3] 320 | landa0 = en_values[scheme][4] 321 | 322 | a = abe_values[scheme][0] 323 | b = abe_values[scheme][1] 324 | e2 = abe_values[scheme][2] 325 | 326 | n = (a-b) / (a+b) 327 | 328 | # Prepare to iterate 329 | M = 0 330 | theta = theta0 331 | # Iterate, 4 times should be enough 332 | for i in range(4): 333 | theta = ((northing - n0 - M) / (a * f0)) + theta 334 | M = b * f0 * ( \ 335 | (1.0 + n + 5.0/4.0 *n*n + 5.0/4.0 *n*n*n) * (theta-theta0) - \ 336 | (3.0*n + 3.0*n*n + 21.0/8.0 *n*n*n) *math.sin(theta-theta0) *math.cos(theta+theta0) + \ 337 | (15.0/8.0*n*n + 15.0/8.0*n*n*n) *math.sin(2.0*(theta-theta0)) *math.cos(2.0*(theta+theta0)) - \ 338 | 35.0/24.0*n*n*n *math.sin(3.0*(theta-theta0)) *math.cos(3.0*(theta+theta0)) \ 339 | ) 340 | 341 | # Compute intermediate values 342 | v = a * f0 * math.pow( (1 - e2 * math.sin(theta)*math.sin(theta)), -0.5 ) 343 | ro = a * f0 * (1 - e2) * math.pow( (1 - e2 * math.sin(theta)*math.sin(theta)), -1.5 ) 344 | nu2 = v/ro - 1 345 | tantheta2 = math.pow(math.tan(theta),2) 346 | 347 | VII = math.tan(theta) / (2 * ro * v) 348 | VIII = math.tan(theta) / (24 * ro * math.pow(v,3)) \ 349 | * (5 + 3 * tantheta2 + nu2 - \ 350 | 9 * tantheta2 * nu2 ) 351 | IX = math.tan(theta) / (720 * ro * math.pow(v,5)) \ 352 | * (61 + 90 * tantheta2 + 45 * tantheta2 * tantheta2) 353 | X = 1 / (math.cos(theta) * v) 354 | XI = 1 / (math.cos(theta) * 6 * math.pow(v,3)) \ 355 | * (v/ro + 2*tantheta2) 356 | XII = 1 / (math.cos(theta) * 120 * math.pow(v,5)) \ 357 | * (5 + 28 * tantheta2 + 24 * tantheta2 * tantheta2) 358 | XIIa = 1 / (math.cos(theta) * 5040 * math.pow(v,7)) \ 359 | * (61 + 662 * tantheta2 + 1320 * tantheta2 * tantheta2 \ 360 | + 720 * tantheta2 * tantheta2 * tantheta2) 361 | 362 | lat_rad = theta - VII * math.pow((easting-e0),2) \ 363 | + VIII * math.pow((easting-e0),4) \ 364 | - IX * math.pow((easting-e0),6) 365 | long_rad = landa0 + X * (easting-e0) \ 366 | - XI * math.pow((easting-e0),3) \ 367 | + XII * math.pow((easting-e0),5) \ 368 | - XIIa * math.pow((easting-e0),7) 369 | 370 | lat = lat_rad / 2.0 / math.pi * 360.0 371 | long = long_rad / 2.0 / math.pi * 360.0 372 | 373 | return (lat,long) 374 | 375 | ############################################################## 376 | # Cassini Easting/Northing Transform Methods # 377 | ############################################################## 378 | 379 | def turn_latlong_into_cassini_en(lat_dec,long_dec,scheme): 380 | """Latitude and Longitude, into Cassini-Soldner easting and northing co-ordinates, in the given scheme. See http://www.posc.org/Epicentre.2_2/DataModel/ExamplesofUsage/eu_cs34g.html for details of the calculation used""" 381 | 382 | a = abe_values[scheme][0] 383 | b = abe_values[scheme][1] 384 | e2 = abe_values[scheme][2] 385 | 386 | e4 = e2 * e2 387 | e6 = e2 * e2 * e2 388 | 389 | theta = float(lat_dec) /360.0 *2.0*math.pi 390 | landa = float(long_dec) /360.0 *2.0*math.pi 391 | 392 | theta0 = cassini_values[scheme][0] 393 | landa0 = cassini_values[scheme][1] 394 | false_easting = cassini_values[scheme][2] 395 | false_northing = cassini_values[scheme][3] 396 | 397 | # Compute intermediate values 398 | A = (landa - landa0) * math.cos(theta) 399 | T = math.tan(theta) * math.tan(theta) 400 | C = e2 / (1.0 - e2) * math.cos(theta) * math.cos(theta) 401 | v = a / math.sqrt( 1 - (e2 * math.sin(theta) * math.sin(theta)) ) 402 | 403 | A2 = A ** 2 404 | A3 = A ** 3 405 | A4 = A ** 4 406 | A5 = A ** 5 407 | 408 | # And M, which is how far along the meridian our latitude is from the origin 409 | def makeM(picked_theta): 410 | return a * ( 411 | (1.0 - e2/4.0 - 3.0*e4/64.0 - 5.0*e6/256.0) * picked_theta 412 | - (3.0*e2/8.0 + 3.0*e4/32.0 + 45.0*e6/1024.0) * math.sin(2.0*picked_theta) 413 | + (15.0*e4/256.0 + 45.0*e6/1024.0) * math.sin(4.0*picked_theta) 414 | - (35.0*e6/3072.0) * math.sin(6.0*picked_theta) 415 | ) 416 | M = makeM(theta) 417 | M0 = makeM(theta0) 418 | 419 | # Now calculate 420 | easting = false_easting + v * ( 421 | A - T * A3 / 6.0 - (8.0 - T + 8.0*C) * T * A5 / 120.0 ) 422 | northing = false_northing + M - M0 + v * math.tan(theta) * ( 423 | A2 / 2.0 + (5.0 - T + 6.0*C) * A4 / 24.0 ) 424 | 425 | return (easting,northing) 426 | 427 | def turn_cassini_en_into_latlong(easting,northing,scheme): 428 | """Cassini-Soldner easting and northing, into Latitude and Longitude, in the given scheme. See http://www.posc.org/Epicentre.2_2/DataModel/ExamplesofUsage/eu_cs34g.html for details of the calculation used""" 429 | 430 | a = abe_values[scheme][0] 431 | b = abe_values[scheme][1] 432 | e2 = abe_values[scheme][2] 433 | 434 | e4 = e2 * e2 435 | e6 = e2 * e2 * e2 436 | 437 | theta0 = cassini_values[scheme][0] 438 | landa0 = cassini_values[scheme][1] 439 | false_easting = cassini_values[scheme][2] 440 | false_northing = cassini_values[scheme][3] 441 | 442 | def makeM(picked_theta): 443 | return a * ( 444 | (1.0 - e2/4.0 - 3.0*e4/64.0 - 5.0*e6/256.0) * picked_theta 445 | - (3.0*e2/8.0 + 3.0*e4/32.0 + 45.0*e6/1024.0) * math.sin(2.0*picked_theta) 446 | + (15.0*e4/256.0 + 45.0*e6/1024.0) * math.sin(4.0*picked_theta) 447 | - (35.0*e6/3072.0) * math.sin(6.0*picked_theta) 448 | ) 449 | 450 | # Compute first batch of intermediate values 451 | M1 = makeM(theta0) + (northing - false_northing) 452 | mu1 = M1 / (a * (1.0 - e2/4.0 - 3.0*e4/64.0 - 5.0*e6/256.0) ) 453 | e1 = (1 - ((1-e2) ** 0.5)) / (1 + ((1-e2) ** 0.5)) 454 | 455 | e1_2 = e1 ** 2 456 | e1_3 = e1 ** 3 457 | e1_4 = e1 ** 4 458 | 459 | # Now compute theta1 at T1 460 | theta1 = mu1 + ( 461 | + (3.0*e1 / 2.0 - 27.0*e1_3 / 32.0) * math.sin(2.0*mu1) 462 | + (21.0*e1_2 / 16.0 - 55.0*e1_4 / 32.0) * math.sin(4.0*mu1) 463 | + (151.0*e1_3 / 96.0) * math.sin(6.0*mu1) 464 | + (1097.0*e1_4 / 512.0) * math.sin(8.0*mu1) 465 | ) 466 | T1 = (math.tan(theta1)) ** 2 467 | 468 | # Now we can find v1, ro1 and D 469 | v1 = a / math.sqrt( 1.0 - (e2 * math.sin(theta1) * math.sin(theta1)) ) 470 | ro1 = a * (1 - e2) / ((1 - e2 * math.sin(theta1) * math.sin(theta1)) ** 1.5) 471 | D = (easting - false_easting) / v1 472 | 473 | # And finally the lat and long 474 | lat = theta1 - (v1 * math.tan(theta1)) / ro1 * ( 475 | D*D/2.0 - (1.0 + 3.0 * T1) * ( (D**4) / 24.0 ) ) 476 | long = landa0 + ( 477 | D - T1 * (D**3) / 3.0 + (1 + 3.0 * T1) * T1 * (D**5) / 15.0 478 | ) / math.cos(theta1) 479 | 480 | # Now make decimal versions 481 | lat_dec = lat * 360.0 / 2.0 / math.pi 482 | long_dec = long * 360.0 / 2.0 / math.pi 483 | 484 | return (lat_dec,long_dec) 485 | 486 | ############################################################## 487 | # OS Specific Methods Follow # 488 | ############################################################## 489 | 490 | def turn_easting_northing_into_six_fig(easting,northing): 491 | """Turn OS easting and northing values into the six figure OS grid refecence. See http://www.jstott.me.uk/jscoord/""" 492 | first_letter = "" 493 | second_letter = "" 494 | 495 | easting = int(easting) 496 | northing = int(northing) 497 | 498 | # Get the 100 km part 499 | hkm_east = int( math.floor(easting / 100000.0) ) 500 | hkm_north = int( math.floor(northing / 100000.0) ) 501 | if hkm_north < 5: 502 | if hkm_east < 5: 503 | first_letter = "S" 504 | else: 505 | first_letter = "T" 506 | elif hkm_north < 10: 507 | if hkm_east < 5: 508 | first_letter = "N" 509 | else: 510 | first_letter = "O" 511 | else: 512 | first_letter = "H" 513 | 514 | # Get the 10km part 515 | index = 65 + ((4 - (hkm_north % 5)) * 5) + (hkm_east % 5) 516 | ti = index 517 | if index >= 73: 518 | index += 1 519 | second_letter = chr(index) 520 | 521 | # Get digits 2-4 on easting and northing 522 | e = math.floor( (easting - (100000.0 * hkm_east)) / 100.0) 523 | n = math.floor( (northing - (100000.0 * hkm_north)) / 100.0) 524 | e = "%03d" % e 525 | n = "%03d" % n 526 | 527 | return first_letter + second_letter + e + n 528 | -------------------------------------------------------------------------------- /checker.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | __author__ = 'sdk' 3 | from sacm import * 4 | from xml.dom import minidom 5 | import pandas as pd 6 | 7 | asdmUID = '' 8 | 9 | 10 | def getASDM(uid=None): 11 | asdmXML = GetXML(uid,'ASDM') 12 | if asdmXML is not False: 13 | asdm = minidom.parseString(asdmXML) 14 | rows = asdm.getElementsByTagName('Table') 15 | asdmList = list() 16 | for i in rows: 17 | if int(i.getElementsByTagName('NumberRows')[0].firstChild.data) != 0: 18 | #print i.getElementsByTagName('Name')[0].firstChild.data,i.getElementsByTagName('NumberRows')[0].firstChild.data 19 | asdmList.append((i.getElementsByTagName('Name')[0].firstChild.data, 20 | i.getElementsByTagName('NumberRows')[0].firstChild.data, 21 | str(i.getElementsByTagName('Entity')[0].getAttribute('entityTypeName')), 22 | str(i.getElementsByTagName('Entity')[0].getAttribute('entityId')))) 23 | toc = asdm.getElementsByTagName('TimeOfCreation')[0].firstChild.data 24 | 25 | return asdmList, pd.DataFrame(asdmList, columns=['table', 'numrows', 'typename', 'uid'],) , datetime.datetime.strptime(toc.strip()[0:19],"%Y-%m-%dT%H:%M:%S" ) 26 | 27 | else: 28 | return False 29 | 30 | 31 | 32 | 33 | def getMain(uid=None): 34 | mainXML = GetXML(uid,'Main') 35 | if mainXML is not False: 36 | main = minidom.parseString(mainXML) 37 | mainList = list() 38 | rows = main.getElementsByTagName('row') 39 | for i in rows: 40 | #print i.getElementsByTagName('time')[0].firstChild.data, i.getElementsByTagName('stateId')[0].firstChild.data 41 | mainList.append((sdmTimeString(int(i.getElementsByTagName('time')[0].firstChild.data)), 42 | #i.getElementsByTagName('numAntenna')[0].firstChild.data, 43 | i.getElementsByTagName('timeSampling')[0].firstChild.data, 44 | int(i.getElementsByTagName('interval')[0].firstChild.data), 45 | #i.getElementsByTagName('numIntegration')[0].firstChild.data, 46 | int(i.getElementsByTagName('scanNumber')[0].firstChild.data), 47 | int(i.getElementsByTagName('subscanNumber')[0].firstChild.data), 48 | int(i.getElementsByTagName('dataSize')[0].firstChild.data), 49 | i.getElementsByTagName('dataUID')[0].getElementsByTagName('EntityRef')[0].getAttribute('entityId'), 50 | i.getElementsByTagName('fieldId')[0].firstChild.data, 51 | i.getElementsByTagName('stateId')[0].firstChild.data)) 52 | return pd.DataFrame(mainList, columns=['time', 'timeSampling', 'interval', 'scanNumber', 53 | 'subscanNumber', 'dataSize', 'dataUID', 'fieldId', 'stateId']) 54 | else: 55 | return False 56 | 57 | def getAntennas(uid=None): 58 | antennaXML = GetXML(uid,'Antenna') 59 | if antennaXML is not False: 60 | antenna = minidom.parseString(antennaXML) 61 | antennaList = list() 62 | rows = antenna.getElementsByTagName('row') 63 | for i in rows: 64 | antennaList.append(( 65 | i.getElementsByTagName('antennaId')[0].firstChild.data, 66 | i.getElementsByTagName('name')[0].firstChild.data, 67 | i.getElementsByTagName('antennaMake')[0].firstChild.data, 68 | i.getElementsByTagName('dishDiameter')[0].firstChild.data, 69 | i.getElementsByTagName('stationId')[0].firstChild.data, 70 | )) 71 | return pd.DataFrame(antennaList,columns=['antennaId','name','antennaMake','dishDiameter','stationId']) 72 | else: 73 | return False 74 | 75 | def getSBSummary(uid=None): 76 | summaryXML = GetXML(uid,'SBSummary') 77 | if summaryXML is not False: 78 | summary = minidom.parseString(summaryXML) 79 | summaryList = list() 80 | rows = summary.getElementsByTagName('row') 81 | for i in rows: 82 | summaryList.append((i.getElementsByTagName('sbSummaryUID')[0].getElementsByTagName('EntityRef')[0].getAttribute('entityId'), 83 | i.getElementsByTagName('projectUID')[0].getElementsByTagName('EntityRef')[0].getAttribute('entityId'), 84 | i.getElementsByTagName('obsUnitSetUID')[0].getElementsByTagName('EntityRef')[0].getAttribute('entityId'), 85 | float(i.getElementsByTagName('frequency')[0].firstChild.data), 86 | i.getElementsByTagName('frequencyBand')[0].firstChild.data, 87 | i.getElementsByTagName('scienceGoal')[0].firstChild.data, 88 | i.getElementsByTagName('weatherConstraint')[0].firstChild.data )) 89 | 90 | return pd.DataFrame(summaryList, columns=['sbSummaryUID', 'projectUID', 'obsUnitSetUID', 'frequency', 91 | 'frequencyBand', 'scienceGoal', 'weatherConstraint']) 92 | else: 93 | return False 94 | 95 | def getScan(uid=None): 96 | scanXML = GetXML(uid,'Scan') 97 | if scanXML is not False: 98 | scan = minidom.parseString(scanXML) 99 | scanList = list() 100 | rows = scan.getElementsByTagName('row') 101 | for i in rows: 102 | try: 103 | scanList.append((int(i.getElementsByTagName('scanNumber')[0].firstChild.data), 104 | int(i.getElementsByTagName('startTime')[0].firstChild.data), 105 | int(i.getElementsByTagName('endTime')[0].firstChild.data), 106 | #i.getElementsByTagName('numIntent')[0].firstChild.data, 107 | int(i.getElementsByTagName('numSubscan')[0].firstChild.data), 108 | arrayParser(i.getElementsByTagName('scanIntent')[0].firstChild.data, 1), 109 | arrayParser(i.getElementsByTagName('calDataType')[0].firstChild.data, 1), 110 | int(i.getElementsByTagName('numField')[0].firstChild.data), 111 | i.getElementsByTagName('fieldName')[0].firstChild.data, 112 | i.getElementsByTagName('sourceName')[0].firstChild.data)) 113 | except IndexError as e: 114 | scanList.append((int(i.getElementsByTagName('scanNumber')[0].firstChild.data), 115 | int(i.getElementsByTagName('startTime')[0].firstChild.data), 116 | int(i.getElementsByTagName('endTime')[0].firstChild.data), 117 | #i.getElementsByTagName('numIntent')[0].firstChild.data, 118 | int(i.getElementsByTagName('numSubscan')[0].firstChild.data), 119 | arrayParser(i.getElementsByTagName('scanIntent')[0].firstChild.data, 1), 120 | arrayParser(i.getElementsByTagName('calDataType')[0].firstChild.data, 1), 121 | 0, 122 | u"None", 123 | u"None")) 124 | 125 | 126 | return pd.DataFrame(scanList, columns=['scanNumber', 'startTime', 'endTime', 'numSubscan', 127 | 'scanIntent', 'calDataType', 'numField', 'fieldName', 'sourceName']) 128 | else: 129 | return False 130 | 131 | def getStation(uid=None): 132 | stationXML = GetXML(uid,'Station') 133 | if stationXML is not False: 134 | station = minidom.parseString(stationXML) 135 | stationList = list() 136 | rows = station.getElementsByTagName('row') 137 | for i in rows: 138 | try: 139 | stationList.append(( 140 | i.getElementsByTagName('stationId')[0].firstChild.data, 141 | i.getElementsByTagName('name')[0].firstChild.data, 142 | i.getElementsByTagName('position')[0].firstChild.data, 143 | i.getElementsByTagName('type')[0].firstChild.data, 144 | )) 145 | except IndexError as error: 146 | print error 147 | return False 148 | return pd.DataFrame(stationList ,columns=['stationId','name','position','type']) 149 | 150 | 151 | 152 | def getSubScan(uid=None): 153 | subscanXML = GetXML(uid,'Subscan') 154 | if subscanXML is not False: 155 | subscan = minidom.parseString(subscanXML) 156 | subscanList = list() 157 | rows = subscan.getElementsByTagName('row') 158 | for i in rows: 159 | subscanList.append((int(i.getElementsByTagName('scanNumber')[0].firstChild.data), 160 | int(i.getElementsByTagName('subscanNumber')[0].firstChild.data), 161 | int(i.getElementsByTagName('startTime')[0].firstChild.data), 162 | int(i.getElementsByTagName('endTime')[0].firstChild.data), 163 | i.getElementsByTagName('fieldName')[0].firstChild.data, 164 | i.getElementsByTagName('subscanIntent')[0].firstChild.data, 165 | i.getElementsByTagName('subscanMode')[0].firstChild.data, 166 | #i.getElementsByTagName('numIntegration')[0].firstChild.data, 167 | #i.getElementsByTagName('numSubintegration')[0].firstChild.data, 168 | #i.getElementsByTagName('correlatorCalibration')[0].firstChild.data 169 | )) 170 | return pd.DataFrame(subscanList, columns=['scanNumber','subscanNumber','startTime','endTime','fieldName', 171 | 'subscanIntent','subscanMode']) 172 | else: 173 | return False 174 | 175 | 176 | def getSource(uid=None): 177 | sourceXML = GetXML(uid,'Source') 178 | if sourceXML is not False: 179 | source = minidom.parseString(sourceXML) 180 | sourceList = list() 181 | rows = source.getElementsByTagName('row') 182 | #there are missing fields in some rows for the Source table. 183 | for i in rows: 184 | sourceList.append((int(i.getElementsByTagName('sourceId')[0].firstChild.data), 185 | i.getElementsByTagName('timeInterval')[0].firstChild.data, 186 | i.getElementsByTagName('direction')[0].firstChild.data, 187 | i.getElementsByTagName('directionCode')[0].firstChild.data, 188 | i.getElementsByTagName('sourceName')[0].firstChild.data, 189 | i.getElementsByTagName('spectralWindowId')[0].firstChild.data)) 190 | return pd.DataFrame(sourceList,columns=['sourceId','timeInterval','direction','directionCode','sourceName', 191 | 'spectralWindowId']) 192 | else: 193 | return False 194 | 195 | 196 | def getSpectralWindow(uid=None): 197 | spwXML = GetXML(uid,'SpectralWindow') 198 | if spwXML is not False: 199 | spw = minidom.parseString(spwXML) 200 | spwList = list() 201 | rows = spw.getElementsByTagName('row') 202 | for i in rows: 203 | if int(i.getElementsByTagName('numChan')[0].firstChild.data) > 4: 204 | try: 205 | spwList.append((i.getElementsByTagName('spectralWindowId')[0].firstChild.data, 206 | i.getElementsByTagName('basebandName')[0].firstChild.data, 207 | i.getElementsByTagName('netSideband')[0].firstChild.data, 208 | int(i.getElementsByTagName('numChan')[0].firstChild.data), 209 | float(i.getElementsByTagName('refFreq')[0].firstChild.data), 210 | i.getElementsByTagName('sidebandProcessingMode')[0].firstChild.data, 211 | float(i.getElementsByTagName('totBandwidth')[0].firstChild.data), 212 | i.getElementsByTagName('chanFreqStart')[0].firstChild.data, 213 | i.getElementsByTagName('chanFreqStep')[0].firstChild.data, 214 | i.getElementsByTagName('chanWidth')[0].firstChild.data, 215 | i.getElementsByTagName('effectiveBw')[0].firstChild.data, 216 | i.getElementsByTagName('name')[0].firstChild.data, 217 | #i.getElementsByTagName('resolutionArray')[0].firstChild.data, 218 | i.getElementsByTagName('assocNature')[0].firstChild.data, 219 | i.getElementsByTagName('assocSpectralWindowId')[0].firstChild.data)) 220 | except IndexError as e: 221 | print e 222 | 223 | return pd.DataFrame(spwList, columns=['spectralWindowId', 'basebandName', 'netSideband', 'numChan', 224 | 'refFreq', 'sidebandProcessingMode', 'totBandwidth', 'chanFreqStart','chanFreqStep','chanWidth', 225 | 'effectiveBw', 'name', 226 | 'assocNature', 'assocSpectralWindowId']) 227 | else: 228 | return False 229 | 230 | 231 | def getField(uid=None): 232 | fieldXML = GetXML(uid,'Field') 233 | if fieldXML is not False: 234 | field = minidom.parseString(fieldXML) 235 | fieldList = list() 236 | rows = field.getElementsByTagName('row') 237 | for i in rows: 238 | fieldList.append((i.getElementsByTagName('fieldId')[0].firstChild.data, 239 | i.getElementsByTagName('fieldName')[0].firstChild.data, 240 | i.getElementsByTagName('numPoly')[0].firstChild.data, 241 | #i.getElementsByTagName('delayDir')[0].firstChild.data, 242 | #i.getElementsByTagName('phaseDir')[0].firstChild.data, 243 | i.getElementsByTagName('referenceDir')[0].firstChild.data, 244 | int(i.getElementsByTagName('time')[0].firstChild.data), 245 | i.getElementsByTagName('code')[0].firstChild.data, 246 | i.getElementsByTagName('directionCode')[0].firstChild.data, 247 | int(i.getElementsByTagName('sourceId')[0].firstChild.data))) 248 | #return pd.DataFrame(fieldList, columns=['fieldId', 'fieldName', 'numPoly','delayDir','phaseDir','referenceDir', 'time', 'code', 'directionCode', 'sourceId']) 249 | return pd.DataFrame(fieldList, columns=['fieldId', 'fieldName', 'numPoly','referenceDir', 'time', 'code', 'directionCode', 'sourceId']) 250 | else: 251 | return False 252 | 253 | def getSysCal(uid=None): 254 | syscalXML = GetXML(uid,'SysCal') 255 | if syscalXML is not False: 256 | syscal = minidom.parseString(syscalXML) 257 | syscalList = list() 258 | rows = syscal.getElementsByTagName('row') 259 | for i in rows: 260 | syscalList.append(( 261 | i.getElementsByTagName('timeInterval')[0].firstChild.data, 262 | i.getElementsByTagName('numReceptor')[0].firstChild.data, 263 | i.getElementsByTagName('numChan')[0].firstChild.data, 264 | i.getElementsByTagName('tcalFlag')[0].firstChild.data, 265 | i.getElementsByTagName('tcalSpectrum')[0].firstChild.data, 266 | i.getElementsByTagName('trxFlag')[0].firstChild.data, 267 | i.getElementsByTagName('trxSpectrum')[0].firstChild.data, 268 | i.getElementsByTagName('tskyFlag')[0].firstChild.data, 269 | i.getElementsByTagName('tskySpectrum')[0].firstChild.data, 270 | i.getElementsByTagName('tsysFlag')[0].firstChild.data, 271 | i.getElementsByTagName('tsysSpectrum')[0].firstChild.data, 272 | i.getElementsByTagName('antennaId')[0].firstChild.data.strip(), 273 | i.getElementsByTagName('feedId')[0].firstChild.data, 274 | i.getElementsByTagName('spectralWindowId')[0].firstChild.data.strip() )) 275 | return pd.DataFrame(syscalList, columns=['timeInterval','numReceptor','numChan','tcalFlag','tcalSpectrum','trxFlag', 276 | 'trxSpectrum','tskyFlag','tskySpectrum','tsysFlag','tsysSpectrum','antennaId', 277 | 'feedId','spectralWindowId']) 278 | else: 279 | return False 280 | 281 | 282 | def getSBData(sbuid=None): 283 | schedXML = GetXML(sbuid, 'SchedBlock') 284 | sched = minidom.parseString(schedXML) 285 | 286 | schedList = list() 287 | rowsBL = sched.getElementsByTagName('sbl:BLSpectralWindow') 288 | rowsACA = sched.getElementsByTagName('sbl:ACASpectralWindow') 289 | rows = rowsBL if len(rowsBL) > len(rowsACA) else rowsACA 290 | for i in rows: 291 | brother = i.parentNode.getElementsByTagName('sbl:BaseBandSpecificationRef') 292 | parent = i.parentNode.parentNode.parentNode 293 | schedList.append(( 294 | parent.getAttribute('entityPartId'), 295 | parent.getAttribute('switchingType'), 296 | #parent.getAttribute('receiverType'), 297 | parent.getElementsByTagName('sbl:name')[0].firstChild.data, 298 | brother[0].getAttribute('entityId'), 299 | brother[0].getAttribute('partId'), 300 | #brother[0].getAttribute('entityTypeName'), 301 | #i.getAttribute('sideBand'), 302 | #i.getAttribute('windowFunction'), 303 | i.getAttribute('polnProducts'), 304 | #i.getAttribute('correlationBits'), 305 | i.getElementsByTagName('sbl:centerFrequency')[0].firstChild.data, 306 | i.getElementsByTagName('sbl:centerFrequency')[0].getAttribute('unit'), 307 | #i.getElementsByTagName('sbl:spectralAveragingFactor')[0].firstChild.data, 308 | #i.getElementsByTagName('sbl:name')[0].firstChild.data, 309 | i.getElementsByTagName('sbl:effectiveBandwidth')[0].firstChild.data, 310 | i.getElementsByTagName('sbl:effectiveBandwidth')[0].getAttribute('unit'), 311 | i.getElementsByTagName('sbl:effectiveNumberOfChannels')[0].firstChild.data, 312 | #i.getElementsByTagName('sbl:useThisSpectralWindow')[0].firstChild.data, 313 | i.getElementsByTagName('sbl:ChannelAverageRegion')[0].getElementsByTagName('sbl:startChannel')[0].firstChild.data, 314 | i.getElementsByTagName('sbl:ChannelAverageRegion')[0].getElementsByTagName('sbl:numberChannels')[0].firstChild.data, 315 | 316 | )) 317 | 318 | specs = pd.DataFrame(schedList) 319 | 320 | rows = sched.getElementsByTagName('sbl:BaseBandSpecification') 321 | bbList = list() 322 | 323 | for i in rows: 324 | parent = i.parentNode 325 | bbList.append(( 326 | parent.getAttribute('receiverBand'), 327 | parent.getAttribute('dopplerReference'), 328 | parent.getElementsByTagName('sbl:restFrequency')[0].firstChild.data, 329 | parent.getElementsByTagName('sbl:restFrequency')[0].getAttribute('unit'), 330 | parent.getElementsByTagName('sbl:frequencySwitching')[0].firstChild.data, 331 | parent.getElementsByTagName('sbl:lO2Frequency')[0].firstChild.data, 332 | parent.getElementsByTagName('sbl:lO2Frequency')[0].getAttribute('unit'), 333 | #parent.getElementsByTagName('sbl:weighting')[0].firstChild.data, 334 | #parent.getElementsByTagName('sbl:useUSB')[0].firstChild.data, 335 | #parent.getElementsByTagName('sbl:use12GHzFilter')[0].firstChild.data, 336 | #parent.getElementsByTagName('sbl:imageCenterFrequency')[0].firstChild.data, 337 | #parent.getElementsByTagName('sbl:imageCenterFrequency')[0].getAttribute('unit'), 338 | i.getAttribute('entityPartId'), 339 | i.getAttribute('baseBandName'), 340 | #i.getAttribute('sideBandPreference'), 341 | i.getElementsByTagName('sbl:centerFrequency')[0].firstChild.data, 342 | i.getElementsByTagName('sbl:lO2Frequency')[0].firstChild.data, 343 | i.getElementsByTagName('sbl:lO2Frequency')[0].getAttribute('unit'), 344 | #i.getElementsByTagName('sbl:weighting')[0].firstChild.data, 345 | #i.getElementsByTagName('sbl:useUSB')[0].firstChild.data, 346 | #i.getElementsByTagName('sbl:use12GHzFilter')[0].firstChild.data, 347 | #i.getElementsByTagName('sbl:imageCenterFrequency')[0].firstChild.data, 348 | #i.getElementsByTagName('sbl:imageCenterFrequency')[0].getAttribute('unit') 349 | )) 350 | 351 | bb = pd.DataFrame(bbList) 352 | targetList = list() 353 | rows = sched.getElementsByTagName('sbl:Target') 354 | for i in rows: 355 | targetList.append(( 356 | i.getAttribute('entityPartId'), 357 | i.getElementsByTagName('sbl:AbstractInstrumentSpecRef')[0].getAttribute('partId'), 358 | i.getElementsByTagName('sbl:FieldSourceRef')[0].getAttribute('partId'), 359 | i.getElementsByTagName('sbl:ObservingParametersRef')[0].getAttribute('partId'), 360 | )) 361 | 362 | target = pd.DataFrame(targetList, columns=['entityPartId', 'InstrumentSpec', 'FieldSource', 'ObsParameter']) 363 | 364 | rows = sched.getElementsByTagName('sbl:ScienceParameters') 365 | scienceList = list() 366 | for i in rows: 367 | scienceList.append(( 368 | i.getAttribute('entityPartId'), 369 | i.getElementsByTagName('sbl:name')[0].firstChild.data, 370 | i.getElementsByTagName('sbl:representativeBandwidth')[0].firstChild.data, 371 | i.getElementsByTagName('sbl:representativeBandwidth')[0].getAttribute('unit'), 372 | i.getElementsByTagName('sbl:representativeFrequency')[0].firstChild.data, 373 | i.getElementsByTagName('sbl:representativeFrequency')[0].getAttribute('unit'), 374 | i.getElementsByTagName('sbl:sensitivityGoal')[0].firstChild.data, 375 | i.getElementsByTagName('sbl:sensitivityGoal')[0].getAttribute('unit'), 376 | i.getElementsByTagName('sbl:integrationTime')[0].firstChild.data, 377 | i.getElementsByTagName('sbl:integrationTime')[0].getAttribute('unit'), 378 | i.getElementsByTagName('sbl:subScanDuration')[0].firstChild.data, 379 | i.getElementsByTagName('sbl:subScanDuration')[0].getAttribute('unit'), 380 | i.getElementsByTagName('sbl:forceAtmCal')[0].firstChild.data 381 | )) 382 | 383 | science = pd.DataFrame(scienceList) 384 | 385 | rows = sched.getElementsByTagName('sbl:PhaseCalParameters') 386 | phaseList = list() 387 | for i in rows: 388 | phaseList.append(( 389 | i.getAttribute('entityPartId'), 390 | #i.getElementsByTagName('sbl:name')[0].firstChild.data, 391 | i.getElementsByTagName('sbl:cycleTime')[0].firstChild.data, 392 | i.getElementsByTagName('sbl:cycleTime')[0].getAttribute('unit'), 393 | i.getElementsByTagName('sbl:defaultIntegrationTime')[0].firstChild.data, 394 | i.getElementsByTagName('sbl:defaultIntegrationTime')[0].getAttribute('unit'), 395 | i.getElementsByTagName('sbl:subScanDuration')[0].firstChild.data, 396 | i.getElementsByTagName('sbl:subScanDuration')[0].getAttribute('unit'), 397 | i.getElementsByTagName('sbl:forceAtmCal')[0].firstChild.data, 398 | i.getElementsByTagName('sbl:forceExecution')[0].firstChild.data 399 | )) 400 | 401 | phase = pd.DataFrame(phaseList) 402 | 403 | rows = sched.getElementsByTagName('sbl:FieldSource') 404 | fieldList = list() 405 | for i in rows: 406 | fieldList.append(( 407 | i.getAttribute('entityPartId'), 408 | i.getAttribute('solarSystemObject'), 409 | i.getElementsByTagName('sbl:sourceName')[0].firstChild.data, 410 | #i.getElementsByTagName('sbl:sourceEphemeris')[0].firstChild.data, 411 | i.getElementsByTagName('sbl:name')[0].firstChild.data, 412 | )) 413 | 414 | field = pd.DataFrame(fieldList) 415 | return bb,specs,target,phase,science,field 416 | 417 | def getSBFields(sbuid=None): 418 | schedXML = GetXML(sbuid, 'SchedBlock') 419 | sched = minidom.parseString(schedXML) 420 | 421 | rows = sched.getElementsByTagName('sbl:FieldSource') 422 | fieldList = list() 423 | for i in rows: 424 | fieldList.append(( 425 | i.getAttribute('entityPartId'), 426 | i.getAttribute('solarSystemObject'), 427 | i.getElementsByTagName('sbl:sourceName')[0].firstChild.data, 428 | #i.getElementsByTagName('sbl:sourceEphemeris')[0].firstChild.data, 429 | i.getElementsByTagName('sbl:name')[0].firstChild.data, 430 | i.getElementsByTagName('sbl:sourceCoordinates')[0].getElementsByTagName('val:longitude')[0].firstChild.data, 431 | i.getElementsByTagName('sbl:sourceCoordinates')[0].getElementsByTagName('val:latitude')[0].firstChild.data, 432 | 433 | )) 434 | 435 | field = pd.DataFrame(fieldList, columns=['entityPartId','solarSystemObject','sourceName','name','longitude','latitude']) 436 | return field 437 | 438 | def getSBScience(sbuid=None): 439 | schedXML = GetXML(sbuid, 'SchedBlock') 440 | sched = minidom.parseString(schedXML) 441 | 442 | rows = sched.getElementsByTagName('sbl:ScienceParameters') 443 | scienceList = list() 444 | for i in rows: 445 | scienceList.append(( 446 | i.getAttribute('entityPartId'), 447 | i.getElementsByTagName('sbl:name')[0].firstChild.data, 448 | i.getElementsByTagName('sbl:representativeBandwidth')[0].firstChild.data, 449 | i.getElementsByTagName('sbl:representativeBandwidth')[0].getAttribute('unit'), 450 | i.getElementsByTagName('sbl:representativeFrequency')[0].firstChild.data, 451 | i.getElementsByTagName('sbl:representativeFrequency')[0].getAttribute('unit'), 452 | i.getElementsByTagName('sbl:sensitivityGoal')[0].firstChild.data, 453 | i.getElementsByTagName('sbl:sensitivityGoal')[0].getAttribute('unit'), 454 | i.getElementsByTagName('sbl:integrationTime')[0].firstChild.data, 455 | i.getElementsByTagName('sbl:integrationTime')[0].getAttribute('unit'), 456 | i.getElementsByTagName('sbl:subScanDuration')[0].firstChild.data, 457 | i.getElementsByTagName('sbl:subScanDuration')[0].getAttribute('unit'), 458 | i.getElementsByTagName('sbl:forceAtmCal')[0].firstChild.data 459 | )) 460 | 461 | science = pd.DataFrame(scienceList, columns=['entityPartId','name','representativeBandwidth','unit_rb','representativeFrequency','unit_rf', 462 | 'sensitivityGoal','unit_sg','integrationTime','unit_it','subScanDuration','unit_sc','forceAtmCal']) 463 | return science 464 | 465 | def getSBTargets(sbuid=None): 466 | schedXML = GetXML(sbuid, 'SchedBlock') 467 | sched = minidom.parseString(schedXML) 468 | targetList = list() 469 | rows = sched.getElementsByTagName('sbl:Target') 470 | for i in rows: 471 | targetList.append(( 472 | i.getAttribute('entityPartId'), 473 | i.getElementsByTagName('sbl:AbstractInstrumentSpecRef')[0].getAttribute('partId'), 474 | i.getElementsByTagName('sbl:FieldSourceRef')[0].getAttribute('partId'), 475 | i.getElementsByTagName('sbl:ObservingParametersRef')[0].getAttribute('partId'), 476 | )) 477 | 478 | target = pd.DataFrame(targetList, columns=['entityPartId', 'InstrumentSpec', 'FieldSource', 'ObsParameter']) 479 | return target 480 | 481 | def getSBOffsets(sbuid=None): 482 | schedXML = GetXML(sbuid, 'SchedBlock') 483 | sched = minidom.parseString(schedXML) 484 | offsetList = list() 485 | rows = sched.getElementsByTagName('sbl:phaseCenterCoordinates') 486 | for i in rows: 487 | offsetList.append(( 488 | i.parentNode.parentNode.getAttribute('entityPartId'), 489 | i.getAttribute('system'), 490 | i.getAttribute('type'), 491 | i.getElementsByTagName('val:longitude')[0].firstChild.data, 492 | i.getElementsByTagName('val:longitude')[0].getAttribute('unit'), 493 | i.getElementsByTagName('val:latitude')[0].firstChild.data, 494 | i.getElementsByTagName('val:latitude')[0].getAttribute('unit'), 495 | )) 496 | 497 | offset = pd.DataFrame(offsetList, columns=['partId','system', 'type', 'longitude','lon_unit', 'latitude','lat_unit']) 498 | return offset 499 | 500 | def getScienceGoal(prjUID=None): 501 | projXML = GetXML(prjUID, 'ObsProject') 502 | proj = minidom.parseString(projXML) 503 | scienceGoalList = list() 504 | rows = proj.getElementsByTagName('prj:ScienceSpectralWindow') 505 | 506 | for i in rows: 507 | scienceGoalList.append(( 508 | i.parentNode.parentNode.getElementsByTagName('prj:name')[0].firstChild.data, 509 | i.parentNode.parentNode.getElementsByTagName('prj:ObsUnitSetRef')[0].getAttribute('entityId'), 510 | i.parentNode.parentNode.getElementsByTagName('prj:ObsUnitSetRef')[0].getAttribute('partId'), 511 | i.parentNode.getElementsByTagName('prj:representativeFrequency')[0].firstChild.data, 512 | i.parentNode.getElementsByTagName('prj:userRepresentativeFrequency')[0].firstChild.data, 513 | i.getElementsByTagName('prj:centerFrequency')[0].firstChild.data, 514 | i.getElementsByTagName('prj:representativeWindow')[0].firstChild.data, 515 | )) 516 | 517 | scienceGoal = pd.DataFrame(scienceGoalList) 518 | return scienceGoal 519 | 520 | 521 | 522 | def getSB_spectralconf(sbuid=None): 523 | schedXML = GetXML(sbuid, 'SchedBlock') 524 | sched = minidom.parseString(schedXML) 525 | 526 | schedList = list() 527 | rowsBL = sched.getElementsByTagName('sbl:BLSpectralWindow') 528 | rowsACA = sched.getElementsByTagName('sbl:ACASpectralWindow') 529 | rows = rowsBL if len(rowsBL) > len(rowsACA) else rowsACA 530 | for i in rows: 531 | brother = i.parentNode.getElementsByTagName('sbl:BaseBandSpecificationRef') 532 | parent = i.parentNode.parentNode.parentNode 533 | schedList.append(( 534 | parent.getAttribute('entityPartId'), 535 | parent.getAttribute('switchingType'), 536 | #parent.getAttribute('receiverType'), 537 | parent.getElementsByTagName('sbl:name')[0].firstChild.data, 538 | brother[0].getAttribute('entityId'), 539 | brother[0].getAttribute('partId'), 540 | #brother[0].getAttribute('entityTypeName'), 541 | #i.getAttribute('sideBand'), 542 | #i.getAttribute('windowFunction'), 543 | #i.getAttribute('polnProducts'), 544 | #i.getAttribute('correlationBits'), 545 | #i.getElementsByTagName('sbl:centerFrequency')[0].firstChild.data, 546 | #i.getElementsByTagName('sbl:centerFrequency')[0].getAttribute('unit'), 547 | #i.getElementsByTagName('sbl:spectralAveragingFactor')[0].firstChild.data, 548 | #i.getElementsByTagName('sbl:name')[0].firstChild.data, 549 | #i.getElementsByTagName('sbl:effectiveBandwidth')[0].firstChild.data, 550 | #i.getElementsByTagName('sbl:effectiveBandwidth')[0].getAttribute('unit'), 551 | #i.getElementsByTagName('sbl:effectiveNumberOfChannels')[0].firstChild.data, 552 | #i.getElementsByTagName('sbl:useThisSpectralWindow')[0].firstChild.data, 553 | #i.getElementsByTagName('sbl:ChannelAverageRegion')[0].getElementsByTagName('sbl:startChannel')[0].firstChild.data, 554 | #i.getElementsByTagName('sbl:ChannelAverageRegion')[0].getElementsByTagName('sbl:numberChannels')[0].firstChild.data, 555 | 556 | )) 557 | 558 | specs = pd.DataFrame(schedList) 559 | 560 | rows = sched.getElementsByTagName('sbl:BaseBandSpecification') 561 | bbList = list() 562 | 563 | for i in rows: 564 | parent = i.parentNode 565 | bbList.append(( 566 | parent.getAttribute('receiverBand'), 567 | #parent.getAttribute('dopplerReference'), 568 | #parent.getElementsByTagName('sbl:restFrequency')[0].firstChild.data, 569 | #parent.getElementsByTagName('sbl:restFrequency')[0].getAttribute('unit'), 570 | #parent.getElementsByTagName('sbl:frequencySwitching')[0].firstChild.data, 571 | #parent.getElementsByTagName('sbl:lO2Frequency')[0].firstChild.data, 572 | #parent.getElementsByTagName('sbl:lO2Frequency')[0].getAttribute('unit'), 573 | #parent.getElementsByTagName('sbl:weighting')[0].firstChild.data, 574 | #parent.getElementsByTagName('sbl:useUSB')[0].firstChild.data, 575 | #parent.getElementsByTagName('sbl:use12GHzFilter')[0].firstChild.data, 576 | #parent.getElementsByTagName('sbl:imageCenterFrequency')[0].firstChild.data, 577 | #parent.getElementsByTagName('sbl:imageCenterFrequency')[0].getAttribute('unit'), 578 | i.getAttribute('entityPartId'), 579 | i.getAttribute('baseBandName'), 580 | #i.getAttribute('sideBandPreference'), 581 | #i.getElementsByTagName('sbl:centerFrequency')[0].firstChild.data, 582 | #i.getElementsByTagName('sbl:lO2Frequency')[0].firstChild.data, 583 | #i.getElementsByTagName('sbl:lO2Frequency')[0].getAttribute('unit'), 584 | #i.getElementsByTagName('sbl:weighting')[0].firstChild.data, 585 | #i.getElementsByTagName('sbl:useUSB')[0].firstChild.data, 586 | #i.getElementsByTagName('sbl:use12GHzFilter')[0].firstChild.data, 587 | #i.getElementsByTagName('sbl:imageCenterFrequency')[0].firstChild.data, 588 | #i.getElementsByTagName('sbl:imageCenterFrequency')[0].getAttribute('unit') 589 | )) 590 | 591 | bb = pd.DataFrame(bbList) 592 | 593 | return bb,specs 594 | 595 | --------------------------------------------------------------------------------