├── docs ├── .nojekyll ├── source │ ├── index.rst │ ├── datamodels.rst │ ├── getting_started.rst │ ├── sms.rst │ └── conf.py └── Makefile ├── .coveragerc ├── tests ├── data │ ├── test_crds_cache │ │ ├── config │ │ │ └── hst │ │ │ │ └── ref_cache_subdir_mode │ │ └── references │ │ │ └── hst │ │ │ ├── 03p1706jl_wcp.fits.gz │ │ │ ├── 23n1744jl_lamp.fits.gz │ │ │ ├── 24914347l_lamp.fits.gz │ │ │ ├── u1t1616ol_lamp.fits.gz │ │ │ ├── u1t1616ql_wcp.fits.gz │ │ │ └── xaf1429el_wcp.fits.gz │ ├── lb4c10niq_spt.fits.gz │ ├── lb4l02m1q_spt.fits.gz │ ├── lb4l02m2q_spt.fits.gz │ ├── lbhx1epiq_spt.fits.gz │ ├── ld3la1csq_spt.fits.gz │ ├── ldi404zsq_spt.fits.gz │ ├── ldi405e3q_spt.fits.gz │ ├── ldng01chq_spt.fits.gz │ ├── ldng05huq_spt.fits.gz │ ├── ldngz2szj_jit.fits.gz │ ├── ldngz2szq_spt.fits.gz │ ├── ldxe02010_jit.fits.gz │ ├── ldxe02ssj_jit.fits.gz │ ├── lb4l02m1q_rawacq.fits.gz │ ├── lb4l02m2q_rawacq.fits.gz │ ├── lbhx1epiq_rawacq.fits.gz │ ├── ld3la1csq_rawacq.fits.gz │ ├── ldi404zsq_rawacq.fits.gz │ ├── ldi405e3q_rawacq.fits.gz │ ├── ldng01chq_rawacq.fits.gz │ ├── ldng05huq_rawacq.fits.gz │ ├── ldngz2szq_rawacq.fits.gz │ ├── lb4c10niq_lampflash.fits.gz │ ├── lbhx26fmq_lampflash.fits.gz │ ├── lbhx97ijq_lampflash.fits.gz │ ├── lcwj02qvq_lampflash.fits.gz │ ├── lcwj05egq_lampflash.fits.gz │ ├── lcwj08e3q_lampflash.fits.gz │ ├── ld1ce1rcq_lampflash.fits.gz │ ├── ld1ce2j1q_lampflash.fits.gz │ ├── ld1ce3yuq_lampflash.fits.gz │ ├── ld1ce4dkq_lampflash.fits.gz │ ├── lddm01tlq_lampflash.fits.gz │ ├── bad_111078a6.txt │ ├── 122687b1.txt │ ├── 180147b1.txt │ ├── 162567b1.txt │ ├── 153067a8.txt │ ├── 181137b3.txt │ ├── 181137b4.txt │ ├── 181137c2.txt │ └── 161368a8.txt ├── cosmoconfig_test.yaml ├── test_acq_monitors.py ├── conftest.py ├── test_osm_monitors.py ├── test_data_models.py ├── test_monitor_helpers.py ├── test_sms_ingest.py └── test_filesystem.py ├── cosmo ├── sms │ ├── __init__.py │ ├── sms_db.py │ └── ingest_sms.py ├── telemetry_support │ ├── COSMnemonics.xls │ ├── telem_mon_env_variables.sh │ ├── default_telemetry_zooms.csv │ └── telemetry_monitors_text.json ├── dark_monitor_list.dat ├── pytest.ini ├── monitors │ ├── __init__.py │ ├── jitter_monitors.py │ └── data_models.py ├── __init__.py ├── run_monitors.py └── monitor_helpers.py ├── readthedocs.yml ├── setup.py ├── .gitignore ├── README.md ├── LICENSE └── .travis.yml /docs/.nojekyll: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = *blub* -------------------------------------------------------------------------------- /tests/data/test_crds_cache/config/hst/ref_cache_subdir_mode: -------------------------------------------------------------------------------- 1 | flat -------------------------------------------------------------------------------- /cosmo/sms/__init__.py: -------------------------------------------------------------------------------- 1 | from .sms_db import SMSFileStats, SMSTable, DB 2 | from .ingest_sms import SMSFinder, SMSFile 3 | -------------------------------------------------------------------------------- /tests/data/lb4c10niq_spt.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/lb4c10niq_spt.fits.gz -------------------------------------------------------------------------------- /tests/data/lb4l02m1q_spt.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/lb4l02m1q_spt.fits.gz -------------------------------------------------------------------------------- /tests/data/lb4l02m2q_spt.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/lb4l02m2q_spt.fits.gz -------------------------------------------------------------------------------- /tests/data/lbhx1epiq_spt.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/lbhx1epiq_spt.fits.gz -------------------------------------------------------------------------------- /tests/data/ld3la1csq_spt.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/ld3la1csq_spt.fits.gz -------------------------------------------------------------------------------- /tests/data/ldi404zsq_spt.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/ldi404zsq_spt.fits.gz -------------------------------------------------------------------------------- /tests/data/ldi405e3q_spt.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/ldi405e3q_spt.fits.gz -------------------------------------------------------------------------------- /tests/data/ldng01chq_spt.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/ldng01chq_spt.fits.gz -------------------------------------------------------------------------------- /tests/data/ldng05huq_spt.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/ldng05huq_spt.fits.gz -------------------------------------------------------------------------------- /tests/data/ldngz2szj_jit.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/ldngz2szj_jit.fits.gz -------------------------------------------------------------------------------- /tests/data/ldngz2szq_spt.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/ldngz2szq_spt.fits.gz -------------------------------------------------------------------------------- /tests/data/ldxe02010_jit.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/ldxe02010_jit.fits.gz -------------------------------------------------------------------------------- /tests/data/ldxe02ssj_jit.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/ldxe02ssj_jit.fits.gz -------------------------------------------------------------------------------- /tests/data/lb4l02m1q_rawacq.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/lb4l02m1q_rawacq.fits.gz -------------------------------------------------------------------------------- /tests/data/lb4l02m2q_rawacq.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/lb4l02m2q_rawacq.fits.gz -------------------------------------------------------------------------------- /tests/data/lbhx1epiq_rawacq.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/lbhx1epiq_rawacq.fits.gz -------------------------------------------------------------------------------- /tests/data/ld3la1csq_rawacq.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/ld3la1csq_rawacq.fits.gz -------------------------------------------------------------------------------- /tests/data/ldi404zsq_rawacq.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/ldi404zsq_rawacq.fits.gz -------------------------------------------------------------------------------- /tests/data/ldi405e3q_rawacq.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/ldi405e3q_rawacq.fits.gz -------------------------------------------------------------------------------- /tests/data/ldng01chq_rawacq.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/ldng01chq_rawacq.fits.gz -------------------------------------------------------------------------------- /tests/data/ldng05huq_rawacq.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/ldng05huq_rawacq.fits.gz -------------------------------------------------------------------------------- /tests/data/ldngz2szq_rawacq.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/ldngz2szq_rawacq.fits.gz -------------------------------------------------------------------------------- /cosmo/telemetry_support/COSMnemonics.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/cosmo/telemetry_support/COSMnemonics.xls -------------------------------------------------------------------------------- /tests/data/lb4c10niq_lampflash.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/lb4c10niq_lampflash.fits.gz -------------------------------------------------------------------------------- /tests/data/lbhx26fmq_lampflash.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/lbhx26fmq_lampflash.fits.gz -------------------------------------------------------------------------------- /tests/data/lbhx97ijq_lampflash.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/lbhx97ijq_lampflash.fits.gz -------------------------------------------------------------------------------- /tests/data/lcwj02qvq_lampflash.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/lcwj02qvq_lampflash.fits.gz -------------------------------------------------------------------------------- /tests/data/lcwj05egq_lampflash.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/lcwj05egq_lampflash.fits.gz -------------------------------------------------------------------------------- /tests/data/lcwj08e3q_lampflash.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/lcwj08e3q_lampflash.fits.gz -------------------------------------------------------------------------------- /tests/data/ld1ce1rcq_lampflash.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/ld1ce1rcq_lampflash.fits.gz -------------------------------------------------------------------------------- /tests/data/ld1ce2j1q_lampflash.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/ld1ce2j1q_lampflash.fits.gz -------------------------------------------------------------------------------- /tests/data/ld1ce3yuq_lampflash.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/ld1ce3yuq_lampflash.fits.gz -------------------------------------------------------------------------------- /tests/data/ld1ce4dkq_lampflash.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/ld1ce4dkq_lampflash.fits.gz -------------------------------------------------------------------------------- /tests/data/lddm01tlq_lampflash.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/lddm01tlq_lampflash.fits.gz -------------------------------------------------------------------------------- /readthedocs.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sphinx: 4 | configuration: docs/conf.py 5 | 6 | conda: 7 | environment: docs/environment.yml 8 | -------------------------------------------------------------------------------- /tests/data/test_crds_cache/references/hst/03p1706jl_wcp.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/test_crds_cache/references/hst/03p1706jl_wcp.fits.gz -------------------------------------------------------------------------------- /tests/data/test_crds_cache/references/hst/23n1744jl_lamp.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/test_crds_cache/references/hst/23n1744jl_lamp.fits.gz -------------------------------------------------------------------------------- /tests/data/test_crds_cache/references/hst/24914347l_lamp.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/test_crds_cache/references/hst/24914347l_lamp.fits.gz -------------------------------------------------------------------------------- /tests/data/test_crds_cache/references/hst/u1t1616ol_lamp.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/test_crds_cache/references/hst/u1t1616ol_lamp.fits.gz -------------------------------------------------------------------------------- /tests/data/test_crds_cache/references/hst/u1t1616ql_wcp.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/test_crds_cache/references/hst/u1t1616ql_wcp.fits.gz -------------------------------------------------------------------------------- /tests/data/test_crds_cache/references/hst/xaf1429el_wcp.fits.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/cosmo/master/tests/data/test_crds_cache/references/hst/xaf1429el_wcp.fits.gz -------------------------------------------------------------------------------- /cosmo/dark_monitor_list.dat: -------------------------------------------------------------------------------- 1 | FUV NUV 2 | 16322 16327 3 | 15771 15776 4 | 15533 15538 5 | 14940 14942 6 | 14520 14521 7 | 14436 14442 8 | 13968 13974 9 | 13521 13528 10 | 13121 13126 11 | 12716 12720 12 | 12423 12420 13 | 11895 11894 14 | -------------------------------------------------------------------------------- /cosmo/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | python_files = run_monitors.py 3 | python_classes = Run* 4 | python_functions = run_* 5 | addopts = -rA -v 6 | markers = 7 | monthly: marks monitors that should be run monthly (deselect with '-m "not monthly") 8 | ingest: marks a runner that ingests new data only (deselect with '-m "not ingest") 9 | -------------------------------------------------------------------------------- /tests/cosmoconfig_test.yaml: -------------------------------------------------------------------------------- 1 | # Monitor data database 2 | data: 3 | db_settings: 4 | database: 'data_test.db' 5 | pragmas: 6 | journal_mode: 'wal' 7 | foreign_keys: 1 8 | ignore_check_constraints: 0 9 | synchronous: 0 10 | 11 | # Monitor status and results database 12 | results: 13 | db_settings: 14 | database: 'results_test.db' 15 | pragmas: 16 | journal_mode: 'wal' 17 | foreign_keys: 1 18 | ignore_check_constraints: 0 19 | synchronous: 0 -------------------------------------------------------------------------------- /cosmo/monitors/__init__.py: -------------------------------------------------------------------------------- 1 | from .acq_monitors import AcqImageMonitor, AcqImageV2V3Monitor, AcqPeakdMonitor, AcqPeakxdMonitor 2 | from .osm_shift_monitors import FuvOsmShift1Monitor, FuvOsmShift2Monitor, NuvOsmShift1Monitor, NuvOsmShift2Monitor 3 | from .osm_drift_monitors import FUVOSMDriftMonitor, NUVOSMDriftMonitor 4 | 5 | __all__ = [ 6 | 'AcqImageMonitor', 7 | 'AcqImageV2V3Monitor', 8 | 'AcqPeakdMonitor', 9 | 'AcqPeakxdMonitor', 10 | 'FuvOsmShift1Monitor', 11 | 'FuvOsmShift2Monitor', 12 | 'NuvOsmShift1Monitor', 13 | 'NuvOsmShift2Monitor', 14 | 'FUVOSMDriftMonitor', 15 | 'NUVOSMDriftMonitor' 16 | ] 17 | -------------------------------------------------------------------------------- /cosmo/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | SETTINGS = { 4 | 'filesystem': {'source': os.environ['COSMO_FILES_SOURCE']}, 5 | 'output': os.environ['COSMO_OUTPUT'], 6 | 'dark_programs': os.environ['DARK_PROGRAMS'], 7 | 'sms': { 8 | 'source': os.environ['COSMO_SMS_SOURCE'], 9 | 'db_settings': { 10 | 'database': os.environ.get('COSMO_SMS_DB', 'sms.db'), 11 | 'pragmas': { 12 | 'journal_mode': os.environ.get('COSMO_SMS_DB_JOURNAL', 'wal'), 13 | 'foreign_keys': os.environ.get('COSMO_SMS_DB_FORIEGN_KEYS', 1), 14 | 'ignore_check_constraints': os.environ.get('COSMO_SMS_DB_IGNORE_CHECK_CONSTRAINTS', 0), 15 | 'synchronous': os.environ.get('COSMO_SMS_DB_SYNCHRONOUS', 0) 16 | } 17 | } 18 | }, 19 | } 20 | -------------------------------------------------------------------------------- /cosmo/telemetry_support/telem_mon_env_variables.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/bash 2 | # Where do the telemetry data files live? (unlikely to change this) 3 | export TELEMETRY_DATADIR="/grp/hst/cos/Telemetry/" 4 | 5 | # Where do you want the output plots to live? (likely to change this) 6 | export TELEMETRY_PLOTSDIR="/grp/hst/cos2/monitoring/telemetry/latest" 7 | 8 | SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 9 | # Where does the mnemonics excel file live? (likely to change this) 10 | export TELEMETRY_MNEMONICS="$SCRIPT_DIR/COSMnemonics.xls" 11 | 12 | # Path to the (JSON) file where you specify the numerical value of each telemetry variable whose value is a text string (i.e. the OSM positions) 13 | export TELEMETRY_TEXTS="$SCRIPT_DIR/telemetry_monitors_text.json" 14 | 15 | # Where does the zoom-specifying csv file live? (likely to change this) 16 | export TELEMETRY_ZOOMS="$SCRIPT_DIR/default_telemetry_zooms.csv" -------------------------------------------------------------------------------- /tests/data/bad_111078a6.txt: -------------------------------------------------------------------------------- 1 | 2 | COS Exposure Report: SMS 111078A6 3 | 4 | Data Exposure Start FUV Mechanism Positions Cent Tsince Tsince Lamp STIMs Flash 5 | Filename Prop Target PRG OB AL EX Conf Opmode ExpTime yyyy.ddd:hh:mm:ss State Aper OSM1 OSM2 Wave FP OSM1 OSM2 Lamp Curr A B Dop # Dur T 6 | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- 7 | lbmum1c5 12420 DARK BMU M1 01 01 NUV TIME-TAG 1330.0 2011.108:00:00:09 PSA G130M G185M 0 0 43209 43209 NO 8 | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- 9 | lbl205gi12315V-TW-HYABL2050101NUVACQ/PEAKXD16.02011.108:09:14:46PSANCM1G285M26760244141NO -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | setup( 4 | name='cosmo', 5 | version='1.1.1', 6 | description='Monitors for HST/COS', 7 | keywords=['astronomy'], 8 | classifiers=[ 9 | 'Programming Language :: Python :: 3', 10 | 'License :: BSD-3 :: Association of Universities for Research in Astronomy', 11 | 'Operating System :: Linux' 12 | ], 13 | python_requires='~=3.7', # 3.7 and higher, but not 4 14 | packages=find_packages(), 15 | install_requires=[ 16 | 'setuptools', 17 | 'numpy>=1.11.1', 18 | 'astropy>=1.0.1', 19 | 'plotly>=4.9.0', 20 | 'dask', 21 | 'pandas>=0.25.0', 22 | 'pytest', 23 | 'pyyaml', 24 | 'peewee>=3.13.3', 25 | 'crds', 26 | 'sunpy[all]', 27 | 'monitorframe @ git+https://github.com/spacetelescope/monitor-framework@v1.2.0#egg=monitorframe' 28 | ], 29 | package_data={'cosmo': ['pytest.ini']}, 30 | entry_points={ 31 | 'console_scripts': 32 | ['cosmo=cosmo.run_monitors:runner'] 33 | } 34 | ) 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Misc 2 | *pkl 3 | 4 | #-- package config files 5 | configure.yaml 6 | 7 | 8 | # Byte-compiled / optimized / DLL files 9 | __pycache__/ 10 | *.py[cod] 11 | 12 | # C extensions 13 | *.so 14 | 15 | # Distribution / packaging 16 | .Python 17 | env/ 18 | build/ 19 | develop-eggs/ 20 | dist/ 21 | downloads/ 22 | eggs/ 23 | .eggs/ 24 | lib/ 25 | lib64/ 26 | parts/ 27 | sdist/ 28 | var/ 29 | *.egg-info/ 30 | .installed.cfg 31 | *.egg 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .tox/ 46 | .coverage 47 | .coverage.* 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | *,cover 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Sphinx documentation 58 | docs/_build/ 59 | 60 | # PyBuilder 61 | target/ 62 | 63 | # Pycharm 64 | *idea 65 | 66 | # pytest 67 | *pytest_cache 68 | 69 | # Config 70 | *config.yaml 71 | 72 | # Finder 73 | .DS_Store 74 | -------------------------------------------------------------------------------- /cosmo/telemetry_support/default_telemetry_zooms.csv: -------------------------------------------------------------------------------- 1 | Mnemonic,min_y,max_y,min_x,max_x 2 | LAPXSTP,-230,230,, 3 | LD2LMP1T,4,15,, 4 | LD2LMP2T,1,14,, 5 | LDCAMPAT,20,27,, 6 | LDCAMPBT,19,27,, 7 | LDCDVAAT,21,26,, 8 | LDCDVABT,21,26,, 9 | LDCEAUXI,31,46,, 10 | LDCEDECA,-10,7000,, 11 | LDCEDECB,-10,7000,, 12 | LDCEFECA,-10,7000,, 13 | LDCEFECB,-10,7000,, 14 | LDCEQANA,-1,44,, 15 | LDCEQANB,39,44,, 16 | LDCESDC1,-10,15000,, 17 | LDCESDC2,-10,14000,, 18 | LDCETEMP,-1,30,, 19 | LDCHVFMT,22,26.5,, 20 | LDCHVMNA,-5350,-2850,, 21 | LDCHVMNB,-5350,-2850,, 22 | LDCHVNMA,-5300,-5000,, 23 | LDCHVNMB,-5350,-5000,, 24 | LDCIMONA,20,40,, 25 | LDCIMONB,20,32,, 26 | LDCLVPCT,22,35,, 27 | LDCTDCAT,22,32,, 28 | LDCTDCBT,22,32,, 29 | LDIBT,20,32,, 30 | LDVABPT,7,25,, 31 | LMCHAMPT,20,30,, 32 | LMCPCMDD,-2100,-1450,, 33 | LMEVENTS,-10,1000,, 34 | LMHIBIAS,140.17,140.23,, 35 | LMMCETMP,15,27,, 36 | LMMCPI,35,49,, 37 | LMMCPV,-2060,-1745,, 38 | LMPCVOLT,-805,21,, 39 | LNELBOXT,5,28,, 40 | LNTUBET,10,24,, 41 | LOM1STP,-13100,11500,, 42 | LOM2STP,-13100,12000,, 43 | LOMFSTP,-2300,1200,, 44 | LPNLMP1T,2.5,13.5,, 45 | LPNLMP2T,3.5,12.5,, 46 | METPSPR,5.20E-08,1.25E-07,, -------------------------------------------------------------------------------- /cosmo/sms/sms_db.py: -------------------------------------------------------------------------------- 1 | from peewee import Model, TextField, IntegerField, FloatField, DateTimeField, ForeignKeyField 2 | from playhouse.sqlite_ext import SqliteExtDatabase 3 | 4 | from .. import SETTINGS 5 | 6 | DB = SqliteExtDatabase(**SETTINGS['sms']['db_settings']) 7 | 8 | 9 | class BaseModel(Model): 10 | 11 | class Meta: 12 | database = DB 13 | 14 | 15 | class SMSFileStats(BaseModel): 16 | SMSID = TextField(primary_key=True) 17 | VERSION = TextField() 18 | FILEID = TextField(unique=True) 19 | FILENAME = TextField() 20 | INGEST_DATE = DateTimeField() 21 | 22 | 23 | class SMSTable(BaseModel): 24 | 25 | EXPOSURE = TextField(primary_key=True) 26 | FILEID = ForeignKeyField(SMSFileStats, field='FILEID', backref='exposures', on_delete='cascade', on_update='cascade') 27 | ROOTNAME = TextField() 28 | PROPOSID = IntegerField(verbose_name='proposal id') 29 | DETECTOR = TextField() 30 | OPMODE = TextField() 31 | EXPTIME = FloatField() 32 | EXPSTART = DateTimeField() 33 | FUVHVSTATE = TextField(verbose_name='fuv hv state') 34 | APERTURE = TextField() 35 | OSM1POS = TextField(verbose_name='OSM1 position') 36 | OSM2POS = TextField(verbose_name='OSM2 position') 37 | CENWAVE = IntegerField() 38 | FPPOS = IntegerField() 39 | TSINCEOSM1 = FloatField(verbose_name='time since OSM1 move') 40 | TSINCEOSM2 = FloatField(verbose_name='time since OSM2 move') 41 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | COSMO (COS Monitoring) 2 | ============================== 3 | Welcome! This is the documentation for COS Monitoring (COSMO). 4 | This document aims to give a high level overview of COSMO, including it's monitors, the back end, and how it all ties 5 | together. 6 | 7 | What is COSMO? 8 | -------------- 9 | COSMO is a software package of monitors and interfaces for their data for the Cosmic Origins Spectrograph (COS) at Space 10 | Telescope Science Institute (STScI). 11 | These monitors are important for ensuring that COS is operating nominally as well as extending its mission lifetime. 12 | 13 | COSMO is built on `monitorframe `_, a light-weight framework for 14 | creating instrument monitors (for more information about ``monitorframe``, check out the 15 | `documentation `_). 16 | 17 | The remote repository for COSMO is hosted on `GitHub `_ and utilizes 18 | `Travis CI `_ for its automated testing needs. 19 | COSMO also tracks testing coverage with `Codecov `_. 20 | 21 | COSMO 22 | ----- 23 | .. toctree:: 24 | :maxdepth: 2 25 | 26 | getting_started 27 | monitors 28 | datamodels 29 | sms 30 | api 31 | 32 | 33 | Indices and tables 34 | ================== 35 | * :ref:`genindex` -------------------------------------------------------------------------------- /tests/test_acq_monitors.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest 3 | 4 | from cosmo.monitors.acq_monitors import AcqPeakdMonitor, AcqImageMonitor, AcqPeakxdMonitor 5 | from cosmo.monitors.data_models import AcqDataModel 6 | 7 | 8 | @pytest.fixture(params=[False, True]) 9 | def set_acqmonitor(request, data_dir, here): 10 | if request.param: 11 | model = AcqDataModel() 12 | model.ingest() 13 | 14 | def _set_monitor(monitor): 15 | AcqDataModel.files_source = data_dir 16 | AcqDataModel.subdir_pattern = None 17 | 18 | monitor.data_model = AcqDataModel 19 | monitor.output = here 20 | 21 | active = monitor() 22 | 23 | return active 24 | 25 | return _set_monitor 26 | 27 | 28 | class TestAcqMonitors: 29 | 30 | @pytest.fixture(autouse=True, params=[AcqImageMonitor, AcqPeakdMonitor, AcqPeakxdMonitor]) 31 | def acqmonitor(self, request, set_acqmonitor): 32 | acqmonitor = set_acqmonitor(request.param) 33 | 34 | request.cls.acqmonitor = acqmonitor 35 | 36 | yield 37 | 38 | if request.cls.acqmonitor.model.model is not None: 39 | request.cls.acqmonitor.model.model.drop_table(safe=True) 40 | 41 | def test_monitor_steps(self): 42 | self.acqmonitor.initialize_data() 43 | self.acqmonitor.run_analysis() 44 | self.acqmonitor.plot() 45 | self.acqmonitor.write_figure() 46 | self.acqmonitor.store_results() 47 | 48 | assert os.path.exists(self.acqmonitor.output) 49 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # COSMO 2 | [![Build Status](https://travis-ci.org/spacetelescope/cosmo.svg?branch=master)](https://travis-ci.org/spacetelescope/cosmo) 3 | [![codecov](https://codecov.io/gh/spacetelescope/cosmo/branch/master/graph/badge.svg)](https://codecov.io/gh/spacetelescope/cosmo) 4 | 5 | COSMO (COS MOnitoring) is a software system that is built specifically for the Cosmic Origins Spectrograph (COS) 6 | team at Space Telescope Science Institute (STScI). 7 | 8 | COSMO is made up of a series of monitors and their data built on the light-weight 9 | [monitorframe framework](https://github.com/spacetelescope/monitor-framework). 10 | These monitors are important for ensuring that COS is operating nominally as well as extending its mission lifetime. 11 | 12 | Check out the COSMO documenation [here](https://spacetelescope.github.io/cosmo/) 13 | 14 | ## Contributing 15 | Please open a new [issue](https://github.com/spacetelescope/cosmo/issues) or new 16 | [pull request](https://github.com/spacetelescope/cosmo/pulls) for 17 | bugs, feedback, or new features you would like to see. 18 | 19 | To contribute code, please use Feature Branching: 20 | 21 | 1. Clone this repository to your personal space 22 | 2. Create a feature branch on your clone for your changes 23 | 3. Make changes and commit into your branch 24 | 4. Issue a pull request to get your changes into the main repo 25 | 26 | For more details on this workflow, check out 27 | [this tutorial](https://www.atlassian.com/git/tutorials/comparing-workflows/feature-branch-workflow). 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2019, Association of Universities for Research in Astronomy. 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest 3 | 4 | from glob import glob 5 | 6 | HERE = os.path.dirname(os.path.abspath(__file__)) 7 | 8 | TEST_CONFIG = os.path.join(HERE, 'cosmoconfig_test.yaml') 9 | 10 | # Check to make sure that the test config file is being used. If not, don't run the tests 11 | if os.environ['MONITOR_CONFIG'] != TEST_CONFIG: 12 | raise TypeError('Tests should only be executed with the testing configuration file') 13 | 14 | if os.environ['COSMO_SMS_DB'] != os.path.join(HERE, 'test.db'): 15 | raise TypeError('Test should only be executed with a test database') 16 | 17 | 18 | @pytest.fixture(scope='session', autouse=True) 19 | def db_cleanup(): 20 | yield # The tests don't actually need this test "value" 21 | 22 | # Cleanup 23 | if os.path.exists('test.db'): 24 | os.remove('test.db') # Delete test database file after the completion of all tests 25 | 26 | # Remove temporary shared memory file if it exists 27 | if os.path.exists('test.db-shm'): 28 | os.remove('test.db-shm') 29 | 30 | # Remove temporary write-ahead log file if it exists 31 | if os.path.exists('test.db-wal'): 32 | os.remove('test.db-wal') 33 | 34 | if os.path.exists('data_test.db'): 35 | os.remove('data_test.db') 36 | 37 | if os.path.exists('data_test.db-shm'): 38 | os.remove('data_test.db-shm') 39 | 40 | if os.path.exists('data_test.db-wal'): 41 | os.remove('data_test.db-wal') 42 | 43 | 44 | @pytest.fixture(scope='session') 45 | def data_dir(): 46 | test_data = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/') 47 | os.environ['CRDS_PATH'] = os.path.join(test_data, 'test_crds_cache') 48 | 49 | return test_data 50 | 51 | 52 | @pytest.fixture(scope='session') 53 | def here(): 54 | return HERE 55 | 56 | 57 | @pytest.fixture(scope='session', autouse=True) 58 | def clean_up_output(here): 59 | yield 60 | 61 | output = glob(os.path.join(here, '*html')) + glob(os.path.join(here, '*csv')) 62 | 63 | if output: 64 | for file in output: 65 | os.remove(file) 66 | -------------------------------------------------------------------------------- /cosmo/telemetry_support/telemetry_monitors_text.json: -------------------------------------------------------------------------------- 1 | {"Name":{"0":"LOSM1POS","1":"LOSM2POS","2":"LAPDSTAT","3":"LSHPOS","4":"LOSMOFST","5":"LAPMPOS","6":"LDCSTATE","7":"LAPXSTAT","8":"LMEVTYPE"},"Data":{"0":{"Name":{"0":"Unknown","1":"--","2":"AbMvFail","3":"RelMvReq","4":"G140L","5":"G130M","6":"G160M","7":"NCM1","8":"NCM1FLAT"},"Data":{"0":-3,"1":-2,"2":-1,"3":0,"4":1,"5":2,"6":3,"7":4,"8":5},"Color":{"0":"darkgray","1":"gray","2":"red","3":"black","4":"chocolate","5":"darkblue","6":"crimson","7":"gold","8":"darkgoldenrod"}},"1":{"Name":{"0":"Unknown","1":"--","2":"AbMvFail","3":"RelMvReq","4":"G230L","5":"G185M","6":"G225M","7":"G285M","8":"TA1Image","9":"TA1Brght"},"Data":{"0":-3,"1":-2,"2":-1,"3":0,"4":1,"5":2,"6":3,"7":4,"8":5,"9":6},"Color":{"0":"darkgray","1":"gray","2":"red","3":"black","4":"chocolate","5":"darkblue","6":"darkgreen","7":"crimson","8":"gold","9":"darkgoldenrod"}},"2":{"Name":{"0":"NotInit","1":"MvTimout","2":"InitSrt","3":"Initlzd","4":"PosValid","5":"InitFail","6":"MoveStrt","7":"MvInPrgs","8":"MvBlockd","9":"Invalid","10":"OutOfTol","11":"HWSUFail","12":"MvComplt"},"Data":{"0":0,"1":1,"2":2,"3":3,"4":4,"5":5,"6":6,"7":7,"8":8,"9":9,"10":10,"11":11,"12":12}},"3":{"Name":{"0":"--","1":"Unknown","2":"Closed","3":"Open"},"Data":{"0":0,"1":1,"2":2,"3":3}},"4":{"Name":{"0":"Invalid","1":"0","2":"-2","3":"-1","4":"1"},"Data":{"0":0,"1":1,"2":2,"3":3,"4":4}},"5":{"Name":{"0":"PSA-FUV","1":"--","2":"Unknown","3":"BOA-FUV","4":"RelMvReq","5":"PSA-NUV","6":"FCA-NUV","7":"WCA-NUV","8":"WCA-FUV","9":"FCA-FUV","10":"BOA-NUV","11":"PSA-FUVB","12":"WCA-FUVB","13":"PSA-NUVB","14":"WCA-NUVB","15":"BOA-NUVB","16":"BOA-FUVB","17":"WCA-NUVO","18":"PSA-NUVO","19":"BOA-NUVO","20":"PSA-FUVA","21":"WCA-FUVO","22":"PSA-FUVO","23":"FCA-NUVO","24":"WCA-FUVA","25":"BOA-FUVA","26":"FCA-NUVB","27":"WCA-NUVA","28":"BOA-NUVA","29":"PSA-NUVA","30":"FCA-FUVB"},"Data":{"0":0,"1":1,"2":2,"3":3,"4":4,"5":5,"6":6,"7":7,"8":8,"9":9,"10":10,"11":11,"12":12,"13":13,"14":14,"15":15,"16":16,"17":17,"18":18,"19":19,"20":20,"21":21,"22":22,"23":23,"24":24,"25":25,"26":26,"27":27,"28":28,"29":29,"30":30}},"6":{"Name":{"0":"Low","1":"Off","2":"On","3":"NomA","4":"NomAB","5":"NomB","6":"Set","7":"Invalid"},"Data":{"0":0,"1":1,"2":2,"3":3,"4":4,"5":5,"6":6,"7":7}},"7":{"Name":{"0":"NotInit","1":"MvInPrgs","2":"MvBlockd","3":"Invalid","4":"Initlzd","5":"PosValid","6":"InitFail","7":"MoveStrt","8":"HWSUFail"},"Data":{"0":0,"1":1,"2":2,"3":3,"4":4,"5":5,"6":6,"7":7,"8":8}},"8":{"Name":{"0":"VECounts","1":"Invalid","2":"ORCounts","3":"WCounts","4":"XCounts","5":"YCounts","6":"ZCounts","7":"EVCounts"},"Data":{"0":0,"1":1,"2":2,"3":3,"4":4,"5":5,"6":6,"7":7}}}} -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: xenial 2 | language: python 3 | python: "3.7" 4 | 5 | compiler: gcc 6 | 7 | # Setting sudo to false opts in to Travis-CI container-based builds. 8 | sudo: false 9 | 10 | # The apt packages below are needed for sphinx builds, which can no longer 11 | # be installed with sudo apt-get. 12 | addons: 13 | apt: 14 | packages: 15 | - graphviz 16 | - texlive-latex-extra 17 | - dvipng 18 | 19 | os: linux 20 | 21 | env: 22 | global: 23 | # SET DEFAULTS TO AVOID REPEATING IN MOST CASES 24 | # For now dark monitor path is set as blank 25 | - CONDA_ARGS='--quiet' 26 | - CONDA_INSTALL="conda install $CONDA_ARGS" 27 | - CONDA_DEPS='stsci coverage' 28 | - CONDA_DOC_DEPS="$CONDA_DEPS sphinx" 29 | - SETUP_CMD='test' 30 | - MONITOR_CONFIG='/home/travis/build/spacetelescope/cosmo/tests/cosmoconfig_test.yaml' 31 | - DARK_PROGRAMS='' 32 | - COSMO_FILES_SOURCE='/home/travis/build/spacetelescope/cosmo/tests/data' 33 | - COSMO_SMS_SOURCE='/home/travis/build/spacetelescope/cosmo/tests/data' 34 | - COSMO_OUTPUT='/home/travis/build/spacetelescope/cosmo/tests' 35 | - COSMO_SMS_DB='/home/travis/build/spacetelescope/cosmo/tests/test.db' 36 | - CRDS_SERVER_URL='https://hst-crds.stsci.edu' 37 | - CRDS_PATH='/home/travis/build/spacetelescope/cosmo/tests/data/test_crds_cache' 38 | 39 | install: 40 | # USE UTF8 ENCODING. SHOULD BE DEFAULT, BUT THIS IS INSURANCE AGAINST FUTURE CHANGES 41 | - export PYTHONIOENCODING=UTF8 42 | 43 | # Install conda 44 | 45 | # http://conda.pydata.org/docs/travis.html#the-travis-yml-file 46 | - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; 47 | - bash miniconda.sh -b -p $HOME/miniconda 48 | - export PATH="$HOME/miniconda/bin:$PATH" 49 | - hash -r 50 | - conda config --set always_yes yes --set changeps1 no 51 | - conda config --add channels http://ssb.stsci.edu/astroconda 52 | - conda install -c anaconda setuptools # Add setuptools install 53 | - conda update -q -n base -c defaults conda # Update base packages 54 | - conda info -a # Check the environment in case there are any issues 55 | 56 | # Create and activate the test environment 57 | - conda create $CONDA_ARGS -n test python=$TRAVIS_PYTHON_VERSION $AP_SELECT $NP_SELECT $CR_SELECT $CONDA_DEPS 58 | - source activate test 59 | 60 | # DOCUMENTATION DEPENDENCIES 61 | - if [[ $SETUP_CMD == build_sphinx* ]]; then $CONDA_INSTALL numpy=$NUMPY_VERSION $CONDA_DOC_DEPS; fi 62 | 63 | # Codecov 64 | - pip install codecov 65 | 66 | # Install package with pip 67 | - pip install . 68 | 69 | script: 70 | - coverage run -m pytest 71 | 72 | after_success: codecov 73 | -------------------------------------------------------------------------------- /tests/test_osm_monitors.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest 3 | 4 | from cosmo.monitors.osm_drift_monitors import FUVOSMDriftMonitor, NUVOSMDriftMonitor 5 | 6 | from cosmo.monitors.osm_shift_monitors import ( 7 | FuvOsmShift1Monitor, FuvOsmShift2Monitor, NuvOsmShift1Monitor, NuvOsmShift2Monitor 8 | ) 9 | 10 | from cosmo.monitors.data_models import OSMDataModel 11 | from cosmo.sms import SMSFinder 12 | 13 | 14 | @pytest.fixture(params=[False, True]) 15 | def set_osmmonitor(request, data_dir, here): 16 | if request.param: 17 | model = OSMDataModel() 18 | model.ingest() 19 | 20 | def _set_monitor(monitor, datamodel): 21 | # OSM Drift model requires that an SMS database exist 22 | finder = SMSFinder(data_dir) 23 | finder.ingest_files() 24 | 25 | datamodel.files_source = data_dir 26 | datamodel.cosmo_layout = False 27 | 28 | monitor.data_model = datamodel 29 | monitor.output = here 30 | 31 | active = monitor() 32 | 33 | return active 34 | 35 | return _set_monitor 36 | 37 | 38 | class TestOSMDriftMonitors: 39 | 40 | @pytest.fixture(autouse=True, params=[FUVOSMDriftMonitor, NUVOSMDriftMonitor]) 41 | def osmdriftmonitor(self, request, set_osmmonitor): 42 | osmdriftmonitor = set_osmmonitor(request.param, OSMDataModel) 43 | 44 | request.cls.osmdriftmonitor = osmdriftmonitor 45 | 46 | yield 47 | 48 | if request.cls.osmdriftmonitor.model.model is not None: 49 | request.cls.osmdriftmonitor.model.model.drop_table(safe=True) 50 | 51 | def test_monitor_steps(self): 52 | self.osmdriftmonitor.initialize_data() 53 | self.osmdriftmonitor.run_analysis() 54 | self.osmdriftmonitor.plot() 55 | self.osmdriftmonitor.write_figure() 56 | self.osmdriftmonitor.store_results() 57 | 58 | assert os.path.exists(self.osmdriftmonitor.output) 59 | 60 | 61 | class TestOsmShiftMonitors: 62 | 63 | @pytest.fixture( 64 | autouse=True, 65 | params=[FuvOsmShift1Monitor, FuvOsmShift2Monitor, NuvOsmShift1Monitor, NuvOsmShift2Monitor] 66 | ) 67 | def osmshiftmonitor(self, request, set_osmmonitor): 68 | osmshiftmonitor = set_osmmonitor(request.param, OSMDataModel) 69 | 70 | request.cls.osmshiftmonitor = osmshiftmonitor 71 | 72 | yield 73 | 74 | if request.cls.osmshiftmonitor.model.model is not None: 75 | request.cls.osmshiftmonitor.model.model.drop_table(safe=True) 76 | 77 | def test_monitor_steps(self): 78 | self.osmshiftmonitor.initialize_data() 79 | self.osmshiftmonitor.run_analysis() 80 | self.osmshiftmonitor.plot() 81 | self.osmshiftmonitor.write_figure() 82 | self.osmshiftmonitor.store_results() 83 | 84 | assert os.path.exists(self.osmshiftmonitor.output) 85 | 86 | if 'Fuv' in self.osmshiftmonitor.name: 87 | assert os.path.exists( 88 | os.path.join( 89 | os.path.dirname(self.osmshiftmonitor.output), 90 | f'{self.osmshiftmonitor._filename}-outliers.csv' 91 | ) 92 | ) 93 | -------------------------------------------------------------------------------- /cosmo/monitors/jitter_monitors.py: -------------------------------------------------------------------------------- 1 | import os 2 | import plotly.graph_objects as go 3 | 4 | from plotly.subplots import make_subplots 5 | 6 | from ..filesystem import JitterFileData 7 | from ..monitor_helpers import absolute_time 8 | 9 | # TODO: Jitter Monitor needs to include 2 (possibly 3) subplots of statistics on the jitter per exposure. 10 | # Plot 1: mean vs time with +/- std "line boundaries" 11 | # Plot 2: max vs time 12 | # Plot 3 (potentially): std vs time 13 | 14 | 15 | def view_jitter(jitter_file: str) -> go.Figure: 16 | """Plot Jitter data for a single jitter root or association.""" 17 | # Get the data 18 | jitter_data = JitterFileData( 19 | jitter_file, 20 | primary_header_keys=('PROPOSID', 'CONFIG'), 21 | ext_header_keys=('EXPNAME',), 22 | table_keys=('SI_V2_AVG', 'SI_V3_AVG', 'SI_V2_RMS', 'SI_V3_RMS', 'Seconds'), 23 | get_expstart=True 24 | ) 25 | 26 | # Start the figure 27 | figure = make_subplots(2, 1, shared_xaxes=True) 28 | 29 | for jit in jitter_data: 30 | for direction, position in zip(['V2', 'V3'], [(1, 1), (2, 1)]): 31 | # Set time 32 | time = absolute_time(expstart=jit['EXPSTART'], time=jit['Seconds']) 33 | 34 | # Lower bound 35 | figure.add_trace( 36 | go.Scatter( 37 | x=time.to_datetime(), 38 | y=jit[f'SI_{direction}_AVG'] - jit[f'SI_{direction}_RMS'], 39 | mode='lines', 40 | line={'width': 0}, 41 | showlegend=False, 42 | legendgroup=jit['EXPNAME'], 43 | name=f'{jit["EXPNAME"]} -RMS' 44 | ), 45 | row=position[0], 46 | col=position[-1] 47 | ) 48 | 49 | # Upper bound 50 | figure.add_trace( 51 | go.Scatter( 52 | x=time.to_datetime(), 53 | y=jit[f'SI_{direction}_AVG'] + jit[f'SI_{direction}_RMS'], 54 | mode='lines', 55 | line={'width': 0}, 56 | fillcolor='rgba(68, 68, 68, 0.1)', 57 | fill='tonexty', 58 | showlegend=False, 59 | legendgroup=jit['EXPNAME'], 60 | name=f'{jit["EXPNAME"]} +RMS' 61 | ), 62 | row=position[0], 63 | col=position[-1] 64 | ) 65 | 66 | # Jitter 67 | figure.add_trace( 68 | go.Scatter( 69 | x=time.to_datetime(), 70 | y=jit[f'SI_{direction}_AVG'], 71 | mode='lines', 72 | legendgroup=jit['EXPNAME'], 73 | name=f'{jit["EXPNAME"]} - {direction}', 74 | hovertemplate='Time=%{x}
Jitter=%{y} arcseconds' 75 | ), 76 | row=position[0], 77 | col=position[-1] 78 | ) 79 | 80 | figure.update_layout( 81 | { 82 | 'title': f'{os.path.basename(jitter_file)} {jitter_data[0]["CONFIG"]}', 83 | 'xaxis': {'title': 'Datetime'}, 84 | 'yaxis': {'title': 'V2 Jitter (averaged over 3 seconds) [arceconds]'}, 85 | 'xaxis2': {'title': 'Datetime'}, 86 | 'yaxis2': {'title': 'V3 Jitter (averaged over 3 seconds) [arcseconds]'} 87 | } 88 | ) 89 | 90 | return figure 91 | -------------------------------------------------------------------------------- /docs/source/datamodels.rst: -------------------------------------------------------------------------------- 1 | DataModels 2 | ========== 3 | The ``monitorframe`` framework uses ``DataModel`` classes (subclasses of the ``BaseDataModel``) to find new data and 4 | interact with the monitor data database. 5 | Each monitor requires a DataModel to function and accesses data through the DataModel object. 6 | 7 | DataModels can also be used independently from the Monitor that they were intended for for exploring the data or 8 | ingesting new data outside of the monitoring workflow. 9 | 10 | For more detailed information on the DataModel, see the 11 | `monitorframe documentation `_. 12 | 13 | AcqDataModel 14 | ------------ 15 | The ``AcqDataModel`` is used with the Target Acquisition monitors. 16 | 17 | AcqDataModel includes the following data: 18 | 19 | - ACQSLEWX: Slew value in the X direction. 20 | Default of 0 for Acquisition types that don't record this keyword. 21 | - ACQSLEWY: Slew value in the Y direction. 22 | Default of 0 for Acquisition types that don't record this keyword. 23 | - EXPSTART: Time at the start of the exposure (mjd). 24 | - ROOTNAME: Exposure rootname. 25 | - PROPOSID: Proposal ID. 26 | - OBSTYPE: Imaging or Spectroscopic. 27 | - NEVENTS: Number of events recorded during the exposure. 28 | - SHUTTER: Open or Closed. 29 | - LAMPEVNT: Number of events recorded from the internal lamp exposure. 30 | - ACQSTAT: Success or Failure. 31 | - EXTENDED: Yes or No. 32 | - LINENUM: Line number where the exposure occurs on the observation schedule. 33 | - APERTURE: Aperture used during the observation. 34 | - OPT_ELEM: Grating or Mirror used during the observation. 35 | - LIFE_ADJ: Lifetime Position. 36 | - CENWAVE: Cenwave of the observation. 37 | - DETECTOR: NUV or FUV. 38 | - EXPTYPE: Type of Acquisition. 39 | ACQ/IMAGE, ACQ/PEAKD, ACQ/PEAKXD, or ACQ/SEARCH 40 | - DGESTAR: Dominant guide star used for the exposure. 41 | - FGS: Fine guidance sensor used during the exposure. 42 | F1, F2, or F3 43 | 44 | These data are gathered from *rawacq* and *spt* files. 45 | 46 | OSMDataModel 47 | ------------ 48 | The ``OSMDataModel`` is used with the OSM Monitors. 49 | 50 | OSMDataModel includes the following data: 51 | 52 | - ROOTNAME: Exposure rootname. 53 | - EXPSTART: Time at the start of the exposure (mjd). 54 | - DETECTOR: NUV or FUV. 55 | - LIFE_ADJ: Lifetime Position. 56 | - OPT_ELEM: Grating or Mirror used during the observation. 57 | - CENWAVE: Cenwave of the observation. 58 | - FPPOS: FPPOS used for the observation. 59 | - PROPOSID: Proposal ID. 60 | - OBSET_ID: Observation set ID. 61 | - TIME: The TIME column of a given exposure (s). 62 | Time of shift measurement (relative to the start of the exposure). 63 | - SHIFT_DISP: The SHIFT_DISP column of a given exposure (pix). 64 | Shift values in the dispersion direction. 65 | - SHIFT_XDISP: The SHIFT_XDISP column of a given exposure (pix). 66 | Shift values in the cross-dispersion direction. 67 | - SEGMENT: The SEGMENT column of a given exposure. 68 | - SEGMENT_LAMPTAB: The SEGMENT column of the matched row in the LAMPTAB file. 69 | - FP_PIXEL_SHIFT: the FP_PIXEL_SHIFT column of the matched rows (all segments) in the LAMPTAB file. 70 | The FP_PIXEL_SHIFT is the offset in pixels for each FPPOS. 71 | - XC_RANGE: The XC_RANGE column of the matched row in the WCPTAB file. 72 | XC_RANGE is the maximum pixel offset to use when doing a cross correlation between the observed data and the template 73 | wavecal. 74 | - SEARCH_OFFSET: The SEARCH_OFFSET column of the matched row in the WCPTAB file. 75 | The SEARCH_OFFSET is the zero-point offset for the search range. 76 | 77 | 78 | These data are gathered from *lampflash* files, LAMPTAB files and WCPTAB files (the LAMPTAB and WCPTAB used depends on 79 | the *lampflash* file). 80 | -------------------------------------------------------------------------------- /tests/test_data_models.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | import pytest 4 | 5 | from cosmo.monitors.data_models import AcqDataModel, OSMDataModel 6 | from cosmo.sms import SMSFinder 7 | 8 | 9 | @pytest.fixture 10 | def make_datamodel(data_dir): 11 | def _make_datamodel(model): 12 | if model == OSMDataModel: 13 | # OSM Drift data model requires that an SMS database exist 14 | test_finder = SMSFinder(data_dir) 15 | test_finder.ingest_files() 16 | 17 | model.files_source = data_dir 18 | model.subdir_pattern = None 19 | test_model = model() 20 | 21 | return test_model 22 | 23 | return _make_datamodel 24 | 25 | 26 | class TestOSMDataModel: 27 | 28 | @pytest.fixture(autouse=True) 29 | def osmmodel(self, request, make_datamodel): 30 | osmmodel = make_datamodel(OSMDataModel) 31 | 32 | request.cls.osmmodel = osmmodel # Add the data model to the test class 33 | 34 | yield 35 | 36 | if request.cls.osmmodel.model is not None: 37 | request.cls.osmmodel.model.drop_table(safe=True) 38 | 39 | def test_data_collection(self): 40 | assert isinstance(self.osmmodel.new_data, pd.DataFrame) 41 | assert len(self.osmmodel.new_data) == 11 # There are 11 test data sets 42 | 43 | def test_content_collected(self): 44 | keys_that_should_be_there = ( 45 | # Header keywords 46 | 'ROOTNAME', 'EXPSTART', 'DETECTOR', 'LIFE_ADJ', 'OPT_ELEM', 'CENWAVE', 'FPPOS', 'PROPOSID', 'OBSET_ID', 47 | 48 | # Data extension keywords 49 | 'TIME', 'SHIFT_DISP', 'SHIFT_XDISP', 'SEGMENT', 50 | 51 | # Data from the SMS files 52 | 'TSINCEOSM1', 'TSINCEOSM2', 53 | 54 | # Reference file data 55 | 'LAMPTAB_SEGMENT', 'FP_PIXEL_SHIFT', 'XC_RANGE' 56 | ) 57 | 58 | for key in keys_that_should_be_there: 59 | assert key in self.osmmodel.new_data 60 | 61 | # Check that entries that have no data have been removed 62 | assert not self.osmmodel.new_data.apply(lambda x: not bool(len(x.SHIFT_DISP)), axis=1).all() 63 | 64 | def test_data_extension_data(self): 65 | data_extension_keys = ('TIME', 'SHIFT_DISP', 'SHIFT_XDISP', 'SEGMENT') 66 | 67 | for key in data_extension_keys: 68 | assert isinstance(self.osmmodel.new_data[key].values, np.ndarray) 69 | 70 | def test_reference_data(self): 71 | data_extension_keys = ('LAMPTAB_SEGMENT', 'FP_PIXEL_SHIFT', 'XC_RANGE') 72 | 73 | for key in data_extension_keys: 74 | assert isinstance(self.osmmodel.new_data[key].values, np.ndarray) 75 | 76 | def test_data_ingest(self): 77 | self.osmmodel.ingest() 78 | 79 | assert self.osmmodel.model is not None 80 | assert len(list(self.osmmodel.model.select())) == 11 81 | 82 | 83 | class TestAcqDataModel: 84 | 85 | @pytest.fixture(autouse=True) 86 | def acqmodel(self, request, make_datamodel): 87 | acqmodel = make_datamodel(AcqDataModel) 88 | 89 | request.cls.acqmodel = acqmodel 90 | 91 | yield 92 | 93 | if request.cls.acqmodel.model is not None: 94 | request.cls.acqmodel.model.drop_table(safe=True) 95 | 96 | def test_data_collection(self): 97 | assert isinstance(self.acqmodel.new_data, pd.DataFrame) 98 | assert len(self.acqmodel.new_data) == 9 # There are 9 test data sets 99 | 100 | def test_content_collected(self): 101 | keys_that_should_be_there = ( 102 | # Header keywords 103 | 'ACQSLEWX', 'ACQSLEWY', 'EXPSTART', 'ROOTNAME', 'PROPOSID', 'OBSTYPE', 'NEVENTS', 'SHUTTER', 'LAMPEVNT', 104 | 'ACQSTAT', 'EXTENDED', 'LINENUM', 'APERTURE', 'OPT_ELEM', 'CENWAVE', 'DETECTOR', 'LIFE_ADJ', 105 | 106 | # SPT header keywords 107 | 'DGESTAR' 108 | ) 109 | 110 | for key in keys_that_should_be_there: 111 | assert key in self.acqmodel.new_data 112 | 113 | # Check that the FGS column was created correctly 114 | assert 'FGS' in self.acqmodel.new_data 115 | assert sorted(self.acqmodel.new_data.FGS.unique()) == ['F1', 'F2', 'F3'] 116 | 117 | def test_data_ingest(self): 118 | self.acqmodel.ingest() 119 | 120 | assert self.acqmodel.model is not None 121 | assert len(list(self.acqmodel.model.select())) == 9 122 | -------------------------------------------------------------------------------- /tests/data/122687b1.txt: -------------------------------------------------------------------------------- 1 | 2 | COS Exposure Report: SMS 122687B1 3 | 4 | Data Exposure Start FUV Mechanism Positions Cent Tsince Tsince Lamp STIMs Flash 5 | Filename Prop Target PRG OB AL EX Conf Opmode ExpTime yyyy.ddd:hh:mm:ss State Aper OSM1 OSM2 Wave FP OSM1 OSM2 Lamp Curr A B Dop # Dur T 6 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 7 | lbx1a2ff 12781 206W3 BX1 A2 01 01 NUV ACQ/IMAGE 60.0 2012.268:18:35:34 PSA O NCM1 MIRRORA 0 0 413 230 NO 8 | lbx1a2fh 12781 206W3 BX1 A2 03 01 NUV ACQ/IMAGE 300.0 2012.268:18:45:54 PSA O NCM1 MIRRORB 0 0 1033 484 NO 9 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 10 | lbhx26fj 12289 J081520.65 BHX 26 01 01 NUV ACQ/IMAGE 70.0 2012.268:19:15:42 PSA B NCM1 MIRRORB 0 0 452 263 NO 11 | lbhx26fm 12289 J081520.65 BHX 26 03 01 FUV TIME-TAG 900.0 2012.268:19:21:00 167/163 PSA B G140L ----- 1280 0 128 581 LOW LOW NO 2 7 S 12 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 13 | lbdo3sja 12046 BDO 3S 01 01 2012.269:00:30:29 COS MEMORY COPY -- DCE CODE IMAGE TO CS RAM 14 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 15 | lc1s04l8 12929 WD1654+160 C1S 04 01 01 FUV ACQ/SEARCH 2.2 2012.273:04:46:37 167/163 PSA B G130M ----- 1291 0 378428 378290 SCANSIZE=2 STEPSIZE=1767 FLUXWT 16 | lc1s04l9 12929 WD1654+160 C1S 04 02 01 FUV ACQ/PEAKXD 2.1 2012.273:04:50:31 167/163 PSA B G130M ----- 1291 0 378662 378524 OFF OFF NO 17 | lc1s04la 12929 WD1654+160 C1S 04 03 01 FUV ACQ/PEAKD 2.2 2012.273:04:51:20 167/163 PSA B G130M ----- 1291 0 378711 378573 DWELLS=3 STEPSIZE=1300 FLUXWT 18 | lc1s04lc 12929 WD1654+160 C1S 04 05 01 FUV TIME-TAG 286.0 2012.273:04:55:46 167/163 PSA B G130M ----- 1291 -2 378977 378839 LOW LOW NO 1 12 S 19 | lc1s04le 12929 WD1654+160 C1S 04 07 01 FUV TIME-TAG 310.0 2012.273:05:02:55 167/163 PSA B G130M ----- 1291 -1 379406 379268 LOW LOW NO 1 12 S 20 | lc1s04lg 12929 WD1654+160 C1S 04 09 01 FUV TIME-TAG 310.0 2012.273:05:10:38 167/163 PSA B G130M ----- 1291 0 379869 379731 LOW LOW NO 1 12 S 21 | lc1s04li 12929 WD1654+160 C1S 04 0B 01 FUV TIME-TAG 310.0 2012.273:05:18:21 167/163 PSA B G130M ----- 1291 1 380332 380194 LOW LOW NO 1 12 S 22 | lc1s04lk 12929 WD1654+160 C1S 04 0D 01 FUV TIME-TAG 393.0 2012.273:05:28:15 167/163 PSA B G160M ----- 1611 -2 144 380788 LOW LOW NO 2 12 S 23 | lc1s04lm 12929 WD1654+160 C1S 04 0F 01 FUV TIME-TAG 871.0 2012.273:06:20:13 167/163 PSA B G160M ----- 1611 -1 3262 383906 LOW LOW NO 2 12 S 24 | lc1s04lo 12929 WD1654+160 C1S 04 0H 01 FUV TIME-TAG 871.0 2012.273:06:36:48 167/163 PSA B G160M ----- 1611 0 4257 384901 LOW LOW NO 2 12 S 25 | lc1s04lq 12929 WD1654+160 C1S 04 0J 01 FUV TIME-TAG 966.0 2012.273:06:53:23 167/163 PSA B G160M ----- 1611 1 5252 385896 LOW LOW NO 2 12 S 26 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 27 | lbs332rf 12603 2MASS-J154 BS3 32 01 01 NUV ACQ/IMAGE 257.0 2012.274:13:20:50 PSA B NCM1 MIRRORB 0 0 639 450 NO 28 | lbs332rh 12603 2MASS-J154 BS3 32 03 01 FUV TIME-TAG 2073.0 2012.274:14:23:59 167/163 PSA B G130M ----- 1291 0 137 4239 LOW LOW NO 3 12 S 29 | lbs332rq 12603 2MASS-J154 BS3 32 06 01 FUV TIME-TAG 2971.0 2012.274:15:46:45 167/163 PSA B G130M ----- 1291 1 5103 9205 LOW LOW NO 2 12 S 30 | lbs332rz 12603 2MASS-J154 BS3 32 08 01 FUV TIME-TAG 2901.0 2012.274:17:23:37 167/163 PSA B G130M ----- 1327 0 10915 15017 LOW LOW NO 2 12 S 31 | lbs332s9 12603 2MASS-J154 BS3 32 0A 01 FUV TIME-TAG 2971.0 2012.274:18:58:13 167/163 PSA B G130M ----- 1327 1 16591 20693 LOW LOW NO 2 12 S 32 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 33 | -------------------------------------------------------------------------------- /tests/test_monitor_helpers.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import pandas as pd 3 | import numpy as np 4 | 5 | from cosmo.monitor_helpers import convert_day_of_year, fit_line, explode_df, absolute_time, create_visibility, v2v3 6 | 7 | 8 | @pytest.fixture(params=[2017.301, '2017.301']) 9 | def good_date(request): 10 | return request.param 11 | 12 | 13 | @pytest.fixture(params=[1, '1']) 14 | def bad_date(request): 15 | return request.param 16 | 17 | 18 | class TestConvertDayofYear: 19 | 20 | def test_ingest(self, good_date): 21 | convert_day_of_year(good_date) 22 | 23 | def test_date_type_fail(self, bad_date): 24 | with pytest.raises((ValueError, TypeError)): # The format shouldn't match 25 | convert_day_of_year(bad_date) 26 | 27 | 28 | class TestFitLine: 29 | 30 | def test_simple_fit(self): 31 | test_x = test_y = [1, 2, 3] 32 | fit, fitline = fit_line(test_x, test_y) 33 | 34 | assert len(fit.coeffs) == 2 # should be a linear fit 35 | assert fit[1] == pytest.approx(1) # slope of 1 36 | assert fit[0] == pytest.approx(0) # intercept of 0 37 | 38 | def test_different_lengths_fail(self): 39 | test_x = [1, 2] 40 | test_y = [1, 2, 3] 41 | 42 | with pytest.raises(TypeError): 43 | fit_line(test_x, test_y) 44 | 45 | 46 | @pytest.fixture 47 | def test_df(): 48 | return pd.DataFrame({'a': 1, 'b': [[1, 2, 3]]}) 49 | 50 | 51 | class TestExplodeDf: 52 | 53 | def test_exploded_length(self, test_df): 54 | exploded = explode_df(test_df, ['b']) 55 | assert len(exploded) == 3 56 | assert all(exploded.a == 1) 57 | 58 | def test_different_lengths_fail(self, test_df): 59 | with pytest.raises(AttributeError): 60 | explode_df(test_df, ['a', 'b']) # Column a is not "explode-able" 61 | 62 | test_df['c'] = [[1, 2]] # Add a column with an array element of a different length than b 63 | 64 | with pytest.raises(ValueError): 65 | explode_df(test_df, ['c', 'b']) 66 | 67 | # If the first column listed is longer, the procedure won't produce an error, but the result will have NaNs 68 | with pytest.raises(ValueError): 69 | explode_df(test_df, ['b', 'c']) 70 | 71 | 72 | ABSTIME_BAD_INPUT = [ 73 | (pd.DataFrame({'EXPSTART': [58484.0, 58485.0, 58486.0], }), None, None, AttributeError), 74 | (pd.DataFrame({'EXPSTART': [58484.0, 58485.0, 58486.0], 'TIME': [1, 2, 3]}), [1, 2, 3], [1, 2, 3], ValueError), 75 | (None, None, None, TypeError), 76 | (pd.DataFrame({'TIME': [1, 2, 3]}), None, None, AttributeError), 77 | ( 78 | pd.DataFrame({'EXPSTART': [58484.0, 58485.0, 58486.0], 'some_other_time': [1, 2, 3]}), 79 | None, 80 | None, 81 | AttributeError 82 | ), 83 | (None, [1, 2, 3], None, TypeError), 84 | (None, None, [1, 2, 3], TypeError) 85 | ] 86 | 87 | ABSTIME_GOOD_INPUT = [ 88 | (pd.DataFrame({'EXPSTART': [58484.0, 58485.0, 58486.0], 'TIME': [1, 2, 3]}), None, None, None), 89 | ( 90 | pd.DataFrame({'EXPSTART': [58484.0, 58485.0, 58486.0], 'some_other_time': [1, 2, 3]}), 91 | None, 92 | None, 93 | 'some_other_time' 94 | ), 95 | (None, [1, 2, 3], [1, 2, 3], None), 96 | (None, np.array([1, 2, 3]), np.array([1, 2, 3]), None) 97 | ] 98 | 99 | 100 | @pytest.fixture(params=ABSTIME_BAD_INPUT) 101 | def bad_input(request): 102 | return request.param 103 | 104 | 105 | @pytest.fixture(params=ABSTIME_GOOD_INPUT) 106 | def good_input(request): 107 | return request.param 108 | 109 | 110 | class TestAbsoluteTime: 111 | 112 | def test_ingest_fails(self, bad_input): 113 | df, expstart, time, error = bad_input 114 | 115 | with pytest.raises(error): 116 | absolute_time(df=df, expstart=expstart, time=time) 117 | 118 | def test_compute_absolute_time(self, good_input): 119 | df, expstart, time, time_key = good_input 120 | absolute_time(df=df, expstart=expstart, time=time, time_key=time_key) 121 | 122 | 123 | class TestCreateVisibility: 124 | 125 | def test_output(self): 126 | test_trace_lengths = [1, 2, 3] 127 | test_visible = [True, False, False] 128 | 129 | visible_options = create_visibility(test_trace_lengths, test_visible) 130 | 131 | assert len(visible_options) == 6 132 | assert visible_options == [True, False, False, False, False, False] 133 | 134 | 135 | class TestV2V3: 136 | 137 | def test_list_input(self): 138 | x = y = [1, 2, 3] 139 | v2, v3 = v2v3(x, y) 140 | 141 | assert isinstance(v2, np.ndarray) and isinstance(v3, np.ndarray) 142 | 143 | def test_calculation(self): 144 | x = y = np.array([1, 2, 3]) 145 | v2, v3 = v2v3(x, y) 146 | 147 | # Given the conversion, v2 should be [sqrt(2), 2*sqrt(2), 3*sqrt(2)] and y should be [0, 0, 0] 148 | for item, expected in zip(v2, [np.sqrt(2), np.sqrt(2) * 2, np.sqrt(2) * 3]): 149 | assert item == pytest.approx(expected) 150 | 151 | for item in v3: 152 | assert item == pytest.approx(0) 153 | -------------------------------------------------------------------------------- /cosmo/run_monitors.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest 3 | import shlex 4 | 5 | from argparse import ArgumentParser 6 | 7 | from . import monitors 8 | from .sms import SMSFinder 9 | 10 | 11 | def collection() -> dict: 12 | """Collect Monitor and DataModel classes. Monitors will only be collected if the monitors package structure is 13 | maintained (i.e. ensuring that new monitors are included in monitors.__init__.py). 14 | 15 | Monitors will only be collected if they adhere to the "*Monitor" naming conventions, and DataModels will only be 16 | collected if they adhere to the "*DataModel" naming convention. Any class with "Base" in the name will be ignored as 17 | it is assumed that these are not complete monitors and should not be executed by themselves. 18 | """ 19 | collection_set = {'monthly': [], 'daily': [], 'all': [], 'datamodels': []} 20 | 21 | for key, value in monitors.__dict__.items(): 22 | if 'Monitor' in key and 'Base' not in key: 23 | collection_set['all'].append(value) 24 | 25 | if 'run' in value.__dict__: 26 | collection_set[value.run].append(value) 27 | 28 | # noinspection PyUnresolvedReferences 29 | for key, value in monitors.data_models.__dict__.items(): 30 | if 'DataModel' in key and 'Base' not in key: 31 | collection_set['all'].append(value) 32 | collection_set['datamodels'].append(value) 33 | 34 | return collection_set 35 | 36 | 37 | COLLECTION = collection() 38 | 39 | 40 | @pytest.fixture 41 | def monitor(): 42 | """Fixture-factory that creates a monitor instance from the input class.""" 43 | def _monitor(monitor_class): 44 | active = monitor_class() 45 | 46 | return active 47 | 48 | return _monitor 49 | 50 | 51 | @pytest.fixture(params=COLLECTION['monthly']) 52 | def monthly_monitor(request, monitor): 53 | """Parametrized fixture for monitors that should be executed monthly.""" 54 | active = monitor(request.param) 55 | 56 | yield active 57 | 58 | # If run_ingest is not included in the test session, ingest any new data into the databases 59 | session_names = [item.name for item in request.session.items] # names of "test" in the session object 60 | 61 | if 'run_ingest' not in session_names: 62 | active.model.ingest() 63 | 64 | 65 | @pytest.fixture(params=COLLECTION['daily']) 66 | def daily_monitor(request, monitor): 67 | """Parametrized fixture for monitors that should be executed daily.""" 68 | active = monitor(request.param) 69 | 70 | yield active 71 | 72 | # If run_ingest is not included in the test session, ingest any new data into the databases 73 | session_names = [item.name for item in request.session.items] # names of "test" in the session object 74 | 75 | if 'run_ingest' not in session_names: 76 | active.model.ingest() 77 | 78 | 79 | @pytest.fixture 80 | def sms(): 81 | """Fixture for the SMSFinder object.""" 82 | finder = SMSFinder() 83 | 84 | return finder 85 | 86 | 87 | @pytest.fixture(params=COLLECTION['datamodels']) 88 | def datamodel(request): 89 | """Parametrized fixture for DataModels for use in ingestion only.""" 90 | active_model = request.param() 91 | 92 | return active_model 93 | 94 | 95 | class RunIngestion: 96 | 97 | @pytest.mark.ingest 98 | @pytest.mark.monthly 99 | def run_sms_ingest(self, sms): 100 | """Execute SMS file ingestion. Will be executed before the monthly monitors (since the OSM monitors require that 101 | new SMS files be ingested first. Additionally, this runner is included in the "ingest" group. 102 | """ 103 | sms.ingest_files() 104 | 105 | @pytest.mark.ingest 106 | def run_ingest(self, datamodel): 107 | """Execute DataModel new data discovery and ingestion. Included in the "ingest" group.""" 108 | datamodel.ingest() 109 | 110 | 111 | class RunMonitors: 112 | """Class for organizing runners.""" 113 | 114 | @pytest.mark.monthly 115 | def run_monthly(self, monthly_monitor): 116 | """Execute monitors marked as monthly.""" 117 | monthly_monitor.monitor() 118 | 119 | 120 | def runner(): 121 | """Function for running the monitors with pytest. Intended as an entry-point for use via the commandline.""" 122 | here = os.path.dirname(os.path.abspath(__file__)) 123 | 124 | default_pytest_args = f'{here}' # Any additional arguments that should be called with pytest. 125 | 126 | # Parse commandline arguments 127 | parser = ArgumentParser() 128 | 129 | parser.add_argument('--monthly', '-mo', action='store_true', help='Execute Monitors marked as "monthly"') 130 | parser.add_argument('--ingest', '-in', action='store_true', help='Execute data ingestion for DataModels and SMS') 131 | 132 | args = parser.parse_args() 133 | 134 | # Execute pytest 135 | if args.monthly: 136 | pytest.main(shlex.split(default_pytest_args + ' -m monthly')) 137 | 138 | return 139 | 140 | if args.ingestion: 141 | pytest.main(shlex.split(default_pytest_args + ' -m ingest')) 142 | 143 | return 144 | 145 | pytest.main(shlex.split(default_pytest_args)) 146 | -------------------------------------------------------------------------------- /docs/source/getting_started.rst: -------------------------------------------------------------------------------- 1 | Getting Started 2 | =============== 3 | COSMO is intended to be installed and its monitors run either manually or via cronjob. 4 | Use of COSMO outside of the scope of executing the monitors that are defined is not recommended, however, the data 5 | models and their API can be used to access the monitoring data for use outside the scope of this project if needed. 6 | 7 | Installing 8 | ---------- 9 | Before installing COSMO, be sure to create an environment with python 3.7+ at minimum. 10 | A good starting point would be:: 11 | 12 | conda create -n cosmo_env python=3.7 stsci 13 | 14 | For developers, also including ``coverage`` is also recommended (but not mandatory). 15 | 16 | After an environment has been prepared, clone the repository:: 17 | 18 | git clone https://github.com/spacetelescope/cosmo.git 19 | 20 | Then install using pip:: 21 | 22 | cd cosmo 23 | pip install -e . 24 | 25 | The ``-e`` argument is required for users who will also be developing and execute tests. 26 | 27 | Configuration 28 | -------------- 29 | COSMO Settings with a ``monitorframe`` Configuration File and Environment Variables 30 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 31 | To manage configurations, COSMO primarily uses environment variables on top of the ``monitorframe`` configuration file. 32 | 33 | Use the following, minimum set of environment variables to configure COSMO: 34 | 35 | .. code-block:: 36 | 37 | COSMO_FILES_SOURCE='path/to/data/files' 38 | COSMO_OUTPUT='path/to/monitor/output' 39 | COSMO_SMS_SOURCE='path/to/sms/files' 40 | 41 | Additional environment variables are available for further configuring the SMS database and are described in the 42 | :ref:`sms-database` section. 43 | 44 | ``monitorframe`` requires a ``yaml`` configuration file with the following: 45 | 46 | .. code-block:: yaml 47 | 48 | # Monitor data database 49 | data: 50 | db_settings: 51 | database: '' 52 | pragmas: 53 | journal_mode: 'wal' 54 | foreign_keys: 1 55 | ignore_check_constraints: 0 56 | synchronous: 0 57 | 58 | # Monitor status and results database 59 | results: 60 | db_settings: 61 | database: '' 62 | pragmas: 63 | journal_mode: 'wal' 64 | foreign_keys: 1 65 | ignore_check_constraints: 0 66 | synchronous: 0 67 | 68 | For more information on sqlite pragma statements, see `this `_. 69 | 70 | This configuration file should be set to an environment variable called ``MONITOR_CONFIG``. 71 | 72 | .. warning:: 73 | 74 | Use proper precautions around your configuration file. 75 | It may or may not contain sensitive information, so please ensure that permissions on that file are restricted to 76 | the intended users. 77 | DON'T push it to GitHub! 78 | 79 | CRDS 80 | ^^^^ 81 | Some of the COSMO DataModels utilize data from reference files, and take advantage of ``crds`` to do so. 82 | For configuration and setup instructions for using ``crds``, see 83 | `the crds user manual `_. 84 | 85 | At minimum, users will need access to a CRDS cache with the following reference file types: 86 | 87 | - LAMPTAB 88 | - WCPTAB 89 | 90 | Since the COSMO monitors use data from reference files across time, it would be best to get all files of those types 91 | available in the *active context*. 92 | 93 | The easiest way to ensure that the local CRDS cache has everything required, users can use:: 94 | 95 | crds sync --contexts hst-cos-operational --fetch-references 96 | 97 | This command with download *all* COS reference files and mappings to the ``CRDS_CACHE`` (see the instructions mentioned 98 | above). 99 | 100 | .. warning:: 101 | 102 | The command given above works well, but there's a caveat: it requires a large amount of available storage space at 103 | the cache location (between 2-3 GB). 104 | 105 | Running Tests 106 | ------------- 107 | COSMO includes a suite of tests for the package. 108 | For developers, it's a good idea to execute these tests whenever there are changes to the code or environment. 109 | 110 | Before executing tests, set the ``MONITOR_CONFIG`` environment variable to the test configuration 111 | that's included with the repository: ``cosmo/tests/cosmoconfig_tests.yaml``, and set the ``COSMO_SMS_DB`` environment 112 | variable to `'test.db'`. 113 | 114 | .. note:: 115 | 116 | If tests are executed before setting the ``MONITOR_CONFIG`` and ``COSMO_SMS_DB`` environment variables to the test 117 | configurations, the tests *will not execute*. 118 | 119 | If you're in the project directory, you can execute the tests with:: 120 | 121 | python -m pytest 122 | 123 | For executing the tests with coverage (after ``coverage`` has been installed), use:: 124 | 125 | coverage run -m pytest 126 | 127 | Executing Monitors 128 | ------------------ 129 | Monitors can be executed by using the monitoring classes directly: 130 | 131 | .. code-block:: python 132 | 133 | from cosmo.monitors import AcqImageMonitor 134 | 135 | monitor = AcqImageMonitor() 136 | 137 | # Run it 138 | monitor.monitor() 139 | 140 | Or, they can be executed from the command line:: 141 | 142 | (cosmoenv) mycomputer:~ user$ cosmo --monthly 143 | 144 | For more command line options:: 145 | 146 | (cosmoenv) mycomputer:~ user$ cosmo --help 147 | 148 | -------------------------------------------------------------------------------- /tests/test_sms_ingest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | 4 | from cosmo.sms import SMSFinder, SMSFile, SMSFileStats, SMSTable 5 | 6 | TEST_DATA = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/') 7 | 8 | 9 | @pytest.fixture 10 | def clean_db_tables(): 11 | yield 12 | 13 | SMSTable.drop_table(safe=True) 14 | SMSFileStats.drop_table(safe=True) 15 | 16 | 17 | @pytest.fixture(params=[os.path.dirname(os.path.abspath(__file__)), '/this/is/not/a/directory']) 18 | def bad_file_path(request): 19 | """Fixture that parametrizes cases of file paths that should result in an error.""" 20 | bad_file_path = request.param 21 | 22 | return bad_file_path 23 | 24 | 25 | @pytest.fixture(params=[os.path.join(TEST_DATA, test_file) for test_file in ['100047aa.txt', '180147b1.txt']]) 26 | def smsfile(request, clean_db_tables): 27 | """Fixture that parametrizes cases of two files (one old format and one new) that should be ingested 28 | successfully. 29 | 30 | Includes a clean up for tests that create tables in the test database. 31 | """ 32 | return request.param 33 | 34 | 35 | @pytest.fixture 36 | def test_finder(clean_db_tables): 37 | """Fixture that yields an SMSFinder object for testing. Clean up removes database tables.""" 38 | test_finder = SMSFinder(TEST_DATA) 39 | 40 | return test_finder 41 | 42 | 43 | class TestSMSFile: 44 | """Test class that includes tests for the SMSFile object.""" 45 | 46 | def test_data_ingest(self, smsfile): 47 | """Test that SMSFile is initialized successfully and that the file data is correctly found and ingested.""" 48 | SMSFile(smsfile) 49 | 50 | def test_ingest_fail(self): 51 | """Test that ingestion fails for a file with an unknown format.""" 52 | bad_file = os.path.join(TEST_DATA, 'bad_111078a6.txt') 53 | 54 | with pytest.raises(ValueError): 55 | SMSFile(bad_file) 56 | 57 | def test_datatypes(self, smsfile): 58 | """Test that the ingested dtypes are correct for each field.""" 59 | correct_dtypes = { 60 | 'FILEID': object, 61 | 'FILENAME': object, 62 | 'EXPOSURE': object, 63 | 'ROOTNAME': object, 64 | 'PROPOSID': int, 65 | 'DETECTOR': object, 66 | 'OPMODE': object, 67 | 'EXPTIME': float, 68 | 'EXPSTART': object, 69 | 'FUVHVSTATE': object, 70 | 'APERTURE': object, 71 | 'OSM1POS': object, 72 | 'OSM2POS': object, 73 | 'CENWAVE': int, 74 | 'FPPOS': int, 75 | 'TSINCEOSM1': float, 76 | 'TSINCEOSM2': float 77 | } 78 | 79 | sms = SMSFile(smsfile) 80 | dtypes = sms.data.dtypes 81 | 82 | for key, value in dtypes.iteritems(): 83 | assert value == correct_dtypes[key] 84 | 85 | def test_database_ingest(self, smsfile): 86 | """Test that the insert_to_db method executes successfully.""" 87 | test_sms = SMSFile(smsfile) 88 | test_sms.insert_to_db() 89 | 90 | 91 | class TestSMSFinder: 92 | """Tests for SMSFinder""" 93 | 94 | def test_found(self, test_finder): 95 | """Test that sms files are found correctly.""" 96 | assert len(test_finder.all_sms) == 13 97 | 98 | def test_ingest_files(self, test_finder): 99 | """Test that sms files are ingested into the database correctly.""" 100 | test_finder.ingest_files() 101 | assert len(list(SMSFileStats.select())) == 13 # check that files are actually ingested 102 | 103 | # Check conflict resolution 104 | test_finder.ingest_files() # ingest the same files again 105 | assert len(list(SMSFileStats.select())) == 13 # check that the same files are not ingested again 106 | 107 | def test_sms_classification(self, test_finder): 108 | """Test that the sms files are correctly determined as new.""" 109 | assert len(test_finder.new_sms) == 13 # All data is new if nothing is in the database 110 | assert test_finder.old_sms is None 111 | 112 | test_finder.ingest_files() 113 | ingested_test_finder = SMSFinder(TEST_DATA) 114 | 115 | assert ingested_test_finder.new_sms is None # All data was ingested 116 | assert len(ingested_test_finder.currently_ingested) == 13 117 | assert len(ingested_test_finder.old_sms) == 13 118 | 119 | def test_fails_on_no_data(self, bad_file_path): 120 | """Test that an error is raised if no files are found.""" 121 | with pytest.raises(OSError): 122 | SMSFinder(bad_file_path) 123 | 124 | def test_version_filter(self, test_finder): 125 | """Test that the sms finder filters the files and only 'finds' the most recent version of the available SMS.""" 126 | # test SMS file set, 181137 includes three versions of the same SMS. 127 | # The only file reported by SMSFinder should be the 'newest' version, c2. 128 | testcase = test_finder.new_sms[test_finder.new_sms.sms_id == '181137'] 129 | 130 | assert len(testcase) == 1 131 | assert testcase.version.values[0] == 'c2' 132 | 133 | def test_entry_is_updated(self, clean_db_tables): 134 | test_sms = SMSFile(os.path.join(TEST_DATA, '181137b4.txt')) 135 | test_sms.insert_to_db() 136 | 137 | # Attempt to insert a newer version of the same SMS 138 | update_sms = SMSFile(os.path.join(TEST_DATA, '181137c2.txt')) 139 | update_sms.insert_to_db() 140 | 141 | record = SMSFileStats.get(SMSFileStats.SMSID == '181137') 142 | assert record.VERSION == 'c2' # After running ingest_files the newer file should've replaced the old version 143 | 144 | records = SMSTable.select().where(SMSTable.FILEID == '181137').dicts().iterator() 145 | for record in records: 146 | assert record['FILEID'] == '181137c2' 147 | -------------------------------------------------------------------------------- /cosmo/monitor_helpers.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | import datetime 4 | 5 | from itertools import repeat 6 | from astropy.time import Time, TimeDelta 7 | from typing import Union, Tuple, Sequence, List 8 | 9 | 10 | def convert_day_of_year(date: Union[float, str]) -> Time: 11 | """Convert day of the year (defined as yyyy.ddd where ddd is the day number of that year) to an astropy Time object. 12 | Some important dates for the COS team were recorded in this format. 13 | """ 14 | return Time( 15 | datetime.datetime.strptime( 16 | f'{date:.3f}' if isinstance(date, float) else date, 17 | '%Y.%j' 18 | ), 19 | format='datetime' 20 | ) 21 | 22 | 23 | def fit_line(x: Sequence, y: Sequence) -> Tuple[np.poly1d, np.ndarray]: 24 | """Given arrays x and y, fit a line.""" 25 | fit = np.poly1d(np.polyfit(x, y, 1)) 26 | 27 | return fit, fit(x) 28 | 29 | 30 | def explode_df(df: pd.DataFrame, list_keywords: list) -> pd.DataFrame: 31 | """If a dataframe contains arrays for the element of a column or columns given by list_keywords, expand the 32 | dataframe to one row per array element. Each row in list_keywords must be the same length. 33 | """ 34 | idx = df.index.repeat(df[list_keywords[0]].str.len()) # Repeat values based on the number of elements in the arrays 35 | unpacked = pd.concat([pd.DataFrame({x: np.concatenate(df[x].values)}) for x in list_keywords], axis=1) 36 | unpacked.index = idx # assigns repeated index to the unpacked dataframe, unpacked. 37 | 38 | # Join unpacked df to the original df and drop the old columns 39 | exploded = unpacked.join(df.drop(list_keywords, 1), how='left').reset_index(drop=True) 40 | 41 | if exploded.isna().values.any(): # If there are NaNs, then it didn't make sense to "explode" the input df 42 | raise ValueError('Elements in columns to be exploded are not the same length across rows.') 43 | 44 | return exploded 45 | 46 | 47 | def absolute_time(df: pd.DataFrame = None, expstart: Sequence = None, time: Sequence = None, time_key: str = None, 48 | time_format: str = 'sec') -> TimeDelta: 49 | """Compute the time sequence relative to the start of the exposure (EXPSTART). Can be computed from a DataFrame that 50 | contains an EXPSTART column and some other time array column, or from an EXPSTART array and time array pair. 51 | """ 52 | # If no input is given raise an error 53 | if df is None and expstart is None and time is None: 54 | raise TypeError('Computing and absolute time requires either a dataframe or set of arrays') 55 | 56 | # Check that expstart and time_array are used together 57 | if bool(expstart is not None or time is not None) and not (expstart is not None and time is not None): 58 | raise TypeError('expstart and time must be used together.') 59 | 60 | # Ingest given dataframe if one is given and check that it's not used with arrays at the same time 61 | if df is not None: 62 | if bool(expstart is not None or time is not None): 63 | raise ValueError('Cannot use a dataframe and arrays as input at the same time. Use one or the other.') 64 | 65 | expstart = df.EXPSTART 66 | time = df.TIME if not time_key else df[time_key] 67 | 68 | zero_points = Time(expstart, format='mjd') 69 | time_delta = TimeDelta(time, format=time_format) 70 | 71 | return zero_points + time_delta 72 | 73 | 74 | def create_visibility(trace_lengths: List[int], visible_list: List[bool]) -> List[bool]: 75 | """Create visibility lists for plotly buttons. trace_lengths and visible_list must be in the correct order. 76 | 77 | :param trace_lengths: List of the number of traces in each "button set". 78 | :param visible_list: Visibility setting for each button set (either True or False). 79 | """ 80 | visibility = [] # Total visibility. Length should match the total number of traces in the figure. 81 | for visible, trace_length in zip(visible_list, trace_lengths): 82 | visibility += list(repeat(visible, trace_length)) # Set each trace per button. 83 | 84 | return visibility 85 | 86 | 87 | def v2v3(slew_x: Sequence, slew_y: Sequence) -> Tuple[Union[np.ndarray, pd.Series], Union[np.ndarray, pd.Series]]: 88 | """Detector coordinates to V2/V3 coordinates.""" 89 | # If input are lists, convert to np arrays so that the operations are completed as expected 90 | if isinstance(slew_x, list): 91 | slew_x = np.array(slew_x) 92 | 93 | if isinstance(slew_y, list): 94 | slew_y = np.array(slew_y) 95 | 96 | rotation_angle = np.radians(45.0) # rotation angle in degrees converted to radians 97 | x_conversion = slew_x * np.cos(rotation_angle) 98 | y_conversion = slew_y * np.sin(rotation_angle) 99 | 100 | v2 = x_conversion + y_conversion 101 | v3 = x_conversion - y_conversion 102 | 103 | return v2, v3 104 | 105 | 106 | def get_osm_data(datamodel, detector: str) -> pd.DataFrame: 107 | """Query for OSM data and append any relevant new data to it.""" 108 | data = pd.DataFrame() 109 | 110 | if datamodel.model is not None: 111 | query = datamodel.model.select().where(datamodel.model.DETECTOR == detector) 112 | 113 | # Need to convert the stored array columns back into... arrays 114 | data = data.append( 115 | datamodel.query_to_pandas( 116 | query, 117 | array_cols=[ 118 | 'TIME', 119 | 'SHIFT_DISP', 120 | 'SHIFT_XDISP', 121 | 'SEGMENT', 122 | 'XC_RANGE', 123 | 'LAMPTAB_SEGMENT', 124 | 'SEARCH_OFFSET', 125 | 'FP_PIXEL_SHIFT' 126 | ], 127 | ), 128 | sort=True, 129 | ignore_index=True 130 | ) 131 | 132 | if datamodel.new_data is None: 133 | return data 134 | 135 | if not datamodel.new_data.empty: 136 | new_data = datamodel.new_data[datamodel.new_data.DETECTOR == detector].reset_index(drop=True) 137 | data = data.append(new_data, sort=True, ignore_index=True) 138 | 139 | return data 140 | -------------------------------------------------------------------------------- /docs/source/sms.rst: -------------------------------------------------------------------------------- 1 | SMS Data Ingestion 2 | ================== 3 | Some of the monitors in COSMO rely on data that are found in Science Mission Schedule (SMS) reports. 4 | These reports are produced outside of the COS Branch and supplied via *txt* files. 5 | 6 | A variety of data on exposures is provided in these report files, and in some cases can actually be used to verify 7 | information in the regular data products themselves. 8 | 9 | SMS File Format and Naming Conventions 10 | -------------------------------------- 11 | The Report files use a human readable table format, which requires special processing to read since it's not a "typical" 12 | table format (such as *csv*). 13 | 14 | The naming convention for these files are comprised of two different ID codes: 15 | 1. A "SMS ID" which uniquely identifies a particular SMS report 16 | 2. A "Version ID" which identifies the version of a particular SMS 17 | 18 | The SMS ID is the start date of the SMS in the form "YYDDD", and is the first six digits in the report filename. 19 | The Version ID is typically an incrementing group of two alpha-numeric characters (with some rare exceptions such as for 20 | when an SMS report is re-created, at which point an "r" will be added to the end of the normal Version ID) that occur 21 | after the SMS ID in the filename. 22 | For example, SMS 091403 version "b4" is a more recent version than "a1." 23 | 24 | SMS reports can have multiple versions available. 25 | Reports are generated at a particular cadence, and can contain "working" versions of an observation schedule that can 26 | change across versions. 27 | The "largest" version alpha-numerically corresponds to the most up-to-date version of the SMS. 28 | This is the version that should be used for comparisons with COS data products. 29 | 30 | .. _sms-database: 31 | 32 | SMS Database 33 | ------------ 34 | Further configuration of the SMS database can be accomplished with the following environment variables (defaults as 35 | comments): 36 | 37 | .. code-block:: 38 | 39 | COSMO_SMS_DB # 'sms.db' 40 | COSMO_SMS_DB_JOURNAL # 'wal' 41 | COSMO_SMS_DB_FORIEGN_KEYS # 1 42 | COSMO_SMS_DB_IGNORE_CHECK_CONSTRAINTS # 0 43 | COSMO_SMS_DB_SYNCHRONOUS # 0 44 | 45 | The first item is the path to the database file, while the remaining options are supported pragma statements. 46 | 47 | The SMS Database itself is comprised of two tables, ``SMSFileStats`` and ``SMSTable``. 48 | 49 | The SMSFileStats table records information about the report files themselves including the SMS ID, Version ID, date of 50 | ingestion and a "FILEID," which is the combination of the SMS ID and Version ID and is used as a reference in SMSTable. 51 | 52 | SMSTable contains the data of the SMS reports. 53 | Each record is identified uniquely based on an exposure key that is generated out of a handful of columns from the 54 | report itself that identifies an exposure (the ROOTNAME cannot be used as this may actually change across SMS versions). 55 | These records are linked to a particular SMS report in the SMSFileStats table. 56 | 57 | SMSFileStats and SMSTable are both ``peewee.Model`` objects that can be used to query SMS data from the database. 58 | 59 | For example: 60 | 61 | .. code-block:: python 62 | 63 | import pandas as pd 64 | from cosmo.sms import SMSFileStats, SMSTable 65 | 66 | # select all sms records 67 | query = SMSFileStats.select() # Returns a query object 68 | 69 | # Put the data in a pandas DataFrame 70 | df = pd.DataFrame(list(query.dicts())) 71 | 72 | # Show data records from a single SMS report using the SMSID 73 | report = SMSFileStats.get(SMSFileStats.SMSID == '123456') 74 | 75 | # Or 76 | report = SMSFileStats.get('123456') # The SMSID is the primary key for SMSFileStats, so this syntax also works 77 | 78 | # Or 79 | report = SMSFileStats['123456'] 80 | 81 | report.exposures # returns a peewee.Query object 82 | 83 | list(report.exposures.dicts()) # returns a list of dictionaries of the table data 84 | 85 | # Query the SMS Data 86 | all_data = SMSTable.select() 87 | 88 | # Query and filter SMS Data 89 | fuv_data = SMSTable.select().where(SMSTable.DETECTOR == 'FUV') 90 | 91 | SMS Ingestion Rules 92 | ------------------- 93 | SMS reports are ingested into the local SMS database, ``sms.db`` using a set of rules to ensure that the correct 94 | information is stored. 95 | 96 | Ingestion rules are as follows: 97 | - If multiple versions of the same SMS are available, the most recent version is used (largest alpha-numeric Version ID). 98 | - If a new version of an SMS is found with a larger Version ID, that new report will supersede the current entry for 99 | that particular SMS. 100 | - If data (identified by the exposure key) from one SMS is also found in another SMS, the data will be ingested from 101 | the SMS report with the larger SMS ID (and therefore more recent), and will replace any existing matching records. 102 | 103 | .. note:: 104 | 105 | These rules ensure that the data in SMSTable is accurate, but can also result in "stale" reports in SMSFileStats if 106 | data occurs in a more recent SMS report. 107 | 108 | It's also possible to have SMSTable records that don't actually correspond to any COS dataset. 109 | This is due to the fact that the SMS reports reflect the observation schedule, which can change a number of times 110 | right up to the planned observation date. 111 | As this happens, ROOTNAME and other exposure information is updated accordingly, and since the ROOTNAME is based on 112 | an incrementing naming system, exposures can be assigned several ROOTNAME until the "final" schedule is established. 113 | Additionally, if an exposure does not execute (for example, when COS enters safe mode), those exposures will still 114 | appear in the SMS. 115 | 116 | Finding and Ingesting SMS Report Files 117 | -------------------------------------- 118 | Reading and ingesting the data of the SMS Report files is done by the ``SMSFile`` class. 119 | SMSFile reads and ingests data from the *txt* files using a series of regular expressions (due to the irregular, human- 120 | readable format) and creates a ``pandas.DataFrame`` to contain the data. 121 | 122 | SMSFile can also ingest the data into the SMS database with the ``ingest_smsfile`` method. 123 | 124 | .. note:: 125 | 126 | If the file already exists in the database, the file will *not* be ingested. 127 | Additionally, the file will be ingested according to the rules described above, and will be ingested (or not) 128 | accordingly. 129 | 130 | The ``SMSFinder`` class can be used for locating the most recent versions of any SMS report found in a given directory. 131 | SMSFinder also classifies the reports that it finds as "new" (not currently in the database) or "old" (currently in the 132 | database), and can ingest all "new" reports into the database. 133 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # For building the GitHub Pages branch 11 | GH_PAGES_SOURCES = source Makefile 12 | GH_PAGES_EXCLUDE = $(BUILDDIR) ../_sources ../_static 13 | 14 | # User-friendly check for sphinx-build 15 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 16 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 17 | endif 18 | 19 | # Internal variables. 20 | PAPEROPT_a4 = -D latex_paper_size=a4 21 | PAPEROPT_letter = -D latex_paper_size=letter 22 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 23 | # the i18n builder cannot share the environment and doctrees with the others 24 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 25 | 26 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 27 | 28 | help: 29 | @echo "Please use \`make ' where is one of" 30 | @echo " html to make standalone HTML files" 31 | @echo " dirhtml to make HTML files named index.html in directories" 32 | @echo " singlehtml to make a single large HTML file" 33 | @echo " pickle to make pickle files" 34 | @echo " json to make JSON files" 35 | @echo " htmlhelp to make HTML files and a HTML help project" 36 | @echo " qthelp to make HTML files and a qthelp project" 37 | @echo " devhelp to make HTML files and a Devhelp project" 38 | @echo " epub to make an epub" 39 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 40 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 41 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 42 | @echo " text to make text files" 43 | @echo " man to make manual pages" 44 | @echo " texinfo to make Texinfo files" 45 | @echo " info to make Texinfo files and run them through makeinfo" 46 | @echo " gettext to make PO message catalogs" 47 | @echo " changes to make an overview of all changed/added/deprecated items" 48 | @echo " xml to make Docutils-native XML files" 49 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 50 | @echo " linkcheck to check all external links for integrity" 51 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 52 | 53 | clean: 54 | rm -rf $(BUILDDIR)/* 55 | 56 | html: 57 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 58 | @echo 59 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 60 | 61 | gh-pages: 62 | make html 63 | git diff --exit-code 64 | git diff --cached --exit-code 65 | git checkout gh-pages 66 | git pull 67 | rm -rf $(GH_PAGES_EXCLUDE) 68 | git checkout master $(GH_PAGES_SOURCES) 69 | git reset HEAD 70 | make html 71 | mv -fv $(BUILDDIR)/html/* ../ 72 | rm -rf $(GH_PAGES_SOURCES) $(BUILDDIR) 73 | # mv -fv .nojekyll ../ 74 | # rm -rf .nojekyll 75 | git add -A 76 | git commit --no-verify -m "Generated gh-pages for `git log master -1 --pretty=short --abbrev-commit`" && git push origin gh-pages ; git checkout master 77 | 78 | gh-pages-clean: 79 | git checkout -- . 80 | rm -rf $(GH_PAGES_SOURCES) build 81 | git checkout master 82 | 83 | dirhtml: 84 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 85 | @echo 86 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 87 | 88 | singlehtml: 89 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 90 | @echo 91 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 92 | 93 | pickle: 94 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 95 | @echo 96 | @echo "Build finished; now you can process the pickle files." 97 | 98 | json: 99 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 100 | @echo 101 | @echo "Build finished; now you can process the JSON files." 102 | 103 | htmlhelp: 104 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 105 | @echo 106 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 107 | ".hhp project file in $(BUILDDIR)/htmlhelp." 108 | 109 | qthelp: 110 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 111 | @echo 112 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 113 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 114 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/smov_cos_redo.qhcp" 115 | @echo "To view the help file:" 116 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/smov_cos_redo.qhc" 117 | 118 | devhelp: 119 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 120 | @echo 121 | @echo "Build finished." 122 | @echo "To view the help file:" 123 | @echo "# mkdir -p $$HOME/.local/share/devhelp/smov_cos_redo" 124 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/smov_cos_redo" 125 | @echo "# devhelp" 126 | 127 | epub: 128 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 129 | @echo 130 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 131 | 132 | latex: 133 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 134 | @echo 135 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 136 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 137 | "(use \`make latexpdf' here to do that automatically)." 138 | 139 | latexpdf: 140 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 141 | @echo "Running LaTeX files through pdflatex..." 142 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 143 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 144 | 145 | latexpdfja: 146 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 147 | @echo "Running LaTeX files through platex and dvipdfmx..." 148 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 149 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 150 | 151 | text: 152 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 153 | @echo 154 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 155 | 156 | man: 157 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 158 | @echo 159 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 160 | 161 | texinfo: 162 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 163 | @echo 164 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 165 | @echo "Run \`make' in that directory to run these through makeinfo" \ 166 | "(use \`make info' here to do that automatically)." 167 | 168 | info: 169 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 170 | @echo "Running Texinfo files through makeinfo..." 171 | make -C $(BUILDDIR)/texinfo info 172 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 173 | 174 | gettext: 175 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 176 | @echo 177 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 178 | 179 | changes: 180 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 181 | @echo 182 | @echo "The overview file is in $(BUILDDIR)/changes." 183 | 184 | linkcheck: 185 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 186 | @echo 187 | @echo "Link check complete; look for any errors in the above output " \ 188 | "or in $(BUILDDIR)/linkcheck/output.txt." 189 | 190 | doctest: 191 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 192 | @echo "Testing of doctests in the sources finished, look at the " \ 193 | "results in $(BUILDDIR)/doctest/output.txt." 194 | 195 | xml: 196 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 197 | @echo 198 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 199 | 200 | pseudoxml: 201 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 202 | @echo 203 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 204 | -------------------------------------------------------------------------------- /tests/data/180147b1.txt: -------------------------------------------------------------------------------- 1 | 2 | COS Exposure Report: SMS 180147B1 3 | 4 | Data Exposure Start FUV Mechanism Positions Cent Tsince Tsince Lamp STIMs Flash 5 | Filename Prop Target PRG OB AL EX Conf Opmode ExpTime yyyy.ddd:hh:mm:ss State Aper OSM1 OSM2 Wave FP OSM1 OSM2 Lamp Curr A B Dop # Dur T 6 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 7 | ldq01kjt 14940 DARK DQ0 1K 01 01 FUV TIME-TAG 1330.0 2018.016:00:00:11 163/163 PSA 4 G130M ----- 1291 0 125111 125111 LOW LOW NO 8 | ldq01ljw 14940 DARK DQ0 1L 01 01 FUV TIME-TAG 1330.0 2018.016:00:29:30 163/163 PSA 4 G130M ----- 1291 0 126870 126870 LOW LOW NO 9 | ldq01mkj 14940 DARK DQ0 1M 01 01 FUV TIME-TAG 1330.0 2018.016:01:04:24 163/163 PSA 4 G130M ----- 1291 0 128964 128964 LOW LOW NO 10 | ldq01nkl 14940 DARK DQ0 1N 01 01 FUV TIME-TAG 1330.0 2018.016:01:46:49 163/163 PSA 4 G130M ----- 1291 0 131509 131509 LOW LOW NO 11 | ldq01ol2 14940 DARK DQ0 1O 01 01 FUV TIME-TAG 1330.0 2018.016:02:20:42 163/163 PSA 4 G130M ----- 1291 0 133542 133542 LOW LOW NO 12 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 13 | lbdodfrp 12046 BDO DF 01 01 2018.017:03:00:28 COS MEMORY COPY -- DCE CODE IMAGE TO CS RAM 14 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 15 | ld6706wp 14684 2DFGRSS394 D67 06 01 01 NUV ACQ/IMAGE 31.0 2018.018:02:04:30 PSA 3 NCM1 MIRRORA 0 0 203 305370 NO 16 | ld6706ws 14684 2DFGRSS394 D67 06 03 01 FUV TIME-TAG 1281.0 2018.018:02:08:53 167/175 PSA 3 G130M ----- 1291 -2 123 305633 LOW LOW NO 3 12 S 17 | ld6706ww 14684 2DFGRSS394 D67 06 05 01 FUV TIME-TAG 1090.0 2018.018:02:32:47 167/175 PSA 3 G130M ----- 1291 -1 1557 307067 LOW LOW NO 2 12 S 18 | ld6706wz 14684 2DFGRSS394 D67 06 07 01 FUV TIME-TAG 1419.0 2018.018:03:37:14 167/175 PSA 3 G130M ----- 1291 0 5424 310934 LOW LOW NO 2 12 S 19 | ld6706x1 14684 2DFGRSS394 D67 06 09 01 FUV TIME-TAG 1419.0 2018.018:04:02:38 167/175 PSA 3 G130M ----- 1291 1 6948 312458 LOW LOW NO 2 12 S 20 | ld6706x3 14684 2DFGRSS394 D67 06 0B 01 FUV TIME-TAG 1419.0 2018.018:05:12:34 167/175 PSA 3 G130M ----- 1327 -2 11144 316654 LOW LOW NO 2 12 S 21 | ld6706x5 14684 2DFGRSS394 D67 06 0D 01 FUV TIME-TAG 1419.0 2018.018:05:37:58 167/175 PSA 3 G130M ----- 1327 -1 12668 318178 LOW LOW NO 2 12 S 22 | ld6706x7 14684 2DFGRSS394 D67 06 0F 01 FUV TIME-TAG 1419.0 2018.018:06:47:54 167/175 PSA 3 G130M ----- 1327 0 16864 322374 LOW LOW NO 2 12 S 23 | ld6706xg 14684 2DFGRSS394 D67 06 0H 01 FUV TIME-TAG 1419.0 2018.018:07:13:18 167/175 PSA 3 G130M ----- 1327 1 18388 323898 LOW LOW NO 2 12 S 24 | ld6706xn 14684 2DFGRSS394 D67 06 0J 01 FUV TIME-TAG 649.0 2018.018:08:23:14 167/175 PSA 3 G130M ----- 1291 -2 22584 328094 LOW LOW NO 1 12 S 25 | ld6706xs 14684 2DFGRSS394 D67 06 0L 01 FUV TIME-TAG 649.0 2018.018:08:35:58 167/175 PSA 3 G130M ----- 1291 -1 23348 328858 LOW LOW NO 1 12 S 26 | ld6706xv 14684 2DFGRSS394 D67 06 0N 01 FUV TIME-TAG 649.0 2018.018:08:48:42 167/175 PSA 3 G130M ----- 1291 0 24112 329622 LOW LOW NO 1 12 S 27 | ld6706yg 14684 2DFGRSS394 D67 06 0P 01 FUV TIME-TAG 649.0 2018.018:09:01:26 167/175 PSA 3 G130M ----- 1291 1 24876 330386 LOW LOW NO 1 12 S 28 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 29 | ldgt01as 15084 Q0025-4047 DGT 01 01 01 NUV ACQ/IMAGE 98.0 2018.018:22:35:32 PSA 4 NCM1 MIRRORB 0 0 346 268 NO 30 | ldgt01av 15084 Q0025-4047 DGT 01 03 01 FUV TIME-TAG 1144.0 2018.018:22:41:32 163/100 PSA 4 G140L ----- 1105 -2 129 628 LOW OFF NO 3 7 S 31 | ldgt01az 15084 Q0025-4047 DGT 01 05 01 FUV TIME-TAG 1144.0 2018.018:23:03:09 163/100 PSA 4 G140L ----- 1105 -1 1426 1925 LOW OFF NO 2 7 S 32 | ldgt01bb 15084 Q0025-4047 DGT 01 07 01 FUV TIME-TAG 1426.0 2018.019:00:07:27 163/100 PSA 4 G140L ----- 1105 0 5284 5783 LOW OFF NO 2 7 S 33 | ldgt01bk 15084 Q0025-4047 DGT 01 09 01 FUV TIME-TAG 1426.0 2018.019:00:33:52 163/100 PSA 4 G140L ----- 1105 0 6869 7368 LOW OFF NO 2 7 S 34 | ldgt01cm 15084 Q0025-4047 DGT 01 0B 01 FUV TIME-TAG 1425.0 2018.019:01:42:46 163/100 PSA 4 G140L ----- 1105 1 11003 11502 LOW OFF NO 2 7 S 35 | ldgt01ct 15084 Q0025-4047 DGT 01 0D 01 FUV TIME-TAG 1425.0 2018.019:02:09:10 163/100 PSA 4 G140L ----- 1105 1 12587 13086 LOW OFF NO 2 7 S 36 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 37 | ldi405e3 15174 GJ436 DI4 05 01 01 NUV ACQ/PEAKXD 425.2 2018.019:09:08:08 PSA 4 NCM1 G230L 2950 0 241 136 NO 38 | ldi405e4 15174 GJ436 DI4 05 02 01 NUV ACQ/PEAKD 34.1 2018.019:09:16:46 PSA 4 NCM1 G230L 3360 0 759 654 DWELLS=3 STEPSIZE=1300 FLUXWT 39 | ldi405e6 15174 GJ436 DI4 05 04 01 FUV TIME-TAG 1850.0 2018.019:10:24:56 163/163 PSA 4 G130M ----- 1291 0 123 4744 LOW LOW NO 3 12 S 40 | ldi406ev 15174 GJ436 DI4 06 01 01 NUV ACQ/PEAKXD 425.2 2018.019:15:12:19 PSA 4 NCM1 G230L 2950 0 241 136 NO 41 | ldi406ew 15174 GJ436 DI4 06 02 01 NUV ACQ/PEAKD 34.1 2018.019:15:20:57 PSA 4 NCM1 G230L 3360 0 759 654 DWELLS=3 STEPSIZE=1300 FLUXWT 42 | ldi406ey 15174 GJ436 DI4 06 04 01 FUV TIME-TAG 1850.0 2018.019:15:27:48 163/163 PSA 4 G130M ----- 1291 0 123 1065 LOW LOW NO 3 12 S 43 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 44 | ldrm02fb 15448 NGC-863 DRM 02 01 01 NUV ACQ/IMAGE 140.0 2018.019:20:10:42 PSA 4 NCM1 MIRRORB 0 0 388 310 NO 45 | ldrm02fd 15448 NGC-863 DRM 02 03 01 FUV TIME-TAG 430.0 2018.019:20:58:46 163/100 PSA 4 G140L ----- 1105 -2 129 3194 LOW OFF NO 2 7 S 46 | ldrm02ff 15448 NGC-863 DRM 02 05 01 FUV TIME-TAG 430.0 2018.019:21:07:41 163/100 PSA 4 G140L ----- 1105 -1 664 3729 LOW OFF NO 2 7 S 47 | ldrm02fh 15448 NGC-863 DRM 02 07 01 FUV TIME-TAG 430.0 2018.019:21:16:36 163/100 PSA 4 G140L ----- 1105 0 1199 4264 LOW OFF NO 2 7 S 48 | ldrm02fj 15448 NGC-863 DRM 02 09 01 FUV TIME-TAG 430.0 2018.019:21:25:31 163/100 PSA 4 G140L ----- 1105 1 1734 4799 LOW OFF NO 2 7 S 49 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 50 | ldi407fl 15174 GJ436 DI4 07 01 01 NUV ACQ/PEAKXD 425.2 2018.019:23:08:59 PSA 4 NCM1 G230L 2950 0 241 136 NO 51 | ldi407fm 15174 GJ436 DI4 07 02 01 NUV ACQ/PEAKD 34.1 2018.019:23:17:37 PSA 4 NCM1 G230L 3360 0 759 654 DWELLS=3 STEPSIZE=1300 FLUXWT 52 | ldi407fo 15174 GJ436 DI4 07 04 01 FUV TIME-TAG 1850.0 2018.019:23:24:28 163/163 PSA 4 G130M ----- 1291 0 123 1065 LOW LOW NO 3 12 S 53 | ldi408if 15174 GJ436 DI4 08 01 01 NUV ACQ/PEAKXD 425.2 2018.020:13:49:43 PSA 4 NCM1 G230L 2950 0 241 136 NO 54 | ldi408ig 15174 GJ436 DI4 08 02 01 NUV ACQ/PEAKD 34.1 2018.020:13:58:21 PSA 4 NCM1 G230L 3360 0 759 654 DWELLS=3 STEPSIZE=1300 FLUXWT 55 | ldi408ii 15174 GJ436 DI4 08 04 01 FUV TIME-TAG 1850.0 2018.020:15:00:54 163/163 PSA 4 G130M ----- 1291 0 123 4407 LOW LOW NO 3 12 S 56 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 57 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # smov_cos_redo documentation build configuration file, created by 4 | # sphinx-quickstart on Tue Dec 8 10:53:41 2015. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | # If extensions (or modules to document with autodoc) are in another directory, 16 | # add these directories to sys.path here. If the directory is relative to the 17 | # documentation root, use os.path.abspath to make it absolute, like shown here. 18 | # -- General configuration ------------------------------------------------ 19 | 20 | # If your documentation needs a minimal Sphinx version, state it here. 21 | # needs_sphinx = '1.0' 22 | 23 | # Add any Sphinx extension module names here, as strings. They can be 24 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 25 | # ones. 26 | extensions = [ 27 | 'sphinx.ext.autodoc', 28 | 'sphinx.ext.autosummary', 29 | 'sphinx.ext.viewcode', 30 | 'sphinx.ext.autosectionlabel', 31 | ] 32 | 33 | # Add any paths that contain templates here, relative to this directory. 34 | templates_path = ['_templates'] 35 | 36 | # The suffix of source filenames. 37 | source_suffix = '.rst' 38 | 39 | # The encoding of source files. 40 | # source_encoding = 'utf-8-sig' 41 | 42 | # The master toctree document. 43 | master_doc = 'index' 44 | 45 | # General information about the project. 46 | project = u'COSMO' 47 | # noinspection PyShadowingBuiltins 48 | copyright = u'2015, Association of Universities for Research in Astronomy.' 49 | 50 | # The version info for the project you're documenting, acts as replacement for 51 | # |version| and |release|, also used in various other places throughout the 52 | # built documents. 53 | # 54 | # The short X.Y version. 55 | version = '1.1.0' 56 | # The full version, including alpha/beta/rc tags. 57 | release = '1.1.0' 58 | 59 | # The language for content autogenerated by Sphinx. Refer to documentation 60 | # for a list of supported languages. 61 | # language = None 62 | 63 | # There are two options for replacing |today|: either, you set today to some 64 | # non-false value, then it is used: 65 | # today = '' 66 | # Else, today_fmt is used as the format for a strftime call. 67 | # today_fmt = '%B %d, %Y' 68 | 69 | # List of patterns, relative to source directory, that match files and 70 | # directories to ignore when looking for source files. 71 | exclude_patterns = ['_build'] 72 | 73 | # The reST default role (used for this markup: `text`) to use for all 74 | # documents. 75 | # default_role = None 76 | 77 | # If true, '()' will be appended to :func: etc. cross-reference text. 78 | # add_function_parentheses = True 79 | 80 | # If true, the current module name will be prepended to all description 81 | # unit titles (such as .. function::). 82 | # add_module_names = True 83 | 84 | # If true, sectionauthor and moduleauthor directives will be shown in the 85 | # output. They are ignored by default. 86 | # show_authors = False 87 | 88 | # The name of the Pygments (syntax highlighting) style to use. 89 | pygments_style = 'sphinx' 90 | 91 | # A list of ignored prefixes for module index sorting. 92 | # modindex_common_prefix = [] 93 | 94 | # If true, keep warnings as "system message" paragraphs in the built documents. 95 | # keep_warnings = False 96 | 97 | 98 | # -- Options for HTML output ---------------------------------------------- 99 | 100 | # The theme to use for HTML and HTML Help pages. See the documentation for 101 | # a list of builtin themes. 102 | html_theme = 'sphinx_rtd_theme' 103 | 104 | # Theme options are theme-specific and customize the look and feel of a theme 105 | # further. For a list of options available for each theme, see the 106 | # documentation. 107 | # html_theme_options = {} 108 | 109 | # Add any paths that contain custom themes here, relative to this directory. 110 | # html_theme_path = [] 111 | 112 | # The name for this set of Sphinx documents. If None, it defaults to 113 | # " v documentation". 114 | # html_title = None 115 | 116 | # A shorter title for the navigation bar. Default is the same as html_title. 117 | # html_short_title = None 118 | 119 | # The name of an image file (relative to this directory) to place at the top 120 | # of the sidebar. 121 | # html_logo = None 122 | 123 | # The name of an image file (within the static path) to use as favicon of the 124 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 125 | # pixels large. 126 | # html_favicon = None 127 | 128 | # Add any paths that contain custom static files (such as style sheets) here, 129 | # relative to this directory. They are copied after the builtin static files, 130 | # so a file named "default.css" will overwrite the builtin "default.css". 131 | html_static_path = ['_static'] 132 | 133 | # Add any extra paths that contain custom files (such as robots.txt or 134 | # .htaccess) here, relative to this directory. These files are copied 135 | # directly to the root of the documentation. 136 | # html_extra_path = [] 137 | 138 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 139 | # using the given strftime format. 140 | # html_last_updated_fmt = '%b %d, %Y' 141 | 142 | # If true, SmartyPants will be used to convert quotes and dashes to 143 | # typographically correct entities. 144 | # html_use_smartypants = True 145 | 146 | # Custom sidebar templates, maps document names to template names. 147 | # html_sidebars = {} 148 | 149 | # Additional templates that should be rendered to pages, maps page names to 150 | # template names. 151 | # html_additional_pages = {} 152 | 153 | # If false, no module index is generated. 154 | # html_domain_indices = True 155 | 156 | # If false, no index is generated. 157 | # html_use_index = True 158 | 159 | # If true, the index is split into individual pages for each letter. 160 | # html_split_index = False 161 | 162 | # If true, links to the reST sources are added to the pages. 163 | # html_show_sourcelink = True 164 | 165 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 166 | # html_show_sphinx = True 167 | 168 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 169 | # html_show_copyright = True 170 | 171 | # If true, an OpenSearch description file will be output, and all pages will 172 | # contain a tag referring to it. The value of this option must be the 173 | # base URL from which the finished HTML is served. 174 | # html_use_opensearch = '' 175 | 176 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 177 | # html_file_suffix = None 178 | 179 | # Output file base name for HTML help builder. 180 | htmlhelp_basename = 'cosmo_redodoc' 181 | 182 | # -- Options for LaTeX output --------------------------------------------- 183 | 184 | latex_elements = { 185 | # The paper size ('letterpaper' or 'a4paper'). 186 | # 'papersize': 'letterpaper', 187 | 188 | # The font size ('10pt', '11pt' or '12pt'). 189 | # 'pointsize': '10pt', 190 | 191 | # Additional stuff for the LaTeX preamble. 192 | # 'preamble': '', 193 | } 194 | 195 | # Grouping the document tree into LaTeX files. List of tuples 196 | # (source start file, target name, title, 197 | # author, documentclass [howto, manual, or own class]). 198 | latex_documents = [ 199 | ('index', 'cosmo_redo.tex', u'cosmo\\_redo Documentation', 200 | u'Justin Ely, Jo Taylor, Mees Fix', 'manual'), 201 | ] 202 | 203 | # The name of an image file (relative to this directory) to place at the top of 204 | # the title page. 205 | # latex_logo = None 206 | 207 | # For "manual" documents, if this is true, then toplevel headings are parts, 208 | # not chapters. 209 | # latex_use_parts = False 210 | 211 | # If true, show page references after internal links. 212 | # latex_show_pagerefs = False 213 | 214 | # If true, show URL addresses after external links. 215 | # latex_show_urls = False 216 | 217 | # Documents to append as an appendix to all manuals. 218 | # latex_appendices = [] 219 | 220 | # If false, no module index is generated. 221 | # latex_domain_indices = True 222 | 223 | 224 | # -- Options for manual page output --------------------------------------- 225 | 226 | # One entry per manual page. List of tuples 227 | # (source start file, name, description, authors, manual section). 228 | man_pages = [ 229 | ('index', 'cosmo_redo', u'cosmo_redo Documentation', 230 | [u'Justin Ely, Jo Taylor, Mees Fix'], 1) 231 | ] 232 | 233 | # If true, show URL addresses after external links. 234 | # man_show_urls = False 235 | 236 | 237 | # -- Options for Texinfo output ------------------------------------------- 238 | 239 | # Grouping the document tree into Texinfo files. List of tuples 240 | # (source start file, target name, title, author, 241 | # dir menu entry, description, category) 242 | texinfo_documents = [ 243 | ('index', 'cosmo_redo', u'cosmo_redo Documentation', 244 | u'Justin Ely, Jo Taylor, Mees Fix', 'cosmo_redo', 'One line description of project.', 245 | 'Miscellaneous'), 246 | ] 247 | 248 | # Documents to append as an appendix to all manuals. 249 | # texinfo_appendices = [] 250 | 251 | # If false, no module index is generated. 252 | # texinfo_domain_indices = True 253 | 254 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 255 | # texinfo_show_urls = 'footnote' 256 | 257 | # If true, do not generate a @detailmenu in the "Top" node's menu. 258 | # texinfo_no_detailmenu = False 259 | -------------------------------------------------------------------------------- /cosmo/monitors/data_models.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | import os 4 | from glob import glob 5 | 6 | from typing import List 7 | from monitorframe.datamodel import BaseDataModel 8 | from peewee import OperationalError 9 | 10 | from ..filesystem import find_files, data_from_exposures, data_from_jitters 11 | from ..sms import SMSTable 12 | from .. import SETTINGS 13 | 14 | FILES_SOURCE = SETTINGS['filesystem']['source'] 15 | PROGRAMS = SETTINGS['dark_programs'] 16 | 17 | 18 | def dgestar_to_fgs(results: List[dict]) -> None: 19 | """Add a FGS key to each row dictionary.""" 20 | for item in results: 21 | item.update({'FGS': item['DGESTAR'][-2:]}) # The dominant guide star key is the last 2 values in the string 22 | 23 | 24 | class AcqDataModel(BaseDataModel): 25 | """Datamodel for Acq files.""" 26 | files_source = FILES_SOURCE 27 | subdir_pattern = '?????' 28 | primary_key = 'ROOTNAME' 29 | 30 | def get_new_data(self): 31 | header_request = { 32 | 0: [ 33 | 'ACQSLEWX', 34 | 'ACQSLEWY', 35 | 'ROOTNAME', 36 | 'PROPOSID', 37 | 'OBSTYPE', 38 | 'SHUTTER', 39 | 'LAMPEVNT', 40 | 'ACQSTAT', 41 | 'EXTENDED', 42 | 'LINENUM', 43 | 'APERTURE', 44 | 'OPT_ELEM', 45 | 'LIFE_ADJ', 46 | 'CENWAVE', 47 | 'DETECTOR', 48 | 'EXPTYPE' 49 | ], 50 | 1: ['EXPSTART', 'NEVENTS'] 51 | } 52 | 53 | # Different ACQ types may not have the full set 54 | header_defaults = {'ACQSLEWX': 0.0, 'ACQSLEWY': 0.0, 'NEVENTS': 0.0, 'LAMPEVNT': 0.0} 55 | 56 | # SPT file header keys, extensions 57 | spt_header_request = {0: ['DGESTAR']} 58 | 59 | files = find_files('*rawacq*', data_dir=self.files_source, subdir_pattern=self.subdir_pattern) 60 | 61 | if self.model is not None: 62 | currently_ingested = [item.FILENAME for item in self.model.select(self.model.FILENAME)] 63 | 64 | ##finds index values for all files that are in the currently_ingested list 65 | ##should remove any repeated instances of the same file (e.g., fits and fits.gz) 66 | del_list = [] 67 | for file in currently_ingested: 68 | indx = file.index('.fits') 69 | prefix = file[:indx] 70 | for i, f in enumerate(files): 71 | if(prefix in f): 72 | del_list.append(i) 73 | 74 | for index in sorted(del_list, reverse=True): 75 | del files[index] 76 | 77 | 78 | if not files: # No new files 79 | return pd.DataFrame() 80 | 81 | data_results = data_from_exposures( 82 | files, 83 | header_request=header_request, 84 | header_defaults=header_defaults, 85 | spt_header_request=spt_header_request, 86 | ) 87 | 88 | dgestar_to_fgs(data_results) 89 | 90 | return data_results 91 | 92 | 93 | class OSMDataModel(BaseDataModel): 94 | """Data model for all OSM Shift monitors.""" 95 | files_source = FILES_SOURCE 96 | subdir_pattern = '?????' 97 | 98 | cosmo_layout = True 99 | 100 | primary_key = 'ROOTNAME' 101 | 102 | def get_new_data(self): 103 | """Retrieve data.""" 104 | header_request = { 105 | 0: ['ROOTNAME', 'DETECTOR', 'LIFE_ADJ', 'OPT_ELEM', 'CENWAVE', 'FPPOS', 'PROPOSID', 'OBSET_ID'], 106 | 1: ['EXPSTART'] 107 | } 108 | 109 | table_request = {1: ['TIME', 'SHIFT_DISP', 'SHIFT_XDISP', 'SEGMENT']} 110 | 111 | reference_request = { 112 | 'LAMPTAB': { 113 | 'match_keys': ['OPT_ELEM', 'CENWAVE', 'FPOFFSET'], 114 | 'table_request': {1: ['SEGMENT', 'FP_PIXEL_SHIFT']}, 115 | }, 116 | 'WCPTAB': {'match_keys': ['OPT_ELEM'], 'table_request': {1: ['XC_RANGE', 'SEARCH_OFFSET']}} 117 | } 118 | 119 | files = find_files('*lampflash*', data_dir=self.files_source, subdir_pattern=self.subdir_pattern) 120 | 121 | if self.model is not None: 122 | currently_ingested = [item.FILENAME for item in self.model.select(self.model.FILENAME)] 123 | 124 | ##finds index values for all files that are in the currently_ingested list 125 | ##should remove any repeated instances of the same file (e.g., fits and fits.gz) 126 | del_list = [] 127 | for file in currently_ingested: 128 | indx = file.index('.fits') 129 | prefix = file[:indx] 130 | for i, f in enumerate(files): 131 | if(prefix in f): 132 | del_list.append(i) 133 | 134 | for index in sorted(del_list, reverse=True): 135 | del files[index] 136 | 137 | if not files: # No new files 138 | return pd.DataFrame() 139 | 140 | data_results = pd.DataFrame( 141 | data_from_exposures( 142 | files, 143 | header_request=header_request, 144 | table_request=table_request, 145 | reference_request=reference_request 146 | ) 147 | ) 148 | 149 | # Remove any rows that have empty data columns 150 | data_results = data_results.drop( 151 | data_results[data_results.apply(lambda x: not bool(len(x.SHIFT_DISP)), axis=1)].index.values 152 | ).reset_index(drop=True) 153 | 154 | # Add tsince data from SMSTable. 155 | try: 156 | sms_data = pd.DataFrame( 157 | SMSTable.select(SMSTable.ROOTNAME, SMSTable.TSINCEOSM1, SMSTable.TSINCEOSM2).where( 158 | # x << y -> x IN y (y must be a list) 159 | SMSTable.ROOTNAME + 'q' << data_results.ROOTNAME.to_list()).dicts() 160 | ) 161 | 162 | except OperationalError as e: 163 | raise type(e)(str(e) + '\nSMS database is required.') 164 | 165 | # It's possible that there could be a lag in between when the SMS data is updated and when new lampflashes 166 | # are added. 167 | # Returning the empty data frame ensures that only files with a match in the SMS data are added... 168 | # This may not be the best idea 169 | if sms_data.empty: 170 | return sms_data 171 | 172 | # Need to add the 'q' at the end of the rootname.. For some reason those are missing from the SMS rootnames 173 | sms_data.ROOTNAME += 'q' 174 | 175 | # Combine the data from the files with the data from the SMS table with an inner merge between the two. 176 | # NOTE: this means that if a file does not have a corresponding entry in the SMSTable, it will not be in the 177 | # dataset used for monitoring. 178 | merged = pd.merge(data_results, sms_data, on='ROOTNAME') 179 | 180 | return merged 181 | 182 | 183 | class JitterDataModel(BaseDataModel): 184 | files_source = FILES_SOURCE 185 | subdir_pattern = '?????' 186 | 187 | def get_new_data(self): 188 | primary_header_keys = ('PROPOSID', 'CONFIG') 189 | extension_header_keys = ('EXPNAME',) 190 | 191 | data_keys = ('SI_V2_AVG', 'SI_V3_AVG') 192 | reduce = {'SI_V2_AVG': ('mean', 'std', 'max'), 'SI_V3_AVG': ('mean', 'std', 'max')} 193 | 194 | files = find_files('*jit*', data_dir=self.files_source, subdir_pattern=self.subdir_pattern) 195 | 196 | if self.model is not None: 197 | currently_ingested = [item.FILENAME for item in self.model.select(self.model.FILENAME)] 198 | 199 | for file in currently_ingested: 200 | files.remove(file) 201 | 202 | if not files: # No new files 203 | return pd.DataFrame() 204 | 205 | data_results = pd.DataFrame( 206 | data_from_jitters( 207 | files, 208 | primary_header_keys, 209 | extension_header_keys, 210 | data_keys, 211 | reduce_to_stats=reduce 212 | ) 213 | ) 214 | 215 | # Remove any NaNs or inf that may occur from the statistics calculations. 216 | data_results = data_results.replace([np.inf, -np.inf], np.nan).dropna().reset_index(drop=True) 217 | 218 | return data_results[~data_results.EXPTYPE.str.contains('ACQ|DARK|FLAT')] 219 | 220 | 221 | def get_program_ids(pid_file): 222 | """Retrieve the program IDs from the given text file.""" 223 | programs_df = pd.read_csv(pid_file, delim_whitespace=True) 224 | all_programs = [] 225 | for col, col_data in programs_df.iteritems(): 226 | all_programs += col_data.to_numpy(dtype=str).tolist() 227 | 228 | return all_programs 229 | 230 | class DarkDataModel(BaseDataModel): 231 | """DataModel for dark corrtag files.""" 232 | cosmo_layout = False 233 | files_source = FILES_SOURCE 234 | 235 | def get_new_data(self): 236 | """Set the model for what data is to be retrieved from each dark 237 | file.""" 238 | # this way when you get new data it will get all the data 239 | header_request = { 240 | 0: ['ROOTNAME', 'SEGMENT'], 1: ['EXPTIME', 'EXPSTART'] 241 | } 242 | table_request = { 243 | 1: ['PHA', 'XCORR', 'YCORR', 'TIME'], 244 | 3: ['TIME', 'LATITUDE', 'LONGITUDE'] 245 | } 246 | 247 | files = [] 248 | 249 | program_ids = get_program_ids(PROGRAMS) 250 | 251 | for prog_id in program_ids: 252 | new_files_source = os.path.join(FILES_SOURCE, prog_id) 253 | files += find_files('*corrtag*', data_dir=new_files_source) 254 | 255 | if self.model is not None: 256 | currently_ingested = [item.FILENAME for item in 257 | self.model.select(self.model.FILENAME)] 258 | 259 | for file in currently_ingested: 260 | files.remove(file) 261 | 262 | if not files: # No new files 263 | return pd.DataFrame() 264 | 265 | data_results = data_from_exposures(files, 266 | header_request=header_request, 267 | table_request=table_request) 268 | 269 | return data_results 270 | -------------------------------------------------------------------------------- /tests/data/162567b1.txt: -------------------------------------------------------------------------------- 1 | 2 | COS Exposure Report: SMS 162567B1 3 | 4 | Data Exposure Start FUV Mechanism Positions Cent Tsince Tsince Lamp STIMs Flash 5 | Filename Prop Target PRG OB AL EX Conf Opmode ExpTime yyyy.ddd:hh:mm:ss State Aper OSM1 OSM2 Wave FP OSM1 OSM2 Lamp Curr A B Dop # Dur T 6 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 7 | ld1ce4dk 14429 ANY D1C E4 04 07 FUV TIME-TAG 2900.0 2016.256:09:27:44 167/169 PSA G130M ----- 1291 -1 34064 34064 LOW LOW NO 2 12 S 8 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 9 | ldao23dw 14597 WD2132+096 DAO 23 01 01 NUV ACQ/IMAGE 32.0 2016.256:11:03:59 PSA NCM1 MIRRORB 0 0 285 201 NO 10 | ldao23dy 14597 WD2132+096 DAO 23 03 01 FUV TIME-TAG 2507.0 2016.256:11:08:45 167/169 PSA G130M ----- 1291 -2 122 487 LOW LOW NO 3 12 S 11 | ldao23e8 14597 WD2132+096 DAO 23 05 01 FUV TIME-TAG 900.0 2016.256:12:37:10 167/169 PSA G130M ----- 1291 -2 5427 5792 LOW LOW NO 2 12 S 12 | ldao23ea 14597 WD2132+096 DAO 23 07 01 FUV TIME-TAG 1912.0 2016.256:12:54:05 167/169 PSA G130M ----- 1291 -1 6442 6807 LOW LOW NO 2 12 S 13 | ldao23ei 14597 WD2132+096 DAO 23 09 01 FUV TIME-TAG 1550.0 2016.256:14:12:35 167/169 PSA G130M ----- 1291 -1 11152 11517 LOW LOW NO 2 12 S 14 | ldao23ek 14597 WD2132+096 DAO 23 0B 01 FUV TIME-TAG 1272.0 2016.256:14:40:10 167/169 PSA G130M ----- 1291 0 12807 13172 LOW LOW NO 2 12 S 15 | ldao23es 14597 WD2132+096 DAO 23 0D 01 FUV TIME-TAG 2250.0 2016.256:15:48:00 167/169 PSA G130M ----- 1291 0 16877 17242 LOW LOW NO 2 12 S 16 | ldao23eu 14597 WD2132+096 DAO 23 0F 01 FUV TIME-TAG 572.0 2016.256:16:27:15 167/169 PSA G130M ----- 1291 1 19232 19597 LOW LOW NO 1 12 S 17 | ldao23ew 14597 WD2132+096 DAO 23 0H 01 FUV TIME-TAG 2927.0 2016.256:17:23:25 167/169 PSA G130M ----- 1291 1 22602 22967 LOW LOW NO 2 12 S 18 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 19 | lda930f7 14675 BI253 DA9 30 01 01 NUV ACQ/IMAGE 40.0 2016.256:21:08:23 BOA NCM1 MIRRORA 0 0 222 10298 NO 20 | lda930f9 14675 BI253 DA9 30 03 01 FUV TIME-TAG 150.0 2016.256:21:13:44 167/169 PSA G160M ----- 1589 -2 125 10619 LOW LOW NO 1 12 S 21 | lda930fb 14675 BI253 DA9 30 05 01 FUV TIME-TAG 150.0 2016.256:21:18:28 167/169 PSA G160M ----- 1589 -1 409 10903 LOW LOW NO 1 12 S 22 | lda930fd 14675 BI253 DA9 30 07 01 FUV TIME-TAG 150.0 2016.256:21:23:12 167/169 PSA G160M ----- 1589 0 693 11187 LOW LOW NO 1 12 S 23 | lda930ff 14675 BI253 DA9 30 09 01 FUV TIME-TAG 150.0 2016.256:21:27:56 167/169 PSA G160M ----- 1589 1 977 11471 LOW LOW NO 1 12 S 24 | lda930fh 14675 BI253 DA9 30 0B 01 FUV TIME-TAG 150.0 2016.256:22:46:15 167/169 PSA G130M ----- 1291 -2 125 16170 LOW LOW NO 1 12 S 25 | lda930fj 14675 BI253 DA9 30 0D 01 FUV TIME-TAG 150.0 2016.256:22:51:18 167/169 PSA G130M ----- 1291 -1 428 16473 LOW LOW NO 1 12 S 26 | lda930fl 14675 BI253 DA9 30 0F 01 FUV TIME-TAG 120.0 2016.256:22:56:21 167/169 PSA G130M ----- 1291 0 731 16776 LOW LOW NO 1 12 S 27 | lda930fn 14675 BI253 DA9 30 0H 01 FUV TIME-TAG 120.0 2016.256:23:00:44 167/169 PSA G130M ----- 1291 1 994 17039 LOW LOW NO 1 12 S 28 | lda935fv 14675 SK-6826 DA9 35 01 01 FUV ACQ/PEAKXD 1.0 2016.257:00:22:29 167/169 PSA G160M ----- 1589 0 146 21944 OFF OFF NO 29 | lda935fw 14675 SK-6826 DA9 35 02 01 FUV ACQ/PEAKD 1.0 2016.257:00:23:10 167/169 PSA G160M ----- 1589 0 187 21985 DWELLS=5 STEPSIZE= 900 FLUXFLR 30 | lda935fz 14675 SK-6826 DA9 35 04 01 FUV TIME-TAG 150.0 2016.257:00:28:27 167/169 PSA G160M ----- 1589 -2 504 22302 LOW LOW NO 1 12 S 31 | lda935g2 14675 SK-6826 DA9 35 06 01 FUV TIME-TAG 150.0 2016.257:00:33:20 167/169 PSA G160M ----- 1589 -1 797 22595 LOW LOW NO 1 12 S 32 | lda935g4 14675 SK-6826 DA9 35 08 01 FUV TIME-TAG 150.0 2016.257:00:38:13 167/169 PSA G160M ----- 1589 0 1090 22888 LOW LOW NO 1 12 S 33 | lda935gd 14675 SK-6826 DA9 35 0A 01 FUV TIME-TAG 150.0 2016.257:01:26:13 167/169 PSA G160M ----- 1589 1 3970 25768 LOW LOW NO 1 12 S 34 | lda935gg 14675 SK-6826 DA9 35 0C 01 FUV TIME-TAG 110.0 2016.257:02:01:11 167/169 PSA G130M ----- 1291 -2 125 27866 LOW LOW NO 1 12 S 35 | lda935gi 14675 SK-6826 DA9 35 0E 01 FUV TIME-TAG 110.0 2016.257:02:05:15 167/169 PSA G130M ----- 1291 -1 369 28110 LOW LOW NO 1 12 S 36 | lda935h7 14675 SK-6826 DA9 35 0G 01 FUV TIME-TAG 110.0 2016.257:02:09:19 167/169 PSA G130M ----- 1291 0 613 28354 LOW LOW NO 1 12 S 37 | lda935h9 14675 SK-6826 DA9 35 0I 01 FUV TIME-TAG 110.0 2016.257:02:13:23 167/169 PSA G130M ----- 1291 1 857 28598 LOW LOW NO 1 12 S 38 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 39 | lbdobfih 12046 BDO BF 01 01 2016.257:14:00:00 COS MEMORY COPY -- DCE CODE IMAGE TO CS RAM 40 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 41 | lcwo03xj 14247 CXO-J03383 CWO 03 01 01 NUV ACQ/IMAGE 400.0 2016.260:10:48:08 PSA NCM1 MIRRORA 0 0 577 318683 NO 42 | lcwo03xl 14247 CXO-J03383 CWO 03 03 01 FUV TIME-TAG 846.0 2016.260:10:59:01 167/100 PSA G140L ----- 1105 -2 134 319336 LOW OFF NO 2 7 S 43 | lcwo03xs 14247 CXO-J03383 CWO 03 05 01 FUV TIME-TAG 846.0 2016.260:11:15:02 167/100 PSA G140L ----- 1105 -1 1095 320297 LOW OFF NO 2 7 S 44 | lcwo03xv 14247 CXO-J03383 CWO 03 07 01 FUV TIME-TAG 1438.0 2016.260:12:13:54 167/100 PSA G140L ----- 1105 0 4627 323829 LOW OFF NO 2 7 S 45 | lcwo03y2 14247 CXO-J03383 CWO 03 09 01 FUV TIME-TAG 1438.0 2016.260:12:39:37 167/100 PSA G140L ----- 1105 1 6170 325372 LOW OFF NO 2 7 S 46 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 47 | ld0w1bza 14442 DARK D0W 1B 01 01 NUV TIME-TAG 1330.0 2016.261:00:00:12 PSA G130M MIRRORA 0 0 39177 366207 NO 48 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 49 | ld0x6fa5 14436 DARK D0X 6F 01 01 FUV TIME-TAG 1330.0 2016.261:01:17:47 167/169 PSA G130M ----- 1309 0 43832 370862 LOW LOW NO 50 | ld0x6gax 14436 DARK D0X 6G 01 01 FUV TIME-TAG 1330.0 2016.261:01:50:58 167/169 PSA G130M ----- 1309 0 45823 372853 LOW LOW NO 51 | ld0x6hb7 14436 DARK D0X 6H 01 01 FUV TIME-TAG 1330.0 2016.261:02:55:46 167/169 PSA G130M ----- 1309 0 49711 376741 LOW LOW NO 52 | ld0x6ib9 14436 DARK D0X 6I 01 01 FUV TIME-TAG 1330.0 2016.261:03:25:05 167/169 PSA G130M ----- 1309 0 51470 378500 LOW LOW NO 53 | ld0x6jbm 14436 DARK D0X 6J 01 01 FUV TIME-TAG 1330.0 2016.261:04:30:14 167/169 PSA G130M ----- 1309 0 55379 382409 LOW LOW NO 54 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 55 | ld0w1ccw 14442 DARK D0W 1C 01 01 NUV TIME-TAG 1330.0 2016.261:08:44:32 PSA G130M MIRRORA 0 0 70637 397667 NO 56 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 57 | lcwo04h9 14247 CXO-J03383 CWO 04 01 01 NUV ACQ/IMAGE 400.0 2016.262:04:08:20 PSA NCM1 MIRRORA 0 0 577 467495 NO 58 | lcwo04hb 14247 CXO-J03383 CWO 04 03 01 FUV TIME-TAG 846.0 2016.262:04:19:13 167/100 PSA G140L ----- 1105 -2 134 468148 LOW OFF NO 2 7 S 59 | lcwo04hi 14247 CXO-J03383 CWO 04 05 01 FUV TIME-TAG 846.0 2016.262:04:35:14 167/100 PSA G140L ----- 1105 -1 1095 469109 LOW OFF NO 2 7 S 60 | lcwo04hl 14247 CXO-J03383 CWO 04 07 01 FUV TIME-TAG 1438.0 2016.262:05:33:57 167/100 PSA G140L ----- 1105 0 4618 472632 LOW OFF NO 2 7 S 61 | lcwo04ht 14247 CXO-J03383 CWO 04 09 01 FUV TIME-TAG 1438.0 2016.262:05:59:40 167/100 PSA G140L ----- 1105 1 6161 474175 LOW OFF NO 2 7 S 62 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 63 | lcvj12hw 14077 WD2350-248 CVJ 12 01 01 FUV ACQ/PEAKXD 1.0 2016.262:07:02:54 167/169 PSA G130M ----- 1291 0 2136 477969 OFF OFF NO 64 | lcvj12hx 14077 WD2350-248 CVJ 12 02 01 FUV ACQ/PEAKD 1.0 2016.262:07:03:41 167/169 PSA G130M ----- 1291 0 2183 478016 DWELLS=5 STEPSIZE= 900 FLUXFLR 65 | lcvj12hz 14077 WD2350-248 CVJ 12 04 01 FUV TIME-TAG 450.0 2016.262:07:08:58 167/169 PSA G130M ----- 1291 -2 2500 478333 LOW LOW NO 1 12 S 66 | lcvj12i1 14077 WD2350-248 CVJ 12 06 01 FUV TIME-TAG 450.0 2016.262:07:19:01 167/169 PSA G130M ----- 1291 1 3103 478936 LOW LOW NO 1 12 S 67 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 68 | -------------------------------------------------------------------------------- /tests/data/153067a8.txt: -------------------------------------------------------------------------------- 1 | 2 | COS Exposure Report: SMS 153067A8 3 | 4 | Data Exposure Start FUV Mechanism Positions Cent Tsince Tsince Lamp STIMs Flash 5 | Filename Prop Target PRG OB AL EX Conf Opmode ExpTime yyyy.ddd:hh:mm:ss State Aper OSM1 OSM2 Wave FP OSM1 OSM2 Lamp Curr A B Dop # Dur T 6 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 7 | ld0x01cq 14436 DARK D0X 01 01 01 FUV TIME-TAG 1330.0 2015.306:03:12:11 167/163 PSA B G130M ----- 1309 0 11531 11531 LOW LOW NO 8 | ld0x02d7 14436 DARK D0X 02 01 01 FUV TIME-TAG 1330.0 2015.306:03:48:20 167/163 PSA B G130M ----- 1309 0 13700 13700 LOW LOW NO 9 | ld0x03dl 14436 DARK D0X 03 01 01 FUV TIME-TAG 1330.0 2015.306:04:19:28 167/163 PSA B G130M ----- 1309 0 15568 15568 LOW LOW NO 10 | ld0x04dz 14436 DARK D0X 04 01 01 FUV TIME-TAG 1330.0 2015.306:04:48:50 167/163 PSA B G130M ----- 1309 0 17330 17330 LOW LOW NO 11 | ld0x05ec 14436 DARK D0X 05 01 01 FUV TIME-TAG 1330.0 2015.306:05:19:38 167/163 PSA B G130M ----- 1309 0 19178 19178 LOW LOW NO 12 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 13 | ld0w01eo 14442 DARK D0W 01 01 01 NUV TIME-TAG 1330.0 2015.306:05:56:00 PSA O G130M MIRRORA 0 0 21360 21360 NO 14 | ld0w02gz 14442 DARK D0W 02 01 01 NUV TIME-TAG 1330.0 2015.306:15:19:21 PSA O G130M MIRRORA 0 0 55161 55161 NO 15 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 16 | lbdoa5hx 12046 BDO A5 01 01 2015.306:21:49:58 COS MEMORY COPY -- DCE CODE IMAGE TO CS RAM 17 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 18 | lcvj05ou 14077 WD0216+143 CVJ 05 01 01 FUV ACQ/PEAKXD 1.0 2015.307:21:41:48 167/163 PSA B G130M ----- 1291 0 164508 164508 OFF OFF NO 19 | lcvj05ov 14077 WD0216+143 CVJ 05 02 01 FUV ACQ/PEAKD 1.0 2015.307:21:42:35 167/163 PSA B G130M ----- 1291 0 164555 164555 DWELLS=5 STEPSIZE= 900 FLUXFLR 20 | lcvj05ox 14077 WD0216+143 CVJ 05 04 01 FUV TIME-TAG 400.0 2015.307:21:47:52 167/163 PSA B G130M ----- 1291 -2 164872 164872 LOW LOW NO 1 12 S 21 | lcvj05p1 14077 WD0216+143 CVJ 05 06 01 FUV TIME-TAG 400.0 2015.307:21:58:30 167/163 PSA B G130M ----- 1291 1 165510 165510 LOW LOW NO 1 12 S 22 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 23 | ld1ce1rc 14429 ANY D1C E1 04 07 FUV TIME-TAG 2900.0 2015.308:03:46:40 167/163 PSA B G130M ----- 1327 -2 186400 186400 LOW LOW NO 2 12 S 24 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 25 | lcxr11ab 14201 GP1454+452 CXR 11 01 01 NUV ACQ/IMAGE 200.0 2015.308:20:48:29 PSA B NCM1 MIRRORA 0 0 377 247709 NO 26 | lcxr11ae 14201 GP1454+452 CXR 11 03 01 FUV TIME-TAG 486.0 2015.308:20:56:34 167/163 PSA B G160M ----- 1623 -2 131 248194 LOW LOW NO 2 12 S 27 | lcxr11ah 14201 GP1454+452 CXR 11 05 01 FUV TIME-TAG 486.0 2015.308:21:06:25 167/163 PSA B G160M ----- 1623 -1 722 248785 LOW LOW NO 2 12 S 28 | lcxr11aj 14201 GP1454+452 CXR 11 07 01 FUV TIME-TAG 486.0 2015.308:21:16:16 167/163 PSA B G160M ----- 1623 0 1313 249376 LOW LOW NO 2 12 S 29 | lcxr11al 14201 GP1454+452 CXR 11 09 01 FUV TIME-TAG 486.0 2015.308:21:26:07 167/163 PSA B G160M ----- 1623 1 1904 249967 LOW LOW NO 2 12 S 30 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 31 | lcvj03ef 14077 WD2328+107 CVJ 03 01 01 FUV ACQ/PEAKXD 1.0 2015.309:16:33:34 167/163 PSA B G130M ----- 1291 0 68252 318814 OFF OFF NO 32 | lcvj03eg 14077 WD2328+107 CVJ 03 02 01 FUV ACQ/PEAKD 1.0 2015.309:16:34:21 167/163 PSA B G130M ----- 1291 0 68299 318861 DWELLS=5 STEPSIZE= 900 FLUXFLR 33 | lcvj03ei 14077 WD2328+107 CVJ 03 04 01 FUV TIME-TAG 570.0 2015.309:16:39:38 167/163 PSA B G130M ----- 1291 -2 68616 319178 LOW LOW NO 1 12 S 34 | lcvj03ek 14077 WD2328+107 CVJ 03 06 01 FUV TIME-TAG 570.0 2015.309:16:51:50 167/163 PSA B G130M ----- 1291 1 69348 319910 LOW LOW NO 1 12 S 35 | lcvj38ih 14077 WD2018-233 CVJ 38 01 01 FUV ACQ/PEAKXD 2.5 2015.310:09:44:35 167/163 PSA B G130M ----- 1291 0 130113 380675 OFF OFF NO 36 | lcvj38ii 14077 WD2018-233 CVJ 38 02 01 FUV ACQ/PEAKD 2.5 2015.310:09:45:24 167/163 PSA B G130M ----- 1291 0 130162 380724 DWELLS=5 STEPSIZE= 900 FLUXFLR 37 | lcvj38ik 14077 WD2018-233 CVJ 38 04 01 FUV TIME-TAG 800.0 2015.310:09:50:49 167/163 PSA B G130M ----- 1291 -2 130487 381049 LOW LOW NO 2 12 S 38 | lcvj38im 14077 WD2018-233 CVJ 38 06 01 FUV TIME-TAG 800.0 2015.310:10:06:13 167/163 PSA B G130M ----- 1291 1 131411 381973 LOW LOW NO 2 12 S 39 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 40 | lcj901ja 13827 SMSS-J0313 CJ9 01 01 01 NUV ACQ/IMAGE 66.0 2015.310:18:26:45 PSA O NCM1 MIRRORA 0 0 243 412005 NO 41 | lcj901jd 13827 SMSS-J0313 CJ9 01 03 01 NUV TIME-TAG 2828.0 2015.310:18:31:05 PSA O NCM1 G225M 2390 -2 503 87 NO 3 7 S 42 | lcj901jl 13827 SMSS-J0313 CJ9 01 05 01 NUV TIME-TAG 3307.0 2015.310:19:46:15 PSA O NCM1 G225M 2390 -1 5013 4597 NO 3 7 S 43 | lcj901jt 13827 SMSS-J0313 CJ9 01 07 01 NUV TIME-TAG 3307.0 2015.310:21:21:43 PSA O NCM1 G225M 2390 0 10741 10325 NO 3 7 S 44 | lcj901k7 13827 SMSS-J0313 CJ9 01 09 01 NUV TIME-TAG 3307.0 2015.310:22:57:10 PSA O NCM1 G225M 2390 0 16468 16052 NO 3 7 S 45 | lcj901kl 13827 SMSS-J0313 CJ9 01 0B 01 NUV TIME-TAG 3307.0 2015.311:00:32:38 PSA O NCM1 G225M 2390 1 22196 21780 NO 3 7 S 46 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 47 | ld0x06l1 14436 DARK D0X 06 01 01 FUV TIME-TAG 1330.0 2015.311:01:36:46 167/163 PSA B G130M ----- 1309 0 442 336 LOW LOW NO 48 | ld0x07m3 14436 DARK D0X 07 01 01 FUV TIME-TAG 1330.0 2015.311:02:06:05 167/163 PSA B G130M ----- 1309 0 2201 2095 LOW LOW NO 49 | ld0x08m8 14436 DARK D0X 08 01 01 FUV TIME-TAG 1330.0 2015.311:02:35:24 167/163 PSA B G130M ----- 1309 0 3960 3854 LOW LOW NO 50 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 51 | lclz05mc 13875 QSO-231145 CLZ 05 01 01 NUV ACQ/IMAGE 150.0 2015.311:03:24:59 PSA B NCM1 MIRRORA 0 0 327 6829 NO 52 | lclz05mf 13875 QSO-231145 CLZ 05 03 01 FUV TIME-TAG 2264.0 2015.311:03:31:42 167/100 PSA B G140L ----- 1105 -2 134 7232 LOW OFF NO 3 7 S 53 | lclz05ml 13875 QSO-231145 CLZ 05 05 01 FUV TIME-TAG 2940.0 2015.311:04:55:55 167/100 PSA B G140L ----- 1105 -1 5187 12285 LOW OFF NO 2 7 S 54 | lclz05mw 13875 QSO-231145 CLZ 05 07 01 FUV TIME-TAG 2940.0 2015.311:06:31:21 167/100 PSA B G140L ----- 1105 0 10913 18011 LOW OFF NO 2 7 S 55 | lclz05n9 13875 QSO-231145 CLZ 05 09 01 FUV TIME-TAG 2940.0 2015.311:08:06:48 167/100 PSA B G140L ----- 1105 1 16640 23738 LOW OFF NO 2 7 S 56 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 57 | ld0x09no 14436 DARK D0X 09 01 01 FUV TIME-TAG 1330.0 2015.311:09:33:11 167/163 PSA B G130M ----- 1309 0 2042 28921 LOW LOW NO 58 | ld0x0ao1 14436 DARK D0X 0A 01 01 FUV TIME-TAG 1330.0 2015.311:10:02:30 167/163 PSA B G130M ----- 1309 0 3801 30680 LOW LOW NO 59 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 60 | lcku01ot 13833 VV2006-J09 CKU 01 01 01 NUV ACQ/IMAGE 10.0 2015.311:16:34:49 PSA B NCM1 MIRRORA 0 0 187 54219 NO 61 | lcku01ow 13833 VV2006-J09 CKU 01 03 01 FUV TIME-TAG 1300.0 2015.311:16:39:02 167/163 PSA B G130M ----- 1309 -2 130 54472 LOW LOW NO 3 12 S 62 | lcku01p2 13833 VV2006-J09 CKU 01 05 01 FUV TIME-TAG 1301.0 2015.311:17:02:56 167/163 PSA B G130M ----- 1309 -1 1564 55906 LOW LOW NO 2 12 S 63 | lcku01p6 13833 VV2006-J09 CKU 01 07 01 FUV TIME-TAG 1495.0 2015.311:18:05:29 167/163 PSA B G130M ----- 1309 0 5317 59659 LOW LOW NO 2 12 S 64 | lcku01pe 13833 VV2006-J09 CKU 01 09 01 FUV TIME-TAG 1494.0 2015.311:18:32:38 167/163 PSA B G130M ----- 1309 1 6946 61288 LOW LOW NO 2 12 S 65 | lcku01ph 13833 VV2006-J09 CKU 01 0B 01 FUV TIME-TAG 1494.0 2015.311:19:40:49 167/163 PSA B G130M ----- 1291 -2 11037 65379 LOW LOW NO 2 12 S 66 | lcku01po 13833 VV2006-J09 CKU 01 0D 01 FUV TIME-TAG 1495.0 2015.311:20:07:57 167/163 PSA B G130M ----- 1291 -1 12665 67007 LOW LOW NO 2 12 S 67 | lcku01pr 13833 VV2006-J09 CKU 01 0F 01 FUV TIME-TAG 1495.0 2015.311:21:16:09 167/163 PSA B G130M ----- 1291 0 16757 71099 LOW LOW NO 2 12 S 68 | lcku01py 13833 VV2006-J09 CKU 01 0H 01 FUV TIME-TAG 1494.0 2015.311:21:43:18 167/163 PSA B G130M ----- 1291 1 18386 72728 LOW LOW NO 2 12 S 69 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 70 | lcj902t2 13827 SMSS-J0313 CJ9 02 01 01 NUV ACQ/IMAGE 66.0 2015.312:13:54:12 PSA O NCM1 MIRRORA 0 0 243 130982 NO 71 | lcj902t4 13827 SMSS-J0313 CJ9 02 03 01 NUV TIME-TAG 2828.0 2015.312:14:43:23 PSA O NCM1 G225M 2390 -2 3194 87 NO 3 7 S 72 | lcj902td 13827 SMSS-J0313 CJ9 02 05 01 NUV TIME-TAG 3307.0 2015.312:16:18:48 PSA O NCM1 G225M 2390 -1 8919 5812 NO 3 7 S 73 | lcj902tr 13827 SMSS-J0313 CJ9 02 07 01 NUV TIME-TAG 3307.0 2015.312:17:54:14 PSA O NCM1 G225M 2390 -1 14645 11538 NO 3 7 S 74 | lcj902tu 13827 SMSS-J0313 CJ9 02 09 01 NUV TIME-TAG 3307.0 2015.312:19:29:40 PSA O NCM1 G225M 2390 0 20371 17264 NO 3 7 S 75 | lcj902u2 13827 SMSS-J0313 CJ9 02 0B 01 NUV TIME-TAG 3307.0 2015.312:21:05:16 PSA O NCM1 G225M 2390 1 26107 23000 NO 3 7 S 76 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 77 | lcxr15uc 14201 GP0822+224 CXR 15 01 01 NUV ACQ/IMAGE 200.0 2015.312:22:55:08 PSA B NCM1 MIRRORA 0 0 377 3080 NO 78 | lcxr15uf 14201 GP0822+224 CXR 15 03 01 FUV TIME-TAG 450.0 2015.312:23:03:12 167/163 PSA B G160M ----- 1577 -2 129 3564 LOW LOW NO 2 12 S 79 | lcxr15uh 14201 GP0822+224 CXR 15 05 01 FUV TIME-TAG 450.0 2015.312:23:12:27 167/163 PSA B G160M ----- 1577 -1 684 4119 LOW LOW NO 2 12 S 80 | lcxr15uj 14201 GP0822+224 CXR 15 07 01 FUV TIME-TAG 450.0 2015.312:23:21:42 167/163 PSA B G160M ----- 1577 0 1239 4674 LOW LOW NO 2 12 S 81 | lcxr15ul 14201 GP0822+224 CXR 15 09 01 FUV TIME-TAG 450.0 2015.312:23:30:57 167/163 PSA B G160M ----- 1577 1 1794 5229 LOW LOW NO 2 12 S 82 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 83 | -------------------------------------------------------------------------------- /cosmo/sms/ingest_sms.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datetime 3 | import glob 4 | import re 5 | import pandas as pd 6 | 7 | from typing import Union 8 | from itertools import repeat 9 | from peewee import chunked, OperationalError, EXCLUDED 10 | 11 | from .sms_db import SMSFileStats, SMSTable, DB 12 | from .. import SETTINGS 13 | 14 | SMS_FILE_LOC = SETTINGS['sms']['source'] 15 | 16 | 17 | class SMSFile: 18 | """Class that encapsulates the SMS file data.""" 19 | _db = DB 20 | _patterns = { 21 | 'ROOTNAME': r'l[a-z0-9]{7}', # String of 7 alpha-numeric characters after 'l' 22 | 'PROPOSID': r'(?<=l[a-z0-9]{7} )\d{5}', # String of 5 digits occurring after a rootname. 23 | # Exposure: Group of three sets of alpha(caps)-numeric characters 24 | 'EXPOSURE': r'(?<= )[A-Z0-9]{3} [A-Z0-9]{2} [A-Z0-9]{2}(?= \d{2} )', 25 | 'DETECTOR': r'(?<= )NUV|FUV(?= )', # Either NUV or FUV 26 | 'OPMODE': r'ACQ\/\S{5,6}|TIME\-TAG|ACCUM', # ACQ followed by 5 or 6 characters or TIME-TAG or ACCUM 27 | 'EXPTIME': r'(?<= )\d+\.\d{1}(?= )', # Float with one decimal place followed and preceded by a space 28 | 'EXPSTART': r'\d{4}\.\d{3}:\d{2}:\d{2}:\d{2}', 29 | # fuvhvstate: After expstart, either 6 spaces, or HV+(3 or 4 characters) or 2 sets of 3 digits separated by 30 | # '/' followed by a single space 31 | 'FUVHVSTATE': r'(?<=\d{4}\.\d{3}:\d{2}:\d{2}:\d{2} ) {6}|HV[a-zA-Z]{3,4}|\d{3}\/\d{3}(?= )', 32 | # aperture: Tuple of (aperture, ap_pos or empty depending on format of the sms file) 33 | 'APERTURE': r'(PSA|BOA|WCA|FCA|RELATIVE|REL) (\w{1}|\s+)', 34 | 'osm': r'(NCM1|G130M|G140L|G160M|NCM1FLAT)\s+(-----|MIRRORA|MIRRORB|G\d{3}M|G\d{3}L)', # tuple (osm1, osm2) 35 | # cenwave_fp_t1_t2: Tuple of (cenwave, fpoffset, tsince1, tsince2) 36 | 'cenwave_fp_t1_t2': r'(?<= )(0|\d{4}|\d{3}) ( 0|-1|-2|-3| 1)\s+(\d{1,6})\s+(\d{1,6})' 37 | } 38 | 39 | # Keywords and dtypes for the sms_db table 40 | _table_keys = { 41 | 'ROOTNAME': str, 42 | 'PROPOSID': int, 43 | 'EXPOSURE': str, 44 | 'DETECTOR': str, 45 | 'OPMODE': str, 46 | 'EXPTIME': float, 47 | 'EXPSTART': str, 48 | 'FUVHVSTATE': str, 49 | 'APERTURE': str, 50 | 'OSM1POS': str, 51 | 'OSM2POS': str, 52 | 'CENWAVE': int, 53 | 'FPPOS': int, 54 | 'TSINCEOSM1': float, 55 | 'TSINCEOSM2': float 56 | } 57 | 58 | # Keys that should return a single item 59 | _single_value_keys = ['ROOTNAME', 'PROPOSID', 'DETECTOR', 'OPMODE', 'EXPTIME', 'EXPSTART'] 60 | 61 | # Keys that return a tuple of items 62 | _grouped_value_keys = [ 63 | 'FUVHVSTATE', 64 | 'APERTURE', 65 | 'OSM1POS', 66 | 'OSM2POS', 67 | 'CENWAVE', 68 | 'FPPOS', 69 | 'TSINCEOSM1', 70 | 'TSINCEOSM2' 71 | ] 72 | 73 | def __init__(self, smsfile: str): 74 | """Initialize and ingest the sms file data.""" 75 | self.datetime_format = '%Y-%m-%d %H:%M:%S' 76 | self.filename = smsfile 77 | self.file_id = os.path.basename(self.filename).split('.')[0] 78 | self.sms_id = self.file_id[:6] # The SMS id is the first 6 digits of the filename 79 | self.version = self.file_id[6:] # The SMS version is the last 2 (or sometimes 3) digits of the filename 80 | self.ingest_date = datetime.datetime.today() 81 | 82 | self._data = self.ingest_smsfile() 83 | self.data: pd.DataFrame = pd.DataFrame(self._data).astype(self._table_keys) 84 | 85 | @property 86 | def _single_value_patterns(self) -> dict: 87 | """Return a dictionary subset of the patterns dictionary for patterns that return a single value.""" 88 | return {key: self._patterns[key] for key in self._single_value_keys} 89 | 90 | def ingest_smsfile(self) -> dict: 91 | """Collect data from the SMS file.""" 92 | data = {} 93 | 94 | # Initialize keys that require breakdown of grouped matches 95 | for key in self._grouped_value_keys: 96 | data[key] = [] 97 | 98 | filtered_lines = [] 99 | with open(self.filename) as sms: 100 | for i, line in enumerate(sms): 101 | # Skip the header of the file 102 | if i <= 5: 103 | continue 104 | 105 | # Skip "special" types 106 | if 'MEMORY' in line or 'ALIGN/OSM' in line or 'ALIGN/APER' in line: 107 | continue 108 | 109 | filtered_lines.append(line) 110 | 111 | sms_string = ''.join(filtered_lines) 112 | 113 | # Ingest single-valued matches 114 | data.update( 115 | { 116 | key: re.findall(pattern, sms_string) 117 | for key, pattern in self._single_value_patterns.items() 118 | } 119 | ) 120 | 121 | # Ingest EXPOSURE 122 | data['EXPOSURE'] = [''.join(item.split()) for item in re.findall(self._patterns['EXPOSURE'], sms_string)] 123 | 124 | # Ingest fuvhvstate since some values are empty 125 | for item in re.findall(self._patterns['FUVHVSTATE'], sms_string): 126 | data['FUVHVSTATE'].append(item if item.strip() else 'N/A') 127 | 128 | # Ingest aperture matches 129 | for item in re.findall(self._patterns['APERTURE'], sms_string): 130 | data['APERTURE'].append(' '.join(item).strip()) 131 | 132 | # Ingest osm1 and osm2 positions 133 | for item in re.findall(self._patterns['osm'], sms_string): 134 | osm1pos, osm2pos = item 135 | data['OSM1POS'].append(osm1pos) 136 | data['OSM2POS'].append(osm2pos if osm2pos.strip('-') else 'N/A') # if osm2 isn't used, the value is ----- 137 | 138 | # Ingest cenwave, fppos, tsinceosm1, and tsinceosm2 139 | for item in re.findall(self._patterns['cenwave_fp_t1_t2'], sms_string): 140 | cenwave, fpoffset, tsinceosm1, tsinceosm2 = item 141 | fppos = int(fpoffset) + 3 # fpoffset is relative to the third position 142 | 143 | data['CENWAVE'].append(cenwave) 144 | # noinspection PyTypeChecker 145 | data['FPPOS'].append(fppos) 146 | data['TSINCEOSM1'].append(tsinceosm1) 147 | data['TSINCEOSM2'].append(tsinceosm2) 148 | 149 | # Add the "sms id" 150 | data.update({'FILEID': list(repeat(self.file_id, len(data['ROOTNAME'])))}) 151 | 152 | return data 153 | 154 | def insert_to_db(self): 155 | """Insert ingested SMS data into the database tables.""" 156 | new_record = { 157 | 'SMSID': self.sms_id, 158 | 'VERSION': self.version, 159 | 'FILEID': self.file_id, 160 | 'FILENAME': self.filename, 161 | 'INGEST_DATE': self.ingest_date.strftime(self.datetime_format) 162 | } 163 | 164 | # Insert into the file stats table 165 | with self._db.atomic(): 166 | if not SMSFileStats.table_exists(): 167 | SMSFileStats.create_table() 168 | SMSFileStats.insert(new_record).execute() 169 | 170 | else: 171 | SMSFileStats.insert(new_record).on_conflict( 172 | action='update', 173 | update=new_record, 174 | conflict_target=[SMSFileStats.SMSID], 175 | where=(EXCLUDED.VERSION > SMSFileStats.VERSION) 176 | ).execute() 177 | 178 | # Insert data into sms data table 179 | row_oriented_data = self.data.to_dict(orient='row') 180 | 181 | with self._db.atomic(): 182 | if not SMSTable.table_exists(): 183 | SMSTable.create_table() 184 | 185 | for batch in chunked(row_oriented_data, 100): 186 | SMSTable.insert_many(batch).execute() 187 | 188 | else: 189 | # For new data with an existing table, need to check whether or not new records are replacing old ones. 190 | for row in row_oriented_data: 191 | SMSTable.insert(**row).on_conflict( 192 | action='update', 193 | update=row, 194 | conflict_target=[SMSTable.EXPOSURE], 195 | # Update if the FILEID is greater. This is only possible if exposures appear in a later SMS 196 | # or a new SMS version is available. 197 | where=(EXCLUDED.FILEID_id > SMSTable.FILEID_id) 198 | ).execute() 199 | 200 | 201 | class SMSFinder: 202 | """Class for finding sms files in the specified filesystem and the database.""" 203 | _sms_pattern = r'\A\d{6}[a-z]{1}[a-z0-9]{1}' 204 | 205 | def __init__(self, source: str = SMS_FILE_LOC): 206 | self.currently_ingested = None 207 | 208 | self.filesource = source 209 | if not os.path.exists(self.filesource): 210 | raise OSError(f'source directory, {self.filesource} does not exist.') 211 | 212 | # Find the most recent ingest date from the database; This is inefficient, but the db is small enough that it 213 | # doesn't matter. 214 | 215 | try: # If the table doesn't exist, don't set _currently_ingested 216 | self.currently_ingested = pd.DataFrame(list(SMSFileStats.select().dicts())) 217 | except OperationalError: 218 | pass 219 | 220 | self.all_sms = self.find_all() 221 | self._grouped_results = self.all_sms.groupby('is_new') 222 | 223 | @property 224 | def new_sms(self) -> Union[pd.Series, None]: 225 | """Return only the sms files that were determined as new.""" 226 | try: 227 | return self._grouped_results.get_group(True) 228 | 229 | except KeyError: 230 | return 231 | 232 | @property 233 | def old_sms(self) -> Union[pd.Series, None]: 234 | """Return only the sms files there were not determined as new.""" 235 | try: 236 | return self._grouped_results.get_group(False) 237 | 238 | except KeyError: 239 | return 240 | 241 | @staticmethod 242 | def _filter_l_exp_files(files_list): 243 | """Filter l-exp files in the file list to eliminate those that also have a txt file copy.""" 244 | txt_files = [file for file in files_list if file.endswith('.txt')] 245 | l_exp_files = [file for file in files_list if file.endswith('.l-exp')] 246 | 247 | filtered_lexp = [file for file in l_exp_files if file.strip('.l-exp') + '.txt' not in txt_files] 248 | 249 | return txt_files + filtered_lexp 250 | 251 | def find_all(self) -> pd.DataFrame: 252 | """Find all SMS files from the source directory. Determine if the file is 'new'.""" 253 | results = {'smsfile': [], 'is_new': [], 'sms_id': [], 'version': []} 254 | all_files = glob.glob(os.path.join(self.filesource, '*')) 255 | 256 | # Filter out l-exp files that have txt counterparts 257 | reduced_files = self._filter_l_exp_files(all_files) 258 | 259 | for file in reduced_files: 260 | # There may be other files or directories in the sms source directory 261 | match = re.findall(self._sms_pattern, os.path.basename(file)) 262 | if match and os.path.isfile(file): 263 | name = match[0] # findall returns a list. There should only be one match per file 264 | sms_id = name[:6] 265 | version = name[6:] 266 | 267 | results['version'].append(version) 268 | results['sms_id'].append(sms_id) 269 | results['smsfile'].append(file) 270 | results['is_new'].append(self._is_new(file)) 271 | 272 | data = pd.DataFrame(results) 273 | 274 | # Filter the dataframe for the most recent version of the files per file id 275 | data = data.iloc[ 276 | [ 277 | index for _, group in data.groupby('sms_id') 278 | for index in group[group.version == group.version.max()].index.values 279 | ] 280 | ].reset_index(drop=True) 281 | 282 | # Raise an error if no sms files were found. Otherwise this will break other stuff in an obscure way 283 | if data.empty: 284 | raise OSError('No SMS files were found in the source directory.') 285 | 286 | return data 287 | 288 | def _is_new(self, smsfile): 289 | """Determine if an sms file is considered 'new'. New is defined as any file that is not in the database 290 | """ 291 | if self.currently_ingested is None or self.currently_ingested.empty: 292 | return True # If there are no ingested files, they're all new 293 | 294 | return smsfile not in self.currently_ingested.FILENAME.values 295 | 296 | def ingest_files(self): 297 | """Add new sms data to the database.""" 298 | if self.new_sms is not None: 299 | for sms in self.new_sms.smsfile: 300 | smsfile = SMSFile(sms) 301 | smsfile.insert_to_db() 302 | -------------------------------------------------------------------------------- /tests/data/181137b3.txt: -------------------------------------------------------------------------------- 1 | 2 | COS Exposure Report: SMS 181137B3 3 | 4 | Data Exposure Start FUV Mechanism Positions Cent Tsince Tsince Lamp STIMs Flash 5 | Filename Prop Target PRG OB AL EX Conf Opmode ExpTime yyyy.ddd:hh:mm:ss State Aper OSM1 OSM2 Wave FP OSM1 OSM2 Lamp Curr A B Dop # Dur T 6 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 7 | ldie05kf 15341 SDSSJ12420 DIE 05 01 01 NUV ACQ/IMAGE 161.0 2018.113:18:25:03 PSA 4 NCM1 MIRRORA 1850 0 333 66303 NO 8 | ldie05kh 15341 SDSSJ12420 DIE 05 03 01 FUV TIME-TAG 2220.0 2018.113:19:24:52 163/100 PSA 4 G140L ----- 1105 -2 129 69892 LOW OFF NO 3 7 S 9 | ldie05kj 15341 SDSSJ12420 DIE 05 05 01 FUV TIME-TAG 1641.0 2018.113:20:57:38 163/100 PSA 4 G140L ----- 1105 -1 5695 75458 LOW OFF NO 2 7 S 10 | ldie05kl 15341 SDSSJ12420 DIE 05 07 01 FUV TIME-TAG 1170.0 2018.113:22:24:44 163/100 PSA 4 G140L ----- 1105 0 10921 80684 LOW OFF NO 2 7 S 11 | ldie05kn 15341 SDSSJ12420 DIE 05 09 01 FUV TIME-TAG 900.0 2018.113:22:45:56 163/100 PSA 4 G140L ----- 1105 0 12193 81956 LOW OFF NO 2 7 S 12 | ldie05kp 15341 SDSSJ12420 DIE 05 0B 01 FUV TIME-TAG 1901.0 2018.114:00:00:06 163/100 PSA 4 G140L ----- 1105 1 16643 86406 LOW OFF NO 2 7 S 13 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 14 | ldq03il5 14940 DARK DQ0 3I 01 01 FUV TIME-TAG 1330.0 2018.114:00:46:22 163/163 PSA 4 G130M ----- 1291 0 731 89182 LOW LOW NO 15 | ldq03jlx 14940 DARK DQ0 3J 01 01 FUV TIME-TAG 1330.0 2018.114:01:20:12 163/163 PSA 4 G130M ----- 1291 0 2761 91212 LOW LOW NO 16 | ldq03km0 14940 DARK DQ0 3K 01 01 FUV TIME-TAG 1330.0 2018.114:01:49:31 163/163 PSA 4 G130M ----- 1291 0 4520 92971 LOW LOW NO 17 | ldq03lm3 14940 DARK DQ0 3L 01 01 FUV TIME-TAG 1330.0 2018.114:02:18:50 163/163 PSA 4 G130M ----- 1291 0 6279 94730 LOW LOW NO 18 | ldq03mm5 14940 DARK DQ0 3M 01 01 FUV TIME-TAG 1330.0 2018.114:02:48:09 163/163 PSA 4 G130M ----- 1291 0 8038 96489 LOW LOW NO 19 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 20 | ldk201pm 15147 IO-LEADING DK2 01 01 02 NUV ACQ/SEARCH 1.0 2018.115:06:27:29 PSA 4 NCM1 G230L 3000 0 180 75 SCANSIZE=3 STEPSIZE=1767 FLUXFLR 21 | ldk201po 15147 IO-LEADING DK2 01 03 02 FUV TIME-TAG 249.0 2018.115:06:35:59 163/163 PSA 4 G130M ----- 1291 0 123 585 LOW LOW NO 2 12 S 22 | ldk2a1qd 15147 IO-OFFSET- DK2 A1 01 02 FUV TIME-TAG 1341.0 2018.115:06:47:05 163/163 PSA 4 G130M ----- 1291 1 789 185 LOW LOW NO 2 12 S 23 | ldk2a1qg 15147 DARK DK2 A1 03 01 NUV TIME-TAG 4.0 2018.115:07:10:59 PSA 1 G130M MIRRORA 0 0 2223 1619 NO 24 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 25 | ldnj02qi 15227 UVQSJ10112 DNJ 02 01 01 NUV ACQ/IMAGE 75.0 2018.115:09:14:14 PSA 4 NCM1 MIRRORB 0 0 323 245 NO 26 | ldnj02ql 15227 UVQSJ10112 DNJ 02 03 01 FUV TIME-TAG 719.0 2018.115:09:20:27 163/163 PSA 4 G160M ----- 1600 -2 130 618 LOW LOW NO 2 12 S 27 | ldnj02qo 15227 UVQSJ10112 DNJ 02 05 01 FUV TIME-TAG 719.0 2018.115:09:34:11 163/163 PSA 4 G160M ----- 1600 -1 954 1442 LOW LOW NO 2 12 S 28 | ldnj02qs 15227 UVQSJ10112 DNJ 02 07 01 FUV TIME-TAG 719.0 2018.115:09:47:55 163/163 PSA 4 G160M ----- 1600 0 1778 2266 LOW LOW NO 2 12 S 29 | ldnj02qv 15227 UVQSJ10112 DNJ 02 09 01 FUV TIME-TAG 719.0 2018.115:10:46:53 163/163 PSA 4 G160M ----- 1600 1 5316 5804 LOW LOW NO 1 12 S 30 | ldnj02r3 15227 UVQSJ10112 DNJ 02 0B 01 FUV TIME-TAG 956.0 2018.115:11:01:53 163/163 PSA 4 G160M ----- 1623 -2 6216 6704 LOW LOW NO 2 12 S 31 | ldnj02r5 15227 UVQSJ10112 DNJ 02 0D 01 FUV TIME-TAG 955.0 2018.115:11:19:34 163/163 PSA 4 G160M ----- 1623 -1 7277 7765 LOW LOW NO 2 12 S 32 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 33 | lbdodtr9 12046 BDO DT 01 01 2018.115:11:44:59 COS MEMORY COPY -- DCE CODE IMAGE TO CS RAM 34 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 35 | ldkm02v5 15220 SDSSJ12304 DKM 02 01 01 NUV ACQ/IMAGE 20.0 2018.116:02:53:01 PSA 4 NCM1 MIRRORB 0 0 268 190 NO 36 | ldkm02v7 15220 SDSSJ12304 DKM 02 03 01 FUV TIME-TAG 547.7 2018.116:02:57:43 163/100 PSA 4 G140L ----- 1105 -2 129 472 LOW OFF NO 2 7 S 37 | ldkm02v9 15220 SDSSJ12304 DKM 02 05 01 FUV TIME-TAG 547.7 2018.116:03:08:36 163/100 PSA 4 G140L ----- 1105 -1 782 1125 LOW OFF NO 2 7 S 38 | ldkm02vb 15220 SDSSJ12304 DKM 02 07 01 FUV TIME-TAG 547.7 2018.116:03:19:29 163/100 PSA 4 G140L ----- 1105 0 1435 1778 LOW OFF NO 2 7 S 39 | ldkm02vd 15220 SDSSJ12304 DKM 02 09 01 FUV TIME-TAG 547.7 2018.116:03:30:22 163/100 PSA 4 G140L ----- 1105 1 2088 2431 LOW OFF NO 2 7 S 40 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 41 | ldgp25xh 15196 SDSSJ10173 DGP 25 01 01 NUV ACQ/IMAGE 30.0 2018.116:18:52:22 PSA 4 NCM1 MIRRORA 0 0 202 54585 NO 42 | ldgp25xj 15196 SDSSJ10173 DGP 25 03 01 FUV TIME-TAG 1081.0 2018.116:18:56:45 163/163 PSA 4 G130M ----- 1291 0 123 54848 LOW LOW NO 3 12 S 43 | ldgp25xp 15196 SDSSJ10173 DGP 25 05 01 FUV TIME-TAG 1081.0 2018.116:20:13:10 163/163 PSA 4 G130M ----- 1291 1 4708 59433 LOW LOW NO 2 12 S 44 | ldgp25xr 15196 SDSSJ10173 DGP 25 07 01 FUV TIME-TAG 1289.0 2018.116:20:34:22 163/163 PSA 4 G130M ----- 1291 0 5980 60705 LOW LOW NO 2 12 S 45 | ldgp25xt 15196 SDSSJ10173 DGP 25 09 01 FUV TIME-TAG 1289.0 2018.116:21:48:30 163/163 PSA 4 G130M ----- 1291 1 10428 65153 LOW LOW NO 2 12 S 46 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 47 | ldhz41zr 15075 SDSSJ12315 DHZ 41 01 01 NUV ACQ/IMAGE 100.0 2018.117:02:44:48 PSA 4 NCM1 MIRRORA 0 0 272 82931 NO 48 | ldhz41zu 15075 SDSSJ12315 DHZ 41 03 01 FUV TIME-TAG 1138.0 2018.117:02:50:32 163/100 PSA 4 G140L ----- 1105 0 129 83275 LOW OFF NO 3 7 S 49 | ldhz41a1 15075 SDSSJ12315 DHZ 41 05 01 FUV TIME-TAG 1138.0 2018.117:03:11:34 163/100 PSA 4 G140L ----- 1105 1 1391 84537 LOW OFF NO 2 7 S 50 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 51 | ldgp26a7 15196 SDSSJ10180 DGP 26 01 01 NUV ACQ/IMAGE 39.0 2018.117:04:12:13 PSA 4 NCM1 MIRRORA 0 0 211 88176 NO 52 | ldgp26a9 15196 SDSSJ10180 DGP 26 03 01 FUV TIME-TAG 1076.0 2018.117:04:16:45 163/163 PSA 4 G130M ----- 1291 0 123 88448 LOW LOW NO 3 12 S 53 | ldgp26ad 15196 SDSSJ10180 DGP 26 05 01 FUV TIME-TAG 1076.0 2018.117:04:36:36 163/163 PSA 4 G130M ----- 1291 1 1314 89639 LOW LOW NO 2 12 S 54 | ldgp26ai 15196 SDSSJ10180 DGP 26 07 01 FUV TIME-TAG 1289.0 2018.117:05:45:09 163/163 PSA 4 G130M ----- 1291 0 5427 93752 LOW LOW NO 2 12 S 55 | ldgp26ao 15196 SDSSJ10180 DGP 26 09 01 FUV TIME-TAG 1289.0 2018.117:06:08:42 163/163 PSA 4 G130M ----- 1291 1 6840 95165 LOW LOW NO 2 12 S 56 | ldgp26ar 15196 SDSSJ10180 DGP 26 0B 01 FUV TIME-TAG 1289.0 2018.117:07:20:28 163/163 PSA 4 G130M ----- 1291 0 11146 99471 LOW LOW NO 2 12 S 57 | ldgp26az 15196 SDSSJ10180 DGP 26 0D 01 FUV TIME-TAG 1289.0 2018.117:07:44:01 163/163 PSA 4 G130M ----- 1291 1 12559 100884 LOW LOW NO 2 12 S 58 | ldgp26b2 15196 SDSSJ10180 DGP 26 0F 01 FUV TIME-TAG 1289.0 2018.117:08:55:48 163/163 PSA 4 G130M ----- 1291 0 16866 105191 LOW LOW NO 2 12 S 59 | ldgp26b7 15196 SDSSJ10180 DGP 26 0H 01 FUV TIME-TAG 1289.0 2018.117:09:19:21 163/163 PSA 4 G130M ----- 1291 1 18279 106604 LOW LOW NO 2 12 S 60 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 61 | ldj105f4 15168 -PHI-LEO DJ1 05 01 01 FUV ACQ/SEARCH 4.0 2018.118:02:27:01 163/163 PSA 4 G130M ----- 1291 0 79939 168264 SCANSIZE=2 STEPSIZE=1767 FLUXWT 62 | ldj105f5 15168 -PHI-LEO DJ1 05 02 01 FUV ACQ/PKD-XD 4.0 2018.118:02:30:05 163/163 PSA 4 G130M ----- 1291 0 80123 168448 DWELLS=5 STEPSIZE=1000 FLUXFLR 63 | ldj105f9 15168 -PHI-LEO DJ1 05 03 01 FUV ACQ/PEAKD 4.0 2018.118:02:33:13 163/163 PSA 4 G130M ----- 1291 0 80311 168636 DWELLS=5 STEPSIZE= 900 FLUXFLR 64 | ldj105fd 15168 -PHI-LEO DJ1 05 05 01 FUV TIME-TAG 2218.0 2018.118:02:37:35 163/163 PSA 4 G130M ----- 1291 0 80573 168898 LOW LOW NO 2 12 S 65 | ldj105g5 15168 -PHI-LEO DJ1 05 07 01 FUV TIME-TAG 2916.0 2018.118:04:01:39 163/163 PSA 4 G130M ----- 1291 1 85617 173942 LOW LOW NO 2 12 S 66 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 67 | ldgt04jd 15084 SDSS-J0915 DGT 04 01 01 NUV ACQ/IMAGE 111.0 2018.119:05:23:37 PSA 4 NCM1 MIRRORB 0 0 359 281 NO 68 | ldgt04jg 15084 SDSS-J0915 DGT 04 03 01 FUV TIME-TAG 1166.0 2018.119:05:29:50 163/100 PSA 4 G140L ----- 1105 -2 129 654 LOW OFF NO 3 7 S 69 | ldgt04jq 15084 SDSS-J0915 DGT 04 05 01 FUV TIME-TAG 1020.0 2018.119:05:51:49 163/100 PSA 4 G140L ----- 1105 -1 1448 1973 LOW OFF NO 2 7 S 70 | ldgt04ke 15084 SDSS-J0915 DGT 04 07 01 FUV TIME-TAG 1385.0 2018.119:06:55:40 163/100 PSA 4 G140L ----- 1105 0 5279 5804 LOW OFF NO 2 7 S 71 | ldgt04kr 15084 SDSS-J0915 DGT 04 09 01 FUV TIME-TAG 1385.0 2018.119:07:21:24 163/100 PSA 4 G140L ----- 1105 0 6823 7348 LOW OFF NO 2 7 S 72 | ldgt04kv 15084 SDSS-J0915 DGT 04 0B 01 FUV TIME-TAG 1385.0 2018.119:08:31:01 163/100 PSA 4 G140L ----- 1105 1 11000 11525 LOW OFF NO 2 7 S 73 | ldgt04kx 15084 SDSS-J0915 DGT 04 0D 01 FUV TIME-TAG 1385.0 2018.119:08:56:45 163/100 PSA 4 G140L ----- 1105 1 12544 13069 LOW OFF NO 2 7 S 74 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 75 | ldi203n9 15185 SDSSJ11500 DI2 03 01 01 NUV ACQ/IMAGE 42.0 2018.119:19:50:29 PSA 1 NCM1 MIRRORB 0 0 290 212 NO 76 | ldi203nb 15185 SDSSJ11500 DI2 03 03 01 NUV TIME-TAG 550.0 2018.119:19:55:53 PSA 1 NCM1 G185M 1913 -2 614 91 NO 2 17 S 77 | ldi203nd 15185 SDSSJ11500 DI2 03 05 01 NUV TIME-TAG 550.0 2018.119:20:06:30 PSA 1 NCM1 G185M 1913 -1 1251 728 NO 2 17 S 78 | ldi203nf 15185 SDSSJ11500 DI2 03 07 01 NUV TIME-TAG 550.0 2018.119:20:17:07 PSA 1 NCM1 G185M 1913 0 1888 1365 NO 2 17 S 79 | ldi203nh 15185 SDSSJ11500 DI2 03 09 01 NUV TIME-TAG 550.0 2018.119:20:27:44 PSA 1 NCM1 G185M 1913 1 2525 2002 NO 2 17 S 80 | ldi203nj 15185 SDSSJ11500 DI2 03 0B 01 FUV TIME-TAG 653.0 2018.119:21:23:43 163/163 PSA 4 G160M ----- 1623 -2 136 5361 LOW LOW NO 2 12 S 81 | ldi203nl 15185 SDSSJ11500 DI2 03 0D 01 FUV TIME-TAG 653.0 2018.119:21:36:21 163/163 PSA 4 G160M ----- 1623 -1 894 6119 LOW LOW NO 2 12 S 82 | ldi203nn 15185 SDSSJ11500 DI2 03 0F 01 FUV TIME-TAG 653.0 2018.119:21:48:59 163/163 PSA 4 G160M ----- 1623 0 1652 6877 LOW LOW NO 2 12 S 83 | ldi203np 15185 SDSSJ11500 DI2 03 0H 01 FUV TIME-TAG 653.0 2018.119:22:01:37 163/163 PSA 4 G160M ----- 1623 1 2410 7635 LOW LOW NO 1 12 S 84 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 85 | -------------------------------------------------------------------------------- /tests/data/181137b4.txt: -------------------------------------------------------------------------------- 1 | 2 | COS Exposure Report: SMS 181137B4 3 | 4 | Data Exposure Start FUV Mechanism Positions Cent Tsince Tsince Lamp STIMs Flash 5 | Filename Prop Target PRG OB AL EX Conf Opmode ExpTime yyyy.ddd:hh:mm:ss State Aper OSM1 OSM2 Wave FP OSM1 OSM2 Lamp Curr A B Dop # Dur T 6 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 7 | ldie05kf 15341 SDSSJ12420 DIE 05 01 01 NUV ACQ/IMAGE 161.0 2018.113:18:25:03 PSA 4 NCM1 MIRRORA 1850 0 333 66303 NO 8 | ldie05kh 15341 SDSSJ12420 DIE 05 03 01 FUV TIME-TAG 2220.0 2018.113:19:24:52 163/100 PSA 4 G140L ----- 1105 -2 129 69892 LOW OFF NO 3 7 S 9 | ldie05kj 15341 SDSSJ12420 DIE 05 05 01 FUV TIME-TAG 1641.0 2018.113:20:57:38 163/100 PSA 4 G140L ----- 1105 -1 5695 75458 LOW OFF NO 2 7 S 10 | ldie05kl 15341 SDSSJ12420 DIE 05 07 01 FUV TIME-TAG 1170.0 2018.113:22:24:44 163/100 PSA 4 G140L ----- 1105 0 10921 80684 LOW OFF NO 2 7 S 11 | ldie05kn 15341 SDSSJ12420 DIE 05 09 01 FUV TIME-TAG 900.0 2018.113:22:45:56 163/100 PSA 4 G140L ----- 1105 0 12193 81956 LOW OFF NO 2 7 S 12 | ldie05kp 15341 SDSSJ12420 DIE 05 0B 01 FUV TIME-TAG 1901.0 2018.114:00:00:06 163/100 PSA 4 G140L ----- 1105 1 16643 86406 LOW OFF NO 2 7 S 13 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 14 | ldq03il5 14940 DARK DQ0 3I 01 01 FUV TIME-TAG 1330.0 2018.114:00:46:22 163/163 PSA 4 G130M ----- 1291 0 731 89182 LOW LOW NO 15 | ldq03jlx 14940 DARK DQ0 3J 01 01 FUV TIME-TAG 1330.0 2018.114:01:20:12 163/163 PSA 4 G130M ----- 1291 0 2761 91212 LOW LOW NO 16 | ldq03km0 14940 DARK DQ0 3K 01 01 FUV TIME-TAG 1330.0 2018.114:01:49:31 163/163 PSA 4 G130M ----- 1291 0 4520 92971 LOW LOW NO 17 | ldq03lm3 14940 DARK DQ0 3L 01 01 FUV TIME-TAG 1330.0 2018.114:02:18:50 163/163 PSA 4 G130M ----- 1291 0 6279 94730 LOW LOW NO 18 | ldq03mm5 14940 DARK DQ0 3M 01 01 FUV TIME-TAG 1330.0 2018.114:02:48:09 163/163 PSA 4 G130M ----- 1291 0 8038 96489 LOW LOW NO 19 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 20 | ldk201oc 15147 IO-LEADING DK2 01 01 02 NUV ACQ/SEARCH 1.0 2018.115:06:27:29 PSA 4 NCM1 G230L 3000 0 180 75 SCANSIZE=3 STEPSIZE=1767 FLUXFLR 21 | ldk201oe 15147 IO-LEADING DK2 01 03 02 FUV TIME-TAG 249.0 2018.115:06:35:59 163/163 PSA 4 G130M ----- 1291 0 123 585 LOW LOW NO 2 12 S 22 | ldk2a1og 15147 IO-OFFSET- DK2 A1 01 02 FUV TIME-TAG 1341.0 2018.115:06:47:05 163/163 PSA 4 G130M ----- 1291 1 789 185 LOW LOW NO 2 12 S 23 | ldk2a1oi 15147 DARK DK2 A1 03 01 NUV TIME-TAG 4.0 2018.115:07:10:59 PSA 1 G130M MIRRORA 0 0 2223 1619 NO 24 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 25 | ldnj02ok 15227 UVQSJ10112 DNJ 02 01 01 NUV ACQ/IMAGE 75.0 2018.115:09:14:14 PSA 4 NCM1 MIRRORB 0 0 323 245 NO 26 | ldnj02on 15227 UVQSJ10112 DNJ 02 03 01 FUV TIME-TAG 719.0 2018.115:09:20:27 163/163 PSA 4 G160M ----- 1600 -2 130 618 LOW LOW NO 2 12 S 27 | ldnj02oq 15227 UVQSJ10112 DNJ 02 05 01 FUV TIME-TAG 719.0 2018.115:09:34:11 163/163 PSA 4 G160M ----- 1600 -1 954 1442 LOW LOW NO 2 12 S 28 | ldnj02ou 15227 UVQSJ10112 DNJ 02 07 01 FUV TIME-TAG 719.0 2018.115:09:47:55 163/163 PSA 4 G160M ----- 1600 0 1778 2266 LOW LOW NO 2 12 S 29 | ldnj02ox 15227 UVQSJ10112 DNJ 02 09 01 FUV TIME-TAG 719.0 2018.115:10:46:53 163/163 PSA 4 G160M ----- 1600 1 5316 5804 LOW LOW NO 1 12 S 30 | ldnj02p5 15227 UVQSJ10112 DNJ 02 0B 01 FUV TIME-TAG 956.0 2018.115:11:01:53 163/163 PSA 4 G160M ----- 1623 -2 6216 6704 LOW LOW NO 2 12 S 31 | ldnj02p7 15227 UVQSJ10112 DNJ 02 0D 01 FUV TIME-TAG 955.0 2018.115:11:19:34 163/163 PSA 4 G160M ----- 1623 -1 7277 7765 LOW LOW NO 2 12 S 32 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 33 | lbdodtt7 12046 BDO DT 01 01 2018.115:23:00:00 COS MEMORY COPY -- DCE CODE IMAGE TO CS RAM 34 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 35 | ldkm02um 15220 SDSSJ12304 DKM 02 01 01 NUV ACQ/IMAGE 20.0 2018.116:02:53:01 PSA 4 NCM1 MIRRORB 0 0 268 190 NO 36 | ldkm02uo 15220 SDSSJ12304 DKM 02 03 01 FUV TIME-TAG 547.7 2018.116:02:57:43 163/100 PSA 4 G140L ----- 1105 -2 129 472 LOW OFF NO 2 7 S 37 | ldkm02uq 15220 SDSSJ12304 DKM 02 05 01 FUV TIME-TAG 547.7 2018.116:03:08:36 163/100 PSA 4 G140L ----- 1105 -1 782 1125 LOW OFF NO 2 7 S 38 | ldkm02us 15220 SDSSJ12304 DKM 02 07 01 FUV TIME-TAG 547.7 2018.116:03:19:29 163/100 PSA 4 G140L ----- 1105 0 1435 1778 LOW OFF NO 2 7 S 39 | ldkm02uu 15220 SDSSJ12304 DKM 02 09 01 FUV TIME-TAG 547.7 2018.116:03:30:22 163/100 PSA 4 G140L ----- 1105 1 2088 2431 LOW OFF NO 2 7 S 40 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 41 | ldgp25wy 15196 SDSSJ10173 DGP 25 01 01 NUV ACQ/IMAGE 30.0 2018.116:18:52:22 PSA 4 NCM1 MIRRORA 0 0 202 54585 NO 42 | ldgp25x0 15196 SDSSJ10173 DGP 25 03 01 FUV TIME-TAG 1081.0 2018.116:18:56:45 163/163 PSA 4 G130M ----- 1291 0 123 54848 LOW LOW NO 3 12 S 43 | ldgp25x6 15196 SDSSJ10173 DGP 25 05 01 FUV TIME-TAG 1081.0 2018.116:20:13:10 163/163 PSA 4 G130M ----- 1291 1 4708 59433 LOW LOW NO 2 12 S 44 | ldgp25x8 15196 SDSSJ10173 DGP 25 07 01 FUV TIME-TAG 1289.0 2018.116:20:34:22 163/163 PSA 4 G130M ----- 1291 0 5980 60705 LOW LOW NO 2 12 S 45 | ldgp25xa 15196 SDSSJ10173 DGP 25 09 01 FUV TIME-TAG 1289.0 2018.116:21:48:30 163/163 PSA 4 G130M ----- 1291 1 10428 65153 LOW LOW NO 2 12 S 46 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 47 | ldhz41z8 15075 SDSSJ12315 DHZ 41 01 01 NUV ACQ/IMAGE 100.0 2018.117:02:44:48 PSA 4 NCM1 MIRRORA 0 0 272 82931 NO 48 | ldhz41zb 15075 SDSSJ12315 DHZ 41 03 01 FUV TIME-TAG 1138.0 2018.117:02:50:32 163/100 PSA 4 G140L ----- 1105 0 129 83275 LOW OFF NO 3 7 S 49 | ldhz41zh 15075 SDSSJ12315 DHZ 41 05 01 FUV TIME-TAG 1138.0 2018.117:03:11:34 163/100 PSA 4 G140L ----- 1105 1 1391 84537 LOW OFF NO 2 7 S 50 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 51 | ldgp26zn 15196 SDSSJ10180 DGP 26 01 01 NUV ACQ/IMAGE 39.0 2018.117:04:12:13 PSA 4 NCM1 MIRRORA 0 0 211 88176 NO 52 | ldgp26zp 15196 SDSSJ10180 DGP 26 03 01 FUV TIME-TAG 1076.0 2018.117:04:16:45 163/163 PSA 4 G130M ----- 1291 0 123 88448 LOW LOW NO 3 12 S 53 | ldgp26zt 15196 SDSSJ10180 DGP 26 05 01 FUV TIME-TAG 1076.0 2018.117:04:36:36 163/163 PSA 4 G130M ----- 1291 1 1314 89639 LOW LOW NO 2 12 S 54 | ldgp26zy 15196 SDSSJ10180 DGP 26 07 01 FUV TIME-TAG 1289.0 2018.117:05:45:09 163/163 PSA 4 G130M ----- 1291 0 5427 93752 LOW LOW NO 2 12 S 55 | ldgp26a5 15196 SDSSJ10180 DGP 26 09 01 FUV TIME-TAG 1289.0 2018.117:06:08:42 163/163 PSA 4 G130M ----- 1291 1 6840 95165 LOW LOW NO 2 12 S 56 | ldgp26a8 15196 SDSSJ10180 DGP 26 0B 01 FUV TIME-TAG 1289.0 2018.117:07:20:28 163/163 PSA 4 G130M ----- 1291 0 11146 99471 LOW LOW NO 2 12 S 57 | ldgp26ag 15196 SDSSJ10180 DGP 26 0D 01 FUV TIME-TAG 1289.0 2018.117:07:44:01 163/163 PSA 4 G130M ----- 1291 1 12559 100884 LOW LOW NO 2 12 S 58 | ldgp26aj 15196 SDSSJ10180 DGP 26 0F 01 FUV TIME-TAG 1289.0 2018.117:08:55:48 163/163 PSA 4 G130M ----- 1291 0 16866 105191 LOW LOW NO 2 12 S 59 | ldgp26ao 15196 SDSSJ10180 DGP 26 0H 01 FUV TIME-TAG 1289.0 2018.117:09:19:21 163/163 PSA 4 G130M ----- 1291 1 18279 106604 LOW LOW NO 2 12 S 60 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 61 | ldj105el 15168 -PHI-LEO DJ1 05 01 01 FUV ACQ/SEARCH 4.0 2018.118:02:27:01 163/163 PSA 4 G130M ----- 1291 0 79939 168264 SCANSIZE=2 STEPSIZE=1767 FLUXWT 62 | ldj105em 15168 -PHI-LEO DJ1 05 02 01 FUV ACQ/PKD-XD 4.0 2018.118:02:30:05 163/163 PSA 4 G130M ----- 1291 0 80123 168448 DWELLS=5 STEPSIZE=1000 FLUXFLR 63 | ldj105eq 15168 -PHI-LEO DJ1 05 03 01 FUV ACQ/PEAKD 4.0 2018.118:02:33:13 163/163 PSA 4 G130M ----- 1291 0 80311 168636 DWELLS=5 STEPSIZE= 900 FLUXFLR 64 | ldj105eu 15168 -PHI-LEO DJ1 05 05 01 FUV TIME-TAG 2218.0 2018.118:02:37:35 163/163 PSA 4 G130M ----- 1291 0 80573 168898 LOW LOW NO 2 12 S 65 | ldj105fo 15168 -PHI-LEO DJ1 05 07 01 FUV TIME-TAG 2916.0 2018.118:04:01:39 163/163 PSA 4 G130M ----- 1291 1 85617 173942 LOW LOW NO 2 12 S 66 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 67 | ldgt04jd 15084 SDSS-J0915 DGT 04 01 01 NUV ACQ/IMAGE 111.0 2018.119:05:23:37 PSA 4 NCM1 MIRRORB 0 0 359 281 NO 68 | ldgt04jg 15084 SDSS-J0915 DGT 04 03 01 FUV TIME-TAG 1166.0 2018.119:05:29:50 163/100 PSA 4 G140L ----- 1105 -2 129 654 LOW OFF NO 3 7 S 69 | ldgt04jq 15084 SDSS-J0915 DGT 04 05 01 FUV TIME-TAG 1020.0 2018.119:05:51:49 163/100 PSA 4 G140L ----- 1105 -1 1448 1973 LOW OFF NO 2 7 S 70 | ldgt04ke 15084 SDSS-J0915 DGT 04 07 01 FUV TIME-TAG 1385.0 2018.119:06:55:40 163/100 PSA 4 G140L ----- 1105 0 5279 5804 LOW OFF NO 2 7 S 71 | ldgt04kr 15084 SDSS-J0915 DGT 04 09 01 FUV TIME-TAG 1385.0 2018.119:07:21:24 163/100 PSA 4 G140L ----- 1105 0 6823 7348 LOW OFF NO 2 7 S 72 | ldgt04kv 15084 SDSS-J0915 DGT 04 0B 01 FUV TIME-TAG 1385.0 2018.119:08:31:01 163/100 PSA 4 G140L ----- 1105 1 11000 11525 LOW OFF NO 2 7 S 73 | ldgt04kx 15084 SDSS-J0915 DGT 04 0D 01 FUV TIME-TAG 1385.0 2018.119:08:56:45 163/100 PSA 4 G140L ----- 1105 1 12544 13069 LOW OFF NO 2 7 S 74 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 75 | ldi203n9 15185 SDSSJ11500 DI2 03 01 01 NUV ACQ/IMAGE 42.0 2018.119:19:50:29 PSA 1 NCM1 MIRRORB 0 0 290 212 NO 76 | ldi203nb 15185 SDSSJ11500 DI2 03 03 01 NUV TIME-TAG 550.0 2018.119:19:55:53 PSA 1 NCM1 G185M 1913 -2 614 91 NO 2 17 S 77 | ldi203nd 15185 SDSSJ11500 DI2 03 05 01 NUV TIME-TAG 550.0 2018.119:20:06:30 PSA 1 NCM1 G185M 1913 -1 1251 728 NO 2 17 S 78 | ldi203nf 15185 SDSSJ11500 DI2 03 07 01 NUV TIME-TAG 550.0 2018.119:20:17:07 PSA 1 NCM1 G185M 1913 0 1888 1365 NO 2 17 S 79 | ldi203nh 15185 SDSSJ11500 DI2 03 09 01 NUV TIME-TAG 550.0 2018.119:20:27:44 PSA 1 NCM1 G185M 1913 1 2525 2002 NO 2 17 S 80 | ldi203nj 15185 SDSSJ11500 DI2 03 0B 01 FUV TIME-TAG 653.0 2018.119:21:23:43 163/163 PSA 4 G160M ----- 1623 -2 136 5361 LOW LOW NO 2 12 S 81 | ldi203nl 15185 SDSSJ11500 DI2 03 0D 01 FUV TIME-TAG 653.0 2018.119:21:36:21 163/163 PSA 4 G160M ----- 1623 -1 894 6119 LOW LOW NO 2 12 S 82 | ldi203nn 15185 SDSSJ11500 DI2 03 0F 01 FUV TIME-TAG 653.0 2018.119:21:48:59 163/163 PSA 4 G160M ----- 1623 0 1652 6877 LOW LOW NO 2 12 S 83 | ldi203np 15185 SDSSJ11500 DI2 03 0H 01 FUV TIME-TAG 653.0 2018.119:22:01:37 163/163 PSA 4 G160M ----- 1623 1 2410 7635 LOW LOW NO 1 12 S 84 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 85 | -------------------------------------------------------------------------------- /tests/data/181137c2.txt: -------------------------------------------------------------------------------- 1 | 2 | COS Exposure Report: SMS 181137C2 3 | 4 | Data Exposure Start FUV Mechanism Positions Cent Tsince Tsince Lamp STIMs Flash 5 | Filename Prop Target PRG OB AL EX Conf Opmode ExpTime yyyy.ddd:hh:mm:ss State Aper OSM1 OSM2 Wave FP OSM1 OSM2 Lamp Curr A B Dop # Dur T 6 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 7 | ldie05kf 15341 SDSSJ12420 DIE 05 01 01 NUV ACQ/IMAGE 161.0 2018.113:18:25:03 PSA 4 NCM1 MIRRORA 1850 0 333 66303 NO 8 | ldie05kh 15341 SDSSJ12420 DIE 05 03 01 FUV TIME-TAG 2220.0 2018.113:19:24:52 163/100 PSA 4 G140L ----- 1105 -2 129 69892 LOW OFF NO 3 7 S 9 | ldie05kj 15341 SDSSJ12420 DIE 05 05 01 FUV TIME-TAG 1641.0 2018.113:20:57:38 163/100 PSA 4 G140L ----- 1105 -1 5695 75458 LOW OFF NO 2 7 S 10 | ldie05kl 15341 SDSSJ12420 DIE 05 07 01 FUV TIME-TAG 1170.0 2018.113:22:24:44 163/100 PSA 4 G140L ----- 1105 0 10921 80684 LOW OFF NO 2 7 S 11 | ldie05kn 15341 SDSSJ12420 DIE 05 09 01 FUV TIME-TAG 900.0 2018.113:22:45:56 163/100 PSA 4 G140L ----- 1105 0 12193 81956 LOW OFF NO 2 7 S 12 | ldie05kp 15341 SDSSJ12420 DIE 05 0B 01 FUV TIME-TAG 1901.0 2018.114:00:00:06 163/100 PSA 4 G140L ----- 1105 1 16643 86406 LOW OFF NO 2 7 S 13 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 14 | ldq03il5 14940 DARK DQ0 3I 01 01 FUV TIME-TAG 1330.0 2018.114:00:46:22 163/163 PSA 4 G130M ----- 1291 0 731 89182 LOW LOW NO 15 | ldq03jlx 14940 DARK DQ0 3J 01 01 FUV TIME-TAG 1330.0 2018.114:01:20:12 163/163 PSA 4 G130M ----- 1291 0 2761 91212 LOW LOW NO 16 | ldq03km0 14940 DARK DQ0 3K 01 01 FUV TIME-TAG 1330.0 2018.114:01:49:31 163/163 PSA 4 G130M ----- 1291 0 4520 92971 LOW LOW NO 17 | ldq03lm3 14940 DARK DQ0 3L 01 01 FUV TIME-TAG 1330.0 2018.114:02:18:50 163/163 PSA 4 G130M ----- 1291 0 6279 94730 LOW LOW NO 18 | ldq03mm5 14940 DARK DQ0 3M 01 01 FUV TIME-TAG 1330.0 2018.114:02:48:09 163/163 PSA 4 G130M ----- 1291 0 8038 96489 LOW LOW NO 19 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 20 | ldk201oc 15147 IO-LEADING DK2 01 01 02 NUV ACQ/SEARCH 1.0 2018.115:06:27:29 PSA 4 NCM1 G230L 3000 0 180 75 SCANSIZE=3 STEPSIZE=1767 FLUXFLR 21 | ldk201oe 15147 IO-LEADING DK2 01 03 02 FUV TIME-TAG 249.0 2018.115:06:35:59 163/163 PSA 4 G130M ----- 1291 0 123 585 LOW LOW NO 2 12 S 22 | ldk2a1og 15147 IO-OFFSET- DK2 A1 01 02 FUV TIME-TAG 1341.0 2018.115:06:47:05 163/163 PSA 4 G130M ----- 1291 1 789 185 LOW LOW NO 2 12 S 23 | ldk2a1oi 15147 DARK DK2 A1 03 01 NUV TIME-TAG 4.0 2018.115:07:10:59 PSA 1 G130M MIRRORA 0 0 2223 1619 NO 24 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 25 | ldnj02ok 15227 UVQSJ10112 DNJ 02 01 01 NUV ACQ/IMAGE 75.0 2018.115:09:14:14 PSA 4 NCM1 MIRRORB 0 0 323 245 NO 26 | ldnj02on 15227 UVQSJ10112 DNJ 02 03 01 FUV TIME-TAG 719.0 2018.115:09:20:27 163/163 PSA 4 G160M ----- 1600 -2 130 618 LOW LOW NO 2 12 S 27 | ldnj02oq 15227 UVQSJ10112 DNJ 02 05 01 FUV TIME-TAG 719.0 2018.115:09:34:11 163/163 PSA 4 G160M ----- 1600 -1 954 1442 LOW LOW NO 2 12 S 28 | ldnj02ou 15227 UVQSJ10112 DNJ 02 07 01 FUV TIME-TAG 719.0 2018.115:09:47:55 163/163 PSA 4 G160M ----- 1600 0 1778 2266 LOW LOW NO 2 12 S 29 | ldnj02ox 15227 UVQSJ10112 DNJ 02 09 01 FUV TIME-TAG 719.0 2018.115:10:46:53 163/163 PSA 4 G160M ----- 1600 1 5316 5804 LOW LOW NO 1 12 S 30 | ldnj02p5 15227 UVQSJ10112 DNJ 02 0B 01 FUV TIME-TAG 956.0 2018.115:11:01:53 163/163 PSA 4 G160M ----- 1623 -2 6216 6704 LOW LOW NO 2 12 S 31 | ldnj02p7 15227 UVQSJ10112 DNJ 02 0D 01 FUV TIME-TAG 955.0 2018.115:11:19:34 163/163 PSA 4 G160M ----- 1623 -1 7277 7765 LOW LOW NO 2 12 S 32 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 33 | lbdodtt7 12046 BDO DT 01 01 2018.115:23:00:00 COS MEMORY COPY -- DCE CODE IMAGE TO CS RAM 34 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 35 | ldkm02um 15220 SDSSJ12304 DKM 02 01 01 NUV ACQ/IMAGE 20.0 2018.116:02:53:01 PSA 4 NCM1 MIRRORB 0 0 268 190 NO 36 | ldkm02uo 15220 SDSSJ12304 DKM 02 03 01 FUV TIME-TAG 547.7 2018.116:02:57:43 163/100 PSA 4 G140L ----- 1105 -2 129 472 LOW OFF NO 2 7 S 37 | ldkm02uq 15220 SDSSJ12304 DKM 02 05 01 FUV TIME-TAG 547.7 2018.116:03:08:36 163/100 PSA 4 G140L ----- 1105 -1 782 1125 LOW OFF NO 2 7 S 38 | ldkm02us 15220 SDSSJ12304 DKM 02 07 01 FUV TIME-TAG 547.7 2018.116:03:19:29 163/100 PSA 4 G140L ----- 1105 0 1435 1778 LOW OFF NO 2 7 S 39 | ldkm02uu 15220 SDSSJ12304 DKM 02 09 01 FUV TIME-TAG 547.7 2018.116:03:30:22 163/100 PSA 4 G140L ----- 1105 1 2088 2431 LOW OFF NO 2 7 S 40 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 41 | ldgp25wy 15196 SDSSJ10173 DGP 25 01 01 NUV ACQ/IMAGE 30.0 2018.116:18:52:22 PSA 4 NCM1 MIRRORA 0 0 202 54585 NO 42 | ldgp25x0 15196 SDSSJ10173 DGP 25 03 01 FUV TIME-TAG 1081.0 2018.116:18:56:45 163/163 PSA 4 G130M ----- 1291 0 123 54848 LOW LOW NO 3 12 S 43 | ldgp25x6 15196 SDSSJ10173 DGP 25 05 01 FUV TIME-TAG 1081.0 2018.116:20:13:10 163/163 PSA 4 G130M ----- 1291 1 4708 59433 LOW LOW NO 2 12 S 44 | ldgp25x8 15196 SDSSJ10173 DGP 25 07 01 FUV TIME-TAG 1289.0 2018.116:20:34:22 163/163 PSA 4 G130M ----- 1291 0 5980 60705 LOW LOW NO 2 12 S 45 | ldgp25xa 15196 SDSSJ10173 DGP 25 09 01 FUV TIME-TAG 1289.0 2018.116:21:48:30 163/163 PSA 4 G130M ----- 1291 1 10428 65153 LOW LOW NO 2 12 S 46 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 47 | ldhz41z8 15075 SDSSJ12315 DHZ 41 01 01 NUV ACQ/IMAGE 100.0 2018.117:02:44:48 PSA 4 NCM1 MIRRORA 0 0 272 82931 NO 48 | ldhz41zb 15075 SDSSJ12315 DHZ 41 03 01 FUV TIME-TAG 1138.0 2018.117:02:50:32 163/100 PSA 4 G140L ----- 1105 0 129 83275 LOW OFF NO 3 7 S 49 | ldhz41zh 15075 SDSSJ12315 DHZ 41 05 01 FUV TIME-TAG 1138.0 2018.117:03:11:34 163/100 PSA 4 G140L ----- 1105 1 1391 84537 LOW OFF NO 2 7 S 50 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 51 | ldgp26zn 15196 SDSSJ10180 DGP 26 01 01 NUV ACQ/IMAGE 39.0 2018.117:04:12:13 PSA 4 NCM1 MIRRORA 0 0 211 88176 NO 52 | ldgp26zp 15196 SDSSJ10180 DGP 26 03 01 FUV TIME-TAG 1076.0 2018.117:04:16:45 163/163 PSA 4 G130M ----- 1291 0 123 88448 LOW LOW NO 3 12 S 53 | ldgp26zt 15196 SDSSJ10180 DGP 26 05 01 FUV TIME-TAG 1076.0 2018.117:04:36:36 163/163 PSA 4 G130M ----- 1291 1 1314 89639 LOW LOW NO 2 12 S 54 | ldgp26zy 15196 SDSSJ10180 DGP 26 07 01 FUV TIME-TAG 1289.0 2018.117:05:45:09 163/163 PSA 4 G130M ----- 1291 0 5427 93752 LOW LOW NO 2 12 S 55 | ldgp26a5 15196 SDSSJ10180 DGP 26 09 01 FUV TIME-TAG 1289.0 2018.117:06:08:42 163/163 PSA 4 G130M ----- 1291 1 6840 95165 LOW LOW NO 2 12 S 56 | ldgp26a8 15196 SDSSJ10180 DGP 26 0B 01 FUV TIME-TAG 1289.0 2018.117:07:20:28 163/163 PSA 4 G130M ----- 1291 0 11146 99471 LOW LOW NO 2 12 S 57 | ldgp26ag 15196 SDSSJ10180 DGP 26 0D 01 FUV TIME-TAG 1289.0 2018.117:07:44:01 163/163 PSA 4 G130M ----- 1291 1 12559 100884 LOW LOW NO 2 12 S 58 | ldgp26aj 15196 SDSSJ10180 DGP 26 0F 01 FUV TIME-TAG 1289.0 2018.117:08:55:48 163/163 PSA 4 G130M ----- 1291 0 16866 105191 LOW LOW NO 2 12 S 59 | ldgp26ao 15196 SDSSJ10180 DGP 26 0H 01 FUV TIME-TAG 1289.0 2018.117:09:19:21 163/163 PSA 4 G130M ----- 1291 1 18279 106604 LOW LOW NO 2 12 S 60 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 61 | ldj105el 15168 -PHI-LEO DJ1 05 01 01 FUV ACQ/SEARCH 4.0 2018.118:02:27:01 163/163 PSA 4 G130M ----- 1291 0 79939 168264 SCANSIZE=2 STEPSIZE=1767 FLUXWT 62 | ldj105em 15168 -PHI-LEO DJ1 05 02 01 FUV ACQ/PKD-XD 4.0 2018.118:02:30:05 163/163 PSA 4 G130M ----- 1291 0 80123 168448 DWELLS=5 STEPSIZE=1000 FLUXFLR 63 | ldj105eq 15168 -PHI-LEO DJ1 05 03 01 FUV ACQ/PEAKD 4.0 2018.118:02:33:13 163/163 PSA 4 G130M ----- 1291 0 80311 168636 DWELLS=5 STEPSIZE= 900 FLUXFLR 64 | ldj105eu 15168 -PHI-LEO DJ1 05 05 01 FUV TIME-TAG 2218.0 2018.118:02:37:35 163/163 PSA 4 G130M ----- 1291 0 80573 168898 LOW LOW NO 2 12 S 65 | ldj105fo 15168 -PHI-LEO DJ1 05 07 01 FUV TIME-TAG 2916.0 2018.118:04:01:39 163/163 PSA 4 G130M ----- 1291 1 85617 173942 LOW LOW NO 2 12 S 66 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 67 | ldgt04jd 15084 SDSS-J0915 DGT 04 01 01 NUV ACQ/IMAGE 111.0 2018.119:05:23:37 PSA 4 NCM1 MIRRORB 0 0 359 281 NO 68 | ldgt04jg 15084 SDSS-J0915 DGT 04 03 01 FUV TIME-TAG 1166.0 2018.119:05:29:50 163/100 PSA 4 G140L ----- 1105 -2 129 654 LOW OFF NO 3 7 S 69 | ldgt04jq 15084 SDSS-J0915 DGT 04 05 01 FUV TIME-TAG 1020.0 2018.119:05:51:49 163/100 PSA 4 G140L ----- 1105 -1 1448 1973 LOW OFF NO 2 7 S 70 | ldgt04ke 15084 SDSS-J0915 DGT 04 07 01 FUV TIME-TAG 1385.0 2018.119:06:55:40 163/100 PSA 4 G140L ----- 1105 0 5279 5804 LOW OFF NO 2 7 S 71 | ldgt04kr 15084 SDSS-J0915 DGT 04 09 01 FUV TIME-TAG 1385.0 2018.119:07:21:24 163/100 PSA 4 G140L ----- 1105 0 6823 7348 LOW OFF NO 2 7 S 72 | ldgt04kv 15084 SDSS-J0915 DGT 04 0B 01 FUV TIME-TAG 1385.0 2018.119:08:31:01 163/100 PSA 4 G140L ----- 1105 1 11000 11525 LOW OFF NO 2 7 S 73 | ldgt04kx 15084 SDSS-J0915 DGT 04 0D 01 FUV TIME-TAG 1385.0 2018.119:08:56:45 163/100 PSA 4 G140L ----- 1105 1 12544 13069 LOW OFF NO 2 7 S 74 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 75 | ldi203n6 15185 SDSSJ11500 DI2 03 01 01 NUV ACQ/IMAGE 42.0 2018.119:19:50:29 PSA 1 NCM1 MIRRORB 0 0 290 212 NO 76 | ldi203n8 15185 SDSSJ11500 DI2 03 03 01 NUV TIME-TAG 550.0 2018.119:19:55:53 PSA 1 NCM1 G185M 1913 -2 614 91 NO 2 17 S 77 | ldi203na 15185 SDSSJ11500 DI2 03 05 01 NUV TIME-TAG 550.0 2018.119:20:06:30 PSA 1 NCM1 G185M 1913 -1 1251 728 NO 2 17 S 78 | ldi203nc 15185 SDSSJ11500 DI2 03 07 01 NUV TIME-TAG 550.0 2018.119:20:17:07 PSA 1 NCM1 G185M 1913 0 1888 1365 NO 2 17 S 79 | ldi203ne 15185 SDSSJ11500 DI2 03 09 01 NUV TIME-TAG 550.0 2018.119:20:27:44 PSA 1 NCM1 G185M 1913 1 2525 2002 NO 2 17 S 80 | ldi203ng 15185 SDSSJ11500 DI2 03 0B 01 FUV TIME-TAG 653.0 2018.119:21:23:43 163/163 PSA 4 G160M ----- 1623 -2 136 5361 LOW LOW NO 2 12 S 81 | ldi203ni 15185 SDSSJ11500 DI2 03 0D 01 FUV TIME-TAG 653.0 2018.119:21:36:21 163/163 PSA 4 G160M ----- 1623 -1 894 6119 LOW LOW NO 2 12 S 82 | ldi203nk 15185 SDSSJ11500 DI2 03 0F 01 FUV TIME-TAG 653.0 2018.119:21:48:59 163/163 PSA 4 G160M ----- 1623 0 1652 6877 LOW LOW NO 2 12 S 83 | ldi203nm 15185 SDSSJ11500 DI2 03 0H 01 FUV TIME-TAG 653.0 2018.119:22:01:37 163/163 PSA 4 G160M ----- 1623 1 2410 7635 LOW LOW NO 1 12 S 84 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 85 | -------------------------------------------------------------------------------- /tests/test_filesystem.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | import numpy as np 4 | 5 | from astropy.io import fits 6 | from shutil import copy 7 | 8 | from cosmo.filesystem import ( 9 | data_from_exposures, 10 | data_from_jitters, 11 | FileData, 12 | SPTData, 13 | ReferenceData, 14 | JitterFileData, 15 | find_files, 16 | get_exposure_data, 17 | get_jitter_data 18 | ) 19 | 20 | 21 | @pytest.fixture(scope='class') 22 | def file_data(data_dir): 23 | file = os.path.join(data_dir, 'ld1ce4dkq_lampflash.fits.gz') 24 | header_request = {0: ['ROOTNAME'], 1: ['EXPSTART']} 25 | table_request = {1: ['TIME', 'SHIFT_DISP', 'SHIFT_XDISP']} 26 | 27 | with fits.open(file) as f: 28 | test_data = FileData(f, header_request=header_request, table_request=table_request) 29 | 30 | return test_data 31 | 32 | 33 | @pytest.fixture(scope='class') 34 | def ref_data(data_dir): 35 | file = os.path.join(data_dir, 'ld1ce4dkq_lampflash.fits.gz') 36 | match_keys = ['OPT_ELEM', 'CENWAVE', 'FPOFFSET'] 37 | header_request = {0: ['DETECTOR']} 38 | table_request = {1: ['SEGMENT', 'FP_PIXEL_SHIFT']} 39 | 40 | with fits.open(file) as f: 41 | ref_data = ReferenceData( 42 | f, 43 | 'LAMPTAB', 44 | match_keys=match_keys, 45 | header_request=header_request, 46 | table_request=table_request 47 | ) 48 | 49 | return ref_data 50 | 51 | 52 | @pytest.fixture(scope='class') 53 | def spt_data(data_dir): 54 | file = os.path.join(data_dir, 'ld3la1csq_rawacq.fits.gz') 55 | header_request = {0: ['INSTRUME']} 56 | 57 | return SPTData(file, header_request=header_request) 58 | 59 | 60 | @pytest.fixture 61 | def cosmo_layout_testdir(data_dir): 62 | # Set up the test cosmo-style directory 63 | cosmo_dir = os.path.join(data_dir, '11111') # 11111 is a fake directory; matches the program directory pattern 64 | 65 | os.mkdir(cosmo_dir) 66 | copy(os.path.join(data_dir, 'lb4c10niq_lampflash.fits.gz'), cosmo_dir) 67 | 68 | yield 69 | 70 | os.remove(os.path.join(cosmo_dir, 'lb4c10niq_lampflash.fits.gz')) 71 | os.rmdir(cosmo_dir) 72 | 73 | 74 | @pytest.fixture(params=['ldngz2szj_jit.fits.gz', 'ldxe02010_jit.fits.gz'], scope='class') 75 | def jitter_file(data_dir, request): 76 | file = os.path.join(data_dir, request.param) 77 | primary_header_keys = ['PROPOSID', 'CONFIG'] 78 | ext_header_keys = ['EXPNAME'] 79 | table_keys = ['SI_V2_AVG'] 80 | 81 | return JitterFileData( 82 | file, 83 | primary_header_keys=primary_header_keys, 84 | ext_header_keys=ext_header_keys, 85 | table_keys=table_keys 86 | ) 87 | 88 | 89 | @pytest.fixture(scope='class') 90 | def exposure_data(data_dir): 91 | file = os.path.join(data_dir, 'lb4c10niq_lampflash.fits.gz') 92 | header_request = {0: ['ROOTNAME'], 1: ['EXPSTART']} 93 | table_request = {1: ['TIME', 'SHIFT_DISP']} 94 | spt_header_request = {1: ['LQTDFINI']} 95 | reference_request = { 96 | 'LAMPTAB': { 97 | 'match_keys': ['OPT_ELEM', 'CENWAVE', 'FPOFFSET'], 98 | 'table_request': {1: ['FP_PIXEL_SHIFT', 'SEGMENT']} 99 | }, 100 | 'WCPTAB': { 101 | 'match_keys': ['OPT_ELEM'], 102 | 'table_request': {1: ['XC_RANGE', 'SEARCH_OFFSET']} 103 | } 104 | } 105 | 106 | return get_exposure_data( 107 | file, 108 | header_request=header_request, 109 | table_request=table_request, 110 | spt_header_request=spt_header_request, 111 | reference_request=reference_request 112 | ) 113 | 114 | 115 | @pytest.fixture(scope='class') 116 | def jitter_data(data_dir): 117 | file = os.path.join(data_dir, 'ldxe02010_jit.fits.gz') 118 | primary_header_keys = ['PROPOSID'] 119 | ext_header_keys = ['EXPNAME'] 120 | table_keys = ['SI_V2_AVG'] 121 | reduce_to_stats = {'SI_V2_AVG': ('mean', 'std', 'max')} 122 | 123 | return get_jitter_data( 124 | file, 125 | primary_header_keys=primary_header_keys, 126 | ext_header_keys=ext_header_keys, 127 | table_keys=table_keys, 128 | get_expstart=False, 129 | reduce_to_stats=reduce_to_stats 130 | ) 131 | 132 | 133 | @pytest.fixture(scope='class') 134 | def multi_exposure_data(data_dir): 135 | files = find_files('*rawacq*', data_dir=data_dir, subdir_pattern=None) 136 | header_request = {0: ['ROOTNAME']} 137 | 138 | return data_from_exposures(files, header_request=header_request) 139 | 140 | 141 | @pytest.fixture(scope='class') 142 | def multi_jitter_data(data_dir): 143 | files = find_files('*jit*', data_dir=data_dir, subdir_pattern=None) 144 | primary_header_keys = ['PROPOSID'] 145 | ext_header_keys = ['EXPNAME'] 146 | 147 | return data_from_jitters( 148 | files, 149 | primary_header_keys=primary_header_keys, 150 | ext_header_keys=ext_header_keys, 151 | get_expstart=False 152 | ) 153 | 154 | 155 | class TestFindFiles: 156 | 157 | def test_fails_for_bad_dir(self): 158 | assert not find_files('*', 'doesnotexist', subdir_pattern=None) 159 | 160 | def test_finds_files(self, data_dir): 161 | files = find_files('*lampflash*', data_dir=data_dir, subdir_pattern=None) 162 | 163 | assert len(files) == 11 164 | 165 | @pytest.mark.usefixtures("cosmo_layout_testdir") 166 | def test_finds_files_cosmo_layout(self, data_dir): 167 | files = find_files('*', data_dir=data_dir) 168 | 169 | assert len(files) == 51 # only one "program" directory with only one file in it 170 | 171 | 172 | class TestFileData: 173 | 174 | def test_header_request(self, file_data): 175 | assert 'ROOTNAME' in file_data 176 | assert file_data['ROOTNAME'] == 'ld1ce4dkq' 177 | 178 | def test_table_request(self, file_data): 179 | assert 'TIME' in file_data and 'SHIFT_DISP' in file_data and 'SHIFT_XDISP' in file_data 180 | assert isinstance(file_data['TIME'], np.ndarray) 181 | 182 | def test_header_request_fails_without_defaults(self, data_dir): 183 | file = os.path.join(data_dir, 'lb4c10niq_lampflash.fits.gz') 184 | header_request = {0: ['fake']} 185 | 186 | with pytest.raises(KeyError): 187 | with fits.open(file) as f: 188 | FileData(f, header_request=header_request) 189 | 190 | def test_header_request_defaults(self, data_dir): 191 | file = os.path.join(data_dir, 'lb4c10niq_lampflash.fits.gz') 192 | header_request = {0: ['fake']} 193 | header_defaults = {'fake': 'definitely fake'} 194 | 195 | with fits.open(file) as f: 196 | test_data = FileData(f, header_request=header_request, header_defaults=header_defaults) 197 | 198 | assert 'fake' in test_data 199 | assert test_data['fake'] == 'definitely fake' 200 | 201 | def test_bytestr_conversion(self, data_dir): 202 | file = os.path.join(data_dir, 'lb4c10niq_lampflash.fits.gz') 203 | header_request = {0: ['byte']} 204 | header_defaults = {'byte': np.array([b'here be bytes'])} 205 | 206 | with fits.open(file) as f: 207 | test_data = FileData(f, header_request=header_request, header_defaults=header_defaults) 208 | 209 | assert 'byte' in test_data 210 | assert test_data['byte'].dtype == np.dtype(' 1: 264 | assert data['EXPSTART'] == 0 and data['EXPTYPE'] == 'N/A' # associated raw files are not in the dataset 265 | 266 | assert 'PROPOSID' in data and 'CONFIG' in data and 'EXPNAME' in data 267 | 268 | def test_extension_data(self, jitter_file): 269 | if len(jitter_file) > 1: 270 | assert len(jitter_file) == 4 # the jitter association file includes 4 extensions apart from the primary 271 | 272 | def test_table_request(self, jitter_file): 273 | for data in jitter_file: 274 | assert 'SI_V2_AVG' in data 275 | assert all(data['SI_V2_AVG'] < 1e30) # Make sure placeholder data is properly removed 276 | 277 | def test_reduce_to_stat(self, jitter_file): 278 | reduce = {'SI_V2_AVG': ('mean', 'std', 'max')} 279 | 280 | jitter_file.reduce_to_stat(reduce) 281 | 282 | for data in jitter_file: 283 | assert 'SI_V2_AVG' not in data 284 | assert 'SI_V2_AVG_mean' in data and 'SI_V2_AVG_std' in data and 'SI_V2_AVG_max' in data 285 | 286 | 287 | class TestGetExposureData: 288 | 289 | def test_recovered_length(self, exposure_data): 290 | assert len(exposure_data) == 10 # All request keys/columns 291 | 292 | def test_header_requests(self, exposure_data): 293 | assert 'ROOTNAME' in exposure_data and exposure_data['ROOTNAME'] == 'lb4c10niq' 294 | assert 'EXPSTART' in exposure_data and exposure_data['EXPSTART'] == 55202.48302439 295 | assert 'LQTDFINI' in exposure_data and exposure_data['LQTDFINI'] == 'TDF Up' 296 | 297 | def test_table_requests(self, exposure_data): 298 | assert ( 299 | 'TIME' in exposure_data and 300 | np.allclose( # Using isclose due to dtype discrepancies 301 | exposure_data['TIME'], 302 | [4.32000017, 4.32000017, 4.32000017, 2404.35205078, 2404.35205078, 2404.35205078] 303 | ) 304 | ) 305 | 306 | assert ( 307 | 'SHIFT_DISP' in exposure_data and 308 | np.allclose( 309 | exposure_data['SHIFT_DISP'], 310 | [-23.67234, -23.992914, -24.230333, -23.85408, -23.90261, -24.0943] 311 | ) 312 | ) 313 | 314 | assert 'SEGMENT' in exposure_data and len(exposure_data['SEGMENT']) == 3 # NUV 315 | assert 'FP_PIXEL_SHIFT' in exposure_data and len(exposure_data['FP_PIXEL_SHIFT']) == 3 316 | assert 'XC_RANGE' in exposure_data 317 | assert 'SEARCH_OFFSET' in exposure_data 318 | 319 | 320 | class TestDataFromExposures: 321 | 322 | def test_length(self, multi_exposure_data): 323 | assert len(multi_exposure_data) == 9 324 | 325 | def test_data_collection(self, multi_exposure_data): 326 | test_data = sorted([filedata['ROOTNAME'] for filedata in multi_exposure_data]) 327 | actual = sorted( 328 | [ 329 | 'ld3la1csq', 330 | 'ldngz2szq', 331 | 'ldi404zsq', 332 | 'ldi405e3q', 333 | 'lb4l02m1q', 334 | 'lbhx1epiq', 335 | 'ldng01chq', 336 | 'lb4l02m2q', 337 | 'ldng05huq' 338 | ] 339 | ) 340 | 341 | assert test_data == actual 342 | 343 | 344 | class TestGetJitterData: 345 | 346 | def test_length(self, jitter_data): 347 | assert len(jitter_data) == 4 348 | 349 | def test_header_data(self, jitter_data): 350 | for data in jitter_data: 351 | assert 'PROPOSID' in data and 'EXPNAME' in data 352 | 353 | def test_table_data(self, jitter_data): 354 | for data in jitter_data: 355 | assert 'SI_V2_AVG_mean' in data and 'SI_V2_AVG_std' in data and 'SI_V2_AVG_max' in data 356 | assert 'SI_V2_AVG' not in data 357 | 358 | 359 | class TestDataFromJitters: 360 | 361 | def test_length(self, multi_jitter_data): 362 | assert len(multi_jitter_data) == 6 363 | 364 | def test_data(self, multi_jitter_data): 365 | for data in multi_jitter_data: 366 | assert 'PROPOSID' in data and 'EXPNAME' in data 367 | -------------------------------------------------------------------------------- /tests/data/161368a8.txt: -------------------------------------------------------------------------------- 1 | 2 | COS Exposure Report: SMS 161368A8 3 | 4 | Data Exposure Start FUV Mechanism Positions Cent Tsince Tsince Lamp STIMs Flash 5 | Filename Prop Target PRG OB AL EX Conf Opmode ExpTime yyyy.ddd:hh:mm:ss State Aper OSM1 OSM2 Wave FP OSM1 OSM2 Lamp Curr A B Dop # Dur T 6 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 7 | lcvw08fs 14140 2MASS-J134 CVW 08 01 01 FUV ACQ/PEAKXD 64.2 2016.137:15:54:54 167/169 PSA B G160M ----- 1577 0 150 58494 OFF OFF NO 8 | lcvw08ft 14140 2MASS-J134 CVW 08 02 01 FUV ACQ/PEAKD 34.3 2016.137:15:56:39 167/169 PSA B G160M ----- 1577 0 255 58599 DWELLS=3 STEPSIZE=1300 FLUXWT 9 | lcvw08fw 14140 2MASS-J134 CVW 08 04 01 FUV TIME-TAG 1149.0 2016.137:16:02:41 167/169 PSA B G160M ----- 1577 -2 617 58961 LOW LOW NO 2 12 S 10 | lcvw08g4 14140 2MASS-J134 CVW 08 06 01 FUV TIME-TAG 1149.0 2016.137:16:23:35 167/169 PSA B G160M ----- 1577 -1 1871 60215 LOW LOW NO 2 12 S 11 | lcvw08g8 14140 2MASS-J134 CVW 08 08 01 FUV TIME-TAG 1419.0 2016.137:17:29:04 167/169 PSA B G160M ----- 1577 0 5800 64144 LOW LOW NO 2 12 S 12 | lcvw08gf 14140 2MASS-J134 CVW 08 0A 01 FUV TIME-TAG 1419.0 2016.137:17:54:28 167/169 PSA B G160M ----- 1577 1 7324 65668 LOW LOW NO 2 12 S 13 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 14 | lcwe06j0 14145 J1127+1154 CWE 06 01 01 NUV ACQ/IMAGE 75.9 2016.138:06:06:29 PSA B NCM1 MIRRORB 0 0 329 245 NO 15 | lcwe06j2 14145 J1127+1154 CWE 06 03 01 FUV TIME-TAG 2376.0 2016.138:06:12:47 167/169 PSA B G160M ----- 1577 -2 129 623 LOW LOW NO 3 12 S 16 | lcwe06j9 14145 J1127+1154 CWE 06 05 01 FUV TIME-TAG 2932.0 2016.138:07:38:55 167/169 PSA B G160M ----- 1577 -1 5297 5791 LOW LOW NO 2 12 S 17 | lcwe06jh 14145 J1127+1154 CWE 06 07 01 FUV TIME-TAG 2932.0 2016.138:09:14:19 167/169 PSA B G160M ----- 1611 0 11021 11515 LOW LOW NO 2 12 S 18 | lcwe06jp 14145 J1127+1154 CWE 06 09 01 FUV TIME-TAG 2932.0 2016.138:10:49:42 167/169 PSA B G160M ----- 1611 1 16744 17238 LOW LOW NO 2 12 S 19 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 20 | lbdoayjx 12046 BDO AY 01 01 2016.138:11:48:04 COS MEMORY COPY -- DCE CODE IMAGE TO CS RAM 21 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 22 | lcvl06jz 14137 Q1328+2159 CVL 06 01 01 NUV ACQ/IMAGE 60.0 2016.138:12:36:01 PSA B NCM1 MIRRORA 0 0 237 3260 NO 23 | lcvl06k1 14137 Q1328+2159 CVL 06 03 01 FUV TIME-TAG 1165.0 2016.138:12:41:06 167/169 PSA B G140L ----- 1280 -2 126 3565 LOW LOW NO 3 7 S 24 | lcvl06k3 14137 Q1328+2159 CVL 06 05 01 FUV TIME-TAG 1171.0 2016.138:13:02:26 167/169 PSA B G140L ----- 1280 -1 1406 4845 LOW LOW NO 2 7 S 25 | lcvl06k7 14137 Q1328+2159 CVL 06 07 01 FUV TIME-TAG 1415.0 2016.138:14:08:25 167/169 PSA B G140L ----- 1280 0 5365 8804 LOW LOW NO 2 7 S 26 | lcvl06k9 14137 Q1328+2159 CVL 06 09 01 FUV TIME-TAG 1406.0 2016.138:14:33:55 167/169 PSA B G140L ----- 1280 1 6895 10334 LOW LOW NO 2 7 S 27 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 28 | lcwj14o9 14265 HE1211-132 CWJ 14 01 01 NUV ACQ/IMAGE 27.0 2016.139:09:09:33 PSA O NCM1 MIRRORB 0 0 280 196 NO 29 | lcwj14ob 14265 HE1211-132 CWJ 14 03 01 NUV TIME-TAG 568.0 2016.139:09:13:38 PSA O NCM1 G225M 2373 -2 525 87 NO 2 22 S 30 | lcwj14of 14265 HE1211-132 CWJ 14 05 01 NUV TIME-TAG 568.0 2016.139:09:24:43 PSA O NCM1 G225M 2373 -1 1190 752 NO 2 22 S 31 | lcwj14oi 14265 HE1211-132 CWJ 14 07 01 NUV TIME-TAG 568.0 2016.139:09:35:48 PSA O NCM1 G225M 2373 0 1855 1417 NO 2 22 S 32 | lcwj14ok 14265 HE1211-132 CWJ 14 09 01 NUV TIME-TAG 567.0 2016.139:09:46:53 PSA O NCM1 G225M 2373 1 2520 2082 NO 2 22 S 33 | lcwj14om 14265 HE1211-132 CWJ 14 0B 01 NUV TIME-TAG 670.0 2016.139:10:42:17 PSA O NCM1 G225M 2373 -2 5844 5406 NO 1 22 S 34 | lcwj14ot 14265 HE1211-132 CWJ 14 0D 01 NUV TIME-TAG 670.0 2016.139:10:55:04 PSA O NCM1 G225M 2373 -1 6611 6173 NO 1 22 S 35 | lcwj14ov 14265 HE1211-132 CWJ 14 0F 01 NUV TIME-TAG 669.0 2016.139:11:07:51 PSA O NCM1 G225M 2373 0 7378 6940 NO 1 22 S 36 | lcwj14ox 14265 HE1211-132 CWJ 14 0H 01 NUV TIME-TAG 669.0 2016.139:11:20:37 PSA O NCM1 G225M 2373 1 8144 7706 NO 1 22 S 37 | lcwj14p0 14265 HE1211-132 CWJ 14 0J 01 NUV TIME-TAG 670.0 2016.139:12:17:43 PSA O NCM1 G225M 2373 -2 11570 11132 NO 1 22 S 38 | lcwj14p7 14265 HE1211-132 CWJ 14 0L 01 NUV TIME-TAG 670.0 2016.139:12:30:30 PSA O NCM1 G225M 2373 -1 12337 11899 NO 1 22 S 39 | lcwj14p9 14265 HE1211-132 CWJ 14 0N 01 NUV TIME-TAG 669.0 2016.139:12:43:17 PSA O NCM1 G225M 2373 0 13104 12666 NO 1 22 S 40 | lcwj14pb 14265 HE1211-132 CWJ 14 0P 01 NUV TIME-TAG 669.0 2016.139:12:56:03 PSA O NCM1 G225M 2373 1 13870 13432 NO 1 22 S 41 | lcwj14pe 14265 HE1211-132 CWJ 14 0R 01 NUV TIME-TAG 670.0 2016.139:13:53:09 PSA O NCM1 G225M 2373 -2 17296 16858 NO 1 22 S 42 | lcwj14pl 14265 HE1211-132 CWJ 14 0T 01 NUV TIME-TAG 670.0 2016.139:14:05:56 PSA O NCM1 G225M 2373 -1 18063 17625 NO 1 22 S 43 | lcwj14pn 14265 HE1211-132 CWJ 14 0V 01 NUV TIME-TAG 669.0 2016.139:14:18:43 PSA O NCM1 G225M 2373 0 18830 18392 NO 1 22 S 44 | lcwj14pp 14265 HE1211-132 CWJ 14 0X 01 NUV TIME-TAG 669.0 2016.139:14:31:29 PSA O NCM1 G225M 2373 1 19596 19158 NO 1 22 S 45 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 46 | ld1ce3yu 14429 ANY D1C E3 04 07 FUV TIME-TAG 2900.0 2016.141:01:22:44 167/169 PSA B G130M ----- 1291 0 124678 124572 LOW LOW NO 2 12 S 47 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 48 | lcya04av 14171 GALEX-J114 CYA 04 01 01 NUV ACQ/IMAGE 17.1 2016.141:06:20:46 PSA B NCM1 MIRRORA 0 0 195 142454 NO 49 | lcya04ax 14171 GALEX-J114 CYA 04 03 01 FUV TIME-TAG 2526.0 2016.141:07:12:52 167/169 PSA B G140L ----- 1280 -2 126 145580 LOW LOW NO 3 7 S 50 | lcya04b4 14171 GALEX-J114 CYA 04 05 01 FUV TIME-TAG 2927.0 2016.141:08:48:15 167/169 PSA B G140L ----- 1280 -1 5849 151303 LOW LOW NO 2 7 S 51 | lcya04bb 14171 GALEX-J114 CYA 04 07 01 FUV TIME-TAG 2927.0 2016.141:10:23:39 167/169 PSA B G140L ----- 1280 0 11573 157027 LOW LOW NO 2 7 S 52 | lcya04bi 14171 GALEX-J114 CYA 04 09 01 FUV TIME-TAG 2927.0 2016.141:11:59:03 167/169 PSA B G140L ----- 1280 1 17297 162751 LOW LOW NO 2 7 S 53 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 54 | lcwj15bq 14265 HE1211-132 CWJ 15 01 01 NUV ACQ/IMAGE 27.0 2016.141:13:38:47 PSA O NCM1 MIRRORB 0 0 280 196 NO 55 | lcwj15bs 14265 HE1211-132 CWJ 15 03 01 NUV TIME-TAG 568.0 2016.141:13:42:52 PSA O NCM1 G225M 2410 -2 525 87 NO 2 7 S 56 | lcwj15bw 14265 HE1211-132 CWJ 15 05 01 NUV TIME-TAG 568.0 2016.141:13:53:57 PSA O NCM1 G225M 2410 -1 1190 752 NO 2 7 S 57 | lcwj15bz 14265 HE1211-132 CWJ 15 07 01 NUV TIME-TAG 568.0 2016.141:14:05:02 PSA O NCM1 G225M 2410 0 1855 1417 NO 2 7 S 58 | lcwj15c1 14265 HE1211-132 CWJ 15 09 01 NUV TIME-TAG 567.0 2016.141:14:16:07 PSA O NCM1 G225M 2410 1 2520 2082 NO 2 7 S 59 | lcwj15c3 14265 HE1211-132 CWJ 15 0B 01 NUV TIME-TAG 670.0 2016.141:15:11:31 PSA O NCM1 G225M 2410 -2 5844 5406 NO 1 7 S 60 | lcwj15ca 14265 HE1211-132 CWJ 15 0D 01 NUV TIME-TAG 670.0 2016.141:15:24:18 PSA O NCM1 G225M 2410 -1 6611 6173 NO 1 7 S 61 | lcwj15cc 14265 HE1211-132 CWJ 15 0F 01 NUV TIME-TAG 669.0 2016.141:15:37:05 PSA O NCM1 G225M 2410 0 7378 6940 NO 1 7 S 62 | lcwj15ce 14265 HE1211-132 CWJ 15 0H 01 NUV TIME-TAG 669.0 2016.141:15:49:51 PSA O NCM1 G225M 2410 1 8144 7706 NO 1 7 S 63 | lcwj15ch 14265 HE1211-132 CWJ 15 0J 01 NUV TIME-TAG 670.0 2016.141:16:46:57 PSA O NCM1 G225M 2410 -2 11570 11132 NO 1 7 S 64 | lcwj15co 14265 HE1211-132 CWJ 15 0L 01 NUV TIME-TAG 670.0 2016.141:16:59:44 PSA O NCM1 G225M 2410 -1 12337 11899 NO 1 7 S 65 | lcwj15cq 14265 HE1211-132 CWJ 15 0N 01 NUV TIME-TAG 669.0 2016.141:17:12:31 PSA O NCM1 G225M 2410 0 13104 12666 NO 1 7 S 66 | lcwj15cs 14265 HE1211-132 CWJ 15 0P 01 NUV TIME-TAG 669.0 2016.141:17:25:17 PSA O NCM1 G225M 2410 1 13870 13432 NO 1 7 S 67 | lcwj15cv 14265 HE1211-132 CWJ 15 0R 01 NUV TIME-TAG 670.0 2016.141:18:22:22 PSA O NCM1 G225M 2410 -2 17295 16857 NO 1 7 S 68 | lcwj15d2 14265 HE1211-132 CWJ 15 0T 01 NUV TIME-TAG 670.0 2016.141:18:35:09 PSA O NCM1 G225M 2410 -1 18062 17624 NO 1 7 S 69 | lcwj15d4 14265 HE1211-132 CWJ 15 0V 01 NUV TIME-TAG 669.0 2016.141:18:47:56 PSA O NCM1 G225M 2410 0 18829 18391 NO 1 7 S 70 | lcwj15d6 14265 HE1211-132 CWJ 15 0X 01 NUV TIME-TAG 669.0 2016.141:19:00:42 PSA O NCM1 G225M 2410 1 19595 19157 NO 1 7 S 71 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 72 | ld0x42f6 14436 DARK D0X 42 01 01 FUV TIME-TAG 1330.0 2016.142:00:00:11 167/169 PSA B G130M ----- 1309 0 17172 17066 LOW LOW NO 73 | ld0x43fm 14436 DARK D0X 43 01 01 FUV TIME-TAG 1330.0 2016.142:00:36:12 167/169 PSA B G130M ----- 1309 0 19333 19227 LOW LOW NO 74 | ld0x44ga 14436 DARK D0X 44 01 01 FUV TIME-TAG 1330.0 2016.142:01:13:58 167/169 PSA B G130M ----- 1309 0 21599 21493 LOW LOW NO 75 | ld0x45gk 14436 DARK D0X 45 01 01 FUV TIME-TAG 1330.0 2016.142:01:53:35 167/169 PSA B G130M ----- 1309 0 23976 23870 LOW LOW NO 76 | ld0x46gq 14436 DARK D0X 46 01 01 FUV TIME-TAG 1330.0 2016.142:02:22:54 167/169 PSA B G130M ----- 1309 0 25735 25629 LOW LOW NO 77 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 78 | lcya06ip 14171 GALEX-J144 CYA 06 01 01 NUV ACQ/IMAGE 9.7 2016.142:13:40:15 PSA B NCM1 MIRRORA 0 0 187 66270 NO 79 | lcya06ir 14171 GALEX-J144 CYA 06 03 01 FUV TIME-TAG 1209.0 2016.142:13:44:30 167/169 PSA B G140L ----- 1280 -2 126 66525 LOW LOW NO 3 7 S 80 | lcya06it 14171 GALEX-J144 CYA 06 05 01 FUV TIME-TAG 1209.0 2016.142:14:06:34 167/169 PSA B G140L ----- 1280 -1 1450 67849 LOW LOW NO 2 7 S 81 | lcya06iv 14171 GALEX-J144 CYA 06 07 01 FUV TIME-TAG 1404.0 2016.142:15:13:29 167/169 PSA B G140L ----- 1280 0 5465 71864 LOW LOW NO 2 7 S 82 | lcya06ix 14171 GALEX-J144 CYA 06 09 01 FUV TIME-TAG 1404.0 2016.142:15:38:48 167/169 PSA B G140L ----- 1280 1 6984 73383 LOW LOW NO 2 7 S 83 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 84 | ld3c01k5 14479 NGC-4051 D3C 01 01 01 NUV ACQ/IMAGE 50.0 2016.143:03:46:15 PSA B NCM1 MIRRORB 0 0 303 219 NO 85 | ld3c01k7 14479 NGC-4051 D3C 01 03 01 FUV TIME-TAG 1060.0 2016.143:03:51:19 167/169 PSA B G130M ----- 1291 0 122 523 LOW LOW NO 3 12 S 86 | ld3c01l7 14479 NGC-4051 D3C 01 05 01 FUV TIME-TAG 1065.0 2016.143:05:19:05 167/169 PSA B G130M ----- 1291 1 5388 5789 LOW LOW NO 2 12 S 87 | ld3c01le 14479 NGC-4051 D3C 01 07 01 FUV TIME-TAG 1294.0 2016.143:06:54:27 167/169 PSA B G130M ----- 1327 -2 11110 11511 LOW LOW NO 2 12 S 88 | ld3c01lg 14479 NGC-4051 D3C 01 09 01 FUV TIME-TAG 1296.0 2016.143:07:17:56 167/169 PSA B G130M ----- 1327 -1 12519 12920 LOW LOW NO 2 12 S 89 | ld3c01ll 14479 NGC-4051 D3C 01 0B 01 FUV TIME-TAG 1294.0 2016.143:08:29:50 167/169 PSA B G160M ----- 1600 0 136 17234 LOW LOW NO 3 12 S 90 | ld3c01lt 14479 NGC-4051 D3C 01 0D 01 FUV TIME-TAG 1294.0 2016.143:08:53:19 167/169 PSA B G160M ----- 1600 1 1545 18643 LOW LOW NO 2 12 S 91 | ld3c01lw 14479 NGC-4051 D3C 01 0F 01 FUV TIME-TAG 1294.0 2016.143:10:05:10 167/169 PSA B G160M ----- 1623 -2 5856 22954 LOW LOW NO 2 12 S 92 | ld3c01ly 14479 NGC-4051 D3C 01 0H 01 FUV TIME-TAG 1296.0 2016.143:10:28:39 167/169 PSA B G160M ----- 1623 -1 7265 24363 LOW LOW NO 2 12 S 93 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- 94 | --------------------------------------------------------------------------------