├── jeta ├── config │ ├── __init__.py │ ├── filetypes.dat │ ├── parameters.hdf5 │ └── properties.py ├── core │ ├── __init__.py │ └── exceptions.py ├── staging │ ├── __init__.py │ ├── validation.py │ └── tests │ │ └── test_staging.py ├── archive │ ├── tests │ │ ├── __init__.py │ │ ├── test_orbit.py │ │ ├── test_archive_status_api.py │ │ └── test_utils.py │ ├── __init__.py │ ├── pagination.py │ ├── derived │ │ ├── __init__.py │ │ ├── test.py │ │ ├── acispow.py │ │ ├── eps.py │ │ └── base.py │ ├── file_defs.py │ ├── files.py │ ├── recovery.py │ ├── fastss.pyx │ └── status.py ├── __init__.py └── tests │ └── file_defs.py ├── scripts ├── filetypes.dat ├── msid_bad_times.dat ├── shell │ ├── clear_archive.bash │ └── build_env.bash ├── python │ ├── create_sample_ingest_file.py │ └── add_msid_to_refdata.py └── sql │ ├── create.archive.meta.sql │ └── update_primary_key_type.sql ├── legacy ├── doc │ ├── ska.jpg │ ├── ska.png │ ├── fetchplots │ │ ├── first.png │ │ ├── obc_rates.png │ │ ├── aorate3_hist.png │ │ ├── basic_func.png │ │ ├── plot_cxctime.png │ │ ├── tephin_daily.png │ │ ├── aca_gyro_rates.png │ │ ├── gyro_sc_rates.png │ │ ├── interpolate_input.png │ │ ├── iplot_aopcadmode.png │ │ ├── load_bus_voltage.png │ │ ├── state_bins_4ohtrz10.png │ │ ├── interpolate_False_True.png │ │ ├── interpolate_True_False.png │ │ ├── interpolate_True_True.png │ │ ├── interpolate_False_False.png │ │ └── make_interpolate.py │ ├── pyplots │ │ ├── pyplot_text.png │ │ ├── pyplot_three.png │ │ ├── pyplot_annotate.png │ │ ├── pyplot_simple.png │ │ ├── pyplot_formatstr.png │ │ └── pyplot_two_subplots.png │ ├── fetch_tutorial_standalone.rst │ ├── utils.rst │ ├── plot.rst │ ├── references.rst │ ├── fetch.rst │ ├── date_time_formats.rst │ ├── index.rst │ ├── examples │ │ └── cmp_obc_gyro.py │ ├── numpy_tutorial.rst │ └── Makefile ├── MANIFEST.in ├── check_integrity.py ├── dev_utils │ ├── remote.py │ ├── plot_safes.py │ ├── find_colname_diffs.py │ ├── print_msidfile_times.py │ ├── find_tdb_diffs.py │ ├── fix_oba_ave.py │ ├── validate_tdb_updates.py │ ├── example_interpolate.py │ ├── fix_neg_dts.py │ └── repro_interval.sh ├── transfer_stage.py ├── correlate_units.py ├── make_units_sci.py ├── NOTES.occ_install ├── validate.py ├── find_rel_temps.py ├── rebuild_stats │ ├── NOTES.rebuild_stats │ └── move_stats.py ├── NOTES.occ_tunnel ├── context_files.py ├── test │ ├── make_plots.py │ ├── compare_regr_vals.py │ └── get_regr_vals.py ├── make_test_eng_archive.sh ├── context_def.py ├── make_units_eng.py ├── Makefile ├── make_h5.py ├── make_qual.py ├── move_fits_archive.py ├── ingest_fits_archive.py ├── update_fits_archive.py ├── NOTES.resync_occ ├── update_h5_archive.py ├── make_test_eng_archive.csh ├── update_h5_archive2.py ├── update_h5_archive3.py ├── alias_rename_files.py ├── task_schedule_occ_gap.cfg ├── task_schedule_gap.cfg ├── task_schedule.cfg ├── task_schedule_occ.cfg ├── make_resync_stage.py ├── convert_headers.py ├── get_products.py ├── make_units_cxc.py ├── NOTES.OBA-wide ├── NOTES.state_bins ├── fix_ingest_h5.py ├── RELEASE.rst ├── ingest_h5_archive.py └── NOTES.add_content ├── .gitmodules ├── docs ├── source │ ├── update.md │ ├── plotting.md │ ├── operations.md │ ├── ingest.md │ ├── index.md │ └── fetch.md ├── Makefile └── make.bat ├── pyproject.toml ├── .hgignore ├── pytest.ini ├── .gitignore ├── .github └── workflows │ └── publish-to-pypi.yml ├── CHANGES.md ├── requirements ├── local.txt └── production.txt ├── README.md ├── LICENSE.rst └── setup.py /jeta/config/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jeta/core/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jeta/staging/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jeta/archive/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jeta/config/filetypes.dat: -------------------------------------------------------------------------------- 1 | instrum content 2 | TELEM TLM -------------------------------------------------------------------------------- /scripts/filetypes.dat: -------------------------------------------------------------------------------- 1 | instrum content 2 | TELEM TLM -------------------------------------------------------------------------------- /legacy/doc/ska.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/ska.jpg -------------------------------------------------------------------------------- /legacy/doc/ska.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/ska.png -------------------------------------------------------------------------------- /scripts/msid_bad_times.dat: -------------------------------------------------------------------------------- 1 | # MSID bad times 2 | # MSID start_bad_time stop_bad_time 3 | 4 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "raven"] 2 | path = raven 3 | url = https://github.com/spacetelescope/RAVEN 4 | -------------------------------------------------------------------------------- /jeta/__init__.py: -------------------------------------------------------------------------------- 1 | # from .version import package_version 2 | # __version__ = package_version.version 3 | 4 | -------------------------------------------------------------------------------- /jeta/config/parameters.hdf5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/jeta/config/parameters.hdf5 -------------------------------------------------------------------------------- /legacy/doc/fetchplots/first.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/fetchplots/first.png -------------------------------------------------------------------------------- /legacy/doc/fetchplots/obc_rates.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/fetchplots/obc_rates.png -------------------------------------------------------------------------------- /legacy/doc/pyplots/pyplot_text.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/pyplots/pyplot_text.png -------------------------------------------------------------------------------- /legacy/doc/pyplots/pyplot_three.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/pyplots/pyplot_three.png -------------------------------------------------------------------------------- /legacy/doc/fetchplots/aorate3_hist.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/fetchplots/aorate3_hist.png -------------------------------------------------------------------------------- /legacy/doc/fetchplots/basic_func.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/fetchplots/basic_func.png -------------------------------------------------------------------------------- /legacy/doc/fetchplots/plot_cxctime.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/fetchplots/plot_cxctime.png -------------------------------------------------------------------------------- /legacy/doc/fetchplots/tephin_daily.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/fetchplots/tephin_daily.png -------------------------------------------------------------------------------- /legacy/doc/pyplots/pyplot_annotate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/pyplots/pyplot_annotate.png -------------------------------------------------------------------------------- /legacy/doc/pyplots/pyplot_simple.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/pyplots/pyplot_simple.png -------------------------------------------------------------------------------- /legacy/doc/fetchplots/aca_gyro_rates.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/fetchplots/aca_gyro_rates.png -------------------------------------------------------------------------------- /legacy/doc/fetchplots/gyro_sc_rates.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/fetchplots/gyro_sc_rates.png -------------------------------------------------------------------------------- /legacy/doc/pyplots/pyplot_formatstr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/pyplots/pyplot_formatstr.png -------------------------------------------------------------------------------- /legacy/doc/fetchplots/interpolate_input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/fetchplots/interpolate_input.png -------------------------------------------------------------------------------- /legacy/doc/fetchplots/iplot_aopcadmode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/fetchplots/iplot_aopcadmode.png -------------------------------------------------------------------------------- /legacy/doc/fetchplots/load_bus_voltage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/fetchplots/load_bus_voltage.png -------------------------------------------------------------------------------- /legacy/doc/pyplots/pyplot_two_subplots.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/pyplots/pyplot_two_subplots.png -------------------------------------------------------------------------------- /legacy/doc/fetchplots/state_bins_4ohtrz10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/fetchplots/state_bins_4ohtrz10.png -------------------------------------------------------------------------------- /legacy/doc/fetchplots/interpolate_False_True.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/fetchplots/interpolate_False_True.png -------------------------------------------------------------------------------- /legacy/doc/fetchplots/interpolate_True_False.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/fetchplots/interpolate_True_False.png -------------------------------------------------------------------------------- /legacy/doc/fetchplots/interpolate_True_True.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/fetchplots/interpolate_True_True.png -------------------------------------------------------------------------------- /legacy/doc/fetchplots/interpolate_False_False.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/jeta/master/legacy/doc/fetchplots/interpolate_False_False.png -------------------------------------------------------------------------------- /docs/source/update.md: -------------------------------------------------------------------------------- 1 | # Update 2 | 3 | ```{py:currentmodule} jeta.archive.update 4 | ``` 5 | 6 | ```{eval-rst} 7 | .. autofunction:: statistics 8 | ``` 9 | 10 | -------------------------------------------------------------------------------- /legacy/doc/fetch_tutorial_standalone.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | Fetch Tutorial 3 | ================================ 4 | 5 | .. include:: fetch_tutorial.rst 6 | -------------------------------------------------------------------------------- /scripts/shell/clear_archive.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | rm ${TELEMETRY_ARCHIVE}/archive.meta.info.db3 \ 4 | && rm ${TELEMETRY_ARCHIVE}/msids.pickle \ 5 | && rm -rf ${TELEMETRY_ARCHIVE}/data; 6 | -------------------------------------------------------------------------------- /docs/source/plotting.md: -------------------------------------------------------------------------------- 1 | # Plotting 2 | 3 | ```{py:currentmodule} jeta.archive.plot 4 | ``` 5 | ## Plotting API Classes 6 | 7 | ```{eval-rst} 8 | .. autoclass:: MsidPlot 9 | :show-inheritance: 10 | :members: 11 | ``` 12 | -------------------------------------------------------------------------------- /legacy/doc/utils.rst: -------------------------------------------------------------------------------- 1 | .. include:: references.rst 2 | 3 | ================================================ 4 | Utilities API documentation 5 | ================================================ 6 | 7 | .. automodule:: Ska.engarchive.utils 8 | :members: 9 | -------------------------------------------------------------------------------- /legacy/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include Ska/engarchive/units_*.pkl 2 | include Ska/engarchive/*.dat 3 | include Ska/engarchive/tests/*.dat 4 | include Makefile 5 | include update_archive.py 6 | include transfer_stage.py 7 | include archfiles_def.sql 8 | include filetypes_all.dat 9 | include task_schedule*.cfg 10 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools>=45", 4 | "setuptools_scm>=6.2", 5 | "wheel", 6 | "Cython==0.29.24", 7 | "numpy==1.20.3" 8 | ] 9 | build-backend = "setuptools.build_meta" 10 | 11 | [tool.setuptools_scm] 12 | 13 | [tool.cibuildwheel] 14 | build = ["cp38-*", "cp39-*"] 15 | -------------------------------------------------------------------------------- /jeta/archive/__init__.py: -------------------------------------------------------------------------------- 1 | # # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | # from jeta.version import __version__, __git_version__ 3 | 4 | 5 | # def test(*args, **kwargs): 6 | # ''' 7 | # Run py.test unit tests. 8 | # ''' 9 | # import testr 10 | # return testr.test(*args, **kwargs) 11 | -------------------------------------------------------------------------------- /legacy/doc/plot.rst: -------------------------------------------------------------------------------- 1 | .. include:: references.rst 2 | 3 | ============================= 4 | Plotting API documentation 5 | ============================= 6 | 7 | .. automodule:: Ska.engarchive.plot 8 | 9 | Classes 10 | ----------- 11 | .. autoclass:: Ska.engarchive.plot.MsidPlot 12 | :show-inheritance: 13 | :members: 14 | -------------------------------------------------------------------------------- /legacy/check_integrity.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 3 | """ 4 | Check integrity of Ska archive 5 | 6 | This is normally run as a cron job. 7 | """ 8 | 9 | if __name__ == '__main__': 10 | from Ska.engarchive import check_integrity 11 | check_integrity.main() 12 | -------------------------------------------------------------------------------- /legacy/dev_utils/remote.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | from IPython.parallel import Client 3 | client = Client() 4 | dview = client[0] # direct view object 5 | 6 | 7 | @dview.remote(block=True) 8 | def remote_fetch(*args, **kwargs): 9 | import Ska.engarchive.fetch_eng as fetch 10 | return fetch.Msid(*args, **kwargs) 11 | -------------------------------------------------------------------------------- /legacy/transfer_stage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 3 | """ 4 | Perform file transfer functions to get files over to GRETA network. 5 | 6 | This is normally run as a cron job. 7 | """ 8 | 9 | if __name__ == '__main__': 10 | from Ska.engarchive import transfer_stage 11 | transfer_stage.main() 12 | -------------------------------------------------------------------------------- /.hgignore: -------------------------------------------------------------------------------- 1 | data 2 | .*\.log 3 | files_def.sql 4 | db.sql3 5 | Ska.engarchive.egg-info 6 | build 7 | dist 8 | .*\.wpr 9 | example_files 10 | test/plot_1crat.png 11 | test/plot_fptemp_11.png 12 | test/plot_orbitephem0_x.png 13 | test/plot_sim_z.png 14 | test/plot_tephin.png 15 | test/regr_vals.flight 16 | test/regr_vals.test 17 | test/eng_archive 18 | test_eng_archive.tar.gz 19 | test_eng_archive/filetypes.dat 20 | -------------------------------------------------------------------------------- /jeta/core/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module is used to define custom exceptions. The intent is to create 3 | exceptions that are more descriptive than the defaults. 4 | """ 5 | 6 | 7 | class ImproperlyConfigured(Exception): 8 | 9 | def __init__(self, message): 10 | super().__init__(message) 11 | 12 | 13 | class StrategyNotImplemented(Exception): 14 | 15 | def __init__(self, message): 16 | super().__init__(message) 17 | -------------------------------------------------------------------------------- /legacy/correlate_units.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | import pickle 3 | 4 | cxcunits = pickle.load(open('cxcunits.pkl')) 5 | occunits = pickle.load(open('occunits.pkl')) 6 | 7 | conversions = {} 8 | for cxcmsid, cxcunit in cxcunits.items(): 9 | if cxcmsid not in occunits: 10 | print cxcmsid, 'missing' 11 | else: 12 | conversions[cxcmsid] = (cxcunit, occunits[cxcmsid]) 13 | 14 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | minversion = 6.0 3 | addopts = -ra -vv --code-highlight=yes 4 | 5 | env = 6 | HOME=~/tmp 7 | TELEMETRY_ARCHIVE={HOME}/jeta-test-cache/archive/ 8 | STAGING_DIRECTORY={HOME}/jeta-test-cache/stage/ 9 | 10 | # test file matching 11 | python_files = tests.py test_*.py *_tests.py 12 | 13 | # ignore these directories when doing test discovery 14 | norecursedirs = .git anaconda* 15 | 16 | # ignore warnings about using a newer version 17 | filterwarnings = ignore:.*:DeprecationWarning -------------------------------------------------------------------------------- /legacy/make_units_sci.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | """ 3 | Make a unit_system consistent with usual CXC Science units where all temperatures 4 | are in degC instead of Kelvins. Otherwise leave the CXC units untouched. 5 | """ 6 | import cPickle as pickle 7 | 8 | units_cxc = pickle.load(open('units_cxc.pkl')) 9 | units_sci = dict((msid, 'DEGC') for msid, unit in units_cxc.items() 10 | if unit == 'K' or unit == 'deltaK') 11 | pickle.dump(units_sci, open('units_sci.pkl', 'wb')) 12 | -------------------------------------------------------------------------------- /legacy/dev_utils/plot_safes.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | def plot_pitch(event, stat=None): 3 | figure() 4 | start = DateTime(event.start) 5 | stop = DateTime(event.stop) 6 | dat = fetch.Msid('pitch', start - 0.5, stop + 1.5, stat=stat) 7 | if len(dat) > 2: 8 | dat.plot('b') 9 | dat = fetch.Msid('aosares1', start - 0.5, stop + 1.5, stat=stat) 10 | dat.plot('r') 11 | plot_cxctime([start.secs, start.secs], ylim(), '--r') 12 | plot_cxctime([stop.secs, stop.secs], ylim(), '--r') 13 | -------------------------------------------------------------------------------- /jeta/archive/pagination.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from jeta.archive import fetch 3 | 4 | def paginate(msid, draw, idx0=0, length=10): 5 | """ A simple function to return a page of the full resolution data. 6 | """ 7 | results = fetch.MSID(msid) 8 | 9 | start = idx0 10 | stop = idx0 + length 11 | 12 | times = results.times[start:stop] 13 | values = results.vals[start:stop] 14 | 15 | return { 16 | 'draw': draw, 17 | 'data': [times, values], 18 | 'recordsTotal': len(results), 19 | 'recordsFiltered': len(times) 20 | } -------------------------------------------------------------------------------- /jeta/archive/derived/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | """ 3 | Derived Parameters 4 | ------------------ 5 | 6 | The engineering archive has pseudo-MSIDs that are derived via computation from 7 | telemetry MSIDs. All derived parameter names begin with the characters "DP_" 8 | (not case sensitive as usual). Otherwise there is no difference from standard 9 | MSIDs. 10 | """ 11 | 12 | from .base import * 13 | 14 | from .thermal import * 15 | from .test import * 16 | from .acispow import * 17 | from .pcad import * 18 | from .orbit import * 19 | from .eps import * 20 | -------------------------------------------------------------------------------- /legacy/doc/references.rst: -------------------------------------------------------------------------------- 1 | .. _`Main`: index.html 2 | .. _`overview`: index.html 3 | .. _`tutorial`: tutorial.html 4 | .. _`fetch manual`: fetch.html 5 | .. _`table definitions`: table_def.html 6 | .. _`telemetry archive tables`: table_def.html 7 | .. _`observation database`: observation_def.html 8 | .. _`observation table definition`: observation_def.html 9 | .. _`matplotlib`: http://matplotlib.sourceforge.net/ 10 | .. _`numpy`: http://numpy.scipy.org/ 11 | .. _`ipython`: http://ipython.scipy.org/moin/ 12 | .. _HEAsoft: http://heasarc.gsfc.nasa.gov/lheasoft/ 13 | .. _CIAO: http://cxc.harvard.edu/ciao/ 14 | .. _SAS: http://xmm.vilspa.esa.es/sas/ 15 | 16 | -------------------------------------------------------------------------------- /jeta/config/properties.py: -------------------------------------------------------------------------------- 1 | import os 2 | from jeta.core.exceptions import ImproperlyConfigured 3 | 4 | def get_env_variable(var_name): 5 | 6 | try: 7 | return os.environ[var_name] 8 | except: 9 | message = "Set the {} environment variable".format(var_name) 10 | raise ImproperlyConfigured(message) 11 | 12 | 13 | STAGING_DIRECTORY=get_env_variable('STAGING_DIRECTORY') 14 | 15 | # FOF file columns from which to parse data. 16 | NAME_COLUMN=get_env_variable('NAME_COLUMN') 17 | TIME_COLUMN=get_env_variable('TIME_COLUMN') 18 | VALUE_COLUMN=get_env_variable('VALUE_COLUMN') 19 | 20 | lock_file_path = f'{get_env_variable("TELEMETRY_ARCHIVE")}/ingest.lock' 21 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = jSka 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /legacy/dev_utils/find_colname_diffs.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | from __future__ import print_function 3 | import os 4 | import pickle 5 | 6 | root = '/proj/sot/ska/data/eng_archive/data' 7 | content_dirs = os.listdir(root) 8 | for content_dir in content_dirs: 9 | f1 = os.path.join(root, content_dir, 'colnames.pickle') 10 | f2 = os.path.join(root, content_dir, 'colnames_all.pickle') 11 | if os.path.exists(f1) and os.path.exists(f2): 12 | colnames = pickle.load(open(f1)) 13 | colnames_all = pickle.load(open(f2)) 14 | diff = colnames_all - colnames - set(['QUALITY']) 15 | if diff: 16 | print(content_dir) 17 | print(diff) 18 | print() 19 | -------------------------------------------------------------------------------- /legacy/NOTES.occ_install: -------------------------------------------------------------------------------- 1 | - Finish rsync-ing content over to OCC 2 | - rsync stage to OCC 3 | - Update HEAD skare and skare-occ to Ska.engarchive 0.12 (but don't push to $hg yet) 4 | - Pull changes to OCC skare-occ 5 | - Install skare-occ and engarchive 0.12 6 | X Test fetch 7 | - Try transfer_stage on HEAD LAN. If ftp is OK, then run transfer stage on OCC. 8 | - Test update_archive on OCC: OR better, use test eng_archive data repository 9 | - Make backup copy of simcoor stage files and simcoor msid files 10 | - update_archive.py --occ --content simcoor 11 | - Test results 12 | - Install engarchive 0.12 on HEAD 13 | - Set up OCC cron job with task_schedule_occ.cfg 14 | - Other testing then, maybe with a manual run of task_schedule on either side? 15 | 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # MacOSX 2 | .DS_Store 3 | 4 | # Testing 5 | test/plot*.png 6 | test/eng_archive 7 | test/*.tar.gz 8 | test/regr_vals.* 9 | .pytest_cache/* 10 | htmlcov 11 | .coverage 12 | 13 | # Project 14 | build 15 | data 16 | dist 17 | doc/_build 18 | www 19 | MANIFEST 20 | *~ 21 | *.log 22 | *.fits.gz 23 | stage 24 | Ska.engarchive.egg-info 25 | sken.egg-info 26 | Ska/engarchive/GIT_VERSION 27 | test/eng_archive* 28 | dev_utils/event_filter_1.png 29 | dev_utils/event_filter_2.png 30 | dev_utils/l0_5icd.pdf 31 | dev_utils/memleak.py 32 | recovery 33 | .env 34 | jeta.egg-info 35 | linux_deploy.sh 36 | VDS.hdf5 37 | fastss.c 38 | jeta/version.py 39 | 40 | # Python 41 | *.pyc 42 | 43 | # vscode 44 | .vscode/ 45 | jeta.code-workspace 46 | jeta/staging/.coverage 47 | -------------------------------------------------------------------------------- /.github/workflows/publish-to-pypi.yml: -------------------------------------------------------------------------------- 1 | name: Publish to PyPI 2 | 3 | on: 4 | release: 5 | types: [released] 6 | 7 | jobs: 8 | publish: 9 | uses: spacetelescope/action-publish_to_pypi/.github/workflows/workflow.yml@master 10 | with: 11 | test: false 12 | build_platform_wheels: true # Set to true if your package contains a C extension 13 | secrets: 14 | user: ${{ secrets.PYPI_USERNAME_STSCI_MAINTAINER }} 15 | password: ${{ secrets.PYPI_PASSWORD_STSCI_MAINTAINER }} # WARNING: Do not hardcode secret values here! If you want to use a different user or password, you can override this secret by creating one with the same name in your Github repository settings. 16 | test_password: ${{ secrets.PYPI_PASSWORD_STSCI_MAINTAINER_TEST }} 17 | -------------------------------------------------------------------------------- /scripts/python/create_sample_ingest_file.py: -------------------------------------------------------------------------------- 1 | """ A tool to generate sample files for ingest. 2 | 3 | These files can be used to test algorithms with more 4 | realistic data. 5 | 6 | """ 7 | import argparse 8 | from astropy.time import Time 9 | 10 | 11 | def get_options(args=None): 12 | parser = argparse.ArgumentParser() 13 | 14 | parser.add_argument( 15 | "--num-of-files", 16 | type=int, 17 | default=1, 18 | help="The number of ingest files to generate for testing." 19 | ) 20 | 21 | parser.add_argument( 22 | "--date-now", 23 | default=Time.now(), 24 | help="Set start time for the first file for testing (default=NOW)" 25 | ) 26 | 27 | return parser.parse_args(args) 28 | 29 | 30 | def main(): 31 | pass 32 | -------------------------------------------------------------------------------- /legacy/validate.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | """ 3 | Find gaps (positive and negative) in telemetry times. Positive gaps are missing data 4 | while negative gaps are duplicate data. The latter need to be excised (via bad quality). 5 | """ 6 | 7 | import tables 8 | from context_def import datadir, ft, files 9 | from Chandra.Time import DateTime 10 | 11 | ft['content'] = 'acis2eng' 12 | ft['msid'] = 'TIME' 13 | 14 | h5 = tables.openFile(files['oldmsid'].abs) 15 | qual = h5.root.quality[:] 16 | times = h5.root.data[:][~ qual] 17 | 18 | dts = times[1:] - times[:-1] 19 | bad = (dts > 66) | (dts < -1) 20 | 21 | for gap_time, dt in zip(times[bad], dts[bad]): 22 | print '%s %12d %7d' % (DateTime(gap_time).date, gap_time, dt) 23 | 24 | h5.close() 25 | -------------------------------------------------------------------------------- /jeta/archive/derived/test.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | from . import base 3 | 4 | class DerivedParameterTest(base.DerivedParameter): 5 | content_root = 'test' 6 | 7 | #-------------------------------------------- 8 | class DP_TEST1(DerivedParameterTest): 9 | rootparams = ['TEPHIN'] 10 | time_step = 32.8 11 | 12 | def calc(self, dataset): 13 | return dataset['TEPHIN'].vals 14 | 15 | class DP_TEST2(DerivedParameterTest): 16 | rootparams = ['TEPHIN'] 17 | time_step = 65.6 18 | 19 | def calc(self, dataset): 20 | return dataset['TEPHIN'].vals * 1.5 21 | 22 | class DP_TEST3(DerivedParameterTest): 23 | rootparams = ['TEPHIN'] 24 | time_step = 65.6 25 | 26 | def calc(self, dataset): 27 | return dataset['TEPHIN'].vals * 2.0 28 | 29 | -------------------------------------------------------------------------------- /legacy/doc/fetch.rst: -------------------------------------------------------------------------------- 1 | .. include:: references.rst 2 | 3 | ==================================== 4 | Fetch API documentation 5 | ==================================== 6 | 7 | .. py:module:: Ska.engarchive.fetch 8 | 9 | Classes 10 | -------- 11 | .. autoclass:: MSID 12 | :show-inheritance: 13 | :members: 14 | 15 | .. autoclass:: MSIDset 16 | :show-inheritance: 17 | :members: 18 | 19 | .. autoclass:: Msid 20 | :show-inheritance: 21 | :members: 22 | 23 | .. autoclass:: Msidset 24 | :show-inheritance: 25 | :members: 26 | 27 | Functions 28 | ----------- 29 | .. autofunction:: get_telem 30 | 31 | .. autofunction:: get_time_range 32 | 33 | .. autofunction:: get_units 34 | 35 | .. autofunction:: msid_glob 36 | 37 | .. autofunction:: read_bad_times 38 | 39 | .. autofunction:: set_units 40 | 41 | Time and Date formats 42 | ---------------------- 43 | .. include:: date_time_formats.rst 44 | 45 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 5 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 6 | 7 | ## [Unreleased] 8 | ### [2.0.0] 9 | ### Added 10 | - Added vim text-editor to support post build modification or testing. 11 | - Added script to create an ingest file with random names and data for 12 | testing large datasets. 13 | 14 | ### Changed 15 | - Update update.py ingest algorithms 16 | 17 | ### Fixed 18 | - TBA 19 | 20 | ### Security 21 | - TBA 22 | ## [1.0.0] - 2020-04-22 23 | 24 | Initial release 25 | ## [1.0.1] - 2020-07-27 26 | ### Fixed 27 | - MSISet interpolate method returning empty list. 28 | 29 | ## [1.0.2] - 2020-07-28 30 | ### Fixed 31 | - Fixed: Issue with syncing home dirs during re-deployment. 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | set SPHINXPROJ=jSka 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 20 | echo.installed, then set the SPHINXBUILD environment variable to point 21 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 22 | echo.may add the Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /legacy/find_rel_temps.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | import pickle 3 | from Ska.engarchive import fetch 4 | 5 | dat = asciitable.read('/proj/sot/ska/ops/TDB/tmsrment.txt', 6 | Reader=asciitable.NoHeader, delimiter=",", 7 | quotechar='"') 8 | 9 | descrs = dict((msid.upper(), descr) for msid, descr, unit in zip(dat['col1'], dat['col2'], dat['col5']) 10 | if unit) 11 | 12 | cxcunits = pickle.load(open('cxcunits.pkl')) 13 | 14 | temp_msids = [msid for msid, unit in cxcunits.items() if unit == 'K'] 15 | 16 | reltemps = [] 17 | for msid in sorted(temp_msids): 18 | try: 19 | dat = fetch.MSID(msid, '2010:001', '2010:002', stat='daily') 20 | except ValueError: 21 | print msid, 'not in archive' 22 | continue 23 | 24 | if dat.means[0] < 100: 25 | print msid, dat.means[0], descrs[msid] 26 | reltemps.append(msid) 27 | 28 | -------------------------------------------------------------------------------- /legacy/dev_utils/print_msidfile_times.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | """ 3 | Print start or start/stop times for MSID content types. 4 | """ 5 | import glob 6 | import tables 7 | 8 | from Chandra.Time import DateTime 9 | 10 | if False: 11 | time_files = glob.glob('/proj/sot/ska/data/eng_archive/data/*/TIME.h5') 12 | for time_file in time_files: 13 | h5 = tables.openFile(time_file) 14 | time0 = h5.root.data[0] 15 | h5.close() 16 | print(DateTime(time0).date, time_file) 17 | 18 | if True: 19 | time_files = glob.glob('/proj/sot/ska/data/eng_archive/1999/data/*/TIME.h5') 20 | for time_file in time_files: 21 | h5 = tables.openFile(time_file) 22 | dat = h5.root.data 23 | if len(dat) > 0: 24 | time0, time1 = dat[0], dat[-1] 25 | print(DateTime([time0, time1]).date, time_file) 26 | else: 27 | print('EMPTY', time_file) 28 | h5.close() 29 | -------------------------------------------------------------------------------- /legacy/dev_utils/find_tdb_diffs.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | from __future__ import print_function 3 | import Ska.tdb 4 | 5 | msid_xref = {} 6 | for line in open('/proj/sot/ska/ops/TDB/p014/msids.xref'): 7 | vals = line.split() 8 | msid_xref[vals[0]] = vals[1] 9 | 10 | versions = (4, 6, 7, 8, 9, 10, 11, 12, 13, 14) 11 | for v1, v2 in zip(versions[:-1], versions[1:]): 12 | Ska.tdb.set_tdb_version(v1) 13 | m1 = set(Ska.tdb.msids.Tmsrment['MSID']) 14 | 15 | Ska.tdb.set_tdb_version(v2) 16 | m2 = set(Ska.tdb.msids.Tmsrment['MSID']) 17 | print('****** {} vs {} *******'.format(v1, v2)) 18 | if m1 - m2: 19 | print('** REMOVED **') 20 | for msid in sorted(m1 - m2): 21 | print('{:15s}'.format(msid)) 22 | if m2 - m1: 23 | print('** ADDED **') 24 | for msid in sorted(m2 - m1): 25 | print('{:15s} {:15s} {:s}'.format(msid, msid_xref[msid], Ska.tdb.msids[msid].technical_name)) 26 | print() 27 | -------------------------------------------------------------------------------- /legacy/rebuild_stats/NOTES.rebuild_stats: -------------------------------------------------------------------------------- 1 | # Make initial dir 2 | cd ~/git/eng_archive 3 | mkdir -p /proj/sot/ska/tmp/eng_archive/data 4 | ln -s /proj/sot/ska/tmp/eng_archive/data ./ 5 | 6 | # Copy column headers and basic directory structure 7 | rsync -av --exclude='*.h5*' --exclude='headers.pickle' --exclude='arch' \ 8 | /proj/sot/ska/data/eng_archive/data/ /proj/sot/ska/tmp/eng_archive/data/ 9 | 10 | # Rebuild stats 11 | source run_rebuild_stats.csh 12 | 13 | # Check AWDxTQI where x=1..6 14 | setenv ENG_ARCHIVE $PWD 15 | ipython --pylab 16 | impska 17 | dat = fetch.Msid('awd1tqi', '2000:001', stat='daily') 18 | dat.plot() 19 | dat = fetch.Msid('awd1tqi', '2010:001', '2012:001', stat='5min') 20 | clf() 21 | dat.plot() 22 | 23 | # Validate (do this for stat='5min' and stat='daily' in following code) 24 | python rebuild_stats/validate_stats.py '5min' 25 | python rebuild_stats/validate_stats.py 'daily' 26 | 27 | # Swap in the new values, putting the old values in a backup location 28 | python move_stats.py 29 | -------------------------------------------------------------------------------- /docs/source/operations.md: -------------------------------------------------------------------------------- 1 | # Operations 2 | 3 | ```{py:currentmodule} jeta.archive.operations 4 | ``` 5 | ## Operations API Classes 6 | ### Public Operation API Functions 7 | ```{eval-rst} 8 | .. autofunction:: backup 9 | ``` 10 | ```{eval-rst} 11 | .. autofunction:: restore 12 | ``` 13 | ```{eval-rst} 14 | .. autofunction:: truncate 15 | ``` 16 | ```{eval-rst} 17 | .. autofunction:: destory 18 | ``` 19 | ```{eval-rst} 20 | .. autofunction:: add_msid_to_archive 21 | ``` 22 | ```{eval-rst} 23 | .. autofunction:: initialize 24 | ``` 25 | ### Private Operation API Functions (Not To Be Called Directly) 26 | ```{eval-rst} 27 | .. autofunction:: _create_archive_database 28 | ``` 29 | ```{eval-rst} 30 | .. autofunction:: _create_root_content 31 | ``` 32 | ```{eval-rst} 33 | .. autofunction:: _create_msid_index 34 | ``` 35 | ```{eval-rst} 36 | .. autofunction:: _create_msid_dataset 37 | ``` 38 | ```{eval-rst} 39 | .. autofunction:: _create_archive_files 40 | ``` 41 | ```{eval-rst} 42 | .. autofunction:: _create_msid_directory 43 | ``` 44 | -------------------------------------------------------------------------------- /legacy/NOTES.occ_tunnel: -------------------------------------------------------------------------------- 1 | - See mail from Paul Grant (search "ssh tunnel") 2 | 3 | gretasot IP: 131.142.113.82 4 | snowman IP: 131.142.113.102 5 | 6 | imac% history 7 | 1 11:09 ssh 131.142.113.82 8 | 2 11:19 ssh -X -N -f -L 1234:131.142.113.82:22 SOT@131.142.113.102 9 | 7 11:21 ssh -Y -p 1234 SOT@127.0.0.1 10 | 11 | None of these work. Not sure why 12 | 14 11:29 rsync --dry-run -av -e ssh eng_archive/ rsync://SOT@127.0.0.1:1234/proj/sot/ska/data/eng_archive/data/ 13 | 19 11:35 rsync --dry-run -av eng_archive/ rsync://SOT@127.0.0.1:1234/proj/sot/ska/data/eng_archive/data/ 14 | 25 11:43 rsync --dry-run -av --port=1234 eng_archive/thm2eng SOT@127.0.0.1:/proj/sot/ska/data/eng_archive/data/ 15 | 16 | These worked (but scp died after 10 minutes) 17 | 18 | scp -rp -P 1234 thm1eng SOT@127.0.0.1:/proj/sot/ska/data/eng_archive/data/ 19 | rsync -av -e 'ssh -p 1234' eng_archive/ SOT@127.0.0.1:/proj/sot/ska/data/eng_archive/ 20 | 21 | Remove the tunnel 22 | ----------------- 23 | ps auxwww| grep ssh 24 | kill 25 | -------------------------------------------------------------------------------- /legacy/context_files.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | from pyyaks.context import ContextDict 3 | 4 | basedir = 'test' 5 | 6 | FT = ContextDict('ft') 7 | ft = FT.accessor() 8 | 9 | FILES = ContextDict('files', basedir=basedir) 10 | FILES.update({'contentdir': '{{ft.content}}/', 11 | 'headers': '{{ft.content}}/headers.pickle', 12 | 'columns': '{{ft.content}}/columns.pickle', 13 | 'columns_all': '{{ft.content}}/columns_all.pickle', 14 | 'msiddir': '{{ft.content}}/msid/', 15 | 'msid': '{{ft.content}}/msid/{{ft.msid}}', 16 | 'archdir': '{{ft.content}}/arch/{{ft.year}}/{{ft.doy}}/', 17 | 'archfile': '{{ft.content}}/arch/{{ft.year}}/{{ft.doy}}/{{ft.basename}}', 18 | }) 19 | files = FILES.accessor() 20 | 21 | # Original archive files 22 | OFILES = ContextDict('of', basedir='/data/cosmos2/tlm') 23 | OFILES.update({'contentdir': '{{ft.content}}/', 24 | }) 25 | ofiles = OFILES.accessor() 26 | -------------------------------------------------------------------------------- /docs/source/ingest.md: -------------------------------------------------------------------------------- 1 | # Ingest Process 2 | ```{eval-rst} 3 | .. py:currentmodule:: jeta.archive.ingest 4 | ``` 5 | ## Public Ingest API 6 | ```{eval-rst} 7 | .. autofunction:: calculate_delta_times 8 | ``` 9 | ```{eval-rst} 10 | .. autofunction:: sort_msid_data_by_time 11 | ``` 12 | ```{eval-rst} 13 | .. autofunction:: move_archive_files 14 | ``` 15 | ```{eval-rst} 16 | .. autofunction:: execute 17 | ``` 18 | 19 | ## Private Ingest Methods 20 | ```{eval-rst} 21 | .. autofunction:: _process_csv 22 | ``` 23 | 24 | ```{eval-rst} 25 | .. autofunction:: _process_hdf 26 | ``` 27 | ## Supported Ingest Modes 28 | 29 | * Single MSID FOF (CSV) - Comma-delimited tabular data for a single msid in FOF format. 30 | * Multiple MSIDs FOF (CSV) - Comma-delimited tabular data for a multiple msids in FOF format. 31 | * HDF5 Multiple MSIDs - HDF5 files with sample data partitioned into datasets. 32 | 33 | ## Manually Starting an Ingest from the CLI 34 | - TBD 35 | ## Ingest Scheduling 36 | 37 | - TBD 38 | 39 | ## Telemetry Archive Structure 40 | 41 | - See the LITA Systems Technical Documentation for acrhive implementation details. -------------------------------------------------------------------------------- /requirements/local.txt: -------------------------------------------------------------------------------- 1 | six==1.16.0 2 | notebook==6.4.5 3 | nbconvert==6.2.0 4 | nbformat==5.1.3 5 | MarkupSafe==2.0.1 6 | 7 | ipympl==0.8.2 8 | ipywidgets==7.6.5 9 | ipykernel==6.5.0 10 | ipython==7.29.0 11 | ipython-genutils==0.2.0 12 | 13 | jupyter-client==7.0.6 14 | jupyter-core==4.9.1 15 | jupyter-telemetry==0.1.0 16 | 17 | Jinja2==3.0.2 18 | jsonschema==4.2.1 19 | ruamel.yaml==0.17.17 20 | 21 | Django==2.2.24 22 | djangorestframework==3.12.4 23 | djangorestframework-simplejwt==4.7.2 24 | 25 | Unipath==1.1 26 | gunicorn==20.1.0 27 | 28 | jhub-remote-user-authenticator==0.1.0 29 | jupyterhub-dummyauthenticator==0.3.1 30 | 31 | Cython==0.29.24 32 | 33 | numpy==1.20.3 34 | pandas==1.3.4 35 | astropy==4.3.1 36 | torch==1.10.0 37 | matplotlib==3.4.3 38 | seaborn==0.11.2 39 | plotly==5.3.1 40 | bokeh==2.3.3 41 | numba==0.54.1 42 | 43 | h5py==3.5.0 44 | tables==3.6.1 45 | pyyaks==3.3.3 46 | celery==5.2.2 47 | 48 | redis==4.1.0 49 | 50 | 51 | jupyterlab==3.2.2 52 | configurable-http-proxy==0.2.3 53 | jupyterhub-idle-culler 54 | 55 | # Documentation 56 | sphinx_rtd_theme 57 | sphinxcontrib-versioning==2.2.1 58 | -------------------------------------------------------------------------------- /requirements/production.txt: -------------------------------------------------------------------------------- 1 | six==1.16.0 2 | notebook==6.4.5 3 | nbconvert==6.2.0 4 | nbformat==5.1.3 5 | MarkupSafe==2.0.1 6 | 7 | ipympl==0.8.2 8 | ipywidgets==7.6.5 9 | ipykernel==6.5.0 10 | ipython==7.29.0 11 | ipython-genutils==0.2.0 12 | 13 | jupyter-client==7.0.6 14 | jupyter-core==4.9.1 15 | jupyter-telemetry==0.1.0 16 | 17 | Jinja2==3.0.2 18 | jsonschema==4.2.1 19 | ruamel.yaml==0.17.17 20 | 21 | Django==2.2.24 22 | djangorestframework==3.12.4 23 | djangorestframework-simplejwt==4.7.2 24 | 25 | Unipath==1.1 26 | gunicorn==20.1.0 27 | 28 | jhub-remote-user-authenticator==0.1.0 29 | jupyterhub-dummyauthenticator==0.3.1 30 | 31 | Cython==0.29.24 32 | 33 | numpy==1.20.3 34 | pandas==1.3.4 35 | astropy==4.3.1 36 | torch==1.10.0 37 | matplotlib==3.4.3 38 | seaborn==0.11.2 39 | plotly==5.3.1 40 | bokeh==2.3.3 41 | numba==0.54.1 42 | 43 | h5py==3.5.0 44 | tables==3.6.1 45 | pyyaks==3.3.3 46 | celery==5.2.2 47 | 48 | redis==4.1.0 49 | 50 | 51 | jupyterlab==3.2.2 52 | configurable-http-proxy==0.2.3 53 | jupyterhub-idle-culler 54 | 55 | # Documentation 56 | sphinx_rtd_theme 57 | sphinxcontrib-versioning==2.2.1 58 | -------------------------------------------------------------------------------- /legacy/test/make_plots.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 3 | 4 | # Make a few plots for a sanity check of test archive correctness 5 | # run in pylab 6 | 7 | import os 8 | import sys 9 | from matplotlib.pyplot import * 10 | 11 | import Ska.engarchive.fetch_sci as fetch 12 | from Ska.Matplotlib import plot_cxctime 13 | 14 | print 'Fetch file is', fetch.__file__ 15 | print 'ENG_ARCHIVE is', os.environ['ENG_ARCHIVE'] 16 | 17 | msids = ('1crat', 'fptemp_11', 'orbitephem0_x', 'sim_z', 'tephin') 18 | rootdir = os.path.dirname(__file__) 19 | 20 | for ifig, msid in enumerate(msids): 21 | figure(ifig+1) 22 | clf() 23 | dat = fetch.MSID(msid, '2010:250', '2011:100', filter_bad=True) 24 | dat5 = fetch.MSID(msid, '2010:250', '2011:100',stat='5min') 25 | datday = fetch.MSID(msid, '2010:250', '2011:100', stat='daily') 26 | subplot(3, 1, 1) 27 | plot_cxctime(dat.times, dat.vals, '-b') 28 | grid() 29 | subplot(3, 1, 2) 30 | plot_cxctime(dat5.times, dat5.means, '-r') 31 | grid() 32 | subplot(3, 1, 3) 33 | plot_cxctime(datday.times, datday.means, '-c') 34 | grid() 35 | savefig(os.path.join(rootdir, 'plot_{0}.png'.format(msid))) 36 | -------------------------------------------------------------------------------- /legacy/dev_utils/fix_oba_ave.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | """ 3 | Apply a scale factor of 36.0/35.0 to account for a mistake in the original specification 4 | of the OBC_AVE thermal derived parameter. 5 | 6 | This is done as part of https://github.com/sot/eng_archive/pull/127, but original discussion 7 | and code is in https://github.com/sot/eng_archive/pull/115 8 | 9 | To start:: 10 | 11 | rsync -av --exclude=arch /proj/sot/ska/data/eng_archive/data/dp_thermal128/ data/dp_thermal128/ 12 | 13 | Then:: 14 | 15 | python fix_oba_ave.py 16 | """ 17 | 18 | import tables 19 | 20 | scale = 36. / 35. 21 | 22 | h5 = tables.openFile('data/dp_thermal128/DP_OBA_AVE.h5', 'a') 23 | val = h5.root.data 24 | val[:] = val[:] * scale 25 | h5.close() 26 | 27 | h5 = tables.openFile('data/dp_thermal128/5min/DP_OBA_AVE.h5', 'a') 28 | for attr in 'val min max mean'.split(): 29 | val = getattr(h5.root.data.cols, attr) 30 | val[:] = val[:] * scale 31 | h5.close() 32 | 33 | h5 = tables.openFile('data/dp_thermal128/daily/DP_OBA_AVE.h5', 'a') 34 | for attr in 'val min max mean std p01 p05 p16 p50 p84 p95 p99'.split(): 35 | val = getattr(h5.root.data.cols, attr) 36 | val[:] = val[:] * scale 37 | h5.close() 38 | -------------------------------------------------------------------------------- /legacy/make_test_eng_archive.sh: -------------------------------------------------------------------------------- 1 | # From within the hg eng_archive repo 2 | # source make_test_eng_archive.csh 3 | 4 | export ENG_ARCHIVE=$PWD/test/eng_archive 5 | mkdir -p test/eng_archive 6 | rm -rf test/eng_archive/* 7 | 8 | rm -f test/make_eng_archive.log 9 | touch test/make_eng_archive.log 10 | 11 | # To make the data from scratch use the following. BUT NORMALLY just use the 12 | # existing copy of the flight eng archive as the baseline start point. 13 | 14 | # echo "Making regr data..." 15 | # ./make_regr_data.py --start 2012:290 --stop 2012:300 --data-root test/eng_archive >>& test/make_eng_archive.log 16 | # 17 | # echo "Tarring..." 18 | # cd test 19 | # tar zcf eng_archive.tar.gz eng_archive 20 | # cd .. 21 | 22 | cd test 23 | tar xvf /proj/sot/ska/data/eng_archive/regr/flight_eng_archive.tar.gz 24 | cd .. 25 | 26 | CONTENTS="--content=acis2eng --content=acis3eng --content=acisdeahk --content=ccdm4eng --content=dp_acispow128 --content=orbitephem0 --content=simcoor --content=thm1eng" 27 | DATAROOT="--data-root=test/eng_archive" 28 | UPDATE_OPTS="$DATAROOT $CONTENTS" 29 | 30 | echo "Updating archive..." 31 | ./update_archive.py --date-now 2012:315 --date-start 2012:300 --max-lookback-time=2 $UPDATE_OPTS >> test/make_eng_archive.log 2>&1 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /jeta/archive/tests/test_orbit.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | import numpy as np 3 | 4 | from ..derived import orbit 5 | 6 | 7 | def test_orbital_elements(): 8 | """ 9 | Test orbital elements calculation vs. reference 10 | http://www.castor2.ca/05_OD/01_Gauss/14_Kepler/index.html 11 | 12 | The example there suffers from some roundoff error by using the printed values instead 13 | of full precision. 14 | """ 15 | # State vector in m, m/s 16 | x = 5052.4587e3 17 | y = 1056.2713e3 18 | z = 5011.6366e3 19 | vx = 3.8589872e3 20 | vy = 4.2763114e3 21 | vz = -4.8070493e3 22 | 23 | out = orbit.calc_orbital_elements(x, y, z, vx, vy, vz) 24 | 25 | expected = {'semi_major_axis': 7310.8163e3, 26 | 'orbit_period': 103.68323 * 60, 27 | 'eccentricity': 0.015985887, 28 | 'inclination': 71.048202, 29 | 'ascending_node': 211.28377, 30 | 'argument_perigee': 137.7561049, # 137.75619 in reference (roundoff in example) 31 | 'mean_anomaly': 354.971325} # 354.9724 in reference (roundoff in example) 32 | 33 | for key in expected: 34 | assert np.allclose(out[key], expected[key], atol=0.0, rtol=1e-6) 35 | -------------------------------------------------------------------------------- /legacy/context_def.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | from pyyaks.context import ContextDict 3 | 4 | # datadir = '/data/baffin3/telem_archive/data' 5 | datadir = 'data' 6 | 7 | ft = ContextDict('ft') 8 | 9 | files = ContextDict('files', basedir=datadir) 10 | files.update({'contentdir': '{{ft.content}}/', 11 | 'headers': '{{ft.content}}/headers.pickle', 12 | 'colnames': '{{ft.content}}/colnames.pickle', 13 | 'colnames_all': '{{ft.content}}/colnames_all.pickle', 14 | 'archfiles': '{{ft.content}}/archfiles.db3', 15 | 'msiddir': '{{ft.content}}/msid/', 16 | 'msid': '{{ft.content}}/msid/{{ft.msid}}.h5', 17 | 'qual': '{{ft.content}}/{{ft.msid}}_qual.h5', 18 | 'oldmsid': '{{ft.content}}/{{ft.msid}}.h5', 19 | # 'oldmsid': 'test.h5', 20 | 'archdir': '{{ft.content}}/arch/{{ft.year}}/{{ft.doy}}/', 21 | 'archfile': '{{ft.content}}/arch/{{ft.year}}/{{ft.doy}}/{{ft.basename}}', 22 | }) 23 | 24 | # Original archive files 25 | ofiles = ContextDict('of', basedir='/data/cosmos2/tlm') 26 | ofiles.update({'contentdir': '{{ft.content}}/', 27 | }) 28 | 29 | -------------------------------------------------------------------------------- /legacy/dev_utils/validate_tdb_updates.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | from __future__ import print_function 3 | 4 | import pickle 5 | import os 6 | import Ska.tdb 7 | from Ska.engarchive import fetch_eng as fetch 8 | from matplotlib import pyplot as plt 9 | 10 | for content in os.listdir('data'): 11 | new_colnames = pickle.load(open(os.path.join('data', content, 'colnames.pickle'))) 12 | cur_colnames = pickle.load(open(os.path.join( 13 | '/proj/sot/ska/data/eng_archive', 'data', content, 'colnames.pickle'))) 14 | print('New {}'.format(content)) 15 | new = new_colnames - cur_colnames 16 | print(', '.join(sorted(new))) 17 | lost = cur_colnames - new_colnames 18 | if lost: 19 | print('LOST: ', lost) 20 | 21 | # Plot representative new vals 22 | d1 = '2016:001' 23 | d2 = '2016:002' 24 | 25 | msids = set(['1AHIRADF']) 26 | msids.update(['POLAEV2BT', 'POLINE07T', 'POM2THV1T']) 27 | msids.update(['OHRTHR35_WIDE', '1OLORADF', '1OHIRADF', '2OLORADF', '2OHIRADF', 'OOBTHR30_WIDE']) 28 | msids.update(['AOACIIRS', 'AOACISPX', 'AOACIDPX', 'AOACIMSS']) 29 | msids.update(['4OAVOBAT_WIDE', '4OAVHRMT_WIDE']) 30 | msids.update(['TFSSHDT1', 'TFSSHDT2']) 31 | 32 | for msid in msids: 33 | m = Ska.tdb.msids[msid] 34 | print(m) 35 | dat = fetch.Msid(msid, d1, d2) 36 | plt.figure() 37 | dat.plot() 38 | plt.title(msid) 39 | 40 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # JWST Engineering Telemetry Archive (JETA) 2 | 3 | A python package for getting data in to and out of the JWST Engineering Telemetery Archive. 4 | 5 | The package, `jeta` is the JWST functional-equivalent of [`Ska.eng_archive`](https://github.com/sot/eng_archive). 6 | [Ska](https://cxc.cfa.harvard.edu/mta/ASPECT/tool_doc/pydocs/) is the "... engineering telemetry archive is a suite of tools and data products" for 7 | that supports the [Chandra](https://chandra.harvard.edu/about/spacecraft.html) X-Ray Observatiory. Where as `jeta` fulfills a similar role for JWST. 8 | 9 | > NOTE: The project does not contain any telemetry data, just the tools. 10 | 11 | ## Getting Started 12 | 13 | TBD 14 | 15 | ### Prerequisites 16 | 17 | TBD 18 | 19 | ### Environment Variables 20 | 21 | TBD 22 | 23 | ### Installing 24 | 25 | TBD 26 | 27 | ## Testing 28 | 29 | TBD 30 | 31 | ## Coding Style 32 | 33 | https://github.com/spacetelescope/style-guides 34 | 35 | ## Built With 36 | 37 | * [Chandra Tools](https://cxc.harvard.edu/mta/ASPECT/tool_doc/pydocs/index.html) - Inspired by Chandra Tools 38 | 39 | ## Versioning 40 | 41 | Project adhears to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). for versioning. 42 | 43 | ## Authors 44 | 45 | * **David Kauffman** - *Initial work* - [David Kauffman](https://github.com/ddkauffman) 46 | 47 | ## Acknowledgments 48 | 49 | * Amanda Arvai 50 | * Tom Aldcroft 51 | * Jean Connelly 52 | * Alex Hunter 53 | -------------------------------------------------------------------------------- /jeta/archive/derived/acispow.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | """ 3 | Derived parameter MSIDs related to ACIS power. 4 | """ 5 | 6 | import numpy as np 7 | from . import base 8 | 9 | class DP_DPA_POWER(base.DerivedParameter): 10 | """ACIS total DPA-A and DPA-B power""" 11 | rootparams = ['1dp28avo', '1dpicacu', '1dp28bvo', '1dpicbcu'] 12 | time_step = 32.8 13 | content_root = 'acispow' 14 | 15 | def calc(self, data): 16 | power = (data['1dp28avo'].vals * data['1dpicacu'].vals + 17 | data['1dp28bvo'].vals * data['1dpicbcu'].vals) 18 | return power 19 | 20 | 21 | class DP_DEA_POWER(base.DerivedParameter): 22 | """ACIS DEA power""" 23 | rootparams = ['1de28avo', '1deicacu'] 24 | time_step = 32.8 25 | content_root = 'acispow' 26 | 27 | def calc(self, data): 28 | power = data['1de28avo'].vals * data['1deicacu'].vals 29 | return power 30 | 31 | 32 | class DP_PSMC_POWER(base.DerivedParameter): 33 | """ACIS PSMC power""" 34 | rootparams = ['1dp28avo', '1dpicacu', '1dp28bvo', '1dpicbcu', '1de28avo', '1deicacu'] 35 | time_step = 32.8 36 | content_root = 'acispow' 37 | 38 | def calc(self, data): 39 | power = (data['1dp28avo'].vals * data['1dpicacu'].vals + 40 | data['1dp28bvo'].vals * data['1dpicbcu'].vals + 41 | data['1de28avo'].vals * data['1deicacu'].vals) 42 | return power 43 | 44 | -------------------------------------------------------------------------------- /legacy/make_units_eng.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | """ 3 | Make a unit_system consistent with usual OCC/FOT engineering units via P009. 4 | """ 5 | 6 | import pickle 7 | from copy import copy 8 | 9 | import Ska.engarchive.converters 10 | import Ska.tdb 11 | 12 | units_cxc = pickle.load(open('units_cxc.pkl', 'rb')) 13 | units_eng = copy(units_cxc) 14 | 15 | # Update with units from TMSRMENT table of the TDB 16 | dat = Ska.tdb.tables['tmsrment'] 17 | tdb_units = {str(msid.upper()): str(unit) # convert from numpy type to pure Python 18 | for msid, unit in zip(dat['MSID'], dat['ENG_UNIT'])} 19 | units_eng.update({msid: tdb_units[msid] for msid, unit in units_eng.items() 20 | if msid in tdb_units}) 21 | 22 | # Any other MSIDs that still have units of 'K' are converted to DEGC 23 | units_eng.update({msid: 'DEGC' for msid, unit in units_eng.items() 24 | if unit in ['K', 'deltaK']}) 25 | 26 | # Use info about DEA HK telemetry from converters to add units 27 | for col in Ska.engarchive.converters._get_deahk_cols(): 28 | if 'unit' in col and col['unit'] == 'K': 29 | units_eng[col['name'].upper()] = 'DEGC' 30 | 31 | # Make STEP unique for SIM MSIDs 32 | units_eng['3LDRTPOS'] = 'TSCSTEP' 33 | units_eng['3TSCPOS'] = 'TSCSTEP' 34 | units_eng['3FAPOS'] = 'FASTEP' 35 | units_eng['3MRMMXMV'] = 'PWMSTEP' 36 | 37 | units_eng['HKEBOXTEMP'] = 'DEGF' 38 | 39 | pickle.dump(units_eng, open('units_eng.pkl', 'wb')) 40 | -------------------------------------------------------------------------------- /legacy/Makefile: -------------------------------------------------------------------------------- 1 | # Define the task name 2 | TASK = eng_archive 3 | 4 | # Set Flight environment to be SKA. The other choice is TST. Include the 5 | # Makefile.FLIGHT make file that does most of the hard work 6 | FLIGHT = SKA 7 | include /proj/sot/ska/include/Makefile.FLIGHT 8 | 9 | WWW = $(INSTALL)/www 10 | # Define the installed executables for the task. This directory is reserved 11 | # for documented tools and not for dedicated scripts, one-off codes etc 12 | # BIN = fetch fetch_server 13 | DOC = doc/_build/html/* 14 | 15 | # Installed data. These are template RDB files, mostly relevant for testing 16 | DATA = archfiles_def.sql filetypes_all.dat task_schedule*.cfg 17 | 18 | # telem_archive uses a number of dedicated perl and IDL scripts 19 | SHARE = update_archive.py transfer_stage.py check_integrity.py fetch_tutorial.py fix_bad_values.py NOTES.* 20 | 21 | .PHONY: doc clean_dp 22 | 23 | doc: 24 | cd doc; make html 25 | 26 | install: $(TEST_DEPS) 27 | # mkdir -p $(INSTALL_BIN) 28 | mkdir -p $(INSTALL_DATA) 29 | mkdir -p $(INSTALL_SHARE) 30 | mkdir -p $(INSTALL_DOC) 31 | # 32 | # rsync --times --cvs-exclude $(BIN) $(INSTALL_BIN)/ 33 | rsync --archive --cvs-exclude $(DATA) $(INSTALL_DATA)/ 34 | rsync --times --cvs-exclude $(SHARE) $(INSTALL_SHARE)/ 35 | rsync --archive --times $(DOC) $(INSTALL_DOC)/ 36 | 37 | install_doc: 38 | rsync --archive --times $(DOC) $(INSTALL_DOC)/ 39 | 40 | test_dp: clean_dp 41 | ./add_derived.py --data-root=$(PWD) 42 | ./update_archive.py --data-root=$(PWD) --content=dp_ --date-now=2000:021 --max-lookback-time=100 43 | 44 | clean_dp: 45 | rm -rf data/dp_* 46 | -------------------------------------------------------------------------------- /legacy/make_h5.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | import tables 3 | import numpy as np 4 | import Ska.Table 5 | import re 6 | 7 | if 'tlm' not in globals(): 8 | print 'Reading tlm' 9 | tlm = Ska.Table.read_fits_table('/proj/sot/ska/analysis/psmc/fitting/telem_08.fits') 10 | cols = tlm.dtype.names 11 | 12 | filters = tables.Filters(complevel=9, complib='zlib') 13 | f32 = tables.Float32Atom() 14 | fileh = tables.openFile('%s.h5' % col, mode='w', filters=filters) 15 | 16 | for col in cols: 17 | # Use ``a`` as the object type for the enlargeable array. 18 | h5col = col if re.match(r'^[a-zA-Z_]', col) else '_' + col 19 | array_f = fileh.createEArray(fileh.root, h5col, f32, (0,), title=col) 20 | array_f.append(tlm[col]) 21 | array_f.flush() 22 | print 'made and appended', col 23 | 24 | fileh.close() 25 | 26 | if 0: 27 | # Use ``a`` as the object type for the enlargeable array. 28 | array_f = fileh.createEArray(fileh.root, 'array_f', f32, (0,), "Floats") 29 | print 'created file' 30 | nxx = 100 31 | for i in range(nxx): 32 | print i 33 | nx = 1000000 34 | x = 2 * np.pi * np.arange(nx) / nx 35 | a = np.int32(np.sin(x) * 2.5 + np.random.normal(scale=0.05, size=len(x))) 36 | print 'made a' 37 | 38 | array_f.append(a) 39 | array_f.flush() 40 | print 'appended a' 41 | 42 | # Close the file. 43 | fileh.close() 44 | print 'closed file' 45 | 46 | if 0: 47 | fileh = tables.openFile('earray1.h5') 48 | print np.mean(fileh.root.array_f) 49 | fileh.close() 50 | 51 | 52 | -------------------------------------------------------------------------------- /LICENSE.rst: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2017, Smithsonian Astrophysical Observatory 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /legacy/dev_utils/example_interpolate.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | import numpy as np 3 | 4 | from Ska.Matplotlib import plot_cxctime 5 | from Ska.engarchive import fetch_eng as fetch 6 | 7 | 8 | def plot_both(x, title_str): 9 | for msid, color in (('aosares1', 'b'), ('pitch_fss', 'r')): 10 | plot_cxctime(x[msid].times, x[msid].vals, '-o' + color) 11 | bads = x[msid].bads 12 | if bads is not None and len(np.flatnonzero(bads)) > 0: 13 | plot_cxctime(x[msid].times[bads], x[msid].vals[bads], 'kx', ms=10., mew=3) 14 | title(title_str) 15 | 16 | 17 | dat = fetch.MSIDset(['aosares1','pitch_fss'], '2010:001:00:00:00','2010:001:00:00:20') 18 | dat['aosares1'].vals = (np.arange(len(dat['aosares1'])) * 4.0)[::-1] + 0.5 19 | dat['aosares1'].bads[2] = True 20 | dat['pitch_fss'].vals = np.arange(len(dat['pitch_fss'])) 21 | dat['pitch_fss'].bads[6] = True 22 | 23 | def plot_em(): 24 | close('all') 25 | figure(1, figsize=(6, 4)) 26 | plot_both(dat, 'Original data') 27 | x0, x1 = xlim() 28 | dx = (x1 - x0) * 0.05 29 | x0, x1 = x0 - dx, x1 + dx 30 | xlim(x0, x1) 31 | ylim(-0.5, 20) 32 | grid() 33 | tight_layout() 34 | 35 | for filter_bad in (False, True): 36 | for bad_union in (False, True): 37 | dat2 = dat.interpolate(dt=0.5, filter_bad=filter_bad, bad_union=bad_union, copy=True) 38 | figure(figsize=(6, 4)) 39 | plot_both(dat2, 'filter_bad={} bad_union={}'.format(filter_bad, bad_union)) 40 | xlim(x0, x1) 41 | ylim(-0.5, 20) 42 | grid() 43 | tight_layout() 44 | -------------------------------------------------------------------------------- /legacy/doc/date_time_formats.rst: -------------------------------------------------------------------------------- 1 | The Ska engineering telemetry archive tools support a wide 2 | range of formats for representing date-time stamps. Note that within this and 3 | other documents for this tool suite, the words 'time' and 'date' are used 4 | interchangably to mean a date-time stamp. 5 | 6 | The available formats are listed in the table below: 7 | 8 | ============ =============================================== ======= 9 | Format Description System 10 | ============ =============================================== ======= 11 | secs Elapsed seconds since 1998-01-01T00:00:00 tt 12 | numday DDDD:hh:mm:ss.ss... Elapsed days and time utc 13 | jd* Julian Day utc 14 | mjd* Modified Julian Day = JD - 2400000.5 utc 15 | date YYYY:DDD:hh:mm:ss.ss.. utc 16 | caldate YYYYMonDD at hh:mm:ss.ss.. utc 17 | fits FITS date/time format YYYY-MM-DDThh:mm:ss.ss.. tt 18 | unix* Unix time (since 1970.0) utc 19 | greta YYYYDDD.hhmmss[sss] utc 20 | ============ =============================================== ======= 21 | 22 | ``*`` Ambiguous for input parsing and only available as output formats. 23 | 24 | For the ``date`` format one can supply only YYYY:DDD in which case 12:00:00.000 25 | is implied. 26 | 27 | The default time "System" for the different formats is either ``tt`` 28 | (Terrestrial Time) or ``utc`` (UTC). Since TT differs from UTC by around 64 29 | seconds it is important to be consistent in specifying the time format. 30 | 31 | -------------------------------------------------------------------------------- /legacy/make_qual.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | """ 3 | Convert boolean quality vectors within each content/MSID.h5 file to an int8 4 | content/MSID_qual.h5 file. This is a one-off for Matt to be able to use files 5 | in Matlab. 6 | """ 7 | from __future__ import with_statement 8 | 9 | import re, os , sys 10 | import glob 11 | import time 12 | import cPickle as pickle 13 | 14 | import Ska.File 15 | import Ska.Table 16 | 17 | import tables 18 | import numpy as np 19 | 20 | from context_def import ft, files 21 | 22 | # Get the archive content filetypes 23 | filetypes = Ska.Table.read_ascii_table('filetypes.dat') 24 | 25 | for filetype in filetypes: 26 | ft.val.content = filetype.content.lower() 27 | 28 | if not os.path.exists(files['colnames'].abs): 29 | continue 30 | 31 | colnames = list(pickle.load(open(files['colnames'].abs))) 32 | for colname in reversed(colnames): 33 | ft.val.msid = colname 34 | if os.path.exists(files['qual'].abs): 35 | continue 36 | 37 | print 'Reading', files['oldmsid'].abs 38 | h5 = tables.openFile(files['oldmsid'].abs) 39 | iqual = np.int8(h5.root.quality[:]) 40 | h5.close() 41 | 42 | filters = tables.Filters(complevel=5, complib='zlib') 43 | 44 | print 'Creating', files['qual'].abs 45 | h5 = tables.openFile(files['qual'].abs, mode='w', filters=filters) 46 | h5shape = (0,) 47 | h5type = tables.Atom.from_dtype(iqual.dtype) 48 | h5.createEArray(h5.root, 'quality', h5type, h5shape, title=colname + ' quality', 49 | expectedrows=len(iqual)) 50 | h5.root.quality.append(iqual) 51 | h5.close() 52 | 53 | -------------------------------------------------------------------------------- /legacy/test/compare_regr_vals.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 3 | 4 | import sys 5 | import os 6 | import optparse 7 | import cPickle as pickle 8 | import numpy as np 9 | 10 | def get_options(): 11 | parser = optparse.OptionParser() 12 | parser.add_option("--root", 13 | default='regr_vals', 14 | help="Input prefix") 15 | return parser.parse_args() 16 | 17 | opt, args = get_options() 18 | 19 | flight = pickle.load(open(opt.root + '.flight')) 20 | test = pickle.load(open(opt.root + '.test')) 21 | 22 | msids = ('1crat', 'fptemp_11', 'orbitephem0_x', 'sim_z', 'tephin', 'cvcductr', 'dp_dpa_power') 23 | attrs = ('times', 'vals', 'quals', 'stds', 'mins', 'maxes', 'means', 24 | 'p01s', 'p05s', 'p16s', 'p50s', 'p84s', 'p95s', 'p99s') 25 | 26 | for msid in msids: 27 | for stat in ('dat', 'dat5', 'datd'): 28 | for attr in attrs: 29 | try: 30 | f = flight[msid][stat] 31 | t = test[msid][stat] 32 | except KeyError: 33 | print 'MSID={} stat={} missing in flight or test data'.format(msid, stat) 34 | continue 35 | if attr not in f: 36 | continue 37 | print 'Checking', msid, stat, attr, 38 | if len(f[attr]) != len(t[attr]): 39 | print 'Length mismatch:', len(f[attr]), len(t[attr]) 40 | continue 41 | if attr == 'quals': 42 | ok = f['quals'] == t['quals'] 43 | else: 44 | ok = np.max(np.abs(f[attr] - t[attr])) < 1e-6 45 | print ('OK' if ok else 'NOT OK') 46 | 47 | -------------------------------------------------------------------------------- /jeta/staging/validation.py: -------------------------------------------------------------------------------- 1 | import numba 2 | import numpy as np 3 | import pandas as pd 4 | 5 | import h5py 6 | import tables 7 | 8 | from jeta.archive.utils import get_env_variable 9 | 10 | STAGING_DIRECTORY = get_env_variable('STAGING_DIRECTORY') 11 | 12 | 13 | @numba.jit(nopython=True) 14 | def _is_sorted_numba(a): 15 | for i in range(a.size-1): 16 | if a[i+1] < a[i]: 17 | return False 18 | return True 19 | 20 | class StagingValidator(): 21 | 22 | 23 | def validate_msid_tracking(self): 24 | pass 25 | 26 | 27 | def validate_ingest_dataset_start_stop(self, h5): 28 | pass 29 | 30 | 31 | def validate_ingest_file_as_consecutive(self, h5file): 32 | try: 33 | results = self.validate_ingest_file_datasets_as_consecutive(h5file) 34 | except Exception as err: 35 | raise err 36 | return np.all(results == True) 37 | 38 | 39 | def validate_ingest_file_datasets_as_consecutive(self, h5file): 40 | """ Open an ingest file and verify the time column for 41 | each dataset is consecutive. 42 | """ 43 | with h5py.File(f"{STAGING_DIRECTORY}/{h5file}") as h5: 44 | dataset_labels = h5['samples'].keys() 45 | 46 | results = [] 47 | 48 | for label in dataset_labels: 49 | all_dset_times = pd.DataFrame(h5['samples'][label][...])['observatoryTime'][...].to_numpy() 50 | results.append(np.atleast_1d(_is_sorted_numba(np.atleast_1d(all_dset_times.byteswap().newbyteorder())))) 51 | 52 | return np.array(results) 53 | 54 | 55 | def validate_consecutive_ingest_file(self, h5): 56 | pass 57 | 58 | 59 | def __init__(self): 60 | pass 61 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | from distutils.extension import Extension 3 | import subprocess 4 | import sys 5 | 6 | from Cython.Build import cythonize 7 | import numpy 8 | 9 | # with open("README.md") as f: 10 | # long_description = f.read() 11 | 12 | setup( 13 | name='jeta', 14 | description='Modules supporting JWST engineering telemetry archiving.', 15 | license='BSD 3-Clause', 16 | long_description='', 17 | long_description_content_type='text/markdown', 18 | author='David Kauffman', 19 | author_email='dkauffman@stsci.edu', 20 | url='https://github.com/spacetelescope/jeta', 21 | packages=find_packages(include=[ 22 | 'jeta', 23 | 'jeta.archive', 24 | 'jeta.archive.*', 25 | 'jeta.staging', 26 | 'jeta.archive.derived', 27 | 'jeta.config', 28 | 'jeta.core', 29 | 'jeta.tests', 30 | ]), 31 | ext_modules = cythonize( 32 | [ 33 | Extension( 34 | "fastss", 35 | ["jeta/archive/fastss.pyx"], 36 | include_dirs=[numpy.get_include()] 37 | ) 38 | ], 39 | compiler_directives={'language_level' : "3"} 40 | ), 41 | install_requires=[ 42 | 'Cython==0.29.24', 43 | 'numpy==1.20.3', 44 | 'pandas==1.3.4', 45 | 'astropy==4.3.1', 46 | 'torch==1.10.0', 47 | 'matplotlib==3.4.3', 48 | 'seaborn==0.11.2', 49 | 'plotly==5.3.1', 50 | 'numba==0.54.1', 51 | 'h5py==3.5.0', 52 | 'tables==3.6.1', 53 | 'pyyaks==3.3.3', 54 | ], 55 | scripts=[ 56 | 'scripts/sql/create.archive.meta.sql' 57 | ] 58 | # classifiers=[ 59 | # 'License :: BSD 3-Clause', 60 | # ] 61 | ) 62 | -------------------------------------------------------------------------------- /scripts/sql/create.archive.meta.sql: -------------------------------------------------------------------------------- 1 | 2 | CREATE TABLE IF NOT EXISTS ingest_history ( 3 | ingest_id text not null, -- unique id for each ingest 4 | discovered_files int, -- the number of files discovered for ingest 5 | processed_files int, -- the number of files discovered for ingest 6 | coverage_start float not null, 7 | coverage_end float not null, 8 | tstart int not null, -- start of ingest 9 | tstop int not null, -- end of ingest 10 | rowstart int, 11 | ingest_status text, -- success, failure, null 12 | new_msids int, -- number of new msids added during this ingest 13 | chunk_size int, -- processing chunks used during this ingest 14 | uuid text, 15 | 16 | CONSTRAINT pk_ingest_id PRIMARY KEY (ingest_id) 17 | ); 18 | 19 | CREATE TABLE IF NOT EXISTS archfiles ( 20 | filename text not null, -- filename for individual the ingest file 21 | year int, -- year file was created 22 | doy int, -- day of year file was created 23 | tstart float not null, -- start of data coverage for the file, in unix time 24 | tstop float not null, -- end of data coverage for the file,unix time 25 | offset int not null, -- offset used 26 | chunk_group int not null, -- the chunk group to which this file was included in for an ingest 27 | startmjf int, 28 | stopmjf int, 29 | processing_date text not null, 30 | ingest_id text not null, 31 | 32 | FOREIGN KEY(ingest_id) REFERENCES ingest_history(ingest_id), 33 | CONSTRAINT pk_archfiles PRIMARY KEY (filename) 34 | ); 35 | 36 | CREATE INDEX IF NOT EXISTS idx_archfiles_tstart ON archfiles (tstart); -------------------------------------------------------------------------------- /scripts/python/add_msid_to_refdata.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue May 10 16:08:42 2022 4 | 5 | @author: ahunter 6 | """ 7 | 8 | import pandas as pd 9 | import h5py 10 | import sys 11 | import numpy as np 12 | from jeta.archive import operations 13 | from jeta.archive.utils import get_env_variable 14 | 15 | epoch = 2459572.5 16 | np_dtype = 'np.float64' 17 | 18 | ALL_KNOWN_MSID_METAFILE = get_env_variable('ALL_KNOWN_MSID_METAFILE') 19 | 20 | if not sys.argv[1].endswith('.xlsx'): 21 | print ('bad argument') 22 | sys.exit() 23 | 24 | prd = pd.read_excel(sys.argv[1], sheet_name=0, skiprows=3) 25 | 26 | with h5py.File(ALL_KNOWN_MSID_METAFILE, 'a') as h5: 27 | print (f'starting length: {len(h5.keys())}') 28 | msid_list = list(h5.keys()) 29 | 30 | for index, row in prd.iterrows(): 31 | mnemonic = row['TlmMnemonic'] 32 | tlmid = int(row['TlmIdentifier']) 33 | prd_dtype = row['TlmParameterDataType'] 34 | nbytes = 8 35 | 36 | if mnemonic not in msid_list: 37 | print (f'adding {mnemonic}') 38 | 39 | h5.create_group(mnemonic) 40 | h5[mnemonic].attrs['id'] = tlmid 41 | h5[mnemonic].attrs['last_ingested_timestamp'] = epoch 42 | h5[mnemonic].attrs['nbytes'] = nbytes 43 | h5[mnemonic].attrs['numpy_datatype'] = np_dtype 44 | h5[mnemonic].attrs['parameter_datatype'] = prd_dtype 45 | 46 | 47 | operations.add_msid_to_archive( 48 | mnemonic, 49 | dtype=np.float64, 50 | nrows=operations.calculate_expected_rows(4), 51 | nbytes=nbytes 52 | ) 53 | 54 | 55 | print (f'ending length: {len(h5.keys())}') 56 | -------------------------------------------------------------------------------- /scripts/shell/build_env.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= # 4 | # THESE VALUES ARE OVERWRITTEN BY THE CI/CD 5 | # PIPELINE DURING A PRODUCTION DEPLOYMENT 6 | # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= # 7 | set -x \ 8 | && echo "EXPORTING VARIABLES" 9 | 10 | # The number of workers that can be spaw 11 | export WORKERS=4 12 | 13 | # CLI Version Information 14 | export VERSION_MAJOR=0 15 | export VERSION_MINOR=0 16 | export VERSION_PATCH=0 17 | export RELEASE=dev 18 | 19 | # Project Root Directory 20 | export PROJECT_PATH=${HOME}/Projects/repos/jeta 21 | 22 | # Archive Root Directory 23 | export ENG_ARCHIVE=${HOME}/local_archive 24 | 25 | # Archive Data Root Directory (i.e. telemetery and archive metadata goes here) 26 | export TELEMETRY_ARCHIVE=${ENG_ARCHIVE}/archive 27 | 28 | # Staging Area Root Directory (i.e. Where files are delivered to be processed) 29 | export STAGING_DIRECTORY=${ENG_ARCHIVE}/staging 30 | 31 | # MSID Data Reference File 32 | export ALL_KNOWN_MSID_METAFILE=${ENG_ARCHIVE}/all_known_msid_sematics.h5 33 | 34 | # Root Directory for System Logs 35 | export JETA_LOGS=${ENG_ARCHIVE}/logs 36 | 37 | # Root Directory for System Support Scripts 38 | export JETA_SCRIPTS=${PROJECT_PATH}/scripts 39 | 40 | # Path to Meta Database Definition File 41 | export JETA_ARCHIVE_DEFINITION_SOURCE=${JETA_SCRIPTS}/sql/create.archive.meta.sql 42 | 43 | # Relevant Field Names 44 | export NAME_COLUMN="Telemetry Mnemonic" 45 | export TIME_COLUMN="Observatory Time" 46 | export VALUE_COLUMN="EU Value" 47 | 48 | # Dummy Web API Session Key 49 | export RAVEN_SECRET_KEY=cGlvdXl0aXVmaGpjbWJuajhoOTdweWlsaG45cDc2ODU3aWRyeWljZmtndmpoYms7amk4dTB5OTdneWl2aGtiamxuamk4MDk3Z3lpYmhrCg 50 | 51 | # JupyterLab Specific Variables 52 | export JUPYTERHUB_HOME=${ENG_ARCHIVE}/jupyterhub 53 | export JUPYTERHUB_CONFIG_DIRECTORY=${JUPYTERHUB_HOME}/config 54 | export JUPYTERHUB_LOG_DIRECTORY=${JUPYTERHUB_HOME}/log 55 | export JUPYTERHUB_USER_SPACE=${JUPYTERHUB_HOME}/user_space 56 | 57 | # Local Network Subnet for Docker 58 | export NETWORK_SUBNET=192.168.1.0/24 59 | -------------------------------------------------------------------------------- /legacy/doc/fetchplots/make_interpolate.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | """ 3 | Make plots illustrating MSIDset.interpolate() 4 | 5 | % skatest (or skadev) 6 | $ export ENG_ARCHIVE=/proj/sot/ska/data/eng_archive 7 | $ ipython --classic 8 | >>> run -i make_interpolate 9 | >>> plot_em() 10 | """ 11 | 12 | import numpy as np 13 | 14 | from Ska.Matplotlib import plot_cxctime 15 | from Ska.engarchive import fetch_eng as fetch 16 | import matplotlib.pyplot as plt 17 | 18 | 19 | dat = fetch.MSIDset(['aosares1', 'pitch_fss'], '2010:001:00:00:00', '2010:001:00:00:20') 20 | dat['aosares1'].vals = (np.arange(len(dat['aosares1'])) * 4.0)[::-1] + 0.5 21 | dat['aosares1'].bads[2] = True 22 | dat['pitch_fss'].vals = np.arange(len(dat['pitch_fss'])) 23 | dat['pitch_fss'].bads[6] = True 24 | 25 | 26 | def plot_both(x, title_str): 27 | for msid, color in (('aosares1', 'b'), ('pitch_fss', 'r')): 28 | plot_cxctime(x[msid].times, x[msid].vals, '-o' + color) 29 | bads = x[msid].bads 30 | if bads is not None and len(np.flatnonzero(bads)) > 0: 31 | plot_cxctime(x[msid].times[bads], x[msid].vals[bads], 'kx', ms=10., mew=2.5) 32 | plt.title(title_str) 33 | 34 | 35 | def plot_em(): 36 | plt.close('all') 37 | plt.figure(1, figsize=(6, 4)) 38 | plot_both(dat, 'Original data') 39 | x0, x1 = plt.xlim() 40 | dx = (x1 - x0) * 0.05 41 | x0, x1 = x0 - dx, x1 + dx 42 | 43 | def fix_plot(): 44 | plt.xlim(x0, x1) 45 | plt.ylim(-0.5, 20) 46 | plt.grid() 47 | plt.tight_layout() 48 | 49 | fix_plot() 50 | plt.savefig('interpolate_input.png') 51 | 52 | for filter_bad in (False, True): 53 | for bad_union in (False, True): 54 | dat2 = dat.interpolate(dt=0.5, filter_bad=filter_bad, bad_union=bad_union, copy=True) 55 | plt.figure(figsize=(6, 4)) 56 | plot_both(dat2, 'filter_bad={} bad_union={}'.format(filter_bad, bad_union)) 57 | fix_plot() 58 | plt.savefig('interpolate_{}_{}.png'.format(filter_bad, bad_union)) 59 | 60 | 61 | if __name__ == '__main__': 62 | plot_em() 63 | -------------------------------------------------------------------------------- /legacy/test/get_regr_vals.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 3 | 4 | import sys 5 | import os 6 | import optparse 7 | import cPickle as pickle 8 | 9 | def get_options(): 10 | parser = optparse.OptionParser() 11 | parser.add_option("--test", 12 | action="store_true", 13 | help="Test") 14 | parser.add_option("--start", 15 | default='2010:273', 16 | help="Start time") 17 | parser.add_option("--stop", 18 | default='2010:285', 19 | help="Stop time") 20 | parser.add_option("--out", 21 | default='regr_vals', 22 | help="Output prefix") 23 | return parser.parse_args() 24 | 25 | opt, args = get_options() 26 | if opt.test: 27 | sys.path.insert(0, '..') 28 | if not 'ENG_ARCHIVE' in os.environ: 29 | os.environ['ENG_ARCHIVE'] = os.path.abspath(os.getcwd() + '/eng_archive') 30 | outfile = opt.out + '.test' 31 | else: 32 | outfile = opt.out + '.flight' 33 | 34 | import Ska.engarchive.fetch as fetch 35 | print 'Fetch file is', fetch.__file__ 36 | print 'ENG_ARCHIVE is', os.environ.get('ENG_ARCHIVE') 37 | 38 | msids = ('1crat', 'fptemp_11', 'orbitephem0_x', 'sim_z', 'tephin', 'cvcductr', 'dp_dpa_power') 39 | attrs = ('times', 'vals', 'quals', 'stds', 'mins', 'maxes', 'means', 40 | 'p01s', 'p05s', 'p16s', 'p50s', 'p84s', 'p95s', 'p99s') 41 | out = dict() 42 | for msid in msids: 43 | print 'Getting', msid 44 | dat = fetch.MSID(msid, opt.start, opt.stop) 45 | dat5 = fetch.MSID(msid, opt.start, opt.stop, stat='5min') 46 | datd = fetch.MSID(msid, opt.start, opt.stop, stat='daily') 47 | out[msid] = dict(dat=dict((x, getattr(dat, x)) for x in attrs if hasattr(dat, x)), 48 | dat5=dict((x, getattr(dat5, x)) for x in attrs if hasattr(dat5, x)), 49 | datd=dict((x, getattr(datd, x)) for x in attrs if hasattr(datd, x))) 50 | 51 | out['ENG_ARCHIVE'] = os.environ.get('ENG_ARCHIVE') 52 | out['file'] = fetch.__file__ 53 | pickle.dump(out, open(outfile, 'w'), protocol=-1) 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /legacy/move_fits_archive.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 3 | """ 4 | One-off tool to move archive files from original flat structure (everything in 5 | one directory per content-type) to /arch///[files] structure. 6 | """ 7 | import re, os , sys 8 | import glob 9 | import time 10 | import shutil 11 | 12 | from Chandra.Time import DateTime 13 | import Ska.Table 14 | import pyyaks.logger 15 | import pyyaks.context 16 | 17 | import Ska.engarchive.file_defs as file_defs 18 | 19 | arch_files = pyyaks.context.ContextDict('arch_files', basedir=file_defs.arch_root) 20 | arch_files.update(file_defs.arch_files) 21 | 22 | orig_arch_files = pyyaks.context.ContextDict('orig_arch_files', basedir=file_defs.orig_arch_root) 23 | orig_arch_files.update(file_defs.orig_arch_files) 24 | 25 | ft = pyyaks.context.ContextDict('ft') 26 | 27 | def main(): 28 | filetypes = Ska.Table.read_ascii_table('filetypes.dat') 29 | if len(sys.argv) == 2: 30 | filetypes = filetypes[ filetypes['content'] == sys.argv[1].upper() ] 31 | 32 | loglevel = pyyaks.logger.INFO 33 | logger = pyyaks.logger.get_logger(level=loglevel, format="%(message)s") 34 | 35 | for filetype in filetypes: 36 | ft.content = filetype.content.lower() 37 | 38 | orig_files_glob = os.path.join(orig_arch_files['contentdir'].abs, filetype['fileglob']) 39 | logger.info('orig_files_glob=%s', orig_files_glob) 40 | for f in glob.glob(orig_files_glob): 41 | ft.basename = os.path.basename(f) 42 | tstart = re.search(r'(\d+)', ft.basename).group(1) 43 | datestart = DateTime(tstart).date 44 | ft.year, ft.doy = re.search(r'(\d\d\d\d):(\d\d\d)', datestart).groups() 45 | 46 | archdir = arch_files['archdir'].abs 47 | archfile = arch_files['archfile'].abs 48 | 49 | if not os.path.exists(archdir): 50 | print 'Making dir', archdir 51 | os.makedirs(archdir) 52 | 53 | if not os.path.exists(archfile): 54 | # logger.info('mv %s %s' % (f, archfile)) 55 | shutil.move(f, archfile) 56 | 57 | if __name__ == '__main__': 58 | main() 59 | -------------------------------------------------------------------------------- /jeta/tests/file_defs.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | """ 3 | Directory and file location definitions for telemetry archive applications. 4 | This is to be used to define corresponding ContextDict objects. 5 | """ 6 | import os 7 | 8 | JETA_TEST_CACHE = os.environ.get('JETA_TEST_CACHE') or '/Users/dkauffman/jeta-test-cache' 9 | 10 | 11 | # Root directories for MSID files. msid_root is prime, others are backups. 12 | # NOTE: msid_root(s) used ONLY in one-off or legacy code, not in update_archive.py or 13 | # transfer_stage.py 14 | msid_root = os.path.join(JETA_TEST_CACHE, 'data', 'jeta') 15 | msid_roots = [msid_root] 16 | 17 | msid_files = { 18 | 'filetypes': 'filetypes.dat', 19 | 'msid_bad_times': 'msid_bad_times.dat', 20 | 'contentdir': 'data/{{ft.content}}/', 21 | 'headers': 'data/{{ft.content}}/headers.pickle', 22 | 'archfiles': 'archive.meta.info.db3', 23 | 'colnames': 'colnames.pickle', 24 | 'colnames_all': 'colnames_all.pickle', 25 | 'msid': 'data/{{ft.content}}/{{ft.msid | upper}}', 26 | 'data': 'data/{{ft.content}}/{{ft.msid | upper}}.h5', 27 | 'statsdir': 'data/{{ft.content}}/stats/{{ft.interval}}/', 28 | 'stats': 'data/tlm/stats/{{ft.interval}}/{{ft.msid | upper }}.h5', 29 | 'processed_files_directory': 'processed_files', 30 | 'mnemonic_index': 'data/{{ft.content}}/{{ft.msid | upper}}/index.h5', 31 | 'mnemonic_value': 'data/{{ft.content}}/{{ft.msid | upper}}/values.h5', 32 | 'mnemonic_times': 'data/{{ft.content}}/{{ft.msid | upper}}/times.h5' 33 | } 34 | 35 | 36 | # NOTE: arch_root used ONLY in one-off or legacy code, not in update_archive.py or 37 | # transfer_stage.py 38 | arch_root = '/data/cosmos2/eng_archive' 39 | arch_files = {'stagedir': 'stage/{{ft.content}}/', 40 | 'rootdir': '', 41 | 'archrootdir': 'data/{{ft.content}}/arch/', 42 | 'archdir': 'data/{{ft.content}}/arch/{{ft.year}}/{{ft.doy}}/', 43 | 'archfile': 'data/{{ft.content}}/arch/{{ft.year}}/{{ft.doy}}/{{ft.basename}}', 44 | } 45 | 46 | # Used when originally creating database. 47 | orig_arch_root = '/data/cosmos2/tlm' 48 | orig_arch_files = {'contentdir': '{{ft.content}}/'} 49 | -------------------------------------------------------------------------------- /legacy/dev_utils/fix_neg_dts.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | """ 3 | Fix residual overlaps within the original archive and h5 ingest. These 4 | are short overlaps that for some reason were not fixed by fix_h5_ingest.py 5 | """ 6 | 7 | import os 8 | import optparse 9 | 10 | import numpy as np 11 | import tables 12 | import asciitable 13 | 14 | from context_def import ft, files 15 | 16 | 17 | def get_options(): 18 | parser = optparse.OptionParser() 19 | parser.add_option("--dry-run", 20 | action="store_true", 21 | help="Dry run (no actual file or database updatees)") 22 | return parser.parse_args() 23 | 24 | opt, args = get_options() 25 | 26 | filetypes = asciitable.read('Ska/engarchive/filetypes.dat') 27 | if len(args) > 0: 28 | filetypes = filetypes[filetypes['content'] == args[0].upper()] 29 | 30 | for filetype in filetypes: 31 | # Update attributes of global ContextValue "ft". This is needed for 32 | # rendering of "files" ContextValue. 33 | print filetype.content 34 | 35 | ft['content'] = filetype['content'].lower() 36 | ft['msid'] = 'TIME' 37 | 38 | # archive files 39 | if not os.path.exists(files['oldmsid'].abs + '.bak'): 40 | print 'Skipping', ft['content'], ' because there is no backup from fix_ingest_h5.py' 41 | continue 42 | 43 | # Open the TIME.h5 file for this content type 44 | h5 = tables.openFile(files['oldmsid'].abs, mode=('r' if opt.dry_run else 'a')) 45 | goods = ~h5.root.quality[:] 46 | times = h5.root.data[:] 47 | print(len(times)) 48 | times = times[goods] 49 | print(len(times)) 50 | while True: 51 | bad_indexes = np.where(np.diff(times) < 0)[0] 52 | print bad_indexes 53 | if len(bad_indexes) == 0: 54 | break 55 | i0 = bad_indexes[0] 56 | time0 = times[i0] 57 | 58 | for i1 in xrange(i0 + 1, len(times) - 1): 59 | if times[i1] < time0: 60 | print 'Setting row {} = {} to bad quality'.format(i1, times[i1]) 61 | if not opt.dry_run: 62 | h5.root.quality[i1] = True 63 | else: 64 | break 65 | break 66 | 67 | h5.close() 68 | -------------------------------------------------------------------------------- /legacy/ingest_fits_archive.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | import time 3 | import os 4 | import sys 5 | 6 | import asciitable 7 | import Ska.arc5gl as arc5gl 8 | 9 | filetypes = asciitable.read('Ska/engarchive/filetypes.dat') 10 | if len(sys.argv) >= 2: 11 | filetypes = filetypes[filetypes['content'] == sys.argv[1].upper()] 12 | outroot = sys.argv[2] if len(sys.argv) >= 3 else '/data/cosmos2/eng_archive/tlm' 13 | 14 | 15 | for filetype in filetypes: 16 | if filetype['content'] in ('ORBITEPHEM', 'LUNAREPHEM', 'SOLAREPHEM'): 17 | doys = range(1, 372, 5) 18 | else: 19 | doys = (1, 367) 20 | 21 | try: 22 | content = filetype['content'].lower() 23 | arc5gl_query = filetype['arc5gl_query'].lower() 24 | 25 | outdir = os.path.abspath(os.path.join(outroot, content)) 26 | if os.path.exists(outdir): 27 | print 'Skipping', content, 'at', time.ctime() 28 | continue 29 | else: 30 | print 'Making dir', outdir 31 | os.makedirs(outdir) 32 | 33 | os.chdir(outdir) 34 | arc5 = arc5gl.Arc5gl() 35 | 36 | print '**********', content, time.ctime(), '***********' 37 | print ' cd ' + outdir 38 | arc5.sendline('cd ' + outdir) 39 | arc5.sendline('version = last') 40 | 41 | for year in range(1999, 2000): 42 | if os.path.exists('/pool14/wink/stoptom'): 43 | raise RuntimeError('Stopped by sherry') 44 | for doy0, doy1 in zip(doys[:-1], doys[1:]): 45 | datestart = '%d:%03d:00:00:00' % (year, doy0) 46 | datestop = '%d:%03d:00:00:00' % (year, doy1) 47 | 48 | sys.stdout.flush() 49 | print ' tstart=%s' % datestart 50 | print ' tstop=%s' % datestop 51 | print ' get %s at %s' % (arc5gl_query, time.ctime()) 52 | 53 | arc5.sendline('tstart=%s' % datestart) 54 | arc5.sendline('tstop=%s;' % datestop) 55 | arc5.sendline('get %s' % arc5gl_query.lower()) 56 | 57 | open('.process', 'w') 58 | 59 | finally: 60 | # explicitly close connection to archive 61 | if 'arc5' in globals(): 62 | del arc5 63 | -------------------------------------------------------------------------------- /legacy/update_fits_archive.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | import time 3 | import re, os , sys 4 | import glob 5 | 6 | import numpy as np 7 | import pexpect 8 | 9 | import Ska.Table 10 | import arc5gl 11 | from Chandra.Time import DateTime 12 | 13 | filetypes = Ska.Table.read_ascii_table('filetypes.dat') 14 | filetypes = filetypes[ filetypes.pipe == 'ENG0' ] 15 | 16 | datestop = DateTime(time.time(), format='unix').date 17 | 18 | FITS_dir = 'test/tlm' 19 | 20 | FT = ContextDict('ft') 21 | ft = FT.accessor() 22 | 23 | FILES = ContextDict('files', basedir=fits_dir) 24 | FILES.update({'outdir': '{{ft.content}}', 25 | 'image': '{{ft_id}}/image' 26 | }) 27 | files = FILES.accessor() 28 | 29 | loglevel = pyyaks.logger.INFO 30 | logfile = 'run.log' 31 | logger = pyyaks.logger.get_logger(level=loglevel, filename=logfile, format="%(message)s") 32 | 33 | 34 | for filetype in filetypes: 35 | try: 36 | ft.content = filetype.content.lower() 37 | ft.instrum = filetype.instrum.lower() 38 | 39 | if os.path.exists(outdir): 40 | print 'Skipping', content, 'at', time.ctime() 41 | continue 42 | else: 43 | os.makedirs(outdir) 44 | 45 | os.chdir(outdir) 46 | arc5 = arc5gl.Arc5gl() 47 | 48 | print '**********', content, time.ctime(), '***********' 49 | print ' cd ' + outdir 50 | arc5.sendline('cd ' + outdir) 51 | 52 | for year in range(2000, 2010): 53 | if os.path.exists('/pool14/wink/stoptom'): 54 | raise RuntimeError('Stopped by sherry') 55 | datestart = '%d:001:00:00:00' % year 56 | datestop = '%d:001:00:00:00' % (year+1) 57 | 58 | print ' tstart=%s' % datestart 59 | print ' tstop=%s' % datestop 60 | print ' get %s_eng_0{%s}' % (instrum, content) 61 | 62 | arc5.sendline('tstart=%s' % datestart) 63 | arc5.sendline('tstop=%s;' % datestop) 64 | arc5.sendline('get %s_eng_0{%s}' % (instrum, content)) 65 | 66 | open('.process', 'w') 67 | 68 | finally: 69 | # explicitly close connection to archive 70 | if 'arc5' in globals(): 71 | del arc5 72 | 73 | 74 | 75 | -------------------------------------------------------------------------------- /legacy/rebuild_stats/move_stats.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | import sys 3 | import os 4 | import shutil 5 | from glob import glob 6 | 7 | old = '/proj/sot/ska/data/eng_archive/data' 8 | bak = '/proj/sot/ska/data/eng_archive/data_stats_bak' 9 | new = '/proj/sot/ska/tmp/eng_archive/data' 10 | 11 | # For testing 12 | #old = '/home/SOT/git/eng_archive/dataold' 13 | #new = '/home/SOT/git/eng_archive/datanew' 14 | #bak = '/home/SOT/git/eng_archive/databak' 15 | 16 | dryrun = True 17 | 18 | dirs = sorted(glob(os.path.join(old, '*'))) 19 | for dir in dirs: 20 | content = os.path.basename(dir) 21 | print 'Content:', content 22 | bak_content = os.path.join(bak, content) 23 | old_content = os.path.join(old, content) 24 | for stat in ('5min', 'daily'): 25 | bak_stat = os.path.join(bak, content, stat) 26 | old_stat = os.path.join(old, content, stat) 27 | new_stat = os.path.join(new, content, stat) 28 | if not os.path.exists(old_stat): 29 | print 'Skipping {}/{} not in old_stat'.format(content, stat) 30 | continue 31 | 32 | if not os.path.exists(new_stat): 33 | print 'Skipping {}/{} not in new_stat'.format(content, stat) 34 | continue 35 | 36 | if os.path.exists(bak_stat): 37 | print 'Skipping {}/{} already done'.format(content, stat) 38 | continue 39 | 40 | if not os.path.exists(bak_content): 41 | print 'Making dir', bak_content 42 | if not dryrun: 43 | os.mkdir(bak_content) 44 | 45 | print 'mv {} {}'.format(old_stat, bak_content) 46 | if not dryrun: 47 | shutil.move(old_stat, bak_content) 48 | if os.path.exists(old_stat): 49 | raise Exception('fail old_stat {} still there'.format(old_stat)) 50 | if not os.path.exists(bak_stat): 51 | raise Exception('fail bak_stat {} not there'.format(bak_stat)) 52 | 53 | print 'mv {} {}'.format(new_stat, old_content) 54 | if not dryrun: 55 | shutil.move(new_stat, old_content) 56 | if os.path.exists(new_stat): 57 | raise Exception('fail new_stat {} still there'.format(new_stat)) 58 | if not os.path.exists(old_stat): 59 | raise Exception('fail old_stat {} not there'.format(old_stat)) 60 | -------------------------------------------------------------------------------- /legacy/doc/index.rst: -------------------------------------------------------------------------------- 1 | ================================== 2 | Ska Engineering Telemetry Archive 3 | ================================== 4 | .. include:: references.rst 5 | 6 | Overview 7 | -------- 8 | 9 | The Ska engineering telemetry archive is a suite of tools and data products 10 | that make available the majority of all Chandra engineering telemetry since the 11 | start of the mission (1999:204). This includes about 6300 MSIDs. The telemetry are stored 12 | in a way that allows for very fast and efficient retrieval into memory. 13 | Typical retrieve rates are around 10^7 samples/sec. For an MSID sampled once 14 | per second this translates to about 3 sec per year of data. 15 | 16 | The engineering telemetry archive consists of: 17 | 18 | * Tools to ingest and compress telemetry from the CXC Chandra archive products. 19 | * Compressed telemetry files in HDF5 format. Each MSID has three associated products: 20 | 21 | - Full time-resolution data: time, value, quality 22 | - 5-minute statistics: min, max, mean, sampled value, number of samples 23 | - Daily statistics: min, max, mean, sampled value, standard deviation, percentiles (1, 24 | 5, 16, 50, 84, 95, 99), number of samples. 25 | * A python module to retrieve telemetry values. 26 | 27 | Pseudo-MSIDs 28 | ---------------------------- 29 | 30 | A small selection of pseudo-MSIDs that do not come in the engineering telemetry 31 | stream are also available in the archive. These are: 32 | 33 | * SIM telemetry: SIM position and moving status (deprecated) 34 | * EPHIN telemetry: level-0 science telemetry from EPHIN 35 | * ACIS DEA housekeeping: status from the DEA (including detector focal plane temperature) 36 | * Ephemeris: predictive and definitive orbital (Chandra), solar, and lunar ephemeris values 37 | * Derived parameters: values computed from other MSIDs in the archive (ACIS power, orbital elements, PCAD, thermal) 38 | 39 | For details see the documentation on Pseudo-MSIDs in the engineering archive. 40 | 41 | Documentation 42 | ------------- 43 | 44 | .. toctree:: 45 | :maxdepth: 1 46 | 47 | tutorial 48 | fetch_tutorial 49 | ska_fetch 50 | pseudo_msids 51 | 52 | API docs 53 | -------- 54 | 55 | .. toctree:: 56 | :maxdepth: 1 57 | 58 | fetch 59 | plot 60 | utils 61 | 62 | .. toctree:: 63 | :hidden: 64 | 65 | date_time_formats 66 | fetch_tutorial_standalone 67 | ipython_tutorial 68 | matplotlib_tutorial 69 | numpy_tutorial 70 | references 71 | -------------------------------------------------------------------------------- /scripts/sql/update_primary_key_type.sql: -------------------------------------------------------------------------------- 1 | 2 | CREATE TABLE IF NOT EXISTS copy_ingest_history ( 3 | ingest_id text not null, -- unique id for each ingest 4 | discovered_files int, -- the number of files discovered for ingest 5 | processed_files int, -- the number of files discovered for ingest 6 | coverage_start float not null, 7 | coverage_end float not null, 8 | tstart int not null, -- start of ingest 9 | tstop int not null, -- end of ingest 10 | rowstart int, 11 | ingest_status text, -- success, failure, null 12 | new_msids int, -- number of new msids added during this ingest 13 | chunk_size int, -- processing chunks used during this ingest 14 | 15 | CONSTRAINT pk_ingest_id PRIMARY KEY (ingest_id) 16 | ); 17 | 18 | CREATE TABLE IF NOT EXISTS copy_archfiles ( 19 | filename text not null, -- filename for individual the ingest file 20 | year int, -- year file was created 21 | doy int, -- day of year file was created 22 | tstart float not null, -- start of data coverage for the file, in unix time 23 | tstop float not null, -- end of data coverage for the file,unix time 24 | offset int not null, -- offset used 25 | chunk_group int not null, -- the chunk group to which this file was included in for an ingest 26 | startmjf int, 27 | stopmjf int, 28 | processing_date text not null, 29 | ingest_id text not null, 30 | 31 | FOREIGN KEY(ingest_id) REFERENCES ingest_history(ingest_id), 32 | CONSTRAINT pk_archfiles PRIMARY KEY (filename) 33 | ); 34 | 35 | CREATE INDEX IF NOT EXISTS idx_archfiles_tstart ON archfiles (tstart); 36 | 37 | INSERT INTO copy_ingest_history (ingest_id, discovered_files, processed_files, coverage_start, coverage_end, tstart, tstop, rowstart, ingest_status, new_msids, chunk_size) 38 | SELECT cast (ingest_id as text), discovered_files, processed_files, coverage_start, coverage_end, tstart, tstop, rowstart, ingest_status, new_msids, chunk_size FROM ingest_history; 39 | 40 | INSERT INTO copy_archfiles (filename, year, doy, tstart, tstop, "offset", chunk_group, startmjf, stopmjf, processing_date, ingest_id) 41 | SELECT filename, year, doy, tstart, tstop, "offset", chunk_group, startmjf, stopmjf, processing_date, cast (ingest_id as text) FROM archfiles ; -------------------------------------------------------------------------------- /legacy/NOTES.resync_occ: -------------------------------------------------------------------------------- 1 | ########################################################################## 2 | ## On gretasot on GRETA 3 | ########################################################################## 4 | 5 | # TRUNCATE BAD records 6 | set content=thm1eng 7 | $ska/share/eng_archive/update_archive.py --occ [--dry-run] [--content $content] \ 8 | --truncate 2011:205 --data-root /proj/sot/ska/data/eng_archive \ 9 | >&! run.log & 10 | tail -f run.log 11 | 12 | 13 | ########################################################################## 14 | ## On HEAD side 15 | ########################################################################## 16 | 17 | # Make an artificial stage dir using archived CXCDS arch files 18 | mkdir stage 19 | ./make_resync_stage --date 20 | # then rsync -a stage/ => mac => GRETA 21 | 22 | 23 | ########################################################################## 24 | ## On gretasot on GRETA 25 | ########################################################################## 26 | 27 | NOTE: if the duration to be resynced is longer than 60 days you must use 28 | the --max-lookback flag! 29 | 30 | NOTE: the --data-root flag is needed, default is wrong for OCC 31 | 32 | # PRACTICE with single content types 33 | cd $ska/share/eng_archive 34 | $ska/share/eng_archive/update_archive.py --occ --content=$content \ 35 | --data-root /proj/sot/ska/data/eng_archive > & ! run.log & 36 | tail -f run.log 37 | 38 | # Only 100 files at a time are ingested, CCDM4 *crashes* because of memory after ~250 files. 39 | 40 | $ska/share/eng_archive/update_archive.py --occ --data-root /proj/sot/ska/data/eng_archive >& run3.log 41 | $ska/share/eng_archive/update_archive.py --occ --data-root /proj/sot/ska/data/eng_archive >& run4.log 42 | $ska/share/eng_archive/update_archive.py --occ --data-root /proj/sot/ska/data/eng_archive >& run5.log 43 | $ska/share/eng_archive/update_archive.py --occ --data-root /proj/sot/ska/data/eng_archive >& run6.log 44 | $ska/share/eng_archive/update_archive.py --occ --data-root /proj/sot/ska/data/eng_archive >& run7.log 45 | $ska/share/eng_archive/update_archive.py --occ --data-root /proj/sot/ska/data/eng_archive >& run8.log 46 | $ska/share/eng_archive/update_archive.py --occ --data-root /proj/sot/ska/data/eng_archive >& run9.log 47 | $ska/share/eng_archive/update_archive.py --occ --data-root /proj/sot/ska/data/eng_archive >& run10.log 48 | $ska/share/eng_archive/update_archive.py --occ --data-root /proj/sot/ska/data/eng_archive >& run11.log 49 | 50 | -------------------------------------------------------------------------------- /jeta/archive/tests/test_archive_status_api.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tempfile 3 | from time import sleep 4 | 5 | import unittest 6 | from unittest import mock 7 | 8 | 9 | JETA_TEST_CACHE = '/Users/dkauffman/jeta-test-cache' 10 | FIXTURE_ARCHIVE = '/Users/dkauffman/jeta-test-cache/archive' 11 | FIXTURE_STAGE = '/Users/dkauffman/jeta-test-cache/stage' 12 | 13 | 14 | class TestArchiveStatusAPI(unittest.TestCase): 15 | 16 | def setUp(self) -> None: 17 | return super().setUp() 18 | 19 | def tearDown(self) -> None: 20 | return super().tearDown() 21 | 22 | def test_smoke_test(self): 23 | assert 1 == 1 24 | 25 | @mock.patch.dict(os.environ, { 26 | "TELEMETRY_ARCHIVE": FIXTURE_ARCHIVE 27 | } 28 | ) 29 | def test_get_msid_count(self): 30 | import pickle 31 | 32 | tmp = tempfile.TemporaryDirectory() 33 | with tmp as tempdir: 34 | os.environ['TELEMETRY_ARCHIVE'] = tempdir 35 | 36 | test_msid_list = ['TEST_MSID'] 37 | expected_result = len(test_msid_list) 38 | with open(os.path.join(tempdir, 'colnames.pickle'), '+wb') as tmp_pickle: 39 | pickle.dump(test_msid_list, tmp_pickle) 40 | 41 | from jeta.archive.status import get_msid_count 42 | actual_result = get_msid_count() 43 | tmp.cleanup() 44 | 45 | assert actual_result == expected_result 46 | 47 | def test_get_list_of_activities(self): 48 | self.fail('Write a test for: test_get_list_of_activities') 49 | 50 | def test_get_list_of_files_in_range(self): 51 | self.fail('Write a test for: test_get_list_of_files_in_range') 52 | 53 | # @mock.patch.dict(os.environ, { 54 | # "TELEMETRY_ARCHIVE": FIXTURE_ARCHIVE 55 | # } 56 | # ) 57 | # def test_get_msid_names(self): 58 | # import pickle 59 | 60 | # tmp = tempfile.TemporaryDirectory() 61 | 62 | # with tmp as tempdir: 63 | # print(tempdir) 64 | # os.environ['TELEMETRY_ARCHIVE'] = tempdir 65 | 66 | # test_msid_list = ['TEST_MSID'] 67 | # expected_result = test_msid_list 68 | 69 | # with open(os.path.join(tempdir, 'colnames.pickle'), '+wb') as tmp_pickle: 70 | # pickle.dump(test_msid_list, tmp_pickle) 71 | 72 | # from jeta.archive.status import get_msid_names 73 | # actual_result = get_msid_names() 74 | # assert actual_result == expected_result 75 | -------------------------------------------------------------------------------- /jeta/archive/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | import numpy as np 3 | 4 | from ..utils import get_fetch_size 5 | from .. import fetch 6 | 7 | 8 | def test_get_fetch_size_functionality(): 9 | """ 10 | Various functionality tests for estimating fetch memory usage. 11 | """ 12 | fetch_mb, out_mb = get_fetch_size('aopcadmd', '2010:001', '2011:001') 13 | assert fetch_mb, out_mb == (399.97, 399.97) 14 | 15 | fetch_mb, out_mb = get_fetch_size('aopcadmd', '2010:001', '2011:001', interpolate_dt=1.025 * 10) 16 | assert fetch_mb, out_mb == (399.97, 40.0) 17 | 18 | fetch_mb, out_mb = get_fetch_size('aopcadmd', '2010:001', '2011:001', fast=False, stat='5min') 19 | assert fetch_mb, out_mb == (1.92, 1.92) 20 | 21 | # 5min stat 22 | fetch_mb, out_mb = get_fetch_size('aopcadmd', '2010:001', '2011:001', stat='5min') 23 | assert fetch_mb, out_mb == (-1, -1) 24 | 25 | # Too short 26 | fetch_mb, out_mb = get_fetch_size('aopcadmd', '2010:001', '2010:030') 27 | assert fetch_mb, out_mb == (-1, -1) 28 | 29 | 30 | def test_get_fetch_size_accuracy(): 31 | """ 32 | Does it give the right answer? 33 | """ 34 | # By hand for stat 35 | dat = fetch.MSID('aopcadmd', '2010:001', '2011:001', stat='5min') 36 | fetch_bytes = sum(getattr(dat, attr).nbytes for attr in dat.colnames) 37 | 38 | fetch_mb, out_mb = get_fetch_size('aopcadmd', '2010:001', '2011:001', stat='5min', 39 | interpolate_dt=328 * 2, fast=False) 40 | assert np.isclose(fetch_mb, fetch_bytes / 1e6, rtol=0.0, atol=0.01) 41 | 42 | # Now interpolate to 10 minute intervals 43 | dat.interpolate(328.0 * 2) 44 | fetch_bytes = sum(getattr(dat, attr).nbytes for attr in dat.colnames) 45 | assert np.isclose(out_mb, fetch_bytes / 1e6, rtol=0.0, atol=0.01) 46 | 47 | # By hand for full resolution 48 | dat = fetch.MSID('aopcadmd', '2011:001', '2011:010') 49 | fetch_bytes = sum(getattr(dat, attr).nbytes for attr in dat.colnames) 50 | 51 | fetch_mb, out_mb = get_fetch_size('aopcadmd', '2011:001', '2011:010', 52 | interpolate_dt=328 * 2, fast=False) 53 | assert np.isclose(fetch_mb, fetch_bytes / 1e6, rtol=0.0, atol=0.01) 54 | 55 | # Now interpolate to 10 minute intervals 56 | dat.interpolate(328.0 * 2) 57 | fetch_bytes = sum(getattr(dat, attr).nbytes for attr in dat.colnames) 58 | assert np.isclose(out_mb, fetch_bytes / 1e6, rtol=0.0, atol=0.01) 59 | -------------------------------------------------------------------------------- /legacy/update_h5_archive.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | import time 3 | import tables 4 | import numpy as np 5 | import Ska.Table 6 | import re 7 | import os 8 | import sys 9 | import glob 10 | 11 | def make_h5_col_file(dat, content, colname): 12 | """Make a new h5 table to hold column from ``dat``.""" 13 | filename = os.path.join('data', content, 'msid', colname + '.h5') 14 | if os.path.exists(filename): 15 | os.unlink(filename) 16 | filedir = os.path.dirname(filename) 17 | if not os.path.exists(filedir): 18 | os.makedirs(filedir) 19 | 20 | filters = tables.Filters(complevel=5, complib='zlib') 21 | h5 = tables.openFile(filename, mode='w', filters=filters) 22 | 23 | col = dat[colname] 24 | h5shape = (0,) + col.shape[1:] 25 | h5type = tables.Atom.from_dtype(col.dtype) 26 | h5.createEArray(h5.root, 'data', h5type, h5shape, title=colname, 27 | expectedrows=86400*365*10) 28 | h5.createEArray(h5.root, 'quality', tables.BoolAtom(), (0,), title='Quality', 29 | expectedrows=86400*365*10) 30 | print 'Made', colname 31 | h5.close() 32 | 33 | def append_h5_col(dat, content, colname, i_colname): 34 | filename = os.path.join('data', content, 'msid', colname + '.h5') 35 | h5 = tables.openFile(filename, mode='a') 36 | h5.root.data.append(dat[colname]) 37 | h5.root.quality.append(dat['QUALITY'][:,i_colname]) 38 | h5.close() 39 | 40 | filetypes = Ska.Table.read_ascii_table('filetypes.dat') 41 | filetypes = filetypes[ filetypes.pipe == 'ENG0' ] 42 | 43 | for filetype in filetypes: 44 | print filetype 45 | content = filetype.content.lower() 46 | instrum = filetype.instrum.lower() 47 | fitsdir = os.path.abspath(os.path.join('data', content, 'fits')) 48 | 49 | fitsfiles = sorted(glob.glob(os.path.join(fitsdir, '*.fits.gz'))) 50 | if not fitsfiles: 51 | continue 52 | 53 | dat = Ska.Table.read_fits_table(fitsfiles[0]) 54 | for colname in dat.dtype.names: 55 | make_h5_col_file(dat, content, colname) 56 | 57 | h5dir = os.path.join('data', content, 'msid') 58 | if not os.path.exists(h5dir): 59 | os.makedirs(h5dir) 60 | 61 | for i, f in enumerate(fitsfiles): 62 | print 'Ingesting', i, len(fitsfiles), f 63 | dat = Ska.Table.read_fits_table(f) 64 | 65 | for i_colname, colname in enumerate(dat.dtype.names): 66 | print '.', 67 | append_h5_col(dat, content, colname, i_colname) 68 | print 69 | 70 | -------------------------------------------------------------------------------- /legacy/doc/examples/cmp_obc_gyro.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | import Ska.engarchive.fetch as fetch 3 | from Ska.Matplotlib import plot_cxctime 4 | import Ska.Numpy 5 | 6 | tstart = '2009:313:16:00:00' 7 | tstop = '2009:313:17:00:00' 8 | 9 | # Get OBC rates and gyro counts 10 | obc = fetch.MSIDset(tstart, tstop, ['aorate1', 'aorate2', 'aorate3'], filter_bad=True) 11 | gyr = fetch.MSIDset(tstart, tstop, ['aogyrct1', 'aogyrct2', 'aogyrct3', 'aogyrct4'], filter_bad=True) 12 | 13 | # Transform delta gyro counts (4 channels) to a body rate (3 axes) 14 | cts2rate = array([[-0.5 , 0.5 , 0.5 , -0.5 ], 15 | [-0.25623091, 0.60975037, -0.25623091, 0.60975037], 16 | [-0.55615682, -0.05620959, -0.55615682, -0.05620959]]) 17 | 18 | # Calculate raw spacecraft rate directly from gyro data 19 | cts = np.array([gyr['aogyrct1'].vals, 20 | gyr['aogyrct2'].vals, 21 | gyr['aogyrct3'].vals, 22 | gyr['aogyrct4'].vals]) 23 | raw_times = (gyr['aogyrct1'].times[1:] + gyr['aogyrct1'].times[:-1]) / 2 24 | delta_times = gyr['aogyrct1'].times[1:] - gyr['aogyrct1'].times[:-1] 25 | delta_cts = cts[:, 1:] - cts[:, :-1] 26 | raw_rates = np.dot(cts2rate, delta_cts) * 0.02 / delta_times 27 | 28 | # Plot the OBC rates 29 | figure(1, figsize=(8,6)) 30 | clf() 31 | for frame, msid, label in ((1, 'aorate1', 'roll'), 32 | (2, 'aorate2', 'pitch'), 33 | (3, 'aorate3', 'yaw')): 34 | subplot(3, 1, frame) 35 | obc_rates = obc[msid].vals * 206254. 36 | plot_cxctime(obc[msid].times, obc_rates, '-') 37 | plot_cxctime(obc[msid].times, Ska.Numpy.smooth(obc_rates, window_len=20), '-r') 38 | ylim(average(obc_rates) + array([-1.5, 1.5])) 39 | title(label.capitalize() + ' rate (arcsec/sec)') 40 | 41 | subplots_adjust(bottom=0.12, top=0.90) 42 | # savefig('obc_rates_' + dur + '.png') 43 | 44 | # Plot the S/C rates from raw gyro data 45 | figure(2, figsize=(8,6)) 46 | clf() 47 | for axis, label in ((0, 'roll'), 48 | (1, 'pitch'), 49 | (2, 'yaw')): 50 | subplot(3, 1, 1+axis) 51 | raw_rate = raw_rates[axis, :] 52 | plot_cxctime(raw_times, raw_rate, '-') 53 | plot_cxctime(raw_times, Ska.Numpy.smooth(raw_rate, window_len=20), '-r') 54 | ylim(np.mean(raw_rate) + np.array([-0.4, 0.4])) 55 | title(label.capitalize() + ' S/C rate (arcsec/sec)') 56 | 57 | subplots_adjust(bottom=0.12, top=0.90) 58 | # savefig('gyro_sc_rates_' + dur + '.png') 59 | 60 | -------------------------------------------------------------------------------- /legacy/make_test_eng_archive.csh: -------------------------------------------------------------------------------- 1 | # From within the hg eng_archive repo 2 | # source make_test_eng_archive.csh 3 | 4 | setenv ENG_ARCHIVE $PWD/test/eng_archive 5 | mkdir -p test/eng_archive 6 | rm -rf test/eng_archive/* 7 | 8 | rm -f test/make_eng_archive.log 9 | touch test/make_eng_archive.log 10 | 11 | echo "Making regr data..." 12 | ./make_regr_data.py --start 2010:260 --stop 2010:270 --data-root test/eng_archive >>& test/make_eng_archive.log 13 | 14 | echo "Tarring..." 15 | cd test 16 | tar zcf eng_archive.tar.gz eng_archive 17 | cd .. 18 | 19 | echo "Updating archive..." 20 | ./update_archive.py --date-now 2010:271 --data-root test/eng_archive >>& test/make_eng_archive.log 21 | ./update_archive.py --date-now 2010:272 --data-root test/eng_archive >>& test/make_eng_archive.log 22 | ./update_archive.py --date-now 2010:273 --data-root test/eng_archive >>& test/make_eng_archive.log 23 | ./update_archive.py --date-now 2010:274 --data-root test/eng_archive >>& test/make_eng_archive.log 24 | ./update_archive.py --date-now 2010:275 --data-root test/eng_archive >>& test/make_eng_archive.log 25 | ./update_archive.py --date-now 2010:276 --data-root test/eng_archive >>& test/make_eng_archive.log 26 | ./update_archive.py --date-now 2010:277 --data-root test/eng_archive >>& test/make_eng_archive.log 27 | ./update_archive.py --date-now 2010:278 --data-root test/eng_archive >>& test/make_eng_archive.log 28 | ./update_archive.py --date-now 2010:279 --data-root test/eng_archive >>& test/make_eng_archive.log 29 | ./update_archive.py --date-now 2010:280 --data-root test/eng_archive >>& test/make_eng_archive.log 30 | ./update_archive.py --date-now 2010:281 --data-root test/eng_archive >>& test/make_eng_archive.log 31 | ./update_archive.py --date-now 2010:282 --data-root test/eng_archive >>& test/make_eng_archive.log 32 | ./update_archive.py --date-now 2010:283 --data-root test/eng_archive >>& test/make_eng_archive.log 33 | ./update_archive.py --date-now 2010:284 --data-root test/eng_archive >>& test/make_eng_archive.log 34 | ./update_archive.py --date-now 2010:285 --data-root test/eng_archive >>& test/make_eng_archive.log 35 | ./update_archive.py --date-now 2010:286 --data-root test/eng_archive >>& test/make_eng_archive.log 36 | ./update_archive.py --date-now 2010:287 --data-root test/eng_archive >>& test/make_eng_archive.log 37 | ./update_archive.py --date-now 2010:288 --data-root test/eng_archive >>& test/make_eng_archive.log 38 | ./update_archive.py --date-now 2010:289 --data-root test/eng_archive >>& test/make_eng_archive.log 39 | ./update_archive.py --date-now 2010:290 --data-root test/eng_archive >>& test/make_eng_archive.log 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /jeta/archive/file_defs.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | """ 3 | Directory and file location definitions for telemetry archive applications. 4 | This is to be used to define corresponding ContextDict objects. 5 | 6 | 'ft' is expected to be a ContexDict represent an archive file as well with 7 | content (e.g. ACIS2ENG), msid (1PDEAAT), year, doy and basename (archive file 8 | basename) defined. 9 | 10 | Msid files are the hdf5 files containing the entire mission telemetry for one MSID. 11 | Arch files are the CXC archive files containing a short interval of telemetry for 12 | all MSIDs in the same content-type group (e.g. ACIS2ENG). 13 | """ 14 | import os 15 | 16 | SKA = os.environ.get('SKA') or '/proj/sot/ska' 17 | 18 | 19 | # Root directories for MSID files. msid_root is prime, others are backups. 20 | # NOTE: msid_root(s) used ONLY in one-off or legacy code, not in update_archive.py or 21 | # transfer_stage.py 22 | msid_root = os.path.join(SKA, 'data', 'jeta') 23 | msid_roots = [msid_root] 24 | 25 | msid_files = { 26 | 'filetypes': 'filetypes.dat', 27 | 'msid_bad_times': 'msid_bad_times.dat', 28 | 'contentdir': 'data/{{ft.content}}/', 29 | 'headers': 'data/{{ft.content}}/headers.pickle', 30 | 'archfiles': 'archive.meta.info.db3', 31 | 'colnames': 'colnames.pickle', 32 | 'colnames_all': 'colnames_all.pickle', 33 | 'msid': 'data/{{ft.content}}/{{ft.msid | upper}}', 34 | 'data': 'data/{{ft.content}}/{{ft.msid | upper}}.h5', 35 | 'statsdir': 'data/{{ft.content}}/stats/{{ft.interval}}/', 36 | 'stats': 'data/tlm/stats/{{ft.interval}}/{{ft.msid | upper }}.h5', 37 | 'processed_files_directory': 'processed_files', 38 | 'mnemonic_index': 'data/{{ft.content}}/{{ft.msid | upper}}/index.h5', 39 | 'mnemonic_value': 'data/{{ft.content}}/{{ft.msid | upper}}/values.h5', 40 | 'mnemonic_times': 'data/{{ft.content}}/{{ft.msid | upper}}/times.h5' 41 | } 42 | 43 | 44 | # NOTE: arch_root used ONLY in one-off or legacy code, not in update_archive.py or 45 | # transfer_stage.py 46 | arch_root = '/data/cosmos2/eng_archive' 47 | arch_files = {'stagedir': 'stage/{{ft.content}}/', 48 | 'rootdir': '', 49 | 'archrootdir': 'data/{{ft.content}}/arch/', 50 | 'archdir': 'data/{{ft.content}}/arch/{{ft.year}}/{{ft.doy}}/', 51 | 'archfile': 'data/{{ft.content}}/arch/{{ft.year}}/{{ft.doy}}/{{ft.basename}}', 52 | } 53 | 54 | # Used when originally creating database. 55 | orig_arch_root = '/data/cosmos2/tlm' 56 | orig_arch_files = {'contentdir': '{{ft.content}}/'} 57 | -------------------------------------------------------------------------------- /legacy/doc/numpy_tutorial.rst: -------------------------------------------------------------------------------- 1 | NumPy has an excellent `basic tutorial 2 | `_ available. Here I just copy 3 | the Quick Tour from that tutorial but you should read the rest as well. In 4 | these examples the python prompt is shown as ">>>" in order to distinguish the 5 | input from the outputs. 6 | 7 | Arrays can be created in different ways:: 8 | 9 | >>> a = array( [ 10, 20, 30, 40 ] ) # create an array out of a list 10 | >>> a 11 | array([10, 20, 30, 40]) 12 | >>> b = arange( 4 ) # create an array of 4 integers, from 0 to 3 13 | >>> b 14 | array([0, 1, 2, 3]) 15 | >>> c = linspace(-pi,pi,3) # create an array of 3 evenly spaced samples from -pi to pi 16 | >>> c 17 | array([-3.14159265, 0. , 3.14159265]) 18 | 19 | New arrays can be obtained by operating with existing arrays:: 20 | 21 | >>> d = a+b**2 # elementwise operations 22 | >>> d 23 | array([10, 21, 34, 49]) 24 | 25 | Arrays may have more than one dimension:: 26 | 27 | >>> x = ones( (3,4) ) 28 | >>> x 29 | array([[1., 1., 1., 1.], 30 | [1., 1., 1., 1.], 31 | [1., 1., 1., 1.]]) 32 | >>> x.shape # a tuple with the dimensions 33 | (3, 4) 34 | 35 | and you can change the dimensions of existing arrays:: 36 | 37 | >>> y = arange(12) 38 | >>> y 39 | array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) 40 | >>> y.shape = 3,4 # does not modify the total number of elements 41 | >>> y 42 | array([[ 0, 1, 2, 3], 43 | [ 4, 5, 6, 7], 44 | [ 8, 9, 10, 11]]) 45 | 46 | It is possible to operate with arrays of different dimensions as long as they fit well (broadcasting):: 47 | 48 | >>> 3*a # multiply each element of a by 3 49 | array([ 30, 60, 90, 120]) 50 | >>> a+y # sum a to each row of y 51 | array([[10, 21, 32, 43], 52 | [14, 25, 36, 47], 53 | [18, 29, 40, 51]]) 54 | 55 | Similar to Python lists, arrays can be indexed, sliced and iterated over:: 56 | 57 | >>> a[2:4] = -7,-3 # modify last two elements of a 58 | >>> for i in a: # iterate over a 59 | ... print i 60 | ... 61 | 10 62 | 20 63 | -7 64 | -3 65 | 66 | When indexing more than one dimension, indices are separated by commas:: 67 | 68 | >>> x[1,2] = 20 69 | >>> x[1,:] # x's second row 70 | array([ 1, 1, 20, 1]) 71 | >>> x[0] = a # change first row of x 72 | >>> x 73 | array([[10, 20, -7, -3], 74 | [ 1, 1, 20, 1], 75 | [ 1, 1, 1, 1]]) 76 | -------------------------------------------------------------------------------- /legacy/update_h5_archive2.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | import time 3 | import tables 4 | import numpy as np 5 | import Ska.Table 6 | import re 7 | import os 8 | import sys 9 | import glob 10 | 11 | def make_h5_col_file(dat, content, colname): 12 | """Make a new h5 table to hold column from ``dat``.""" 13 | filename = os.path.join('data', content, 'msid', colname + '.h6') 14 | if os.path.exists(filename): 15 | os.unlink(filename) 16 | filedir = os.path.dirname(filename) 17 | if not os.path.exists(filedir): 18 | os.makedirs(filedir) 19 | 20 | filters = tables.Filters(complevel=5, complib='zlib') 21 | h5 = tables.openFile(filename, mode='w', filters=filters) 22 | 23 | col = dat[colname] 24 | h5shape = (0,) + col.shape[1:] 25 | h5type = tables.Atom.from_dtype(col.dtype) 26 | h5.createEArray(h5.root, 'data', h5type, h5shape, title=colname, 27 | expectedrows=86400*365*10) 28 | h5.createEArray(h5.root, 'quality', tables.BoolAtom(), (0,), title='Quality', 29 | expectedrows=86400*365*10) 30 | print 'Made', colname 31 | h5.close() 32 | 33 | def append_h5_col(dats, content, colname, i_colname): 34 | filename = os.path.join('data', content, 'msid', colname + '.h6') 35 | h5 = tables.openFile(filename, mode='a') 36 | h5.root.data.append(np.hstack([x[colname] for x in dats])) 37 | h5.root.quality.append(np.hstack([x['QUALITY'][:,i_colname] for x in dats])) 38 | h5.close() 39 | 40 | filetypes = Ska.Table.read_ascii_table('filetypes.dat') 41 | filetypes = filetypes[ filetypes.pipe == 'ENG0' ] 42 | 43 | for filetype in filetypes: 44 | if filetype.content != 'PCAD3ENG': 45 | continue 46 | print filetype 47 | content = filetype.content.lower() 48 | instrum = filetype.instrum.lower() 49 | fitsdir = os.path.abspath(os.path.join('data', content, 'fits')) 50 | 51 | fitsfiles = sorted(glob.glob(os.path.join(fitsdir, '*.fits.gz'))) 52 | if not fitsfiles: 53 | continue 54 | 55 | dat = Ska.Table.read_fits_table(fitsfiles[0]) 56 | for colname in dat.dtype.names: 57 | make_h5_col_file(dat, content, colname) 58 | 59 | h5dir = os.path.join('data', content, 'msid') 60 | if not os.path.exists(h5dir): 61 | os.makedirs(h5dir) 62 | 63 | dats = [] 64 | for i, f in enumerate(fitsfiles): 65 | print 'Reading', i, len(fitsfiles), f 66 | dats.append(Ska.Table.read_fits_table(f)) 67 | 68 | for i_colname, colname in enumerate(dat.dtype.names): 69 | if colname != 'QUALITY': 70 | print '.', 71 | append_h5_col(dats, content, colname, i_colname) 72 | print 73 | 74 | -------------------------------------------------------------------------------- /legacy/update_h5_archive3.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | import pyfits 3 | import time 4 | import tables 5 | import numpy as np 6 | import Ska.Table 7 | import re 8 | import os 9 | import sys 10 | import glob 11 | 12 | def make_h5_col_file(dat, content, colname): 13 | """Make a new h5 table to hold column from ``dat``.""" 14 | filename = os.path.join('data', content, 'msid', colname + '.h7') 15 | if os.path.exists(filename): 16 | os.unlink(filename) 17 | filedir = os.path.dirname(filename) 18 | if not os.path.exists(filedir): 19 | os.makedirs(filedir) 20 | 21 | filters = tables.Filters(complevel=5, complib='zlib') 22 | h5 = tables.openFile(filename, mode='w', filters=filters) 23 | 24 | col = dat[colname] 25 | h5shape = (0,) + col.shape[1:] 26 | h5type = tables.Atom.from_dtype(col.dtype) 27 | h5.createEArray(h5.root, 'data', h5type, h5shape, title=colname, 28 | expectedrows=86400*365*10) 29 | h5.createEArray(h5.root, 'quality', tables.BoolAtom(), (0,), title='Quality', 30 | expectedrows=86400*365*10) 31 | print 'Made', colname 32 | h5.close() 33 | 34 | def append_h5_col(hdus, content, colname, i_colname): 35 | filename = os.path.join('data', content, 'msid', colname + '.h7') 36 | h5 = tables.openFile(filename, mode='a') 37 | h5.root.data.append(np.hstack([x[1].field(colname) for x in hdus])) 38 | h5.root.quality.append(np.hstack([x[1].field('QUALITY')[:,i_colname] for x in hdus])) 39 | h5.close() 40 | 41 | filetypes = Ska.Table.read_ascii_table('filetypes.dat') 42 | filetypes = filetypes[ filetypes.pipe == 'ENG0' ] 43 | 44 | for filetype in filetypes: 45 | if filetype.content != 'PCAD3ENG': 46 | continue 47 | print filetype 48 | content = filetype.content.lower() 49 | instrum = filetype.instrum.lower() 50 | fitsdir = os.path.abspath(os.path.join('data', content, 'fits')) 51 | 52 | fitsfiles = sorted(glob.glob(os.path.join(fitsdir, '*.fits.gz'))) 53 | if not fitsfiles: 54 | continue 55 | 56 | dat = Ska.Table.read_fits_table(fitsfiles[0]) 57 | for colname in dat.dtype.names: 58 | make_h5_col_file(dat, content, colname) 59 | 60 | h5dir = os.path.join('data', content, 'msid') 61 | if not os.path.exists(h5dir): 62 | os.makedirs(h5dir) 63 | 64 | hdus = [] 65 | for i, f in enumerate(fitsfiles): 66 | print 'Reading', i, len(fitsfiles), f 67 | hdus.append(pyfits.open(f)) 68 | 69 | for i_colname, colname in enumerate(dat.dtype.names): 70 | if colname != 'QUALITY': 71 | print '.', 72 | append_h5_col(hdus, content, colname, i_colname) 73 | print 74 | 75 | -------------------------------------------------------------------------------- /legacy/alias_rename_files.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 3 | """ 4 | Rename MSID, 5min and daily h5 files based on a set of MSID aliases. 5 | Also updates colnames.pickle and colnames_all.pickle. 6 | """ 7 | 8 | import os 9 | import argparse 10 | import pickle 11 | 12 | import pyyaks.context 13 | 14 | import Ska.engarchive.fetch as fetch 15 | from Ska.engarchive.converters import ALIASES 16 | import Ska.engarchive.file_defs as file_defs 17 | 18 | 19 | def get_options(args=None): 20 | parser = argparse.ArgumentParser() 21 | parser.add_argument("--dry-run", 22 | action="store_true", 23 | help="Dry run (no actual file or database updatees)") 24 | parser.add_argument("--data-root", 25 | default=".", 26 | help="Engineering archive root directory for MSID and arch files") 27 | parser.add_argument("--content", 28 | default="sim_mrg", 29 | help="Content type to process (default='sim_mrg')") 30 | return parser.parse_args(args) 31 | 32 | opt = get_options() 33 | 34 | ft = fetch.ft 35 | ft['content'] = opt.content.lower() 36 | msid_files = pyyaks.context.ContextDict('msid_files', 37 | basedir=opt.data_root) 38 | msid_files.update(file_defs.msid_files) 39 | 40 | colnames = [x for x in pickle.load(open(msid_files['colnames'].abs))] 41 | 42 | aliases = ALIASES[opt.content] 43 | 44 | for colname in (x for x in colnames if x in aliases): 45 | for filetype, interval in (('msid', ''), 46 | ('stats', '5min'), 47 | ('stats', 'daily')): 48 | ft['interval'] = interval 49 | ft['msid'] = colname 50 | oldname = msid_files[filetype].rel 51 | ft['msid'] = aliases[colname] 52 | newname = msid_files[filetype].rel 53 | if os.path.exists(oldname): 54 | print 'mv {} {}'.format(oldname, newname) 55 | if not opt.dry_run: 56 | os.rename(oldname, newname) 57 | else: 58 | print 'No file for {} ({} does not exist)'.format(colname, oldname) 59 | 60 | colnames = pickle.load(open(msid_files['colnames'].abs)) 61 | colnames = set(aliases.get(x, x) for x in colnames) 62 | colnames_all = pickle.load(open(msid_files['colnames_all'].abs)) 63 | colnames_all = set(aliases.get(x, x) for x in colnames_all) 64 | 65 | print 'Updating {} and {}'.format(msid_files['colnames'].rel, 66 | msid_files['colnames_all'].rel) 67 | if not opt.dry_run: 68 | with open(msid_files['colnames'].rel, 'w') as f: 69 | pickle.dump(colnames, f) 70 | with open(msid_files['colnames_all'].rel, 'w') as f: 71 | pickle.dump(colnames_all, f) 72 | -------------------------------------------------------------------------------- /legacy/task_schedule_occ_gap.cfg: -------------------------------------------------------------------------------- 1 | # Configuration file for task_schedule.pl to run astromon jobs 2 | 3 | subject Engineering telemetry archive 4 | timeout 80000 # Default tool timeout 5 | heartbeat_timeout 10 # Maximum age of heartbeat file (seconds) 6 | iterations 1 # Run once then shut down task_schedule 7 | print_error 1 # Print full log of errors 8 | disable_alerts 0 # Don't disable alerts since this jobs runs just once/day 9 | loud 0 # Run loudly or quietly (production mode) 10 | 11 | # Data files and directories. The *_dir vars can have $ENV{} vars which 12 | # get interpolated. (Note lack of task name after TST_DATA because this is just for test). 13 | 14 | data_dir $ENV{SKA_DATA}/eng_archive # Data file directory 15 | log_dir $ENV{SKA_DATA}/eng_archive/logs # Log file directory 16 | bin_dir $ENV{SKA_SHARE}/eng_archive # Bin dir (optional, see task def'n) 17 | master_log eng_archive.log # Composite master log (created in log_dir) 18 | 19 | # Email addresses that receive an alert if there was a severe error in 20 | # running jobs (i.e. couldn't start jobs or couldn't open log file). 21 | # Processing errors *within* the jobs are caught with watch_cron_logs 22 | 23 | alert SOT 24 | 25 | # Define task parameters 26 | # cron: Job repetition specification ala crontab 27 | # exec: Name of executable. Can have $ENV{} vars which get interpolated. 28 | # If bin_dir is defined then bin_dir is prepended to non-absolute exec names. 29 | # log: Name of log. Can have $ENV{} vars which get interpolated. 30 | # If log is set to '' then no log file will be created 31 | # If log is not defined it is set to .log. 32 | # If log_dir is defined then log_dir is prepended to non-absolute log names. 33 | # timeout: Maximum time (seconds) for job before timing out 34 | 35 | # This has multiple jobs which get run in specified order 36 | # Note the syntax 'exec : cmd', which means that the given command is 37 | # executed only once for each of times the task is executed. In the 38 | # example below, the commands are done once each 1, 2, and 4 minutes, respectively. 39 | 40 | 41 | cron * * * * * 42 | check_cron * * * * * 43 | exec transfer_stage.py --occ --data-root /proj/sot/ska/data/eng_archive 44 | exec update_archive.py --occ --data-root /proj/sot/ska/data/eng_archive --max-gap 1000000 45 | 46 | 47 | # File Expression 48 | # ---------- --------------------------- 49 | eng_archive.log error 50 | eng_archive.log warning 51 | eng_archive.log fatal 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /legacy/task_schedule_gap.cfg: -------------------------------------------------------------------------------- 1 | # Configuration file for task_schedule.pl to run astromon jobs 2 | 3 | subject Engineering telemetry archive 4 | timeout 80000 # Default tool timeout 5 | heartbeat_timeout 10 # Maximum age of heartbeat file (seconds) 6 | iterations 1 # Run once then shut down task_schedule 7 | print_error 1 # Print full log of errors 8 | disable_alerts 0 # Don't disable alerts since this jobs runs just once/day 9 | loud 0 # Run loudly or quietly (production mode) 10 | 11 | # Data files and directories. The *_dir vars can have $ENV{} vars which 12 | # get interpolated. (Note lack of task name after TST_DATA because this is just for test). 13 | 14 | data_dir $ENV{SKA_DATA}/eng_archive # Data file directory 15 | log_dir $ENV{SKA_DATA}/eng_archive/logs # Log file directory 16 | bin_dir $ENV{SKA_SHARE}/eng_archive # Bin dir (optional, see task def'n) 17 | master_log eng_archive.log # Composite master log (created in log_dir) 18 | 19 | # Email addresses that receive an alert if there was a severe error in 20 | # running jobs (i.e. couldn't start jobs or couldn't open log file). 21 | # Processing errors *within* the jobs are caught with watch_cron_logs 22 | 23 | alert aca@head.cfa.harvard.edu 24 | 25 | # Define task parameters 26 | # cron: Job repetition specification ala crontab 27 | # exec: Name of executable. Can have $ENV{} vars which get interpolated. 28 | # If bin_dir is defined then bin_dir is prepended to non-absolute exec names. 29 | # log: Name of log. Can have $ENV{} vars which get interpolated. 30 | # If log is set to '' then no log file will be created 31 | # If log is not defined it is set to .log. 32 | # If log_dir is defined then log_dir is prepended to non-absolute log names. 33 | # timeout: Maximum time (seconds) for job before timing out 34 | 35 | # This has multiple jobs which get run in specified order 36 | # Note the syntax 'exec : cmd', which means that the given command is 37 | # executed only once for each of times the task is executed. In the 38 | # example below, the commands are done once each 1, 2, and 4 minutes, respectively. 39 | 40 | 41 | cron * * * * * 42 | check_cron * * * * * 43 | exec update_archive.py --data-root /proj/sot/ska/data/eng_archive --max-gap 1000000 44 | exec transfer_stage.py --data-root /proj/sot/ska/data/eng_archive 45 | 46 | 47 | # File Expression 48 | # ---------- --------------------------- 49 | eng_archive.log error 50 | eng_archive.log warning 51 | eng_archive.log fatal 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /legacy/task_schedule.cfg: -------------------------------------------------------------------------------- 1 | # Configuration file for task_schedule.pl to run astromon jobs 2 | 3 | subject Engineering telemetry archive 4 | timeout 80000 # Default tool timeout 5 | heartbeat_timeout 10 # Maximum age of heartbeat file (seconds) 6 | iterations 1 # Run once then shut down task_schedule 7 | print_error 1 # Print full log of errors 8 | disable_alerts 0 # Don't disable alerts since this jobs runs just once/day 9 | loud 0 # Run loudly or quietly (production mode) 10 | 11 | # Data files and directories. The *_dir vars can have $ENV{} vars which 12 | # get interpolated. (Note lack of task name after TST_DATA because this is just for test). 13 | 14 | data_dir $ENV{SKA_DATA}/eng_archive # Data file directory 15 | log_dir $ENV{SKA_DATA}/eng_archive/logs # Log file directory 16 | bin_dir $ENV{SKA_SHARE}/eng_archive # Bin dir (optional, see task def'n) 17 | master_log eng_archive.log # Composite master log (created in log_dir) 18 | 19 | # Email addresses that receive an alert if there was a severe error in 20 | # running jobs (i.e. couldn't start jobs or couldn't open log file). 21 | # Processing errors *within* the jobs are caught with watch_cron_logs 22 | 23 | alert aca@head.cfa.harvard.edu 24 | 25 | # Define task parameters 26 | # cron: Job repetition specification ala crontab 27 | # exec: Name of executable. Can have $ENV{} vars which get interpolated. 28 | # If bin_dir is defined then bin_dir is prepended to non-absolute exec names. 29 | # log: Name of log. Can have $ENV{} vars which get interpolated. 30 | # If log is set to '' then no log file will be created 31 | # If log is not defined it is set to .log. 32 | # If log_dir is defined then log_dir is prepended to non-absolute log names. 33 | # timeout: Maximum time (seconds) for job before timing out 34 | 35 | # This has multiple jobs which get run in specified order 36 | # Note the syntax 'exec : cmd', which means that the given command is 37 | # executed only once for each of times the task is executed. In the 38 | # example below, the commands are done once each 1, 2, and 4 minutes, respectively. 39 | 40 | 41 | cron * * * * * 42 | check_cron * * * * * 43 | exec update_archive.py --data-root /proj/sot/ska/data/eng_archive 44 | exec transfer_stage.py --data-root /proj/sot/ska/data/eng_archive 45 | exec check_integrity.py --data-root /proj/sot/ska/data/eng_archive 46 | 47 | 48 | # File Expression 49 | # ---------- --------------------------- 50 | eng_archive.log error 51 | eng_archive.log warning 52 | eng_archive.log fatal 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /legacy/task_schedule_occ.cfg: -------------------------------------------------------------------------------- 1 | # Configuration file for task_schedule.pl to run astromon jobs 2 | 3 | subject Engineering telemetry archive 4 | timeout 80000 # Default tool timeout 5 | heartbeat_timeout 10 # Maximum age of heartbeat file (seconds) 6 | iterations 1 # Run once then shut down task_schedule 7 | print_error 1 # Print full log of errors 8 | disable_alerts 0 # Don't disable alerts since this jobs runs just once/day 9 | loud 0 # Run loudly or quietly (production mode) 10 | 11 | # Data files and directories. The *_dir vars can have $ENV{} vars which 12 | # get interpolated. (Note lack of task name after TST_DATA because this is just for test). 13 | 14 | data_dir $ENV{SKA_DATA}/eng_archive # Data file directory 15 | log_dir $ENV{SKA_DATA}/eng_archive/logs # Log file directory 16 | bin_dir $ENV{SKA_SHARE}/eng_archive # Bin dir (optional, see task def'n) 17 | master_log eng_archive.log # Composite master log (created in log_dir) 18 | 19 | # Email addresses that receive an alert if there was a severe error in 20 | # running jobs (i.e. couldn't start jobs or couldn't open log file). 21 | # Processing errors *within* the jobs are caught with watch_cron_logs 22 | 23 | alert SOT,mdahmer,aarvai 24 | 25 | # Define task parameters 26 | # cron: Job repetition specification ala crontab 27 | # exec: Name of executable. Can have $ENV{} vars which get interpolated. 28 | # If bin_dir is defined then bin_dir is prepended to non-absolute exec names. 29 | # log: Name of log. Can have $ENV{} vars which get interpolated. 30 | # If log is set to '' then no log file will be created 31 | # If log is not defined it is set to .log. 32 | # If log_dir is defined then log_dir is prepended to non-absolute log names. 33 | # timeout: Maximum time (seconds) for job before timing out 34 | 35 | # This has multiple jobs which get run in specified order 36 | # Note the syntax 'exec : cmd', which means that the given command is 37 | # executed only once for each of times the task is executed. In the 38 | # example below, the commands are done once each 1, 2, and 4 minutes, respectively. 39 | 40 | 41 | cron * * * * * 42 | check_cron * * * * * 43 | exec transfer_stage.py --occ --data-root /proj/sot/ska/data/eng_archive 44 | exec update_archive.py --occ --data-root /proj/sot/ska/data/eng_archive 45 | exec check_integrity.py --data-root /proj/sot/ska/data/eng_archive 46 | 47 | 48 | # File Expression 49 | # ---------- --------------------------- 50 | eng_archive.log error 51 | eng_archive.log warning 52 | eng_archive.log fatal 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /legacy/dev_utils/repro_interval.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script allows controlled truncation and reprocessing of the live Ska engineering 4 | # archive data. The loop structure is used so that only one content type is truncated and 5 | # reprocessed at once, thus minimizing impact to users and reducing the footprint in case 6 | # something goes wrong. 7 | # 8 | # To use this, most likely you will first want to do a test where CONTENTS is set to just 9 | # one type, for instance misc8eng is a good one because it is very small. 10 | # 11 | # Usage (from the dev_utils directory within a persistent NX window): 12 | # 13 | # $ ./repro_interval.sh 14 | # 15 | # This will slowly go through each content type, truncate and re-ingest. Open another 16 | # window and `tail -f trunc.log`. 17 | 18 | # The following have been removed from the CONTENTS list below because quite often 19 | # they don't need repro in the same way as the others. But put them back in if 20 | # it makes sense. 21 | # 22 | # lunarephem0 23 | # solarephem0 24 | # orbitephem1 25 | # lunarephem1 26 | # solarephem1 27 | # angleephem 28 | 29 | CONTENTS=" 30 | acisdeahk 31 | acis2eng 32 | acis3eng 33 | acis4eng 34 | ccdm1eng 35 | ccdm2eng 36 | ccdm3eng 37 | ccdm4eng 38 | ccdm5eng 39 | ccdm7eng 40 | ccdm8eng 41 | ccdm10eng 42 | ccdm11eng 43 | ccdm12eng 44 | ccdm13eng 45 | ccdm14eng 46 | ccdm15eng 47 | ephin1eng 48 | ephin2eng 49 | eps1eng 50 | eps2eng 51 | eps3eng 52 | eps4eng 53 | eps5eng 54 | eps6eng 55 | eps7eng 56 | eps9eng 57 | eps10eng 58 | hrc0hk 59 | hrc0ss 60 | hrc2eng 61 | hrc4eng 62 | hrc5eng 63 | misc1eng 64 | misc2eng 65 | misc3eng 66 | misc4eng 67 | misc5eng 68 | misc6eng 69 | misc7eng 70 | misc8eng 71 | obc3eng 72 | obc4eng 73 | obc5eng 74 | pcad3eng 75 | pcad4eng 76 | pcad5eng 77 | pcad6eng 78 | pcad7eng 79 | pcad8eng 80 | pcad10eng 81 | pcad11eng 82 | pcad12eng 83 | pcad13eng 84 | pcad14eng 85 | pcad15eng 86 | prop1eng 87 | prop2eng 88 | sim1eng 89 | sim2eng 90 | sim3eng 91 | sim21eng 92 | sms1eng 93 | sms2eng 94 | tel1eng 95 | tel2eng 96 | tel3eng 97 | thm1eng 98 | thm2eng 99 | thm3eng 100 | simdiag 101 | simcoor 102 | sim_mrg 103 | ephhk 104 | cpe1eng 105 | dp_thermal1 106 | dp_thermal128 107 | dp_acispow128 108 | dp_pcad1 109 | dp_pcad4 110 | dp_pcad16 111 | dp_pcad32 112 | dp_orbit1280 113 | dp_eps8 114 | dp_eps16" 115 | 116 | touch trunc.log 117 | 118 | TRUNCATE=2017:018 119 | DATAROOT=/proj/sot/ska/data/eng_archive 120 | # DRY="--dry-run" 121 | DRY="" 122 | 123 | for CONTENT in $CONTENTS 124 | do 125 | echo "" 126 | 127 | CMD="/proj/sot/ska/share/eng_archive/update_archive.py $DRY --data-root=$DATAROOT --truncate=$TRUNCATE --content=$CONTENT" 128 | echo $CMD 129 | $CMD &>> trunc.log 130 | 131 | CMD="/proj/sot/ska/share/eng_archive/update_archive.py $DRY --data-root=$DATAROOT --content=$CONTENT --max-arch-files=1000" 132 | echo $CMD 133 | $CMD &>> trunc.log 134 | 135 | done 136 | -------------------------------------------------------------------------------- /legacy/make_resync_stage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 3 | 4 | """Create stage directory structure resync the OCC eng archive with the 5 | HEAD archive. 6 | """ 7 | 8 | import os 9 | import shutil 10 | import glob 11 | import Ska.Table 12 | from Chandra.Time import DateTime 13 | import Ska.engarchive.fetch as fetch 14 | import Ska.engarchive.file_defs as file_defs 15 | import optparse 16 | import pyyaks.context 17 | 18 | def get_options(): 19 | parser = optparse.OptionParser() 20 | parser.add_option("--dry-run", 21 | action="store_true", 22 | help="Dry run (no actual file or database updatees)") 23 | parser.add_option("--date", 24 | default="2011:200", 25 | help="Starting year:doy for assembling resync archive files") 26 | parser.add_option("--content", 27 | action='append', 28 | help="Content type to process (default = all)") 29 | parser.add_option("--outdir", 30 | default="stage", 31 | help="Output directory") 32 | return parser.parse_args() 33 | 34 | opt, args = get_options() 35 | 36 | msid_files = pyyaks.context.ContextDict('msid_files', file_defs.msid_root) 37 | msid_files.update(file_defs.msid_files) 38 | 39 | arch_files = pyyaks.context.ContextDict('arch_files', basedir=file_defs.arch_root) 40 | arch_files.update(file_defs.arch_files) 41 | 42 | ft = fetch.ft 43 | 44 | date = DateTime(opt.date).date 45 | year0, doy0 = date[0:4], date[5:8] 46 | 47 | # Get the archive content filetypes 48 | filetypes = Ska.Table.read_ascii_table(msid_files['filetypes'].abs) 49 | if opt.content: 50 | contents = [x.upper() for x in opt.content] 51 | filetypes = [x for x in filetypes if x['content'] in contents] 52 | 53 | for filetype in filetypes: 54 | ft['content'] = filetype['content'].lower() 55 | print "Copying {} content".format(ft['content']) 56 | archtime = DateTime("{}:{}".format(year0, doy0)).secs 57 | 58 | outdir = os.path.join(opt.outdir, ft['content'].val) 59 | if not os.path.exists(outdir): 60 | if not opt.dry_run: 61 | os.mkdir(outdir) 62 | 63 | while True: 64 | date = DateTime(archtime).date 65 | ft['year'], ft['doy'] = date[0:4], date[5:8] 66 | print " {}:{}".format(ft['year'], ft['doy']) 67 | archdir = arch_files['archdir'].abs 68 | if os.path.exists(archdir): 69 | for filename in glob.glob(os.path.join(archdir, '*.fits.gz')): 70 | if not os.path.exists(os.path.join(outdir, os.path.basename(filename))): 71 | # print "Copying {} to {}".format(filename, outdir) 72 | if not opt.dry_run: 73 | shutil.copy2(filename, outdir) 74 | 75 | archtime += 86400 76 | if archtime > DateTime().secs: 77 | break 78 | 79 | -------------------------------------------------------------------------------- /legacy/convert_headers.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | """ 3 | Convert dict of archive files and associated info from headers.pickle to 4 | sqlite3 'archfiles' table. This is a one-off after ingest_h5_archive.py 5 | to correct an original hasty decision about storing file info. 6 | """ 7 | import re, os , sys 8 | import cPickle as pickle 9 | import collections 10 | import sqlite3 11 | 12 | import Ska.Table 13 | import Ska.DBI 14 | from context_def import datadir, ft, files 15 | 16 | from Chandra.Time import DateTime 17 | import asciitable 18 | 19 | filetypes = asciitable.read('Ska/engarchive/filetypes.dat') 20 | if len(sys.argv) == 2: 21 | filetypes = filetypes[ filetypes['content'] == sys.argv[1].upper() ] 22 | 23 | cols_map = dict(TSTART = 'tstart', 24 | TSTOP = 'tstop', 25 | row0 = 'rowstart', 26 | row1 = 'rowstop', 27 | STARTMJF = 'startmjf', 28 | STARTMNF = 'startmnf', 29 | STOPMJF = 'stopmjf', 30 | STOPMNF = 'stopmnf', 31 | CHECKSUM = 'checksum', 32 | TLMVER = 'tlmver', 33 | ASCDSVER = 'ascdsver', 34 | REVISION = 'revision', 35 | DATE = 'date') 36 | 37 | archfiles_def = open('archfiles_def.sql').read() 38 | 39 | for filetype in filetypes: 40 | ft['content'] = filetype['content'].lower() 41 | 42 | if not os.path.exists(files['contentdir'].abs) or os.path.exists(files['archfiles'].abs): 43 | print 'Skipping', ft['content'].val, files['contentdir'].abs, files['archfiles'].abs 44 | continue 45 | 46 | print 'Processing', ft['content'].val 47 | 48 | print 'Creating db', files['archfiles'].abs 49 | db = Ska.DBI.DBI(dbi='sqlite', server=files['archfiles'].abs, autocommit=False) 50 | db.execute(archfiles_def) 51 | db.commit() 52 | 53 | print 'Reading', files['headers'].abs 54 | headers = pickle.load(open(files['headers'].abs)) 55 | 56 | 57 | filetime_counts = collections.defaultdict(int) 58 | 59 | n = len(headers) 60 | for i, filename in enumerate(sorted(headers)): 61 | header = headers[filename] 62 | out = dict((outkey, header.get(inkey)) for inkey, outkey in cols_map.items()) 63 | out['filename'] = filename 64 | filetime = int(re.search(r'(\d+)', filename).group(1)) 65 | out['filetime'] = filetime 66 | filedate = DateTime(filetime).date 67 | out['year'], out['doy'] = re.search(r'(\d\d\d\d):(\d\d\d)', filedate).groups() 68 | 69 | if i % 100 == 0: 70 | print i, n, filetime, filename, header['row0'], header['row1'], '\r', 71 | 72 | filetime_counts[filetime] += 1 73 | db.insert(out, 'archfiles') 74 | 75 | db.commit() 76 | 77 | print 78 | print 'Repeats', [(k, v) for k, v in filetime_counts.items() if v > 1] 79 | 80 | -------------------------------------------------------------------------------- /legacy/get_products.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | import tables 3 | import numpy as np 4 | import Ska.Table 5 | import re 6 | import os 7 | import sys 8 | import glob 9 | 10 | def get_h5_colname(colname): 11 | # return colname 12 | return colname if re.match(r'^[a-zA-Z_]', colname) else '_' + colname 13 | 14 | def make_h5_file(dat, filename): 15 | """Make a new h5 table to hold columns from ``dat``.""" 16 | if os.path.exists(filename): 17 | os.unlink(filename) 18 | 19 | colnames = dat.dtype.names 20 | filters = tables.Filters(complevel=5, complib='zlib') 21 | h5 = tables.openFile(filename, mode='w', filters=filters) 22 | 23 | for colname in colnames: 24 | col = dat[colname] 25 | h5shape = (0,) + col.shape[1:] 26 | h5colname = get_h5_colname(colname) 27 | h5type = tables.Atom.from_dtype(col.dtype) 28 | h5.createEArray(h5.root, h5colname, h5type, h5shape, title=colname, 29 | expectedrows=86400*30) 30 | print 'Made', colname 31 | 32 | return h5 33 | 34 | def make_h5_col_file(dat, colname): 35 | """Make a new h5 table to hold column from ``dat``.""" 36 | filename = 'thm1eng/msid/' + colname + '.h5' 37 | if os.path.exists(filename): 38 | os.unlink(filename) 39 | 40 | filters = tables.Filters(complevel=5, complib='zlib') 41 | h5 = tables.openFile(filename, mode='w', filters=filters) 42 | 43 | col = dat[colname] 44 | h5shape = (0,) + col.shape[1:] 45 | h5colname = get_h5_colname(colname) 46 | h5type = tables.Atom.from_dtype(col.dtype) 47 | h5.createEArray(h5.root, h5colname, h5type, h5shape, title=colname, 48 | expectedrows=86400*30) 49 | print 'Made', colname 50 | h5.close() 51 | 52 | def append_h5_col(dat, colname): 53 | filename = 'thm1eng/msid/' + colname + '.h5' 54 | h5 = tables.openFile(filename, mode='a') 55 | h5colname = get_h5_colname(colname) 56 | h5col = h5.root.__getattr__(h5colname) 57 | h5col.append(dat[colname]) 58 | h5.close() 59 | 60 | dat = Ska.Table.read_fits_table('thm_1_eng0.fits.gz') 61 | for colname in dat.dtype.names: 62 | make_h5_col_file(dat, colname) 63 | 64 | for f in sorted(glob.glob('thm1eng/*.fits.gz')): 65 | print 'Reading', f 66 | dat = Ska.Table.read_fits_table(f) 67 | 68 | for colname in dat.dtype.names: 69 | print '.', 70 | append_h5_col(dat, colname) 71 | print 72 | 73 | if 0: 74 | filename = 'test2.h5' 75 | dat = Ska.Table.read_fits_table('pcadf_proto_3_eng0.fits.gz') 76 | h5 = make_h5_file(dat, filename) 77 | 78 | for f in glob.glob('pcad3/pcadf*'): 79 | print 'Reading', f 80 | dat = Ska.Table.read_fits_table(f) 81 | 82 | for colname in dat.dtype.names: 83 | h5colname = get_h5_colname(colname) 84 | h5col = h5.root.__getattr__(h5colname) 85 | h5col.append(dat[colname]) 86 | h5.flush() 87 | 88 | -------------------------------------------------------------------------------- /jeta/archive/derived/eps.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | """ 3 | Derived parameter MSIDs related to EPS subsystem. 4 | 5 | Author: B. Bissell 6 | 7 | Revision History:: 8 | 9 | Jul 2014 Initial Version 10 | """ 11 | 12 | from . import base 13 | 14 | 15 | class DerivedParameterEps(base.DerivedParameter): 16 | content_root = 'eps' 17 | 18 | 19 | #-------------------------------------------- 20 | class DP_BATT1_TAVE(DerivedParameterEps): 21 | """Battery 1 Average Temperature. Derived from average of all three battery temperature sensors. 22 | Telemetry 16x / MF 23 | """ 24 | rootparams = ['TB1T1', 'TB1T2', 'TB1T3'] 25 | time_step = 2.05 26 | 27 | def calc(self, data): 28 | BATT1_TAVE = (data['TB1T1'].vals + data['TB1T2'].vals + 29 | data['TB1T3'].vals) / 3 30 | return BATT1_TAVE 31 | 32 | 33 | #-------------------------------------------- 34 | class DP_BATT2_TAVE(DerivedParameterEps): 35 | """Battery 2 Average Temperature. Derived from average of all three battery temperature sensors. 36 | Telemetry 16x / MF 37 | """ 38 | rootparams = ['TB2T1', 'TB2T2', 'TB2T3'] 39 | time_step = 2.05 40 | 41 | def calc(self, data): 42 | BATT2_TAVE = (data['TB2T1'].vals + data['TB2T2'].vals + 43 | data['TB2T3'].vals) / 3 44 | return BATT2_TAVE 45 | 46 | 47 | #-------------------------------------------- 48 | class DP_BATT3_TAVE(DerivedParameterEps): 49 | """Battery 3 Average Temperature. Derived from average of all three battery temperature sensors. 50 | Telemetry 16x / MF 51 | """ 52 | rootparams = ['TB3T1', 'TB3T2', 'TB3T3'] 53 | time_step = 2.05 54 | 55 | def calc(self, data): 56 | BATT3_TAVE = (data['TB3T1'].vals + data['TB3T2'].vals + 57 | data['TB3T3'].vals) / 3 58 | return BATT3_TAVE 59 | 60 | 61 | #-------------------------------------------- 62 | class DP_EPOWER1(DerivedParameterEps): 63 | """Bus Power = ELBI_LOW * ELBV 64 | Telemetry 8x / MF 65 | """ 66 | rootparams = ['ELBI_LOW', 'ELBV'] 67 | time_step = 4.1 68 | 69 | def calc(self, data): 70 | EPOWER1 = (data['ELBI_LOW'].vals * data['ELBV'].vals) 71 | return EPOWER1 72 | 73 | 74 | #-------------------------------------------- 75 | class DP_MYSAPOW(DerivedParameterEps): 76 | """-Y Solar Array Power = ESAMYI * ELBV 77 | Telemetry 8x / MF 78 | """ 79 | rootparams = ['ESAMYI', 'ELBV'] 80 | time_step = 4.1 81 | 82 | def calc(self, data): 83 | MYSAPOW = (data['ESAMYI'].vals * data['ELBV'].vals) 84 | return MYSAPOW 85 | 86 | 87 | #-------------------------------------------- 88 | class DP_PYSAPOW(DerivedParameterEps): 89 | """+Y Solar Array Power = ESAPYI * ELBV 90 | Telemetry 8x / MF 91 | """ 92 | rootparams = ['ESAPYI', 'ELBV'] 93 | time_step = 4.1 94 | 95 | def calc(self, data): 96 | PYSAPOW = (data['ESAPYI'].vals * data['ELBV'].vals) 97 | return PYSAPOW 98 | -------------------------------------------------------------------------------- /jeta/staging/tests/test_staging.py: -------------------------------------------------------------------------------- 1 | # base modules 2 | import os 3 | import time 4 | from pathlib import Path 5 | 6 | # testing, mocks, and fixtures modules 7 | import unittest 8 | import pytest 9 | from unittest.mock import patch 10 | 11 | # jeta specific modules 12 | from jeta.archive.utils import get_env_variable 13 | 14 | from jeta.staging.manage import ( 15 | get_staged_files_by_date, 16 | remove_activity, 17 | _format_activity_destination, 18 | _create_activity_staging_area 19 | ) 20 | 21 | HOME = str(Path.home()) 22 | STAGING_DIRECTORY = get_env_variable('STAGING_DIRECTORY') 23 | 24 | 25 | class TestStaging(unittest.TestCase): 26 | 27 | def setUp(self) -> None: 28 | return super().setUp() 29 | 30 | def tearDown(self) -> None: 31 | return super().tearDown() 32 | 33 | def test_smoke_test(self): 34 | assert 1 == 1 35 | 36 | def test_can_import_manage(self): 37 | import importlib 38 | result = importlib.find_loader('jeta.staging.manage.py') 39 | 40 | assert result is not None 41 | 42 | def test_get_files_by_date(self): 43 | 44 | result = get_staged_files_by_date(0, time.time()) 45 | 46 | assert len(result) == 1 47 | 48 | @patch('jeta.staging.manage.os.path.exists') 49 | @patch('jeta.staging.manage.os.mkdir') 50 | def test_create_activity_call_mkdir(self, mock_mkdir, mock_exists): 51 | mock_exists.return_value = False 52 | 53 | _activity = 'TEST_ACTIVITY' 54 | expected_path = f"{STAGING_DIRECTORY}{'TEST_ACTIVITY'}" 55 | 56 | _create_activity_staging_area(_activity) 57 | 58 | mock_mkdir.assert_called_once_with(expected_path) 59 | 60 | @patch('jeta.staging.manage.os.path.exists') 61 | def test_create_activity_already_exists_raise_error(self, mock_exists): 62 | 63 | with pytest.raises(IOError): 64 | mock_exists.return_value = True 65 | 66 | _activity = 'TEST_ACTIVITY' 67 | 68 | _create_activity_staging_area(_activity) 69 | 70 | @patch('jeta.staging.manage.os.path.exists') 71 | @patch('jeta.staging.manage.os.remove') 72 | def test_remove_activity(self, mock_remove, mock_exists): 73 | 74 | mock_exists.return_value = True 75 | 76 | _activity = 'TEST_ACTIVITY' 77 | expected_path = f"{STAGING_DIRECTORY}{'TEST_ACTIVITY'}" 78 | 79 | remove_activity(_activity) 80 | 81 | mock_remove.assert_called_once_with(expected_path) 82 | 83 | def test_add_activity(self): 84 | pytest.fail('No test implemented') 85 | 86 | def test_add_ingest_file_to_activity(self): 87 | pytest.fail('No test implemented') 88 | 89 | def test_get_files_for_activity(self): 90 | pytest.fail('No test implemented') 91 | 92 | def restore_activity_to_staging(self): 93 | pytest.fail('No test implemented') 94 | 95 | def test_format_activity_destination(self): 96 | _activity = "TEST_ACTIVITY" 97 | 98 | expected = f'{STAGING_DIRECTORY}{_activity}/' 99 | actual = _format_activity_destination(_activity) 100 | print(actual) 101 | 102 | assert expected == actual 103 | -------------------------------------------------------------------------------- /legacy/make_units_cxc.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | """ 3 | Create the default unit system as found in the CXC telemetry FITS files. 4 | """ 5 | from __future__ import print_function 6 | 7 | import os 8 | import re 9 | import glob 10 | import argparse 11 | import cPickle as pickle 12 | import pyfits 13 | import pyyaks 14 | 15 | from Ska.engarchive.converters import _get_deahk_cols, CXC_TO_MSID 16 | from Ska.engarchive import file_defs 17 | from Ska.engarchive import fetch 18 | 19 | 20 | def get_options(args=None): 21 | parser = argparse.ArgumentParser() 22 | parser.add_argument("--dry-run", 23 | action="store_true", 24 | help="Dry run (no actual file or database updates)") 25 | return parser.parse_args(args) 26 | 27 | opt = get_options() 28 | 29 | ft = fetch.ft 30 | arch_files = pyyaks.context.ContextDict('update_archive.arch_files', 31 | basedir=os.path.join(file_defs.SKA, 'data', 'eng_archive')) 32 | arch_files.update(file_defs.arch_files) 33 | 34 | units = {} 35 | 36 | # CXC content types 37 | contents = set(fetch.content.values()) 38 | 39 | for content in sorted(contents): 40 | if content.startswith('dp_'): 41 | continue 42 | ft['content'] = content 43 | 44 | dir_ = arch_files['archrootdir'].abs 45 | print('Finding units in', dir_) 46 | 47 | # Get the most recent directory 48 | years = sorted([yr for yr in os.listdir(dir_) if re.match(r'\d{4}', yr)]) 49 | dir_ = os.path.join(dir_, years[-1]) 50 | 51 | days = sorted([day for day in os.listdir(dir_) if re.match(r'\d{3}', day)]) 52 | dir_ = os.path.join(dir_, days[-1]) 53 | 54 | files = glob.glob(os.path.join(dir_, '*.fits.gz')) 55 | if not files: 56 | print('No {} fits files in {}'.format(content, dir_)) 57 | continue 58 | 59 | print('Reading', files[0]) 60 | hdus = pyfits.open(os.path.join(dir_, files[0])) 61 | cols = hdus[1].columns 62 | for msid, unit in zip(cols.names, cols.units): 63 | unit = unit.strip() 64 | if unit: 65 | msid = msid.upper() 66 | if content in CXC_TO_MSID: 67 | msid = CXC_TO_MSID[content].get(msid, msid) 68 | if re.match(r'(orbit|lunar|solar|angle)ephem', content): 69 | msid = '{}_{}'.format(content.upper(), msid) 70 | units[msid.upper()] = unit 71 | hdus.close() 72 | 73 | # AFAIK these are the only temperature MSIDs that are actually temperature 74 | # differences and which require special handling on conversion. 75 | relative_temp_msids = ( 76 | 'OHRMGRD3', # RT 500 TO RT 502: CAP GRADIENT MONITOR 77 | 'OHRMGRD6', # RT 503 TO RT 501: CAP GRADIENT MONITOR 78 | 'OOBAGRD3', # RT 505 TO RT 504: PERISCOPE GRADIENT MONITOR 79 | 'OOBAGRD6', # RT 507 TO RT 506: PERISCOPE GRADIENT MONITOR 80 | ) 81 | 82 | for msid in relative_temp_msids: 83 | units[msid] = 'deltaK' 84 | 85 | units['3MRMMXMV'] = 'PWM' 86 | 87 | # Use info about DEA HK telemetry from converters to add units 88 | for col in _get_deahk_cols(): 89 | if 'unit' in col: 90 | units[col['name'].upper()] = col['unit'] 91 | 92 | if not opt.dry_run: 93 | pickle.dump(units, open('units_cxc.pkl', 'wb')) 94 | else: 95 | print(repr(units)) 96 | -------------------------------------------------------------------------------- /legacy/doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | 9 | # Internal variables. 10 | PAPEROPT_a4 = -D latex_paper_size=a4 11 | PAPEROPT_letter = -D latex_paper_size=letter 12 | ALLSPHINXOPTS = -d _build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 13 | 14 | .PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest 15 | 16 | help: 17 | @echo "Please use \`make ' where is one of" 18 | @echo " html to make standalone HTML files" 19 | @echo " dirhtml to make HTML files named index.html in directories" 20 | @echo " pickle to make pickle files" 21 | @echo " json to make JSON files" 22 | @echo " htmlhelp to make HTML files and a HTML help project" 23 | @echo " qthelp to make HTML files and a qthelp project" 24 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 25 | @echo " changes to make an overview of all changed/added/deprecated items" 26 | @echo " linkcheck to check all external links for integrity" 27 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 28 | 29 | clean: 30 | -rm -rf _build/* 31 | 32 | html: 33 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) _build/html 34 | @echo 35 | @echo "Build finished. The HTML pages are in _build/html." 36 | 37 | dirhtml: 38 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) _build/dirhtml 39 | @echo 40 | @echo "Build finished. The HTML pages are in _build/dirhtml." 41 | 42 | pickle: 43 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) _build/pickle 44 | @echo 45 | @echo "Build finished; now you can process the pickle files." 46 | 47 | json: 48 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) _build/json 49 | @echo 50 | @echo "Build finished; now you can process the JSON files." 51 | 52 | htmlhelp: 53 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) _build/htmlhelp 54 | @echo 55 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 56 | ".hhp project file in _build/htmlhelp." 57 | 58 | qthelp: 59 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) _build/qthelp 60 | @echo 61 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 62 | ".qhcp project file in _build/qthelp, like this:" 63 | @echo "# qcollectiongenerator _build/qthelp/Eng archive.qhcp" 64 | @echo "To view the help file:" 65 | @echo "# assistant -collectionFile _build/qthelp/Eng archive.qhc" 66 | 67 | latex: 68 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) _build/latex 69 | @echo 70 | @echo "Build finished; the LaTeX files are in _build/latex." 71 | @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ 72 | "run these through (pdf)latex." 73 | 74 | changes: 75 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) _build/changes 76 | @echo 77 | @echo "The overview file is in _build/changes." 78 | 79 | linkcheck: 80 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) _build/linkcheck 81 | @echo 82 | @echo "Link check complete; look for any errors in the above output " \ 83 | "or in _build/linkcheck/output.txt." 84 | 85 | doctest: 86 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) _build/doctest 87 | @echo "Testing of doctests in the sources finished, look at the " \ 88 | "results in _build/doctest/output.txt." 89 | -------------------------------------------------------------------------------- /legacy/NOTES.OBA-wide: -------------------------------------------------------------------------------- 1 | Update process 2 | ================ 3 | 4 | - Prior to patch uplink, copy obc4eng content directory to temp dir. 5 | - Update code to add an obc4eng converter 6 | 7 | Update 8 | ------ 9 | 10 | cd ~/git/eng_archive 11 | export ENG_ARCHIVE=$PWD 12 | python ./update_archive.py --content=obc4eng 13 | 14 | Replace flight archive on HEAD 15 | ------------------------------- 16 | 17 | cd ~/git/eng_archive 18 | rsync -av data/obc4eng $ska/data/eng_archive/data/obc4eng-post 19 | cd $ska/data/eng_archive/data 20 | mv obc4eng obc4eng-pre-PR361 21 | mv obc4eng-post/obc4eng ./ 22 | rmdir obc4eng-post 23 | mv obc4eng/arch{,-patch} 24 | cd obc4eng 25 | ln -s /data/cosmos2/eng_archive/data/obc4eng/arch 26 | rsync -av arch-patch/ arch/ 27 | 28 | Test 29 | ---- 30 | 31 | ipython --pylab 32 | impska 33 | msid_nums = {'a': '08 09 10 11 12 13 14 15 17 18 19 20 21 22 23 24 25 26 27 28 29'.split(), 34 | 'b': '30 31 33 34 35 36 37 38 39 40 41 44 45 46 49 50 51 52 53 54'.split() 35 | } 36 | for patch in ('a', 'b'): 37 | for msid_num in msid_nums[patch]: 38 | msid = 'OOBTHR' + msid_num 39 | dat = fetch.Msid(msid, '2014:341', stat='5min') 40 | dat.iplot() 41 | a = raw_input(msid + '> ') 42 | 43 | Copy to GRETA (from SOT@chimchim) by rsync-ing from HEAD 44 | -------------------------------------------------------- 45 | 46 | Update Ska.engarchive and install via skare 47 | ------------------------------------------- 48 | 49 | Definition 50 | ============= 51 | 52 | From the PR-361 patch request, here are the 41 MSIDs that are being 53 | patched to use wide mode: 54 | 55 | OOBTHR08 KT.Tel_Therm_Database(OBA, 6) 56 | OOBTHR09 KT.Tel_Therm_Database(OBA, 7) 57 | OOBTHR10 KT.Tel_Therm_Database(OBA, 8) 58 | OOBTHR11 KT.Tel_Therm_Database(OBA, 9) 59 | OOBTHR12 KT.Tel_Therm_Database(OBA, 10) 60 | OOBTHR13 KT.Tel_Therm_Database(OBA, 11) 61 | OOBTHR14 KT.Tel_Therm_Database(OBA, 12) 62 | OOBTHR15 KT.Tel_Therm_Database(OBA, 13) 63 | OOBTHR17 KT.Tel_Therm_Database(OBA, 14) 64 | OOBTHR18 KT.Tel_Therm_Database(OBA, 15) 65 | OOBTHR19 KT.Tel_Therm_Database(OBA, 16) 66 | OOBTHR20 KT.Tel_Therm_Database(OBA, 17) 67 | OOBTHR21 KT.Tel_Therm_Database(OBA, 18) 68 | OOBTHR22 KT.Tel_Therm_Database(OBA, 19) 69 | OOBTHR23 KT.Tel_Therm_Database(OBA, 20) 70 | OOBTHR24 KT.Tel_Therm_Database(OBA, 21) 71 | OOBTHR25 KT.Tel_Therm_Database(OBA, 22) 72 | OOBTHR26 KT.Tel_Therm_Database(OBA, 23) 73 | OOBTHR27 KT.Tel_Therm_Database(OBA, 24) 74 | OOBTHR28 KT.Tel_Therm_Database(OBA, 25) 75 | OOBTHR29 KT.Tel_Therm_Database(OBA, 26) 76 | ^^^ Patch A ^^^^ 77 | --------- 78 | ,,,, Patch B ,,,, 79 | OOBTHR30 KT.Tel_Therm_Database(OBA, 27) 80 | OOBTHR31 KT.Tel_Therm_Database(OBA, 28) 81 | OOBTHR33 KT.Tel_Therm_Database(OBA, 29) 82 | OOBTHR34 KT.Tel_Therm_Database(OBA, 30) 83 | OOBTHR35 KT.Tel_Therm_Database(OBA, 31) 84 | OOBTHR36 KT.Tel_Therm_Database(OBA, 32) 85 | OOBTHR37 KT.Tel_Therm_Database(OBA, 33) 86 | OOBTHR38 KT.Tel_Therm_Database(OBA, 34) 87 | OOBTHR39 KT.Tel_Therm_Database(OBA, 35) 88 | OOBTHR40 KT.Tel_Therm_Database(OBA, 36) 89 | OOBTHR41 KT.Tel_Therm_Database(OBA, 37) 90 | OOBTHR44 KT.Tel_Therm_Database(OBA,40) 91 | OOBTHR45 KT.Tel_Therm_Database(OBA,41) 92 | OOBTHR46 KT.Tel_Therm_Database(OBA,42) 93 | OOBTHR49 KT.Tel_Therm_Database(OBA,44) 94 | OOBTHR50 KT.Tel_Therm_Database(OBA,45) 95 | OOBTHR51 KT.Tel_Therm_Database(OBA,46) 96 | OOBTHR52 KT.Tel_Therm_Database(OBA,47) 97 | OOBTHR53 KT.Tel_Therm_Database(OBA,48) 98 | OOBTHR54 KT.Tel_Therm_Database(OBA,49) 99 | -------------------------------------------------------------------------------- /legacy/NOTES.state_bins: -------------------------------------------------------------------------------- 1 | ********************************************************************************* 2 | Log in to a machine with at least 16 Gb RAM and get into ska environment 3 | ********************************************************************************* 4 | 5 | ssh kadi 6 | cd ~/git/eng_archive 7 | ska 8 | unset ENG_ARCHIVE 9 | 10 | Content types with just one fileset (no distinct 1999 dir): 11 | 12 | dp_eps16, dp_eps8, hrc0hk, hrc0ss, simdiag, ccdm14eng, ccdm15eng 13 | 14 | First replicate the data structure and existing files that are required. 15 | 16 | cd /proj/sot/ska/data/eng_archive 17 | rsync -av --exclude=arch/ --exclude='*.h5' --exclude=archfiles.db3 --exclude=NEW --exclude=OLD 1999/ /data/aca/tmp/eng_archive/1999/ 18 | rsync -av --exclude=arch/ --exclude='*.h5' --exclude=archfiles.db3 --exclude=NEW --exclude=OLD data/ /data/aca/tmp/eng_archive/data/ 19 | 20 | cd ~/git/eng_archive 21 | ln -s /data/aca/tmp/eng_archive/data ./ 22 | ln -s /data/aca/tmp/eng_archive/1999 ./ 23 | 24 | Now do the computations:: 25 | 26 | ./update_archive.py --no-full --state-codes-only --date-now=2000:005 --max-lookback-time=180 --data-root=1999 >& run1.log & 27 | ./update_archive.py --no-full --state-codes-only --date-now=2000:010 --max-lookback-time=15 --data-root=. >& run2.log & 28 | ./update_archive.py --no-full --state-codes-only --date-start=2000:020 --max-lookback-time=180 --data-root=. >& run3.log & 29 | 30 | ./update_archive.py --no-full --state-codes-only --date-now=2000:005 --max-lookback-time=180 --data-root=1999 >& run10.log & 31 | ./update_archive.py --no-full --state-codes-only --date-now=2000:010 --max-lookback-time=15 --data-root=. >& run20.log & 32 | ./update_archive.py --no-full --state-codes-only --date-start=2000:020 --max-lookback-time=180 --data-root=. >& run30.log & 33 | 34 | Bring local archive up to date (after taking a break from this):: 35 | 36 | ./update_archive.py --no-full --state-codes-only --max-lookback-time=200 --data-root=. >& run50.log & 37 | 38 | Install 39 | ======= 40 | 41 | HEAD:: 42 | 43 | # First make list of state code MSIDs that are getting updated 44 | cd ~/git/eng_archive 45 | /bin/ls -1 data/*/{5min,daily}/*.h5 > state_msids 46 | /bin/ls -1 1999/data/*/{5min,daily}/*.h5 >> state_msids 47 | 48 | # Make backup directory 49 | tmproot=/data/aca/tmp/eng_archive 50 | mkdir -p $tmproot/stats_bin_backup 51 | 52 | # Make backup copy of relevant 5min/daily MSIDs 53 | rsync -v --files-from=state_msids \ 54 | /proj/sot/ska/data/eng_archive/ \ 55 | $tmproot/stats_bin_backup/ 56 | 57 | # Practice and check 58 | head -1 state_msids > state_msids_one 59 | 60 | # just the first entry 61 | rsync --dry-run -v --files-from=state_msids_one ./ /proj/sot/ska/data/eng_archive/ 62 | 63 | # Do it 64 | rsync --dry-run -v --files-from=state_msids ./ /proj/sot/ska/data/eng_archive/ 65 | 66 | - Install version 0.41 of Ska.engarchive to Ska flight (or test on GRETA). 67 | 68 | CONTINGENCY:: 69 | 70 | # Restore backup copy of relevant 5min/daily MSIDs 71 | rsync -v --files-from=state_msids \ 72 | $tmproot/stats_bin_backup/ \ 73 | /proj/sot/ska/data/eng_archive/ 74 | 75 | 76 | GRETA:: 77 | 78 | set tmproot=/proj/sot/ska/tmp/eng_archive 79 | 80 | # Copy over new stats files for state-values MSIDs 81 | cd /home/SOT/git/eng_archive 82 | rm -rf data 1999 83 | mkdir data 84 | mkdir 1999 85 | rsync -av aldcroft@ccosmos:/home/aldcroft/git/eng_archive/data/ data/ 86 | rsync -av aldcroft@ccosmos:/home/aldcroft/git/eng_archive/1999/ 1999/ 87 | 88 | Now follow the HEAD instructions, but remembering to instal to Ska test. 89 | -------------------------------------------------------------------------------- /legacy/fix_ingest_h5.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | """ 3 | Fix overlaps within the original archive and h5 ingest. These are due to 4 | problems with different versioned files that cover same time range being 5 | retrieved by the archive. 6 | """ 7 | 8 | import os 9 | import shutil 10 | import optparse 11 | 12 | import numpy 13 | import tables 14 | import Ska.DBI 15 | import asciitable 16 | 17 | from context_def import ft, files 18 | 19 | 20 | def get_options(): 21 | parser = optparse.OptionParser() 22 | parser.add_option("--dry-run", 23 | action="store_true", 24 | help="Dry run (no actual file or database updatees)") 25 | parser.add_option("--overwrite-partial", 26 | action="store_true", 27 | help="Overwrite the overlapping bits of file (e.g. for EPHEM)") 28 | return parser.parse_args() 29 | 30 | opt, args = get_options() 31 | 32 | filetypes = asciitable.read('Ska/engarchive/filetypes.dat') 33 | if len(args) > 0: 34 | filetypes = filetypes[filetypes['content'] == args[0].upper()] 35 | 36 | for filetype in filetypes: 37 | # Update attributes of global ContextValue "ft". This is needed for 38 | # rendering of "files" ContextValue. 39 | print filetype.content 40 | 41 | ft['content'] = filetype['content'].lower() 42 | ft['msid'] = 'TIME' 43 | 44 | # archive files 45 | if not os.path.exists(files['archfiles'].abs) or os.path.exists(files['oldmsid'].abs + '.bak'): 46 | print 'Skipping', ft['content'] 47 | continue 48 | 49 | # Make backup! 50 | print 'Copying ', files['oldmsid'].abs, files['oldmsid'].abs + '.bak' 51 | if not opt.dry_run: 52 | shutil.copy2(files['oldmsid'].abs, files['oldmsid'].abs + '.bak') 53 | 54 | # Read the list of archive files that were used to build the MSID h5 files 55 | db = Ska.DBI.DBI(dbi='sqlite', server=files['archfiles'].abs) 56 | archfiles = db.fetchall('select * from archfiles order by tstart asc') 57 | db.conn.close() 58 | 59 | # Find archive files that overlap in time (tstart : tstop). 60 | overlaps = archfiles[1:].tstart - archfiles[:-1].tstop < -1 61 | file0s = archfiles[:-1][overlaps] 62 | file1s = archfiles[1:][overlaps] 63 | 64 | # Open the TIME.h5 file for this content type 65 | h5 = tables.openFile(files['oldmsid'].abs, mode=('r' if opt.dry_run else 'a')) 66 | 67 | # Iterate through overlapping files and set bad quality in TIME for rows 68 | # coming from the file with lower revision 69 | for file0, file1 in zip(file0s, file1s): 70 | if opt.overwrite_partial: 71 | # Effectively overwrite the file0 values with file1 values 72 | # for the file0 rows from file1['tstart'] to file0['tstop'] 73 | times = h5.root.data[file0['rowstart']:file0['rowstop']] 74 | bad_rowstart = numpy.searchsorted(times, file1['tstart']) + file0['rowstart'] 75 | bad_rowstop = file0['rowstop'] 76 | else: 77 | badfile = file0 if file0['revision'] < file1['revision'] else file1 78 | bad_rowstart = badfile['rowstart'] 79 | bad_rowstop = badfile['rowstop'] 80 | print file0['filename'], file0['revision'], file1['revision'], \ 81 | '%9d %9d %9d %9d' % (file0['tstart'], file0['tstop'], 82 | file1['tstart'], file1['tstop']), \ 83 | 'Setting TIME rows', bad_rowstart, bad_rowstop, 'to bad quality' 84 | if not opt.dry_run: 85 | h5.root.quality[bad_rowstart:bad_rowstop] = True 86 | 87 | h5.close() 88 | -------------------------------------------------------------------------------- /docs/source/index.md: -------------------------------------------------------------------------------- 1 | # Getting Started with jeta 2 | 3 | ## What is jeta? 4 | 5 | The `jeta` package is an API for monitoring and accessing FOS Engineering Telemetry Archive. The API exposes functions for ingesting/processing telemetry (tlm), 6 | archive status/managment, and data fetching. Tools are built around the python data science stack (i.e. numpy, pandas, matplotlib ... etc.) 7 | and well as HDF5 standard and supporting python implementations (h5py, pytables). There are multiple ways to access and use the jeta package 8 | including directly after installing with `pip`, inside a pre-built enviroment with jupyter lab, a web API using HTTP request, and a web application 9 | that provides a front-end for the most common use cases for the tools. 10 | 11 | The package, `jeta` is a LITA functional-equivalent of [`Ska.eng_archive`](https://github.com/sot/eng_archive). 12 | [Ska](https://cxc.cfa.harvard.edu/mta/ASPECT/tool_doc/pydocs/) is the "engineering telemetry archive is a suite of tools and data products" for 13 | that supports the [Chandra](https://chandra.harvard.edu/about/spacecraft.html) X-Ray Observatiory. Where as `jeta` fulfills a similar role for JWST. 14 | The package is still dependant on the Chandra package environment, [skare3](https://github.com/sot/skare3). 15 | 16 | ## Installing `jeta` 17 | 18 | The jeta package can be installed and run locally against a local copy of the archive (WARNING. This is not recommend as data volumns will get extremely large. This feature is mainly for the purpose of development and testing). There are currently two methods of using `jeta` locally, first by installing directly on the system (or in a virtual envirnment) and second running a containerized version. In 19 | either case, a collection of environment variables must be set. A list of required environment variables and there descriptions 20 | can be found in `/scripts/build_env.bash`. You can run this script in a terminal in the following ways to set the 21 | environment variables automatically: 22 | 23 | ```bash 24 | source /scripts/build_env.bash 25 | # or, and not the . prefix 26 | . /scripts/build_env.bash 27 | ``` 28 | 29 | ### Using setuptools 30 | You can install the jeta package locally on your system or in a conda or virtualenv. First clone a copy of the source repo 31 | and the use the `setup.py` module to install. 32 | 33 | ```bash 34 | git clone https://github.com/spacetelescope/jeta 35 | cd jeta 36 | # a list of python packages required to use jeta 37 | # are listed in the requirements/local.txt file 38 | python setup.py install 39 | ``` 40 | ### Using docker 41 | 42 | ```bash 43 | # After setting the environment variables 44 | # or creating a .env with them in the same dir 45 | # as the docker-compose.yml 46 | docker-compose -f docker-compose.yml build 47 | docker-compose -f docker-compose.yml up 48 | ``` 49 | 50 | ### Verify the Installation 51 | 52 | ```bash 53 | >>> import jeta 54 | >>> jeta.__version__ 55 | '2.11.0' 56 | ``` 57 | 58 | ## Using on in JupyterLab 59 | 60 | Navigate to: https://jlab.stsci.edu/notebook/user//lab 61 | 62 | From there you will be presented with your own area to run jupyter notebooks 63 | with the `jeta` package already available. 64 | 65 | ## Basic Usage 66 | 67 | ```python 68 | from jeta.archive import fetch 69 | msid = fetch.MSID(MSID, tstart, tstop) 70 | print(msid.vals) 71 | print(msid.times) 72 | print(msid.means) 73 | print(msid.mins) 74 | print(msid.maxes) 75 | ``` 76 | 77 | ```{toctree} 78 | --- 79 | maxdepth: 2 80 | caption: Contents 81 | --- 82 | fetch.md 83 | plotting.md 84 | operations.md 85 | ingest.md 86 | update.md 87 | ``` 88 | 89 | 90 | 91 | 92 | 93 | Indices and tables 94 | ================== 95 | 96 | * :ref:`genindex` 97 | * :ref:`modindex` 98 | * :ref:`search` 99 | -------------------------------------------------------------------------------- /docs/source/fetch.md: -------------------------------------------------------------------------------- 1 | # Fetching 2 | 3 | The target audience of this document is a member of the FOT who wishes to access archived engineering data for 4 | the purpose of data analysis. 5 | 6 | ```{py:currentmodule} jeta.archive.fetch 7 | ``` 8 | 9 | ## Acronyms and Terms 10 | * MSID - an object model of an mnemonic (or measurand). 11 | * API - Application Programming Interface 12 | * CLI - Command-Line Interface 13 | 14 | ## Fetch API Classes 15 | ### MSID Class API 16 | ```{eval-rst} 17 | .. autoclass:: MSID 18 | :show-inheritance: 19 | :members: 20 | ``` 21 | 22 | ### MSIDSet Class API 23 | ```{eval-rst} 24 | .. autoclass:: MSIDset 25 | :show-inheritance: 26 | :members: 27 | ``` 28 | 29 | ### Msid Class API 30 | ```{eval-rst} 31 | .. autoclass:: Msid 32 | :show-inheritance: 33 | :members: 34 | ``` 35 | 36 | ### Msidset 37 | ```{eval-rst} 38 | .. autoclass:: Msidset 39 | :show-inheritance: 40 | :members: 41 | ``` 42 | 43 | ## CLI Example 44 | 45 | This next section will demostrate the usage of the fetch module. These examples are presented 46 | as if being executed on a command-line such as a unix terminal. However, the example should work 47 | the same in a Jupyter notebook environment. 48 | 49 | ### Ex. 1: Fetch The Archive Coverage For An MSID 50 | 51 | It maybe useful to learn the archive coverage interval for a specific msid. This can be accomplished by using the 52 | `fetch.get_time_range` function. 53 | 54 | ```python 55 | # Returns date range as unix timestamp by default 56 | fetch.get_time_range() 57 | (2458664.5208374076, 2458666.5249855556) 58 | 59 | # Returns date range in ISO format 60 | fetch.get_time_range(, "iso") 61 | ('2019-06-30 00:30:00.352', '2019-07-02 00:35:58.752') 62 | ``` 63 | 64 | 65 | ### Ex. 2: Fetch All Archived Data For A Single Mnemonic 66 | 67 | Import the fetch submodule from the archive module. You'll get a message indicating the location where 68 | the system will attempt to fetch telemetry. 69 | 70 | ```python 71 | from jeta.archive import fetch 72 | fetch: using ENG_ARCHIVE= for archive path 73 | msid = fetch.MSID() # where is the MSID 74 | ``` 75 | 76 | ### Ex. 3: Fetch A Range Of Archived Data for Single Mnemonic 77 | 78 | Import the fetch submodule from the archive module. You'll get a message indicating the location where 79 | the system will attempt to fetch telemetry. As part of the fetch you will specific the `tstart` and `tstop` 80 | in the Day Of Year (DOY) -- `yday` in the `astropy.time.Time` module -- format. Percision for the user supplied times 81 | is to the millisecond level (i.e. YYYY:DOY:HH:MM:SS.sss). 82 | 83 | ```python 84 | from jeta.archive import fetch 85 | fetch: using ENG_ARCHIVE= for archive path 86 | msid = fetch.Msid('', '2019:001', '2019:002') # where is the MSID 87 | ``` 88 | 89 | ### Ex. 4: Plotting An MSID 90 | 91 | To fetch data for a single mnenmonic create a instance of the *Msid (Mnemonic System Identifier)* class. 92 | 93 | ```python 94 | msid = fetch.Msid('', '2019:001', '2019:002') 95 | msid.plot() # will create an inline/widget plot in a jupyter notebook or new plot window in a desktop environment 96 | ``` 97 | 98 | This returns the python data structure (Msid Object) containing the data requested in the specified range. The 99 | objects ``plot()`` method can then be called to display a plot of the data. The plot is generated by matplotlib. 100 | 101 | 102 | ### Ex. 5: Fetching Multiple MSIDs At Once Using MSIDSet 103 | 104 | For some operations it may be useful to retrieve a set of Msid. To do this use the **MsidSet Object** in place of the Msid. 105 | ```python 106 | fetch.Msidset([MNEMONIC_1, 'MNEMONIC_2'], '2019:001', '2019:365') 107 | Msidset( 108 | [ 109 | ( 110 | 'MNEMONIC_1', 111 | 112 | ), 113 | ( 114 | 'MNEMONIC_2', 115 | 116 | ) 117 | ] 118 | ) 119 | ``` -------------------------------------------------------------------------------- /jeta/archive/files.py: -------------------------------------------------------------------------------- 1 | import os 2 | import h5py 3 | 4 | from bisect import bisect_left, bisect_right 5 | 6 | import numpy as np 7 | 8 | class IndexFile: 9 | 10 | def _get_start_index(self, tstart): 11 | """ Returns the index of the epoch (timestamp) <= the given start time. 12 | """ 13 | i = bisect_right(self.data['epoch'], tstart) 14 | if i and not (i-1 < 0): 15 | return i - 1 16 | else: 17 | return 0 18 | raise ValueError(f'{tstart} did not match any value.') 19 | 20 | def _get_stop_index(self, tstop): 21 | """ Returns the index of the epoch (timestamp) >= the given stop time. 22 | """ 23 | i = bisect_right(self.data['epoch'], tstop) 24 | if not (i >= self.data.shape[0]): 25 | return i 26 | else: 27 | return -1 28 | raise ValueError(f'{tstop} did not match any value.') 29 | 30 | def get_index_range(self, tstart, tstop): 31 | """ Returns a tuple of indices given an start and stop time range. 32 | """ 33 | return (self._get_start_index(tstart), self._get_stop_index(tstop)) 34 | 35 | def get_archive_index_range(self, tstart, tstop): 36 | idx0 = self.data['index'][self._get_start_index(tstart)] 37 | idxN = self.data['index'][self._get_stop_index(tstop)] 38 | return (idx0, idxN) 39 | 40 | 41 | def get_data(self, tstart=None, tstop=None): 42 | """ Returns a either the full data for the file or a block given a range. 43 | """ 44 | if tstart is None or tstop is None: 45 | return self.data 46 | else: 47 | idx0, idxN = self.get_index_range(tstart, tstop) 48 | return self.data[idx0:idxN] 49 | 50 | def __init__(self, msid): 51 | self.msid = msid 52 | self.file_path = '{}/data/tlm/{}/index.h5'.format(os.environ['TELEMETRY_ARCHIVE'], msid) 53 | assert os.path.exists(self.file_path), f'Could not create index file reference. File path does not exist: {self.file_path}' 54 | with h5py.File(self.file_path, 'r') as h5: 55 | self.data = h5['epoch'][...] 56 | self.data = np.array([row for row in self.data], dtype=([('epoch', np.float64), ('index', np.int32)])) 57 | self.idx0 = self.data['index'][0] 58 | self.idxN = self.data['index'][-1] 59 | self.epoch0 = self.data['epoch'][0] 60 | self.epochN = self.data['epoch'][-1] 61 | 62 | 63 | class ValueFile: 64 | 65 | def get_file_data_range(self, tstart=None, tstop=None): 66 | 67 | if tstart is None or tstop is None: 68 | idx0 = 0 69 | idxN = -1 70 | else: 71 | idx0, idxN = self.index_file.get_archive_index_range(tstart, tstop) 72 | with h5py.File(self.file_path, 'r') as h5: 73 | self.selection = h5['values'][idx0:idxN] 74 | self.selection_length = len(self.selection) 75 | self.length = len(h5['values']) 76 | 77 | def __init__(self, msid): 78 | self.msid = msid 79 | self.file_path = self.file_path = '{}/data/tlm/{}/values.h5'.format(os.environ['TELEMETRY_ARCHIVE'], msid) 80 | assert os.path.exists(self.file_path), f'Could not create values file reference. File path does not exist: {self.file_path}' 81 | self.index_file = IndexFile(self.msid) 82 | self.selection = None 83 | 84 | 85 | class TimeFile: 86 | 87 | def get_file_data_range(self, tstart=None, tstop=None): 88 | if tstart is None and tstop is None: 89 | idx0 = 0 90 | idxN = -1 91 | else: 92 | idx0, idxN = self.index_file.get_archive_index_range(tstart, tstop) 93 | with h5py.File(self.file_path, 'r') as h5: 94 | self.selection = h5['times'][idx0:idxN] 95 | self.selection_length = len(self.selection) 96 | self.length = len(h5['times']) 97 | 98 | def __init__(self, msid): 99 | self.msid = msid 100 | self.file_path = self.file_path = '{}/data/tlm/{}/times.h5'.format(os.environ['TELEMETRY_ARCHIVE'], msid) 101 | assert os.path.exists(self.file_path), f'Could not create values file reference. File path does not exist: {self.file_path}' 102 | self.index_file = IndexFile(self.msid) 103 | self.selection = None 104 | -------------------------------------------------------------------------------- /jeta/archive/recovery.py: -------------------------------------------------------------------------------- 1 | def fix_state_code(state_code): 2 | """ 3 | Return a version of ``state_code`` that has only alphanumeric chars. This 4 | can be used as a column name, unlike e.g. "n_+1/2". Since this gets called 5 | in an inner loop cache the result. 6 | """ 7 | try: 8 | out = _fix_state_code_cache[state_code] 9 | except KeyError: 10 | out = state_code 11 | for sub_in, sub_out in ((r'\+', 'PLUS_'), 12 | (r'\-', 'MINUS_'), 13 | (r'>', '_GREATER_'), 14 | (r'/', '_DIV_')): 15 | out = re.sub(sub_in, sub_out, out) 16 | _fix_state_code_cache[state_code] = out 17 | 18 | return out 19 | 20 | 21 | def fix_misorders(filetype): 22 | """Fix problems in the eng archive where archive files were ingested out of 23 | time order. This results in a non-monotonic times in the MSID hdf5 files 24 | and subsequently corrupts the stats files. This routine looks for 25 | discontinuities in rowstart assuming filename ordering and swaps neighbors. 26 | One needs to verify in advance (--dry-run --fix-misorders --content ...) 27 | that this will be an adequate fix. 28 | Example:: 29 | update_archive.py --dry-run --fix-misorders --content misc3eng 30 | update_archive.py --fix-misorders --content misc3eng >& fix_misc3.log 31 | update_archive.py --content misc3eng --max-lookback-time 100 >>& fix_misc3.log 32 | In the --dry-run it is important to verify that the gap is really just from 33 | two mis-ordered files that can be swapped. Look at the rowstart,rowstop values 34 | in the filename-ordered list. 35 | :param filetype: filetype 36 | :returns: minimum time for all misorders found 37 | """ 38 | colnames = pickle.load(open(msid_files['colnames'].abs, 'rb')) 39 | 40 | # Setup db handle with autocommit=False so that error along the way aborts insert transactions 41 | db = Ska.DBI.DBI(dbi='sqlite', server=msid_files['archfiles'].abs, autocommit=False) 42 | 43 | # Get misordered archive files 44 | archfiles = db.fetchall('SELECT * FROM archfiles order by filename') 45 | bads = archfiles['rowstart'][1:] - archfiles['rowstart'][:-1] < 0 46 | 47 | if not np.any(bads): 48 | logger.info('No misorders') 49 | return 50 | 51 | for bad in np.flatnonzero(bads): 52 | i2_0, i1_0 = archfiles['rowstart'][bad:bad + 2] 53 | i2_1, i1_1 = archfiles['rowstop'][bad:bad + 2] 54 | 55 | # Update hdf5 file for each column (MSIDs + TIME, MJF, etc) 56 | for colname in colnames: 57 | ft['msid'] = colname 58 | logger.info('Fixing %s', msid_files['msid'].abs) 59 | if not opt.dry_run: 60 | filepath = msid_files['mnemonic_value'].abs 61 | h5 = tables.open_file(filepath, mode='a') 62 | #h5 = tables.open_file(msid_files['msid'].abs, mode='a') 63 | hrd = h5.root.data 64 | hrq = h5.root.quality 65 | 66 | hrd1 = hrd[i1_0:i1_1] 67 | hrd2 = hrd[i2_0:i2_1] 68 | hrd[i1_0:i1_0 + len(hrd2)] = hrd2 69 | hrd[i1_0 + len(hrd2): i2_1] = hrd1 70 | 71 | hrq1 = hrq[i1_0:i1_1] 72 | hrq2 = hrq[i2_0:i2_1] 73 | hrq[i1_0:i1_0 + len(hrq2)] = hrq2 74 | hrq[i1_0 + len(hrq2): i2_1] = hrq1 75 | 76 | h5.close() 77 | 78 | # Update the archfiles table 79 | cmd = 'UPDATE archfiles SET ' 80 | cols = ['rowstart', 'rowstop'] 81 | cmd += ', '.join(['%s=?' % x for x in cols]) 82 | cmd += ' WHERE filename=?' 83 | rowstart1 = i1_0 84 | rowstop1 = rowstart1 + i2_1 - i2_0 85 | rowstart2 = rowstop1 + 1 86 | rowstop2 = i2_1 87 | vals1 = [rowstart1, rowstop1, archfiles['filename'][bad]] 88 | vals2 = [rowstart2, rowstop2, archfiles['filename'][bad + 1]] 89 | logger.info('Running %s %s', cmd, vals1) 90 | logger.info('Running %s %s', cmd, vals2) 91 | 92 | logger.info('Swapping rows %s for %s', [i1_0, i1_1, i2_0, i2_1], filetype.content) 93 | logger.info('%s', archfiles[bad - 3:bad + 5]) 94 | logger.info('') 95 | 96 | if not opt.dry_run: 97 | db.execute(cmd, [x.tolist() for x in vals1]) 98 | db.execute(cmd, [x.tolist() for x in vals2]) 99 | db.commit() 100 | 101 | return np.min(archfiles['tstart'][bads]) -------------------------------------------------------------------------------- /jeta/archive/fastss.pyx: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cython 3 | cimport numpy as np 4 | 5 | DTYPE = np.int 6 | 7 | ctypedef np.int_t DTYPE_t 8 | ctypedef np.double_t DTYPE_double_t 9 | 10 | @cython.wraparound(False) 11 | @cython.boundscheck(False) 12 | def _search_both_sorted(np.ndarray[dtype=DTYPE_double_t, ndim=1] a not None, 13 | np.ndarray[dtype=DTYPE_double_t, ndim=1] v not None): 14 | """ 15 | Find indices where elements should be inserted to maintain order. 16 | 17 | Find the indices into a sorted array `a` such that, if the corresponding 18 | elements in `v` were inserted before the indices, the order of `a` would 19 | be preserved. 20 | 21 | Parameters 22 | ---------- 23 | a : 1-D array_like 24 | Input array, sorted in ascending order. 25 | v : array_like 26 | Values to insert into `a`. 27 | """ 28 | cdef int nv = v.shape[0] 29 | cdef np.ndarray[dtype=DTYPE_t, ndim=1] idx = np.empty(nv, dtype=DTYPE) 30 | cdef int na = a.shape[0] 31 | cdef unsigned int ia = 0 32 | cdef unsigned int iv 33 | cdef double vi 34 | 35 | for iv in range(nv): 36 | vi = v[iv] 37 | while True: 38 | if ia < na: 39 | if vi <= a[ia]: 40 | idx[iv] = ia 41 | break 42 | else: 43 | ia += 1 44 | else: 45 | idx[iv] = na 46 | break 47 | 48 | return idx 49 | 50 | @cython.wraparound(False) 51 | @cython.boundscheck(False) 52 | def _nearest_index(np.ndarray[dtype=DTYPE_double_t, ndim=1] a not None, 53 | np.ndarray[dtype=DTYPE_double_t, ndim=1] v not None): 54 | """ 55 | Find indices where elements should be inserted to maintain order. 56 | 57 | Find the indices into a sorted array `a` such that, if the corresponding 58 | elements in `v` were inserted before the indices, the order of `a` would 59 | be preserved. 60 | 61 | Parameters 62 | ---------- 63 | a : 1-D array_like 64 | Input array, sorted in ascending order. 65 | v : array_like 66 | Values to insert into `a`. 67 | """ 68 | cdef int nv = v.shape[0] 69 | cdef np.ndarray[dtype=DTYPE_t, ndim=1] idx = np.empty(nv, dtype=DTYPE) 70 | cdef int na = a.shape[0] 71 | cdef unsigned int ia = 0 72 | cdef unsigned int iv 73 | cdef double vi 74 | 75 | for iv in range(nv): 76 | vi = v[iv] 77 | while True: 78 | if ia < na: 79 | if vi <= a[ia]: 80 | if ia == 0: 81 | idx[iv] = ia 82 | elif abs(vi - a[ia - 1]) < abs(vi - a[ia]): 83 | idx[iv] = ia - 1 84 | else: 85 | idx[iv] = ia 86 | break 87 | else: 88 | ia += 1 89 | else: # ia == na without vi ever being less than a[ia] 90 | # Thus vi > all values of a 91 | idx[iv] = na - 1 92 | break 93 | 94 | return idx 95 | 96 | @cython.wraparound(False) 97 | @cython.boundscheck(False) 98 | def _interp_linear(np.ndarray[dtype=DTYPE_double_t, ndim=1] yin not None, 99 | np.ndarray[dtype=DTYPE_double_t, ndim=1] a not None, 100 | np.ndarray[dtype=DTYPE_double_t, ndim=1] v not None): 101 | """ 102 | yout = (xout - x0) / (x1 - x0) * (y1 - y0) + y0 103 | 104 | Parameters 105 | ---------- 106 | yin : 1-D array_like 107 | Input y values 108 | a : 1-D array_like 109 | Input x values corresponding to yin 110 | v : 1-D array_like 111 | Output x values corresponding to yout 112 | """ 113 | cdef double vi 114 | cdef int nv = v.shape[0] 115 | cdef np.ndarray[dtype=DTYPE_double_t, ndim=1] yout = np.empty(nv, dtype=np.float64) 116 | cdef unsigned int na = a.shape[0] 117 | cdef unsigned int ia = 0 118 | cdef unsigned int iv 119 | cdef unsigned int na1 = na - 1 120 | cdef unsigned int na2 = na - 2 121 | cdef unsigned int ia1 122 | 123 | for iv in range(nv): 124 | vi = v[iv] 125 | while True: 126 | if ia < na: 127 | if vi <= a[ia]: 128 | if ia == 0: 129 | yout[iv] = (vi - a[0]) / (a[1] - a[0]) * (yin[1] - yin[0]) + yin[0] 130 | else: 131 | ia1 = ia - 1 132 | yout[iv] = (vi - a[ia1]) / (a[ia] - a[ia1]) * (yin[ia] - yin[ia1]) + yin[ia1] 133 | break 134 | else: 135 | ia += 1 136 | else: # ia == na without vi ever being less than a[ia] 137 | # Thus vi > all values of a 138 | yout[iv] = (vi - a[na1]) / (a[na1] - a[na2]) * (yin[na1] - yin[na2]) + yin[na1] 139 | break 140 | 141 | return yout 142 | -------------------------------------------------------------------------------- /jeta/archive/derived/base.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | from __future__ import print_function, division, absolute_import 3 | 4 | from astropy.time import Time 5 | from .. import fetch 6 | 7 | import numpy as np 8 | from .. import cache 9 | 10 | __all__ = ['MNF_TIME', 'times_indexes', 'DerivedParameter'] 11 | 12 | MNF_TIME = 0.25625 # Minor Frame duration (seconds) 13 | 14 | def times_indexes(start, stop, dt): 15 | index0 = Time(start, format='unix').unix // dt 16 | index1 = Time(stop, format='unix').unix // dt + 1 17 | indexes = np.arange(index0, index1, dtype=np.int64) 18 | times = indexes * dt 19 | return times, indexes 20 | 21 | @cache.lru_cache(20) 22 | def interpolate_times(keyvals, len_data_times, data_times=None, times=None): 23 | return np.interpolate(np.arange(len_data_times), 24 | data_times, times, method='nearest') 25 | 26 | class DerivedParameter(object): 27 | max_gap = 66.0 # Max allowed data gap (seconds) 28 | max_gaps = {} 29 | unit_system = 'eng' 30 | dtype = None # If not None then cast to this dtype 31 | 32 | def calc(self, data): 33 | raise NotImplementedError 34 | 35 | def fetch(self, start, stop): 36 | unit_system = fetch.get_units() # cache current units and restore after fetch 37 | fetch.set_units(self.unit_system) 38 | dataset = fetch.MSIDset(self.rootparams, start, stop) 39 | fetch.set_units(unit_system) 40 | 41 | # Translate state codes "ON" and "OFF" to 1 and 0, respectively. 42 | for data in dataset.values(): 43 | if (data.vals.dtype.name == 'string24' 44 | and set(data.vals).issubset(set(['ON ', 'OFF']))): 45 | data.vals = np.where(data.vals == 'OFF', np.int8(0), np.int8(1)) 46 | 47 | times, indexes = times_indexes(start, stop, self.time_step) 48 | bads = np.zeros(len(times), dtype=np.bool) # All data OK (false) 49 | 50 | for msidname, data in dataset.items(): 51 | # If no data are found in specified interval then stub two fake 52 | # data points that are both bad. All interpolated points will likewise 53 | # be bad. 54 | if len(data) < 2: 55 | data.vals = np.zeros(2, dtype=data.vals.dtype) # two null points 56 | data.bads = np.ones(2, dtype=np.bool) # all points bad 57 | data.times = np.array([times[0], times[-1]]) 58 | print('No data in {} between {} and {} (setting all bad)' 59 | .format(msidname, Time(start, format='unix').yday, Time(stop, format='unix').yday)) 60 | keyvals = (data.content, data.times[0], data.times[-1], 61 | len(times), times[0], times[-1]) 62 | idxs = interpolate_times(keyvals, len(data.times), 63 | data_times=data.times, times=times) 64 | 65 | # Loop over data attributes like "bads", "times", "vals" etc and 66 | # perform near-neighbor interpolation by indexing 67 | for attr in data.colnames: 68 | vals = getattr(data, attr) 69 | if vals is not None: 70 | setattr(data, attr, vals[idxs]) 71 | 72 | bads = bads | data.bads 73 | # Reject near-neighbor points more than max_gap secs from available data 74 | max_gap = self.max_gaps.get(msidname, self.max_gap) 75 | gap_bads = abs(data.times - times) > max_gap 76 | if np.any(gap_bads): 77 | print("Setting bads because of gaps in {} between {} to {}" 78 | .format(msidname, 79 | Time(times[gap_bads][0], format='unix').yday, 80 | Time(times[gap_bads][-1], format='unix').yday)) 81 | bads = bads | gap_bads 82 | 83 | dataset.times = times 84 | dataset.bads = bads 85 | dataset.indexes = indexes 86 | 87 | return dataset 88 | 89 | def __call__(self, start, stop): 90 | dataset = fetch.MSIDset(self.rootparams, start, stop, filter_bad=True) 91 | 92 | # Translate state codes "ON" and "OFF" to 1 and 0, respectively. 93 | for data in dataset.values(): 94 | if (data.vals.dtype.name == 'string24' 95 | and set(data.vals) == set(('ON ', 'OFF'))): 96 | data.vals = np.where(data.vals == 'OFF', np.int8(0), np.int8(1)) 97 | 98 | dataset.interpolate(dt=self.time_step) 99 | 100 | # Return calculated values. Np.asarray will copy the array only if 101 | # dtype is not None and different from vals.dtype; otherwise a 102 | # reference is returned. 103 | vals = self.calc(dataset) 104 | return np.asarray(vals, dtype=self.dtype) 105 | 106 | @property 107 | def mnf_step(self): 108 | return int(round(self.time_step / MNF_TIME)) 109 | 110 | @property 111 | def content(self): 112 | return 'dp_{}{}'.format(self.content_root.lower(), self.mnf_step) 113 | -------------------------------------------------------------------------------- /legacy/RELEASE.rst: -------------------------------------------------------------------------------- 1 | 0.21 - 2012-12-09 2 | ================= 3 | 4 | - Add three new content types: SIM SEA, EPHIN housekeeping, and CPE. 5 | - Numerous infrastructure improvements in ingest and update process. 6 | 7 | 0.20 - 2012-08-11 8 | ================= 9 | 10 | - Use weighted mean and stddev for calculating stats. 11 | - Use float64 to accumulate sum for computing stats mean. 12 | - Rebuild stats files for the full mission. 13 | - Fix bug that number of samples for daily stats was incorrect. 14 | - Add notes and regression testing code for re-building stats files. 15 | - This release fixes issues #38, #39, and #41. 16 | 17 | 0.19.1 2012-06-22 18 | ================= 19 | 20 | - Add ``MSID.interpolate()`` method which is like ``MSIDset.interpolate()`` 21 | - Speed up ``interpolate()`` methods using the new ``Ska.Numpy.interpolate``. 22 | - Add ``MSIDset.filter_bad_times()`` method that applies the bad 23 | times filter to all MSIDs in a set. 24 | - Speed up `filter_bad_times()` by using a single mask array over 25 | all bad time filters. 26 | - Add some unit / regression tests. 27 | 28 | 0.19 - 2012-05-04 29 | ================= 30 | 31 | - Fix MSID.raw_vals() so it handles state codes with different lengths 32 | - Fix problem in iplot where units not tied to fetched MSID 33 | - Add units to DEA housekeeping MSIDs 34 | 35 | 0.18 - 2012-04-30 36 | ================= 37 | 38 | - Make it possible to reliably use the import mechanism to select different 39 | unit systems within the same script or Python process with no interactions. 40 | 41 | 0.17 - 2012-04-17 42 | ================= 43 | 44 | - Improve tutorial documentation 45 | - Modify PCAD derived parameters to use only predictive ephemeris 46 | - Redefine DP_ROLL_FSS and DP_PITCH_FSS to improve accuracy 47 | - Allow for filter_bad_times to function for data that have already been 48 | filtered. 49 | 50 | 0.16 and 0.15 51 | ============= 52 | 53 | A number of major new features are available in the release 0.16 of the 54 | Ska engineering archive: 55 | 56 | - Built-in plotting capability and an interactive plot browser that 57 | allows arbitrary zooming and panning. See tutorial sections: 58 | Plotting time data (http://goo.gl/6tGNZ) and 59 | Interactive plotting (http://goo.gl/2dG4c) 60 | 61 | - Support for plotting state-valued MSIDs and for accessing 62 | the raw count values. See: 63 | State valued MSIDs (http://goo.gl/9R0Tz) 64 | 65 | - Support for accessing Telemetry Database attributes related 66 | to an MSID. See: 67 | Telemetry database (http://goo.gl/pPo0s) 68 | 69 | - PCAD derived parameters (main code from A. Arvai). See: 70 | Derived PCAD parameters (http://goo.gl/iKDUK) 71 | 72 | API Changes 73 | ----------- 74 | 75 | - MSIDset.interpolate() now behaves more intuitively. This was done 76 | by making the ``times`` attribute of interpolated MSIDs correspond 77 | to the new (linearly-spaced) interpolated times. Previously 78 | times was set to the nearest-neighbor times, which is not generally 79 | useful. See https://github.com/sot/eng_archive/issues/20 and 80 | Interpolation (http://goo.gl/5U5Kp) 81 | 82 | Bug fixes 83 | --------- 84 | - Fixed problem where Msidset fails for 5min and daily values. 85 | 86 | 0.14 87 | ==== 88 | 89 | Minor updates and bug fixes: 90 | 91 | - Change max_gap for ACISDEAHK from 3600 to 10000 sec 92 | - Add midvals attr for stat fetch 93 | - Fix ss_vector() to use quaternion midvals and handle missing telemetry 94 | - Fix typo in fetch.logical_intervals 95 | - Explicitly set --data-root when calling update_archive.py in task_schedule.cfg 96 | 97 | 0.13 98 | ==== 99 | 100 | Version 0.13 of the Ska.engineering archive contains a number of 101 | new features: 102 | 103 | - Support for `derived parameter pseudo-MSIDs `_. 104 | Currently there are number of thermal values and ACIS power values. 105 | 106 | - Two new `fetch.MSID `_ methods: 107 | 108 | - logical_intervals() returns contiguous intervals during which a logical 109 | expression is true. 110 | - state_intervals() determines contiguous intervals during which the MSID 111 | value is unchanged. 112 | 113 | - Two new classes fetch.Msid and fetch.Msidset. These are just like fetch.MSID 114 | or fetch.MSIDset except that filter_bad=True by default. You can use the 115 | new classes just like before but you'll always only get good data values. 116 | 117 | - Changed definition of the "vals" attribute for '5min' and 'daily' stat values 118 | so "vals" is now the sames as "means". Previously the "vals" attribute for a 119 | statistics fetch was the exact telemetry value at the interval midpoint. 120 | This quantity is not that useful and prone to errors since one frequently 121 | does things like:: 122 | 123 | dat = fetch.MSID('tephin', '2011:150', stat='5min') 124 | plot_cxctime(dat.times, dat.vals) 125 | 126 | - Caching of previously fetched data values. This is disabled by default 127 | but used for telemetry and derived parameter ingest. In certain 128 | circumstances caching can be useful. 129 | 130 | - New function for truncating the engineering archive at a certain date. 131 | This is useful for fixing the database in the event of a corruption. 132 | -------------------------------------------------------------------------------- /jeta/archive/status.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import h5py 4 | import sqlite3 5 | import glob 6 | import ntpath 7 | 8 | import pyyaks.logger 9 | import pyyaks.context 10 | 11 | import jeta.archive.file_defs as file_defs 12 | from jeta.archive.utils import get_env_variable 13 | 14 | ENG_ARCHIVE = get_env_variable('TELEMETRY_ARCHIVE') 15 | 16 | ALL_KNOWN_MSID_METAFILE = get_env_variable('ALL_KNOWN_MSID_METAFILE') 17 | 18 | msid_files = pyyaks.context.ContextDict('update.msid_files', 19 | basedir=ENG_ARCHIVE) 20 | msid_files.update(file_defs.msid_files) 21 | 22 | 23 | def create_connection(db_file=msid_files['archfiles'].abs): 24 | """ create a database connection to the SQLite database 25 | specified by the db_file 26 | :param db_file: database file 27 | :return: Connection object or None 28 | """ 29 | conn = None 30 | try: 31 | conn = sqlite3.connect(db_file) 32 | except Exception as e: 33 | print(e) 34 | 35 | return conn 36 | 37 | 38 | def get_msid_count(): 39 | """Get a count of the number of msids in msid reference file. 40 | 41 | :return: a count of the number of msids known to the system 42 | :rtype: int 43 | """ 44 | with h5py.File(ALL_KNOWN_MSID_METAFILE, 'r') as h5: 45 | return len(h5.keys()) 46 | 47 | 48 | def get_msid_names(): 49 | """Get the authoritative list of msids known to the system 50 | 51 | :return: a list of msid 52 | :rtype: list 53 | """ 54 | with h5py.File(ALL_KNOWN_MSID_METAFILE, 'r') as h5: 55 | return sorted(list(h5.keys())) 56 | 57 | 58 | def get_list_of_staged_files(include_path=False): 59 | """Returns a list of staged file records. 60 | 61 | The records are stored as tuples and contain: 62 | - the filename 63 | - the files size on disk 64 | - the ctime for the file 65 | - the start/stop coverage of the file 66 | 67 | :param include_path: [description], defaults to False 68 | :type include_path: bool, optional 69 | :return: a list of staged files as tuples 70 | :rtype: list 71 | """ 72 | from pathlib import Path 73 | from jeta.staging.manage import get_file_coverage 74 | 75 | filenames = [(ntpath.basename(paths), Path(paths).stat().st_size, Path(paths).stat().st_ctime, *get_file_coverage(os.path.basename(paths))) for paths in sorted(glob.glob(f"{get_env_variable('STAGING_DIRECTORY')}/*.h5"))] 76 | 77 | return filenames 78 | 79 | 80 | def get_list_of_files_in_range(tstart, tstop, target_dir=get_env_variable('STAGING_DIRECTORY')): 81 | """!!!Not Implemented!!! 82 | 83 | :param tstart: [description] 84 | :type tstart: [type] 85 | :param tstop: [description] 86 | :type tstop: [type] 87 | :param target_dir: [description], defaults to get_env_variable('STAGING_DIRECTORY') 88 | :type target_dir: [type], optional 89 | """ 90 | pass 91 | 92 | 93 | def get_current_ingest_id(): 94 | """Get the uuid of the active ingest 95 | 96 | :return: the id of the active ingest 97 | :rtype: uuid 98 | """ 99 | return os.getenv('JETA_CURRENT_INGEST_ID') 100 | 101 | 102 | def get_ingest_state(): 103 | """Get the state code of the active ingest 104 | 105 | :return: (STARTING, PREPROCESSING, PROCESSING, POSTPROCESSING, COMPLETE) 106 | :rtype: str 107 | """ 108 | return os.getenv('JETA_INGEST_STATE') 109 | 110 | 111 | def get_ingest_files(ingest_id): 112 | """Get a list of files ingested as part of a specific ingest operation 113 | 114 | :param ingest_id: uuid of an ingest operation 115 | :type ingest_id: uuid 116 | :return: a table representing ingest files linked to an ingest 117 | :rtype: list 118 | """ 119 | conn = create_connection() 120 | cur = conn.cursor() 121 | cur.execute(f"SELECT * FROM archfiles WHERE ingest_id={ingest_id};") 122 | 123 | rows = cur.fetchall() 124 | 125 | return rows 126 | 127 | 128 | def get_ingest_history(): 129 | """Get the full ingest history for the archive 130 | 131 | :return: a table representing ingest history 132 | :rtype: list 133 | """ 134 | conn = create_connection() 135 | cur = conn.cursor() 136 | cur.execute("SELECT * FROM ingest_history") 137 | 138 | rows = cur.fetchall() 139 | return rows 140 | 141 | 142 | def get_total_archive_area_size(area="archive"): 143 | """Get the size of a archive area in bytes 144 | 145 | :param area: the area in the archive to get for which to get the size, defaults to "archive" 146 | :type area: str, optional 147 | :return: size of the archive area in bytes 148 | :rtype: float 149 | """ 150 | from pathlib import Path 151 | 152 | area_map = { 153 | 'archive': Path('/srv/telemetry/archive/data/tlm'), 154 | 'staging': Path(get_env_variable('STAGING_DIRECTORY')), 155 | } 156 | 157 | root_directory = area_map[area] 158 | 159 | return sum(f.stat().st_size for f in root_directory.glob('**/*') if f.is_file()) 160 | 161 | 162 | def is_in_archive(msid): 163 | """Checks if a msid has been ingested into the archive 164 | 165 | :param msid: the msid to check 166 | :type msid: str 167 | :return: True if an msid has had data recorded as ingested 168 | :rtype: bool 169 | """ 170 | with open(msid_files['colnames'].abs, 'rb') as f: 171 | colnames = list(pickle.load(f)) 172 | return msid in colnames 173 | -------------------------------------------------------------------------------- /legacy/ingest_h5_archive.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | import time 3 | import tables 4 | import pyfits 5 | import numpy as np 6 | import Ska.Table 7 | import asciitable 8 | import re 9 | import os 10 | import sys 11 | import glob 12 | import cPickle as pickle 13 | 14 | import Ska.engarchive.converters as converters 15 | 16 | def make_h5_col_file(dat, content, colname, n_rows): 17 | """Make a new h5 table to hold column from ``dat``.""" 18 | filename = os.path.join('data', content, colname + '.h5') 19 | if os.path.exists(filename): 20 | os.unlink(filename) 21 | filedir = os.path.dirname(filename) 22 | if not os.path.exists(filedir): 23 | os.makedirs(filedir) 24 | 25 | filters = tables.Filters(complevel=5, complib='zlib') 26 | h5 = tables.openFile(filename, mode='w', filters=filters) 27 | 28 | col = dat[colname] 29 | h5shape = (0,) + col.shape[1:] 30 | h5type = tables.Atom.from_dtype(col.dtype) 31 | h5.createEArray(h5.root, 'data', h5type, h5shape, title=colname, 32 | expectedrows=n_rows) 33 | h5.createEArray(h5.root, 'quality', tables.BoolAtom(), (0,), title='Quality', 34 | expectedrows=n_rows) 35 | print 'Made', colname, 'shape=', h5shape, 'with n_rows(1e6) =', n_rows / 1.0e6 36 | h5.close() 37 | 38 | def append_h5_col(dats, content, colname): 39 | def i_colname(dat): 40 | return list(dat.dtype.names).index(colname) 41 | filename = os.path.join('data', content, colname + '.h5') 42 | h5 = tables.openFile(filename, mode='a') 43 | newdata = np.hstack([x[colname] for x in dats]) 44 | print 'Appending to', colname, 'with shape', newdata.shape 45 | h5.root.data.append(newdata) 46 | h5.root.quality.append(np.hstack([x['QUALITY'][:,i_colname(x)] for x in dats])) 47 | h5.close() 48 | 49 | filetypes = asciitable.read('Ska/engarchive/filetypes.dat') 50 | if len(sys.argv) >= 2: 51 | filetypes = filetypes[filetypes['content'] == sys.argv[1].upper()] 52 | outroot = sys.argv[2] if len(sys.argv) >= 3 else '/data/cosmos2/eng_archive/tlm' 53 | 54 | for filetype in filetypes: 55 | content = filetype['content'].lower() 56 | fitsdir = os.path.abspath(os.path.join(outroot, content)) 57 | 58 | if os.path.exists(os.path.join('data', content)): 59 | print "Skipping", filetype 60 | continue 61 | print filetype 62 | 63 | # If files are already in the final cxc archive location: 64 | # fitsfiles = sorted(glob.glob('/data/cosmos2/eng_archive/data/acisdeahk/arch/????/???/*.fits.gz')) 65 | fitsfiles = sorted(glob.glob(os.path.join(fitsdir, filetype['fileglob']))) 66 | if not fitsfiles: 67 | print 'No files' 68 | continue 69 | 70 | dat = Ska.Table.read_fits_table(fitsfiles[-1]) 71 | dat = converters.convert(dat, filetype['content']) 72 | dt = np.median(dat['TIME'][1:] - dat['TIME'][:-1]) 73 | print 'dt=',dt 74 | n_rows = int(86400 * 365 * 12 / dt) 75 | colnames = set(dat.dtype.names) 76 | colnames_all = set(dat.dtype.names) 77 | for colname in colnames_all: 78 | if len(dat[colname].shape) > 1: 79 | print 'Removing column', colname 80 | colnames.remove(colname) 81 | for colname in colnames: 82 | make_h5_col_file(dat, content, colname, n_rows) 83 | 84 | headers = dict() 85 | max_size = 1e8 86 | dats_size = 0 87 | dats = [] 88 | row = 0 89 | 90 | # fitsfiles = fitsfiles[:50] # for testing a few 91 | for i, f in enumerate(fitsfiles): 92 | print 'Reading', i, len(fitsfiles), f 93 | sys.stdout.flush() 94 | hdus = pyfits.open(f) 95 | hdu = hdus[1] 96 | try: 97 | dat = converters.convert(hdu.data, filetype['content']) 98 | except converters.NoValidDataError: 99 | print 'WARNING: skipping because of no valid data' 100 | continue 101 | header = dict((x, hdu.header[x]) for x in hdu.header.keys() if not re.match(r'T.+\d+', x)) 102 | header['row0'] = row 103 | header['row1'] = row + len(dat) 104 | if dat['QUALITY'].shape[1] != len(dat.dtype.names): 105 | print 'WARNING: skipping because of quality size mismatch: ', \ 106 | dat['QUALITY'].shape[1], len(dat.dtype.names) 107 | hdus.close() 108 | continue 109 | headers[os.path.basename(f)] = header 110 | dats.append(dat) 111 | del hdu 112 | hdus.close() 113 | 114 | colnames_all.update(dat.dtype.names) 115 | colnames.intersection_update(dat.dtype.names) 116 | 117 | row += len(dat) 118 | dats_size += dat.itemsize * len(dat) 119 | if dats_size > max_size or i == len(fitsfiles) - 1: 120 | print 'Writing accumulated column data to h5 file at', time.ctime() 121 | sys.stdout.flush() 122 | for colname in colnames: 123 | append_h5_col(dats, content, colname) 124 | sys.stdout.flush() 125 | print 126 | dats = [] 127 | dats_size = 0 128 | 129 | # Not really necessary to write each time, but helps in case of problems 130 | print 'Writing pickle files' 131 | pickle.dump(headers, open(os.path.join('data', content, 'headers.pickle'), 'w'), protocol=2) 132 | pickle.dump(colnames, open(os.path.join('data', content, 'colnames.pickle'), 'w')) 133 | pickle.dump(colnames_all, open(os.path.join('data', content, 'colnames_all.pickle'), 'w')) 134 | print 'Done writing pickle files' 135 | sys.stdout.flush() 136 | -------------------------------------------------------------------------------- /legacy/NOTES.add_content: -------------------------------------------------------------------------------- 1 | ********************************************************************************* 2 | Log in to a machine with at least 16 Gb RAM and get into ska environment 3 | ********************************************************************************* 4 | 5 | ssh kadi 6 | cd ~/git/eng_archive 7 | ln -s /export/tom/tmp/eng_archive data # some local disk 8 | 9 | # Do the following in general, but NOT if just updating stats data. 10 | # Fetch ends up seeing the >= 2000 archfiles.db and it cannot fetch 11 | # 1999 data properly. 12 | mkdir 1999; cd 1999; ln -s ../data data; cd .. 13 | 14 | ska 15 | export ENG_ARCHIVE=$PWD 16 | export CONTENT= # Upper case 17 | 18 | In order to use a local install of pyfits 2.3, which is *much* faster, do: 19 | 20 | export PYTHONPATH=$PWD/local/lib/python2.7/site-packages 21 | 22 | Note that for adding new content the trick of soft links for the 1999 data is used. 23 | The archive update always goes to the ./data (non-1999) directory but fetch queries 24 | for pre-2000 data always look in 1999. It's a little hacky but works. 25 | 26 | ********************************************************************************* 27 | Add a new content type to eng_archive 28 | ********************************************************************************* 29 | 30 | * Add content definition to filestypes.dat. The CONTENT value must be upper case 31 | 32 | ********************************************************************************* 33 | * Get FITS archive files from 1999:001 through 2000:001 34 | ********************************************************************************* 35 | ./update_archive.py --date-now=2000:001 --date-start=1999:210 --create --content=$CONTENT 36 | 37 | This puts files into ./data. In --create mode there is no limit on file gaps. 38 | 39 | ********************************************************************************* 40 | * Test in ska env within git/eng_archive 41 | ********************************************************************************* 42 | 43 | ipython --pylab 44 | impska 45 | dat = fetch.MSID(, '1999:300', '1999:310') # etc 46 | dat.iplot() 47 | 48 | ********************************************************************************* 49 | * Update/create the stats files in ska 50 | ********************************************************************************* 51 | ./update_archive.py --no-full --content $CONTENT --max-lookback-time 1e20 52 | 53 | Probably worth checking the stats files quickly before going forward. 54 | 55 | ********************************************************************************* 56 | * Bring archive up to current date 57 | ********************************************************************************* 58 | ./update_archive.py --content $CONTENT --date-start 2000:010 > $CONTENT.log 2>&1 & 59 | 60 | ********************************************************************************* 61 | * Move FITS files into archive 62 | ********************************************************************************* 63 | python move_fits_archive.py $CONTENT 64 | 65 | ********************************************************************************* 66 | * Copy h5 files into Ska 67 | ********************************************************************************* 68 | cp -rp data/$CONTENT $SKA/data/eng_archive/data/ 69 | 70 | ********************************************************************************* 71 | * Update version and install new package 72 | ********************************************************************************* 73 | 74 | - Update Ska/engarchive/version.py 75 | 76 | python setup.py sdist 77 | cp -p dist/...tar.gz $SKA/pkgs/ 78 | 79 | - Update skare and install 80 | 81 | =========================================== 82 | DONE 83 | =========================================== 84 | 85 | -------Appendix ------------------------------------------------------------------- 86 | 87 | * Old-style initial ingest 88 | 89 | ********************************************************************************* 90 | * Get FITS archive files from 1999:001 through 2000:001 91 | ********************************************************************************* 92 | export TMP_FITS=/export/tom/tmp/eng_archive/tlm/ # some local disk 93 | python ingest_fits_archive.py $CONTENT $TMP_FITS 94 | 95 | This puts files into $TMP_FITS/$CONTENT 96 | 97 | ********************************************************************************* 98 | * Ingest to a local H5 repo for $CONTENT 99 | ********************************************************************************* 100 | python ingest_h5_archive.py $CONTENT $TMP_FITS 101 | 102 | This puts files into data/ which should be a link to a local disk. 103 | 104 | ********************************************************************************* 105 | * Generate sql archfiles file 106 | ********************************************************************************* 107 | python convert_headers.py $CONTENT 108 | 109 | ********************************************************************************* 110 | * Fix file overlaps. Do a dry run perhaps. 111 | ********************************************************************************* 112 | python fix_ingest_h5.py $CONTENT --dry-run 113 | python fix_ingest_h5.py $CONTENT 114 | 115 | # For EPHEM content (and others in future??) use the --overwrite-partial 116 | # flag to make it overwrite overlapping rows instead of tossing the 117 | # entire archive file: 118 | python fix_ingest_h5.py $CONTENT --overwrite-partial 119 | 120 | =========================================== 121 | OLD-STYLE INITIAL INGEST (END) 122 | =========================================== 123 | --------------------------------------------------------------------------------