├── docs ├── index.rst ├── Makefile ├── make.bat └── conf.py ├── MOSFIRE ├── Util.py ├── __init__.py ├── Filters.py ├── Detector.py ├── conftest.py ├── Options.py ├── numerixenv.py ├── Bspline.py ├── MosfireDrpLog.py ├── data │ └── 10March2011.4.972.db ├── _astropy_init.py ├── Combine.py └── Longslit.py ├── bin ├── pyraf │ └── clcache.v2 │ │ ├── Q0FDSEVfVkVSU0lPTg== │ │ ├── +n4asH1RYaX40IlzoqR5MA== │ │ ├── 1M-t5BQ+Q2lQZPqGKbzgaA== │ │ ├── 2FuqbO+l6P0oA1+If5D3OQ== │ │ ├── FpTX5U4EkQ2vHAHvjuJx0w== │ │ ├── JtgwqzI6SyCPW1kMaqMW9Q== │ │ ├── Ni7iGHlF4UuCULtdITuLaA== │ │ ├── TRaS+P5nIkRvfxx2S7IFwQ== │ │ ├── bY6loQiPQIiojsAywxUb0Q== │ │ ├── cqbeZc2AqKZwRM66IV3kjg== │ │ ├── jPA0cvnBqGQ0t3hgB2QZWg== │ │ ├── m-a-9w9PJIR0BceaUeenYA== │ │ ├── mNap4ryaca97pYawSxgNZg== │ │ ├── nMvjMrY8i0-GKbpP9rZEYA== │ │ ├── rA35OdKgMipLZ-qtNAUAbg== │ │ ├── vd5To6yC7aNdoCIY4Gah9A== │ │ └── yNHTc+V20A3B892Qp3Uirg== └── mospy_test~ ├── manual ├── docs │ ├── eq1.png │ ├── eq2.png │ ├── image1.png │ ├── image2.png │ ├── image3.png │ ├── image4.png │ ├── image5.png │ ├── image6.png │ ├── image7.png │ ├── image8.png │ ├── image9.png │ ├── extract1.png │ ├── extract2.png │ ├── image10.png │ ├── image11.png │ ├── image12.png │ ├── image13.png │ ├── MOSFIRE_DRP_Manual.pdf │ ├── generate_pdf │ ├── index.md │ ├── hints.md │ ├── preface.md │ ├── manual.md │ ├── headercomments.md │ ├── rectify.md │ ├── retrieve.md │ ├── longslit.md │ ├── autodriver.md │ ├── handle.md │ ├── installing.md │ ├── extract.md │ ├── flats.md │ ├── installing_2016.md │ ├── driver.md │ ├── changes.md │ ├── background.md │ ├── wavelengthK.md │ ├── example.md │ ├── long2pos.md │ ├── installing_2015A.md │ └── wavelengthYJH.md └── mkdocs.yml ├── .gitmodules ├── mospy.yaml ├── README.md ├── MANIFEST.in ├── setup.cfg ├── scripts ├── mospy ├── mospy_what.py └── mospy_handle.py ├── .gitignore ├── apps ├── mospy~ ├── show_img.py ├── mospy ├── what.py ├── mospy_anaconda ├── plot_scale.py ├── audit.py ├── CSU_plot_confirm.py ├── CSU_Check.py ├── handle.py~ ├── handle.py └── check_CSU_pos.py ├── drivers ├── Driver.py ├── Longslit_driver.py ├── K_driver.py ├── Long2pos.py └── Long2pos_K_driver.py ├── platescale └── 10March2011.4.972.db └── setup.py /docs/index.rst: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MOSFIRE/Util.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /bin/pyraf/clcache.v2/Q0FDSEVfVkVSU0lPTg==: -------------------------------------------------------------------------------- 1 | Uv2q. -------------------------------------------------------------------------------- /manual/docs/eq1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/manual/docs/eq1.png -------------------------------------------------------------------------------- /manual/docs/eq2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/manual/docs/eq2.png -------------------------------------------------------------------------------- /manual/docs/image1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/manual/docs/image1.png -------------------------------------------------------------------------------- /manual/docs/image2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/manual/docs/image2.png -------------------------------------------------------------------------------- /manual/docs/image3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/manual/docs/image3.png -------------------------------------------------------------------------------- /manual/docs/image4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/manual/docs/image4.png -------------------------------------------------------------------------------- /manual/docs/image5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/manual/docs/image5.png -------------------------------------------------------------------------------- /manual/docs/image6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/manual/docs/image6.png -------------------------------------------------------------------------------- /manual/docs/image7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/manual/docs/image7.png -------------------------------------------------------------------------------- /manual/docs/image8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/manual/docs/image8.png -------------------------------------------------------------------------------- /manual/docs/image9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/manual/docs/image9.png -------------------------------------------------------------------------------- /manual/docs/extract1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/manual/docs/extract1.png -------------------------------------------------------------------------------- /manual/docs/extract2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/manual/docs/extract2.png -------------------------------------------------------------------------------- /manual/docs/image10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/manual/docs/image10.png -------------------------------------------------------------------------------- /manual/docs/image11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/manual/docs/image11.png -------------------------------------------------------------------------------- /manual/docs/image12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/manual/docs/image12.png -------------------------------------------------------------------------------- /manual/docs/image13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/manual/docs/image13.png -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "astropy_helpers"] 2 | path = astropy_helpers 3 | url = http://github.com/astropy/astropy-helpers.git 4 | -------------------------------------------------------------------------------- /manual/docs/MOSFIRE_DRP_Manual.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/manual/docs/MOSFIRE_DRP_Manual.pdf -------------------------------------------------------------------------------- /bin/pyraf/clcache.v2/+n4asH1RYaX40IlzoqR5MA==: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/bin/pyraf/clcache.v2/+n4asH1RYaX40IlzoqR5MA== -------------------------------------------------------------------------------- /bin/pyraf/clcache.v2/1M-t5BQ+Q2lQZPqGKbzgaA==: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/bin/pyraf/clcache.v2/1M-t5BQ+Q2lQZPqGKbzgaA== -------------------------------------------------------------------------------- /bin/pyraf/clcache.v2/2FuqbO+l6P0oA1+If5D3OQ==: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/bin/pyraf/clcache.v2/2FuqbO+l6P0oA1+If5D3OQ== -------------------------------------------------------------------------------- /bin/pyraf/clcache.v2/FpTX5U4EkQ2vHAHvjuJx0w==: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/bin/pyraf/clcache.v2/FpTX5U4EkQ2vHAHvjuJx0w== -------------------------------------------------------------------------------- /bin/pyraf/clcache.v2/JtgwqzI6SyCPW1kMaqMW9Q==: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/bin/pyraf/clcache.v2/JtgwqzI6SyCPW1kMaqMW9Q== -------------------------------------------------------------------------------- /bin/pyraf/clcache.v2/Ni7iGHlF4UuCULtdITuLaA==: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/bin/pyraf/clcache.v2/Ni7iGHlF4UuCULtdITuLaA== -------------------------------------------------------------------------------- /bin/pyraf/clcache.v2/TRaS+P5nIkRvfxx2S7IFwQ==: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/bin/pyraf/clcache.v2/TRaS+P5nIkRvfxx2S7IFwQ== -------------------------------------------------------------------------------- /bin/pyraf/clcache.v2/bY6loQiPQIiojsAywxUb0Q==: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/bin/pyraf/clcache.v2/bY6loQiPQIiojsAywxUb0Q== -------------------------------------------------------------------------------- /bin/pyraf/clcache.v2/cqbeZc2AqKZwRM66IV3kjg==: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/bin/pyraf/clcache.v2/cqbeZc2AqKZwRM66IV3kjg== -------------------------------------------------------------------------------- /bin/pyraf/clcache.v2/jPA0cvnBqGQ0t3hgB2QZWg==: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/bin/pyraf/clcache.v2/jPA0cvnBqGQ0t3hgB2QZWg== -------------------------------------------------------------------------------- /bin/pyraf/clcache.v2/m-a-9w9PJIR0BceaUeenYA==: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/bin/pyraf/clcache.v2/m-a-9w9PJIR0BceaUeenYA== -------------------------------------------------------------------------------- /bin/pyraf/clcache.v2/mNap4ryaca97pYawSxgNZg==: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/bin/pyraf/clcache.v2/mNap4ryaca97pYawSxgNZg== -------------------------------------------------------------------------------- /bin/pyraf/clcache.v2/nMvjMrY8i0-GKbpP9rZEYA==: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/bin/pyraf/clcache.v2/nMvjMrY8i0-GKbpP9rZEYA== -------------------------------------------------------------------------------- /bin/pyraf/clcache.v2/rA35OdKgMipLZ-qtNAUAbg==: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/bin/pyraf/clcache.v2/rA35OdKgMipLZ-qtNAUAbg== -------------------------------------------------------------------------------- /bin/pyraf/clcache.v2/vd5To6yC7aNdoCIY4Gah9A==: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/bin/pyraf/clcache.v2/vd5To6yC7aNdoCIY4Gah9A== -------------------------------------------------------------------------------- /bin/pyraf/clcache.v2/yNHTc+V20A3B892Qp3Uirg==: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Keck-DataReductionPipelines/MosfireDRP/HEAD/bin/pyraf/clcache.v2/yNHTc+V20A3B892Qp3Uirg== -------------------------------------------------------------------------------- /MOSFIRE/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | MOSFIRE Data Reduction Pipeline and instrument control software 3 | ''' 4 | 5 | 6 | ''' Version number must be a float''' 7 | __version__ = 0.11 8 | -------------------------------------------------------------------------------- /mospy.yaml: -------------------------------------------------------------------------------- 1 | name: mospy 2 | channels: 3 | - conda-forge 4 | - defaults 5 | dependencies: 6 | - ccdproc=1.3.0 7 | - numpy=1.13.3 8 | - scipy=1.0.0 9 | - python=3.6.3 10 | - astropy=2.0.3 11 | - pyqt=5.6.0 12 | - ipython=6.2.1 13 | 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | MosfireDRP 2 | ========== 3 | This is the central repository for the MOSFIRE DRP. 4 | 5 | 6 | Installation 7 | ============ 8 | 9 | Recommended installation instructions are located in the manual at https://keck-datareductionpipelines.github.io/MosfireDRP/installing/ -------------------------------------------------------------------------------- /manual/docs/generate_pdf: -------------------------------------------------------------------------------- 1 | pandoc -o MOSFIRE_DRP_Manual.pdf manual.md changes.md preface.md installing.md retrieve.md handle.md autodriver.md driver.md flats.md wavelengthYJH.md wavelengthK.md background.md rectify.md extract.md long2pos.md longslit.md headercomments.md hints.md example.md 2 | -------------------------------------------------------------------------------- /MOSFIRE/Filters.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ''' 5 | MOSFIRE Filter information 6 | 7 | [YJHK]hpp : the half power points of filters in micron 8 | 9 | ''' 10 | 11 | hpp = { 12 | 'Y': [9750.0, 11240.0], 13 | 'J': [11530.0, 13520.0], 14 | 'H': [14500.0, 18225.0], 15 | 'K': [19210.0, 24040.0] 16 | } 17 | 18 | -------------------------------------------------------------------------------- /MOSFIRE/Detector.py: -------------------------------------------------------------------------------- 1 | ''' 2 | MOSFIRE Detector Code 3 | 4 | File contains values associated with the Hawaii-2RG detector. 5 | 6 | pixelsize of 18 micron is provided by Teledyne. 7 | 8 | ''' 9 | 10 | import numpy as np 11 | 12 | mm = 1.0 13 | 14 | pixelsize = 0.018 * mm 15 | npix = (2048, 2048) 16 | 17 | gain = 2.15 # From MOSFIRE pre ship review pg. 125 18 | RN = 21.0 19 | 20 | 21 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | include CHANGES.rst 3 | 4 | include ez_setup.py 5 | include ah_bootstrap.py 6 | include setup.cfg 7 | 8 | recursive-include *.pyx *.c *.pxd 9 | recursive-include docs * 10 | recursive-include licenses * 11 | recursive-include cextern * 12 | recursive-include scripts * 13 | 14 | prune build 15 | prune docs/_build 16 | prune docs/api 17 | 18 | recursive-include astropy_helpers * 19 | exclude astropy_helpers/.git 20 | exclude astropy_helpers/.gitignore 21 | 22 | global-exclude *.pyc *.o 23 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [build_sphinx] 2 | source-dir = docs 3 | build-dir = docs/_build 4 | all_files = 1 5 | 6 | [upload_docs] 7 | upload-dir = docs/_build/html 8 | show-response = 1 9 | 10 | [pytest] 11 | minversion = 2.2 12 | norecursedirs = build docs/_build 13 | 14 | [ah_bootstrap] 15 | auto_use = True 16 | 17 | [metadata] 18 | package_name = MOSFIRE 19 | description = MOSFIRE data reduction pipeline 20 | long_description = MOSFIRE data reduction pipeline 21 | author = MOSFIRE 22 | author_email = mosfiredrp@gmail.com 23 | license = BSD 24 | url = http://mosfire-datareductionpipeline.github.io/MosfireDRP/ 25 | edit_on_github = False 26 | github_project = Mosfire-DataReductionPipeline/MosfireDRP 27 | -------------------------------------------------------------------------------- /manual/docs/index.md: -------------------------------------------------------------------------------- 1 | # MOSFIRE DRP 2 | 3 | This is the central repository for the MOSFIRE DRP originally developed by N. Konidaris and C. Steidel at Caltech, and currently hosted at the W.M. Keck Observatory. 4 | 5 | If you need help with the pipeline or to report a problem, please visit our [issue tracking page](https://github.com/Keck-DataReductionPipelines/MosfireDRP/issues) hosted at GitHub. Please note that you need a free GitHub account to submit a ticket. 6 | 7 | The currently released installation and reduction instructions are provided in the [DRP manual](manual). 8 | 9 | The support team currently includes: Luca Rizzi, Josh Walawender, Jim Lyke, Marc Kassis (W. M. Keck Observatory), Leo Alcorn (Texas A&M University), Chuck Steidel (Caltech), and Tuan Do (UCLA). 10 | -------------------------------------------------------------------------------- /MOSFIRE/conftest.py: -------------------------------------------------------------------------------- 1 | # this contains imports plugins that configure py.test for astropy tests. 2 | # by importing them here in conftest.py they are discoverable by py.test 3 | # no matter how it is invoked within the source tree. 4 | 5 | from astropy.tests.pytest_plugins import * 6 | 7 | ## Uncomment the following line to treat all DeprecationWarnings as 8 | ## exceptions 9 | #enable_deprecations_as_exceptions() 10 | 11 | ## Uncomment and customize the following lines to add/remove entries 12 | ## from the list of packages for which version numbers are displayed 13 | ## when running the tests 14 | # try: 15 | # PYTEST_HEADER_MODULES['scikit-image'] = 'skimage' 16 | # del PYTEST_HEADER_MODULES['h5py'] 17 | # except NameError: # needed to support Astropy < 1.0 18 | # pass 19 | -------------------------------------------------------------------------------- /manual/docs/hints.md: -------------------------------------------------------------------------------- 1 | # Hints 2 | 3 | ## Pay attention to the wavelength fitting output 4 | 5 | ![Screenshot](image10.png "The output above shows that up to pixel 1015 the RMS was 0.27 Angstrom level, and then dramatically jumped to 60 angstrom. Look at the image and examine pixel 1016, figure out what happened. You may have to adjust your input files or remove a file from the set.") 6 | 7 | The output above shows that up to pixel 1015 the RMS was 0.27 Angstrom level, and then dramatically jumped to 60 angstrom. Look at the image and examine pixel 1016, figure out what happened. You may have to adjust your input files or remove a file from the set. 8 | 9 | ## Look at rectified_wave_stack files 10 | 11 | Look at rectified_wave_stack* files and make sure the night sky lines are vertical on the detector. 12 | -------------------------------------------------------------------------------- /scripts/mospy: -------------------------------------------------------------------------------- 1 | #!/bin/csh -f 2 | 3 | # INSTRCUCTIONS. Edit the 4 | 5 | if ($#argv == 0) then 6 | echo "Starting MOSFIRE Python" 7 | ipython --colors Linux 8 | else 9 | switch ( $argv[1] ) 10 | case "libs": 11 | python -c "import sys; print sys.path" 12 | breaksw 13 | 14 | case "db": 15 | mospy_db.py $* 16 | breaksw 17 | 18 | case "handle": 19 | mospy_handle.py $* 20 | breaksw 21 | 22 | case "AutoDriver": 23 | AutoDriver.py 24 | breaksw 25 | 26 | case "what": 27 | mospy_what.py $* 28 | breaksw 29 | 30 | default 31 | python $* 32 | breaksw 33 | endsw 34 | endif 35 | 36 | onintr - 37 | 38 | 39 | -------------------------------------------------------------------------------- /manual/docs/preface.md: -------------------------------------------------------------------------------- 1 | # Preface 2 | 3 | This manual describes the installation and usage of the MOSFIRE data reduction pipeline on a unix-like computer. Although primarily tested and developed on a Mac, the pipeline operates on both OSX and Linux systems. In the section 4, we describe an installation procedure for a Mac OSX system. Later sections describe code usage, execution, and outputs. 4 | 5 | The MOSFIRE spectrograph data reduction pipeline was architected by the MOSFIRE commissioning team and written by Nick Konidaris with extensive checking and feedback from Chuck Steidel and other MOSFIRE team members. The pipeline is maintained on an online code repository [https://github.com/Keck-DataReductionPipelines/MosfireDRP](https://github.com/Keck-DataReductionPipelines/MosfireDRP). Please use this website to track issues and and submit requests. 6 | -------------------------------------------------------------------------------- /MOSFIRE/Options.py: -------------------------------------------------------------------------------- 1 | ''' 2 | =================== 3 | MOSFIRE Options 4 | =================== 5 | 6 | ''' 7 | 8 | import getpass 9 | import os 10 | __version__ = '2014.06.10' 11 | 12 | npix = 2048 13 | path_bpm = os.path.join(os.path.dirname(__file__), "data", "badpix_10sep2012.fits") 14 | 15 | flat = { 16 | "version": 1, 17 | "edge-order": 4, # Polynomial order for edge of slit 18 | "edge-fit-width": 20, 19 | "flat-field-order": 7 # Order of polynomial for fitting the flat field profile 20 | } 21 | 22 | wavelength = { 23 | "datadir" : os.path.join(os.path.dirname(__file__), "data"), 24 | "version": 2, 25 | "fractional-wavelength-search": 0.99935, # used in determining oned wavelength solutions 26 | "chebyshev-degree": 5, # polynomial order for fitting wavelengths 27 | } 28 | 29 | -------------------------------------------------------------------------------- /MOSFIRE/numerixenv.py: -------------------------------------------------------------------------------- 1 | from __future__ import division # confidence high 2 | import os 3 | 4 | def check_input(xxx): 5 | """Check if input is a Numarray Array.""" 6 | try: 7 | import numarray 8 | return isinstance(xxx,numarray.numarraycore.NumArray) 9 | except ImportError: 10 | pass 11 | 12 | def check(): 13 | """Check for running numarray version of pyfits with numpy code.""" 14 | try: 15 | import pyfits 16 | if pyfits.__version__ < '1.1': 17 | raise EnvironmentError("Pyfits 1.1 or later required, pyfits version %s detected\n" % pyfits.__version__) 18 | except ImportError: 19 | pass 20 | try: 21 | if os.environ['NUMERIX'] == 'numarray': 22 | raise EnvironmentError("NUMERIX/numarray environment detected; numpy environment required") 23 | except KeyError: 24 | pass 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | bin/pyraf/clcache.v2/* 2 | 3 | REDUX 4 | DATA 5 | *.pyc 6 | 7 | 8 | # Compiled files 9 | *.py[co] 10 | *.a 11 | *.o 12 | *.so 13 | __pycache__ 14 | 15 | # Ignore .c files by default to avoid including generated code. If you want to 16 | # add a non-generated .c extension, use `git add -f filename.c`. 17 | *.c 18 | 19 | # Other generated files 20 | */version.py 21 | */cython_version.py 22 | htmlcov 23 | .coverage 24 | MANIFEST 25 | 26 | # Sphinx 27 | docs/api 28 | docs/_build 29 | 30 | # Eclipse editor project files 31 | .project 32 | .pydevproject 33 | .settings 34 | 35 | # Pycharm editor project files 36 | .idea 37 | 38 | 39 | # Packages/installer info 40 | *.egg 41 | *.egg-info 42 | dist 43 | build 44 | eggs 45 | parts 46 | bin 47 | var 48 | sdist 49 | develop-eggs 50 | .installed.cfg 51 | distribute-*.tar.gz 52 | 53 | # Other 54 | .*.swp 55 | *~ 56 | 57 | # Mac OSX 58 | .DS_Store 59 | 60 | /.project 61 | 62 | site/ -------------------------------------------------------------------------------- /manual/mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: MOSFIRE DRP Documentation 2 | theme: readthedocs 3 | pages: 4 | - 'Home': index.md 5 | - 'DRP Manual': 6 | - 'Contents': manual.md 7 | - 'Changes': changes.md 8 | - 'Preface': preface.md 9 | - 'Installing': installing.md 10 | - 'Retrieve Your Data': retrieve.md 11 | - 'Handle': handle.md 12 | - 'Automatic Generation of the Driver File': autodriver.md 13 | - 'The driver.py File': driver.md 14 | - 'Flats': flats.md 15 | - 'Wavelength Calibration – Y, J, and H Bands': wavelengthYJH.md 16 | - 'Wavelength Calibration – K Band': wavelengthK.md 17 | - 'Background Subtraction': background.md 18 | - 'Rectify': rectify.md 19 | - 'Spectral Extraction': extract.md 20 | - 'Long2pos Reductions': long2pos.md 21 | - 'Longslit Reductions': longslit.md 22 | - 'A Word About Header Comments': headercomments.md 23 | - 'Some Hints': hints.md 24 | - 'Examples of Running the Pipeline': example.md 25 | -------------------------------------------------------------------------------- /manual/docs/manual.md: -------------------------------------------------------------------------------- 1 | # Table of Contents 2 | 3 | 1. [Latest Changes](changes) 4 | 2. [Preface](preface) 5 | 3. [Installing the Data Reduction Pipeline](installing) 6 | 4. [Retrieve Your Data](retrieve) 7 | 5. [DRP First Reduction Step: Handle](handle) 8 | 6. [Automatic Generation of the Driver File](autodriver) 9 | 7. [The driver.py File](driver) 10 | 8. [Flats](flats) 11 | 9. [Wavelength Calibration – Y, J, and H Bands](wavelengthYJH) 12 | 10. [Wavelength Calibration – K Band](wavelengthK) 13 | 11. [Background Subtraction](background) 14 | 12. [Rectify](rectify) 15 | 13. [Spectral Extraction](extract) 16 | 14. [Long2pos Reductions](long2pos) 17 | 15. [Longslit Reductions](longslit) 18 | 16. [A Word About Header Comments](headercomments) 19 | 17. [Some Hints](hints) 20 | 18. [Examples](example) 21 | 22 | The manual is also available in pdf form: [MOSFIRE_DRP_Manual.pdf](MOSFIRE_DRP_Manual.pdf) 23 | 24 |
25 | Note: pdf version of the manual is generated from the individual markdown files for each section by running the `generate_pdf` script in the `docs/` subdirectory of this repository. 26 |
27 | -------------------------------------------------------------------------------- /MOSFIRE/Bspline.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Convenience functions for Bspline 3 | ''' 4 | 5 | import numpy as np 6 | try: 7 | from astropy.io import fits as pf 8 | except: 9 | import pyfits as pf 10 | import CSU 11 | 12 | import unittest 13 | 14 | def array_to_tuple(A): 15 | 16 | xs = [] 17 | ys = [] 18 | zs = [] 19 | 20 | for x in np.arange(A.shape[0]): 21 | for y in np.arange(A.shape[1]): 22 | xs.append(x) 23 | ys.append(y) 24 | zs.append(A[x,y]) 25 | 26 | return list(map(np.array, (xs, ys, zs))) 27 | 28 | 29 | 30 | class TestBsplineFunctions(unittest.TestCase): 31 | 32 | def setUp(self): 33 | pass 34 | 35 | def test_array_to_tuple(self): 36 | sa = self.assertTrue 37 | 38 | A = np.array([[1,2], [3,4]]) 39 | 40 | (xs,ys,zs) = array_to_tuple(A) 41 | sa(zs[0] == 1) 42 | sa(zs[1] == 2) 43 | sa(zs[2] == 3) 44 | sa(zs[3] == 4) 45 | 46 | for cnt in range(len(xs)): 47 | i = xs[cnt] 48 | j = ys[cnt] 49 | sa(A[i, j] == zs[cnt]) 50 | cnt += 1 51 | 52 | if __name__ == '__main__': 53 | unittest.main() 54 | 55 | -------------------------------------------------------------------------------- /manual/docs/headercomments.md: -------------------------------------------------------------------------------- 1 | # Header Comments 2 | 3 | Files produced by the DRP have a series of information in the FITS header that helps users determine the pedigree of files involved in the reduction. Since many files go into reductions, FITS headers are enormous and some documentation about them is useful. 4 | 5 | The derived product FITS headers are organized as follows. The header of the first science file in the ‘A’ frame goes directly into the header. As the rest of the ‘A’ frames go into the header, the new keyword is checked against the current header. If the value of the keyword is different, a new keyword is added with the key postpended by _img### where ### is the file number. A special keyword called imfno### is created showing the full path to the file in the data reduction set. An example is shown below: 6 | 7 | ![Screenshot](image9.png "ds9 output of the FITS header. Note that the first "A" frame file is located in /scr2/mosfire/2013nov26/m131126_0135.fits. The second file (#137) has a similar path. The keywords which follow from file #137 have different values than those in file #135 and are thus named KEY_img###.") 8 | 9 | ds9 output of the FITS header. Note that the first "A" frame file is located in /scr2/mosfire/2013nov26/m131126_0135.fits. The second file (#137) has a similar path. The keywords which follow from file #137 have different values than those in file #135 and are thus named KEY_img###. 10 | -------------------------------------------------------------------------------- /bin/mospy_test~: -------------------------------------------------------------------------------- 1 | #!/bin/csh -f 2 | 3 | # INSTRCUCTIONS. Edit the 4 | 5 | # 1. Update the full path to the Ureka install for the two aliases 6 | # below 7 | alias ur_setup 'eval `/home/npk/.ureka/ur_setup -csh \!*`' 8 | alias ur_forget 'eval `/home/npk/.ureka/ur_forget -csh \!*`' 9 | 10 | 11 | # 2. Set MOSPATH to the full path to the MOSFIRE DRP code repository 12 | # example: 13 | # Change /scr2/mosfire/DRP/mosfire to /Users/myname/MOSFIRE/DRP 14 | setenv MOSPATH /scr2/mosfire/DRP/mosfire 15 | if (! $?PYTHONPATH ) setenv PYTHONPATH ${PYTHONPATH}:${MOSPATH} 16 | 17 | if (! -d $MOSPATH) then 18 | echo "Path '$MOSPATH' does not exist. Modify $0" 19 | echo "and change MOSPATH to the DRP directory." 20 | exit 1 21 | endif 22 | 23 | ur_setup 24 | 25 | if ($#argv == 0) then 26 | echo "Starting MOSFIRE Python" 27 | ipython --pylab --colors Linux 28 | else 29 | switch ( $argv[1] ) 30 | case "libs": 31 | python -c "import sys; print sys.path" 32 | breaksw 33 | 34 | case "db": 35 | python ${MOSPATH}/apps/db.py $* 36 | breaksw 37 | 38 | case "handle": 39 | python ${MOSPATH}/apps/handle.py $* 40 | breaksw 41 | 42 | case "what": 43 | python ${MOSPATH}/apps/what.py $* 44 | breaksw 45 | 46 | default 47 | python $* 48 | breaksw 49 | endsw 50 | endif 51 | 52 | onintr - 53 | 54 | 55 | -------------------------------------------------------------------------------- /apps/mospy~: -------------------------------------------------------------------------------- 1 | #!/bin/csh -f 2 | 3 | # INSTRCUCTIONS. Edit the 4 | 5 | # 1. Update the full path to the Ureka install for the two aliases 6 | # below 7 | alias ur_setup 'eval `/home/npk/.ureka/ur_setup -csh \!*`' 8 | alias ur_forget 'eval `/home/npk/.ureka/ur_forget -csh \!*`' 9 | 10 | # If pythonpath is not previously defined, define it so that 11 | # the setenv works below.. 12 | if (! $?PYTHONPATH ) setenv PYTHONPATH 13 | 14 | # 2. Set MOSPATH to the full path to the MOSFIRE DRP code repository 15 | # example: 16 | # Change /scr2/mosfire/DRP/mosfire to /Users/myname/MOSFIRE/DRP 17 | setenv MOSPATH /scr2/mosfire/DRP/mosfire 18 | setenv PYTHONPATH ${PYTHONPATH}:${MOSPATH} 19 | 20 | if (! -d $MOSPATH) then 21 | echo "Path '$MOSPATH' does not exist. Modify $0" 22 | echo "and change MOSPATH to the DRP directory." 23 | exit 1 24 | endif 25 | 26 | ur_setup 27 | 28 | if ($#argv == 0) then 29 | echo "Starting MOSFIRE Python" 30 | ipython --pylab --colors Linux 31 | else 32 | switch ( $argv[1] ) 33 | case "libs": 34 | python -c "import sys; print sys.path" 35 | breaksw 36 | 37 | case "db": 38 | python ${MOSPATH}/apps/db.py $* 39 | breaksw 40 | 41 | case "handle": 42 | python ${MOSPATH}/apps/handle.py $* 43 | breaksw 44 | 45 | case "what": 46 | python ${MOSPATH}/apps/what.py $* 47 | breaksw 48 | 49 | default 50 | python $* 51 | breaksw 52 | endsw 53 | endif 54 | 55 | onintr - 56 | 57 | 58 | -------------------------------------------------------------------------------- /apps/show_img.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | ''' 4 | Written March 15th 2011 by npk 5 | ''' 6 | 7 | 8 | import os, numpy, scipy, sys 9 | path = "/users/npk/desktop/c9/" 10 | def generate_fname(num): 11 | global path 12 | files = os.listdir(path) 13 | for fn in files: 14 | if fn.find("12_%4.4i" % num) > 0: 15 | return fn 16 | if fn.find("13_%4.4i" % num) > 0: 17 | return fn 18 | if fn.find("14_%4.4i" % num) > 0: 19 | return fn 20 | if fn.find("15_%4.4i" % num) > 0: 21 | return fn 22 | if fn.find("16_%4.4i" % num) > 0: 23 | return fn 24 | if fn.find("17_%4.4i" % num) > 0: 25 | return fn 26 | if fn.find("18_%4.4i" % num) > 0: 27 | return fn 28 | if fn.find("19_%4.4i" % num) > 0: 29 | return fn 30 | if fn.find("20_%4.4i" % num) > 0: 31 | return fn 32 | if fn.find("21_%4.4i" % num) > 0: 33 | return fn 34 | if fn.find("24_%4.4i" % num) > 0: 35 | return fn 36 | 37 | 38 | 39 | fn = generate_fname(int(sys.argv[1])) 40 | print fn 41 | 42 | regpath = "/users/npk/dropbox/mosfire/cooldown\ 9/csu_meas/" 43 | os.system("ds9 %s -regions %s -regions %s" %(path+fn, regpath+fn+".reg", regpath+fn+".meas.reg")) 44 | #os.system("ds9 %s -regions %s" %(path+fn, regpath+fn+".reg")) 45 | 46 | -------------------------------------------------------------------------------- /manual/docs/rectify.md: -------------------------------------------------------------------------------- 1 | # Rectify 2 | 3 | The next step in the reduction process is to combine the wavelength solution with the backgroun subtracted images and then shift and combine the nod positions. If reducing Kband, be sure to use the merged wave_stack solution. To do this we uncomment the following lines in the Driver.py file: 4 | 5 | redfiles = ["eps_" + file + ".fits" for file in obsfiles] 6 | Rectify.handle_rectification(maskname, redfiles, 7 | 'lambda_solution_wave_stack_J_m130114_0443-0445.fits', 8 | band, 9 | waveops) 10 | 11 | The output from this procedure produces four files for every slit. 12 | 13 | | Filename | Content (units) | 14 | |----------------------------------|-----------------------------------------------------------------------------| 15 | | `[maskname]_[band]_[object]_eps.fits` | Signal () | 16 | | `[maskname]_[band]_[object]_itime.fits` | Integration time | 17 | | `[maskname]_[band]_[object]_sig.fits` | Variance ? | 18 | | `[maskname]_[band]_{object}_snrs.fits` | Signal to noise () | 19 | 20 | There is also four images without the “object” in the name. These four files contain the composit spectra with all spectra aligned spectrally and both beams combined. In the *eps.fits files, you will see two negative traces and one positive trace. For a two position nod, the eps files is (A-B) +((B-A)shifted). 21 | 22 | When extracting the emission from an object or measuring the position of an emission line, you should be accessing the *eps.fits files with the wavelength solution written into the WCS information. 23 | 24 | -------------------------------------------------------------------------------- /MOSFIRE/MosfireDrpLog.py: -------------------------------------------------------------------------------- 1 | import logging, sys 2 | 3 | LOG_FILENAME = "mosfire_DRP.log" 4 | 5 | 6 | formatter = logging.Formatter('%(asctime)s - %(module)12s.%(funcName)20s - %(levelname)s: %(message)s') 7 | # set up logging to STDOUT for all levels DEBUG and higher 8 | sh = logging.StreamHandler(sys.stdout) 9 | sh.setLevel(logging.INFO) 10 | sh.setFormatter(formatter) 11 | 12 | # set up logging to a file for all levels DEBUG and higher 13 | fh = logging.FileHandler(LOG_FILENAME) 14 | fh.setLevel(logging.DEBUG) 15 | fh.setFormatter(formatter) 16 | 17 | # create Logger object 18 | mylogger = logging.getLogger('MyLogger') 19 | mylogger.setLevel(logging.DEBUG) 20 | mylogger.addHandler(sh) # enabled: stdout 21 | mylogger.addHandler(fh) # enabled: file 22 | 23 | 24 | # create shortcut functions 25 | debug = mylogger.debug 26 | info = mylogger.info 27 | warning = mylogger.warning 28 | error = mylogger.error 29 | critical = mylogger.critical 30 | 31 | 32 | #logger.info ("log started") 33 | 34 | # Add log entries for versions of numpy, matplotlib, astropy, ccdproc 35 | info(sys.version) 36 | info('python version = {}.{}.{}'.format(sys.version_info.major, 37 | sys.version_info.minor, 38 | sys.version_info.micro)) 39 | import numpy as np 40 | info('numpy version = {}'.format(np.__version__)) 41 | import matplotlib 42 | info('matplotlib version = {}'.format(matplotlib.__version__)) 43 | import astropy 44 | info('astropy version = {}'.format(astropy.__version__)) 45 | import ccdproc 46 | info('ccdproc version = {}'.format(ccdproc.__version__)) 47 | -------------------------------------------------------------------------------- /manual/docs/retrieve.md: -------------------------------------------------------------------------------- 1 | # Retrieve 2 | 3 | Before running the drp, you will need a set of spectroscopic data to reduce that includes flats, science observations, and if the observations are K-band, arcs and thermal flats. NOTE: You must preserve Keck’s file naming convention as the DRP uses the file name to parse data sets. The standard naming convention is `mYYMMDD_####.fits`. 4 | 5 | If you need to retrieve your data, you may either use a secure copy (scp) assumine your data is still accessible from the Keck/MOSFIRE data directory (contact your SA if you need assistance) or use KOA –the Keck Observatory Archive to navigate to the KOA log in page. From there, KOA has forms where you specify the data to retrieve and will create a tar ball for you to download. 6 | 7 | A useful tool is the file translator script that will convert your KOA file names to the standard filenames as they were written to disk during your observing session (koa_translator). Again, your filenames must preserve the standard naming convention and the koa_translator script does this for you. 8 | 9 | If you do not have data of your own and wish to download the example: 10 | Grab the data from: [https://www2.keck.hawaii.edu/realpublic/inst/mosfire/DRP_Test_Case_Hband.zip](https://www2.keck.hawaii.edu/realpublic/inst/mosfire/DRP_Test_Case_Hband.zip). 11 | 12 | Move the test case zip file to a directory of your choice (we will assume `~/Data` for the examples in this manual) and unzip the DRP test case file: 13 | 14 | unzip DRP_Test_Case_Hband.zip 15 | 16 | This will create a `DRP_Test_Case_Hband` subdirectory under your current directory which will contain the raw data which you can use to follow along in subsequent steps of the DRP manual. 17 | 18 | -------------------------------------------------------------------------------- /drivers/Driver.py: -------------------------------------------------------------------------------- 1 | import os, time, logging 2 | import MOSFIRE 3 | from MOSFIRE import Background, Combine, Detector, Flats, IO, Options, Rectify, Wavelength 4 | from MOSFIRE.MosfireDrpLog import info, debug, warning, error 5 | logger = logging.getLogger(__name__) 6 | import numpy as np 7 | from matplotlib import pyplot as pl 8 | try: 9 | from astropy.io import fits as pf 10 | except: 11 | import pyfits as pf 12 | np.seterr(all='ignore') 13 | flatops = Options.flat 14 | waveops = Options.wavelength 15 | 16 | #Driver file automatically generated on Sat Jul 25 17:33:42 2015 17 | #For questions and comments, email mosfiredrp@gmail.com, submit a ticket on the ticketing system, or contact Luca Rizzi @ WMKO 18 | 19 | maskname = 'maskname' 20 | band = 'band' 21 | 22 | #Set noninteractive to True to autofit wavelenth solution instead of manually fitting. 23 | noninteractiveflag=False 24 | obsfiles=['Offset_1.25.txt','Offset_-1.25.txt'] 25 | 26 | Flats.handle_flats('Flat.txt', maskname, band, flatops) 27 | 28 | Wavelength.imcombine(obsfiles, maskname, band, waveops) 29 | Wavelength.fit_lambda_interactively(maskname, band, obsfiles,waveops, noninteractive=noninteractiveflag) 30 | Wavelength.fit_lambda(maskname, band, obsfiles, obsfiles,waveops) 31 | Wavelength.apply_lambda_simple(maskname, band, obsfiles, waveops) 32 | 33 | # modify this variable to point to the correct wavelength file created on the previous step 34 | Wavelength_file = 'lambda_solution_wave_stack_H_m141130_0323-0338.fits' 35 | 36 | Background.handle_background(obsfiles,Wavelength_file,maskname,band,waveops) 37 | 38 | redfiles = ["eps_" + file + ".fits" for file in obsfiles] 39 | Rectify.handle_rectification(maskname, redfiles,Wavelength_file,band,obsfiles,waveops) 40 | 41 | -------------------------------------------------------------------------------- /apps/mospy: -------------------------------------------------------------------------------- 1 | #!/bin/csh -f 2 | 3 | # INSTRCUCTIONS. Edit the 4 | 5 | # 1. Update the full path to the Ureka install for the two aliases 6 | # below 7 | alias ur_setup 'eval `/Users/lrizzi/.ureka/ur_setup -csh \!*`' 8 | alias ur_forget 'eval `/Users/lrizzi/.ureka/ur_forget -csh \!*`' 9 | 10 | 11 | 12 | # If pythonpath is not previously defined, define it so that 13 | # the setenv works below.. 14 | if (! $?PYTHONPATH ) setenv PYTHONPATH 15 | 16 | 17 | # 2. Set MOSPATH to the full path to the MOSFIRE DRP code repository 18 | # example: 19 | # Change /scr2/mosfire/DRP/mosfire to /Users/myname/MOSFIRE/DRP 20 | setenv MOSPATH /Volumes/PromiseRAID/MOSFIRE/DRP_CODE 21 | setenv PYTHONPATH ${PYTHONPATH}:${MOSPATH} 22 | 23 | if (! -d $MOSPATH) then 24 | echo "Path '$MOSPATH' does not exist. Modify $0" 25 | echo "and change MOSPATH to the DRP directory." 26 | exit 1 27 | endif 28 | 29 | ur_setup common primary 30 | 31 | if ($#argv == 0) then 32 | echo "Starting MOSFIRE Python" 33 | ipython --colors Linux 34 | else 35 | switch ( $argv[1] ) 36 | case "libs": 37 | python -c "import sys; print sys.path" 38 | breaksw 39 | 40 | case "db": 41 | python ${MOSPATH}/apps/db.py $* 42 | breaksw 43 | 44 | case "handle": 45 | python ${MOSPATH}/apps/handle.py $* 46 | breaksw 47 | 48 | case "AutoDriver": 49 | python ${MOSPATH}/apps/AutoDriver.py $* 50 | breaksw 51 | 52 | case "what": 53 | python ${MOSPATH}/apps/what.py $* 54 | breaksw 55 | 56 | default 57 | python $* 58 | breaksw 59 | endsw 60 | endif 61 | 62 | onintr - 63 | 64 | 65 | -------------------------------------------------------------------------------- /apps/what.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python 2 | 3 | ''' 4 | MOSFIRE 'what' command: 5 | Spits out an informative summary of files 6 | in the current directory. Or files selected 7 | via a glob. 8 | 9 | i.e. what *0311.fits 10 | 11 | npk - March 23 2011 12 | ''' 13 | 14 | import MOSFIRE 15 | import MOSFIRE.IO 16 | import glob 17 | import sys 18 | 19 | 20 | 21 | files = [] 22 | if len(sys.argv) == 1: 23 | files = glob.iglob('*') 24 | else: 25 | for i in range(1, len(sys.argv)): 26 | files.extend(glob.iglob(sys.argv[i])) 27 | 28 | 29 | 30 | #print("filename object exptime maskname lamp filt Turret") 31 | for fname in files: 32 | 33 | try: 34 | header = MOSFIRE.IO.readheader(fname) 35 | except IOError, err: 36 | print("Couldn't IO %s" % fname) 37 | continue 38 | except: 39 | print("%s is unreadable" % fname) 40 | continue 41 | 42 | lamps = "" 43 | try: 44 | if header["pwstata7"] == 1: 45 | lamps += header["pwloca7"][0:2] 46 | if header["pwstata8"] == 1: 47 | lamps += header["pwloca8"][0:2] 48 | except KeyError: 49 | lamps = "???" 50 | 51 | header.update("lamps", lamps) 52 | 53 | try: 54 | if header["aborted"]: 55 | header.update("object", "ABORTED") 56 | except: 57 | print("Missing header file in: %s" % fname) 58 | 59 | try: 60 | print("%(datafile)12s %(object)40s %(truitime)6.1f s %(maskname)35s %(lamps)3s %(filter)4s %(mgtname)7s" % (header)) 61 | except: 62 | try: 63 | print("%(datafile)12s %(object)25s %(truitime)6.1f s %(lamps)3s %(filter)6s %(mgtname)7s" % (header)) 64 | except: 65 | print("%s Skipped" % fname) 66 | 67 | -------------------------------------------------------------------------------- /scripts/mospy_what.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python 2 | 3 | ''' 4 | MOSFIRE 'what' command: 5 | Spits out an informative summary of files 6 | in the current directory. Or files selected 7 | via a glob. 8 | 9 | i.e. what *0311.fits 10 | 11 | npk - March 23 2011 12 | ''' 13 | 14 | import MOSFIRE 15 | import MOSFIRE.IO 16 | import glob 17 | import sys 18 | 19 | 20 | 21 | files = [] 22 | if len(sys.argv) == 1: 23 | files = glob.iglob('*') 24 | else: 25 | for i in range(1, len(sys.argv)): 26 | files.extend(glob.iglob(sys.argv[i])) 27 | 28 | 29 | 30 | #print("filename object exptime maskname lamp filt Turret") 31 | for fname in files: 32 | 33 | try: 34 | header = MOSFIRE.IO.readheader(fname) 35 | except IOError, err: 36 | print("Couldn't IO %s" % fname) 37 | continue 38 | except: 39 | print("%s is unreadable" % fname) 40 | continue 41 | 42 | lamps = "" 43 | try: 44 | if header["pwstata7"] == 1: 45 | lamps += header["pwloca7"][0:2] 46 | if header["pwstata8"] == 1: 47 | lamps += header["pwloca8"][0:2] 48 | except KeyError: 49 | lamps = "???" 50 | 51 | header.update("lamps", lamps) 52 | 53 | try: 54 | if header["aborted"]: 55 | header.update("object", "ABORTED") 56 | except: 57 | print("Missing header file in: %s" % fname) 58 | 59 | try: 60 | print("%(datafile)12s %(object)40s %(truitime)6.1f s %(maskname)35s %(lamps)3s %(filter)4s %(mgtname)7s" % (header)) 61 | except: 62 | try: 63 | print("%(datafile)12s %(object)25s %(truitime)6.1f s %(lamps)3s %(filter)6s %(mgtname)7s" % (header)) 64 | except: 65 | print("%s Skipped" % fname) 66 | 67 | -------------------------------------------------------------------------------- /apps/mospy_anaconda: -------------------------------------------------------------------------------- 1 | #!/bin/csh -f 2 | 3 | # INSTRCUCTIONS. Edit the 4 | 5 | # 1. Update the full path to the Ureka install for the two aliases 6 | # below 7 | #alias ur_setup 'eval `/Users/lrizzi/.ureka/ur_setup -csh \!*`' 8 | #alias ur_forget 'eval `/Users/lrizzi/.ureka/ur_forget -csh \!*`' 9 | 10 | 11 | 12 | # If pythonpath is not previously defined, define it so that 13 | # the setenv works below.. 14 | if (! $?PYTHONPATH ) setenv PYTHONPATH 15 | 16 | 17 | # 2. Set MOSPATH to the full path to the MOSFIRE DRP code repository 18 | # example: 19 | # Change /scr2/mosfire/DRP/mosfire to /Users/myname/MOSFIRE/DRP 20 | setenv MOSPATH /Users/lrizzi/MosfireDRP 21 | setenv PYTHONPATH ${PYTHONPATH}:${MOSPATH} 22 | 23 | if (! -d $MOSPATH) then 24 | echo "Path '$MOSPATH' does not exist. Modify $0" 25 | echo "and change MOSPATH to the DRP directory." 26 | exit 1 27 | endif 28 | 29 | setenv MYPYTHON "/Users/lrizzi/anaconda/bin/python" 30 | setenv MYIPYTHON "/Users/lrizzi/anaconda/bin/ipython" 31 | 32 | if ($#argv == 0) then 33 | echo "Starting MOSFIRE Python" 34 | $MYIPYTHON --pylab --colors Linux 35 | else 36 | switch ( $argv[1] ) 37 | case "libs": 38 | $MYPYTHON -c "import sys; print sys.path" 39 | breaksw 40 | 41 | case "db": 42 | $MYPYTHON ${MOSPATH}/apps/db.py $* 43 | breaksw 44 | 45 | case "handle": 46 | $MYPYTHON ${MOSPATH}/apps/handle.py $* 47 | breaksw 48 | 49 | case "AutoDriver": 50 | $MYPYTHON ${MOSPATH}/apps/AutoDriver.py $* 51 | breaksw 52 | 53 | case "what": 54 | $MYPYTHON ${MOSPATH}/apps/what.py $* 55 | breaksw 56 | 57 | default 58 | $MYPYTHON $* 59 | breaksw 60 | endsw 61 | endif 62 | 63 | onintr - 64 | 65 | 66 | -------------------------------------------------------------------------------- /apps/plot_scale.py: -------------------------------------------------------------------------------- 1 | 2 | from MOSFIRE import CSU 3 | from matplotlib import pyplot as pl 4 | import numpy as np 5 | from numpy import sin, cos 6 | 7 | reload(CSU) 8 | 9 | pl.ion() 10 | 11 | pl.figure(1) 12 | pl.clf() 13 | pl.xlim([-100,2200]) 14 | pl.ylim([-200,2200]) 15 | 16 | dxs = [] 17 | for s in range(4): 18 | for x in range(0,281,280): 19 | p1 = CSU.csu_mm_to_pix_poly(x, s) 20 | p2 = CSU.csu_mm_to_pix(x,s,Please_Use=True) 21 | 22 | dxs.append(p1[0]-p2[0]) 23 | 24 | dx = p1[0]-p2[0] 25 | dy = p1[1]-p2[1] 26 | scale=50 27 | pl.arrow(p1[0], p1[1], dx*scale, dy*scale) 28 | if s == 0: 29 | pl.axvline(p2[0],color='black',ls='-.',lw=.5) 30 | 31 | 32 | 33 | pl.figure(2) 34 | pl.clf() 35 | 36 | 37 | try: 38 | f = open("../platescale/finalshift.xyXrotYrot.4.972.120k.dat") 39 | lines = f.readlines() 40 | f.close() 41 | except: 42 | print "Failed to read file" 43 | 44 | 45 | pl.figure(3) 46 | pl.clf() 47 | pl.xlim([-100,2200]) 48 | pl.ylim([-100,2200]) 49 | for line in lines: 50 | [px,py,xmm,ymm]= map(np.float,line.split()) 51 | 52 | px_lin = 7.670726 * xmm 53 | py_lin = 7.670726 * ymm 54 | 55 | d = np.radians(360-359.762) 56 | px_lin = cos(d) * px_lin - sin(d) * py_lin 57 | py_lin = sin(d) * px_lin + cos(d) * py_lin 58 | 59 | px_lin += 1042.986 60 | py_lin += 1035.879 61 | 62 | dx = px-px_lin 63 | dy = py-py_lin 64 | 65 | scale =200 66 | print "%3.2f %3.2f" % (dx, dy) 67 | pl.arrow(px_lin, py_lin, scale*dx, scale*dy) 68 | pl.plot([px_lin],[py_lin],'ok') 69 | 70 | pl.arrow(100,1800,scale/2.,0) 71 | pl.title("Distortion Map: ccs solution - linear solution") 72 | -------------------------------------------------------------------------------- /apps/audit.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import time 4 | import traceback 5 | import getpass 6 | import os 7 | import pdb 8 | import pprint 9 | import sets 10 | import sqlite3 11 | import sys 12 | import textwrap 13 | 14 | import numpy as np 15 | from matplotlib import pyplot as pl 16 | 17 | from operator import itemgetter 18 | from itertools import groupby 19 | 20 | import MOSFIRE 21 | 22 | from MOSFIRE import Options, IO, Wavelength 23 | 24 | 25 | def audit(filename): 26 | 27 | header, data = IO.readfits(filename) 28 | 29 | 30 | ll0 = header['crval1'] 31 | dlam = header['cd1_1'] 32 | 33 | ls = ll0 + dlam * np.arange(data.shape[1]) 34 | 35 | linelist = Wavelength.pick_linelist(header) 36 | 37 | 38 | deltas = [] 39 | sigs = [] 40 | xpos = [] 41 | ys = [] 42 | for y in np.linspace(750, 1100, 30): 43 | #for y in np.linspace(5, 640, 50): 44 | sp = np.ma.array(data[y,:]) 45 | 46 | xs, sxs, sigmas = Wavelength.find_known_lines(linelist, ls, sp, 47 | Options.wavelength) 48 | 49 | xpos.append(xs) 50 | ys.append([y] * len(xs)) 51 | 52 | deltas.append(xs - (linelist - ll0)/dlam) 53 | sigs.append(sxs) 54 | 55 | 56 | xpos, ys, deltas, sigs = map(np.array, [xpos, ys, deltas, sigs]) 57 | 58 | deltas[np.abs(deltas) > .75] = np.nan 59 | sigs[np.abs(sigs) > .001] = np.nan 60 | 61 | pl.clf() 62 | size = 0.003/sigs 63 | size[size > 30] = 30 64 | size[size < 1] = 1 65 | pl.scatter( xpos, ys, c=deltas, s=size) 66 | pl.xlim([0, data.shape[1]]) 67 | pl.ylim([0, data.shape[0]]) 68 | pl.xlabel("Spectral pixel") 69 | pl.ylabel("Spatial pixel") 70 | pl.title("Night sky line deviation from solution [pixel]") 71 | pl.colorbar() 72 | 73 | pl.savefig("audit.pdf") 74 | 75 | pdb.set_trace() 76 | 77 | 78 | def usage(): 79 | print(""" 80 | audit [filename] 81 | Commands: """) 82 | 83 | print("\n") 84 | 85 | if __name__ == '__main__': 86 | 87 | if len(sys.argv) != 3: 88 | usage() 89 | sys.exit() 90 | 91 | audit(sys.argv[-1]) 92 | -------------------------------------------------------------------------------- /drivers/Longslit_driver.py: -------------------------------------------------------------------------------- 1 | import os, time, logging 2 | import MOSFIRE 3 | from MOSFIRE import Background, Combine, Detector, Flats, IO, Options, Rectify, Wavelength 4 | from MOSFIRE.MosfireDrpLog import info, debug, warning, error 5 | logger = logging.getLogger(__name__) 6 | import numpy as np 7 | from matplotlib import pyplot as pl 8 | try: 9 | from astropy.io import fits as pf 10 | except: 11 | import pyfits as pf 12 | np.seterr(all='ignore') 13 | flatops = Options.flat 14 | waveops = Options.wavelength 15 | 16 | #Driver file automatically generated on Wed Jul 29 15:11:22 2015 17 | #For questions and comments, email mosfiredrp@gmail.com, submit a ticket on the ticketing system, or contact Luca Rizzi @ WMKO 18 | 19 | maskname = 'LONGSLIT-3x0.7' 20 | band = 'H' 21 | 22 | #Set noninteractive to True to autofit wavelenth solution instead of manually fitting. 23 | noninteractiveflag=False 24 | # modify the target name to match your observations 25 | obsfiles=['Offset_5_HIP17971.txt','Offset_-5_HIP17971.txt'] 26 | target="HIP17971" 27 | 28 | 29 | # modify the yrange to match the size of your longslit 30 | # row position is the extraction line used for the initial wavelength solution. It should be away from your target 31 | longslit = {'yrange':[968,1100],'row_position':1034,'mode':'longslit'} 32 | Flats.handle_flats('Flat.txt', maskname, band, flatops,longslit=longslit) 33 | 34 | Wavelength.imcombine(obsfiles, maskname, band, waveops) 35 | Wavelength.fit_lambda_interactively(maskname, band, obsfiles,waveops,longslit=longslit, noninteractive=noninteractiveflag) 36 | Wavelength.fit_lambda(maskname, band, obsfiles, obsfiles,waveops,longslit=longslit) 37 | Wavelength.apply_lambda_simple(maskname, band, obsfiles, waveops,longslit=longslit) 38 | 39 | # make sure you use the file generated on the previous step 40 | Wavelength_file = 'lambda_solution_wave_stack_H_m121227_0162-0311.fits' 41 | 42 | Background.handle_background(obsfiles,Wavelength_file,maskname,band,waveops,target=target) 43 | 44 | redfiles = ["eps_" + file + ".fits" for file in obsfiles] 45 | Rectify.handle_rectification(maskname, redfiles,Wavelength_file,band,obsfiles,waveops, target=target) 46 | 47 | -------------------------------------------------------------------------------- /manual/docs/longslit.md: -------------------------------------------------------------------------------- 1 | # Longslit Reductions 2 | 3 | The longslit reductions require transferring the Longslit_Driver.py file into the reduction directory. A few key parameters have to be adjusted in Longslit_Driver.py to help the pipeline figure out where to extract the longslit from. 4 | 5 | * `cd /path/to/LONGSLIT/` 6 | * `cp ~/MosfireDRP-master/drivers/Longslit_driver.py .` 7 | * Check all the .txt files to make sure your observations are included. You may have to merge files from various LONGSLIT* directories. This happens when your observations use a shorter longslit than the calibrations. 8 | * Note that mospy handle generates separate offset files for each of your targets, using the target name, but does NOT separate repeated observations of the same targets at different times of the night. 9 | * edit `Driver_Longslit.py` 10 | * Examine a longslit image (see figure below) and adjust 'yrange': [709, 1350] to the vertical range covered by the slit 11 | * From the same examined longslit, select ‘row_position’ so that it is uncontaminated by the spectrum. See Figure 1. 12 | * make sure that ‘mode’:’longslit’ is specified in the longslit variable 13 | * The result should look like Figure 2. 14 | * For each step in a section, uncomment the necessary line and run mosdrp on the Driver file. Once the apply_lambda_simple step is complete, fill in the 'lambda_solution_wave_stack_...' line with the correct wave stack file. 15 | 16 | You now have two options based on the results. If the night sky lines are not bright enough to identify in the interactive step you should use arclamps. In the following instructions, replace wavefiles with ‘Ne.txt’ or ‘Ar.txt’ and specify neon=True or argon=True. 17 | 18 | ![Screenshot](image7.png "An example of an uncontaminated row (#1127) in the longslit.") 19 | 20 | An example of an uncontaminated row (#1127) in the longslit. 21 | 22 | ![Screenshot](image8.png "Example of a modified Driver_Longslit.py. Notice that pixel 991 is selected as the row to perform the initial wavelength solution on. In Figure 2, this is the equivalent of 1127.") 23 | 24 | Example of a modified Driver_Longslit.py. Notice that pixel 991 is selected as the row to perform the initial wavelength solution on. In Figure 2, this is the equivalent of 1127. -------------------------------------------------------------------------------- /manual/docs/autodriver.md: -------------------------------------------------------------------------------- 1 | # AutoDriver 2 | 3 | The pipeline is able to produce a driver file automatically for most cases, thus removing the need to copy one of the standard files and manually edit it. 4 | 5 | To generate the driver file, go to the directory where your Offset files live, and where the reduction is going to happen and type: 6 | 7 | mospy AutoDriver 8 | 9 | This will generate a file called Driver.py, which you should inspect before running it. Highly specialized cases such as particular combinations of sky lines and arcs might not be dealt with correctly. Note that the automatic generation of the driver file works for long2pos and longslit as well. 10 | 11 | To handle special cases, the automatic generation of the driver file makes a number of assumptions, that might not be correct for your science case. 12 | 13 | 1. If either Ar.txt or Ne.txt or both are available, they are being used. 14 | 2. If the band is K, and FlatThermal.txtis available, it is used 15 | 3. For long2pos: if no arcs are available, only specphot in non spectrophotometric mode can be reduced and the pipeline will attempt to use the sky lines. Note that is likely to fail, as sky lines are too faint in short exposures for an accurate wavelength determination. The spectrophotometric mode contains wide slits that cannot be reduced using sky lines only. 16 | 4. In longslit, the pipeline will try to determine the size of the slit using the mask name. For example, if the maskname is LONGSLIT-3x0.7, the pipeline assumes that you have used 3 slits to generate the longslit and that they are centered around the middle line of the detector. 17 | 5. If any of the mandatory observations are missing (such as the flat fields), the pipeline will still generate a Driver.py file, but it will contain warnings about the missing files, and it will NOT run correctly. 18 | 6. If multiple observations of different stars are done in long2pos or in longslit mode, the pipeline will generate multiple driver files, one for each object. If the same object is observed multiple times during the same night, all the observations will end up in the same driver file. If you are observing a telluric standard at different times and you need to have separate spectra, you need to manually create Offset files and Driver files. 19 | -------------------------------------------------------------------------------- /drivers/K_driver.py: -------------------------------------------------------------------------------- 1 | import os, time, logging 2 | import MOSFIRE 3 | from MOSFIRE import Background, Combine, Detector, Flats, IO, Options, Rectify, Wavelength 4 | from MOSFIRE.MosfireDrpLog import info, debug, warning, error 5 | logger = logging.getLogger(__name__) 6 | import numpy as np 7 | from matplotlib import pyplot as pl 8 | try: 9 | from astropy.io import fits as pf 10 | except: 11 | import pyfits as pf 12 | np.seterr(all='ignore') 13 | flatops = Options.flat 14 | waveops = Options.wavelength 15 | 16 | #Driver file automatically generated on Sat Jul 25 17:46:43 2015 17 | #For questions and comments, email mosfiredrp@gmail.com, submit a ticket on the ticketing system, or contact Luca Rizzi @ WMKO 18 | 19 | maskname = 'maskname' 20 | band = 'band' 21 | 22 | #Set noninteractive to True to autofit wavelenth solution instead of manually fitting. 23 | noninteractiveflag=False 24 | obsfiles=['Offset_1.25.txt','Offset_-1.25.txt'] 25 | 26 | Flats.handle_flats('Flat.txt', maskname, band, flatops,lampOffList='FlatThermal.txt') 27 | 28 | Wavelength.imcombine(obsfiles, maskname, band, waveops) 29 | # if you have Ar 30 | Wavelength.imcombine('Ar.txt', maskname, band, waveops) 31 | # if you have Ne 32 | Wavelength.imcombine('Ne.txt', maskname, band, waveops) 33 | Wavelength.fit_lambda_interactively(maskname, band, obsfiles,waveops, noninteractive=noninteractiveflag) 34 | 35 | Wavelength.apply_interactive(maskname, band, waveops, apply=obsfiles, to='Ar.txt', argon=True) 36 | Wavelength.apply_interactive(maskname, band, waveops, apply=obsfiles, to='Ne.txt', neon=True) 37 | 38 | Wavelength.fit_lambda(maskname, band, obsfiles, obsfiles,waveops) 39 | Wavelength.fit_lambda(maskname, band, 'Ne.txt', 'Ne.txt',waveops, wavenames2='Ar.txt') 40 | LROI = [[21000,22800]]*1 41 | LROIs = Wavelength.check_wavelength_roi(maskname, band, obsfiles, 'Ne.txt', LROI, waveops) 42 | 43 | Wavelength.apply_lambda_simple(maskname, band, obsfiles, waveops) 44 | Wavelength.apply_lambda_sky_and_arc(maskname, band, obsfiles, 'Ne.txt', LROIs, waveops) 45 | 46 | Wavelength_file = 'merged_lambda_solution_wave_stack_K_*.fits' 47 | 48 | Background.handle_background(obsfiles,Wavelength_file,maskname,band,waveops) 49 | 50 | redfiles = ["eps_" + file + ".fits" for file in obsfiles] 51 | Rectify.handle_rectification(maskname, redfiles,Wavelength_file,band,obsfiles,waveops) 52 | 53 | -------------------------------------------------------------------------------- /platescale/10March2011.4.972.db: -------------------------------------------------------------------------------- 1 | # Thu 11:33:17 10-Mar-2011 2 | begin final.mm2pix.4.972.120k 3 | xrefmean 1046.505 4 | yrefmean 1012.029 5 | xmean 0.4466459 6 | ymean -3.111027 7 | geometry general 8 | function legendre 9 | xshift -136.4909 10 | yshift -134.5126 11 | xmag 0.1303632 12 | ymag 0.1303667 13 | xrotation 0.2235465 14 | yrotation 0.2227183 15 | xrms 0.007421817 16 | yrms 0.009591535 17 | surface1 11 18 | 2. 2. 19 | 2. 2. 20 | 2. 2. 21 | 0. 0. 22 | 1. 1. 23 | 2048. 2048. 24 | 1. 1. 25 | 2048. 2048. 26 | -2.415665 -1.473981 27 | 133.4257 -0.5205792 28 | 0.518665 133.4294 29 | surface2 29 30 | 2. 2. 31 | 6. 6. 32 | 6. 6. 33 | 2. 2. 34 | 1. 1. 35 | 2048. 2048. 36 | 1. 1. 37 | 2048. 2048. 38 | 0.003701102 -0.002260678 39 | -0.06516604 0.002980762 40 | 0.01680987 -0.009382407 41 | -0.09525164 0.001952664 42 | 0.001301059 -0.006842875 43 | -0.0249715 0.001543188 44 | 0.003321849 -0.06174274 45 | 0.004812567 -0.01276854 46 | 0.006835532 -0.1938907 47 | -0.003903005 -0.01236681 48 | 0.004609621 -0.04301402 49 | 0.008481519 -0.01143874 50 | -0.2039385 0.005151763 51 | -0.003815467 -0.02138967 52 | -0.1099565 0.005565406 53 | 0.002185637 -0.09095713 54 | 0.005690682 -0.004090087 55 | 0.00830204 -0.09770827 56 | -0.005605454 -8.554335E-4 57 | -0.05965742 0.007455383 58 | 0.002974942 -0.02954251 59 | # Thu 11:39:31 10-Mar-2011 60 | begin final.pix2mm.4.972.120k 61 | xrefmean 0.4466459 62 | yrefmean -3.111027 63 | xmean 1046.505 64 | ymean 1012.029 65 | geometry general 66 | function legendre 67 | xshift 1042.986 68 | yshift 1035.879 69 | xmag 7.670879 70 | ymag 7.670669 71 | xrotation 359.7765 72 | yrotation 359.7773 73 | xrms 0.05689785 74 | yrms 0.0735955 75 | surface1 11 76 | 2. 2. 77 | 2. 2. 78 | 2. 2. 79 | 0. 0. 80 | -137. -137. 81 | 137. 137. 82 | -137. -137. 83 | 137. 137. 84 | 1042.986 1035.879 85 | 1050.902 4.100106 86 | -4.085054 1050.874 87 | surface2 29 88 | 2. 2. 89 | 6. 6. 90 | 6. 6. 91 | 2. 2. 92 | -137. -137. 93 | 137. 137. 94 | -137. -137. 95 | 137. 137. 96 | -0.003519427 0.04111919 97 | 0.6503384 -0.01270313 98 | -0.04435552 0.1140394 99 | 0.8596014 -0.004460136 100 | 0.02288385 0.06082749 101 | 0.2194757 -0.01182392 102 | -0.03575225 0.6321526 103 | 0.03145307 0.2228013 104 | -0.0738007 1.750792 105 | 0.05956091 0.1508475 106 | -0.04349946 0.3796116 107 | -0.01592333 0.1551957 108 | 1.84551 -0.02639246 109 | 0.1113619 0.225195 110 | 0.9689353 -0.04159353 111 | -0.02733423 0.8293015 112 | -0.0120037 0.07495303 113 | -0.07806052 0.8610189 114 | 0.05433064 0.0317068 115 | 0.5247607 -0.05944379 116 | -0.02807729 0.2596714 117 | -------------------------------------------------------------------------------- /MOSFIRE/data/10March2011.4.972.db: -------------------------------------------------------------------------------- 1 | # Thu 11:33:17 10-Mar-2011 2 | begin final.mm2pix.4.972.120k 3 | xrefmean 1046.505 4 | yrefmean 1012.029 5 | xmean 0.4466459 6 | ymean -3.111027 7 | geometry general 8 | function legendre 9 | xshift -136.4909 10 | yshift -134.5126 11 | xmag 0.1303632 12 | ymag 0.1303667 13 | xrotation 0.2235465 14 | yrotation 0.2227183 15 | xrms 0.007421817 16 | yrms 0.009591535 17 | surface1 11 18 | 2. 2. 19 | 2. 2. 20 | 2. 2. 21 | 0. 0. 22 | 1. 1. 23 | 2048. 2048. 24 | 1. 1. 25 | 2048. 2048. 26 | -2.415665 -1.473981 27 | 133.4257 -0.5205792 28 | 0.518665 133.4294 29 | surface2 29 30 | 2. 2. 31 | 6. 6. 32 | 6. 6. 33 | 2. 2. 34 | 1. 1. 35 | 2048. 2048. 36 | 1. 1. 37 | 2048. 2048. 38 | 0.003701102 -0.002260678 39 | -0.06516604 0.002980762 40 | 0.01680987 -0.009382407 41 | -0.09525164 0.001952664 42 | 0.001301059 -0.006842875 43 | -0.0249715 0.001543188 44 | 0.003321849 -0.06174274 45 | 0.004812567 -0.01276854 46 | 0.006835532 -0.1938907 47 | -0.003903005 -0.01236681 48 | 0.004609621 -0.04301402 49 | 0.008481519 -0.01143874 50 | -0.2039385 0.005151763 51 | -0.003815467 -0.02138967 52 | -0.1099565 0.005565406 53 | 0.002185637 -0.09095713 54 | 0.005690682 -0.004090087 55 | 0.00830204 -0.09770827 56 | -0.005605454 -8.554335E-4 57 | -0.05965742 0.007455383 58 | 0.002974942 -0.02954251 59 | # Thu 11:39:31 10-Mar-2011 60 | begin final.pix2mm.4.972.120k 61 | xrefmean 0.4466459 62 | yrefmean -3.111027 63 | xmean 1046.505 64 | ymean 1012.029 65 | geometry general 66 | function legendre 67 | xshift 1042.986 68 | yshift 1035.879 69 | xmag 7.670879 70 | ymag 7.670669 71 | xrotation 359.7765 72 | yrotation 359.7773 73 | xrms 0.05689785 74 | yrms 0.0735955 75 | surface1 11 76 | 2. 2. 77 | 2. 2. 78 | 2. 2. 79 | 0. 0. 80 | -137. -137. 81 | 137. 137. 82 | -137. -137. 83 | 137. 137. 84 | 1042.986 1035.879 85 | 1050.902 4.100106 86 | -4.085054 1050.874 87 | surface2 29 88 | 2. 2. 89 | 6. 6. 90 | 6. 6. 91 | 2. 2. 92 | -137. -137. 93 | 137. 137. 94 | -137. -137. 95 | 137. 137. 96 | -0.003519427 0.04111919 97 | 0.6503384 -0.01270313 98 | -0.04435552 0.1140394 99 | 0.8596014 -0.004460136 100 | 0.02288385 0.06082749 101 | 0.2194757 -0.01182392 102 | -0.03575225 0.6321526 103 | 0.03145307 0.2228013 104 | -0.0738007 1.750792 105 | 0.05956091 0.1508475 106 | -0.04349946 0.3796116 107 | -0.01592333 0.1551957 108 | 1.84551 -0.02639246 109 | 0.1113619 0.225195 110 | 0.9689353 -0.04159353 111 | -0.02733423 0.8293015 112 | -0.0120037 0.07495303 113 | -0.07806052 0.8610189 114 | 0.05433064 0.0317068 115 | 0.5247607 -0.05944379 116 | -0.02807729 0.2596714 117 | -------------------------------------------------------------------------------- /manual/docs/handle.md: -------------------------------------------------------------------------------- 1 | # Handle 2 | 3 | Now that you have data to reduce, we need to set up the pipeline with the appropriate files so that the drp knows what files to use in the reduction. The handle step will parses the FITS header information and determine what files are associated with each of your masks. 4 | 5 | Because the DRP no longer has a designated output directory, you will need to run handle in your designated reduction sub-directory (`reduced` in our example). 6 | 7 | mkdir reduced 8 | cd reduced 9 | mospy handle /home/[yourhomedir]/Data/DRP_Test_Case_Hband/2012sep10/*fits 10 | 11 | Please use the full path to the raw data when invoking `mospy handle`. 12 | 13 | A lot of data summarizing the observations is output. This includes a table of the observations: 14 | 15 | m130514_0132 Flat:mos Y mosmaskA 16.0 s mosmaskA Y YJ 16 | m130114_0133 Flat:mos Y mosmaskA 16.0 s mosmaskA Y YJ 17 | m130114_0134 Flat:mos Y mosmaskA 16.0 s mosmaskA Y YJ 18 | m130114_0135 Flat:mos Y mosmaskA 16.0 s mosmaskA Y YJ 19 | m130114_0136 Flat:mos Y mosmaskA 16.0 s mosmaskA Y YJ 20 | m140114_0137 Flat:mos Y mosmaskA 16.0 s mosmaskA Y YJ 21 | ... 22 | 23 | and file lists that organize the observation types: 24 | 25 | mosmaskA /2013jan14/Y/Unknown.txt 26 | mosmaskA /2013jan14/Y/Align.txt 27 | mosmaskA /2013jan14/Y/MIRA.txt 28 | mosmaskA /2013jan14/Y/Ne.txt 29 | mosmaskA /2013jan14/Y/Offset_2.txt 30 | mosmaskA /2013jan14/Y/Offset_-2.txt 31 | mosmaskA /2013jan14/Y/Flat.txt 32 | mosmaskA /2013jan14/Y/Image.txt 33 | mosmaskA /2013jan14/Y/FlatThermal.txt 34 | mosmaskA /2013jan14/Y/Dark.txt 35 | mosmaskA /2013jan14/Y/Ar.txt 36 | mosmaskA 2013jan14/Y/Aborted.txt 37 | ... 38 | 39 | The handle step creates a set of directories organized as 40 | 41 | [maskname]/[date]/[band]/ 42 | 43 | Containing 44 | 45 | * Aborted.txt: Aborted files 46 | * Align.txt: Alignment frames 47 | * Ar.txt: Argon spectra 48 | * Dark.txt: Darks 49 | * Flat.txt: Flat fields 50 | * FlatThermal.txt: Thermal Flats (lamps off) 51 | * Image.txt: Imaging mode 52 | * MIRA.txt: MIRA focus images 53 | * Ne.txt: Neon lamp spectra 54 | * Unknown.txt: Unknown files 55 | * Offset_[p].txt: Science frames 56 | 57 | The output directory structure is designed to make finding reduced data easy, and to separate reductions of the same mask across multiple dates. Below is a screen shot showing the example output from an Offset*.txt file. 58 | 59 | ![Screenshot](image1.png "Directory structure screenshot") 60 | 61 | For longslit observations, separate offsets files are created for each object, but the same offset files are used if the same object is observed many times during the night. You might want to separate the different observations of the same object. 62 | 63 | For long2pos observations, again different offset files are created for each object. Besides, the suffixes _PosA and _PosC are added to the offset files to identify the two left and right positions used in the observations. 64 | 65 | The following section describes in details how to use the driver file to control the pipeline processes, and points at a number of standard driver files that we provide for reference. 66 | 67 | 68 | 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /manual/docs/installing.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | ## Requirements 4 | 5 | The pipeline requires the following python modules: 6 | 7 | * numpy 8 | * astropy 9 | * ccdproc 10 | * scipy 11 | 12 | ## Installing Python 13 | 14 | ### Using Anaconda Cloud and Conda Environments 15 | 16 | Install Anaconda as per the instructions on the [Anaconda web site](https://www.anaconda.com/download). 17 | 18 | Now we will create a conda [environment](https://conda.io/docs/user-guide/tasks/manage-environments.html) specifically for the MOSFIRE DRP. Rather than specify everything on the command line, we will get the specification for the environment from the Anaconda Cloud service. There are two specifications, one for linux (tested on a CentOS 7 system) and one for macOS (tested on macOS 10.12.6). Get the one appropriate for your system using one of the commands below: 19 | 20 | conda env create KeckObservatory/mospy 21 | 22 | Now we will invoke that environment: 23 | 24 | source activate mospy 25 | 26 | Now we will install the DRP itself. From now on, if you want to run the DRP, first invoke the appropriate environment using `source activate mospy`. 27 | 28 | 29 | ## Download and Install the DRP 30 | 31 | Download the zip file of the released version from [GitHub](https://github.com/Keck-DataReductionPipelines/MosfireDRP/releases/download/Release2018/MosfireDRP-2018release.zip). 32 | 33 | Move the zip file to a location on your computer where you want the source code to reside, then unzip the file: 34 | 35 | unzip MosfireDRP-2018release.zip 36 | 37 | Change in to the resulting ```MosfireDRP-2018release/``` directory: 38 | 39 | cd MosfireDRP-2018release 40 | 41 | Run the install program: 42 | 43 | python setup.py install 44 | 45 | The executable `mospy` should now be in your path. If you used the Anaconda based install, it will be in the Anaconda bin directory (e.g. `~/anaconda/envs/mospy/bin/mospy`). 46 | 47 | 48 | ## Alternate Methods of Installing Python 49 | 50 | Note, these are no longer the recommended methods of installing the DRP as they do not gauranttee that the various package versions are compatible with the DRP. 51 | 52 | ### Using the Anaconda Distribution 53 | 54 | Install Anaconda as per the instructions on the [Anaconda web site](https://www.continuum.io/downloads). The pipeline currently (2016 release) only runs on python 2.7, so download and install that version, not the python 3.x version. 55 | 56 | To generate an environment similar to the one in the recommended anaconda cloud based install, you can use the following command: 57 | 58 | ``` 59 | conda create --no-default-packages -c astropy -n mospy python=3.6.3 astropy=2.0.3 ccdproc=1.3.0 ipython=6.2.1 numpy=1.13.3 scipy=1.0.0 PyQt=5.6.0 60 | ``` 61 | 62 | You should now have all the requirements to run the MOSFIRE DRP. This should work on any anaconda install, even if the pre-packaged linux and macOS environments are incompatible with your machine. 63 | 64 | ### Using Other Python Install Methods 65 | 66 | The DRP support group recommends the anaconda python install and has tested the DRP using that installer, but if an appropriate version of python is installed via some other package manager (e.g. apt-get, brew, yum, etc.), then you should be able to install the python package dependencies using either that package manager (if they are available via that package manager) or using `pip`. For example: 67 | 68 | pip install numpy 69 | pip install astropy 70 | pip install ccdproc 71 | 72 | -------------------------------------------------------------------------------- /manual/docs/extract.md: -------------------------------------------------------------------------------- 1 | # Spectral Extraction 2 | 3 | ## Interactive Spectral Extraction Instructions 4 | 5 | The final step is to extract a 1D spectrum for each object in each slit. The final line of the `Driver.py` (or equivalent) file will looks something like this: 6 | 7 | ``` 8 | Extract.extract_spectra(maskname, band, interactive=(not bypassflag)) 9 | ``` 10 | 11 | This will iterate through all the slits for this mask-band combination and if the interactive flag is set, then it will show a plot of the spatial profile of each slit (collapsed in the spectral direction). By default, the software will make a guess at one aperture for each slit. The apertures are indicated by a yellow shaded region and their center position and half width in pixels is annotated near the top of each shaded region. 12 | 13 | ![Screenshot](extract1.png "Screenshot of a spatial profile with a single aperture defined.") 14 | 15 | The apertures define the pixels which will be used as input to the optimal spectral extraction (Horne 1986) algorithm. Having wide a wide aperture should not add additional noise as that will be optimized during the spectral extraction step. The apertures are shown here in order for the user to verify 1) that there is no overlap between adjacent objects, 2) that the apertures are wide enough to reasonably encompass all flux from the object, and 3) that all objects have properly defined apertures. 16 | 17 | The user can add, modify, or delete apertures interactively using this plot window. 18 | 19 | To delete an existing aperture: place the mouse near the center of the aperture and press the "d" key. 20 | 21 | To add an aperture by fitting a gaussian to the profile: place the mouse near the peak of the profile and press the "g" key. The half width of the aperture will be set at 5 times the sigma of the fitted gaussian. 22 | 23 | ![Screenshot](extract2.png "Screenshot of a spatial profile with a second aperture added by hitting the 'g' key.") 24 | 25 | To add an aperture manually: place the mouse in the X position where the new aperture should be centered and press the "a" key. Then type the half width (in pixels) for that aperture in response to the query in the terminal. 26 | 27 | To modify the half width of an existing aperture: place the mouse near the center of the aperture and press the "w" key. Then type the half width (in pixels) for that aperture in response to the query in the terminal. 28 | 29 | To modify the center position of an existing aperture: place the mouse near the center of the aperture and press the "p" key. Then type the position (in pixels) for that aperture in response to the query in the terminal. 30 | 31 | When you are done adding or removing apertures, close the interactive plot window by clicking the close button in the upper right corner (or by whatever method is typical for your OS or windowing system) or press the "q" or "n" keys (for "quit" or "next" respectively). 32 | 33 | ## Spectral Extraction Results 34 | 35 | Whether you used the interactive tool for spectral extraction or allowed the software to automatically guess at the apertures to extract, the software will output both a FITS version of the resulting 1D spectrum and an PNG plot. 36 | 37 | These filenames will have the form: 38 | 39 | ``` 40 | [maskname]_[band]_[targetname]_[aperture].png 41 | [maskname]_[band]_[targetname]_1D_[aperture].fits 42 | ``` 43 | 44 | where `[aperture]` is a two digit integer indication which aperture for that slit this corresponds to (zero based). If the apertures were determined automatically by the software, then only one aperture will have been generated for each slit, so all files will end in `_00.png` or `_1D_00.fits`. 45 | 46 | 47 | -------------------------------------------------------------------------------- /manual/docs/flats.md: -------------------------------------------------------------------------------- 1 | # Flats 2 | 3 | The first action the driver file will take is to generate a pixel flat and slit edge tracing. To initiate the flat generation, uncomment the line below in the 4 | Driver.py file: 5 | 6 | #Flats.handle_flats('Flat.txt', maskname, band, flatops) 7 | 8 | and in your xterm run the DRP 9 | 10 | > mospy Driver.py 11 | 12 | Example output from the xterm session 13 | 14 | > mospy Driver.py 15 | ... Truncated output ... 16 | Flat written to combflat_2d_H.fits 17 | 18 | 00] Finding Slit Edges for BX113 ending at 1901. Slit composed of 3 CSU slits 19 | 01] Finding Slit Edges for BX129 ending at 1812. Slit composed of 2 CSU slits 20 | 02] Finding Slit Edges for xS15 ending at 1768. Slit composed of 1 CSU slits 21 | Skipping (wavelength pixel): 10 22 | 03] Finding Slit Edges for BX131 ending at 1680. Slit composed of 2 CSU slits 23 | 24 | The slit names output to the screen should look familiar as they originated from the mask design process. The output files from this process are the following: 25 | 26 | | Filename | Contains | 27 | |-----------------------|-----------------------------------------------------------------------------------------------| 28 | | `combflat_2d_J.fits` | FITS image of the flats | 29 | | `flatcombine.lst` | The list of files used in the creation of the flat. Contains the full path name to the files. | 30 | | `pixelflat_2d_J.fits` | FITS image of the normalized flat. This is the flat used in other redution steps. | 31 | | `slit-edges_J.npy` | File containing the slit edge information | 32 | | `slit-edges_J.reg` | DS9 regions file that may be overlayed to show the locations of the slits. | 33 | 34 | 35 | At the end, check the output in ds9. For example: 36 | 37 | > ds9 pixelflat_2d_H.fits -region slit-edges_H.reg 38 | 39 | The regions file overlayed on the pixelflat image should look something like: 40 | 41 | ![Screenshot](image2.png "ds9 screenshot") 42 | 43 | The green lines must trace the edge of the slit. If they don’t, then the flat step failed. All values should be around 1.0. There are some big features in the detector that you will become familiar with over time. 44 | 45 | ## K-band flats 46 | 47 | At K-band, the dome is hot enough that light is detected at the longest wavelengths at a level of a few hundred counts. Little to no light is seen at the shortest wavelengths. The light from the dome is not entering MOSFIRE at the same angles that the light from the spot illuminated on the dome by the dome lights. Some observers may wish to correct for this difference by subtracting the thermal flat emission from the dome flat emission before normalizing the flats. To complete this flat subtraction, you use the optional keyword lampsofflist in the flat process as seen in the command below: 48 | 49 | Flats.handle_flats('Flat.txt', maskname, band, flatops, lampOffList='FlatThermal.txt') 50 | 51 | If thermal flats were included in your calibration sequence (default behavior for K-band), then the FlatThermal.txt file should be populated with a list of thermal flats. Use FlatThermal.txt as the list or modify it as you see necessary. 52 | 53 | The outputs from the flat process will include two additional files. 54 | 55 | * combflat_lamps_off_2d_K.fits 56 | * combflat_lamps_on_2d_K.fits 57 | 58 | and now the combflat_2d_K.fits being the difference between the two files. 59 | -------------------------------------------------------------------------------- /manual/docs/installing_2016.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | ## Requirements 4 | 5 | The 2016 version of the pipeline no longer requires IRAF/PyRAF, so the installation should be simpler than previous versions. 6 | 7 | The pipeline requires the following python modules: 8 | 9 | * numpy 10 | * astropy 11 | * ccdproc 12 | * scipy 13 | 14 | ## Installing Python 15 | 16 | ### Using Anaconda Cloud and Conda Environments 17 | 18 | Install Anaconda as per the instructions on the [Anaconda web site](https://www.continuum.io/downloads). 19 | 20 | Now we will create a conda [environment](https://conda.io/docs/user-guide/tasks/manage-environments.html) specifically for the MOSFIRE DRP. Rather than specify everything on the command line, we will get the specification for the environment from the Anaconda Cloud service. There are two specifications, one for linux (tested on a CentOS 7 system) and one for macOS (tested on macOS 10.12.6). Get the one appropriate for your system using one of the commands below: 21 | 22 | conda env create KeckObservatory/mospy_2016_linux 23 | 24 | or 25 | 26 | conda env create KeckObservatory/mospy_2016_macOS 27 | 28 | Now we will invoke that environment: 29 | 30 | source activate mospy_2016_linux 31 | 32 | or 33 | 34 | source activate mospy_2016_macOS 35 | 36 | Now we will install the DRP itself. From now on, if you want to run the DRP, first invoke the appropriate environment using `source activate mospy_2016_linux` or `source activate mospy_2016_macOS`. 37 | 38 | 39 | ## Download and Install the DRP 40 | 41 | Download the zip file of the released version from [GitHub](https://github.com/Keck-DataReductionPipelines/MosfireDRP/releases/download/Release2016A/MosfireDRP-2016release.zip). 42 | 43 | Move the zip file to a location on your computer where you want the source code to reside, then unzip the file: 44 | 45 | unzip MosfireDRP-2016release.zip 46 | 47 | Change in to the resulting ```MosfireDRP-2016release/``` directory: 48 | 49 | cd MosfireDRP-2016release 50 | 51 | Run the install program: 52 | 53 | python setup.py install 54 | 55 | The executable `mospy` should now be in your path. If you used the Anaconda based install, it will be in the Anaconda bin directory (e.g. `~/anaconda2/bin/mospy`). 56 | 57 | 58 | ## Alternate Methods of Installing Python 59 | 60 | Note, these are no longer the recommended methods of installing the DRP as they do not gauranttee that the various package versions are compatible with the DRP. 61 | 62 | ### Using the Anaconda Distribution 63 | 64 | Install Anaconda as per the instructions on the [Anaconda web site](https://www.continuum.io/downloads). The pipeline currently (2016 release) only runs on python 2.7, so download and install that version, not the python 3.x version. 65 | 66 | To generate an environment similar to the one in the recommended anaconda cloud based install, you can use the following command: 67 | 68 | ``` 69 | conda create --no-default-packages -c astropy -n mospy_2016 python=2.7.13 astropy=2.0.2 ccdproc=1.3.0 ipython=5.4.1 numpy=1.11.2 scipy=0.18.1 PyQt=5.6.0 70 | ``` 71 | 72 | You should now have all the requirements to run the MOSFIRE DRP. This should work on any anaconda install, even if the pre-packaged linux and macOS environments are incompatible with your machine. 73 | 74 | ### Using Other Python Install Methods 75 | 76 | The DRP support group recommends the anaconda python install and has tested the DRP using that installer, but if an appropriate version of python (e.g. python 2.7) is installed via some other package manager (e.g. apt-get, brew, yum, etc.), then you should be able to install the python package dependencies using either that package manager (if they are available via that package manager) or using `pip`. For example: 77 | 78 | pip install numpy 79 | pip install astropy 80 | pip install ccdproc 81 | 82 | -------------------------------------------------------------------------------- /manual/docs/driver.md: -------------------------------------------------------------------------------- 1 | # The driver.py File 2 | 3 | The driver file controls all the pipeline steps, and in the drivers sub-directory, you will find a number of driver files: `Driver.py`, `K_Driver.py`, `Long2pos_driver.py`, and `Longslit_Driver.py`. The `Driver` and `K_Driver` will reduce your science data for bands Y,J, and H (this includes the sample data set). The K band requires a special approach because there are too few bright night-sky emission lines at the red end and so the `K_Driver` synthesizes arclamps and night sky lines. The `Long2pos_driver.py` handles `long2pos` and `long2pos_specphot` observations, while the `Longslit_driver.py` deals with observations of single objects using a longslit configuration. 4 | 5 | The driver.py files included with the code download contains execution lines that are commented out. For this example, we will run the driver file one line at a time, but as you become familiar with the DRP process, you will develop your own driver file execution sequencing. Although in the future we hope to further automate the driver file, currently some steps require you to update the inputs with filenames created from previous steps. 6 | 7 | Below is a driver.py file: 8 | 9 | import os, time 10 | import MOSFIRE 11 | 12 | from MOSFIRE import Background, Combine, Detector, Flats, IO, Options, \ 13 | Rectify 14 | from MOSFIRE import Wavelength 15 | 16 | import numpy as np, pylab as pl, pyfits as pf 17 | 18 | np.seterr(all="ignore") 19 | 20 | #Update the insertmaskname with the name of the mask 21 | #Update S with the filter band Y,J,H,or K 22 | maskname = 'insertmaskname' 23 | band = 'S' 24 | 25 | flatops = Options.flat 26 | waveops = Options.wavelength 27 | 28 | obsfiles = ['Offset_1.5.txt', 'Offset_-1.5.txt'] 29 | 30 | #Flats.handle_flats('Flat.txt', maskname, band, flatops) 31 | #Wavelength.imcombine(obsfiles, maskname, band, waveops) 32 | #Wavelength.fit_lambda_interactively(maskname, band, obsfiles, 33 | #waveops) 34 | #Wavelength.fit_lambda(maskname, band, obsfiles, obsfiles, 35 | #waveops) 36 | 37 | #Wavelength.apply_lambda_simple(maskname, band, obsfiles, waveops) 38 | #Background.handle_background(obsfiles, 39 | #'lambda_solution_wave_stack_H_m130429_0224-0249.fits', 40 | #maskname, band, waveops) 41 | 42 | redfiles = ["eps_" + file + ".fits" for file in obsfiles] 43 | #Rectify.handle_rectification(maskname, redfiles, 44 | # "lambda_solution_wave_stack_H_m130429_0224-0249.fits", 45 | # band, 46 | # "/scr2/npk/mosfire/2013apr29/m130429_0224.fits", 47 | # waveops) 48 | # 49 | 50 | To set up your driver file do the following: 51 | 52 | 1. Navigate to the desired output directory created by handle: `cd ~/Data/reducedMOSFIRE_DRP_MASK/2012sep10/H` 53 | 2. Copy the appropriate driver file: `cp ~/MosfireDRP-master/drivers/Driver.py .` NOTE: If you are observing a K band mask you’ll want to copy the `K_driver.py` file over. 54 | 3. Edit driver.py (see bold text in driver file example) 55 | * Update maskname 56 | * Update band to be Y,J,H 57 | * Update the `Offset_#.txt` name. Handle creates offset files with names that are specific to the nod throw. The default driver file uses 1.5 arcsec offsets in the file name. 58 | 59 | In the sections that follow, we will describe the function and outputs of the commented lines found in the driver file starting with the creation of flats. 60 | 61 | If you prefer to override the standard naming convention of the output files, you can specify 62 | 63 | target = “targetname” 64 | 65 | at the beginning of the driver file. If you do so, remember to also add target=target to both the Background and Rectify steps. Example: 66 | 67 | Background.handle_background(obsfiles, 68 | 'lambda_solution_wave_stack_H_m150428_0091-0091.fits', 69 | maskname, band, waveops, target=target) 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | -------------------------------------------------------------------------------- /drivers/Long2pos.py: -------------------------------------------------------------------------------- 1 | import os, time, logging 2 | import MOSFIRE 3 | from MOSFIRE import Background, Combine, Detector, Flats, IO, Options, Rectify, Wavelength 4 | from MOSFIRE.MosfireDrpLog import info, debug, warning, error 5 | logger = logging.getLogger(__name__) 6 | import numpy as np 7 | from matplotlib import pyplot as pl 8 | try: 9 | from astropy.io import fits as pf 10 | except: 11 | import pyfits as pf 12 | np.seterr(all='ignore') 13 | flatops = Options.flat 14 | waveops = Options.wavelength 15 | 16 | #Driver file automatically generated on Wed Jul 29 15:04:02 2015 17 | #For questions and comments, email mosfiredrp@gmail.com, submit a ticket on the ticketing system, or contact Luca Rizzi @ WMKO 18 | 19 | # THIS DRIVER IS SETUP TO REDUCE A STAR CALLED HIP85871_7.25, change this string on the entried below 20 | 21 | 22 | 23 | maskname = 'long2pos_specphot (align)' 24 | band = 'H' 25 | 26 | #Set noninteractive to True to autofit wavelenth solution instead of manually fitting. 27 | noninteractiveflag=False 28 | 29 | # these are the narrow slits 30 | obsfiles_posCnarrow = ['Offset_-21_HIP85871_7.25_PosC.txt', 'Offset_-7_HIP85871_7.25_PosC.txt'] 31 | target_posCnarrow = "HIP85871_7.25_POSC_NARROW" 32 | IO.fix_long2pos_headers(obsfiles_posCnarrow) 33 | obsfiles_posAnarrow = ['Offset_7_HIP85871_7.25_PosA.txt', 'Offset_21_HIP85871_7.25_PosA.txt'] 34 | target_posAnarrow = "HIP85871_7.25_POSA_NARROW" 35 | IO.fix_long2pos_headers(obsfiles_posAnarrow) 36 | # these are the wide slits, comment out if you are not using specphot 37 | obsfiles_posCwide = ['Offset_-14_HIP85871_7.25_PosC.txt', 'Offset_-7_HIP85871_7.25_PosC.txt'] 38 | target_posCwide = "HIP85871_7.25_POSC_WIDE" 39 | IO.fix_long2pos_headers(obsfiles_posCwide) 40 | obsfiles_posAwide = ['Offset_14_HIP85871_7.25_PosA.txt', 'Offset_21_HIP85871_7.25_PosA.txt'] 41 | target_posAwide = "HIP85871_7.25_POSA_WIDE" 42 | IO.fix_long2pos_headers(obsfiles_posAwide) 43 | 44 | # Note: for long2pos, the row position is ignored, and the middle point of the slit is used 45 | longslit = {'yrange': [[1062,1188],[887,1010]], 'row_position': 0, 'mode':'long2pos'} 46 | Flats.handle_flats('Flat.txt', maskname, band, flatops,longslit=longslit) 47 | 48 | # in this case, we are using the argon lines. 49 | # replace this with neon=['Ne.txt'] if you prefer to use Ne, and edit the following lines accordingly 50 | argon = ['Ar.txt'] 51 | Wavelength.imcombine(argon, maskname, band, waveops) 52 | Wavelength.fit_lambda_interactively(maskname, band, argon,waveops,longslit=longslit, argon=True, noninteractive=noninteractiveflag) 53 | Wavelength.fit_lambda(maskname, band, argon,argon,waveops,longslit=longslit) 54 | Wavelength.apply_lambda_simple(maskname, band, argon, waveops, longslit=longslit, smooth=True) 55 | 56 | # make sure you use the correct wavelength file generated before 57 | Wavelength_file = 'lambda_solution_wave_stack_H_m150428_0091-0091.fits' 58 | 59 | # narrow 60 | Background.handle_background(obsfiles_posAnarrow,Wavelength_file,maskname,band,waveops, target=target_posAnarrow) 61 | Background.handle_background(obsfiles_posCnarrow,Wavelength_file,maskname,band,waveops, target=target_posCnarrow) 62 | # wide 63 | Background.handle_background(obsfiles_posAwide,Wavelength_file,maskname,band,waveops, target=target_posAwide) 64 | Background.handle_background(obsfiles_posCwide,Wavelength_file,maskname,band,waveops, target=target_posCwide) 65 | 66 | # narrow 67 | redfiles = ["eps_" + file + ".fits" for file in obsfiles_posAnarrow] 68 | Rectify.handle_rectification(maskname, redfiles,Wavelength_file,band,obsfiles_posAnarrow,waveops, target=target_posAnarrow) 69 | redfiles = ["eps_" + file + ".fits" for file in obsfiles_posCnarrow] 70 | Rectify.handle_rectification(maskname, redfiles,Wavelength_file,band,obsfiles_posCnarrow,waveops, target=target_posCnarrow) 71 | # wide 72 | redfiles = ["eps_" + file + ".fits" for file in obsfiles_posAwide] 73 | redfiles = [redfiles[0]] 74 | Rectify.handle_rectification(maskname, redfiles,Wavelength_file,band,obsfiles_posAwide,waveops, target=target_posAwide) 75 | redfiles = ["eps_" + file + ".fits" for file in obsfiles_posCwide] 76 | redfiles = [redfiles[0]] 77 | Rectify.handle_rectification(maskname, redfiles,Wavelength_file,band,obsfiles_posCwide,waveops, target=target_posCwide) 78 | 79 | -------------------------------------------------------------------------------- /manual/docs/changes.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | ## Changes in Version 2018 4 | 5 | ### New features 6 | 7 | * Python3 compatibility! 8 | * Improved installation procedure uses anaconda cloud 9 | * The 1D spectral extraction code now provides the user the ability to set the default aperture width. 10 | * Revamped documentation using mkdocs including the addition of an examples section. 11 | 12 | __Note__: This release of the DRP is only tested and supported on python 3.6+. If you must use python 2, use the 2016 release of the DRP. We encourage all users to use python 3 and we make the install of python 3 easy using anaconda python and the anaconda cloud (see the installation instructions). 13 | 14 | ### Improvements and bug fixes 15 | 16 | * Fixed bug where 1D extraction apertures could not be interactively positioned. 17 | * Fixed astropy deprecation warning related to clobber option 18 | * Fixed bug where slit nods with an odd number of positions would fail 19 | * Fixed numpy deprecation related to indexing arrays with non-integers 20 | * Fixed header info bug which would cause DS9 coordinates to display incorrectly 21 | 22 | ## Changes in Version 2016 23 | 24 | __Important Note__: The [Ureka package has been deprecated](http://ssb.stsci.edu/ureka/) as of April 26, 2016. As a result, the MOSFIRE pipeline has migrated to a version which is not dependent on IRAF/PyRAF, but only on python packages. It should work with any python install which provides the required packages and versions. 25 | 26 | ### New features 27 | 28 | * DRP is no longer dependent on IRAF/PyRAF 29 | * The use of IRAF's `geoxytran`, `imcombine`, and `imarith` tasks have been replaced with python equivalents. 30 | * The DRP should now work with any python install which has the [required python packages](/manual/installing#Requirements) 31 | * Improved slit tracing using a better thresholding algorithm 32 | * An updated (and now web based) [instruction manual](http://keck-datareductionpipelines.github.io/MosfireDRP/) 33 | * The DRP now performs optimal spectral extraction [Horne 1986](http://adsabs.harvard.edu/abs/1986PASP...98..609H) and outputs a 1D spectrum. Please note that this is intended as a quick look tool, not for final science use. 34 | * The `handle` step now writes `filelist.txt` which contains a list of all the files processed by `handle` instead of printing that output to the screen. The file also contains messages for files not categorized for processing explaining why. In addition, `handle` now no longer writes list files with no content. This is intended to make it easier to quickly see what files are available for reduction. 35 | 36 | 37 | ### Improvements and bug fixes 38 | 39 | * Changed dependence on `pylab` to `matplotlib.pyplot` 40 | * Uses `astropy.io.fits` instead of `pyfits` when available 41 | * Adjust log messages to send more to DEBUG instead of INFO. Leads to less clutter in messages visible to user. 42 | 43 | 44 | ## Changes in Version 2015A 45 | 46 | ### New features 47 | 48 | * Reduction of long2pos and long2pos_specphot 49 | * Reduction of longslit data 50 | * Automatic generation of driver file 51 | * Logging and diagnostic information on screen and on disk 52 | * Package-style installation as a Ureka sub-package 53 | * Support for Ureka 1.5.1 54 | 55 | ### Improvements and bug fixes 56 | 57 | * Fix incorrect determination of the slit parameters which prevented the use of large slits 58 | * Fix incorrect determination of the average wavelength dispersion for the long2pos mode 59 | * Added ability of specifying the output name of the files 60 | * Improved robustness of non-interactive wavelength solution, added possibilty of switching from interactive to non-interactive during the reduction, added k-sigma clipping of the sky or arc lines 61 | * Fixed the problem with the interactive wavelength window not closing at the end of the fit 62 | * Fixed the problem with the interactive fitting window showing up empty on the first fit (no need to use the x key to unzoom) 63 | * Added procedure to fix the header of old images acquired with an outdated version of long2pos 64 | * Disabled cosmic ray rejection for the case of less than 5 exposures 65 | * There is no need to specify one of the observations in Rectify: Rectify will use the first of the files listed in the Offset files. 66 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 3 | 4 | import glob 5 | import os 6 | import sys 7 | 8 | import ah_bootstrap 9 | from setuptools import setup 10 | 11 | #A dirty hack to get around some early import/configurations ambiguities 12 | if sys.version_info[0] >= 3: 13 | import builtins 14 | else: 15 | import __builtin__ as builtins 16 | builtins._ASTROPY_SETUP_ = True 17 | 18 | from astropy_helpers.setup_helpers import ( 19 | register_commands, adjust_compiler, get_debug_option, get_package_info) 20 | from astropy_helpers.git_helpers import get_git_devstr 21 | from astropy_helpers.version_helpers import generate_version_py 22 | 23 | # Get some values from the setup.cfg 24 | import configparser as config 25 | conf = config.ConfigParser() 26 | conf.read(['setup.cfg']) 27 | metadata = dict(conf.items('metadata')) 28 | 29 | PACKAGENAME = metadata.get('package_name', 'packagename') 30 | DESCRIPTION = metadata.get('description', 'Astropy affiliated package') 31 | AUTHOR = metadata.get('author', '') 32 | AUTHOR_EMAIL = metadata.get('author_email', '') 33 | LICENSE = metadata.get('license', 'unknown') 34 | URL = metadata.get('url', 'http://astropy.org') 35 | 36 | # Get the long description from the package's docstring 37 | __import__(PACKAGENAME) 38 | package = sys.modules[PACKAGENAME] 39 | LONG_DESCRIPTION = package.__doc__ 40 | 41 | # Store the package name in a built-in variable so it's easy 42 | # to get from other parts of the setup infrastructure 43 | builtins._ASTROPY_PACKAGE_NAME_ = PACKAGENAME 44 | 45 | # VERSION should be PEP386 compatible (http://www.python.org/dev/peps/pep-0386) 46 | VERSION = '1.0.dev' 47 | 48 | # Indicates if this version is a release version 49 | RELEASE = 'dev' not in VERSION 50 | 51 | if not RELEASE: 52 | VERSION += get_git_devstr(False) 53 | 54 | # Populate the dict of setup command overrides; this should be done before 55 | # invoking any other functionality from distutils since it can potentially 56 | # modify distutils' behavior. 57 | cmdclassd = register_commands(PACKAGENAME, VERSION, RELEASE) 58 | 59 | # Adjust the compiler in case the default on this platform is to use a 60 | # broken one. 61 | adjust_compiler(PACKAGENAME) 62 | 63 | # Freeze build information in version.py 64 | generate_version_py(PACKAGENAME, VERSION, RELEASE, 65 | get_debug_option(PACKAGENAME)) 66 | 67 | # Treat everything in scripts except README.rst as a script to be installed 68 | scripts = [fname for fname in glob.glob(os.path.join('scripts', '*')) 69 | if os.path.basename(fname) != 'README.rst'] 70 | 71 | 72 | # Get configuration information from all of the various subpackages. 73 | # See the docstring for setup_helpers.update_package_files for more 74 | # details. 75 | package_info = get_package_info() 76 | 77 | # Add the project-global data 78 | package_info['package_data'].setdefault(PACKAGENAME, []) 79 | package_info['package_data'][PACKAGENAME].append('data/*') 80 | 81 | # Define entry points for command-line scripts 82 | entry_points = {} 83 | entry_points['console_scripts'] = [ 84 | 'astropy-package-template-example = packagename.example_mod:main', 85 | ] 86 | 87 | # Include all .c files, recursively, including those generated by 88 | # Cython, since we can not do this in MANIFEST.in with a "dynamic" 89 | # directory name. 90 | c_files = [] 91 | for root, dirs, files in os.walk(PACKAGENAME): 92 | for filename in files: 93 | if filename.endswith('.c'): 94 | c_files.append( 95 | os.path.join( 96 | os.path.relpath(root, PACKAGENAME), filename)) 97 | package_info['package_data'][PACKAGENAME].extend(c_files) 98 | 99 | setup(name=PACKAGENAME, 100 | version=VERSION, 101 | description=DESCRIPTION, 102 | scripts=scripts, 103 | requires=['astropy'], 104 | install_requires=['astropy'], 105 | provides=[PACKAGENAME], 106 | author=AUTHOR, 107 | author_email=AUTHOR_EMAIL, 108 | license=LICENSE, 109 | url=URL, 110 | long_description=LONG_DESCRIPTION, 111 | cmdclass=cmdclassd, 112 | zip_safe=False, 113 | use_2to3=True, 114 | entry_points=entry_points, 115 | **package_info 116 | ) 117 | -------------------------------------------------------------------------------- /apps/CSU_plot_confirm.py: -------------------------------------------------------------------------------- 1 | 2 | import numpy as np 3 | from matplotlib import pyplot as pl 4 | import scipy as sp 5 | import scipy.io 6 | import os 7 | from MOSFIRE import CSU 8 | from matplotlib.backends.backend_pdf import PdfPages 9 | 10 | from IPython.Shell import IPShellEmbed 11 | 12 | start_shell = IPShellEmbed() 13 | 14 | def fit_line_with_sigclip(xs, data, i = 0): 15 | 16 | ps = np.polyfit(xs, data, 1) 17 | pf = np.poly1d(ps) 18 | 19 | 20 | residual = np.abs(pf(xs) - data) 21 | sd = np.std(residual) 22 | 23 | ok = np.where(residual < 3.0*sd)[0] 24 | 25 | ps = np.polyfit(xs[ok], data[ok], 1) 26 | pf = np.poly1d(ps) 27 | return [pf, ok] 28 | 29 | path = "/users/npk/dropbox/mosfire/cooldown 9/csu_meas/" 30 | proto = "m11031%1.1i_%4.4i.fits.sav.mat" 31 | 32 | pl.ion() 33 | rs = [] 34 | for i in range(1754, 1793): 35 | if os.path.exists(path + proto % (6, i)): 36 | ld = scipy.io.loadmat(path + proto % (6, i)) 37 | elif os.path.exists(path + proto % (7, i)): 38 | ld = scipy.io.loadmat(path + proto % (7, i)) 39 | elif os.path.exists(path + proto % (8, i)): 40 | ld = scipy.io.loadmat(path + proto % (8, i)) 41 | elif os.path.exists(path + proto % (9, i)): 42 | ld = scipy.io.loadmat(path + proto % (9, i)) 43 | else: 44 | print "No luck" 45 | continue 46 | 47 | ld["img_num"] = i 48 | rs.append(ld) 49 | 50 | 51 | 52 | bar_posns = [[] for i in range(92)] 53 | 54 | for r in rs: 55 | for i in range(92): 56 | assert(r["bars"][i] == i+1) 57 | p = r["poss_mm"][i] 58 | d = r["deltas_mm"][i] 59 | 60 | if not np.isfinite(p): continue 61 | if not np.isfinite(d): continue 62 | if p < 20: continue 63 | if p > 200: continue 64 | bar_posns[i].append([p, d, r["bars"][i]]) 65 | 66 | 67 | even = np.arange(1, 92, 2) 68 | odd = np.arange(0, 92, 2) 69 | 70 | ds = [] 71 | ff = np.poly1d([.08/45, -0.04]) 72 | pl.figure(1) 73 | pl.clf() 74 | fits = [] 75 | for r in rs: 76 | pm = r["poss_mm"] 77 | ok = np.where(np.isfinite(pm))[0] 78 | if len(ok) < 1: continue 79 | 80 | b = r["bars"][ok] 81 | #pl.plot(r["deltas_mm"][ok] + 0.0416 - 0.0010392*b,'o') 82 | pl.plot(r["deltas_mm"][ok],'o') 83 | fits.append(np.polyfit(r["bars"][ok].ravel(), r["deltas_mm"][ok].ravel(), 1)) 84 | #ds.extend(r["deltas_mm"][ok] + 0.0416 - 0.0010392*b) 85 | ds.extend(r["deltas_mm"][ok]) 86 | 87 | 88 | ds = np.array(ds).ravel() 89 | 90 | # Measured by hand 91 | groups = [75, 100, 125, 150, 172, 196] 92 | groups = [25, 50, 75, 100, 125, 150, 172, 196, 220, 244] 93 | 94 | pl.figure(2) 95 | pl.clf() 96 | pl.axis("scaled") 97 | pl.ylim(-140,140) 98 | pl.xlim(-140,140) 99 | pl.xlabel("Keck Field Position (mm)") 100 | pl.ylabel("Keck Field Position (mm)") 101 | pl.grid() 102 | 103 | scale = 300 104 | mn = ds.mean() 105 | for bar in range(92): 106 | bv = np.array(bar_posns[bar]) 107 | if len(bv) == 0: continue 108 | pos = bv[:,0] 109 | delts = bv[:,1] 110 | yshift = (bar % 2)*1. 111 | for group in groups: 112 | p = np.where(np.isfinite(pos) & (np.abs(pos-group)<5))[0] 113 | position = pos[p].mean() - mn - 137.4 114 | delta = delts[p].mean() - mn 115 | 116 | ypos = (np.round(bar/2.)-23.0) * 5.8 * CSU.tempscale + yshift 117 | if delta > 0: pl.arrow(position, ypos, delta*scale, 0, color='red', head_width=.8) 118 | else: pl.arrow(position, ypos, delta*scale, 0, color='blue',head_width=.8) 119 | 120 | pl.text(-132,-117,"0.1 arcsecond") 121 | pl.arrow(-120,-120,scale*.725*.1,0, head_width=.9) 122 | 123 | pl.figure(3) 124 | pl.clf() 125 | arc = (ds-mn)/.725 126 | pl.hist(arc,40,color='w') 127 | pl.title("(Achieved - Requested) Bar Positions") 128 | pl.xlabel("Bar Offsets [arcsecond]") 129 | pl.ylabel("N (total = %i)" % len(arc)) 130 | pl.text(-0.1, 50, "SD: %3.2f arcsecond\nP-V: %3.2f arcsecond" % (arc.std(), arc.max()-arc.min())) 131 | 132 | print 133 | print "Mean offset of: %4.3f" % mn 134 | 135 | pdf = PdfPages("CSU_confirmation_mar_16_2011.pdf") 136 | pl.figure(2).savefig(pdf, format="pdf") 137 | pl.figure(3).savefig(pdf, format="pdf") 138 | pdf.close() 139 | -------------------------------------------------------------------------------- /apps/CSU_Check.py: -------------------------------------------------------------------------------- 1 | ''' 2 | 3 | Written March 3rd 2011 by npk 4 | ''' 5 | 6 | import numpy as np 7 | try: 8 | from astropy.io import fits as pf 9 | except: 10 | import pyfits as pf 11 | from matplotlib import pyplot as pl 12 | from pyraf import iraf 13 | from MOSFIRE import CSU, Detector, IO, Fit 14 | 15 | 16 | def fit_line_with_sigclip(xs, data, i = 0): 17 | 18 | ps = np.polyfit(xs, data, 1) 19 | pf = np.poly1d(ps) 20 | 21 | residual = np.abs(pf(xs) - data) 22 | sd = np.std(residual) 23 | 24 | ok = np.where(residual < 2*sd)[0] 25 | 26 | ps = np.polyfit(xs[ok], data[ok], 1) 27 | pf = np.poly1d(ps) 28 | return [pf, ok] 29 | 30 | if len(ok) == len(residual): 31 | return [pf, ok] 32 | elif i > 2: 33 | return [pf, ok] 34 | else: 35 | return fit_line_with_sigclip(xs[ok], data[ok], i+1) 36 | 37 | def median_tails(v): 38 | a = np.median(v[0:2]) 39 | b = np.median(v[-3:-1]) 40 | 41 | t = v - np.float(a+b)/2. 42 | 43 | return t 44 | 45 | def make_slice(pos, w, h): 46 | '''Returns [ [xslice, ylsice], [x0, x1, y0, x1] ] where 47 | xslice is used as Array[x0:x1] 48 | yslice is used as Array[y0:y1]''' 49 | x0 = pos[0]-w 50 | if x0 < 0: x0 = 0 51 | x1 = pos[0] + w 52 | if x1 > 2047: x1 = 2047 53 | if x0 > x1: x0 = x1 54 | xs = slice(x0, x1) 55 | 56 | y0 = pos[1]-h 57 | if y0 < 0: y0 = 0 58 | y1 = pos[1]+h 59 | if y1 > 2047: y1 = 2047 60 | if y0 > y1: y0 = y1 61 | ys = slice(y0,y1) 62 | 63 | return [[xs,ys],[x0,x1,y0,y1]] 64 | 65 | (header, data) = IO.readfits("/users/npk/desktop/c8/m101029_0233.ref.fits") 66 | (header2, data2) = IO.readfits("/users/npk/desktop/c8/m101029_0425.ref.fits") 67 | (header3, data3) = IO.readfits("/users/npk/desktop/c8/m101029_0427.ref.fits") 68 | 69 | data = data3 70 | 71 | 72 | deg = np.pi/180. 73 | np.set_printoptions(precision=3) 74 | np.set_printoptions(suppress=True) 75 | 76 | reload(CSU) 77 | reload(IO) 78 | reload(Fit) 79 | reload(Detector) 80 | 81 | pl.ion() 82 | 83 | bs = CSU.Barset() 84 | bs.set_header(header) 85 | 86 | pl.figure(1) 87 | pl.clf() 88 | 89 | means = [] 90 | sds = [] 91 | deltas = [] 92 | poss = [] 93 | qs = [] 94 | 95 | cnt = 1 96 | 97 | cntfit = 1 98 | pl.figure(5) 99 | pl.clf() 100 | pl.subplot(7, 7, cntfit) 101 | pl.figure(6) 102 | pl.clf() 103 | pl.subplot(7, 7, cntfit) 104 | 105 | for bar in range(4, 92, 2): 106 | print bar 107 | pos = bs.get_bar_pix(bar) 108 | [[xslice, yslice],extent] = make_slice(pos, 6,25) 109 | if extent[0] == extent[1]: 110 | cnt += 1 111 | continue 112 | if extent[2] == extent[3]: 113 | cnt += 1 114 | continue 115 | cnt+=1 116 | 117 | fits = [] 118 | xs = np.arange(-10,10) 119 | for i in xs: 120 | tofit = data[pos[1]-i, xslice] 121 | y = median_tails(tofit) 122 | 123 | ps = Fit.do_fit(y, Fit.residual_pair) 124 | fits.append(ps[0]) 125 | 126 | fits = np.array(fits) 127 | 128 | m = [np.mean(fits[:,i]) for i in range(5)] 129 | s = [np.std(fits[:,i]) for i in range(5)] 130 | means.append(m) 131 | sds.append(s) 132 | 133 | [ff, ok] = fit_line_with_sigclip(xs, fits[:,1]) 134 | 135 | pl.figure(5) 136 | pl.subplot(7,7,cntfit) 137 | pl.plot(xs, fits[:,1] - ff(xs), '*-') 138 | pl.plot(xs[ok], fits[ok,1] - ff(xs[ok]), 'or-') 139 | pl.ylim([-.1,.1]) 140 | pl.title("%2i" % bar) 141 | 142 | pl.figure(6) 143 | pl.subplot(7,7,cntfit) 144 | pl.plot(xs, fits[:,4],'*-') 145 | pl.plot(xs[ok], fits[ok,4],'or-') 146 | pl.ylim([2.9,4]) 147 | 148 | cntfit += 1 149 | 150 | delta = (extent[0] + ff[0]) - pos[0] 151 | poss.append(extent[0] + ff[0]) 152 | deltas.append(delta) 153 | q = np.degrees(ff[1]) - np.degrees(CSU.rotation) 154 | qs.append(q) 155 | 156 | pl.figure(1) 157 | pl.text(pos[0], pos[1], 'b%2.0i: w=%3.2f p=%5.2f q=%3.2f d=%1.3f' % (bar, np.mean(fits[:,4]), extent[0]+ff[0], q, delta), fontsize=11, family='monospace', horizontalalignment='center') 158 | 159 | pl.xlim([0,2048]) 160 | pl.ylim([0,2048]) 161 | 162 | 163 | means = np.array(means) 164 | sds = np.array(sds) 165 | pl.draw() 166 | -------------------------------------------------------------------------------- /manual/docs/background.md: -------------------------------------------------------------------------------- 1 | # Background Subtraction 2 | 3 | This DRP assumes that targets are nodded along the slit with integration times as described on the instrument web page. The integration times described were selected such that the shot-noise in the region between night sky lines is over 5x larger than the read noise of a 16-fowler sample. For MOSFIRE, we define this as background limited. 4 | 5 | Despite MOSFIRE’s (unprescedented) f/2.0 camera, the desired integration time for background-limited operation is longer than the time for the atmosphere to vary by several percent. As a result, a further background subtraction step is required to remove the residual features. The step is performed by a function called background_subtract_helper() and follows the notation and procedure outlined in Kasen (2003; PASP 115). For most users, you’ll want to use the standard Driver file and not worry about the details. 6 | 7 | In the Driver.py file you want to uncomment the following: 8 | 9 | Background.handle_background(obsfiles, 'lambda_solution_wave_stack_J_m130114_0443-0445.fits', maskname, band, waveops) 10 | 11 | The lambda_solution_wave_stack file needs to be updated in your driver file. If reducing Kband, be sure to use the merged wave_stack solution. It is one of the outputs from the last wavelength step (see section 8). 12 | 13 | In this step: 14 | 15 | * Apply the flat field corrections 16 | * A position files are combined (Offset_*.txt) 17 | * B postion files are combined (Offset_-*.txt) 18 | * Subtract A-B 19 | * Correct for small differences in the background sky emission 20 | 21 | ## Output Files 22 | 23 | The background subtraction step produces the following files. As usual elements in [brackets] are replaced with the value for that mask. 24 | 25 | | Filename | Content (units) | 26 | |----------------------------------|-----------------------------------------------------------------------------| 27 | | `eps_Offset_[###].txt.fits` | Average signal in the ### stack () | 28 | | `var_Offset_[###].txt.fits` | Total variance in each pixel of above file () | 29 | | `itimes_Offset_[###].txt.fits` | Total exposure time in each pixel of above files () | 30 | | `sub_[maskname] _[bandname]_[plan].fits` | Difference (but non background subtracted) file () | 31 | | `bsub_[maskname]_{ bandname]_[plan].fits` | Background subtracted signal () | 32 | | `bmod_[maskname]_{ bandname]_[plan].fits` | Background model signal () | 33 | | `var_[maskname]_{ bandname]_[plan].fits` | Total variance | 34 | | `itime_[maskname]_{ bandname]_[plan].fits` | Average integration time | 35 | 36 | There is redundant information in the above set of files. For instance: 37 | 38 | sub_Mask_K_A-B.fits = eps_Offset_1.5.txt.fits – eps_Offset_-1.5.txt.fits 39 | var_Mask_K_A-B.fits = var_Offset_1.5.txt.fits + var_Offset_1.5.txt.fits 40 | itime_Mask_K_A-B.fits = mean(itime_Offset_1.5.txt.fits, itime_Offset_1.5.t.xt.fits) 41 | 42 | If you want to drill further into how these are constructed, examine the Background.py imcombine and handle_background functions. 43 | 44 | Recitified outputs are also computed as tabulated in the table below. 45 | 46 | | Filename | Content (units) | 47 | |----------------------------------|-----------------------------------------------------------------------------| 48 | | `[maskname]_rectified_[bandname]_[plan].fits` | Signal () | 49 | | `[maskname]_rectified_itime_[bandname]_[plan].fits` | Integration time | 50 | | `[maskname]_rectified_var_[bandname]_[plan].fits` | Variance | 51 | | `[maskname]_rectified_sn_[bandname]_[plan].fits` | Signal to noise () | 52 | 53 | Note that signal to noise is computed as follows: 54 | 55 | ![Equation](eq1.png "Equation1: sn = \frac{signal * integration time}{\sqrt{variance}}") 56 | 57 | yes, we violate the first normal form for convenience. Also note that the STD is computed assuming the detector has a read noise of Detector.RN (documented in the MOSFIRE Pre Ship Review as 21 electron) per fowler sample. Thus, the final STD is 58 | 59 | ![Equation](eq2.png "Equation2: STD = \frac{21 electrons}{\sqrt{N_{reads}} + \sqrt{N_{detected electrons}}") 60 | 61 | assuming the gain in Detector.gain. Note that there is no shot noise from dark current, which was measured to be negligible at pre-ship review. 62 | 63 | An example of what the output looks like is here: 64 | 65 | ![Screenshot](image6.png "Image showing the itime, bsub, and rectified wavelength images. The green crosses are marking the location of the same pixel in each image.") 66 | 67 | Image showing the itime, bsub, and rectified wavelength images. The green crosses are marking the location of the same pixel in each image. 68 | 69 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | 15 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest 16 | 17 | #This is needed with git because git doesn't create a dir if it's empty 18 | $(shell [ -d "_static" ] || mkdir -p _static) 19 | 20 | help: 21 | @echo "Please use \`make ' where is one of" 22 | @echo " html to make standalone HTML files" 23 | @echo " dirhtml to make HTML files named index.html in directories" 24 | @echo " singlehtml to make a single large HTML file" 25 | @echo " pickle to make pickle files" 26 | @echo " json to make JSON files" 27 | @echo " htmlhelp to make HTML files and a HTML help project" 28 | @echo " qthelp to make HTML files and a qthelp project" 29 | @echo " devhelp to make HTML files and a Devhelp project" 30 | @echo " epub to make an epub" 31 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 32 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 33 | @echo " text to make text files" 34 | @echo " man to make manual pages" 35 | @echo " changes to make an overview of all changed/added/deprecated items" 36 | @echo " linkcheck to check all external links for integrity" 37 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 38 | 39 | clean: 40 | -rm -rf $(BUILDDIR) 41 | -rm -rf api 42 | 43 | html: 44 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 45 | @echo 46 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 47 | 48 | dirhtml: 49 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 50 | @echo 51 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 52 | 53 | singlehtml: 54 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 55 | @echo 56 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 57 | 58 | pickle: 59 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 60 | @echo 61 | @echo "Build finished; now you can process the pickle files." 62 | 63 | json: 64 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 65 | @echo 66 | @echo "Build finished; now you can process the JSON files." 67 | 68 | htmlhelp: 69 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 70 | @echo 71 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 72 | ".hhp project file in $(BUILDDIR)/htmlhelp." 73 | 74 | qthelp: 75 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 76 | @echo 77 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 78 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 79 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Astropy.qhcp" 80 | @echo "To view the help file:" 81 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Astropy.qhc" 82 | 83 | devhelp: 84 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 85 | @echo 86 | @echo "Build finished." 87 | @echo "To view the help file:" 88 | @echo "# mkdir -p $$HOME/.local/share/devhelp/Astropy" 89 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Astropy" 90 | @echo "# devhelp" 91 | 92 | epub: 93 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 94 | @echo 95 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 96 | 97 | latex: 98 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 99 | @echo 100 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 101 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 102 | "(use \`make latexpdf' here to do that automatically)." 103 | 104 | latexpdf: 105 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 106 | @echo "Running LaTeX files through pdflatex..." 107 | make -C $(BUILDDIR)/latex all-pdf 108 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 109 | 110 | text: 111 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 112 | @echo 113 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 114 | 115 | man: 116 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 117 | @echo 118 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 119 | 120 | changes: 121 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 122 | @echo 123 | @echo "The overview file is in $(BUILDDIR)/changes." 124 | 125 | linkcheck: 126 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 127 | @echo 128 | @echo "Link check complete; look for any errors in the above output " \ 129 | "or in $(BUILDDIR)/linkcheck/output.txt." 130 | 131 | doctest: 132 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 133 | @echo "Testing of doctests in the sources finished, look at the " \ 134 | "results in $(BUILDDIR)/doctest/output.txt." 135 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | if NOT "%PAPER%" == "" ( 11 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 12 | ) 13 | 14 | if "%1" == "" goto help 15 | 16 | if "%1" == "help" ( 17 | :help 18 | echo.Please use `make ^` where ^ is one of 19 | echo. html to make standalone HTML files 20 | echo. dirhtml to make HTML files named index.html in directories 21 | echo. singlehtml to make a single large HTML file 22 | echo. pickle to make pickle files 23 | echo. json to make JSON files 24 | echo. htmlhelp to make HTML files and a HTML help project 25 | echo. qthelp to make HTML files and a qthelp project 26 | echo. devhelp to make HTML files and a Devhelp project 27 | echo. epub to make an epub 28 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 29 | echo. text to make text files 30 | echo. man to make manual pages 31 | echo. changes to make an overview over all changed/added/deprecated items 32 | echo. linkcheck to check all external links for integrity 33 | echo. doctest to run all doctests embedded in the documentation if enabled 34 | goto end 35 | ) 36 | 37 | if "%1" == "clean" ( 38 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 39 | del /q /s %BUILDDIR%\* 40 | goto end 41 | ) 42 | 43 | if "%1" == "html" ( 44 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 45 | if errorlevel 1 exit /b 1 46 | echo. 47 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 48 | goto end 49 | ) 50 | 51 | if "%1" == "dirhtml" ( 52 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 53 | if errorlevel 1 exit /b 1 54 | echo. 55 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 56 | goto end 57 | ) 58 | 59 | if "%1" == "singlehtml" ( 60 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 61 | if errorlevel 1 exit /b 1 62 | echo. 63 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 64 | goto end 65 | ) 66 | 67 | if "%1" == "pickle" ( 68 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 69 | if errorlevel 1 exit /b 1 70 | echo. 71 | echo.Build finished; now you can process the pickle files. 72 | goto end 73 | ) 74 | 75 | if "%1" == "json" ( 76 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 77 | if errorlevel 1 exit /b 1 78 | echo. 79 | echo.Build finished; now you can process the JSON files. 80 | goto end 81 | ) 82 | 83 | if "%1" == "htmlhelp" ( 84 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 85 | if errorlevel 1 exit /b 1 86 | echo. 87 | echo.Build finished; now you can run HTML Help Workshop with the ^ 88 | .hhp project file in %BUILDDIR%/htmlhelp. 89 | goto end 90 | ) 91 | 92 | if "%1" == "qthelp" ( 93 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 94 | if errorlevel 1 exit /b 1 95 | echo. 96 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 97 | .qhcp project file in %BUILDDIR%/qthelp, like this: 98 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\Astropy.qhcp 99 | echo.To view the help file: 100 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\Astropy.ghc 101 | goto end 102 | ) 103 | 104 | if "%1" == "devhelp" ( 105 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 106 | if errorlevel 1 exit /b 1 107 | echo. 108 | echo.Build finished. 109 | goto end 110 | ) 111 | 112 | if "%1" == "epub" ( 113 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 117 | goto end 118 | ) 119 | 120 | if "%1" == "latex" ( 121 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 122 | if errorlevel 1 exit /b 1 123 | echo. 124 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 125 | goto end 126 | ) 127 | 128 | if "%1" == "text" ( 129 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 130 | if errorlevel 1 exit /b 1 131 | echo. 132 | echo.Build finished. The text files are in %BUILDDIR%/text. 133 | goto end 134 | ) 135 | 136 | if "%1" == "man" ( 137 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 138 | if errorlevel 1 exit /b 1 139 | echo. 140 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 141 | goto end 142 | ) 143 | 144 | if "%1" == "changes" ( 145 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 146 | if errorlevel 1 exit /b 1 147 | echo. 148 | echo.The overview file is in %BUILDDIR%/changes. 149 | goto end 150 | ) 151 | 152 | if "%1" == "linkcheck" ( 153 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 154 | if errorlevel 1 exit /b 1 155 | echo. 156 | echo.Link check complete; look for any errors in the above output ^ 157 | or in %BUILDDIR%/linkcheck/output.txt. 158 | goto end 159 | ) 160 | 161 | if "%1" == "doctest" ( 162 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 163 | if errorlevel 1 exit /b 1 164 | echo. 165 | echo.Testing of doctests in the sources finished, look at the ^ 166 | results in %BUILDDIR%/doctest/output.txt. 167 | goto end 168 | ) 169 | 170 | :end 171 | -------------------------------------------------------------------------------- /manual/docs/wavelengthK.md: -------------------------------------------------------------------------------- 1 | # Wavelength Calibration (K) 2 | 3 | The night sky lines at the red end of the K-band are too faint to achieve small-fraction of a pixel RMS wavelength calibration. You will have to observe a Neon and Argon arc lamps during your afternoon calibrations. By default, the calibration script at the observatory is setup to acquire both the Ne and Argon arcs. 4 | 5 | Because the beams emminating from the arclamp do not follow the same path as the beams coming from the sky, there will be a slight difference between the two solutions. For the afformentioned beam matching reason, the most accurate solution is the night sky lines. Thus, the code has to be clever about merging the two solutions. 6 | 7 | The following subsections describe the additional steps that are necessary to process the arcline data and combine the arcs and night sky line wavelength solutions. 8 | 9 | ## Combine the arc line spectra 10 | 11 | Just like the step in section 8.1 where you combined the science frames to create nightsky line spectra, we first need to combine the arcline data. The arcs are typically three files and you should see them listed in the Ne.txt and Ar.txt file lists in your K band sub directory. To combine the images simply uncomment and run: 12 | 13 | Wavelength.imcombine('Ne.txt', maskname, band, waveops) 14 | Wavelength.imcombine('Ar.txt', maskname, band, waveops) 15 | 16 | ## Identify arc lines using night sky solution 17 | 18 | Instead of having to interactively determine the wavelenth solution for the arcs like we did in section 8.2 for the night sky lines, we are going to use the solutions for the night sky lines as a first approximation for the arcs. This may usually be done because the arcs differ from the night sky lines by a fractions of pixels. You are welcome to interactively solve the neon lamp solution with the Wavelength.fit_lambda_interactively routine; however, the need to run the interactive solution method should be rare. 19 | 20 | To apply the solution from the night sky lines to the arcs center slit position, uncomment and run the following lines. 21 | 22 | Wavelength.apply_interactive(maskname, band, waveops, apply=obsfiles, to='Ne.txt', neon=True) 23 | Wavelength.apply_interactive(maskname, band, waveops, apply=obsfiles, to='Ar.txt', argon=True) 24 | 25 | This step, when run will produce output like: 26 | 27 | slitno 1 STD: 0.16 MAD: 0.06 28 | slitno 2 STD: 0.03 MAD: 0.02 29 | slitno 3 STD: 0.04 MAD: 0.04 30 | slitno 4 STD: 0.05 MAD: 0.01 31 | 32 | For each slit, a new solution is generated for the neon line. The output mimics that described previously where STD is the standard deviation and MAD is the median absolute deviation in angstroms. 33 | 34 | ## Wavelength fitting for the entire slit using arcs 35 | 36 | The next step in the wavelength fitting process is to propogate the arc solution spatially along each slit. Again this is the same process essentially as the fit for the night sky lines. This moves along each row for the slit to determine a wavelenth solution. The output files are comperable to those in step 8.3 37 | 38 | Wavelength.fit_lambda(maskname, band, 'Ne.txt', 'Ne.txt', waveops, wavenames2='Ar.txt') 39 | 40 | You will note that the Ar.txt file is listed as an optional argument. If you choose not to use the Argon lamps, then you may simply remove the optional wavenames2 and execute this using only the Ne arcs. 41 | 42 | Again, this process takes some time to complete. 43 | 44 | ## Merge the arc and sky lists 45 | 46 | In this portion of the procedure, we merge the two lists. These commands may not be run individually. Instead any command containing the variable LROI needs to be run in one mospy driver file session in order to pass the LROI variable. In this section we determin the offsets between the region of overlap between the nightskylines and the arclines. A plot of that region is displayed. To move on you will have to close the plot. 47 | 48 | To execute this step you will need to uncomment the following lines in the driver file. 49 | 50 | LROI = [[21000, 22800]] * 1 51 | LROIs = Wavelength.check_wavelength_roi(maskname, band, obsfiles, 'Ne.txt', LROI, waveops) 52 | Wavelength.apply_lambda_simple(maskname, band, 'Ne.txt', waveops) 53 | Wavelength.apply_lambda_sky_and_arc(maskname, band, obsfiles, 'Ne.txt', LROIs, waveops, neon=True) 54 | 55 | The merged output solution will have a filename that looks like: 56 | 57 | merged_lambda_coeffs_wave_stack_K_m130114_0451-0453_and_wave_stack_K_m140508_0197-0199.npy 58 | merged_lambda_solution_wave_stack_K_m130114_0451-0453_and_wave_stack_K_m140508_0197-0199.fits 59 | merged_rectified_wave_stack_K_m130114_0451-0453_and_wave_stack_K_m140508_0197-0199.fits.gz 60 | 61 | The ouput files have the same format as those in section 8.4 and will need to be used as inputs to the Background and Rectify section below. 62 | 63 | -------------------------------------------------------------------------------- /manual/docs/example.md: -------------------------------------------------------------------------------- 1 | # Examples of Running the Pipeline 2 | 3 | Here we demonstrate three walkthroughs of how to run the MOSFIRE pipeline. We include a longslit reduction, a slitmask reduction, and a long2pos reduction in the H and K bands. Example datasets can be downloaded from this [link](http://www2.keck.hawaii.edu/realpublic/inst/mosfire/example_dataset.zip), and includes all three data types. Details of each step in the reduction process can be seen in the manual, so please read the previous sections in the manual to understand the reductions being performed on these data. 4 | 5 | ## Getting Started 6 | 7 | After downloading and unzipping all of the test data, make a directory in your preferred location to perform your reduction and run [handle](handle). 8 | 9 | mkdir reduced 10 | cd reduced 11 | mospy handle /path/to/data/test_dataset/*.fits 12 | 13 | You should see five new directories after handle is done. 14 | 15 | LONGSLIT-3x0.7 <-- Longslit observations 16 | LONGSLIT-46x0.7 <-- Longslit calibrations 17 | MOSFIRE_DRP_MASK <-- Slitmask calibrations and observations 18 | long2pos <-- long2pos calibrations 19 | long2pos_specphot <-- long2pos_specphot observations 20 | 21 | ## Longslit Reduction 22 | 23 | For more information on the reduction of longslits, go to the [longslit][longslit] section. 24 | 25 | Move to the Longslit observation directory and copy the calibrations to the observation directory: 26 | 27 | cd LONGSLIT-3x0.7/2012nov28/K 28 | cp ../../../LONGSLIT-46x0.7/2013oct15/K/*.txt ./ 29 | 30 | Run the [autodriver](autodriver) to create the driver file Longslit_HIP17971.py 31 | 32 | Edit the driver file so the y-range so it covers the range of the slit, and 'row_position' is not contaminated with the spectrum (you only want sky lines). Look at the files in Offset_-5_HIP17971.txt and Offset_5_HIP17971.txt to determine where these values should be. 33 | 34 | Run the pipeline using: 35 | 36 | mospy Longslit_HIP17971.py 37 | 38 | You can run step by step by commenting out reduction steps in the driver file, or you can run all at once. First the pipeline will create a flat field, as described in the [Flats](flats) section. Check out your combined and pixel flats to make sure they look reasonable, no odd edges to slits. 39 | 40 | We do not recommend non-interactive wavelength reductions unless you have a previous reliable wavelength solution in the same directory ("lambda_coeffs_wave_stack..."). Refer to the Wavelength Calibration sections (for H,J, and Y, or K band) for information on how to perform the interactive fit. In this case, we will be performing a K band solution, so check that [section](wavelengthK) for more details. 41 | 42 | After [background subtraction](background) and [rectification](rectify), the output (notably, HIP17971_K_eps.fits, HIP17971_K_itime.fits, HIP17971_K_sig.fits, HIP17971_K_snrs.fits) you get should look like this: 43 | 44 | ![Longslit DRP output](image10.png "Image showing the output of the DRP for a longslit.") 45 | 46 | For longslits with bright continuua (such as in this example), a spectral [extraction](extract) if you should wish to perform one will be very simple. In the case of a bright continuum like this, you may or may not choose to perform the aperture selection manually, but we encourage manual control of the apertures whenever reasonable, even if just to check the aperture is correct. 47 | 48 | ## Slitmask Reduction 49 | 50 | Move to the MOSFIRE_DRP_MASK directory and run the autodriver: 51 | 52 | cd MOSFIRE_DRP_MASK/2012sep10/H/ 53 | mospy AutoDriver 54 | 55 | The driver file will be called Driver.py. Open the file to make sure there are no abnormalities, and proceed to call Driver.py with either the full reduction, or step by step. 56 | 57 | mospy Driver.py 58 | 59 | Again, we recommend a manual [wavelength solution](wavelengthYJH). Once the wavelength solution is propogated, we recommend checking the wavelength propagation by opening lambda_solution_wave_stack_H_m120910_0163-0176.fits to check that no slits are missing and as you check the intensity levels (which will be the wavelength values) for reasonable numbers for your filter (H band in this case). 60 | 61 | After rectification, the output whole masks will look like this: 62 | 63 | ![Slitmask DRP output](image11.png "Image showing the output of the DRP for a slitmask.") 64 | 65 | And the entire mask (zoomed out) will look like this: 66 | 67 | ![Slitmask eps output](image12.png "Image showing the eps image for a slitmask.") 68 | 69 | Even with bright continuua like this slitmask has, we recommend manually controlling the apertures in case secondary objects are in the slits. In the case of faint or emission line only objects, it might be necessary to manually enter where an aperture should be located and the width of the extraction. 70 | 71 | ## Long2pos_specphot Reduction 72 | 73 | Move to the long2pos_specphot directory. Since calibrations for [long2pos](long2pos) can be used for long2pos_specphot, copy those calibration files to your long2pos_specphot directory: 74 | 75 | cd long2pos_specphot/2017sep10/K 76 | cp ../../../long2pos/2017sep29/K/*.txt ./ 77 | 78 | Run the autodriver and open its output, Long2pos_ZW_TEL.py 79 | 80 | Either comment out or copy the extract command for each desired output target (positions A or C, wide or narrow). Then run the driver file step by step or as a full reduction. 81 | 82 | After rectification, your output eps files for PosA and PosC narrow and wide will look like this: 83 | 84 | ![Output long2pos_specphot images](image13.png "Output long2pos_specphot images") -------------------------------------------------------------------------------- /MOSFIRE/_astropy_init.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | 3 | __all__ = ['__version__', '__githash__', 'test'] 4 | 5 | # this indicates whether or not we are in the package's setup.py 6 | try: 7 | _ASTROPY_SETUP_ 8 | except NameError: 9 | from sys import version_info 10 | if version_info[0] >= 3: 11 | import builtins 12 | else: 13 | import __builtin__ as builtins 14 | builtins._ASTROPY_SETUP_ = False 15 | 16 | try: 17 | from .version import version as __version__ 18 | except ImportError: 19 | __version__ = '' 20 | try: 21 | from .version import githash as __githash__ 22 | except ImportError: 23 | __githash__ = '' 24 | 25 | # set up the test command 26 | def _get_test_runner(): 27 | import os 28 | from astropy.tests.helper import TestRunner 29 | return TestRunner(os.path.dirname(__file__)) 30 | 31 | def test(package=None, test_path=None, args=None, plugins=None, 32 | verbose=False, pastebin=None, remote_data=False, pep8=False, 33 | pdb=False, coverage=False, open_files=False, **kwargs): 34 | """ 35 | Run the tests using `py.test `__. A proper set 36 | of arguments is constructed and passed to `pytest.main`_. 37 | 38 | .. _py.test: http://pytest.org/latest/ 39 | .. _pytest.main: http://pytest.org/latest/builtin.html#pytest.main 40 | 41 | Parameters 42 | ---------- 43 | package : str, optional 44 | The name of a specific package to test, e.g. 'io.fits' or 'utils'. 45 | If nothing is specified all default tests are run. 46 | 47 | test_path : str, optional 48 | Specify location to test by path. May be a single file or 49 | directory. Must be specified absolutely or relative to the 50 | calling directory. 51 | 52 | args : str, optional 53 | Additional arguments to be passed to pytest.main_ in the ``args`` 54 | keyword argument. 55 | 56 | plugins : list, optional 57 | Plugins to be passed to pytest.main_ in the ``plugins`` keyword 58 | argument. 59 | 60 | verbose : bool, optional 61 | Convenience option to turn on verbose output from py.test_. Passing 62 | True is the same as specifying ``'-v'`` in ``args``. 63 | 64 | pastebin : {'failed','all',None}, optional 65 | Convenience option for turning on py.test_ pastebin output. Set to 66 | ``'failed'`` to upload info for failed tests, or ``'all'`` to upload 67 | info for all tests. 68 | 69 | remote_data : bool, optional 70 | Controls whether to run tests marked with @remote_data. These 71 | tests use online data and are not run by default. Set to True to 72 | run these tests. 73 | 74 | pep8 : bool, optional 75 | Turn on PEP8 checking via the `pytest-pep8 plugin 76 | `_ and disable normal 77 | tests. Same as specifying ``'--pep8 -k pep8'`` in ``args``. 78 | 79 | pdb : bool, optional 80 | Turn on PDB post-mortem analysis for failing tests. Same as 81 | specifying ``'--pdb'`` in ``args``. 82 | 83 | coverage : bool, optional 84 | Generate a test coverage report. The result will be placed in 85 | the directory htmlcov. 86 | 87 | open_files : bool, optional 88 | Fail when any tests leave files open. Off by default, because 89 | this adds extra run time to the test suite. Requires the 90 | `psutil` package. 91 | 92 | parallel : int, optional 93 | When provided, run the tests in parallel on the specified 94 | number of CPUs. If parallel is negative, it will use the all 95 | the cores on the machine. Requires the 96 | `pytest-xdist `_ plugin 97 | installed. Only available when using Astropy 0.3 or later. 98 | 99 | kwargs 100 | Any additional keywords passed into this function will be passed 101 | on to the astropy test runner. This allows use of test-related 102 | functionality implemented in later versions of astropy without 103 | explicitly updating the package template. 104 | 105 | """ 106 | test_runner = _get_test_runner() 107 | return test_runner.run_tests( 108 | package=package, test_path=test_path, args=args, 109 | plugins=plugins, verbose=verbose, pastebin=pastebin, 110 | remote_data=remote_data, pep8=pep8, pdb=pdb, 111 | coverage=coverage, open_files=open_files, **kwargs) 112 | 113 | if not _ASTROPY_SETUP_: 114 | import os 115 | from warnings import warn 116 | from astropy import config 117 | 118 | # add these here so we only need to cleanup the namespace at the end 119 | config_dir = None 120 | 121 | if not os.environ.get('ASTROPY_SKIP_CONFIG_UPDATE', False): 122 | config_dir = os.path.dirname(__file__) 123 | config_template = os.path.join(config_dir, __package__ + ".cfg") 124 | if os.path.isfile(config_template): 125 | try: 126 | config.configuration.update_default_config( 127 | __package__, config_dir, version=__version__) 128 | except TypeError as orig_error: 129 | try: 130 | config.configuration.update_default_config( 131 | __package__, config_dir) 132 | except config.configuration.ConfigurationDefaultMissingError as e: 133 | wmsg = (e.args[0] + " Cannot install default profile. If you are " 134 | "importing from source, this is expected.") 135 | warn(config.configuration.ConfigurationDefaultMissingWarning(wmsg)) 136 | del e 137 | except: 138 | raise orig_error 139 | -------------------------------------------------------------------------------- /drivers/Long2pos_K_driver.py: -------------------------------------------------------------------------------- 1 | 2 | # Help, bugs to: http://mosfire.googlecode.com 3 | # 4 | # Instructions 5 | # 1. edit band = '' to band = 'Y' or 'J' or 'H' or 'K' 6 | # e.g. band = 'J' 7 | # 2. edit [709, 1350] to be the pixel values at the beginning and end 8 | # of the long slit. Look at the raw data. 9 | # 3. edit row_position to be a location where the standard star is not. 10 | # 4. Decide if you want to use sky lines or Neon lamps for lambda calibration 11 | # 5. Uncomment one line at a time and run mospy on the driver file 12 | # 13 | 14 | import os, time 15 | import MOSFIRE 16 | 17 | from MOSFIRE import Background, Combine, Detector, Flats, IO, Options, Rectify 18 | from MOSFIRE import Wavelength, Longslit 19 | 20 | import numpy as np 21 | from matplotlib import pyplot as pl 22 | try: 23 | from astropy.io import fits as pf 24 | except: 25 | import pyfits as pf 26 | 27 | np.seterr(all="ignore") 28 | 29 | maskname = 'long2pos' 30 | band = 'K' 31 | 32 | flatops = Options.flat 33 | waveops = Options.wavelength 34 | 35 | # Note: for long2pos, the row position is ignored, and the middle point of the slit is used 36 | longslit = {'yrange': [[1062,1188],[887,1010]], 'row_position': 0, 'mode':'long2pos'} 37 | 38 | 39 | # SETUP FILES FOR DATES before June 10, 2015 40 | 41 | #obsfiles_posC_narrow = ['Offset_-21_HIP85871_7.25.txt', 'Offset_-7_HIP85871_7.25.txt'] 42 | #targetCnarrow = "HIP85871_posC_narrow" 43 | 44 | #obsfiles_posA_narrow = ['Offset_7_HIP85871_7.25.txt', 'Offset_21_HIP85871_7.25.txt'] 45 | #targetAnarrow = "HIP85871_posA_narrow" 46 | 47 | #obsfiles_posC_wide = ['Offset_-14_HIP85871_7.25.txt','Offset_-7_HIP85871_7.25.txt'] 48 | #targetCwide = "HIP85871_posC_wide" 49 | 50 | #obsfiles_posA_wide = ['Offset_14_HIP85871_7.25.txt','Offset_21_HIP85871_7.25.txt'] 51 | #targetAwide = "HIP85871_posA_wide" 52 | 53 | # SETUP FILES for DATES after June 10,2015 54 | 55 | obsfiles_posC_narrow = ['Offset_7_FS134_posC.txt','Offset_-7_FS134_PosC.txt'] 56 | targetCnarrow = "FS134_posC_narrow" 57 | obsfiles_posA_narrow = ['Offset_7_FS134_posA.txt','Offset_-7_FS134_PosA.txt'] 58 | targetAnarrow = "FS134_posA_narrow" 59 | 60 | 61 | # Argon files 62 | argon = ['Ar.txt'] 63 | 64 | #Flats.handle_flats('Flat.txt', maskname, band, flatops, longslit = longslit) 65 | #Flats.handle_flats('Flat.txt', maskname, band, flatops,lampOffList='FlatThermal.txt', longslit=longslit) 66 | 67 | # Uses the argon calibration taken in the afternoon with long2pos for the wavelength calibration 68 | 69 | #Wavelength.imcombine(argon, maskname, band, waveops) 70 | #Wavelength.fit_lambda_interactively(maskname, band, argon, waveops, longslit=longslit, argon=True) 71 | #Wavelength.fit_lambda(maskname, band, argon, argon, waveops, longslit=longslit) 72 | #Wavelength.apply_lambda_simple(maskname, band, argon, waveops, longslit=longslit, smooth=True) 73 | 74 | 75 | ########### NARROW SLITS ############ 76 | 77 | # Long2POS: NARROW SLITS: use the -7 and -21 for posC and 7 and 21 for posA 78 | 79 | obsfiles = obsfiles_posC_narrow 80 | target = targetCnarrow 81 | 82 | #IO.fix_long2pos_headers(obsfiles) 83 | 84 | #Background.handle_background(obsfiles, 85 | # 'lambda_solution_wave_stack_K_m150610_0168-0170.fits', 86 | # maskname, band, waveops, plan=[["A","B"]], target=target) 87 | 88 | redfiles = ["eps_" + file + ".fits" for file in obsfiles] 89 | #update the "lambda_solution_wave_stack_K*.fits" file name 90 | # to the output file name from the apply_lambda process above. 91 | # Update the name of the first file in the offset file (use the full path name. 92 | # e.g. "/Users/user1/MOSFIRE/DRP_CODE/DATA/2014may08/m130114_0451.fits", 93 | 94 | #Rectify.handle_rectification(maskname, redfiles, 95 | # "lambda_solution_wave_stack_K_m150610_0168-0170.fits", 96 | # band, 97 | # obsfiles, 98 | # waveops, target=target) 99 | 100 | 101 | obsfiles = obsfiles_posA_narrow 102 | target = targetAnarrow 103 | 104 | #IO.fix_long2pos_headers(obsfiles) 105 | 106 | #Background.handle_background(obsfiles, 107 | # 'lambda_solution_wave_stack_K_m150610_0168-0170.fits', 108 | # maskname, band, waveops, plan=[["A","B"]], target=target) 109 | 110 | redfiles = ["eps_" + file + ".fits" for file in obsfiles] 111 | #update the "lambda_solution_wave_stack_K*.fits" file name 112 | # to the output file name from the apply_lambda process above. 113 | # Update the name of the first file in the offset file (use the full path name. 114 | # e.g. "/Users/user1/MOSFIRE/DRP_CODE/DATA/2014may08/m130114_0451.fits", 115 | 116 | Rectify.handle_rectification(maskname, redfiles, 117 | "lambda_solution_wave_stack_K_m150610_0168-0170.fits", 118 | band, 119 | obsfiles, 120 | waveops, target=target) 121 | 122 | ########### WIDE SLITS ############ 123 | 124 | # SPECTROPHOTOMETRIC Long2POS: THIS SECTION IS FOR THE REDUCTION OF THE WIDE SLITS. 125 | 126 | #obsfiles = obsfiles_posC_wide 127 | #target = targetCwide 128 | 129 | #IO.fix_long2pos_headers(obsfiles) 130 | 131 | #Background.handle_background(obsfiles, 132 | # 'lambda_solution_wave_stack_H_m150428_0091-0091.fits', 133 | # maskname, band, waveops, plan=[["A","B"]], target=target) 134 | 135 | #obsfiles = obsfiles_posA_wide 136 | #target = targetAwide 137 | 138 | #IO.fix_long2pos_headers(obsfiles) 139 | 140 | #Background.handle_background(obsfiles, 141 | # 'lambda_solution_wave_stack_H_m150428_0091-0091.fits', 142 | # maskname, band, waveops, plan=[["A","B"]], target=target) 143 | 144 | 145 | 146 | 147 | 148 | 149 | # NEON 150 | # Use neon for wavelength calibrations 151 | #Wavelength.imcombine('Ne.txt', maskname, band, waveops) 152 | #Wavelength.fit_lambda_interactively(maskname, band, 'Ne.txt', waveops, longslit=longslit, neon=True) 153 | #Wavelength.fit_lambda(maskname, band, 'Ne.txt', 'Ne.txt', waveops, longslit=longslit) 154 | #Wavelength.apply_lambda_simple(maskname, band, 'Ne.txt', waveops, longslit=longslit, smooth=True) 155 | #print redfiles 156 | #obsfiles = ['eps_off_-10.txt', 'eps_off_10.txt'] 157 | # Update the following line after the apply_lambda_simple step 158 | #Longslit.go(maskname, band, obsfiles, 159 | # 'lambda_solution_wave_stack_H_m150112_0199-0201.fits', 160 | # waveops, longslit, extension='/Volumes/PromiseRAID/MOSFIRE/DRP_CODE/DATA/2015jan12/m150112_0199.fits') 161 | 162 | 163 | 164 | -------------------------------------------------------------------------------- /manual/docs/long2pos.md: -------------------------------------------------------------------------------- 1 | # Long2pos Reductions 2 | 3 | A special driver is provided for long2pos reductions. The driver can also be generated automatically. 4 | 5 | As s reminder, these observations are taken using a script which is run either from the command line (acq_long2pos) or via the background menu. The script produces different results depending on whether the long2pos mask was setup in science mode (only narrow slits) or in alignment mode (narrow and wide slits). 6 | 7 | In the general case of a combination of narrow and wide slits, each run of the script generates 6 images, 3 for each of the two slits. We will refer to the two positions as position A and position C (position B is the intial position used only for alignment). 8 | 9 | Depending on when your data was generated, you might find different Offset files in your directory. Files generated before June 10, 2015 use a different set of YOFFSET keywords than files generated after that date. Unfortunately, the set of keywords generated before June 10, 2015 is not compatible with the pipeline and must be updated: for this we provide a special set of instructions as part of as the driver file to automatically update the keywords. 10 | 11 | For files generated before June 10, 2015, you will find 6 Offset files, named Offset_XXX_object_name_PosY.txt, where XXX can be -21, -14,-7, 7, 14 and 21, the object name is taken from the object keyword, and Y can be either A or C. Similar names are produced if the observations has the correct keywords, but in that case XXX will be one of -7, 0, or 7. 12 | 13 | It is important to notice that the reduction described here is based on the assumption that proper arc lamps are obtained in the afternoon. Specifically, either a Ne or Ar calibration must be obtained with the long2pos mask executed in science mode, and not in alignment mode. In science mode the wide part of the slits is not present. If the slit was executed in alignment mode, the wide part of the slits would prevent a wavelength calibration. 14 | 15 | Note that this also means that if you took your science data at night in long2pos_specphot mode, the mask name of your science file might be long2pos_specphot, rather than long2pos, and the arcs and flats might end up in the wrong subdirectory when the files are processed via mospy handle. In this case it will be necessary to copy Ar.txt, Ne.txt and Flat*.txt from the directory long2pos to your long2pos_specphot directory. 16 | 17 | Let’s now look at the driver file. The declaration "longslit =" is used to define the pixel boundaries of the long2pos observations. In general, it is correct and should not be changed. It might need to be updated in the future is a new long2pos mask is used. Note that it is important to specify ‘mode’=’long2pos’ 18 | 19 | The following section describes the rather long list of Offset files that we will use for the reduction. 20 | 21 | For observations obtained before June 10, 2015, this section might look like this: 22 | 23 | obsfiles_posC_narrow = ['Offset_-21_HIP85871_PosC.txt', 'Offset_-7_HIP85871_PosC.txt'] 24 | targetCnarrow = "HIP85871_posC_narrow" 25 | 26 | obsfiles_posA_narrow = ['Offset_7_HIP85871_PosA.txt', 'Offset_21_HIP85871_PosA.txt'] 27 | targetAnarrow = "HIP85871_posA_narrow" 28 | 29 | obsfiles_posC_wide = ['Offset_-14_HIP85871_PosC.txt','Offset_-7_HIP85871_PosC.txt'] 30 | targetCwide = "HIP85871_posC_wide" 31 | 32 | obsfiles_posA_wide = ['Offset_14_HIP85871_PosA.txt','Offset_21_HIP85871_PosA.txt'] 33 | targetAwide = "HIP85871_posA_wide" 34 | 35 | Files -21_PosC and -7_PosC are the A and B positions for the C pointing, files 7 and 21 are the A and B positions for the A pointing. For the wise slits, file 7_PosC is used as a sky (B) for the -14_PosC position, and file 21_PosA is used as a sky for the 14_PosA position. The target keywords must also be specified to avoid accidental overwrite of intermediate files. 36 | 37 | For files obtained after June 10, 2015, the same section would look like this: 38 | 39 | obsfiles_posC_narrow = ['Offset_7_FS134_posC.txt','Offset_-7_FS134_PosC.txt'] 40 | targetCnarrow = "FS134_posC_narrow" 41 | obsfiles_posA_narrow = ['Offset_7_FS134_posA.txt','Offset_-7_FS134_PosA.txt'] 42 | targetAnarrow = "FS134_posA_narrow" 43 | obsfiles_posC_wide = ['Offset_0_FS134_posC.txt','Offset_-7_FS134_PosC.txt'] 44 | targetCwide = "FS134_posC_wide" 45 | obsfiles_posA_wide = ['Offset_0_FS134_posA.txt','Offset_-7_FS134_PosA.txt'] 46 | targetAwide = "FS134_posA_wide" 47 | 48 | The first step is to produce a flat field. 49 | 50 | Flats.handle_flats('Flat.txt', maskname, band, flatops, longslit = longslit) 51 | 52 | or 53 | 54 | Flats.handle_flats('Flat.txt', maskname, band, flatops,lampOffList='FlatThermal.txt', longslit=longslit) 55 | 56 | Using argon (or neon) lines, we can now produce a wavelength calibration. 57 | 58 | Wavelength.imcombine(argon, maskname, band, waveops) 59 | Wavelength.fit_lambda_interactively(maskname, band, argon, waveops, longslit=longslit, argon=True) 60 | Wavelength.fit_lambda(maskname, band, argon, argon, waveops, longslit=longslit) 61 | Wavelength.apply_lambda_simple(maskname, band, argon, waveops, longslit=longslit, smooth=True) 62 | 63 | While using the interactive fitting, note that there are two slits to fit. 64 | 65 | The next section of the driver reduces the narrow slits. The optional line 66 | 67 | IO.fix_long2pos_headers(obsfiles) 68 | 69 | is ONLY necessary if your observations were taken before June 10, 2015. It is safe to leave this line on for a second run: the script will not modify the same files twice. 70 | 71 | Rememeber to update the lambda_solution_wave_stack file: you can update this in the variable wavelength_file, which will be used by the following instructions. 72 | 73 | The driver contains instructions on how to perform background subtraction and finally rectification, in a similar way as for a normal mask. 74 | 75 | The resulting files are the same as in the standard reduction, but the main results are contained in: 76 | 77 | {object}_posA_narrow_{filter}_eps.fits 78 | 79 | and 80 | 81 | {object}_posC_narrow_{filter}_eps.fits 82 | 83 | For the wide slits, since there is no AB pattern, we use the sky provided by one of the observations in the narrow slits, and we do not perform the final rectification. 84 | 85 | In this case the final science results are contained in: 86 | 87 | bsub_{object}_posC_wide_{filter}_A-B.fits 88 | 89 | and 90 | 91 | bsub_{object}_posA_wide_{filter}_A-B.fits 92 | 93 | 94 | -------------------------------------------------------------------------------- /apps/handle.py~: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python 2 | 3 | ''' 4 | MOSFIRE 'handle' command: 5 | 6 | (c) npk - Dec 2013 7 | ''' 8 | import MOSFIRE 9 | import MOSFIRE.IO as IO 10 | import os 11 | import numpy as np 12 | import pyfits as pf 13 | import sys 14 | import glob 15 | 16 | 17 | 18 | if len(sys.argv) < 3: 19 | print '''Usage: mospy handle [target]''' 20 | sys.exit() 21 | 22 | files = [] 23 | for i in range(1, len(sys.argv)): 24 | files.extend(glob.iglob(sys.argv[i])) 25 | 26 | masks = {} 27 | 28 | for fname in files: 29 | 30 | try: 31 | header = MOSFIRE.IO.readheader(fname) 32 | except IOError, err: 33 | print "Couldn't IO %s" % fname 34 | continue 35 | except: 36 | print "%s is unreadable" % fname 37 | continue 38 | 39 | lamps = "" 40 | try: 41 | if header["pwstata7"] == 1: 42 | lamps += header["pwloca7"][0:2] 43 | if header["pwstata8"] == 1: 44 | lamps += header["pwloca8"][0:2] 45 | except KeyError: 46 | lamps = "???" 47 | 48 | header.update("lamps", lamps) 49 | 50 | try: 51 | if header["aborted"]: 52 | header.update("object", "ABORTED") 53 | except: 54 | print "Missing header file in: %s" % fname 55 | 56 | try: 57 | print "%(datafile)12s %(object)40s %(truitime)6.1f s %(maskname)35s %(lamps)3s %(filter)4s %(mgtname)7s" % (header) 58 | except: 59 | try: 60 | print "%(datafile)12s %(object)25s %(truitime)6.1f s %(lamps)3s %(filter)6s %(mgtname)7s" % (header) 61 | except: 62 | print "%s Skipped" % fname 63 | continue 64 | 65 | 66 | datafile = header['datafile'] + '.fits' 67 | maskname = header['maskname'] 68 | filter = header['filter'] 69 | yr,mn,dy = IO.fname_to_date_tuple(datafile) 70 | date = str(yr)+mn+str(dy) 71 | object = header['object'] 72 | 73 | itime = header['truitime'] 74 | grating_turret = header['mgtname'] 75 | 76 | if object.find("MIRA") == -1: mira = False 77 | else: mira = True 78 | 79 | if maskname.find(" (align)") == -1: 80 | align = False 81 | else: 82 | maskname = maskname.replace(" (align)", "") 83 | align = True 84 | 85 | if maskname.find('LONGSLIT') != -1: 86 | align = False 87 | 88 | if maskname.find('long2pos') != -1: 89 | if grating_turret != 'mirror': 90 | align = False 91 | 92 | empty_files = {'Align': [], 'Ne': [], 'Ar': [], 'Flat': [], 'FlatThermal': [], 93 | 'Dark': [], 'Aborted': [], 'Image': [], 'MIRA': [], 'Unknown': []} 94 | 95 | if maskname not in masks: 96 | masks[maskname] = {date: {filter: empty_files}} 97 | 98 | if date not in masks[maskname]: 99 | masks[maskname][date] = {filter: empty_files} 100 | 101 | if filter not in masks[maskname][date]: 102 | masks[maskname][date][filter] = empty_files 103 | 104 | 105 | offset = 'Offset_' + str(header['YOFFSET']) 106 | 107 | 108 | 109 | if mira: 110 | masks[maskname][date][filter]['MIRA'].append(fname) 111 | elif align: 112 | masks[maskname][date][filter]['Align'].append(fname) 113 | elif 'Ne' in header['lamps']: 114 | masks[maskname][date][filter]['Ne'].append(fname) 115 | elif 'Ar' in header['lamps']: 116 | masks[maskname][date][filter]['Ar'].append(fname) 117 | elif header['ABORTED']: 118 | masks[maskname][date][filter]['Aborted'].append(fname) 119 | elif header['FILTER'] == 'Dark': 120 | masks[maskname][date][filter]['Dark'].append(fname) 121 | elif header['FLATSPEC'] == 1: 122 | masks[maskname][date][filter]['Flat'].append(fname) 123 | elif object.find("Flat:") != -1 and ( object.find("lamps off") != -1 or object.find("Flats:Off")) != -1 : 124 | masks[maskname][date][filter]['FlatThermal'].append(fname) 125 | elif header['mgtname'] == 'mirror': 126 | masks[maskname][date][filter]['Image'].append(fname) 127 | elif offset != 0: 128 | if offset in masks[maskname][date][filter]: 129 | masks[maskname][date][filter][offset].append((fname, itime)) 130 | else: 131 | masks[maskname][date][filter][offset] = [(fname, itime)] 132 | else: 133 | masks[maskname][date][filter]['Unknown'].append(fname) 134 | 135 | 136 | 137 | ##### Now handle mask dictionary 138 | 139 | def descriptive_blurb(): 140 | import getpass, time 141 | 142 | uid = getpass.getuser() 143 | date = time.asctime() 144 | 145 | return "# Created by '%s' on %s\n" % (uid, date) 146 | 147 | 148 | # Write out the list of files in filepath 149 | # list = ['/path/to/mYYmmDD_####.fits' ...] 150 | # filepath is absolute path to the file name to write to 151 | # 152 | # Result, is a file called filepath is written with 153 | # fits files in the list. 154 | def handle_file_list(output_file, files): 155 | '''Write a list of paths to MOSFIRE file to output_file.''' 156 | 157 | if os.path.isfile(output_file): 158 | print "%s: already exists, skipping" % output_file 159 | pass 160 | 161 | print "\t", output_file 162 | f = open(output_file, "w") 163 | f.write(descriptive_blurb()) 164 | if len(files) == 0: 165 | f.close() 166 | return 167 | 168 | picker = lambda x: x 169 | if len(files[0]) == 2: picker = lambda x: x[0] 170 | 171 | # Identify unique path to files: 172 | paths = [os.path.dirname(picker(file)) for file in files] 173 | paths = list(set(paths)) 174 | 175 | if len(paths) == 1: 176 | path_to_all = paths[0] 177 | converter = os.path.basename 178 | f.write("%s # Abs. path to files [optional]\n" % path_to_all) 179 | else: 180 | converter = lambda x: x 181 | 182 | 183 | for path in files: 184 | if len(path) == 2: to_write = "%s # %s s\n" % (converter(path[0]), path[1]) 185 | else: to_write = "%s\n" % converter(path) 186 | 187 | f.write("%s" % to_write) 188 | 189 | 190 | f.close() 191 | 192 | def handle_date_and_filter(mask, date, filter, mask_info): 193 | 194 | path = os.path.join(mask,date,filter) 195 | try: os.makedirs(path) 196 | except OSError: pass 197 | 198 | for type in mask_info.keys(): 199 | handle_file_list(os.path.join(path, type + ".txt"), mask_info[type]) 200 | 201 | 202 | for mask in masks.keys(): 203 | for date in masks[mask].keys(): 204 | for filter in masks[mask][date].keys(): 205 | handle_date_and_filter(mask, date, filter, masks[mask][date][filter]) 206 | 207 | 208 | -------------------------------------------------------------------------------- /MOSFIRE/Combine.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | import pdb 4 | 5 | import numpy as np 6 | try: 7 | from astropy.io import fits as pf 8 | except: 9 | import pyfits as pf 10 | 11 | 12 | import MOSFIRE 13 | 14 | from MOSFIRE import Background, IO, Wavelength 15 | 16 | 17 | def imcombine(filelist, maskname, fname, options, sum_type): 18 | ''' combine the images in file list into fname. 19 | 20 | Sum type: 21 | rate -- filelist is in cnt/s 22 | ivar-rate -- filelist is in s/cnt 23 | snr-rate -- filelist is in SNR 24 | ''' 25 | 26 | ARR = None 27 | hdr = None 28 | i = 1 29 | 30 | itime = 0 31 | 32 | for file in filelist: 33 | 34 | this_hdr, img = IO.readfits(file) 35 | cards = this_hdr.ascardlist() 36 | 37 | thisitime = this_hdr['truitime'] 38 | itime += thisitime 39 | 40 | if ARR is None: ARR = np.zeros(img.shape) 41 | 42 | if sum_type == 'rate': ARR += img * thisitime 43 | if sum_type == 'ivar-rate': ARR += thisitime/img 44 | if sum_type == 'snr-rate': ARR += img * thisitime 45 | 46 | 47 | if hdr is None: 48 | hdr = this_hdr 49 | hdr["fno%2.2i" % i] = (file, "--") 50 | for card in cards: 51 | key, value, comment = (card.key, card.value, card.comment) 52 | 53 | if hdr.has_key(key) and hdr[key] != value: 54 | key = key + ("_img%2.2i" % i) 55 | 56 | if len(key) > 8: key = 'HIERARCH ' + key 57 | 58 | try: 59 | hdr[key] = (value, comment) 60 | except ValueError: 61 | pass 62 | 63 | hdr['itime'] = (itime, 'Itime for %i rectified images' % len(filelist)) 64 | if sum_type == 'rate': ARR /= itime 65 | if sum_type == 'ivar-rate': ARR = itime/ARR 66 | if sum_type == 'snr-rate': ARR /= itime 67 | 68 | IO.writefits(ARR, maskname, fname, options, header=hdr, overwrite=True, 69 | lossy_compress=True) 70 | 71 | def get_path(a): 72 | if os.path.exists(a): return a 73 | 74 | a += ".gz" 75 | if os.path.exists(a): return a 76 | 77 | raise Exception("No such path: %s" % a) 78 | 79 | def gz(name): 80 | if name[-2:] == 'gz': return '.gz' 81 | else: return '' 82 | 83 | def stack_rectified(wavenames, maskname, band, wavops): 84 | N = len(wavenames) 85 | lamnames = [] 86 | suffixs = [] 87 | 88 | for i in xrange(N): 89 | lamnames.append( Wavelength.filelist_to_wavename(wavenames[i], band, 90 | maskname, wavops).rstrip(".fits")) 91 | suffixs.append(lamnames[i].lstrip("wave_stack_%s_" % band)) 92 | 93 | path = os.path.join(wavops["outdir"], maskname) 94 | 95 | 96 | recs = [] 97 | ivars = [] 98 | sns = [] 99 | 100 | 101 | try: 102 | ls = [get_path(os.path.join(path, "eps_%s_%s_%s.fits") % (maskname, 103 | suffix, band)) for suffix in suffixs] 104 | imcombine(ls, maskname, "eps_%s_%s.fits" % (maskname, band), 105 | wavops, 'rate') 106 | except: 107 | pass 108 | 109 | try: 110 | ls = [get_path(os.path.join(path, "ivars_%s_%s_%s.fits") % (maskname, 111 | suffix, band)) for suffix in suffixs] 112 | imcombine(ls, maskname, "ivars_%s_%s.fits" % (maskname, 113 | band), wavops, 'ivar-rate') 114 | except: 115 | pass 116 | 117 | try: 118 | ls = [get_path(os.path.join(path, "snrs_%s_%s_%s.fits") % (maskname, 119 | suffix, band)) for suffix in suffixs] 120 | imcombine(ls, maskname, "snrs_%s_%s.fits" % (maskname, band), 121 | wavops, 'snr-rate') 122 | except: 123 | pass 124 | 125 | 126 | 127 | 128 | def stack_slits(wavenames, maskname, band, wavops): 129 | pass 130 | 131 | def stack_full(wavenames, maskname, band, wavops): 132 | pass 133 | 134 | def stack_files(wavenames, maskname, band, wavops): 135 | stack_rectified(wavenames, maskname, band, wavops) 136 | stack_slits(wavenames, maskname, band, wavops) 137 | stack_full(wavenames, maskname, band, wavops) 138 | 139 | def rename_files(wavenames, maskname, band, wavops): 140 | 141 | lamname = Wavelength.filelist_to_wavename(wavenames[0], band, maskname, 142 | wavops).rstrip(".fits") 143 | 144 | suffix = lamname.lstrip("wave_stack_%s_" % band) 145 | 146 | path = os.path.join(wavops["outdir"], maskname) 147 | 148 | fnames = ["rectified_%s%s.fits", "rectified_ivar_%s%s.fits", 149 | "rectified_sn_%s%s.fits"] 150 | 151 | for fname in fnames: 152 | try: 153 | a = get_path(os.path.join(path, fname % (band, "_" + suffix))) 154 | b = os.path.join(path, fname % (band, "")) + gz(a) 155 | os.rename(a, b) 156 | except: 157 | print("Ignoring renaming of: ", fname) 158 | pass 159 | 160 | 161 | edges = IO.load_edges(maskname, band, wavops) 162 | n_slits = len(edges[0]) 163 | 164 | for i in xrange(1, n_slits+1): 165 | S = "S%2.2i" % (i) 166 | a = get_path(os.path.join(path, 167 | "eps_%s_%s_%s.fits" % (band, suffix, S))) 168 | 169 | a_h = pf.open(a)[0].header 170 | obj = a_h['object'] 171 | 172 | b = os.path.join(path, "%s_%s_%s_eps.fits" % (maskname, band, obj)) + \ 173 | gz(a) 174 | os.rename(a,b) 175 | 176 | a = get_path(os.path.join(path, 177 | "ivar_%s_%s_%s.fits" % (band, suffix, S))) 178 | a_h = pf.open(a)[0].header 179 | obj = a_h['object'] 180 | 181 | b = os.path.join(path, "%s_%s_%s_ivar.fits" % (maskname, band, obj)) + \ 182 | gz(a) 183 | os.rename(a,b) 184 | 185 | a = get_path(os.path.join(path, 186 | "eps_%s_%s_%s.fits" % (maskname, suffix, band))) 187 | b = os.path.join(path, 188 | "%s_%s_eps.fits" % (maskname, band)) + gz(a) 189 | os.rename(a,b) 190 | 191 | a = get_path(os.path.join(path, 192 | "snrs_%s_%s_%s.fits" % (maskname, suffix, band))) 193 | b = os.path.join(path, 194 | "%s_%s_snrs.fits" % (maskname, band)) + gz(a) 195 | os.rename(a, b) 196 | 197 | a = get_path(os.path.join(path, 198 | "ivars_%s_%s_%s.fits" % (maskname, suffix, band))) 199 | b = os.path.join(path, 200 | "%s_%s_ivars.fits" % (maskname, band)) + gz(a) 201 | os.rename(a, b) 202 | 203 | def handle_combine(wavenames, maskname, band, wavops): 204 | 205 | N = len(wavenames) 206 | assert(N > 0) 207 | 208 | print("Starting") 209 | 210 | if N == 1: rename_files(wavenames, maskname, band, wavops) 211 | if N > 1: stack_files(wavenames, maskname, band, wavops) 212 | 213 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 3 | # 4 | # Astropy documentation build configuration file. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this file. 9 | # 10 | # All configuration values have a default. Some values are defined in 11 | # the global Astropy configuration which is loaded here before anything else. 12 | # See astropy.sphinx.conf for which values are set there. 13 | 14 | # If extensions (or modules to document with autodoc) are in another directory, 15 | # add these directories to sys.path here. If the directory is relative to the 16 | # documentation root, use os.path.abspath to make it absolute, like shown here. 17 | # sys.path.insert(0, os.path.abspath('..')) 18 | # IMPORTANT: the above commented section was generated by sphinx-quickstart, but 19 | # is *NOT* appropriate for astropy or Astropy affiliated packages. It is left 20 | # commented out with this explanation to make it clear why this should not be 21 | # done. If the sys.path entry above is added, when the astropy.sphinx.conf 22 | # import occurs, it will import the *source* version of astropy instead of the 23 | # version installed (if invoked as "make html" or directly with sphinx), or the 24 | # version in the build directory (if "python setup.py build_sphinx" is used). 25 | # Thus, any C-extensions that are needed to build the documentation will *not* 26 | # be accessible, and the documentation will not build correctly. 27 | 28 | import datetime 29 | import os 30 | import sys 31 | 32 | try: 33 | import astropy_helpers 34 | except ImportError: 35 | # Building from inside the docs/ directory? 36 | if os.path.basename(os.getcwd()) == 'docs': 37 | a_h_path = os.path.abspath(os.path.join('..', 'astropy_helpers')) 38 | if os.path.isdir(a_h_path): 39 | sys.path.insert(1, a_h_path) 40 | 41 | # Load all of the global Astropy configuration 42 | from astropy_helpers.sphinx.conf import * 43 | 44 | # Get configuration information from setup.cfg 45 | from distutils import config 46 | conf = config.ConfigParser() 47 | conf.read([os.path.join(os.path.dirname(__file__), '..', 'setup.cfg')]) 48 | setup_cfg = dict(conf.items('metadata')) 49 | 50 | # -- General configuration ---------------------------------------------------- 51 | 52 | # If your documentation needs a minimal Sphinx version, state it here. 53 | #needs_sphinx = '1.2' 54 | 55 | # To perform a Sphinx version check that needs to be more specific than 56 | # major.minor, call `check_sphinx_version("x.y.z")` here. 57 | # check_sphinx_version("1.2.1") 58 | 59 | # List of patterns, relative to source directory, that match files and 60 | # directories to ignore when looking for source files. 61 | exclude_patterns.append('_templates') 62 | 63 | # This is added to the end of RST files - a good place to put substitutions to 64 | # be used globally. 65 | rst_epilog += """ 66 | """ 67 | 68 | # -- Project information ------------------------------------------------------ 69 | 70 | # This does not *have* to match the package name, but typically does 71 | project = setup_cfg['package_name'] 72 | author = setup_cfg['author'] 73 | copyright = '{0}, {1}'.format( 74 | datetime.datetime.now().year, setup_cfg['author']) 75 | 76 | # The version info for the project you're documenting, acts as replacement for 77 | # |version| and |release|, also used in various other places throughout the 78 | # built documents. 79 | 80 | __import__(setup_cfg['package_name']) 81 | package = sys.modules[setup_cfg['package_name']] 82 | 83 | # The short X.Y version. 84 | version = package.__version__.split('-', 1)[0] 85 | # The full version, including alpha/beta/rc tags. 86 | release = package.__version__ 87 | 88 | 89 | # -- Options for HTML output --------------------------------------------------- 90 | 91 | # A NOTE ON HTML THEMES 92 | # The global astropy configuration uses a custom theme, 'bootstrap-astropy', 93 | # which is installed along with astropy. A different theme can be used or 94 | # the options for this theme can be modified by overriding some of the 95 | # variables set in the global configuration. The variables set in the 96 | # global configuration are listed below, commented out. 97 | 98 | # Add any paths that contain custom themes here, relative to this directory. 99 | # To use a different custom theme, add the directory containing the theme. 100 | #html_theme_path = [] 101 | 102 | # The theme to use for HTML and HTML Help pages. See the documentation for 103 | # a list of builtin themes. To override the custom theme, set this to the 104 | # name of a builtin theme or the name of a custom theme in html_theme_path. 105 | #html_theme = None 106 | 107 | # Custom sidebar templates, maps document names to template names. 108 | #html_sidebars = {} 109 | 110 | # The name of an image file (within the static path) to use as favicon of the 111 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 112 | # pixels large. 113 | #html_favicon = '' 114 | 115 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 116 | # using the given strftime format. 117 | #html_last_updated_fmt = '' 118 | 119 | # The name for this set of Sphinx documents. If None, it defaults to 120 | # " v documentation". 121 | html_title = '{0} v{1}'.format(project, release) 122 | 123 | # Output file base name for HTML help builder. 124 | htmlhelp_basename = project + 'doc' 125 | 126 | 127 | # -- Options for LaTeX output -------------------------------------------------- 128 | 129 | # Grouping the document tree into LaTeX files. List of tuples 130 | # (source start file, target name, title, author, documentclass [howto/manual]). 131 | latex_documents = [('index', project + '.tex', project + u' Documentation', 132 | author, 'manual')] 133 | 134 | 135 | # -- Options for manual page output -------------------------------------------- 136 | 137 | # One entry per manual page. List of tuples 138 | # (source start file, name, description, authors, manual section). 139 | man_pages = [('index', project.lower(), project + u' Documentation', 140 | [author], 1)] 141 | 142 | 143 | ## -- Options for the edit_on_github extension ---------------------------------------- 144 | 145 | if eval(setup_cfg.get('edit_on_github')): 146 | extensions += ['astropy.sphinx.ext.edit_on_github'] 147 | 148 | versionmod = __import__(setup_cfg['package_name'] + '.version') 149 | edit_on_github_project = setup_cfg['github_project'] 150 | if versionmod.version.release: 151 | edit_on_github_branch = "v" + versionmod.version.version 152 | else: 153 | edit_on_github_branch = "master" 154 | 155 | edit_on_github_source_root = "" 156 | edit_on_github_doc_root = "docs" 157 | -------------------------------------------------------------------------------- /MOSFIRE/Longslit.py: -------------------------------------------------------------------------------- 1 | 2 | # MOSFIRE Longslit Reductions 3 | # 5 Aug 2012 4 | # Nick Konidaris 5 | 6 | import os 7 | import pdb 8 | import numpy as np 9 | import scipy 10 | 11 | from MOSFIRE import Detector, IO, Filters, Wavelength 12 | 13 | def rectify(dname, lamdat, A, B, maskname, band, wavoptions, 14 | longoptions): 15 | 16 | header, data = IO.readfits(dname) 17 | raw_img = data * Detector.gain / header['TRUITIME'] 18 | 19 | dlam = Wavelength.grating_results(band) 20 | hpp = np.array(Filters.hpp[band]) 21 | ll_fid = np.arange(hpp[0], hpp[1], dlam) 22 | 23 | rectified = np.zeros((2048, len(ll_fid))) 24 | 25 | from scipy.interpolate import interp1d 26 | 27 | for i in xrange(2048): 28 | ll = lamdat[i,:] 29 | ss = raw_img[i,:] 30 | ok = np.isfinite(ll) & np.isfinite(ss) & (ll < hpp[1]) & (ll > 31 | hpp[0]) 32 | 33 | if len(np.where(ok)[0]) < 30: 34 | continue 35 | 36 | f = interp1d(ll[ok], ss[ok], bounds_error=False) 37 | rectified[i,:] = f(ll_fid) 38 | 39 | header["wat0_001"] = "system=world" 40 | header["wat1_001"] = "wtype=linear" 41 | header["wat2_001"] = "wtype=linear" 42 | header["dispaxis"] = 1 43 | header["dclog1"] = "Transform" 44 | header["dc-flag"] = 0 45 | header["ctype1"] = "AWAV" 46 | header["cunit1"] = "Angstrom" 47 | header["crval1"] = ll_fid[0] 48 | header["crval2"] = 0 49 | header["crpix1"] = 1 50 | header["crpix2"] = 1 51 | header["cdelt1"] = 1 52 | header["cdelt2"] = 1 53 | header["cname1"] = "angstrom" 54 | header["cname2"] = "pixel" 55 | header["cd1_1"] = dlam 56 | header["cd1_2"] = 0 57 | header["cd2_1"] = 0 58 | header["cd2_2"] = 1 59 | 60 | 61 | header["object"] = "rectified [eps]" 62 | IO.writefits(rectified, maskname, "rectified_%s" % (dname), 63 | wavoptions, header=header, overwrite=True, lossy_compress=True) 64 | 65 | 66 | def imdiff(A, B, maskname, band, header, options): 67 | s = "[0]" 68 | 69 | targname = A[1]["targname"].rstrip(" ") 70 | if targname == "": 71 | objname = A[1]["object"].replace(" ", "_") 72 | else: 73 | objname = targname.replace(" ", "_") 74 | 75 | operand1 = A[0] + '[0]' 76 | operand2 = B[0] + '[0]' 77 | 78 | imnumA = A[0].split('_')[-1].rstrip(".fits") 79 | imnumB = B[0].split('_')[-1].rstrip(".fits") 80 | 81 | dname = "{0}_{1}_{2}_{3}-{4}_{5}-{6}.fits".format(maskname, objname, band, 82 | A[1]["frameid"], B[1]["frameid"], imnumA, imnumB) 83 | 84 | try: os.remove(dname) 85 | except:pass 86 | print("Data Diff {0}-{1}".format(operand1,operand2)) 87 | IO.imarith(operand1, '-', operand2, dname) 88 | 89 | ''' Now handle variance ''' 90 | numreads = header["READS0"] 91 | RN_adu = Detector.RN / np.sqrt(numreads) / Detector.gain 92 | varname = "var_{0}_{1}_{2}_{3}+{4}_{5}+{6}.fits".format(maskname, objname, band, 93 | A[1]["frameid"], B[1]["frameid"], imnumA, imnumB) 94 | 95 | 96 | print("Var Sum {0}+{1}".format(operand1,operand2)) 97 | IO.imarith(operand1, '+', operand2, "tmp_" + varname) 98 | try: os.remove(varname) 99 | except: pass 100 | print("Var add RN {0}+{1}".format(operand1,RN_adu**2)) 101 | IO.imarith("tmp_" + varname, '+', RN_adu**2, varname) 102 | 103 | try: os.remove("tmp_" + varname) 104 | except: pass 105 | 106 | 107 | return dname, varname 108 | 109 | def apply_flat(scifilename, maskname, band): 110 | ''' Divides the contents of scifilename by the flat field and 111 | overwrites scifilename with the same file divided by the flat 112 | 113 | Args: 114 | scifilename: Path to science file name. 115 | maskname: The mask name 116 | band: The filter bands 117 | 118 | Results: 119 | Overwrites scifilename where the data contents of the file 120 | are divided by the pixel flat 121 | ''' 122 | 123 | 124 | pixelflat_file = "pixelflat_2d_{0}.fits".format(band) 125 | flat = readfits(pixelflat_file, use_bpm=True) 126 | flat_data = flat[1].filled(1.0) 127 | 128 | header, data = IO.readfits(scifilename) 129 | 130 | print("Applying flat to file {0}".format(scifilename)) 131 | IO.writefits(data/flat_data, maskname, scifilename, {}, header=header, 132 | overwrite=True) 133 | 134 | 135 | 136 | def go(maskname, 137 | band, 138 | filenames, 139 | wavefile, 140 | wavoptions, 141 | longoptions, 142 | use_flat=False): 143 | 144 | ''' 145 | The go command is the main entry point into this module. 146 | 147 | Inputs: 148 | maskname: String of the mask name 149 | band: String of 'Y', 'J', 'H', or 'K' 150 | filenames: List of filenames to reduce 151 | wavefile: String of path to FITS file with the wavelength solution 152 | wavoptions: The Wavelength Options dictionary 153 | longoptions: Dictionary containing: 154 | {'yrange': The pixel range to extract over 155 | 'row_position': The row to solve the initial wavelength solution on} 156 | use_flat: Boolean False [default] means to use no flat field 157 | Boolean True means to divide by the pixelflat 158 | ''' 159 | wavename = Wavelength.filelist_to_wavename(filenames, band, maskname, 160 | wavoptions).rstrip(".fits") 161 | 162 | print("Wavefile: {0}".format(wavefile)) 163 | lamhdr, lamdat = IO.readfits(wavefile) 164 | 165 | positions = [] 166 | objname = None 167 | for listfile in filenames: 168 | fnames = IO.list_file_to_strings(listfile) 169 | if len(fnames) != 1: 170 | raise Exception("I currently expect only one file per position. Remove multiple entries and try again") 171 | 172 | header, data, bs = IO.readmosfits(fnames[0], wavoptions) 173 | 174 | if objname is None: 175 | objname = header["object"] 176 | 177 | if objname != header["object"]: 178 | print ("Trying to combine longslit stack of object {0} " 179 | "with object {1}".format(objname, header["object"])) 180 | 181 | print("{0:18s} {1:30s} {2:2s} {3:4.1f}".format(file, header["object"], 182 | header["frameid"], header["yoffset"])) 183 | 184 | positions.append([fnames[0], header, data, bs]) 185 | 186 | print("{0:2g} nod positions found. Producing stacked difference" \ 187 | " image.".format(len(positions))) 188 | 189 | for i in xrange(len(positions)-1): 190 | A = positions[i] 191 | B = positions[i+1] 192 | 193 | print("----------- -----".format(A,B)) 194 | 195 | dname, varname = imdiff(A, B, maskname, band, header, wavoptions) 196 | if use_flat: 197 | apply_flat(dname, maskname, band) 198 | apply_flat(varname, maskname, band) 199 | 200 | rectify(dname, lamdat, A, B, maskname, band, wavoptions, 201 | longoptions) 202 | rectify(varname, lamdat, A, B, maskname, band, wavoptions, 203 | longoptions) 204 | print dname 205 | 206 | dname, vname = imdiff(B, A, maskname, band, header, wavoptions) 207 | if use_flat: 208 | apply_flat(dname, maskname, band) 209 | apply_flat(vname, maskname, band) 210 | rectify(dname, lamdat, B, A, maskname, band, wavoptions, 211 | longoptions) 212 | rectify(vname, lamdat, B, A, maskname, band, wavoptions, 213 | longoptions) 214 | 215 | if False: 216 | fname = os.path.join(path, wavename + ".fits") 217 | B = IO.readfits(fname) 218 | B = [fname, B[0], B[1]] 219 | for i in xrange(len(positions)): 220 | A = positions[i] 221 | imdiff(A, B, maskname, band, wavoptions) 222 | rectify(path, dname, lamdat, A, B, maskname, band, wavoptions, 223 | longoptions) 224 | imdiff(B, A, maskname, band, wavoptions) 225 | rectify(path, dname, lamdat, B, A, maskname, band, wavoptions, 226 | longoptions) 227 | 228 | 229 | -------------------------------------------------------------------------------- /manual/docs/installing_2015A.md: -------------------------------------------------------------------------------- 1 | As of version 2015A, the pipeline can be installed as Ureka package. If you prefer to use this installation procedure, download the pipeline tar file and unpack it. Then execute: 2 | 3 | ur_setup 4 | 5 | Go to the mosfire python directory where the setup.py file is located (it is the main directory above the MOSFIRE and apps directories). Run: 6 | 7 | python setup.py install 8 | 9 | This will copy the MOSFIRE pipeline as a python package located into your ureka enviornment. 10 | 11 | Alternatively, if you would like to modify the MOSFIRE pipeline files, run the following instead: 12 | 13 | python setup.py develop 14 | 15 | This will make symoblic links to the MOSFIRE files instead of copying them. Your changes to the pipeline file will now automatically be used when loading the MOSFIRE package. 16 | 17 | Directories created by you 18 | 19 | * DATA – sub directory in which you can store your data. This is not a necessary sub-directory but may help you manage files. Raw data may be stored in other areas on your disk. 20 | * REDUX – sub directory where reductions will be stored. Also not critical, but helpful. 21 | 22 | From now on, if you want to run any pipeline commands, you will always execute “mospy” as seen in our first step in section 5 below. Remember to run ur_setup before running the MOSFIRE pipleine. 23 | 24 | ## Alternate Installation Method 25 | 26 | If you prefer the previous installation method, follow these instructions: 27 | 28 | **1) Install Ureka** 29 | 30 | The pipeline relies on the Ureka Python distribution produced by STScI and Gemini Observatory: [http://ssb.stsci.edu/ureka/](http://ssb.stsci.edu/ureka/) 31 | 32 | The DRP was developed using UREKA version 1.0. Navigate to the 1.0 distribution using the url listed above. Follow the instructions at the links to install the package. The UREKA instructions indicate that you need to run `ur_setup` to put ureka in the path. This is automatically completed when you run the drp and it is found in the mospy code. However, if you want to test the ureka package yourself, you will need to run ur_setup manually. The latest version of Ureka that is confirmed to work with the pipeline is 1.5.1 33 | 34 | **2) Download the pipeline** 35 | 36 | 1. Start an xterm session on your local machine 37 | 2. Run “cd” to navigate to the home directory 38 | 3. Download either the .zip file, or the .tar.gz file from the website [https://keck-datareductionpipelines.github.io/MosfireDRP/](https://keck-datareductionpipelines.github.io/MosfireDRP/). Note that this is the stable and supported version of the pipeline. Alternatively, if you are a github user, you can just clone the repository using: `https://github.com/keck-DataReductionPipelines/MosfireDRP.git`. This is the development version, and it is NOT supported. 39 | 4. Expand the zip or tar file and rename the resulting directory. For example: 40 | 41 | ``` 42 | mkdir ~/MOSFIRE 43 | mv MosfireDRP-1.1 ~/MOSFIRE/DRP_CODE 44 | cd ~/MOSFIRE/DRP_CODE # to navigate to that directory 45 | ``` 46 | 47 | 48 | **3) Create Data Directories** 49 | 50 | Create sub directories for raw data, and reduced data. These sub directories are not specific. You can set up sub directories any way you would like. For the purposes of this manual, we have choosen generic directory names. You may choose to store the raw and reduced data using andy directory structure you would prefer. For our example, we created a raw data directory in the code repository: 51 | 52 | mkdir ~/MOSFIRE/DRP_CODE/DATA 53 | 54 | and a reduction directory in the code repository that will store reduced data: 55 | 56 | mkdir ~/MOSFIRE/DRP_CODE/REDUX 57 | 58 | 59 | **4) Copy the mospy file into your bin dir** 60 | 61 | Navigate to the newly creted bin dir: 62 | 63 | cd ~/MOSFIRE/DRP_CODE/bin 64 | 65 | Copy the mospy executeable to the bin dir 66 | 67 | cp ../apps/mospy . 68 | 69 | **5) edit mospy in your bin dir and update a few lines of code** 70 | 71 | Using your favorite editor (emacs ../bin/mospy), update the path for the `ur_setup`. Replace `/home/npk/.ureka/ur_setup` with your `/your_full_path_name/.ureka/ur_setup` full path. 72 | 73 | Update the path for the `ur_forget`. Replace `/home/npk/.ureka/ur_setup` with `/your_full_path_name/.ureka/ur_forget` 74 | 75 | Update the MOSPATH with the full path to the source code directory. Replace `/src2/mosfire/DRP/mosfire` with `/your_full_path_name/MOSFIRE/DRP_CODE` 76 | 77 | As an example, the original file might look like the following: 78 | 79 | #Update the full path to the ureka install for the 80 | # two aliases below. 81 | alias ur_setup 'eval `/home/npk/.ureka/ur_setup -csh \!*`' 82 | alias ur_forget 'eval `/home/npk/.ureka/ur_forget -csh \!*`' 83 | 84 | # If pythonpath is not previously defined, define it so that 85 | # the setenv works below.. 86 | if (! $?PYTHONPATH ) setenv PYTHONPATH 87 | 88 | #Update the full path to the mosfire DRP code repository 89 | # example: /src2/mosfire/DRP/mosfire change to /Users/myname/MOSFIRE/DRP_CODE 90 | # in which the sub dirs drivers, apps, badpixel, etc. live 91 | setenv MOSPATH /scr2/mosfire/DRP/mosfire 92 | setenv PYTHONPATH ${PYTHONPATH}:${MOSPATH} 93 | 94 | And the modified version for an observers particular setup may look something like this: 95 | 96 | #Update the full path to the ureka install for the 97 | # two aliases below. 98 | alias ur_setup 'eval `/Users/mkassis/.ureka/ur_setup -csh \!*`' 99 | alias ur_forget 'eval `/Users/mkassis/.ureka/ur_forget -csh \!*`' 100 | 101 | # If pythonpath is not previously defined, define it so that 102 | # the setenv works below.. 103 | if (! $?PYTHONPATH ) setenv PYTHONPATH 104 | 105 | # Update the full path to the mosfire DRP code repository 106 | # example: /src2/mosfire/DRP/mosfire 107 | # in which the sub dirs drivers, apps, badpixel, etc. live 108 | setenv MOSPATH /Users/mkassis/Documents/KeckInstrs/MOSFIRE/DRP_CODE_March2014/ 109 | setenv PYTHONPATH ${PYTHONPATH}:${MOSPATH} 110 | 111 | **6) Ensure that mospy is executable** 112 | 113 | chmod +x mospy 114 | 115 | **7) Update your .cshrc file** 116 | 117 | Update your `.cshrc` file with the code bin dir in the path. Add the following line to your `.cshrc` file: 118 | 119 | set path = ( #mosfire_drp_bin_dir# $path ) 120 | 121 | for example: 122 | 123 | set path = ( ~/MOSFIRE/DRP_CODE/bin $path ) 124 | 125 | If you do not normally run csh or tcsh, you may not have a `.cshrc` file. You will need to create one or download an example file like this one: [http://www2.keck.hawaii.edu/inst/mosfire/.cshrc](http://www2.keck.hawaii.edu/inst/mosfire/.cshrc). The `.cshrc` file must be in your home directory. By default, MacOSX does not show files that start with a `.` But you can access them via the terminal. 126 | 127 | For a bash shell: 128 | 129 | # Adding MOSFIRE pipeline 130 | PATH="/pathtomosfiredrp/MOSFIRE/DRP_CODE/bin:${PATH}" 131 | export PATH 132 | 133 | **8) Now source your .cshrc file** 134 | 135 | source ~/.cshrc 136 | 137 | This will put your bin dir into your executable path. 138 | 139 | The installation is now complete. Take a moment to inventory your directory structure. 140 | 141 | DRP_CODE – Main Code Directory containing all sub-dirs for executeable code and in our example the raw and reduced sub-directories. 142 | 143 | * MOSFIRE – directory containing the reduction code 144 | * apps – directory containing a few additional applications: 145 | * what – useful pretty printer for files 146 | * handle – the entry point for creating driver files (more later) 147 | * badpixels – directory containing badpixel maps. 148 | * Drivers – directory containing example driver files. These files are used to initiate the redution process and you will modify them for your specific data sets. This will be discussed in more detail later. 149 | * Driver.py – used for YJH reductions 150 | * K_driver.py – Contains code specific to K band observations 151 | * Longslit_driver.py – Longslit reductions 152 | * Long2pos_driver.py – long2pos and long2pos_specphot reductions 153 | * Platescale – contains a file that describes the detector plate scale 154 | 155 | Directories created by you 156 | 157 | * DATA – sub directory in which you can store your data. This is not a necessary sub-directory but may help you manage files. Raw data may be stored in other areas on your disk. 158 | * REDUX – sub directory where reductions will be stored. Also not critical, but helpful. 159 | * bin – has the modified mospy executable command 160 | 161 | From now on, if you want to run any pipeline commands, you will always execute “mospy” as seen in our first step in section 5 below. 162 | -------------------------------------------------------------------------------- /apps/handle.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python 2 | 3 | ''' 4 | MOSFIRE 'handle' command: 5 | 6 | (c) npk - Dec 2013 7 | ''' 8 | import MOSFIRE.IO as IO 9 | import os 10 | import numpy as np 11 | import sys 12 | import glob 13 | 14 | from MOSFIRE.MosfireDrpLog import debug, info, warning, error 15 | 16 | if len(sys.argv) < 3: 17 | print('''Usage: mospy handle [target]''') 18 | sys.exit() 19 | 20 | ## Output the file list to a text file for later examination 21 | if os.path.exists('filelist.txt'): 22 | debug('Removing old filelist.txt') 23 | os.remove('filelist.txt') 24 | fl = open('filelist.txt', 'w') 25 | 26 | 27 | files = [] 28 | for i in range(1, len(sys.argv)): 29 | files.extend(glob.iglob(sys.argv[i])) 30 | 31 | masks = {} 32 | 33 | 34 | info('Examining {} files'.format(len(files))) 35 | for fname in files: 36 | 37 | try: 38 | header = IO.readheader(fname) 39 | except IOError:#, err: 40 | fl.write("Couldn't IO %s\n" % fname) 41 | continue 42 | except: 43 | fl.write("%s is unreadable\n" % fname) 44 | continue 45 | 46 | lamps = "" 47 | try: 48 | if header["pwstata7"] == 1: 49 | lamps += header["pwloca7"][0:2] 50 | if header["pwstata8"] == 1: 51 | lamps += header["pwloca8"][0:2] 52 | except KeyError: 53 | lamps = "???" 54 | 55 | header['lamps'] = lamps 56 | 57 | try: 58 | if header["aborted"]: 59 | header['object' ] = 'ABORTED' 60 | except: 61 | fl.write("Missing header file in: %s\n" % fname) 62 | 63 | try: 64 | fl.write("%(datafile)12s %(object)35s %(truitime)6.1fs %(maskname)35s %(lamps)3s %(filter)4s %(mgtname)7s\n" % (header)) 65 | except: 66 | try: 67 | fl.write("%(datafile)12s %(object)25s %(truitime)6.1fs %(lamps)3s %(filter)6s %(mgtname)7s\n" % (header)) 68 | except: 69 | fl.write("%s Skipped\n" % fname) 70 | continue 71 | 72 | 73 | datafile = header['datafile'] + '.fits' 74 | maskname = str(header['maskname']) 75 | target = str(header['targname']) 76 | filter = header['filter'] 77 | yr,mn,dy = IO.fname_to_date_tuple(datafile) 78 | date = str(yr)+mn+str(dy) 79 | object = header['object'] 80 | frameid = header['FRAMEID'].strip() 81 | 82 | itime = header['truitime'] 83 | grating_turret = header['mgtname'] 84 | 85 | if object.find("MIRA") == -1: 86 | mira = False 87 | else: 88 | mira = True 89 | 90 | if header['MGTNAME'] is not 'mirror': 91 | mira = False 92 | 93 | if maskname.find(" (align)") == -1: 94 | align = False 95 | else: 96 | maskname = maskname.replace(" (align)", "") 97 | align = True 98 | 99 | if maskname.find('LONGSLIT') != -1: 100 | # print("longslit file") 101 | align = False 102 | 103 | if maskname.find('long2pos') != -1: 104 | if grating_turret != 'mirror': 105 | align = False 106 | 107 | empty_files = {'Align': [], 'Ne': [], 'Ar': [], 'Flat': [], 'FlatThermal': [], 108 | 'Dark': [], 'Aborted': [], 'Image': [], 'MIRA': [], 'Unknown': []} 109 | 110 | if maskname not in masks: 111 | masks[maskname] = {date: {filter: empty_files}} 112 | 113 | if date not in masks[maskname]: 114 | masks[maskname][date] = {filter: empty_files} 115 | 116 | if filter not in masks[maskname][date]: 117 | masks[maskname][date][filter] = empty_files 118 | 119 | # convert numbers such as 1.0 to 1, but leaves 1.5 as 1.5 120 | # - added to match AutoDriver.py code 121 | offset_hdr = float(header['YOFFSET']) 122 | if offset_hdr % 1 == 0: 123 | offsetvalue = int(offset_hdr) 124 | else: 125 | offsetvalue = offset_hdr 126 | 127 | offset = 'Offset_' + str(offsetvalue) 128 | if (maskname.find('long2pos') != -1 and align is False) or maskname.find('LONGSLIT') != -1: 129 | # if the target name contains a /, replace it with _ 130 | target_name = target.replace("/","_") 131 | # if the target name contains a space, remove it 132 | target_name = target_name.replace(" ","") 133 | # add a posC and posA to the offset names 134 | position = '' 135 | if header['XOFFSET']>0: 136 | position = 'PosC' 137 | if header['XOFFSET']<0: 138 | position = 'PosA' 139 | offset = offset+'_'+str(target_name) 140 | if position is not '': 141 | offset = offset+'_'+position 142 | 143 | 144 | if mira: 145 | masks[maskname][date][filter]['MIRA'].append(fname) 146 | elif align: 147 | masks[maskname][date][filter]['Align'].append(fname) 148 | elif 'Ne' in header['lamps']: 149 | masks[maskname][date][filter]['Ne'].append(fname) 150 | elif 'Ar' in header['lamps']: 151 | masks[maskname][date][filter]['Ar'].append(fname) 152 | elif header['ABORTED']: 153 | masks[maskname][date][filter]['Aborted'].append(fname) 154 | elif header['FILTER'] == 'Dark': 155 | masks[maskname][date][filter]['Dark'].append(fname) 156 | elif header['FLATSPEC'] == 1: 157 | masks[maskname][date][filter]['Flat'].append(fname) 158 | elif object.find("Flat:") != -1 and ( object.find("lamps off") != -1 or object.find("Flat:Off")) != -1 : 159 | masks[maskname][date][filter]['FlatThermal'].append(fname) 160 | elif header['mgtname'] == 'mirror': 161 | masks[maskname][date][filter]['Image'].append(fname) 162 | elif offset != 0: 163 | # print "offset is now:"+str(offset) 164 | if frameid in ["A", "B", "A'", "B'","D","C", "E"]: 165 | if offset in masks[maskname][date][filter]: 166 | masks[maskname][date][filter][offset].append((fname, itime)) 167 | # print("adding file to existing offset file") 168 | else: 169 | masks[maskname][date][filter][offset] = [(fname, itime)] 170 | # print("creating new offset file") 171 | else: 172 | fl.write('{} has unexpected FRAMEID: {}\n'.format(fname, frameid)) 173 | else: 174 | masks[maskname][date][filter]['Unknown'].append(fname) 175 | 176 | 177 | 178 | ##### Now handle mask dictionary 179 | 180 | def descriptive_blurb(): 181 | import getpass, time 182 | 183 | uid = getpass.getuser() 184 | date = time.asctime() 185 | 186 | return "# Created by '%s' on %s\n" % (uid, date) 187 | 188 | 189 | # Write out the list of files in filepath 190 | # list = ['/path/to/mYYmmDD_####.fits' ...] 191 | # filepath is absolute path to the file name to write to 192 | # 193 | # Result, is a file called filepath is written with 194 | # fits files in the list. 195 | def handle_file_list(output_file, files): 196 | '''Write a list of paths to MOSFIRE file to output_file.''' 197 | 198 | if os.path.isfile(output_file): 199 | print("%s: already exists, skipping" % output_file ) 200 | # pass 201 | 202 | if len(files) > 0: 203 | with open(output_file, "w") as f: 204 | f = open(output_file, "w") 205 | f.write(descriptive_blurb()) 206 | 207 | picker = lambda x: x 208 | if len(files[0]) == 2: picker = lambda x: x[0] 209 | 210 | # Identify unique path to files: 211 | paths = [os.path.dirname(picker(file)) for file in files] 212 | paths = list(set(paths)) 213 | 214 | if len(paths) == 1: 215 | path_to_all = paths[0] 216 | converter = os.path.basename 217 | f.write("%s # Abs. path to files [optional]\n" % path_to_all) 218 | else: 219 | converter = lambda x: x 220 | 221 | info('Writing {} files to {}'.format(len(files), output_file)) 222 | for path in files: 223 | if len(path) == 2: 224 | to_write = "%s # %s s\n" % (converter(path[0]), path[1]) 225 | else: 226 | to_write = "%s\n" % converter(path) 227 | f.write("%s" % to_write) 228 | 229 | 230 | def handle_date_and_filter(mask, date, filter, mask_info): 231 | 232 | path = os.path.join(mask,date,filter) 233 | try: 234 | os.makedirs(path) 235 | except OSError: 236 | pass 237 | 238 | for type in list(mask_info.keys()): 239 | handle_file_list(os.path.join(path, type + ".txt"), mask_info[type]) 240 | 241 | 242 | for mask in list(masks.keys()): 243 | for date in list(masks[mask].keys()): 244 | for filter in list(masks[mask][date].keys()): 245 | handle_date_and_filter(mask, date, filter, masks[mask][date][filter]) 246 | 247 | 248 | -------------------------------------------------------------------------------- /scripts/mospy_handle.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python 2 | 3 | ''' 4 | MOSFIRE 'handle' command: 5 | 6 | (c) npk - Dec 2013 7 | ''' 8 | import MOSFIRE.IO as IO 9 | import os 10 | import numpy as np 11 | import sys 12 | import glob 13 | 14 | from MOSFIRE.MosfireDrpLog import debug, info, warning, error 15 | 16 | if len(sys.argv) < 3: 17 | print('''Usage: mospy handle [target]''') 18 | sys.exit() 19 | 20 | ## Output the file list to a text file for later examination 21 | if os.path.exists('filelist.txt'): 22 | debug('Removing old filelist.txt') 23 | os.remove('filelist.txt') 24 | fl = open('filelist.txt', 'w') 25 | 26 | 27 | files = [] 28 | for i in range(1, len(sys.argv)): 29 | files.extend(glob.iglob(os.path.abspath(sys.argv[i]))) 30 | 31 | files = [file for file in files if os.path.splitext(file)[1] != '.original'] 32 | 33 | masks = {} 34 | 35 | 36 | info('Examining {} files'.format(len(files))) 37 | for fname in files: 38 | 39 | try: 40 | header = IO.readheader(fname) 41 | except IOError:#, err: 42 | fl.write("Couldn't IO %s\n" % fname) 43 | continue 44 | except: 45 | fl.write("%s is unreadable\n" % fname) 46 | continue 47 | 48 | lamps = "" 49 | try: 50 | if header["pwstata7"] == 1: 51 | lamps += header["pwloca7"][0:2] 52 | if header["pwstata8"] == 1: 53 | lamps += header["pwloca8"][0:2] 54 | except KeyError: 55 | lamps = "???" 56 | 57 | header['lamps'] = lamps 58 | 59 | try: 60 | if header["aborted"]: 61 | header['object' ] = 'ABORTED' 62 | except: 63 | fl.write("Missing header file in: %s\n" % fname) 64 | 65 | try: 66 | fl.write("%(datafile)12s %(object)35s %(truitime)6.1fs %(maskname)35s %(lamps)3s %(filter)4s %(mgtname)7s\n" % (header)) 67 | except: 68 | try: 69 | fl.write("%(datafile)12s %(object)25s %(truitime)6.1fs %(lamps)3s %(filter)6s %(mgtname)7s\n" % (header)) 70 | except: 71 | fl.write("%s Skipped\n" % fname) 72 | continue 73 | 74 | 75 | datafile = header['datafile'] + '.fits' 76 | maskname = str(header['maskname']) 77 | target = str(header['targname']) 78 | filter = header['filter'] 79 | yr,mn,dy = IO.fname_to_date_tuple(datafile) 80 | date = str(yr)+mn+str(dy) 81 | object = header['object'] 82 | frameid = header['FRAMEID'].strip() 83 | 84 | itime = header['truitime'] 85 | grating_turret = header['mgtname'] 86 | 87 | if object.find("MIRA") == -1: 88 | mira = False 89 | else: 90 | mira = True 91 | 92 | if header['MGTNAME'] is not 'mirror': 93 | mira = False 94 | 95 | if maskname.find(" (align)") == -1: 96 | align = False 97 | else: 98 | maskname = maskname.replace(" (align)", "") 99 | align = True 100 | 101 | if maskname.find('LONGSLIT') != -1: 102 | # print("longslit file") 103 | align = False 104 | 105 | if maskname.find('long2pos') != -1: 106 | if grating_turret != 'mirror': 107 | align = False 108 | 109 | empty_files = {'Align': [], 'Ne': [], 'Ar': [], 'Flat': [], 'FlatThermal': [], 110 | 'Dark': [], 'Aborted': [], 'Image': [], 'MIRA': [], 'Unknown': []} 111 | 112 | if maskname not in masks: 113 | masks[maskname] = {date: {filter: empty_files}} 114 | 115 | if date not in masks[maskname]: 116 | masks[maskname][date] = {filter: empty_files} 117 | 118 | if filter not in masks[maskname][date]: 119 | masks[maskname][date][filter] = empty_files 120 | 121 | # convert numbers such as 1.0 to 1, but leaves 1.5 as 1.5 122 | # - added to match AutoDriver.py code 123 | offset_hdr = float(header['YOFFSET']) 124 | if offset_hdr % 1 == 0: 125 | offsetvalue = int(offset_hdr) 126 | else: 127 | offsetvalue = offset_hdr 128 | 129 | offset = 'Offset_' + str(offsetvalue) 130 | if (maskname.find('long2pos') != -1 and align is False) or maskname.find('LONGSLIT') != -1: 131 | # if the target name contains a /, replace it with _ 132 | target_name = target.replace("/","_") 133 | # if the target name contains a space, remove it 134 | target_name = target_name.replace(" ","") 135 | # add a posC and posA to the offset names 136 | position = '' 137 | if header['XOFFSET']>0: 138 | position = 'PosC' 139 | if header['XOFFSET']<0: 140 | position = 'PosA' 141 | offset = offset+'_'+str(target_name) 142 | if position is not '': 143 | offset = offset+'_'+position 144 | 145 | 146 | if mira: 147 | masks[maskname][date][filter]['MIRA'].append(fname) 148 | elif align: 149 | masks[maskname][date][filter]['Align'].append(fname) 150 | elif 'Ne' in header['lamps']: 151 | masks[maskname][date][filter]['Ne'].append(fname) 152 | elif 'Ar' in header['lamps']: 153 | masks[maskname][date][filter]['Ar'].append(fname) 154 | elif header['ABORTED']: 155 | masks[maskname][date][filter]['Aborted'].append(fname) 156 | elif header['FILTER'] == 'Dark': 157 | masks[maskname][date][filter]['Dark'].append(fname) 158 | elif header['FLATSPEC'] == 1: 159 | masks[maskname][date][filter]['Flat'].append(fname) 160 | elif object.find("Flat:") != -1 and ( object.find("lamps off") != -1 or object.find("Flat:Off")) != -1 : 161 | masks[maskname][date][filter]['FlatThermal'].append(fname) 162 | elif header['mgtname'] == 'mirror': 163 | masks[maskname][date][filter]['Image'].append(fname) 164 | elif offset != 0: 165 | # print "offset is now:"+str(offset) 166 | if frameid in ["A", "B", "A'", "B'","D","C", "E"]: 167 | if offset in masks[maskname][date][filter]: 168 | masks[maskname][date][filter][offset].append((fname, itime)) 169 | # print("adding file to existing offset file") 170 | else: 171 | masks[maskname][date][filter][offset] = [(fname, itime)] 172 | # print("creating new offset file") 173 | else: 174 | fl.write('{} has unexpected FRAMEID: {}\n'.format(fname, frameid)) 175 | else: 176 | masks[maskname][date][filter]['Unknown'].append(fname) 177 | 178 | 179 | 180 | ##### Now handle mask dictionary 181 | 182 | def descriptive_blurb(): 183 | import getpass, time 184 | 185 | uid = getpass.getuser() 186 | date = time.asctime() 187 | 188 | return "# Created by '%s' on %s\n" % (uid, date) 189 | 190 | 191 | # Write out the list of files in filepath 192 | # list = ['/path/to/mYYmmDD_####.fits' ...] 193 | # filepath is absolute path to the file name to write to 194 | # 195 | # Result, is a file called filepath is written with 196 | # fits files in the list. 197 | def handle_file_list(output_file, files): 198 | '''Write a list of paths to MOSFIRE file to output_file.''' 199 | 200 | if os.path.isfile(output_file): 201 | print("%s: already exists, skipping" % output_file ) 202 | # pass 203 | 204 | if len(files) > 0: 205 | with open(output_file, "w") as f: 206 | f = open(output_file, "w") 207 | f.write(descriptive_blurb()) 208 | 209 | picker = lambda x: x 210 | if len(files[0]) == 2: picker = lambda x: x[0] 211 | 212 | # Identify unique path to files: 213 | paths = [os.path.dirname(picker(file)) for file in files] 214 | paths = list(set(paths)) 215 | 216 | if len(paths) == 1: 217 | path_to_all = paths[0] 218 | converter = os.path.basename 219 | f.write("%s # Abs. path to files [optional]\n" % path_to_all) 220 | else: 221 | converter = lambda x: x 222 | 223 | info('Writing {} files to {}'.format(len(files), output_file)) 224 | for path in files: 225 | if len(path) == 2: 226 | to_write = "%s # %s s\n" % (converter(path[0]), path[1]) 227 | else: 228 | to_write = "%s\n" % converter(path) 229 | f.write("%s" % to_write) 230 | 231 | 232 | def handle_date_and_filter(mask, date, filter, mask_info): 233 | 234 | path = os.path.join(mask,date,filter) 235 | try: 236 | os.makedirs(path) 237 | except OSError: 238 | pass 239 | 240 | for type in list(mask_info.keys()): 241 | handle_file_list(os.path.join(path, type + ".txt"), mask_info[type]) 242 | 243 | 244 | for mask in list(masks.keys()): 245 | for date in list(masks[mask].keys()): 246 | for filter in list(masks[mask][date].keys()): 247 | handle_date_and_filter(mask, date, filter, masks[mask][date][filter]) 248 | 249 | 250 | -------------------------------------------------------------------------------- /manual/docs/wavelengthYJH.md: -------------------------------------------------------------------------------- 1 | # Wavelength Calibration (Y, J, H) 2 | 3 | In the shorter wavebands, when using the recommended exposure times, the wavelength calibration is performed on night sky lines. The mospy Wavelength module is responsbile for these operations. See the example driver file in section 7. 4 | 5 | ## Combine files 6 | 7 | First step is to produce a file with which you will train your wavelength solution. Since we’re using night sky lines for training, the approach is to combine individual science exposures. This is performed by the python Wavelength.imcombine routine. For a lot of users, this will look something like in the Driver.py file: 8 | 9 | Wavelength.imcombine(obsfiles, maskname, band, waveops) 10 | 11 | The first parameter is obsfiles which is a python string array indicating the list of files in the offset positions. Note that obsfiles has defaults of “Offset_1.5.txt” and “Offset_-1.5.txt” and may need to be updated as described in section 6. 12 | 13 | Suppose you want to exclude a file for reasons such as weather or telescope fault, simply remove the offending file from the appropriate Offset_*.txt. Likewise, you are welcome to add files in as you like, such as observations from the previous night. 14 | 15 | Outputs of this step are: 16 | 17 | | Filename | Contains | 18 | |----------------------------------|-----------------------------------------------------------------------------| 19 | | `wave_stack_[band]_[range].fits` | A median-combined image of the files to be used for the wavelength solution.| 20 | 21 | ## Interactive wavelength fitting 22 | 23 | The next step is to use the wave_stack_*.fits file and determine an initial wavelength solution for each slit. During this process, we interactively fit the lines using a gui that displays. To initiate this process, uncomment the line in the Driver.py file: 24 | 25 | #Wavelength.fit_lambda(maskname, band, obsfiles, obsfiles, waveops) 26 | 27 | And then re-execute the driver file: 28 | 29 | mospy Driver.py 30 | 31 | when you run this step, a GUI window appears. 32 | 33 | ![Screenshot](image3.png "The interactive wavelength solving window. This is a J-band night sky spectrum.") 34 | 35 | The interactive wavelength solving window. This is a J-band night sky spectrum. 36 | 37 | ![Screenshot](image4.png "The interactive wavelength solving window showing an initial fit. This is a J-band night sky spectrum and one of the night sky lines on the right hand side is clearly a poor fit compared to the rest of the identified lines.") 38 | 39 | The interactive wavelength solving window showing an initial fit. This is a J-band night sky spectrum and one of the night sky lines on the right hand side is clearly a poor fit compared to the rest of the identified lines. 40 | 41 | ![Screenshot](image5.png "The interactive wavelength solving window showing a good fit with the initial poor line removed from the calculation. ") 42 | 43 | The interactive wavelength solving window showing a good fit with the initial poor line removed from the calculation. 44 | The interactive wavelength solving window showing an initial fit. This is a J-band night sky spectrum and one of the night sky lines on the right hand side is clearly a poor fit compared to the rest of the identified lines. 45 | 46 | Plotted in the gui will be a sky line spectrum and vertical lines denoting positions and wavelengths of the sky lines. Your goal is to help the pipeline by identifying the night sky lines in the center of each slit. Once you come up with a good solution in the center, the pipeline will propagate it spatially along the slit. In the gui, Press ? to see a list of commands in the console window. The list of commands available on the GUI are: 47 | 48 | * c - to center on the nearest peak (first thing to do to shift the initial wavelength guess) 49 | * c - Center the nearest line at the cursor position 50 | * \ - Fit fit the data 51 | * f – Alternate way to fit the data, equivalent to \ but may cause the spectrum to become full screen. 52 | * k – Toggles k-sigma clipping on and off. Also performs a new fit. 53 | * b - Turns on bypass mode. From now on, the interactive window is not displayed and the fit continues automatically. 54 | * d - Delete a point (remove the wackadoos) 55 | * n - proceed to the Next object 56 | * p - return to back to the Previous object 57 | * r - Reset the current slit (try this if the plot looks strange) 58 | * z - Zoom at cursor position 59 | * x - Unzoom: full screen 60 | * s - Save figure to disk 61 | * h - Help 62 | * q - Quit and save results 63 | 64 | Below is a rough procedure for completing the interactive fitting process. The steps you need to take are as follows. 65 | 66 | * First, check to see if the orange lines match up with obvious night sky lines. If not the expected position does not match the actually position of the line do the following: 67 | * Place your cursor over a line 68 | * Press the “c’ button that will shift the predicted position to the observed line. 69 | * Press “f” to fit. An initial fit is done automatically (starting with version 2015A) A Chebyshev polynomial, f, such that f(pixel #) returns the wavelength in Angstroms. 70 | * You can chose to use k-sigma clipping to avoid the manual operation of removing bad lines with the “k” key. 71 | * Press "x" to unzoom to the full size region 72 | * Assess the fit: 73 | * If a line is poorly fit and should be removed 74 | * Move the cursor to the line 75 | * Press “d” to delete the line from the fit 76 | * For good fits, the residual points turn green. 77 | * When the satisfied with the fit, press “n” to move to the next object. 78 | * If you want to disable the interactive fit and switch to the automatic fit, press “b” (bypass) 79 | * Repeat the process above until you see the red Done! text in the center of your screen. 80 | * Press “q” to quit the interactive gui and move to the next step. 81 | 82 | The prompt should return following the fitting process. The outputs from this process are: 83 | 84 | | Filename | Contains | 85 | |-----------------------|------------------------------------------------------------------------------------------------| 86 | | `barset.npy` | bar positions for each slit are specified | 87 | | `lambda_center_coeffs_wave_stack_band_filenames.npy` | The coefficients of the fit and positions of the measured lines.| 88 | 89 | ## Wavelength fitting for the entire slit 90 | 91 | The next step in the wavelength fitting process is to propogate the solution spatially along each slit. To complete this process we uncomment the line in the Driver.py file: 92 | 93 | #Wavelength.fit_lambda(maskname, band, obsfiles, obsfiles, waveops) 94 | 95 | This is one of the longer running processes and the output should look something like: 96 | 97 | ... 98 | resid ang S09 @ p 978: 0.10 rms 0.07 mad [shift-22] 99 | resid ang S09 @ p 979: 0.09 rms 0.06 mad [shift-22] 100 | resid ang S09 @ p 980: 0.10 rms 0.06 mad [shift-22] 101 | resid ang S09 @ p 981: 0.09 rms 0.06 mad [shift-22] 102 | resid ang S09 @ p 982: 0.08 rms 0.05 mad [shift-22] 103 | resid ang S09 @ p 983: 0.08 rms 0.04 mad [shift-22] 104 | ... 105 | 106 | The prompt should return following the fitting process. The outputs from this process are: 107 | 108 | | Filename | Contains | 109 | |-----------------------|------------------------------------------------------------------------------------| 110 | | `lambda_coeffs_wave_stack_J_m130114_0443-0445.npy` | coefficients of the fit for each row within the slit | 111 | 112 | ## Apply the wavelength solution 113 | 114 | The last step in the wavelength fitting process is to apply the solution and create maps of the wavelength for the data set. To complete this process we uncomment the line in the Driver.py file: 115 | 116 | #Wavelength.apply_lambda_simple(maskname, band, obsfiles, waveops) 117 | 118 | The prompt should return following the fitting process. The outputs from this process are: 119 | 120 | | Filename | Contains | 121 | |-----------------------|------------------------------------------------------------------------------------| 122 | | `lambda_solution_wave_stack_J_m130114_0443-0445.fits` | contains a map of the wavelength for each pixel in the spectra | 123 | | `sigs_solution_wave_stack_J_m130114_0443-0445.fits` | contains the uncertainty in the measured wavelength position for each pixel in the spectra | 124 | | `rectified_wave_stack_J_m130114_0443-0445.fits` | contains the spatially and wavelength rectified resampled sky emission. A column in the image contains all pixels at the same wavelength.| 125 | 126 | 127 | 128 | 129 | 130 | -------------------------------------------------------------------------------- /apps/check_CSU_pos.py: -------------------------------------------------------------------------------- 1 | ''' 2 | 3 | Written March 18th 2011 by npk 4 | 5 | 6 | ''' 7 | 8 | import sys, datetime, getpass, os 9 | import numpy as np 10 | try: 11 | from astropy.io import fits as pf 12 | except: 13 | import pyfits as pf 14 | from matplotlib import pyplot as pl 15 | import scipy.ndimage.filters, scipy.io 16 | from pyraf import iraf 17 | from MOSFIRE import CSU, Detector, IO, Fit 18 | 19 | from IPython.Shell import IPShellEmbed 20 | 21 | start_shell = IPShellEmbed() 22 | 23 | NaN = np.nan 24 | 25 | 26 | def save_region(bs, fname): 27 | s = bs.to_ds9_region() 28 | try: 29 | f = open(fname, "w") 30 | f.writelines(s) 31 | f.close() 32 | except: 33 | return Exception("Could not write barset region file to: %s" % fname) 34 | 35 | def fit_line_with_sigclip(xs, data, i=0): 36 | 37 | 38 | ps = Fit.do_fit_edge(xs, data) 39 | pf = lambda x: Fit.fit_bar_edge(ps, x) 40 | 41 | 42 | residual = np.abs(pf(xs) - data) 43 | sd = np.std(residual) 44 | 45 | ok = np.where(residual < 2.5 * sd)[0] 46 | if len(ok) == 0: 47 | return [lambda x: NaN, []] 48 | 49 | ps = Fit.do_fit_edge(xs[ok], data[ok]) 50 | pf = lambda x: Fit.fit_bar_edge(ps, x) 51 | 52 | return [pf, ok] 53 | 54 | 55 | 56 | if False: 57 | def fit_line_with_sigclip(xs, data, i = 0): 58 | 59 | ps = np.polyfit(xs, data, 1) 60 | pf = np.poly1d(ps) 61 | 62 | 63 | residual = np.abs(pf(xs) - data) 64 | sd = np.std(residual) 65 | 66 | ok = np.where(residual < 2.5*sd)[0] 67 | 68 | ps = np.polyfit(xs[ok], data[ok], 1) 69 | pf = np.poly1d(ps) 70 | return [pf, ok] 71 | 72 | if len(ok) == len(residual): 73 | return [pf, ok] 74 | elif i > 2: 75 | return [pf, ok] 76 | else: 77 | return fit_line_with_sigclip(xs[ok], data[ok], i+1) 78 | 79 | def median_tails(v): 80 | a = np.median(v[0:2]) 81 | b = np.median(v[-3:-1]) 82 | 83 | t = v - np.float(a+b)/2. 84 | 85 | return t 86 | 87 | def make_slice(pos, offset, w, h): 88 | '''Returns [ [xslice, ylsice], [x0, x1, y0, x1] ] where 89 | xslice is used as Array[x0:x1] 90 | yslice is used as Array[y0:y1]''' 91 | x0 = round(pos[0]- w + offset) 92 | if x0 < 0: x0 = 0 93 | x1 = round(pos[0] + w + offset) 94 | if x1 > 2047: x1 = 2047 95 | if x0 > x1: x0 = x1 96 | xs = slice(x0, x1) 97 | 98 | y0 = round(pos[1]-h) 99 | if y0 < 0: y0 = 0 100 | y1 = round(pos[1]+h) 101 | if y1 > 2047: y1 = 2047 102 | if y0 > y1: y0 = y1 103 | ys = slice(y0,y1) 104 | 105 | 106 | return [[xs,ys],[x0,x1,y0,y1],round(pos[1])] 107 | 108 | deg = np.pi/180. 109 | np.set_printoptions(precision=2) 110 | np.set_printoptions(suppress=True) 111 | 112 | reload(CSU) 113 | reload(IO) 114 | reload(Fit) 115 | reload(Detector) 116 | 117 | 118 | def sigclip(data, low=4, high=4): 119 | c = data.ravel() 120 | delta = 1 121 | while delta: 122 | s = c.std() 123 | m = c.mean() 124 | size = c.size 125 | 126 | c = c[(c > (m - s*low)) & (c < (m + s*high))] 127 | 128 | delta = size-c.size 129 | 130 | return c.mean() 131 | 132 | 133 | def is_in_bounds(extent): 134 | if extent[1] > 2047: return False 135 | if extent[0] < 0: return False 136 | if extent[2] > 2047: return False 137 | if extent[0] < 0: return False 138 | if extent[0] == extent[1]: 139 | return False 140 | if extent[2] == extent[3]: 141 | return False 142 | 143 | for i in [0,1]: 144 | for j in [2,3]: 145 | if not CSU.in_field(extent[i],extent[j]): return False 146 | return True 147 | 148 | def is_odd(n): 149 | return (n % 2) == 1 150 | 151 | 152 | def go(fname): 153 | global plot_arr 154 | 155 | print fname 156 | 157 | (header, data) = IO.readfits(fname) 158 | bs = CSU.Barset() 159 | bs.set_header(header) 160 | 161 | bars = [] 162 | means = [] 163 | sds = [] 164 | deltas = [] 165 | deltas_mm = [] 166 | poss = [] 167 | poss_mm = [] 168 | poss_y = [] 169 | request = [] 170 | qs = [] 171 | bars = range(1, 93) 172 | fit_fun = Fit.residual_single 173 | 174 | for bar in bars: 175 | pos = bs.get_bar_pix(bar) 176 | if bar % 8 == 0: 177 | print "%2.0i: (%7.2f, %7.2f)" % (bar, pos[0], pos[1]) 178 | 179 | 180 | if is_odd(bar): 181 | if (bs.pos[bar] - bs.pos[bar-1]) < 2.7: 182 | fit_fun = Fit.residual_pair 183 | else: 184 | fit_fun = Fit.residual_single 185 | 186 | width = 19 187 | [[xslice, yslice],extent,ystart] = make_slice(pos,0,width,30) 188 | 189 | 190 | 191 | if not is_in_bounds(extent): 192 | fits = [0,0,0,0,0] 193 | [ff,ok] = [np.poly1d(0,0), []] 194 | means.append(fits) 195 | sds.append(fits) 196 | drop_this = True 197 | else: 198 | drop_this = False 199 | fits = [] 200 | ys = np.arange(-10,10, dtype=np.float32) 201 | for i in ys: 202 | tofit = data[ystart-i, xslice] 203 | y = median_tails(tofit) 204 | 205 | ps = Fit.do_fit(y, fit_fun) 206 | fits.append(ps[0]) 207 | 208 | fits = np.array(fits) 209 | fits[:,1] += 1 210 | 211 | # fit to the ridgeline 212 | [ff, ok] = fit_line_with_sigclip(ys, fits[:,1]) 213 | m = [np.mean(fits[:,i]) for i in range(5)] 214 | s = [np.std(fits[:,i]) for i in range(5)] 215 | means.append(m) 216 | sds.append(s) 217 | 218 | 219 | slit_center_offset = pos[1] - ystart 220 | fc = ff(slit_center_offset) 221 | slit_center_pos = np.float(extent[0] + fc ) 222 | 223 | if drop_this: 224 | poss.append(NaN) 225 | poss_y.append(NaN) 226 | poss_mm.append(NaN) 227 | else: 228 | poss.append(slit_center_pos) 229 | poss_y.append(ystart) 230 | poss_mm.append(CSU.csu_pix_to_mm_poly(slit_center_pos, ystart)[0]) 231 | 232 | delta = np.float(slit_center_pos - pos[0]) 233 | if drop_this: 234 | deltas.append(NaN) 235 | deltas_mm.append(NaN) 236 | else: 237 | deltas.append(delta) 238 | b = CSU.csu_pix_to_mm_poly(slit_center_pos + delta, ystart)[0] 239 | deltas_mm.append(b - poss_mm[-1]) 240 | 241 | q = np.float(np.degrees(np.tan(ff(1)-ff(0)))) 242 | if drop_this: qs.append(NaN) 243 | qs.append(q) 244 | 245 | 246 | means = np.array(means) 247 | f = lambda x: np.array(x).ravel() 248 | sds = f(sds) 249 | deltas = f(deltas) 250 | poss = f(poss) 251 | poss_y = f(poss_y) 252 | poss_mm = f(poss_mm) 253 | deltas_mm = f(deltas_mm) 254 | qs = f(qs) 255 | bars = f(bars) 256 | 257 | 258 | 259 | fout = "/users/npk/dropbox/mosfire/cooldown 9/csu_meas/" + fname.split("/")[-1] + ".sav" 260 | print "saving" 261 | tosav = {"bars": bars, "request": bs.pos, "deltas_mm": deltas_mm, "poss": poss, "poss_mm": poss_mm, "deltas": deltas, "means": means, "qs": qs} 262 | scipy.io.savemat(fout, tosav) 263 | save_region(bs, "/users/npk/dropbox/mosfire/cooldown 9/csu_meas/" + fname.split("/")[-1] + ".reg") 264 | print "saved" 265 | 266 | regout = "/users/npk/dropbox/mosfire/cooldown 9/csu_meas/" + fname.split("/")[-1] + ".meas.reg" 267 | pairs = np.array([poss,poss_y]).transpose() 268 | s = CSU.to_ds9_region(pairs, dash=0, color="blue", label=False) 269 | try: 270 | f = open(regout, "w") 271 | f.writelines(s) 272 | f.close() 273 | except: 274 | print "Couldn't write: %s" % regout 275 | 276 | 277 | return [tosav, bs] 278 | 279 | 280 | 281 | path = "/users/npk/desktop/c9/" 282 | def generate_fname(num): 283 | global path 284 | files = os.listdir(path) 285 | for fn in files: 286 | if fn.find("12_%4.4i" % num) > 0: 287 | return fn 288 | if fn.find("13_%4.4i" % num) > 0: 289 | return fn 290 | if fn.find("16_%4.4i" % num) > 0: 291 | return fn 292 | if fn.find("17_%4.4i" % num) > 0: 293 | return fn 294 | if fn.find("18_%4.4i" % num) > 0: 295 | return fn 296 | if fn.find("19_%4.4i" % num) > 0: 297 | return fn 298 | if fn.find("20_%4.4i" % num) > 0: 299 | return fn 300 | if fn.find("21_%4.4i" % num) > 0: 301 | return fn 302 | if fn.find("24_%4.4i" % num) > 0: 303 | return fn 304 | 305 | [ts, bs] = go(path + generate_fname(2846)) 306 | --------------------------------------------------------------------------------