├── smili ├── run.sh ├── README.md ├── example_driver.py └── smili_imaging_pipeline.py ├── difmap ├── run.sh ├── difmap.sh ├── CircMask_r30_x-0.002_y0.022.win ├── README.md └── EHT_Difmap ├── eht-imaging ├── run.sh ├── README.md └── eht-imaging_pipeline.py ├── run.sh └── README.md /smili/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright (C) 2019 The Event Horizon Telescope Collaboration 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | ./example_driver.py --uvfitsdir ../data/uvfits 19 | -------------------------------------------------------------------------------- /difmap/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright (C) 2019 The Event Horizon Telescope Collaboration 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | cp ../data/uvfits/*.uvfits . 19 | for f in *.uvfits; do 20 | ./difmap.sh $f 21 | done 22 | rm *.uvfits 23 | -------------------------------------------------------------------------------- /eht-imaging/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright (C) 2019 The Event Horizon Telescope Collaboration 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | for d in 095 096 100 101; do 19 | python eht-imaging_pipeline.py \ 20 | -i ../data/uvfits/SR1_M87_2017_${d}_lo_hops_netcal_StokesI.uvfits \ 21 | -i2 ../data/uvfits/SR1_M87_2017_${d}_hi_hops_netcal_StokesI.uvfits \ 22 | -o SR1_M87_2017_${d}.fits 23 | done 24 | -------------------------------------------------------------------------------- /difmap/difmap.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright (C) 2019 The Event Horizon Telescope Collaboration 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | if [ $# -eq 0 ]; then 19 | echo "usage: $0 [input].uvfits" 20 | exit 0 21 | fi 22 | 23 | in_name=${1%.uvfits} 24 | 25 | expect <" 31 | send -- "@EHT_Difmap ${in_name},CircMask_r30_x-0.002_y0.022,-10,0.5,0.1,2,-1\r" 32 | 33 | expect "*0>" 34 | send -- "exit\r" 35 | 36 | expect "*quit without saving: " 37 | send -- "\r" 38 | 39 | expect eof 40 | EOF 41 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright (C) 2019 The Event Horizon Telescope Collaboration 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | REPO=https://github.com/eventhorizontelescope/2019-D01-01/raw/master 19 | FILE=EHTC_FirstM87Results_Apr2019_uvfits.tgz 20 | 21 | echo "Download uvfits files for the EHT first M87 results" 22 | wget $REPO/$FILE 23 | 24 | echo "Extract uvfits files to disk" 25 | mkdir -p data 26 | tar -vzxf $FILE -C data --strip-components=1 27 | 28 | echo "Run imaging pipelines" 29 | for d in difmap/ eht-imaging/ smili/; do 30 | pushd $d 31 | ./run.sh 32 | popd > /dev/null 33 | done 34 | -------------------------------------------------------------------------------- /difmap/CircMask_r30_x-0.002_y0.022.win: -------------------------------------------------------------------------------- 1 | ! CLEAN windows written by wwins in difmap. 2 | ! Columns specify xmin xmax ymin ymax (mas) of each CLEAN window. 3 | -0.0319833287 0.0279833287 0.0210000000 0.0230000000 4 | -0.0318496231 0.0278496231 0.0230000000 0.0250000000 5 | -0.0315803989 0.0275803989 0.0250000000 0.0270000000 6 | -0.0311719043 0.0271719043 0.0270000000 0.0290000000 7 | -0.0306181760 0.0266181760 0.0290000000 0.0310000000 8 | -0.0299105715 0.0259105715 0.0310000000 0.0330000000 9 | -0.0290370117 0.0250370117 0.0330000000 0.0350000000 10 | -0.0279807621 0.0239807621 0.0350000000 0.0370000000 11 | -0.0267184142 0.0227184142 0.0370000000 0.0390000000 12 | -0.0252163735 0.0212163735 0.0390000000 0.0410000000 13 | -0.0234242853 0.0194242853 0.0410000000 0.0430000000 14 | -0.0212613603 0.0172613603 0.0430000000 0.0450000000 15 | -0.0185831240 0.0145831240 0.0450000000 0.0470000000 16 | -0.0150766968 0.0110766968 0.0470000000 0.0490000000 17 | -0.0096811457 0.0056811457 0.0490000000 0.0510000000 18 | -0.0319833287 0.0279833287 0.0230000000 0.0210000000 19 | -0.0318496231 0.0278496231 0.0210000000 0.0190000000 20 | -0.0315803989 0.0275803989 0.0190000000 0.0170000000 21 | -0.0311719043 0.0271719043 0.0170000000 0.0150000000 22 | -0.0306181760 0.0266181760 0.0150000000 0.0130000000 23 | -0.0299105715 0.0259105715 0.0130000000 0.0110000000 24 | -0.0290370117 0.0250370117 0.0110000000 0.0090000000 25 | -0.0279807621 0.0239807621 0.0090000000 0.0070000000 26 | -0.0267184142 0.0227184142 0.0070000000 0.0050000000 27 | -0.0252163735 0.0212163735 0.0050000000 0.0030000000 28 | -0.0234242853 0.0194242853 0.0030000000 0.0010000000 29 | -0.0212613603 0.0172613603 0.0010000000 -0.0010000000 30 | -0.0185831240 0.0145831240 -0.0010000000 -0.0030000000 31 | -0.0150766968 0.0110766968 -0.0030000000 -0.0050000000 32 | -0.0096811457 0.0056811457 -0.0050000000 -0.0070000000 33 | -------------------------------------------------------------------------------- /difmap/README.md: -------------------------------------------------------------------------------- 1 | # DIFMAP M87 Stokes I Imaging Pipeline for EHT observations in April 2017 2 | 3 | **Authors:** The Event Horizon Telescope Collaboration et al. 4 | 5 | **Date:** April 10, 2019 6 | 7 | **Primary Reference:** [The Event Horizon Telescope Collaboration, et al. 2019d, ApJL, 875, L4 (M87 Paper IV)](https://doi.org/10.3847/2041-8213/ab0e85) 8 | 9 | **Data Product Code:** [2019-D01-02](https://eventhorizontelescope.org/for-astronomers/data) 10 | 11 | **Brief Description:** 12 | DIFMAP script for imaging EHT data using a given mask (set of cleaning windows) 13 | 14 | Requires Caltech's DIFMAP software for CLEAN imaging reconstruction, version v2.5b or above, that can be obtained from [ftp://ftp.astro.caltech.edu/pub/difmap/difmap.html](ftp://ftp.astro.caltech.edu/pub/difmap/difmap.html) 15 | 16 | To reproduce fiducial EHT DIFMAP images of M87: 17 | 18 | 1. Place i) uv FITS file "your_file_name.uvfits", ii) Difmap script EHT_Difmap (this file), and iii) mask CircMask_r30_x-0.002_y0.022.win in the same directory 19 | 20 | 2. Run Difmap and use the calling sequence: 21 | 22 | @EHT_Difmap your_file_name,CircMask_r30_x-0.002_y0.022,-10,0.5,0.1,2,-1 23 | 24 | The header of the EHT_Difmap file contains further information regarding the choice of script parameters 25 | 26 | **References:** 27 | 28 | - [EHT Collaboration Data Portal Website](https://eventhorizontelescope.org/for-astronomers/data) 29 | - [The Event Horizon Telescope Collaboration, et al. 2019a, ApJL, 875, L1 (M87 Paper I)](https://doi.org/10.3847/2041-8213/ab0ec7) 30 | - [The Event Horizon Telescope Collaboration, et al. 2019b, ApJL, 875, L2 (M87 Paper II)](https://doi.org/10.3847/2041-8213/ab0c96) 31 | - [The Event Horizon Telescope Collaboration, et al. 2019c, ApJL, 875, L3 (M87 Paper III)](https://doi.org/10.3847/2041-8213/ab0c57) 32 | - [The Event Horizon Telescope Collaboration, et al. 2019d, ApJL, 875, L4 (M87 Paper IV)](https://doi.org/10.3847/2041-8213/ab0e85) 33 | - [The Event Horizon Telescope Collaboration, et al. 2019e, ApJL, 875, L5 (M87 Paper V)](https://doi.org/10.3847/2041-8213/ab0f43) 34 | - [The Event Horizon Telescope Collaboration, et al. 2019f, ApJL, 875, L6 (M87 Paper VI)](https://doi.org/10.3847/2041-8213/ab1141) 35 | - [Shepherd, M., Pearson, T., & Taylor, G. B., 1994, BAAS, 26, 987S](https://ui.adsabs.harvard.edu/abs/1994BAAS...26..987S/abstract) 36 | - [Shepherd, M., Pearson, T., & Taylor, G. B., 1995, BAAS, 27, 903S](https://ui.adsabs.harvard.edu/abs/1995BAAS...27..903S/abstract) 37 | - [DIFMAP: ftp://ftp.astro.caltech.edu/pub/difmap/difmap.html](ftp://ftp.astro.caltech.edu/pub/difmap/difmap.html) 38 | -------------------------------------------------------------------------------- /eht-imaging/README.md: -------------------------------------------------------------------------------- 1 | # eht-imaging M87 Stokes I Imaging Pipeline for EHT observations in April 2017 2 | 3 | **Authors:** The Event Horizon Telescope Collaboration et al. 4 | 5 | **Date:** April 10, 2019 6 | 7 | **Primary Reference:** [The Event Horizon Telescope Collaboration, et al. 2019d, ApJL, 875, L4 (M87 Paper IV)](https://doi.org/10.3847/2041-8213/ab0e85) 8 | 9 | **Data Product Code:** [2019-D01-02](https://eventhorizontelescope.org/for-astronomers/data) 10 | 11 | **Brief Description:** 12 | 13 | The pipeline reconstructs an image from uvfits files simultaneously released in the EHT website (data release ID: 2019-D01-01) using a python-interfaced imaging package eht-imaging (Chael et al. 2016,2018). 14 | 15 | To run the pipeline, specify the input uvfits data file. Multiple bands can be specified separately. Additional flags control the output, which is only the reconstructed image as a FITS image by default. 16 | 17 | Example call: 18 | 19 | python eht-imaging_pipeline.py -i ../../data/uvfits/SR1_M87_2017_101_lo_hops_netcal_StokesI.uvfits -i2 ../../data/uvfits/SR1_M87_2017_101_hi_hops_netcal_StokesI.uvfits --savepdf 20 | 21 | For additional details, please read the help document associated in the imaging script: "python eht-imaging_pipeline.py --help". 22 | 23 | **Additional References:** 24 | 25 | - [EHT Collaboration Data Portal Website](https://eventhorizontelescope.org/for-astronomers/data) 26 | - [The Event Horizon Telescope Collaboration, et al. 2019a, ApJL, 875, L1 (M87 Paper I)](https://doi.org/10.3847/2041-8213/ab0ec7) 27 | - [The Event Horizon Telescope Collaboration, et al. 2019b, ApJL, 875, L2 (M87 Paper II)](https://doi.org/10.3847/2041-8213/ab0c96) 28 | - [The Event Horizon Telescope Collaboration, et al. 2019c, ApJL, 875, L3 (M87 Paper III)](https://doi.org/10.3847/2041-8213/ab0c57) 29 | - [The Event Horizon Telescope Collaboration, et al. 2019d, ApJL, 875, L4 (M87 Paper IV)](https://doi.org/10.3847/2041-8213/ab0e85) 30 | - [The Event Horizon Telescope Collaboration, et al. 2019e, ApJL, 875, L5 (M87 Paper V)](https://doi.org/10.3847/2041-8213/ab0f43) 31 | - [The Event Horizon Telescope Collaboration, et al. 2019f, ApJL, 875, L6 (M87 Paper VI)](https://doi.org/10.3847/2041-8213/ab1141) 32 | - [Chael, A., Johnson, M., Narayan, R. et al. 2016, ApJ, 829, 11C](https://ui.adsabs.harvard.edu/abs/2016ApJ...829...11C/abstract) 33 | - [Chael, A., Johnson, M., Bouman, K. et al. 2018, ApJ, 857, 23C](https://ui.adsabs.harvard.edu/abs/2018ApJ...857...23C/abstract) 34 | - [Chael, A., Bouman, K., Johnson, M. et al., Zenodo (eht-imaging version 1.1.0)](https://zenodo.org/record/2614016) 35 | - [eht-imaging: https://github.com/achael/eht-imaging](https://github.com/achael/eht-imaging) 36 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # First M87 EHT Results: Imaging Pipelines 2 | 3 | **Authors:** The Event Horizon Telescope Collaboration et al. 4 | 5 | **Date:** April 10, 2019 6 | 7 | **Primary Reference:** [The Event Horizon Telescope Collaboration, et al. 2019d, ApJL, 875, L4 (M87 Paper IV)](https://doi.org/10.3847/2041-8213/ab0e85) 8 | 9 | **Data Product Code:** [2019-D01-02](https://eventhorizontelescope.org/for-astronomers/data) 10 | 11 | **Brief Description:** 12 | 13 | We release three imaging pipelines (DIFMAP, eht-imaging and SMILI) 14 | used in the parameter survey of M87 Paper IV Section 6 and later. All 15 | imaging pipelines create images from calibrated uvfits files (see M87 16 | Paper III) simultaneously released ([data product code: 17 | 2019-D01-01](https://eventhorizontelescope.org/for-astronomers/data)). For 18 | more detailed instructions, please see the README file in the 19 | sub-directory for each pipeline. 20 | 21 | We note that, as described in 22 | [2019-D01-01](https://eventhorizontelescope.org/for-astronomers/data), 23 | released visibility data sets have only Stokes *I*, which are slightly 24 | different from data sets used in Paper IV that have dual polarization 25 | at Stokes `RR` and `LL`. This slight difference in released data sets 26 | will provide no net changes in DIFMAP and eht-imaging pipelines, while 27 | it will change self-calibration procedures slightly for SMILI 28 | calibrating `R` and `L` gains separately for the latter dual 29 | polarization data sets. We confirm that reconstructed images are 30 | consistent with images presented in Paper IV on fiducial parameters 31 | (see Paper IV Section 6) for all three pipelines, and will not affect 32 | our conclusions in the M87 publications (Paper I, II, III, IV, V and 33 | VI). 34 | 35 | **Notes:** 36 | 37 | These data files only include Stokes *I* visibilities, while the 38 | published results used data files from the full SR1 release which 39 | include Stokes `RR` and `LL`. The slight difference in the underlying 40 | data from the conversion to Stokes *I* in the single-precision 41 | `*.uvfits` files, as well as differences in the python dependencies 42 | used by `eht-imaging` and `SMILI` (e.g. `numpy`, `astropy`, `scipy`), 43 | may slightly affect the final image. 44 | 45 | **References:** 46 | 47 | - [EHT Collaboration Data Portal Website](https://eventhorizontelescope.org/for-astronomers/data) 48 | - [The Event Horizon Telescope Collaboration, et al. 2019a, ApJL, 875, L1 (M87 Paper I)](https://doi.org/10.3847/2041-8213/ab0ec7) 49 | - [The Event Horizon Telescope Collaboration, et al. 2019b, ApJL, 875, L2 (M87 Paper II)](https://doi.org/10.3847/2041-8213/ab0c96) 50 | - [The Event Horizon Telescope Collaboration, et al. 2019c, ApJL, 875, L3 (M87 Paper III)](https://doi.org/10.3847/2041-8213/ab0c57) 51 | - [The Event Horizon Telescope Collaboration, et al. 2019d, ApJL, 875, L4 (M87 Paper IV)](https://doi.org/10.3847/2041-8213/ab0e85) 52 | - [The Event Horizon Telescope Collaboration, et al. 2019e, ApJL, 875, L5 (M87 Paper V)](https://doi.org/10.3847/2041-8213/ab0f43) 53 | - [The Event Horizon Telescope Collaboration, et al. 2019f, ApJL, 875, L6 (M87 Paper VI)](https://doi.org/10.3847/2041-8213/ab1141) 54 | -------------------------------------------------------------------------------- /smili/README.md: -------------------------------------------------------------------------------- 1 | # SMILI M87 Stokes I Imaging Pipeline for EHT observations in April 2017 2 | 3 | **Authors:** The Event Horizon Telescope Collaboration et al. 4 | 5 | **Date:** April 10, 2019 6 | 7 | **Primary Reference:** [The Event Horizon Telescope Collaboration, et al. 2019d, ApJL, 875, L4 (M87 Paper IV)](https://doi.org/10.3847/2041-8213/ab0e85) 8 | 9 | **Data Product Code:** [2019-D01-02](https://eventhorizontelescope.org/for-astronomers/data) 10 | 11 | **Brief Description:** 12 | 13 | The pipeline reconstructs an image from a specified uvfits file simultaneously 14 | released in the EHT website (data release ID: [2019-D01-01](https://eventhorizontelescope.org/for-astronomers/data)) using a python-interfaced imaging package SMILI (Akiyama et al. 2017a,b) version 0.0.0 (Akiyama et al. 2019). This script assumes to load calibrated data sets at Stokes I (see M87 Paper III). As described in M87 Paper IV Section 6.2.3, the SMILI pipeline also uses the eht-imaging library (Chael et al. 2016, 2018) version 1.1.0 (Chael et al. 2019) solely for pre-imaging calibrations including time averaging, LMT calibration to use the input visibility data sets consistent with the eht-imaging imaging pipeline. The script has been tested in Python 2.7 installed with Anaconda on Ubuntu 18.04 LTS and MacOS 10.13 & 10.14 (with macport or homebrew). 15 | 16 | The pipeline will output three files. 17 | 18 | - image (specified with -o option; assumed to be xxxx.fits here) 19 | - pre-calibrated uvfits file (xxxx.precal.uvfits) 20 | - self-calibrated uvfits file (xxxx.selfcal.uvfits) 21 | 22 | For usage and detail parameters of the pipeline, please read the help document associated in the imaging script "*python smili_imaging_pipeline.py --help*". 23 | 24 | We also include an example driver *example_driver.py* to run *smili_imaging_pipeline.py* on all released data sets. You can run it by "*python example_driver.py --uvfitsdir < xxxx/uvfits > --nproc < number of the processors >*" For details, please have a look at the help document by "*python example_driver.py --help*". 25 | 26 | **Notes** 27 | 28 | We note that, as described in 2019-D01-01, released visibility data sets are 29 | slightly different from data sets used in Paper IV for two reasons. 30 | 31 | (1) They have only Stokes I, while Paper IV data sets have dual polarization at 32 | Stokes RR and LL. This slight difference in released data sets will change 33 | self-calibration procedures slightly in this pipeline since R and L gains are 34 | calibrated separately for the latter dual polarization data sets. We find that 35 | this does not produce any significant differences (within < ~5%) in resultant images. 36 | 37 | (2) In Paper IV, this pipeline strictly uses Stoke I data; JCMT which has a 38 | single polarization is flagged when Stokes I visibilities are computed from RR 39 | and LL, while JCMT is included for self-calibration at the corresponding 40 | polarization. On the other hand, in the released data sets, JCMT has pseudo 41 | Stokes RR/LL data, such that Stokes I computed from them is identical to the 42 | original single Stokes. This will make JCMT be used in imaging, which change 43 | images slightly. For reproducibility of Paper IV results using this data set, in 44 | default, we will not use JCMT for imaging, as we did in Paper IV. If you want 45 | to use it to use all of data sets, you can specify --keepsinglepol to keep it. 46 | 47 | **References:** 48 | 49 | - [EHT Collaboration Data Portal Website](https://eventhorizontelescope.org/for-astronomers/data) 50 | - [The Event Horizon Telescope Collaboration, et al. 2019a, ApJL, 875, L1 (M87 Paper I)](https://doi.org/10.3847/2041-8213/ab0ec7) 51 | - [The Event Horizon Telescope Collaboration, et al. 2019b, ApJL, 875, L2 (M87 Paper II)](https://doi.org/10.3847/2041-8213/ab0c96) 52 | - [The Event Horizon Telescope Collaboration, et al. 2019c, ApJL, 875, L3 (M87 Paper III)](https://doi.org/10.3847/2041-8213/ab0c57) 53 | - [The Event Horizon Telescope Collaboration, et al. 2019d, ApJL, 875, L4 (M87 Paper IV)](https://doi.org/10.3847/2041-8213/ab0e85) 54 | - [The Event Horizon Telescope Collaboration, et al. 2019e, ApJL, 875, L5 (M87 Paper V)](https://doi.org/10.3847/2041-8213/ab0f43) 55 | - [The Event Horizon Telescope Collaboration, et al. 2019f, ApJL, 875, L6 (M87 Paper VI)](https://doi.org/10.3847/2041-8213/ab1141) 56 | - [Akiyama, K., Ikeda, S., Pleau, M. et al. 2017a, ApJ, 838, 1](https://ui.adsabs.harvard.edu/#abs/2017ApJ...838....1A) 57 | - [Akiyama, K., Kuramochi, K., Ikeda, S. et al. 2017b, AJ, 153, 159](https://ui.adsabs.harvard.edu/#abs/2017AJ....153..159A) 58 | - [Akiyama, K., Tazaki, F., Moriyama, K. et al. 2019, Zenodo (SMILI version 0.0.0)](https://zenodo.org/record/2616725) 59 | - [Chael, A., Johnson, M., Narayan, R. et al. 2016, ApJ, 829, 11C](https://ui.adsabs.harvard.edu/abs/2016ApJ...829...11C/abstract) 60 | - [Chael, A., Johnson, M., Bouman, K. et al. 2018, ApJ, 857, 23C](https://ui.adsabs.harvard.edu/abs/2018ApJ...857...23C/abstract) 61 | - [Chael, A., Bouman, K., Johnson, M. et al., Zenodo (eht-imaging version 1.1.0)](https://zenodo.org/record/2614016) 62 | - [Anaconda: https://www.anaconda.com/](https://www.anaconda.com/) 63 | - [eht-imaging: https://github.com/achael/eht-imaging](https://github.com/achael/eht-imaging) 64 | - [SMILI: https://github.com/astrosmili/smili](https://github.com/astrosmili/smili) 65 | -------------------------------------------------------------------------------- /smili/example_driver.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -*- coding: utf-8 -*- 3 | """Example driver for the SMILI M87 Stokes I Imaging Pipeline for EHT observations in April 2017 4 | 5 | Authors: The Event Horizon Telescope Collaboration et al. 6 | Date: April 10, 2019 7 | Primary Reference: The Event Horizon Telescope Collaboration, et al. 2019d, ApJL, 875, L4 (M87 Paper IV) 8 | Data Product Code: 2019-D01-02 9 | 10 | Brief Description: 11 | The script is an example driver of the SMILI M87 Stokes I Imaging Pipeline 12 | (smili_imaging_pipeline.py) for EHT observations in April 2017 attached in the 13 | same directory. For more detail instructions for smili_imaging_pipeline.py 14 | please read the help document associated in the imaging script 15 | "python smili_imaging_pipeline.py --help". 16 | 17 | This sript reconstructs images on fiducial parameters (see Section 6 of M87 18 | Paper IV) from calibratd uvfits files released in the Data Product Release 19 | (2019-D01-01; see also M87 Paper III). You can run it by 20 | "python example_driver.py --uvfitsdir --nproc ." 21 | For details, please have a look at the help document by 22 | "python example_driver.py --help". 23 | 24 | References: 25 | - EHT Collaboration Data Portal Website: 26 | https://eventhorizontelescope.org/for-astronomers/data 27 | - The Event Horizon Telescope Collaboration, et al. 2019a, ApJL, 875, L1 (M87 Paper I) 28 | - The Event Horizon Telescope Collaboration, et al. 2019b, ApJL, 875, L2 (M87 Paper II) 29 | - The Event Horizon Telescope Collaboration, et al. 2019c, ApJL, 875, L3 (M87 Paper III) 30 | - The Event Horizon Telescope Collaboration, et al. 2019d, ApJL, 875, L4 (M87 Paper IV) 31 | - The Event Horizon Telescope Collaboration, et al. 2019e, ApJL, 875, L5 (M87 Paper V) 32 | - The Event Horizon Telescope Collaboration, et al. 2019f, ApJL, 875, L6 (M87 Paper VI) 33 | - Akiyama, K., Ikeda, S., Pleau, M., et al. 2017a, ApJ, 838, 1 34 | - Akiyama, K., Kuramochi, K., Ikeda, S., et al. 2017b, AJ, 153, 159 35 | - Akiyama, K., Tazaki, F., Moriyama, K., et al. 2019, Zenodo (SMILI version 0.0.0) 36 | - SMILI: https://github.com/astrosmili/smili 37 | """ 38 | 39 | 40 | #------------------------------------------------------------------------------- 41 | # Information of Authors 42 | #------------------------------------------------------------------------------- 43 | __author__ = "The Event Horizon Telescope Collaboration et al." 44 | __copyright__ = "Copyright 2019, the Event Horizon Telescope Collaboration et al." 45 | __license__ = "GPL version 3" 46 | __version__ = "1.0" 47 | __date__ = "April 10 2019" 48 | 49 | 50 | #------------------------------------------------------------------------------- 51 | # Modules 52 | #------------------------------------------------------------------------------- 53 | import os 54 | import argparse 55 | import glob 56 | import tqdm 57 | from multiprocessing import Pool 58 | 59 | 60 | #------------------------------------------------------------------------------- 61 | # Loading command line arguments 62 | #------------------------------------------------------------------------------- 63 | parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) 64 | parser.add_argument('--uvfitsdir',metavar='input uvfits directory',type=str,default='../../data/uvfits/', 65 | help='input uvfits directory of the data product release 2019-XX-XXXX') 66 | parser.add_argument('--outputdir',metavar='output fits file',type=str,default="./smili_reconstructions", 67 | help='output directory. The default is "./smili_reconstructions".') 68 | parser.add_argument('--pipeline',metavar='pipeline script',type=str,default="smili_imaging_pipeline.py", 69 | help='the filename of the pipeline scripts. The default is "smili_imaging_pipeline.py"') 70 | parser.add_argument("--nproc",metavar="Number of parallel processors. Default=4.",type=int,default=4) 71 | args = parser.parse_args() 72 | 73 | nproc = args.nproc 74 | uvfitsdir = args.uvfitsdir 75 | outputdir = args.outputdir 76 | pipeline = args.pipeline 77 | 78 | 79 | #------------------------------------------------------------------------------- 80 | # Preparation 81 | #------------------------------------------------------------------------------- 82 | # create the output directory 83 | if not os.path.isdir(outputdir): 84 | command = "mkdir -p %s"%(outputdir) 85 | print(command) 86 | os.system(command) 87 | 88 | # get uvfits files 89 | uvfitsfiles = sorted(glob.glob(os.path.join(uvfitsdir, "*_hops_netcal_StokesI.uvfits"))) 90 | if len(uvfitsfiles) < 1: 91 | raise ValueError("No input uvfits files in %s"%(uvfitsdir)) 92 | 93 | # copy uvfits files 94 | for uvfitsfile in uvfitsfiles: 95 | uvfitsfilebase = os.path.basename(uvfitsfile) 96 | command = "cp -p %s %s"%(uvfitsfile, os.path.join(outputdir, uvfitsfilebase)) 97 | print(command) 98 | os.system(command) 99 | 100 | # generating commands 101 | commands = [] 102 | for uvfitsfile in uvfitsfiles: 103 | uvfitsfilebase = os.path.basename(uvfitsfile) 104 | 105 | # check observing errors 106 | obsdate = None 107 | for day in [95,96,100,101]: 108 | if "_%03d_"%(day) in uvfitsfilebase: 109 | obsdate = day - 90 110 | break 111 | if obsdate is None: 112 | raise ValueError("Apparently there is no obsday information in the filename of the uvfits: %s"%(uvfitsfilebase)) 113 | 114 | command = [] 115 | command.append("python") 116 | command.append(pipeline) 117 | command.append("-i %s"%(os.path.join(outputdir, uvfitsfilebase))) 118 | command.append("--day %d"%(obsdate)) 119 | command.append("--nproc 1") 120 | commands.append(" ".join(command)) 121 | 122 | #------------------------------------------------------------------------------- 123 | # Running the pipeline 124 | #------------------------------------------------------------------------------- 125 | # OMP NUM THREADS must be set to 1, so that each command will use only a single processor. 126 | os.environ['OMP_NUM_THREADS'] = '1' 127 | 128 | # running imaging in parallel 129 | pool = Pool(processes=nproc) 130 | for _ in tqdm.tqdm(pool.imap_unordered(os.system,commands),total=len(commands)): 131 | pass 132 | exit() 133 | -------------------------------------------------------------------------------- /difmap/EHT_Difmap: -------------------------------------------------------------------------------- 1 | ! DIFMAP script for imaging EHT data using a given mask (set of cleaning windows) 2 | ! 3 | ! Requires Caltech's DIFMAP software for CLEAN imaging reconstruction 4 | ! version v2.5b or above that can be obtained from 5 | ! ftp://ftp.astro.caltech.edu/pub/difmap/difmap.html 6 | ! =============================================================================== 7 | ! 8 | ! DIFMAP script developed by the EHT Collaboration 9 | ! 10 | ! ===>> Calling sequence 11 | ! @EHT_difmap uf_file,Mask,rms_target,compact_flux,ALMA_weight,uvbin,uvpower 12 | ! 13 | ! To reproduce fiducial EHT images of M87: 14 | ! 1.- Place i) uv FITS file "your_file_name.uvfits", ii) Difmap script EHT_Difmap (this file), 15 | ! and iii) mask CircMask_r30_x-0.002_y0.022.win in the same directory 16 | ! 2.- Run Difmap and use the calling sequence: 17 | ! @EHT_difmap your_file_name,CircMask_r30_x-0.002_y0.022,-10,0.5,0.1,2,-1 18 | ! (For your convenience, it is better to do not give spaces in between the parameters.) 19 | ! (There is no problem at all in running the script but the relevant outputs will have spaces in its names.) 20 | ! 21 | 22 | 23 | ! Definition of variables 24 | ! ----------------------- 25 | logical interactive 26 | logical LMT_amp_gains 27 | logical remove_gaussian 28 | logical save_results 29 | integer clean_niter 30 | float clean_gain 31 | float compact_flux 32 | float cp_fl 33 | float ALMA_weight 34 | float uvz_flux 35 | float uvz_weight 36 | integer nx,ny 37 | float dx,dy 38 | float res_rms_noise 39 | float rms_target 40 | float pico 41 | float old_chisq 42 | float dynam 43 | float solint(20) 44 | 45 | 46 | 47 | ! Set of parameters to choose 48 | ! =========================== 49 | 50 | ! Cleaning parameters: gain and number of iterations 51 | clean_gain = 0.02 52 | clean_niter = 100 53 | 54 | ! Target relative rms reduction during cleaning. Use a large negative number 55 | ! to ignore this, using only a target compact flux criterium 56 | rms_target = %3 57 | 58 | ! Target compact flux. Use a large value (i.e. > 10) to ignore this 59 | compact_flux = %4 60 | 61 | ! Possibility to down-weight ALMA. If not, use ALMA_weight=1.0 62 | ALMA_weight = %5 63 | 64 | ! Possibility to force LMT amplitude gains to initial phase-only image 65 | LMT_amp_gains = True 66 | 67 | ! Interactive (shows clean images) 68 | interactive = False 69 | 70 | ! Save results 71 | save_results = True 72 | 73 | ! To remove Gaussian in final image 74 | remove_gaussian = True 75 | 76 | ! Dynamic range in the residual map cleaning stopping criteria. Use 0 to ignore 77 | dynam = 0 78 | 79 | ! Zero flux and weight 80 | uvz_flux = 1.2 81 | uvz_weight = 10 82 | 83 | ! Mapsize 84 | nx=1024 85 | ny=1024 86 | 87 | ! Pixelsize 88 | dx=0.002 89 | dy=0.002 90 | 91 | 92 | ! ### Routines ### 93 | 94 | ! => Clean+selfcal with several stopping criteria: 95 | ! 1) Relative decrease in the residuals rms is below a given threshold 96 | ! 2) A desired cleaned compact flux is reached 97 | ! 3) The visibilities chi2 increases 98 | ! 4) Residual map dynamic range is below a given threshold 99 | ! Keyword %3 establishes whether intermediate selfcal is performed 100 | #+clean_rms_or_flux \ 101 | repeat ;\ 102 | res_rms_noise = imstat(rms)/imstat(noise) ;\ 103 | old_chisq = uvstat(chisq) ;\ 104 | clean clean_niter,clean_gain ;\ 105 | cp_fl = model_flux ;\ 106 | if (%3) selfcal ;\ 107 | if (model_flux >= %1 | peak(flux,max) <= imstat(rms) * dynam | \ 108 | uvstat(chisq) >= old_chisq) ;\ 109 | break ;\ 110 | end if ;\ 111 | until ((res_rms_noise-imstat(rms)/imstat(noise))/res_rms_noise <= %2) ;\ 112 | selfcal 113 | 114 | ! => Clean map without contours 115 | #+limp docont=false;mapl clean;docont=true 116 | 117 | ! => Flagging and unflagging of intra-site baselines 118 | #+flag_intra flag aa-ap,true;flag jc-sm,true 119 | #+unflag_intra unflag aa-ap,true;unflag jc-sm,true 120 | 121 | ! => Adding a new gaussian component to recover the zero-flux followed by a 122 | ! round of model-fitting and phase self-calibration. Size fixed 123 | #+add_gauss \ 124 | addcmp uvz_flux-model_flux,true,0,0,true,2.0,true ;\ 125 | modelfit 50 ;\ 126 | selfcal 127 | 128 | 129 | ! ### Main script ### 130 | ! ################### 131 | 132 | 133 | ! Selects input uv data 134 | observe %1.uvfits 135 | 136 | ! Cleaning Stokes I image 137 | select pi 138 | 139 | ! New weight for ALMA 140 | selfant aa,false,ALMA_weight 141 | 142 | ! Map size 143 | mapsize nx,dx,ny,dy 144 | 145 | ! Zero flux and weight 146 | uvzero uvz_flux,uvz_weight 147 | 148 | ! Uniform weight 149 | uvweight %6,%7 150 | 151 | ! First phase self-cal 152 | startmod 153 | 154 | ! Reading the file specifying clean windows (mask) 155 | rwin %2.win 156 | 157 | !>> First cleaning and phase self-calibration 158 | clean_rms_or_flux compact_flux,rms_target,true 159 | add_gauss 160 | if (interactive) ;\ 161 | print ""; print " ==> Phase-only self-calibration image <=="; print "" ;\ 162 | delwin; device /xw; limp; rwin %2.win; end if 163 | 164 | !<< Option for forcing LMT amplitude gains to initial phase-only map >> 165 | ! solint of 10 minutes 166 | if (LMT_amp_gains) ;\ 167 | selfant ,true ;\ 168 | selfant LM,false ;\ 169 | selfcal true,true,10 ;\ 170 | selfant ,false ;\ 171 | clrmod true,true ;\ 172 | clean_rms_or_flux compact_flux,rms_target,true ;\ 173 | add_gauss ;\ 174 | print ""; print " ==> Phase-only (fixed LMT gains) image <=="; print "" ;\ 175 | if (interactive); delwin; limp; rwin %2.win; end if ;\ 176 | end if 177 | 178 | !>> Global amplitude calibration not normalized (observed flux may change) 179 | gscale true 180 | 181 | !>> Cleaning first gscale map 182 | clrmod true,true 183 | clean_rms_or_flux compact_flux,rms_target,false 184 | add_gauss 185 | 186 | if (interactive) ;\ 187 | print ""; print " ==> gscale-only self-calibration image <=="; print "" ;\ 188 | delwin; limp ; rwin %2.win ;\ 189 | end if 190 | 191 | !>> Rounds of amplitude self-calibration 192 | solint = 240,120,60,30,30,20,10,10,5,5,2,2,1,1,0.5,0.5,0,0,0,0 193 | do i=1,20 ;\ 194 | selflims 1.2 ;\ 195 | selfcal true,true,solint(i) ;\ 196 | clrmod true,true ;\ 197 | clean_rms_or_flux compact_flux,rms_target,false ;\ 198 | add_gauss ;\ 199 | if (interactive) ;\ 200 | print ""; print " ==> Amp+phs",solint(i),"self-cal image <=="; print "" ;\ 201 | delwin; limp; rwin %2.win ;\ 202 | end if ;\ 203 | end do 204 | 205 | ! If removing Gaussian in final image 206 | if (remove_gaussian) ;\ 207 | clrmod true,true ;\ 208 | clean_rms_or_flux compact_flux,rms_target,false ;\ 209 | flag_intra ;\ 210 | end if 211 | 212 | !>> Final statistics 213 | restore 214 | pico=peak(flux,max) 215 | invert 216 | print ""; print " ==> Image statistic <=="; print "" 217 | print " ==> Cleaned compact flux ",cp_fl 218 | print " ==> Image total flux ",model_flux 219 | if (remove_gaussian) ;\ 220 | print " ==> Gaussian component removed and intra-sites flagged" ; end if 221 | print " ==> Chi2: ", uvstat(chisq) 222 | print " ==> rms data-model ", uvstat(rms),"Jy" 223 | print " ==> Expected image noise", imstat(noise),"Jy/beam" 224 | print " ==> Actual image noise", imstat(rms),"Jy/beam" 225 | print " ==> Dynamic range (over 3-sigma)",pico/(3*imstat(rms)) 226 | 227 | !>> Saving final files 228 | if (save_results) ;\ 229 | integer of ;\ 230 | of = outfile(%1.%2.RT%3.CF%4.ALMA%5.UVW%6_%7.stat) ;\ 231 | fprintf of, "ALMA weight=" // strnum(ALMA_weight) // "\n" ;\ 232 | fprintf of, "Compact flux="//strnum(cp_fl) // "\n" ;\ 233 | fprintf of, "Total flux=" // strnum(model_flux) // "\n" ;\ 234 | fprintf of, "Chi2= " // strnum(uvstat(chisq)) // "\n" ;\ 235 | fprintf of, "Image rms=" // strnum(imstat(rms)) // "\n" ;\ 236 | fprintf of, "Dynamic range=" // strnum(pico/(3*imstat(rms))) ;\ 237 | close(of) ; 238 | save %1.%2.RT%3.CF%4.ALMA%5.UVW%6_%7 239 | restore 0,0,0,true ;\ 240 | wmap %1.%2.RT%3.CF%4.ALMA%5.UVW%6_%7.noresiduals.fits ;\ 241 | end if 242 | 243 | if (interactive) ;\ 244 | loglevs 3*imstat(rms)/pico*100. ;\ 245 | mapl cln ;\ 246 | print ""; print " ==> Final image <=="; print "" ;\ 247 | end if 248 | -------------------------------------------------------------------------------- /eht-imaging/eht-imaging_pipeline.py: -------------------------------------------------------------------------------- 1 | """eht-imaging M87 Stokes I Imaging Pipeline for EHT observations in April 2017 2 | 3 | Authors: The Event Horizon Telescope Collaboration et al. 4 | Date: April 10, 2019 5 | Primary Reference: The Event Horizon Telescope Collaboration, et al. 2019d, ApJL, 875, L4 (M87 Paper IV) 6 | Data Product Code: 2019-D01-02 7 | 8 | Brief Description: 9 | The pipeline reconstructs an image from uvfits files simultaneously 10 | released in the EHT website (data release ID: 2019-D01-01) using a 11 | python-interfaced imaging package eht-imaging (Chael et al. 2016,2018). 12 | 13 | To run the pipeline, specify the input uvfits data file. Multiple bands 14 | can be specified separately. Additional flags control the output, which 15 | is only the reconstructed image as a FITS image by default. 16 | 17 | Example call: 18 | python eht-imaging_pipeline.py -i ../SR1_M87_2017_101_lo_hops_netcal_StokesI.uvfits -i2 ../SR1_M87_2017_101_hi_hops_netcal_StokesI.uvfits --savepdf 19 | 20 | For additional details, please read the help document associated in the 21 | imaging script: "python eht-imaging_pipeline.py --help". 22 | 23 | Additional References: 24 | - EHT Collaboration Data Portal Website: 25 | https://eventhorizontelescope.org/for-astronomers/data 26 | - The Event Horizon Telescope Collaboration, et al. 2019a, ApJL, 875, L1 (M87 Paper I) 27 | - The Event Horizon Telescope Collaboration, et al. 2019b, ApJL, 875, L2 (M87 Paper II) 28 | - The Event Horizon Telescope Collaboration, et al. 2019c, ApJL, 875, L3 (M87 Paper III) 29 | - The Event Horizon Telescope Collaboration, et al. 2019d, ApJL, 875, L4 (M87 Paper IV) 30 | - The Event Horizon Telescope Collaboration, et al. 2019e, ApJL, 875, L5 (M87 Paper V) 31 | - The Event Horizon Telescope Collaboration, et al. 2019f, ApJL, 875, L6 (M87 Paper VI) 32 | - Chael, A., Johnson, M., Narayan, R., et al. 2016, ApJ, 829, 11C 33 | - Chael, A., Johnson, M., Bouman, K., et al. 2018, ApJ, 857, 23C 34 | - Chael, A., Bouman, K., Johnson, M. et al., Zenodo (eht-imaging version 1.1.0) 35 | - eht-imaging: https://github.com/achael/eht-imaging 36 | """ 37 | 38 | #------------------------------------------------------------------------------- 39 | # Author Information 40 | #------------------------------------------------------------------------------- 41 | __author__ = "The Event Horizon Telescope Collaboration et al." 42 | __copyright__ = "Copyright 2019, the Event Horizon Telescope Collaboration et al." 43 | __license__ = "GPL version 3" 44 | __version__ = "1.0" 45 | __date__ = "April 10 2019" 46 | 47 | #------------------------------------------------------------------------------- 48 | # Modules 49 | #------------------------------------------------------------------------------- 50 | import matplotlib 51 | matplotlib.use('Agg') 52 | 53 | import os 54 | import argparse 55 | import ehtim as eh 56 | import numpy as np 57 | 58 | #------------------------------------------------------------------------------- 59 | # Load command-line arguments 60 | #------------------------------------------------------------------------------- 61 | parser = argparse.ArgumentParser(description="Fiducial eht-imaging script for M87") 62 | parser.add_argument('-i', '--infile', default="obs.uvfits",help="input UVFITS file") 63 | parser.add_argument('-i2', '--infile2', default="", help="optional 2nd input file (different band) for imaging") 64 | parser.add_argument('-o', '--outfile', default='out.fits', help='output FITS image') 65 | parser.add_argument('--savepdf', default=False, help='saves image pdf (True or False)?',action='store_true') 66 | parser.add_argument('--imgsum', default=False, help='generate image summary pdf',action='store_true') 67 | args = parser.parse_args() 68 | 69 | #------------------------------------------------------------------------------- 70 | # Fiducial imaging parameters obtained from the eht-imaging parameter survey 71 | #------------------------------------------------------------------------------- 72 | zbl = 0.60 # Total compact flux density (Jy) 73 | prior_fwhm = 40.0*eh.RADPERUAS # Gaussian prior FWHM (radians) 74 | sys_noise = 0.02 # fractional systematic noise 75 | # added to complex visibilities 76 | 77 | # constant regularization weights 78 | reg_term = {'simple' : 100, # Maximum-Entropy 79 | 'tv' : 1.0, # Total Variation 80 | 'tv2' : 1.0, # Total Squared Variation 81 | 'l1' : 0.0, # L1 sparsity prior 82 | 'flux' : 1e4} # compact flux constraint 83 | 84 | # initial data weights - these are updated throughout the imaging pipeline 85 | data_term = {'amp' : 0.2, # visibility amplitudes 86 | 'cphase' : 1.0, # closure phases 87 | 'logcamp': 1.0} # log closure amplitudes 88 | 89 | 90 | #------------------------------------------------------------------------------- 91 | # Fixed imaging parameters 92 | #------------------------------------------------------------------------------- 93 | obsfile = args.infile # Pre-processed observation file 94 | ttype = 'nfft' # Type of Fourier transform ('direct', 'nfft', or 'fast') 95 | npix = 64 # Number of pixels across the reconstructed image 96 | fov = 128*eh.RADPERUAS # Field of view of the reconstructed image 97 | maxit = 100 # Maximum number of convergence iterations for imaging 98 | stop = 1e-4 # Imager stopping criterion 99 | gain_tol = [0.02,0.2] # Asymmetric gain tolerance for self-cal; we expect larger values 100 | # for unaccounted sensitivity loss 101 | # than for unaccounted sensitivity improvement 102 | uv_zblcut = 0.1e9 # uv-distance that separates the inter-site "zero"-baselines 103 | # from intra-site baselines 104 | reverse_taper_uas = 5.0 # Finest resolution of reconstructed features 105 | 106 | # Specify the SEFD error budget 107 | # (reported in First M87 Event Horizon Telescope Results III: Data Processing and Calibration) 108 | SEFD_error_budget = {'AA':0.10, 109 | 'AP':0.11, 110 | 'AZ':0.07, 111 | 'LM':0.22, 112 | 'PV':0.10, 113 | 'SM':0.15, 114 | 'JC':0.14, 115 | 'SP':0.07} 116 | 117 | # Add systematic noise tolerance for amplitude a-priori calibration errors 118 | # Start with the SEFD noise (but need sqrt) 119 | # then rescale to ensure that final results respect the stated error budget 120 | systematic_noise = SEFD_error_budget.copy() 121 | for key in systematic_noise.keys(): 122 | systematic_noise[key] = ((1.0+systematic_noise[key])**0.5 - 1.0) * 0.25 123 | 124 | # Extra noise added for the LMT, which has much more variability than the a-priori error budget 125 | systematic_noise['LM'] += 0.15 126 | 127 | #------------------------------------------------------------------------------- 128 | # Define helper functions 129 | #------------------------------------------------------------------------------- 130 | 131 | # Rescale short baselines to excise contributions from extended flux. 132 | # setting zbl < zbl_tot assumes there is an extended constant flux component of zbl_tot-zbl Jy 133 | def rescale_zerobaseline(obs, totflux, orig_totflux, uv_max): 134 | multiplier = zbl / zbl_tot 135 | for j in range(len(obs.data)): 136 | if (obs.data['u'][j]**2 + obs.data['v'][j]**2)**0.5 >= uv_max: continue 137 | for field in ['vis','qvis','uvis','vvis','sigma','qsigma','usigma','vsigma']: 138 | obs.data[field][j] *= multiplier 139 | 140 | # repeat imaging with blurring to assure good convergence 141 | def converge(major=3, blur_frac=1.0): 142 | for repeat in range(major): 143 | init = imgr.out_last().blur_circ(blur_frac*res) 144 | imgr.init_next = init 145 | imgr.make_image_I(show_updates=False) 146 | 147 | #------------------------------------------------------------------------------- 148 | # Prepare the data 149 | #------------------------------------------------------------------------------- 150 | 151 | # Load a single uvfits file 152 | if args.infile2 == '': 153 | # load the uvfits file 154 | obs = eh.obsdata.load_uvfits(obsfile) 155 | 156 | # scan-average the data 157 | # identify the scans (times of continous observation) in the data 158 | obs.add_scans() 159 | 160 | # coherently average the scans, which can be averaged due to ad-hoc phasing 161 | obs = obs.avg_coherent(0.,scan_avg=True) 162 | 163 | # If two uvfits files are passed as input (e.g., high and low band) then use both datasets, 164 | # but do not form closure quantities between the two datasets 165 | else: 166 | # load the two uvfits files 167 | obs1 = eh.obsdata.load_uvfits(obsfile) 168 | obs2 = eh.obsdata.load_uvfits(args.infile2) 169 | 170 | # Average data based on individual scan lengths 171 | obs1.add_scans() 172 | obs2.add_scans() 173 | obs1 = obs1.avg_coherent(0.,scan_avg=True) 174 | obs2 = obs2.avg_coherent(0.,scan_avg=True) 175 | 176 | # Add a slight offset to avoid mixed closure products 177 | obs2.data['time'] += 0.00001 178 | 179 | # concatenate the observations into a single observation object 180 | obs = obs1.copy() 181 | obs.data = np.concatenate([obs1.data,obs2.data]) 182 | 183 | # Estimate the total flux density from the ALMA(AA) -- APEX(AP) zero baseline 184 | zbl_tot = np.median(obs.unpack_bl('AA','AP','amp')['amp']) 185 | if zbl > zbl_tot: 186 | print('Warning: Specified total compact flux density ' + 187 | 'exceeds total flux density measured on AA-AP!') 188 | 189 | # Flag out sites in the obs.tarr table with no measurements 190 | allsites = set(obs.unpack(['t1'])['t1'])|set(obs.unpack(['t2'])['t2']) 191 | obs.tarr = obs.tarr[[o in allsites for o in obs.tarr['site']]] 192 | obs = eh.obsdata.Obsdata(obs.ra, obs.dec, obs.rf, obs.bw, obs.data, obs.tarr, 193 | source=obs.source, mjd=obs.mjd, 194 | ampcal=obs.ampcal, phasecal=obs.phasecal) 195 | 196 | obs_orig = obs.copy() # save obs before any further modifications 197 | 198 | # Rescale short baselines to excize contributions from extended flux 199 | if zbl != zbl_tot: 200 | rescale_zerobaseline(obs, zbl, zbl_tot, uv_zblcut) 201 | 202 | # Order the stations by SNR. 203 | # This will create a minimal set of closure quantities 204 | # with the highest snr and smallest covariance. 205 | obs.reorder_tarr_snr() 206 | 207 | #------------------------------------------------------------------------------- 208 | # Pre-calibrate the data 209 | #------------------------------------------------------------------------------- 210 | 211 | obs_sc = obs.copy() # From here on out, don't change obs. Use obs_sc to track gain changes 212 | res = obs.res() # The nominal array resolution: 1/(longest baseline) 213 | 214 | # Make a Gaussian prior image for maximum entropy regularization 215 | # This Gaussian is also the initial image 216 | gaussprior = eh.image.make_square(obs_sc, npix, fov) 217 | gaussprior = gaussprior.add_gauss(zbl, (prior_fwhm, prior_fwhm, 0, 0, 0)) 218 | 219 | # To avoid gradient singularities in the first step, add an additional small Gaussians 220 | gaussprior = gaussprior.add_gauss(zbl*1e-3, (prior_fwhm, prior_fwhm, 0, prior_fwhm, prior_fwhm)) 221 | 222 | # Reverse taper the observation: this enforces a maximum resolution on reconstructed features 223 | if reverse_taper_uas > 0: 224 | obs_sc = obs_sc.reverse_taper(reverse_taper_uas*eh.RADPERUAS) 225 | 226 | # Add non-closing systematic noise to the observation 227 | obs_sc = obs_sc.add_fractional_noise(sys_noise) 228 | 229 | # Make a copy of the initial data (before any self-calibration but after the taper) 230 | obs_sc_init = obs_sc.copy() 231 | 232 | # Self-calibrate the LMT to a Gaussian model 233 | # (Refer to Section 4's "Pre-Imaging Considerations") 234 | print("Self-calibrating the LMT to a Gaussian model for LMT-SMT...") 235 | 236 | obs_LMT = obs_sc_init.flag_uvdist(uv_max=2e9) # only consider the short baselines (LMT-SMT) 237 | if reverse_taper_uas > 0: 238 | # start with original data that had no reverse taper applied. 239 | # Re-taper, if necessary 240 | obs_LMT = obs_LMT.taper(reverse_taper_uas*eh.RADPERUAS) 241 | 242 | # Make a Gaussian image that would result in the LMT-SMT baseline visibility amplitude 243 | # as estimated in Section 4's "Pre-Imaging Considerations". 244 | # This is achieved with a Gaussian of size 60 microarcseconds and total flux of 0.6 Jy 245 | gausspriorLMT = eh.image.make_square(obs, npix, fov) 246 | gausspriorLMT = gausspriorLMT.add_gauss(0.6, (60.0*eh.RADPERUAS, 60.0*eh.RADPERUAS, 0, 0, 0)) 247 | 248 | # Self-calibrate the LMT visibilities to the gausspriorLMT image 249 | # to enforce the estimated LMT-SMT visibility amplitude 250 | caltab = eh.selfcal(obs_LMT, gausspriorLMT, sites=['LM'], gain_tol=1.0, 251 | method='both', ttype=ttype, caltable=True) 252 | 253 | # Spply the calibration solution to the full (and potentially tapered) dataset 254 | obs_sc = caltab.applycal(obs_sc, interp='nearest', extrapolate=True) 255 | 256 | #------------------------------------------------------------------------------- 257 | # Reconstruct an image 258 | #------------------------------------------------------------------------------- 259 | 260 | 261 | # First Round of Imaging 262 | #------------------------- 263 | print("Round 1: Imaging with visibility amplitudes and closure quantities...") 264 | 265 | # Initialize imaging with a Gaussian image 266 | imgr = eh.imager.Imager(obs_sc, gaussprior, prior_im=gaussprior, 267 | flux=zbl, data_term=data_term, maxit=maxit, 268 | norm_reg=True, systematic_noise=systematic_noise, 269 | reg_term=reg_term, ttype=ttype, cp_uv_min=uv_zblcut, stop=stop) 270 | 271 | # Imaging 272 | imgr.make_image_I(show_updates=False) 273 | converge() 274 | 275 | # Self-calibrate to the previous model (phase-only); 276 | # The solution_interval is 0 to align phases from high and low bands if needed 277 | obs_sc = eh.selfcal(obs_sc, imgr.out_last(), method='phase', ttype=ttype, solution_interval=0.0) 278 | 279 | 280 | # Second Round of Imaging 281 | #------------------------- 282 | print("Round 2: Imaging with visibilities and closure quantities...") 283 | 284 | # Blur the previous reconstruction to the intrinsic resolution of ~25 uas 285 | init = imgr.out_last().blur_circ(res) 286 | 287 | # Increase the weights on the data terms and reinitialize imaging 288 | data_term_intermediate = {'vis':imgr.dat_terms_last()['amp']*10, 289 | 'cphase':imgr.dat_terms_last()['cphase']*10, 290 | 'logcamp':imgr.dat_terms_last()['logcamp']*10} 291 | 292 | imgr = eh.imager.Imager(obs_sc, init, prior_im=gaussprior, flux=zbl, 293 | data_term=data_term_intermediate, maxit=maxit, norm_reg=True, 294 | systematic_noise=systematic_noise, reg_term = reg_term, ttype=ttype, 295 | cp_uv_min=uv_zblcut, stop=stop) 296 | # Imaging 297 | imgr.make_image_I(show_updates=False) 298 | converge() 299 | 300 | # Self-calibrate to the previous model starting from scratch 301 | # phase for all sites; amplitudes for LMT 302 | obs_sc = eh.selfcal(obs_sc_init, imgr.out_last(), method='phase', ttype=ttype) 303 | caltab = eh.selfcal(obs_sc, imgr.out_last(), sites=['LM'], method='both', gain_tol=gain_tol, 304 | ttype=ttype, caltable=True) 305 | obs_sc = caltab.applycal(obs_sc, interp='nearest',extrapolate=True) 306 | 307 | 308 | # Third and Fourth Rounds of Imaging 309 | #----------------------------------- 310 | print("Rounds 3+4: Imaging with visibilities and closure quantities...") 311 | 312 | # Increase the data weights before imaging again 313 | data_term_final = {'vis':imgr.dat_terms_last()['vis']*5, 314 | 'cphase':imgr.dat_terms_last()['cphase']*2, 315 | 'logcamp':imgr.dat_terms_last()['logcamp']*2} 316 | 317 | # Repeat imaging twice 318 | for repeat_selfcal in range(2): 319 | # Blur the previous reconstruction to the intrinsic resolution of ~25 uas 320 | init = imgr.out_last().blur_circ(res) 321 | 322 | # Reinitialize imaging now using complex visibilities; common systematic noise 323 | imgr = eh.imager.Imager(obs_sc, init, prior_im=gaussprior, flux=zbl, 324 | data_term=data_term_final, maxit=maxit, norm_reg=True, 325 | systematic_noise=0.01, reg_term=reg_term, ttype=ttype, 326 | cp_uv_min=uv_zblcut, stop=stop) 327 | # Imaging 328 | imgr.make_image_I(show_updates=False) 329 | converge() 330 | 331 | # Self-calibrate 332 | caltab = eh.selfcal(obs_sc_init, imgr.out_last(), method='both', 333 | sites=['SM','JC','AA','AP','LM','SP','AZ'], 334 | ttype=ttype, gain_tol=gain_tol, caltable=True) 335 | obs_sc = caltab.applycal(obs_sc_init, interp='nearest',extrapolate=True) 336 | 337 | #------------------------------------------------------------------------------- 338 | # Output the results 339 | #------------------------------------------------------------------------------- 340 | 341 | # Save the reconstructed image 342 | im_out = imgr.out_last().copy() 343 | 344 | # If an inverse taper was used, restore the final image 345 | # to be consistent with the original data 346 | if reverse_taper_uas > 0.0: 347 | im_out = im_out.blur_circ(reverse_taper_uas*eh.RADPERUAS) 348 | 349 | # Save the final image 350 | im_out.save_fits(args.outfile) 351 | 352 | # Optionally save a pdf of the final image 353 | if args.savepdf: 354 | pdfout = os.path.splitext(args.outfile)[0] + '.pdf' 355 | im_out.display(cbar_unit=['Tb'],label_type='scale',export_pdf=pdfout) 356 | 357 | # Optionally generate a summary of the final image and associated data consistency metrics 358 | if args.imgsum: 359 | 360 | # Add a large gaussian component to account for the missing flux 361 | # so the final image can be compared with the original data 362 | im_addcmp = im_out.add_zblterm(obs_orig, uv_zblcut, debias=True) 363 | obs_sc_addcmp = eh.selfcal(obs_orig, im_addcmp, method='both', ttype=ttype) 364 | 365 | # Save an image summary sheet 366 | # Note! these chi2 values will not be identical to what is shown in the paper 367 | # because they are combining high and low band 368 | matplotlib.pyplot.close('all') 369 | outimgsum = os.path.splitext(args.outfile)[0] + '_imgsum.pdf' 370 | eh.imgsum(im_addcmp, obs_sc_addcmp, obs_orig, outimgsum ,cp_uv_min=uv_zblcut) 371 | -------------------------------------------------------------------------------- /smili/smili_imaging_pipeline.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -*- coding: utf-8 -*- 3 | """SMILI M87 Stokes I Imaging Pipeline for EHT observations in April 2017 4 | 5 | Authors: The Event Horizon Telescope Collaboration et al. 6 | Date: April 10, 2019 7 | Primary Reference: The Event Horizon Telescope Collaboration, et al. 2019d, ApJL, 875, L4 (M87 Paper IV) 8 | Data Product Code: 2019-D01-02 9 | 10 | Brief Description: 11 | The pipeline reconstructs an image from a specified uvfits file simultaneously 12 | released in the EHT website (data release ID: 2019-D01-01) using a 13 | python-interfaced imaging package SMILI (Akiyama et al. 2017a,b) version 0.0.0 14 | (Akiyama et al. 2019).This script assumes to load calibrated data sets at Stokes 15 | I (see M87 Paper III). As described in M87 Paper IV Section 6.2.3, the SMILI 16 | pipeline also uses the eht-imaging library (Chael et al. 2016, 2018) 17 | version 1.1.0 (Chael et al. 2019) solely for pre-imaging calibrations including 18 | time averaging, LMT calibration to use the input visibility data sets consistent 19 | with the eht-imaging imaging pipeline. The script has been tested in Python 2.7 20 | installed with Anaconda on Ubuntu 18.04 LTS and MacOS 10.13 & 10.14 (with 21 | macport or homebrew). 22 | 23 | Notes: 24 | We note that, as described in 2019-D01-01, released visibility data sets are 25 | slightly different from data sets used in Paper IV for two reasons. 26 | 27 | (1) They have only Stokes I, while Paper IV data sets have dual polarization at 28 | Stokes RR and LL. This slight difference in released data sets will change 29 | self-calibration procedures slightly in this pipeline since R and L gains are 30 | calibrated separately for the latter dual polarization data sets. We find that 31 | this does not produce any significant differences (within < ~5%) in resultant images. 32 | 33 | (2) In Paper IV, this pipeline strictly uses Stoke I data; JCMT which has a 34 | single polarization is flagged when Stokes I visibilities are computed from RR 35 | and LL, while JCMT is included for self-calibration at the corresponding 36 | polarization. On the other hand, in the released data sets, JCMT has pseudo 37 | Stokes RR/LL data, such that Stokes I computed from them is identical to the 38 | original single Stokes. This will make JCMT be used in imaging, which change 39 | images slightly. For reproducibility of Paper IV results using this data set, in 40 | default, we will not use JCMT for imaging, as we did in Paper IV. If you want 41 | to use it to use all of data sets, you can specify --keepsinglepol to keep it. 42 | 43 | The pipeline will output three files. 44 | - image (specified with -o option; assumed to be xxxx.fits here) 45 | - pre-calibrated uvfits file (xxxx.precal.uvfits) 46 | - self-calibrated uvfits file (xxxx.selfcal.uvfits) 47 | For usage and detail parameters of the pipeline, please 48 | read the help document associated in the imaging script 49 | "python smili_imaging_pipeline.py --help". 50 | 51 | References: 52 | - EHT Collaboration Data Portal Website: 53 | https://eventhorizontelescope.org/for-astronomers/data 54 | - The Event Horizon Telescope Collaboration, et al. 2019a, ApJL, 875, L1 (M87 Paper I) 55 | - The Event Horizon Telescope Collaboration, et al. 2019b, ApJL, 875, L2 (M87 Paper II) 56 | - The Event Horizon Telescope Collaboration, et al. 2019c, ApJL, 875, L3 (M87 Paper III) 57 | - The Event Horizon Telescope Collaboration, et al. 2019d, ApJL, 875, L4 (M87 Paper IV) 58 | - The Event Horizon Telescope Collaboration, et al. 2019e, ApJL, 875, L5 (M87 Paper V) 59 | - The Event Horizon Telescope Collaboration, et al. 2019f, ApJL, 875, L6 (M87 Paper VI) 60 | - Akiyama, K., Ikeda, S., Pleau, M., et al. 2017a, ApJ, 838, 1 61 | - Akiyama, K., Kuramochi, K., Ikeda, S., et al. 2017b, AJ, 153, 159 62 | - Akiyama, K., Tazaki, F., Moriyama, K., et al. 2019, Zenodo (SMILI version 0.0.0) 63 | - Chael, A., Johnson, M., Narayan, R., et al. 2016, ApJ, 829, 11C 64 | - Chael, A., Johnson, M., Bouman, K., et al. 2018, ApJ, 857, 23C 65 | - Chael, A., Bouman, K., Johnson, M. et al., Zenodo (eht-imaging version 1.1.0) 66 | - Anaconda: https://www.anaconda.com/ 67 | - eht-imaging: https://github.com/achael/eht-imaging 68 | - SMILI: https://github.com/astrosmili/smili 69 | """ 70 | 71 | 72 | #------------------------------------------------------------------------------- 73 | # Information of Authors 74 | #------------------------------------------------------------------------------- 75 | __author__ = "The Event Horizon Telescope Collaboration et al." 76 | __copyright__ = "Copyright 2019, the Event Horizon Telescope Collaboration et al." 77 | __license__ = "GPL version 3" 78 | __version__ = "1.0" 79 | __date__ = "April 10 2019" 80 | 81 | 82 | #------------------------------------------------------------------------------- 83 | # Modules 84 | #------------------------------------------------------------------------------- 85 | from smili import uvdata,imdata,imaging,util 86 | import ehtim as eh 87 | import pandas as pd 88 | import numpy as np 89 | import copy 90 | import os 91 | import argparse 92 | 93 | 94 | #------------------------------------------------------------------------------- 95 | # Loading command line arguments 96 | #------------------------------------------------------------------------------- 97 | parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) 98 | parser.add_argument('-i','--input',metavar='input uvfits file',type=str,required=True, 99 | help='input uvfits file name ') 100 | parser.add_argument('-o','--output',metavar='output fits file',type=str,default=None, 101 | help='output fits file name') 102 | parser.add_argument('--day',metavar='observing day of April 2017',type=int,required=True, 103 | help='Observational Day of April: 5, 6, 10 or 11. This must be specified correctly to apply the correct total flux density used in the network calibration.') 104 | parser.add_argument("--prior",metavar='prior FWHM size',type=float,default=40., 105 | help='Prior FWHM (uas)') 106 | parser.add_argument("--tfd",metavar='total flux density',type=float,default=0.5, 107 | help='Target total flux density (Jy)') 108 | parser.add_argument("--sys",metavar='fractional systematic error',type=float,default=0., 109 | help='Fractional systematic error to be added to visibilities in quadrature (percent)') 110 | parser.add_argument("--keepsinglepol", action='store_true', default=False, 111 | help='This option uses single pol station(s) for imaging; this will cause slight difference in the resultant images from M87 Paper IV.') 112 | parser.add_argument("--lambl1",metavar='lambda l1',type=float,default=1., 113 | help='lambda l1') 114 | parser.add_argument("--lambtv",metavar='lambda tv',type=float,default=1e3, 115 | help='lambda tv') 116 | parser.add_argument("--lambtsv",metavar='lambda tsv',type=float,default=1e4, 117 | help='lambda tsv') 118 | parser.add_argument("--nproc",metavar="Number of parallel processors. Default=1.",type=int,default=1) 119 | args = parser.parse_args() 120 | 121 | # Input uvfits file 122 | inputuvfits = args.input 123 | 124 | # Output directory 125 | outputfits = args.output 126 | if outputfits is None: 127 | outputfits = os.path.splitext(inputuvfits)[0]+".fits" 128 | outputhead = os.path.splitext(outputfits)[0] 129 | 130 | # Observational date 131 | day = args.day 132 | if day not in [5,6,10,11]: 133 | raise ValueError("The observational day (--day integer) must be (April) 5, 6, 10 or 11.") 134 | 135 | # Prior size 136 | prior_fwhm = args.prior 137 | 138 | # Target total flux density 139 | totalflux = args.tfd 140 | 141 | # Systematic error 142 | syserr = args.sys 143 | 144 | # Lambda L1, TV and TSV 145 | lambl1 = args.lambl1 146 | lambtv = args.lambtv 147 | lambtsv = args.lambtsv 148 | 149 | # Number of processors 150 | nproc = args.nproc 151 | os.environ['OMP_NUM_THREADS'] = '%d'%(nproc) 152 | 153 | #------------------------------------------------------------------------------- 154 | # Other tuning parameters fixed in the paper. 155 | #------------------------------------------------------------------------------- 156 | # The station(s) with single pol data 157 | singlepol = ['JC'] 158 | 159 | # Number of iterative imaging inside a single selfcal cycle 160 | Nweig = 3 161 | 162 | # Number of closure imaging attempted in this script (must be < Nself) 163 | Nflcl = 2 164 | 165 | # Number of selfcals 166 | Nself = 4 167 | 168 | # Imaging pixel size 169 | dx_uas = 2 170 | 171 | # Number of pixels 172 | nx = 64 173 | 174 | 175 | #------------------------------------------------------------------------------- 176 | # Set default imaging parameters 177 | #------------------------------------------------------------------------------- 178 | # Definition of the prior image 179 | def gen_prior_gauss(x0=0.,y0=0.,fwhm=40.,dx=dx_uas,nx=nx): 180 | ''' 181 | This function provides a Gaussain image with a specified FWHM in uas. 182 | ''' 183 | # create a blank image 184 | outimage = imdata.IMFITS(dx=dx,nx=nx,angunit="uas") 185 | outimage = outimage.add_gauss(majsize=fwhm, overwrite=True) 186 | return outimage 187 | 188 | # Set a circular Gaussian prior for L1 189 | l1_prior = gen_prior_gauss(fwhm=prior_fwhm,dx=dx_uas,nx=nx) 190 | 191 | # Default imaging parameters 192 | imprm_init={} 193 | 194 | # Iteration numbers 195 | imprm_init["niter"] = 1000 196 | 197 | # Regularization parameters 198 | # Flat prior TSV 199 | if lambtsv > 0: 200 | imprm_init["tsv_lambda"] = lambtsv 201 | else: 202 | imprm_init["tsv_lambda"] = -1 203 | 204 | # Flat prior TV 205 | if lambtv > 0: 206 | imprm_init["tv_lambda"] = lambtv 207 | else: 208 | imprm_init["tv_lambda"] = -1 209 | 210 | # Weighted L1 211 | if lambl1 > 0: 212 | imprm_init["l1_lambda"] = lambl1 213 | imprm_init["l1_prior"] = l1_prior 214 | else: 215 | imprm_init["l1_lambda"] = -1 216 | 217 | # Maximum Entropy Methods: not used in SMILI pipeline 218 | # GS-MEM 219 | imprm_init["gs_lambda"] = -1 220 | # KL-MEM 221 | imprm_init["kl_lambda"] = -1 222 | 223 | # Total flux density constraint 224 | imprm_init["totalflux"] = totalflux # Target total flux density 225 | imprm_init["tfd_lambda"] = 1 # Lambda parameter 226 | imprm_init["tfd_tgterror"] = 1e-2 # Target fractional accuracy 227 | 228 | 229 | #------------------------------------------------------------------------------- 230 | # Step 1: Pre-calibration 231 | # To make sure that we use visibility data sets pre-calibrated consistently with 232 | # two other pipelines, this pipeline also uses eht-imaging library. 233 | #------------------------------------------------------------------------------- 234 | # Load the uvfits file 235 | obs = eh.obsdata.load_uvfits(inputuvfits).switch_polrep('stokes') 236 | 237 | # Rescale short baselines to excite contributions from extended flux. 238 | # Total flux density adopted in the Network Calibration 239 | # see EHT Collaboration et al. 2019c, M87 Paper III 240 | if day < 7: 241 | netcal_tfd = 1.1 # 1.1 Jy was adopted for Apr 5 and 6 242 | else: 243 | netcal_tfd = 1.2 # 1.2 Jy was adopted for Apr 10 and 11 244 | # Scaling the total flux density: 245 | # setting totalflux < netcal_tfd assumes there is an extended constant flux 246 | # component of netcal_tfd-totalflux Jy 247 | for j in range(len(obs.data)): 248 | if (obs.data['u'][j]**2 + obs.data['v'][j]**2)**0.5 < 0.1e9: 249 | for k in range(-8,0): 250 | obs.data[j][k] *= totalflux/netcal_tfd 251 | 252 | # Do scan averaging 253 | obs.add_scans() # this seperates the data into scans, if it isn't done so already with an NX table 254 | obs = obs.avg_coherent(0.,scan_avg=True) # average each scan coherantly 255 | 256 | # Order stations. this is to create a minimal set of closure quantities with the highest snr 257 | obs.reorder_tarr_snr() 258 | 259 | # Self-calibrate the LMT to a Gaussian model if LMT is included in the input uvfits file 260 | if "LM" in obs.data["t1"] or "LM" in obs.data["t2"]: 261 | obs_selfcal = obs.copy() 262 | # Add systematic noise for leakage and other non-closing errors 263 | # (reminder: this must be done *after* any averaging) 264 | obs_selfcal = obs_selfcal.add_fractional_noise(syserr) 265 | obs_selfcal = obs_selfcal.switch_polrep('stokes') 266 | # Set a circular Gaussian with a FWHM of 60 uas as the model image 267 | gausspriormodel = eh.image.make_square(obs, int(nx), nx*dx_uas*eh.RADPERUAS) # Create empty image 268 | gausspriormodel = gausspriormodel.add_gauss(0.6, (60*eh.RADPERUAS, 60*eh.RADPERUAS, 0, 0, 0)) # Add gaussian 269 | # Self-calibrate amplitudes 270 | for repeat in range(3): 271 | caltab = eh.self_cal.self_cal( 272 | obs_selfcal.flag_uvdist(uv_max=2e9), 273 | gausspriormodel, 274 | sites=['LM','LM'], 275 | method='vis', 276 | ttype='nfft', 277 | processes=nproc, 278 | caltable=True, 279 | gain_tol=1.0) 280 | obs_selfcal = caltab.applycal(obs_selfcal, interp='nearest', extrapolate=True) 281 | # Save calibrated uvfits data sets 282 | obs_selfcal.save_uvfits(outputhead+".precal.uvfits") 283 | del obs, obs_selfcal, caltab, gausspriormodel, netcal_tfd 284 | else: 285 | obs.save_uvfits(outputhead+".precal.uvfits") 286 | del obs 287 | 288 | 289 | #--------------------------------------------------------------------------- 290 | # Step 2: Generating Data Tables for Imaging 291 | #--------------------------------------------------------------------------- 292 | # Create the full complex visibility table 293 | vtable = uvdata.UVFITS(outputhead+".precal.uvfits").select_stokes("I").make_vistable() 294 | if not args.keepsinglepol: 295 | for stname in singlepol: 296 | vtable = vtable.query("st1name != '"+stname+"' and st2name != '"+stname+"'").reset_index(drop=True) 297 | 298 | # Check the number of data sets 299 | if len(vtable.amp) == 0: 300 | raise ValueError("No data points exist in the input visibility data set.") 301 | 302 | # Create tables for non-trivial/non-redundant bi-spectrum and closure amplitudes 303 | btable = vtable.make_bstable(redundant=[["AA","AP"],["JC","SM"]]) 304 | ctable = vtable.make_catable(redundant=[["AA","AP"],["JC","SM"]]) 305 | 306 | # Check the number of data sets 307 | if len(btable.amp) == 0: 308 | btable = None 309 | if len(ctable.amp) == 0: 310 | ctable = None 311 | 312 | #--------------------------------------------------------------------------- 313 | # Step 3: Imaging 314 | #--------------------------------------------------------------------------- 315 | def edit_image(image,imprm): 316 | ''' 317 | Image editing (shifting, blurring) 318 | ''' 319 | image_edited = copy.deepcopy(image) 320 | orgtotalflux = image_edited.totalflux() 321 | 322 | # Shifting images such that the ring or dominant structure comes to the center 323 | if "vistable" not in imprm.keys(): 324 | image_edited = image_edited.comshift() 325 | 326 | # Blurring with 20 uas Gaussian beam 327 | image_edited = image_edited.convolve_gauss(majsize=20,angunit="uas") 328 | 329 | # Scaling images 330 | if "totalflux" in imprm.keys(): 331 | image_edited.data *= imprm["totalflux"]/image_edited.totalflux() 332 | else: 333 | image_edited.data *= orgtotalflux/image_edited.totalflux() 334 | 335 | image_edited.update_fits() 336 | return image_edited 337 | 338 | # Loop for self-calibration 339 | for iself in xrange(Nself): 340 | # Set the initial image 341 | # At each iteration of self-calibrations, imaging starts from a circular 342 | # Gaussian. 343 | initimage = imdata.IMFITS( 344 | dx=dx_uas, 345 | nx=nx, 346 | angunit="uas", 347 | uvfits=outputhead+".precal.uvfits", 348 | ) 349 | initimage = initimage.add_gauss(totalflux=totalflux,majsize=prior_fwhm,overwrite=True) 350 | 351 | # Iterative Imaging: 352 | # At each iteration of self-calibration, imaging iterations are attempted 353 | # for Nweig(=5) times. 354 | for iweig in xrange(Nweig): 355 | # Imaging parameters: copied from the default parameter sets 356 | imprm = copy.deepcopy(imprm_init) 357 | 358 | # Data to be used in default 359 | # Closure phases/amplitudes are used always to avoid over-fitting to 360 | # full complex visibilities 361 | imprm["bstable"]=btable 362 | imprm["catable"]=ctable 363 | 364 | # Use amplitudes or full complex visibilities: 365 | # Parameters depending on self-cal stages 366 | if iself < Nflcl: 367 | # The first half for self-calibration: using closure quantities 368 | 369 | # Get Amplitude Data sets 370 | atable = copy.deepcopy(vtable) 371 | 372 | # Flag the intra-site to avoid conflict with with the total-flux-density regularization. 373 | atable = atable.query("uvdist > 50e6") 374 | 375 | # Adding a fractional error to the data 376 | # For LMT (adding 30% error) 377 | idx = atable["st1name"] == "LM" 378 | idx|= atable["st2name"] == "LM" 379 | atable.loc[idx,"sigma"] = np.sqrt(atable.loc[idx,"sigma"]**2+(atable.loc[idx,"amp"]*0.3)**2) 380 | # For other stations (adding an error specified with amp_err) 381 | idx = idx == False 382 | atable.loc[idx,"sigma"] = np.sqrt(atable.loc[idx,"sigma"]**2+(atable.loc[idx,"amp"]*0.05)**2) 383 | 384 | # Use amplitudes 385 | imprm["amptable"]=atable 386 | else: 387 | # The latter half for self-calibration: using full complex 388 | # visibilities. Note that we still add closure quantities 389 | # to prevent over-fitting to full complex visibilities 390 | vtable_in = vtable.query("uvdist > 50e6") 391 | imprm["vistable"] = vtable_in 392 | 393 | # Imaging 394 | outimage = imaging.lbfgs.imaging(initimage,**imprm) 395 | initimage = edit_image(outimage,imprm) 396 | 397 | # Save image 398 | outimage.to_fits(outputfits) 399 | 400 | # If iself==Nself, self-calibration won't be performed and exit the script. 401 | if iself == Nself-1: 402 | break 403 | 404 | # Self-calibration 405 | # 1. Set the file name 406 | if iself == 0: 407 | uvfitsold = outputhead+".precal.uvfits" 408 | else: 409 | uvfitsold = outputhead+".selfcal.uvfits" 410 | uvfitsnew = outputhead+".selfcal.uvfits" 411 | 412 | # 2. Load uvfits 413 | uvfits = uvdata.UVFITS(uvfitsold) 414 | # 415 | # 3. Edit images 416 | # Before selfcal, we bring the center of the mass to the image center 417 | # and also normalize the image with the target total flux density. 418 | outimage = outimage.comshift() 419 | # Normalize the total flux density 420 | # *** Thanks to the total-flux-density regularization, this will provide only a 421 | # *** slight change in the total flux density by a few pecent or even < 1% 422 | outimage.data[0,0] *= totalflux/outimage.totalflux() 423 | 424 | # 4. Do selfcal 425 | # This is a regularized selfcal function using Gaussian priors 426 | # for gain amplitudes and phases. std_amp is the standard deviation of 427 | # the Gaussian prior for amplitude gains. std_amp=10000 works as almost 428 | # the flat prior, so it is essentially selfcal without any gain constraints. 429 | caltable = uvfits.selfcal(outimage,std_amp=10000) 430 | uvfits = uvfits.apply_cltable(caltable) 431 | 432 | # 5. Save the self-calibrated uvfits file 433 | uvfits.to_uvfits(uvfitsnew) 434 | 435 | # 6. Get a new visibility table 436 | vtable = uvfits.select_stokes("I").make_vistable() 437 | if not args.keepsinglepol: 438 | for stname in singlepol: 439 | vtable = vtable.query("st1name != '"+stname+"' and st2name != '"+stname+"'").reset_index(drop=True) 440 | --------------------------------------------------------------------------------