├── .gitignore
├── listmoduleparams.py
├── loofilters.py
├── varyz.py
├── postprocess.py
├── template_pcigale.ini
├── pcigale.ini
├── README.rst
├── generate_config.py
├── simplesampler.py
└── dualsampler.py


/.gitignore:
--------------------------------------------------------------------------------
1 | modelspectrum*.png
2 | *.cache
3 | prof
4 | 


--------------------------------------------------------------------------------
/listmoduleparams.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python3
 2 | """
 3 | This script gives an example config for each known and recommended module.
 4 | """
 5 | import importlib
 6 | 
 7 | modules = 'sfhdelayed bc03 m2005 sfhdelayed nebular dustatt_calzleit dustatt_powerlaw galdale2014 activate activatelines activategtorus activatepl activatebol biattenuation galdale2014 redshifting'.split()
 8 | 
 9 | for module_name in modules:
10 |      module = importlib.import_module('pcigale.creation_modules.' + module_name)
11 |      klass = module.Module
12 |      print(f"[{module_name}]")
13 |      for param, (dtype, comment, default_value) in klass.parameter_list.items():
14 |         print(f"    # {comment} (type: {dtype})")
15 |         print(f"    {param} = {default_value}")
16 |      print()
17 |      print(f"  # the code of this module is in: {module.__file__}")
18 |      print()
19 |      
20 | 
21 | 


--------------------------------------------------------------------------------
/loofilters.py:
--------------------------------------------------------------------------------
 1 | """
 2 | Create a modified photometry input file, with new rows that leave out one filter at a time.
 3 | 
 4 | The new ids are <origid>-no-<filtername> where <origid> is
 5 | the original id, and filtername is the name of the filter left out.
 6 | 
 7 | Synopsis: python3 loofilter.py input.fits input-new.fits
 8 | """
 9 | 
10 | from astropy.table import Table, vstack
11 | import sys
12 | 
13 | infile = sys.argv[1]
14 | outfile = sys.argv[2]
15 | 
16 | f = Table.read(infile)
17 | flux_columns = [col for col in f.colnames if col + '_err' in f.colnames]
18 | 
19 | tables = [f]
20 | for col in flux_columns:
21 | 	# add a identical row, with original_row[col] set to a negative number (-9999)
22 | 	modf = f.copy()
23 | 	modf[col] = -9999
24 | 	modf['id'] = [str(i) + '-no-' + str(col) for i in modf['id']]
25 | 	tables.append(modf)
26 | 	print(modf)
27 | 
28 | vstack(tables).write(outfile, overwrite=True)
29 | 


--------------------------------------------------------------------------------
/varyz.py:
--------------------------------------------------------------------------------
 1 | """
 2 | Create a modified photometry input file, with new rows with slightly altered redshifts.
 3 | 
 4 | The new rows are for example <origid>-deltaz0.3 where <origid> is
 5 | the original id with the redshift changed by +0.3.
 6 | 
 7 | Synopsis: python3 varyz.py input.fits input-new.fits
 8 | """
 9 | 
10 | import numpy as np
11 | from astropy.table import Table, vstack
12 | import sys
13 | 
14 | infile = sys.argv[1]
15 | outfile = sys.argv[2]
16 | 
17 | f = Table.read(infile)
18 | flux_columns = [col for col in f.colnames if col + '_err' in f.colnames]
19 | 
20 | tables = [f]
21 | for deltaz in np.arange(-0.7, +0.8, 0.1):
22 | 	if deltaz == 0: continue
23 | 	# add a identical row, with redshift changed
24 | 	modf = f.copy()
25 | 	modf['id'] = [str(i) + '-deltaz%.1f' % deltaz for i in modf['id']]
26 | 	modf['redshift'] = f['redshift'] + deltaz
27 | 	tables.append(modf[modf['redshift'] > 0])
28 | 
29 | vstack(tables).write(outfile, overwrite=True)
30 | 


--------------------------------------------------------------------------------
/postprocess.py:
--------------------------------------------------------------------------------
  1 | """
  2 | 
  3 | """
  4 | 
  5 | import sys
  6 | import argparse
  7 | import itertools
  8 | import numpy as np
  9 | #from numpy import log, log10
 10 | 
 11 | import matplotlib.pyplot as plt
 12 | import matplotlib as mpl
 13 | 
 14 | import pcigale
 15 | from pcigale.session.configuration import Configuration
 16 | from pcigale.utils import read_table
 17 | from pcigale.data import Database
 18 | import astropy.cosmology
 19 | #import astropy.units as units
 20 | #from astropy.table import Table
 21 | import pcigale.creation_modules.redshifting
 22 | 
 23 | # parse command line arguments
 24 | 
 25 | 
 26 | class HelpfulParser(argparse.ArgumentParser):
 27 |     def error(self, message):
 28 |         sys.stderr.write('error: %s\n' % message)
 29 |         self.print_help()
 30 |         sys.exit(2)
 31 | 
 32 | 
 33 | parser = HelpfulParser(
 34 |     description=__doc__,
 35 |     epilog="""Johannes Buchner (C) 2013-2023 <johannes.buchner.acad@gmx.com>""",
 36 |     formatter_class=argparse.RawDescriptionHelpFormatter
 37 | )
 38 | 
 39 | 
 40 | args = parser.parse_args()
 41 | 
 42 | # keeping it called pcigale.ini allows running pcigale with the same file
 43 | # if the user wants to run cigale
 44 | print("GRAHSP version %s | parsing pcigale.ini..." % pcigale.__version__)
 45 | config = Configuration("pcigale.ini")
 46 | 
 47 | data_file = config.configuration['data_file']
 48 | column_list = config.configuration['column_list']
 49 | 
 50 | inputs = read_table(data_file)
 51 | for c in inputs.colnames:
 52 |     inputs.rename_column(c, c + '_in')
 53 | print("%d inputs" % len(inputs))
 54 | print("reading output ...")
 55 | outputs = astropy.io.ascii.read(data_file + '_analysis_results.txt', format='commented_header', delimiter="\t")
 56 | if len(outputs) != len(inputs):
 57 |     print("WARNING: %d input rows, but %d output rows" % (len(inputs), len(outputs)))
 58 | 
 59 | ids_are_strings = outputs['id'].dtype.kind in 'US' or inputs['id_in'].dtype.kind in 'US'
 60 | if ids_are_strings:
 61 |     print("stripping id strings")
 62 |     inputs['id_in'] = [i.rstrip() for i in inputs['id_in']]
 63 |     outputs['id'] = [i.rstrip() for i in outputs['id']]
 64 | 
 65 | print("joining ...")
 66 | # join by id, write as fits file
 67 | full = astropy.table.join(
 68 |     inputs, outputs, keys_left='id_in', keys_right='id',
 69 |     table_names=['_in', ''], uniq_col_name='{col_name}{table_name}')
 70 | del full['id_in']
 71 | print("writing %s" % (data_file + '_analysis_results.fits'), "%d outputs" % len(full))
 72 | full.write(data_file + '_analysis_results.fits', overwrite=True)
 73 | 
 74 | print("diagnosing residuals...")
 75 | # get list of user-selected filters
 76 | filters = [name for name in column_list if not name.endswith('_err')]
 77 | 
 78 | plt.figure(figsize=(12, 3))
 79 | default_color_cycle = itertools.cycle(plt.rcParams['axes.prop_cycle'].by_key()['color'])
 80 | depths = []
 81 | 
 82 | with Database() as base:
 83 |     for color, filtername in zip(default_color_cycle, filters):
 84 |         f = base.get_filter(filtername.rstrip('_'))
 85 |         wl_eff = f.effective_wavelength / 1000
 86 |         
 87 |         flux_obs = full[filtername + '_in']
 88 |         flux_obs_err = full[filtername + '_err_in']
 89 |         if np.any(flux_obs_err > 0):
 90 |             depths.append((filtername, color, wl_eff, np.percentile(np.array(flux_obs_err[flux_obs_err>0]), 50)))
 91 |         flux_model = full['totalflux_%s_mean' % filtername]
 92 |         with np.errstate(invalid='ignore', divide='ignore'):
 93 |             delta = -2.5 * np.log10(flux_obs) - -2.5 * np.log10(flux_model + 1e-10)
 94 |         mask_good = np.isfinite(delta)
 95 |         if not mask_good.any():
 96 |             continue
 97 |         plt.text(wl_eff, 2, filtername, rotation=90, va='top', ha='center', color=color)
 98 |         props = dict(color=color)
 99 |         bplot = plt.boxplot(
100 |             [delta[mask_good]],
101 |             positions=[wl_eff], widths=0.1 * wl_eff,
102 |             meanline=True, showmeans=True, showcaps=True, showbox=True,
103 |             notch=True, sym='x', vert=True, whis=1.5, 
104 |             showfliers=True, tick_labels=[filtername], #patch_artist=True,
105 |             capprops=props, boxprops=props, whiskerprops=props,
106 |             flierprops=dict(ms=2, mew=1, mec=color, **props),
107 |             medianprops=dict(ls='-', **props), meanprops=dict(ls=':', **props))
108 |             #[patch.set_facecolor(color) for patch in bplot['boxes']]
109 | 
110 | #plt.yscale('log')
111 | plt.xscale('log')
112 | xlo, xhi = plt.xlim()
113 | plt.xlim(xlo, xhi)
114 | plt.hlines(0, xlo, xhi, colors=['k'], lw=1)
115 | plt.ylim(-2, 2)
116 | plt.ylabel(r'Observed - Model magnitude')
117 | plt.xlabel(r'Observed Wavelength [$\mu{}m$]')
118 | 
119 | def to_mag(x):
120 |     with np.errstate(invalid='ignore', divide='ignore'):
121 |         return -2.5 * np.log10(x)
122 | def from_mag(x):
123 |     return 10**(x/-2.5)
124 | 
125 | secax = plt.gca().secondary_yaxis('right', functions=(from_mag, to_mag))
126 | secax.set_ylabel(r'Observed / Model flux')
127 | secax.set_yticks([0.2, 0.5, 0.8, 1, 1.2, 2, 5])
128 | 
129 | plt.gca().get_xaxis().set_major_formatter(mpl.ticker.ScalarFormatter())
130 | plt.tight_layout()
131 | plt.savefig(data_file + '_filterresiduals.pdf')
132 | plt.close()
133 | 
134 | plt.figure(figsize=(5, 3))
135 | for filtername, color, wl_eff, flux_lo in depths:
136 |     plt.plot(wl_eff, -2.5 * np.log10(flux_lo / 3631000), 'o ', mfc='none', c='k')
137 | 
138 | plt.xscale('log')
139 | plt.gca().invert_yaxis()
140 | plt.xlabel(r'Observed Wavelength [$\mu{}m$]')
141 | plt.ylabel('Depth (AB mag; 50%)')
142 | plt.gca().get_xaxis().set_major_formatter(mpl.ticker.ScalarFormatter())
143 | plt.tight_layout()
144 | plt.savefig(data_file + '_filterdepth.pdf')
145 | plt.close()
146 | 


--------------------------------------------------------------------------------
/template_pcigale.ini:
--------------------------------------------------------------------------------
  1 | # File containing the input data. The columns are 'id' (name of the
  2 | # object), 'redshift' (if 0 the distance is assumed to be 10 pc), the
  3 | # filter names for the fluxes, and the filter names with the '_err'
  4 | # suffix for the uncertainties. The fluxes and the uncertainties must be
  5 | # in mJy. This file is optional to generate the configuration file, in
  6 | # particular for the savefluxes module.
  7 | data_file = 
  8 | 
  9 | # Order of the modules use for SED creation.
 10 | # Choose between bc03 and m2005 for the stellar population
 11 | creation_modules = 
 12 | 
 13 | # Method used for statistical analysis. Available methods: pdf_analysis,
 14 | # savefluxes.
 15 | analysis_method = 
 16 | 
 17 | # This argument is not used in GRAHSP, to parallelise, please indicate in the command
 18 | # line "--cores X"
 19 | cores = -2
 20 | 
 21 | # List of the columns in the observation data file to use for the fitting.
 22 | column_list = 
 23 | 
 24 | # Configuration of the SED creation modules.
 25 | [sed_creation_modules]
 26 | 
 27 |   [[sfhdelayed]]
 28 |      # e-folding time of the main stellar population model in Myr.
 29 |      tau_main = 100, 200, 500, 1000, 3000, 5000, 7000, 10000
 30 |      # Age of the oldest stars in the galaxy in Myr. The precision is 1 Myr.
 31 |      age_main = "eval np.logspace(2.2, 4.0, 18).astype(int)"
 32 |      # Multiplicative factor controlling the amplitude of SFR.
 33 |      sfr_A = 1.0
 34 |      # Normalise the SFH to produce one solar mass.
 35 |      normalise = True
 36 |  
 37 |    [[bc03]]
 38 |      # Initial mass function: 0 (Salpeter) or 1 (Chabrier).
 39 |      imf = 0
 40 |      # Metalicity. Possible values are: 0.0001, 0.0004, 0.004, 0.008, 0.02,
 41 |      metallicity = 0.02
 42 |      # Age [Myr] of the separation between the young and the old star
 43 |      # populations. The default value in 10^7 years (10 Myr). Set to 0 not to
 44 |      # differentiate ages (only an old population).
 45 |      separation_age = 10
 46 |  
 47 |    [[m2005]]
 48 |      # Initial mass function: 0 (Salpeter) or 1 (Kroupa)
 49 |      imf = 0
 50 |      # Metallicity. Possible values are: 0.001, 0.01, 0.02, 0.04.
 51 |      metallicity = 0.02
 52 |      # Age [Myr] of the separation between the young and the old star
 53 |      # populations. The default value in 10^7 years (10 Myr). Set to 0 not to
 54 |      # differentiate ages (only an old population).
 55 |      separation_age = 10
 56 |   
 57 |    [[nebular]]
 58 |      # Ionisation parameter
 59 |      logU = -2.0
 60 |      # Fraction of Lyman continuum photons escaping the galaxy
 61 |      f_esc = 0.0
 62 |      # Fraction of Lyman continuum photons absorbed by dust
 63 |      f_dust = 0.0
 64 |      # Line width in km/s
 65 |      lines_width = 300.0
 66 | 
 67 |   [[biattenuation]]
 68 |     # Galaxy template to use
 69 |     # E(B-V) = 0, 0.001, 0.002, 0.003, 0.004, 0.005, 0.007, 0.01, 0.013, 0.016, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.1, 0.14, 0.2, 0.3, 0.4
 70 |     # E(B-V)-AGN = 0, 0.1, 0.3, 0.5, 1.0
 71 |     E(B-V) = "eval np.logspace(-2, 1, 80)"
 72 |     E(B-V)-AGN = "eval np.logspace(-2, 1, 80)"
 73 | 
 74 |   [[galdale2014]]
 75 |     # Alpha slope. Possible values are: 0.0625, 0.1250, 0.1875, 0.2500,
 76 |     # 0.3125, 0.3750, 0.4375, 0.5000, 0.5625, 0.6250, 0.6875, 0.7500,
 77 |     # 0.8125, 0.8750, 0.9375, 1.0000, 1.0625, 1.1250, 1.1875, 1.2500,
 78 |     # 1.3125, 1.3750, 1.4375, 1.5000, 1.5625, 1.6250, 1.6875, 1.7500,
 79 |     # 1.8125, 1.8750, 1.9375, 2.0000, 2.0625, 2.1250, 2.1875, 2.2500,
 80 |     # 2.3125, 2.3750, 2.4375, 2.5000, 2.5625, 2.6250, 2.6875, 2.7500,
 81 |     # 2.8125, 2.8750, 2.9375, 3.0000, 3.0625, 3.1250, 3.1875, 3.2500,
 82 |     # 3.3125, 3.3750, 3.4375, 3.5000, 3.5625, 3.6250, 3.6875, 3.7500,
 83 |     # 3.8125, 3.8750, 3.9375, 4.0000
 84 |     alpha = 1.5, 2.0, 2.5
 85 |     # maximum wavelength up to which to compute. here: 100 um.
 86 |     lam_max = 100000
 87 |   
 88 |   [[activate]]
 89 |     # fracAGN = 0.0001, 0.001, 0.003, 0.01, 0.03, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.97, 0.99, 0.997, 0.999, 0.9999
 90 |     fracAGN = -1
 91 | 
 92 |   [[activatepl]]
 93 |     plslope = "eval np.arange(-2.7, -1., 0.01)"
 94 |     plbendloc = 50, 80, 90, 100, 120, 150
 95 |     plbendwidth = "eval np.logspace(-1, 1, 10)"
 96 |     uvslope = 0
 97 | 
 98 |   [[activatelines]]
 99 |     # AGN lines
100 |     AFeII = "eval np.logspace(-0.2, 1.5, 10)"
101 |     Alines = 0.3, 0.5, 0.7, 1, 1.5, 2, 4, 10, 20
102 |     linewidth = 10000
103 | 
104 |   [[activatetorus]]
105 |     Si = -2, -1, 0, 1, 2
106 |     fcov = 0.1, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.9
107 |     TORtemp = -3, -1.5, -0.5, 0, 0.5, 1, 1.5, 2, 3
108 |     TORcutoff = 1.0, 1.2, 1.5, 1.7, 1.9, 2.1
109 | 
110 |   [[activategtorus]]
111 |     Si = "eval np.arange(-4, 4.1, 0.2)"
112 |     fcov = "eval np.arange(0.05, 1, 0.05)"
113 |     # Gaussian prior: 0.5 +- 0.2:
114 |     # fcov = 0.0355, 0.0901, 0.1247, 0.1508, 0.172 , 0.1901, 0.2059, 0.2201,        0.233 , 0.2448, 0.2559, 0.2662, 0.276 , 0.2852, 0.294 , 0.3024,        0.3105, 0.3183, 0.3258, 0.3331, 0.3402, 0.3471, 0.3538, 0.3603,        0.3667, 0.3729, 0.3791, 0.3851, 0.391 , 0.3969, 0.4026, 0.4083,         0.4139, 0.4194, 0.4248, 0.4302, 0.4356, 0.4409, 0.4462, 0.4514,        0.4566, 0.4618, 0.4669, 0.4721, 0.4772, 0.4823, 0.4873, 0.4924,        0.4975, 0.5025, 0.5076, 0.5127, 0.5177, 0.5228, 0.5279, 0.5331,        0.5382, 0.5434, 0.5486, 0.5538, 0.5591, 0.5644, 0.5698, 0.5752,        0.5806, 0.5861, 0.5917, 0.5974, 0.6031, 0.609 , 0.6149, 0.6209,         0.6271, 0.6333, 0.6397, 0.6462, 0.6529, 0.6598, 0.6669, 0.6742,        0.6817, 0.6895, 0.6976, 0.706 , 0.7148, 0.724 , 0.7338, 0.7441,        0.7552, 0.767 , 0.7799, 0.7941, 0.8099, 0.828 , 0.8492, 0.8753,        0.9099, 0.9645]
115 |     
116 |     COOLlam = "eval np.arange(10, 30, 0.01)"
117 |     COOLwidth = "eval np.arange(0.2, 0.7, 0.05)"
118 |     HOTlam = "eval np.arange(1, 5.51, 0.01)"
119 |     HOTwidth = "eval np.arange(0.2, 0.7, 0.05)"
120 |     HOTfcov = 0.04, 0.1, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 2.0, 2.5, 3, 5, 10
121 | 
122 |   [[activatebol]]
123 | 
124 | 
125 |   [[redshifting]]
126 |     # Redshift to apply to the galaxy. Leave empty to use the redshifts from
127 |     # the input file.
128 |     redshift = 
129 | 
130 | # Configuration of the statistical analysis method.
131 | #[analysis_configuration]
132 | #  # Name of the output file.
133 | #  output_file = computed_fluxes.txt
134 | #  # If True, save the generated SED.
135 | #  save_sed = True
136 | #  # Format of the output file. Any format supported by astropy.table e.g.
137 | #  # votable or ascii.
138 | #  output_format = votable
139 | 
140 | [analysis_configuration]
141 |   # List of the variables (in the SEDs info dictionaries) for which the
142 |   # statistical analysis will be done.
143 |   analysed_variables = sfh.sfr, stellar.lum, agn.lum6um, agn.lumBolBBB, agn.lumBolTOR
144 |   # list of filters for which the flux will be computed from the best SED mmodels
145 |   additional_bands =  
146 |   # If true, save the best SED for each observation to a file.
147 |   save_best_sed = True
148 |   # If true, for each observation and each analysed variable save the
149 |   # reduced chi2.
150 |   save_chi2 = False
151 |   # If true, for each observation and each analysed variable save the
152 |   # probability density function.
153 |   save_pdf = True
154 |   # If true, for each object check whether upper limits are present and
155 |   # analyse them.
156 |   lim_flag = True
157 |   # If true, for each object we create a mock object and analyse them.
158 |   mock_flag = False
159 | 
160 | 
161 | [statistics]
162 |   exponent = 2
163 |   # calibration suggests the AGN model is good to 20% for the vast majority
164 |   # a exponential distribution of scale 0.2 has mean 0.2
165 |   systematics_width = 0.2
166 |   attenuation_model_uncertainty = False
167 |   variability_uncertainty = False
168 | 
169 | 
170 | [scaling_limits]
171 |   # Minimum log stellar mass.
172 |   mass_min = 5
173 |   # Maximum log stellar mass.
174 |   mass_max = 15
175 |   # Minimum SFR, averaged over the last 100Myrs, in Msun/yr.
176 |   sfr_min = 0
177 |   # Maximum SFR, averaged over the last 100Myrs, in Msun/yr.
178 |   sfr_max = 100000
179 |   # Minimum log AGN Luminosity.
180 |   L_min = 38
181 |   # Maximum log AGN Luminosity.
182 |   L_max = 50
183 | 


--------------------------------------------------------------------------------
/pcigale.ini:
--------------------------------------------------------------------------------
  1 | # File containing the input data. The columns are 'id' (name of the
  2 | # object), 'redshift' (if 0 the distance is assumed to be 10 pc), the
  3 | # filter names for the fluxes, and the filter names with the '_err'
  4 | # suffix for the uncertainties. The fluxes and the uncertainties must be
  5 | # in mJy. This file is optional to generate the configuration file, in
  6 | # particular for the savefluxes module.
  7 | data_file = input.fits
  8 | 
  9 | # Order of the modules use for SED creation. Available modules: SFH:
 10 | # sfh2exp, sfhdelayed, sfhfromfile ; SSP: bc03, m2005 ; Nebular: nebular
 11 | # ; Attenuation: dustatt_calzleit, dustatt_powerlaw ; Dust model:
 12 | # casey2012, dale2014, dl2007, dl2014 ; AGN: dale2014, fritz2006 ;
 13 | # Radio: radio ; redshift: redshifting (mandatory!).
 14 | # dustatt_calzleit
 15 | creation_modules = sfhdelayed, m2005, nebular, activate, activatelines, activategtorus, activatepl, activatebol, biattenuation, galdale2014, redshifting
 16 | 
 17 | # Method used for statistical analysis. Available methods: pdf_analysis,
 18 | # savefluxes.
 19 | analysis_method = pdf_analysis
 20 | 
 21 | # Number of CPU cores available. NOT USED but crashes without this argument
 22 | cores = -2
 23 | 
 24 | # List of the columns in the observation data file to use for the
 25 | # fitting.
 26 | #column_list = u_sdss, u_sdss_err, r_sdss, r_sdss_err, i_sdss, i_sdss_err, z_sdss, z_sdss_err, J_2mass, J_2mass_err, H_2mass, H_2mass_err, Ks_2mass, Ks_2mass_err, IRAC1, IRAC1_err, IRAC2, IRAC2_err
 27 | #column_list = u_sdss, u_sdss_err, r_sdss, r_sdss_err, i_sdss, i_sdss_err, z_sdss, z_sdss_err, J_2mass, J_2mass_err, H_2mass, H_2mass_err, Ks_2mass, Ks_2mass_err, WISE1, WISE1_err, WISE2, WISE2_err
 28 | #column_list = r_sdss, r_sdss_err, i_sdss, i_sdss_err, z_sdss, z_sdss_err, J_2mass, J_2mass_err, H_2mass, H_2mass_err, Ks_2mass, Ks_2mass_err, WISE1, WISE1_err, WISE2, WISE2_err
 29 | column_list = u_sdss, u_sdss_err, r_sdss, r_sdss_err, i_sdss, i_sdss_err, z_sdss, z_sdss_err, Ks_2mass, Ks_2mass_err, WISE1, WISE1_err, WISE2, WISE2_err
 30 | 
 31 | #cosmology = concordance
 32 | cosmology = Planck18
 33 | #cosmology = WMAP7
 34 | 
 35 | # Configuration of the SED creation modules.
 36 | [sed_creation_modules]
 37 | 
 38 |   [[sfhdelayed]]
 39 |      # e-folding time of the main stellar population model in Myr.
 40 |      tau_main = 100, 200, 500, 1000, 3000, 5000, 7000, 10000
 41 |      # Age of the oldest stars in the galaxy in Myr. The precision is 1 Myr.
 42 |      age_main = "eval np.logspace(2.2, 4.0, 18).astype(int)"
 43 |      # Multiplicative factor controlling the amplitude of SFR.
 44 |      sfr_A = 1.0
 45 |      # Normalise the SFH to produce one solar mass.
 46 |      normalise = True
 47 |  
 48 |    [[bc03]]
 49 |      # Initial mass function: 0 (Salpeter) or 1 (Chabrier).
 50 |      imf = 0
 51 |      # Metalicity. Possible values are: 0.0001, 0.0004, 0.004, 0.008, 0.02,
 52 |      metallicity = 0.02
 53 |      # Age [Myr] of the separation between the young and the old star
 54 |      # populations. The default value in 10^7 years (10 Myr). Set to 0 not to
 55 |      # differentiate ages (only an old population).
 56 |      separation_age = 10
 57 |  
 58 |    [[m2005]]
 59 |      # Initial mass function: 0 (Salpeter) or 1 (Kroupa)
 60 |      imf = 0
 61 |      # Metallicity. Possible values are: 0.001, 0.01, 0.02, 0.04.
 62 |      metallicity = 0.02
 63 |      # Age [Myr] of the separation between the young and the old star
 64 |      # populations. The default value in 10^7 years (10 Myr). Set to 0 not to
 65 |      # differentiate ages (only an old population).
 66 |      separation_age = 10
 67 |   
 68 |    [[nebular]]
 69 |      # Ionisation parameter
 70 |      logU = -2.0
 71 |      # Fraction of Lyman continuum photons escaping the galaxy
 72 |      f_esc = 0.0
 73 |      # Fraction of Lyman continuum photons absorbed by dust
 74 |      f_dust = 0.0
 75 |      # Line width in km/s
 76 |      lines_width = 300.0
 77 | 
 78 |   [[biattenuation]]
 79 |     # Galaxy template to use
 80 |     # E(B-V) = 0, 0.001, 0.002, 0.003, 0.004, 0.005, 0.007, 0.01, 0.013, 0.016, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.1, 0.14, 0.2, 0.3, 0.4
 81 |     # E(B-V)-AGN = 0, 0.1, 0.3, 0.5, 1.0
 82 |     E(B-V) = "eval np.logspace(-2, 1, 80)"
 83 |     E(B-V)-AGN = "eval np.logspace(-2, 1, 80)"
 84 | 
 85 |   [[galdale2014]]
 86 |     # Alpha slope. Possible values are: 0.0625, 0.1250, 0.1875, 0.2500,
 87 |     # 0.3125, 0.3750, 0.4375, 0.5000, 0.5625, 0.6250, 0.6875, 0.7500,
 88 |     # 0.8125, 0.8750, 0.9375, 1.0000, 1.0625, 1.1250, 1.1875, 1.2500,
 89 |     # 1.3125, 1.3750, 1.4375, 1.5000, 1.5625, 1.6250, 1.6875, 1.7500,
 90 |     # 1.8125, 1.8750, 1.9375, 2.0000, 2.0625, 2.1250, 2.1875, 2.2500,
 91 |     # 2.3125, 2.3750, 2.4375, 2.5000, 2.5625, 2.6250, 2.6875, 2.7500,
 92 |     # 2.8125, 2.8750, 2.9375, 3.0000, 3.0625, 3.1250, 3.1875, 3.2500,
 93 |     # 3.3125, 3.3750, 3.4375, 3.5000, 3.5625, 3.6250, 3.6875, 3.7500,
 94 |     # 3.8125, 3.8750, 3.9375, 4.0000
 95 |     alpha = 1.5, 2.0, 2.5
 96 |   
 97 |   [[activate]]
 98 |     # fracAGN = 0.0001, 0.001, 0.003, 0.01, 0.03, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.97, 0.99, 0.997, 0.999, 0.9999
 99 |     fracAGN = -1
100 | 
101 |   [[activatepl]]
102 |     plslope = "eval np.arange(-2.7, -1., 0.01)"
103 |     plbendloc = 50, 80, 90, 100, 120, 150
104 |     plbendwidth = "eval np.logspace(-1, 1, 10)"
105 |     uvslope = 0
106 | 
107 |   [[activatelines]]
108 |     # AGN lines
109 |     AFeII = "eval np.logspace(-0.2, 1.5, 10)"
110 |     Alines = 0.3, 0.5, 0.7, 1, 1.5, 2, 4, 10, 20
111 |     linewidth = 10000
112 | 
113 |   [[activatetorus]]
114 |     Si = -2, -1, 0, 1, 2
115 |     fcov = 0.1, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.9
116 |     TORtemp = -3, -1.5, -0.5, 0, 0.5, 1, 1.5, 2, 3
117 |     TORcutoff = 1.0, 1.2, 1.5, 1.7, 1.9, 2.1
118 | 
119 |   [[activategtorus]]
120 |     Si = "eval np.arange(-4, 4.1, 0.2)"
121 |     fcov = "eval np.arange(0.05, 1, 0.05)"
122 |     COOLlam = "eval np.arange(10, 30, 0.01)"
123 |     COOLwidth = "eval np.arange(0.2, 0.7, 0.05)"
124 |     HOTlam = "eval np.arange(1, 5.51, 0.01)"
125 |     HOTwidth = "eval np.arange(0.2, 0.7, 0.05)"
126 |     HOTfcov = 0.04, 0.1, 0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 2.0, 2.5, 3, 5, 10
127 | 
128 |   [[activatebol]]
129 | 
130 | 
131 |   [[redshifting]]
132 |     # Redshift to apply to the galaxy. Leave empty to use the redshifts from
133 |     # the input file.
134 |     redshift = 
135 | 
136 | # Configuration of the statistical analysis method.
137 | #[analysis_configuration]
138 | #  # Name of the output file.
139 | #  output_file = computed_fluxes.txt
140 | #  # If True, save the generated SED.
141 | #  save_sed = True
142 | #  # Format of the output file. Any format supported by astropy.table e.g.
143 | #  # votable or ascii.
144 | #  output_format = votable
145 | 
146 | [analysis_configuration]
147 |   # List of the variables (in the SEDs info dictionaries) for which the
148 |   # statistical analysis will be done.
149 |   analysed_variables = sfh.sfr, sfh.sfr10Myrs, sfh.sfr100Myrs, sfh.age, stellar.lum, dust.luminosity, agn.lum12um, agn.lum6um, agn.lumBolBBB, agn.lumBolTOR
150 |   # analysed_variables = sfh.sfr, sfh.sfr10Myrs, sfh.sfr100Myrs, sfh.age, sfh.tau_main, stellar.m_star_young, stellar.m_star_old, stellar.lum, dust.luminosity, agn.lum5100A, agn.lum12um, agn.lum6um
151 |   # analysed_variables = sfh.sfr, stellar.m_star, agn.M, agn.a, agn.type, agn.fcov, agn.Mdot, agn.lum5100A, agn.lum12um
152 |   # If true, save the best SED for each observation to a file.
153 |   save_best_sed = True
154 |   # If true, for each observation and each analysed variable save the
155 |   # reduced chi2.
156 |   save_chi2 = False
157 |   # If true, for each observation and each analysed variable save the
158 |   # probability density function.
159 |   save_pdf = True
160 |   # If true, for each object check whether upper limits are present and
161 |   # analyse them.
162 |   lim_flag = True
163 |   # If true, for each object we create a mock object and analyse them.
164 |   mock_flag = False
165 | 
166 | [statistics]
167 |   exponent = 2
168 |   # calibration suggests the AGN model is good to 20% for the vast majority
169 |   # a exponential distribution of scale 0.2 has mean 0.2
170 |   systematics_width = 0.2
171 |   attenuation_model_uncertainty = False
172 |   variability_uncertainty = False
173 | 
174 | [scaling_limits]
175 |   # Minimum log stellar mass.
176 |   mass_min = 5
177 |   # Maximum log stellar mass.
178 |   mass_max = 15
179 |   # Minimum SFR, averaged over the last 100Myrs, in Msun/yr.
180 |   sfr_min = 0
181 |   # Maximum SFR, averaged over the last 100Myrs, in Msun/yr.
182 |   sfr_max = 100000
183 |   # Minimum log AGN Luminosity.
184 |   L_min = 38
185 |   # Maximum log AGN Luminosity.
186 |   L_max = 50
187 | 
188 | 


--------------------------------------------------------------------------------
/README.rst:
--------------------------------------------------------------------------------
  1 | GRAHSP Installation
  2 | ===================
  3 | 
  4 | Current use policy and the GRAHSP collaboration
  5 | -----------------------------------------------
  6 | 
  7 | GRAHSP is an open source code developed with several years of effort.
  8 | 
  9 | In this repository you find how to
 10 | 
 11 | - access the GRAHSP code (GRAHSP, GRAHSP-run repositories)
 12 | - access code documentation and example data sets (GRAHSP-example repository)
 13 | - resources include scripts and notebooks in GRAHSP-example, and user issues for questions and answers
 14 | 
 15 | **If you would like help applying the code and interpreting the results**, you are welcome to
 16 | join the GRAHSP collaboration by emailing Johannes Buchner <johannes.buchner.acad@gmx.com>, and as a community, we will help each other and improve GRAHSP together.
 17 | As a collaboration member, you are expected to:
 18 | 
 19 | - help make improvements to the GRAHSP collaboration, such as:
 20 | 
 21 |   - opening issues pointing out how to improve the documentation for newcomers (for example you do not understand something while reading a file)
 22 |   - help resolve open issues, helping other users
 23 |   - sharing code or data sets with other users, such as a Jupyter notebook or data set
 24 | 
 25 | - offer co-authorship to core GRAHSP developers (Johannes, Mara) and anyone else who helps you with the code during the project
 26 | 
 27 | As a collaboration member, you are welcome to send questions on our Slack. Ideally, and especially if your questions are generic and may help other users, please open an issue 
 28 | in the relevant repository:
 29 | 
 30 | - https://github.com/JohannesBuchner/GRAHSP-run/ for most issues, 
 31 | - https://github.com/JohannesBuchner/GRAHSP-examples/ for the scripts there and for new example data or notebooks, 
 32 | - https://github.com/JohannesBuchner/GRAHSP/ only for model bugs
 33 | 
 34 | Scientific publications are encouraged to demonstrate the capabilities and benefits of the code and to obtain scientific insights.
 35 | 
 36 | *Beware*: SED fitting is a subtle endevour where one can make many 
 37 | mistakes. When preparing the photometry, watch out for 
 38 | 
 39 | * type of magnitude (petrosian, Hall, total/aperture, )
 40 | * that the aperture matches across bands so you look at the same physical region
 41 | * flux conversion (units nanomaggies, mJy, uJy, ...)
 42 | * AB vs Vega
 43 | * Milky way extinction correction
 44 | * redshifts (photo-z, spec-z, reliability)
 45 | * galaxy and AGN modelling assumptions
 46 | 
 47 | Experts co-authors reviewing the final manuscript can improve your work.
 48 | 
 49 | You may be interested in RainbowLasso as well: https://github.com/JohannesBuchner/RainbowLasso
 50 | 
 51 | Preliminaries
 52 | ---------------
 53 | 
 54 | 1. Have a look at the `Cigale Documentation <https://cigale.lam.fr/documentation/>`_ and `manual <https://gitlab.lam.fr/cigale/manual/-/jobs/artifacts/master/raw/CIGALE_Manual.pdf?job=build>`_. 
 55 |    
 56 | GRAHSP is built on top of CIGALE, so there are many commonalities,
 57 | such as the input data format.
 58 | 
 59 | Currently, GRAHSP and CIGALE cannot be installed alongside each other.
 60 | 
 61 | 2. Install necessary packages with pip or conda:
 62 | 
 63 | * ultranest 
 64 | * getdist
 65 | * tqdm
 66 | * joblib
 67 | * numba
 68 | * matplotlib
 69 | * scipy
 70 | * astropy
 71 | * sqlalchemy
 72 | * configobj
 73 | * h5py
 74 | 
 75 | For example::
 76 | 
 77 | 	conda install -c conda-forge ultranest tqdm joblib numba h5py sqlalchemy matplotlib configobj astropy
 78 | 	pip3 install getdist
 79 | 
 80 | You need about 4 GB of free space in your python site-package directories 
 81 | (usually inside ~/.local or conda folder).
 82 | 
 83 | Download instructions
 84 | ---------------------
 85 | 
 86 | 3. Download GRAHSP components
 87 | 
 88 | * Option 1: get a release tarball from https://github.com/JohannesBuchner/GRAHSP/releases/
 89 | * Option 2: clone the latest version from the git repositories
 90 | 
 91 |   * GRAHSP: contains the SED model engine: "git clone https://github.com/JohannesBuchner/GRAHSP"
 92 |   * GRAHSP-run: contains the SED fitting engine, visualisations, typically updates more often than GRAHSP: "git clone https://github.com/JohannesBuchner/GRAHSP-run"
 93 |   * GRAHSP-examples: example data and scripts: "git clone https://github.com/JohannesBuchner/GRAHSP-examples"
 94 | 
 95 | Installation instructions
 96 | --------------------------
 97 | 
 98 | 4. go into GRAHSP/ folder: "cd GRAHSP"
 99 | 
100 | 5. install with conda or pip3::
101 | 
102 | 	$ SPEED=2 pip3 install -v .
103 | 	...
104 | 	##############################################################################
105 | 	1- Importing filters...
106 | 
107 | 	Importing BX_B90... (21 points)
108 | 	Importing B_B90... (21 points)
109 | 	...
110 | 
111 | This takes a while as all the filters and models are imported into the
112 | database (data.db). 
113 | 
114 | The SPEED environment variable controls how many models to include::
115 | 
116 |     2 -- quick and small GRAHSP install, 700MB
117 |     1 -- full physical AGN models (Fritz,Netzer), 2800MB, or 
118 |     0 -- like 1 but also include non-solar metallicity galaxies and Draine&Li dust models), 3200MB, 
119 | 
120 | Building the database may fail on NFS-mounted file systems. Use a local file system if this happens.
121 | 
122 | Verifying the installation
123 | ---------------------------
124 | 
125 | Verify that the installation was successful:
126 | 
127 | In the GRAHSP-examples/DR16QWX folder, run::
128 | 
129 | 	$ python3 ../../GRAHSP-run/dualsampler.py --help
130 | 	$ python3 ../../GRAHSP-run/dualsampler.py list-filters
131 | 
132 | 
133 | Plotting the model
134 | ------------------
135 | 
136 | The GRAHSP-examples/ directory contains python scripts that allow plotting the 
137 | model and its components, and playing with parameter settings.
138 | 
139 | In particular:
140 | 
141 | - **plotgagn.py**: plots the full GRAHSP model and its components
142 | 
143 | - **plotstellarpop.py**: allows you to play with stellar populations
144 | 
145 |   - set the age and tau of the star formation history
146 |   - compare Maraston2005 and BC03 templates
147 | 
148 | - **plotattenuation.py**: illustrates the impact of different levels of attenuation
149 | 
150 | Running GRAHSP
151 | ---------------
152 | 
153 | The fitting is performed with **dualsampler.py**. To understand the interface, run::
154 | 
155 | 	$ python3 ../../GRAHSP-run/dualsampler.py --help
156 | 
157 | * The model setup is described with a pcigale.ini file. This is virtually identical to CIGALE.
158 | * The data is described with a data file (pointed to in the pcigale.ini). This is virtually identical to CIGALE.
159 | * What to do is set by command line options.
160 | * How to do it (performance settings) is set by environment variables (see below).
161 | 
162 | In a directory with pcigale.ini file, the following command::
163 | 
164 | 	$ python3 ../../GRAHSP-run/dualsampler.py analyse --cores=2 --plot
165 | 
166 | does:
167 | 
168 | * load data file and filters
169 | * run in parallel with 2 cores
170 | * for each catalog entry
171 | 
172 |   * run fitting with ultranest+slice sampling 
173 |   * create posterior chains
174 |   * create plots and diagnostics
175 |   * output files for custom plots
176 |   * create summary file (analysis_results.txt)
177 | 
178 | * output the fit summary file <inputfilename>_analysis_results.txt. You can convert this to a fits file with::
179 | 
180 | 	$ stilts tpipe in=inputfilename_analysis_results.txt ifmt=CSV out=analysis_results.fits
181 | 
182 | To obtain a file which contains also the input file columns, the following may be useful::
183 | 
184 | 	$ stilts tmatch2 in1=input.fits_analysis_results.txt ifmt1=ASCII suffix1= values1=id \
185 | 		in2=input.fits suffix2=_in values2=id \
186 | 		out=analysis_results.fits fixcols=all matcher=exact 
187 | 
188 | There is also a post-processing script which does the same, and makes
189 | a diagnostic plot of the fit residuals::
190 | 
191 | 	$ python3 ../../GRAHSP-run/postprocess.py
192 | 
193 | 
194 | Environment flags
195 | -----------------
196 | 
197 | On large machines, speed-ups are possible with more memory and CPUs. 
198 | This can be enabled by setting the following environment variables:
199 | 
200 | * OMP_NUM_THREADS: numpy and other libraries also parallelise some of their functions. 
201 |   If you already parallelise with --cores, 
202 |   you should prevent double parallelisation (which causes slowdown)::
203 | 
204 | 	# do not parallelize within each process
205 | 	export OMP_NUM_THREADS=1
206 | 
207 | * MP_METHOD: This controls how parallelisation is performed, see:
208 |   https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods
209 |   It is 'forkserver' by default, alternatives are 'spawn' and 'fork.
210 |   If set to 'joblib', joblib is used for parallelisation: https://joblib.readthedocs.io/en/latest/parallel.html::
211 | 
212 | 	# run with many cores
213 | 	python3 ~/workspace/Storage/jbuchner/persistent/GRAHSP/sampler/dualsampler.py analyse --cores=40 --plot --randomize
214 | 
215 | * CACHE_MAX (default: 10000, suitable for laptops): How many SEDs to cache. Try to increase this. 
216 |   If you see crashes (processes killed), it is likely that you have exceeded the 
217 |   available memory. Then, reduce CACHE_MAX and/or the number of cores::
218 | 
219 | 	# the following relaxes cache constraints since we have huge memory
220 | 	# number of models to keep in cache
221 | 	export CACHE_MAX=200000
222 | 	# report when the cache maximum is reached?
223 | 	export CACHE_VERBOSE=1
224 | 
225 | * DB_IN_MEMORY (default: 0): copy database to memory to avoid mutual blocking of processes::
226 | 
227 | 	# copy database to memory to avoid mutual blocking of processes
228 | 	export DB_IN_MEMORY=1
229 | 
230 | * HDF5_USE_FILE_LOCKING: Most shared remote machines use NFS-mounted file systems. 
231 |   HDF5 (used by ultranest for resume files) has issues with this::
232 | 
233 | 	# locking if running on multiple machines is problematic on NFS
234 | 	export HDF5_USE_FILE_LOCKING=FALSE
235 | 
236 | Other scripts
237 | --------------
238 | 
239 | Here you can also find:
240 | 
241 | - loofilter.py: Create a modified photometry input file, with new rows that leave out one filter at a time.
242 | 
243 |   - Usage: python3 loofilter.py input.fits input-new.fits
244 | 
245 | - varyz.py: Create a modified photometry input file, with new rows with slightly altered redshifts.
246 | 
247 |   - Usage: python3 varyz.py input.fits input-new.fits
248 | 


--------------------------------------------------------------------------------
/generate_config.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | from pcigale.data import Database
  3 | from astropy.table import Table
  4 | import argparse
  5 | import os
  6 | import sys
  7 | from collections import Counter
  8 | import importlib
  9 | from pathlib import Path
 10 | import astropy.units as u
 11 | 
 12 | 
 13 | def Prepare_Data(path, SAVE=True, savepath=None, delete_missing=False):
 14 |     if path.suffix in ['.fits', '.csv']:
 15 |         T = Table.read(path)
 16 |     else:
 17 |         T = Table.read(path, format='ascii')
 18 |         
 19 |     with Database() as base:
 20 |         k, dico = base.get_filter_list()
 21 |     keys = [key for key in dico.keys()]
 22 |     vals = [val for val in dico.values()]
 23 |     
 24 |     list_old_names = []
 25 |     list_new_names = []
 26 |     list_del_names = []
 27 |     
 28 |     id_name = T.colnames[0]
 29 |     redshift_name = T.colnames[1]
 30 |     assert 'id' in id_name.lower(), "The 'id' column is missing."
 31 |     assert 'z' in redshift_name.lower() or 'redshift' in redshift_name.lower(), "the 'redshift' column is missing."
 32 |     T.rename_columns(T.colnames[:2], ['id', 'redshift'])
 33 |     
 34 |     for name in [colname for colname in T.colnames 
 35 |                  if colname not in ['id', 'redshift', 'redshift_err', 'zphot_distrib', 'flag_zphot', 'PZ', 'ZGRID']]:
 36 |         if name[-3:] != 'err':
 37 |             if name not in keys:
 38 |                 if delete_missing:
 39 |                     print(f"\nFilter '{name}' not registered. Data deleted.")
 40 |                     list_del_names += [name, name + '_err']
 41 |                 else:
 42 |                     L_possibilities = []
 43 |                     for part in name.replace('.', '_').split('_'):
 44 |                         if len(part) == 1:
 45 |                             L_possibilities += [name for name in keys if part.lower() in name.lower().replace('.', '_').split('_')]
 46 |                         else:
 47 |                             L_possibilities += [name for name in keys if part.lower() in name.lower()]
 48 | 
 49 |                     print(f"\nFilter '{name}' not registered!")
 50 |                     if len(L_possibilities) > 0:
 51 |                         C = Counter(L_possibilities)
 52 |                         
 53 |                         L_possibilities = np.unique(L_possibilities)
 54 |                         print('Available filters that might correspond:')
 55 |                         print(L_possibilities)
 56 |                         
 57 |                         best_option = None
 58 |                         if Counter(C.values())[max(C.values())] == 1: # unique most likely filter
 59 |                             print('Best option: ', C.most_common()[0][0])
 60 |                             best_option = C.most_common()[0][0]
 61 |                             
 62 |                         print()
 63 |                         question = f'If filter available, please write its name.\nIf not, type "delete" to delete the filter of your data. \n("Q" to exit)\nFilter name (default: {best_option}): '
 64 |                         
 65 |                         valid_answer = False
 66 |                         while not valid_answer:
 67 |                             answer = input(question)
 68 |                             if answer.lower() == 'delete':
 69 |                                 list_del_names += [name, name + '_err']
 70 |                                 valid_answer = True
 71 |                             elif answer in L_possibilities:
 72 |                                 list_old_names += [name, name + '_err']
 73 |                                 list_new_names += [answer, answer + '_err']
 74 |                                 valid_answer = True
 75 |                             elif answer.lower() == 'q':
 76 |                                 return
 77 |                             elif best_option is not None and answer == '':
 78 |                                 list_old_names += [name, name + '_err']
 79 |                                 list_new_names += [best_option, best_option + '_err']
 80 |                                 valid_answer = True
 81 |                             else:
 82 |                                 print('Not a valid anser. Please retry.')
 83 |                         print()
 84 |                     else:
 85 |                         print(f"No match to the '{name}' filter was found. \nPlease, verify column names and/or register corresponding filter.")
 86 |                         list_del_names += [name, name + '_err']
 87 | 
 88 |         old_units = T[name].unit
 89 |         if old_units is None:
 90 |             print(f"\nNo units were given for the {name} column. \nIt is assumed to be flux in mJy.")
 91 |             T[name] = T[name] * u.mJy
 92 |         elif old_units != u.mJy:
 93 |             T[name] = T[name].to(u.mJy)
 94 |             print(f"\nThe flux units of the {name} column have been changed from {old_units} to mJy.")
 95 |                 
 96 |     
 97 |     T.remove_columns(list_del_names)
 98 |     print('The following columns have been removed:')
 99 |     print(list_del_names)
100 |     print()
101 | 
102 |     if len(list_new_names) == 0:
103 |         print('All the column names were correct.')
104 |     elif not delete_missing:
105 |         T.rename_columns(list_old_names, list_new_names)
106 |         print('The columns are renamed.')
107 |     else:
108 |         print('The unknown columns were deleted.')
109 |     
110 |     if SAVE:
111 |         if savepath is None:
112 |             question = "No path to save indicated, please indicate it: "
113 |             savepath = input(question)
114 |         T.write(savepath, overwrite=True)
115 |         print('New file saved.')
116 |         
117 |     txt = 'column_list = '
118 |     for filt in [name for name in T.colnames[2:] if name not in ['redshift_err', 'zphot_distrib', 'flag_zphot', 'PZ', 'ZGRID']]:
119 |         txt += filt + ', '
120 |     txt = txt[:-2]    
121 |         
122 |     return T, txt
123 | 
124 | def get_stellar_pop():
125 |     question = "What stellar population do you want to use?\n\
126 | Type 'bc03' for Bruzual&Charlot (2003) or 'm2005' for Maraston (2005)\n\
127 | Stellar pop (default: m2005): "
128 |     valid_answer = False
129 |     while not valid_answer:
130 |         answer = input(question)
131 |         if answer.lower() == 'bc03':
132 |             valid_answer = True
133 |             stellar_pop = 'bc03'
134 |         elif answer.lower() == 'm2005' or answer == '':
135 |             valid_answer = True
136 |             stellar_pop = 'm2005'
137 |     return stellar_pop
138 | 
139 | 
140 | 
141 | def create_pcigale(path_pcigale_empty, path_new_pcigale, filename, txt):
142 |     with open(path_pcigale_empty, 'r') as f:
143 |         pcigale_empty = f.readlines()
144 |     with open(path_new_pcigale, 'w') as fout:
145 |         for line in pcigale_empty:
146 |             if line[:9] == 'data_file':
147 |                 line = f'data_file = {filename}\n'
148 |             elif line[:16] == 'creation_modules':
149 |                 stellar_pop = get_stellar_pop()
150 |                 list_modules = f'sfhdelayed, {stellar_pop}, nebular, activate, activatelines, activategtorus, activatepl, activatebol, biattenuation, galdale2014, redshifting'
151 |                 line = f'creation_modules = {list_modules}\n'
152 |             elif line[:11] == 'column_list':
153 |                 line = txt + '\n'
154 |             elif line[:15] == 'analysis_method':
155 |                 line = "analysis_method = pdf_analysis\n"
156 | #            elif '[sed_creation_modules]' in line:
157 | #                for module_name in list_modules.split():
158 | #                     module_name = module_name.replace(',', '')
159 | #                     module = importlib.import_module('pcigale.creation_modules.' + module_name)
160 | #                     klass = module.Module
161 | #                     line += f"\n  [[{module_name}]]\n"
162 | #                     for param, (dtype, comment, default_value) in klass.parameter_list.items():
163 | #                        line += f"      # {comment} (type: {dtype})\n"
164 | #                        line += f"      {param} = {default_value}\n"
165 | #                     line += f"    # the code of this module is in: {module.__file__}\n"
166 |             fout.write(line)
167 | 
168 | 
169 | 
170 | class HelpfulParser(argparse.ArgumentParser):
171 |     def error(self, message):
172 |         sys.stderr.write('error: %s\n' % message)
173 |         self.print_help()
174 |         sys.exit(2)
175 | 
176 | 
177 | parser = HelpfulParser(
178 |     description=__doc__,
179 |     epilog="""Johannes Buchner (C) 2013-2023 <johannes.buchner.acad@gmx.com>""",
180 |     formatter_class=argparse.RawDescriptionHelpFormatter
181 | )
182 | 
183 | 
184 | parser.add_argument(
185 |     '-p', '--path', type=str, required=True,
186 |     help='Path of the catalogue to prepare (required).')
187 | args = parser.parse_args()
188 | assert os.path.exists(args.path), "The given path does not exist."
189 | 
190 | 
191 | 
192 | def main():
193 |     filename = Path(args.path)
194 |     current_path = Path.cwd()
195 |     new_filename = current_path / ('new_' + filename.name)
196 |     path_pcigale_empty = Path(__file__).parent / 'template_pcigale.ini'
197 |     path_new_pcigale = current_path / 'pcigale.ini'
198 | 
199 |     data, txt = Prepare_Data(filename, savepath=new_filename)
200 | 
201 |     need_pcigale_file = True
202 |     if path_new_pcigale.exists():
203 |         question = f"The file {path_new_pcigale} already exists.\nDo you want to replace it? [y]/n "
204 |         valid_answer = False
205 |         while not valid_answer:
206 |             answer = input(question)
207 |             if answer.lower() == 'y' or answer == '':
208 |                 valid_answer = True
209 |                 print(f"The file {path_new_pcigale} has been replaced.")
210 |             if answer.lower() == 'n':
211 |                 valid_answer = True
212 |                 need_pcigale_file = False
213 |                 
214 |     if need_pcigale_file:
215 |         create_pcigale(path_pcigale_empty, path_new_pcigale, new_filename, txt)
216 |                 
217 | 
218 | 
219 | if __name__ == '__main__':
220 |     main()
221 | 
222 | 
223 | 
224 | 
225 | 
226 | 
227 | 
228 | 
229 | 
230 | 
231 | 
232 | 
233 | 
234 | 
235 | 
236 | 
237 | 
238 | 
239 | 
240 | 
241 | 
242 | 
243 | 
244 | 
245 | 
246 | 
247 | 
248 | 
249 | 
250 | 
251 | 
252 | 
253 | 
254 | 
255 | 
256 | 
257 | 


--------------------------------------------------------------------------------
/simplesampler.py:
--------------------------------------------------------------------------------
  1 | 
  2 | import numpy as np
  3 | from numpy import log
  4 | from math import erf
  5 | from pcigale.session.configuration import Configuration
  6 | from pcigale.analysis_modules import get_module as get_analysis_module
  7 | from pcigale.utils import read_table
  8 | from pcigale.analysis_modules import complete_obs_table
  9 | from pcigale.warehouse import SedWarehouse
 10 | from pcigale.analysis_modules.pdf_analysis import TOLERANCE
 11 | from pcigale import creation_modules
 12 | from pcigale.sed import SED
 13 | from pcigale.data import Database
 14 | from ultranest import ReactiveNestedSampler
 15 | from ultranest.plot import PredictionBand
 16 | import scipy.stats
 17 | from scipy.constants import c
 18 | import matplotlib.pyplot as plt
 19 | import matplotlib.gridspec as gridspec
 20 | 
 21 | 
 22 | gbl_warehouse = SedWarehouse()
 23 | 
 24 | config = Configuration("pcigale.ini.galonly")
 25 | 
 26 | data_file = config.configuration['data_file']
 27 | column_list = config.configuration['column_list']
 28 | module_list = config.configuration['creation_modules']
 29 | parameter_list = config.configuration['creation_modules_params']
 30 | print(parameter_list)
 31 | 
 32 | analysis_module = get_analysis_module(config.configuration[
 33 |     'analysis_method'])
 34 | analysis_module_params = config.configuration['analysis_method_params']
 35 | 
 36 | analysed_variables = analysis_module_params["analysed_variables"]
 37 | n_variables = len(analysed_variables)
 38 | save = {key: analysis_module_params["save_{}".format(key)].lower() == "true"
 39 |         for key in ["best_sed", "chi2", "pdf"]}
 40 | lim_flag = analysis_module_params["lim_flag"].lower() == "true"
 41 | mock_flag = analysis_module_params["mock_flag"].lower() == "true"
 42 | 
 43 | filters = [name for name in column_list if not name.endswith('_err')]
 44 | with Database() as base:
 45 |     filters_wl_orig = np.array([base.get_filter(name).effective_wavelength for name in filters])
 46 | n_filters = len(filters)
 47 | 
 48 | 
 49 | param_names = []
 50 | for module_name, module_parameters in zip(module_list, parameter_list):
 51 |     for k, v in module_parameters.items():
 52 |         if len(v) > 1:
 53 |             param_names.append("%s.%s" % (module_name, k))
 54 |         del k, v
 55 |     del module_name, module_parameters
 56 | 
 57 | param_names.append("stellar_mass")
 58 | param_names.append("redshift")
 59 | #param_names.append("systematics")
 60 | #rv_systematics = scipy.stats.halfnorm(scale=0.05)
 61 | 
 62 | def make_parameter_list(parameters):
 63 |     parameter_list_first = []
 64 |     i = 0
 65 |     for module_parameters in parameter_list:
 66 |         parameter_list_here = {}
 67 |         for k, v in module_parameters.items():
 68 |             if len(v) == 1:
 69 |                 parameter_list_here[k] = v[0]
 70 |             else:
 71 |                 parameter_list_here[k] = parameters[i]
 72 |                 i += 1
 73 | 
 74 |         parameter_list_first.append(parameter_list_here)
 75 |     return parameter_list_first
 76 | 
 77 | def get_model_fluxes(sed):
 78 |     if 'sfh.age' in sed.info and sed.info['sfh.age'] > sed.info['universe.age']:
 79 |         model_fluxes = -99. * np.ones(len(filters))
 80 |         model_variables = -99. * np.ones(len(analysed_variables))
 81 |     else:
 82 |         model_fluxes = np.array([sed.compute_fnu(filter_) for filter_ in filters])
 83 |         model_variables = np.array([sed.info[name] for name in analysed_variables])
 84 | 
 85 |     return model_fluxes, model_variables
 86 | 
 87 | def compute_model(parameter_list_here):
 88 |     sed = SED()
 89 |     for module, module_parameters in zip(module_list, parameter_list_here):
 90 |         if module == 'redshifting':
 91 |             module_parameters = dict(redshift=0.1)
 92 |         # print(module, module_parameters)
 93 |         module_instance = creation_modules.get_module(module, **module_parameters)
 94 |         module_instance.process(sed)
 95 | 
 96 |     model_fluxes, model_variables = get_model_fluxes(sed)
 97 |     return sed, model_fluxes, model_variables
 98 | 
 99 | #@functools.lru_cache(maxsize=None)
100 | 
101 | #import joblib
102 | #mem = joblib.Memory('.', verbose=False)
103 | 
104 | #@mem.cache
105 | #def compute_model_cached(parameters):
106 | #   parameter_list_here = make_parameter_list(parameters)
107 | #   sed, model_fluxes_full, model_variables = compute_model(parameter_list_here)
108 | #   return model_fluxes_full
109 | 
110 | # Wavelength limits (restframe) when plotting the best SED.
111 | PLOT_L_MIN = 0.1
112 | PLOT_L_MAX = 50
113 | 
114 | def with_attenuation(keys):
115 |     return keys + ['attenuation.' + key for key in keys]
116 | 
117 | def plot_results(sampler, obs, obs_fluxes, obs_errors, wobs, cache_filters):
118 |     results = sampler.results
119 |     plot_dir = sampler.logs['plots']
120 |     prior_samples = sampler.transform(np.random.uniform(size=(10000, len(param_names))))
121 |     assert np.isfinite(prior_samples).all(), (np.where(~np.isfinite(prior_samples).all(axis=0)), np.where(~np.isfinite(prior_samples).all(axis=1)), prior_samples[~np.isfinite(prior_samples)])
122 |     assert np.isfinite(results['samples']).all()
123 |     
124 |     plt.figure(figsize=(12, 12))
125 |     for i, (param_name, samples) in enumerate(zip(param_names, results['samples'].transpose())):
126 |         plt.subplot(4, len(param_names) // 4 + 1, i + 1)
127 |         plt.hist(samples, histtype='step', density=True, bins=40)
128 |         xlo, xhi = plt.xlim()
129 |         plt.hist(prior_samples[:,i], histtype='step', 
130 |             density=True, bins=40, color='gray', ls='-')
131 |         plt.xlim(xlo, xhi)
132 |         plt.yticks([])
133 |         plt.xlabel(param_names[i])
134 |         
135 |     plt.savefig('%s/posteriors.pdf' % plot_dir, bbox_inches='tight')
136 |     plt.close()
137 |     
138 |     bands = {'lum':{}, 'mJy':{}}
139 | 
140 |     plot_elements = [
141 |         dict(keys=with_attenuation(['stellar.young', 'stellar.old']),
142 |              label="Stellar attenuated", color='orange', marker=None, nonposy='clip', linestyle='-',),
143 |         dict(keys=['stellar.young', 'stellar.old'],
144 |              label="Stellar unattenuated", color='b', marker=None, nonposy='clip', linestyle='--', linewidth=0.5),
145 |         dict(keys=with_attenuation(['nebular.lines_young', 'nebular.lines_old', 'nebular.continuum_young', 'nebular.continuum_old']),
146 |              label="Nebular emission", color='y', marker=None, nonposy='clip', linewidth=.5),
147 |         dict(keys=['agn.activate_Disk'],
148 |              label="AGN disk", color=[0.90, 0.90, 0.72], marker=None, nonposy='clip', linestyle='-', linewidth=1.5),
149 |         dict(keys=['agn.activate_Torus'],
150 |              label="AGN torus", color=[0.90, 0.77, 0.42], marker=None, nonposy='clip', linestyle='-', linewidth=1.5),
151 |         dict(keys=['agn.activate_EmLines_BL', 'agn.activate_EmLines_NL', 'agn.activate_FeLines', 'agn.activate_EmLines_LINER'],
152 |              label="AGN lines", color=[0.90, 0.50, 0.21], marker=None, nonposy='clip', linestyle='-', linewidth=0.5),
153 |         dict(keys=['F_lambda_total'],
154 |              label="Model spectrum", color='k', marker=None, nonposy='clip', linestyle='-', linewidth=1.5, alpha=0.7),
155 |     ]
156 |     
157 |     z = obs['redshift']
158 |     chi2_best = -2 * sampler.results['weighted_samples']['logl'].max()
159 |     # chi2_reduced = chi2_best / wobs.sum()
160 |     
161 |     posteriors = []
162 |     
163 |     for parameters in sampler.results['samples'][:100,:]:
164 |         stellar_mass = 10**parameters[-2]
165 |         redshift = parameters[-1]
166 |         # systematic_flux_error = parameters[-1]
167 |         parameter_list_here = make_parameter_list(parameters)
168 |         parameter_list_here[-1] = dict(redshift=redshift)
169 |         sed = gbl_warehouse.get_sed(module_list, parameter_list_here)
170 |         sed.cache_filters = cache_filters
171 | 
172 |         model_fluxes_full, model_variables = get_model_fluxes(sed)
173 |         posteriors.append(model_variables)
174 | 
175 |         mod_fluxes = model_fluxes_full[wobs] * stellar_mass
176 |         
177 |         # print(dir(sed), sed.info.keys())
178 |         wavelength_spec = sed.wavelength_grid
179 |         DL = sed.info['universe.luminosity_distance']
180 | 
181 |         for sed_type in 'mJy', 'lum':
182 |             wavelength_spec2 = wavelength_spec.copy()
183 |             if sed_type == 'lum':
184 |                 sed_multiplier = wavelength_spec2.copy()
185 |                 wavelength_spec2 /= 1. + z
186 |             elif sed_type == 'mJy':
187 |                 sed_multiplier = (wavelength_spec2 * 1e29 /
188 |                                    (c / (wavelength_spec2 * 1e-9)) /
189 |                                    (4. * np.pi * DL * DL))
190 | 
191 |             sed_multiplier *= stellar_mass
192 |             assert (sed_multiplier >= 0).all(), (stellar_mass, DL, wavelength_spec2)
193 |             wavelength_spec2 /= 1000
194 | 
195 |             for j, plot_element in enumerate(plot_elements):
196 |                 keys = plot_element['keys']
197 |                 if not all(k in sed.contribution_names for k in keys):
198 |                     continue
199 |                 if j not in bands[sed_type]:
200 |                     print("building", sed_type, plot_element['label'])
201 |                     bands[sed_type][j] = PredictionBand(wavelength_spec2)
202 |                 pred = sum(sed.get_lumin_contribution(k) * sed_multiplier for k in keys)
203 |                 assert np.isfinite(pred).all(), pred
204 |                 assert bands[sed_type][j].x.shape == pred.shape, (bands[sed_type][j].x.shape, pred.shape)
205 |                 # print(plot_element['label'], pred)
206 |                 bands[sed_type][j].add(pred)
207 | 
208 | 
209 |     for sed_type in 'mJy', 'lum':
210 |         filters_wl = filters_wl_orig[wobs] / 1000
211 |         # wsed = np.where((wavelength_spec2 > xmin) & (wavelength_spec2 < xmax))
212 | 
213 |         figure = plt.figure()
214 |         gs = gridspec.GridSpec(2, 1, height_ratios=[3, 1])
215 | 
216 |         ax1 = plt.subplot(gs[0])
217 |         ax2 = plt.subplot(gs[1])
218 | 
219 |         plt.sca(ax1)
220 |         for j, plot_element in enumerate(plot_elements):
221 |             if j in bands[sed_type]:
222 |                 print("plotting", sed_type, plot_element['label'], np.shape(bands[sed_type][j].ys))
223 |                 bands[sed_type][j].shade(0.45, color=plot_element['color'], 
224 |                     #label=plot_element['label'] % sed.info, 
225 |                     alpha=0.1)
226 |                 line_kwargs = dict(plot_element)
227 |                 del line_kwargs['keys'], line_kwargs['nonposy']
228 |                 bands[sed_type][j].line(**line_kwargs)
229 | 
230 |         if sed_type == 'lum':
231 |             xmin = PLOT_L_MIN / (1 + z)
232 |             xmax = PLOT_L_MAX / (1 + z)
233 | 
234 |             filters_wl /= 1. + z
235 |             k_corr_SED = 1e-29 * (4.*np.pi*DL*DL) * c / (filters_wl*1e-9) / 1000
236 |             obs_fluxes = obs_fluxes * k_corr_SED
237 |             obs_fluxes_err = obs_errors * k_corr_SED
238 |             mod_fluxes = mod_fluxes * k_corr_SED
239 |         elif sed_type == 'mJy':
240 |             xmin = PLOT_L_MIN
241 |             xmax = PLOT_L_MAX
242 | 
243 |             k_corr_SED = 1.
244 |             obs_fluxes_err = obs_errors
245 | 
246 |         ax1.set_autoscale_on(False)
247 |         ax1.scatter(filters_wl, mod_fluxes, marker='o', color='r', s=8,
248 |                     zorder=3, label="Model fluxes")
249 |         mask_ok = np.logical_and(obs_fluxes > 0., obs_errors > 0.)
250 |         ax1.errorbar(filters_wl[mask_ok], obs_fluxes[mask_ok],
251 |                      yerr=obs_fluxes_err[mask_ok]*3, ls='', marker='s',
252 |                      label='Observed fluxes', markerfacecolor='None',
253 |                      markersize=6, markeredgecolor='b', capsize=0.)
254 |         mask_uplim = np.logical_and(np.logical_and(obs_fluxes > 0.,
255 |                                                obs_fluxes_err < 0.),
256 |                                 obs_fluxes_err > -9990. * k_corr_SED)
257 | 
258 |         if not mask_uplim.any() == False:
259 |             ax1.errorbar(filters_wl[mask_uplim], obs_fluxes[mask_uplim],
260 |                          yerr=obs_fluxes_err[mask_uplim]*3, ls='',
261 |                          marker='v', label='Observed upper limits',
262 |                          markerfacecolor='None', markersize=6,
263 |                          markeredgecolor='g', capsize=0.)
264 |         mask_noerr = np.logical_and(obs_fluxes > 0.,
265 |                                     obs_fluxes_err < -9990. * k_corr_SED)
266 |         if not mask_noerr.any() == False:
267 |             ax1.errorbar(filters_wl[mask_noerr], obs_fluxes[mask_noerr],
268 |                          ls='', marker='s', markerfacecolor='None',
269 |                          markersize=6, markeredgecolor='r',
270 |                          label='Observed fluxes, no errors', capsize=0.)
271 |         mask = np.where(obs_fluxes > 0.)
272 |         ax2.errorbar(filters_wl[mask],
273 |                      (obs_fluxes[mask]-mod_fluxes[mask])/obs_fluxes[mask],
274 |                      yerr=obs_fluxes_err[mask]/obs_fluxes[mask]*3,
275 |                      marker='_', label="(Obs-Mod)/Obs", color='k',
276 |                      capsize=0., linestyle=' ')
277 |         ax2.plot([xmin, xmax], [0., 0.], ls='--', color='k')
278 |         ax2.set_xscale('log')
279 |         ax1.set_xscale('log')
280 |         ax2.minorticks_on()
281 | 
282 |         figure.subplots_adjust(hspace=0.2, wspace=0.)
283 | 
284 |         ax1.set_xlim(xmin, xmax)
285 |         ax2.set_xlim(xmin, xmax)
286 |         ymin = min(np.nanmin(obs_fluxes[mask_ok]),
287 |                    np.nanmin(mod_fluxes[mask_ok]))
288 |         
289 |         if not mask_uplim.any() == False:
290 |             ymax = max(max(np.nanmax(obs_fluxes[mask_ok]),
291 |                            np.nanmax(obs_fluxes[mask_uplim])),
292 |                        max(np.nanmax(mod_fluxes[mask_ok]),
293 |                            np.nanmax(mod_fluxes[mask_uplim])))
294 |         else:
295 |             ymax = max(np.nanmax(obs_fluxes[mask_ok]),
296 |                        np.nanmax(mod_fluxes[mask_ok]))
297 |         if np.isinf(ymax):
298 |             ymax = 100 * ymin
299 |         if np.isinf(ymin):
300 |             ymin = ymax / 100
301 |         ax1.set_ylim(1e-2*ymin, 1e1*ymax)
302 |         ax1.set_yscale('log')
303 |         ax2.set_ylim(-1.0, 1.0)
304 |         if sed_type == 'lum':
305 |             ax2.set_xlabel("Rest-frame wavelength [$\mu$m]")
306 |             ax1.set_ylabel("Luminosity [W]")
307 |             ax2.set_ylabel("Relative residual luminosity")
308 |         else:
309 |             ax2.set_xlabel("Observed wavelength [$\mu$m]")
310 |             ax1.set_ylabel("Flux [mJy]")
311 |             ax2.set_ylabel("Relative residual flux")
312 |         ax1.legend(fontsize=6, loc='best', fancybox=True, framealpha=0.5)
313 |         ax2.legend(fontsize=6, loc='best', fancybox=True, framealpha=0.5)
314 |         plt.setp(ax1.get_xticklabels(), visible=False)
315 |         plt.setp(ax1.get_yticklabels()[1], visible=False)
316 |         figure.suptitle(
317 |             "Best model for %s at z = %.3f. $\chi^2$=%.1f/%d" %
318 |                 (obs['id'], obs['redshift'], chi2_best, len(obs_fluxes)))
319 |         figure.savefig("%s/sed_%s.pdf" % (plot_dir, sed_type))
320 |         plt.close(figure)
321 | 
322 | 
323 | def main():
324 |     # Read the observation table and complete it by adding error where
325 |     # none is provided and by adding the systematic deviation.
326 |     obs_table = complete_obs_table(read_table(data_file), column_list,
327 |                                    filters, TOLERANCE, lim_flag)
328 | 
329 |     # pick observation
330 |     for obs in obs_table:
331 |         np.random.seed(1)
332 |         redshift_mean = obs['redshift']
333 |         if 'redshift_err' in obs.colnames:
334 |             redshift_err = obs['redshift_err']
335 |         else:
336 |             # put a 1% error on 1+z, at least
337 |             redshift_err = 0.1 * redshift_mean
338 |         redshift_samples = np.random.normal(redshift_mean, redshift_err, size=1000)
339 |         redshift_samples = redshift_samples[redshift_samples>0]
340 |         redshift_shape, _, redshift_scale = scipy.stats.weibull_min.fit(
341 |             redshift_samples, floc=0,
342 |         )
343 |         print("redshift:", redshift_shape, redshift_scale)
344 |         rv_redshift = scipy.stats.weibull_min(redshift_shape, scale=redshift_scale)
345 | 
346 |         def prior_transform(cube):
347 |             params = cube.copy()
348 |             i = 0
349 |             for module_parameters in parameter_list:
350 |                 for k, v in module_parameters.items():
351 |                     if len(v) > 1:
352 |                         params[i] = v[int(len(v) * cube[i])]
353 |                         i += 1
354 |             
355 |             # stellar mass from 10^5 to 10^15
356 |             params[i] = cube[i] * 10 + 5
357 | 
358 |             # redshift. Approximate redshift with points on the CDF
359 |             params[i+1] = rv_redshift.ppf((1 + np.round(cube[i+1] * 40)) / 42)
360 |             
361 |             # systematic uncertainty
362 |             # params[i+2] = rv_systematics.ppf(cube[i+2])
363 |             return params
364 | 
365 |         
366 |         # select the filters from the list of active filters
367 | 
368 |         obs_fluxes_full = np.array([obs[name] for name in filters])
369 |         obs_errors_full = np.array([obs[name + "_err"] for name in filters])
370 | 
371 |         wobs = np.where(obs_fluxes_full > TOLERANCE)
372 |         obs_fluxes = obs_fluxes_full[wobs]
373 |         obs_errors = obs_errors_full[wobs]
374 | 
375 |         # Some observations may not have flux values in some filter(s), but
376 |         # they can have upper limit(s). To process upper limits, the user
377 |         # is asked to put the upper limit as flux value and an error value with
378 |         # (obs_errors>=-9990. and obs_errors<0.).
379 |         # Next, the user has two options:
380 |         # 1) s/he puts True in the boolean lim_flag
381 |         # and the limits are processed as upper limits below.
382 |         # 2) s/he puts False in the boolean lim_flag
383 |         # and the limits are processed as no-data below.
384 |         cache_filters = {}
385 | 
386 |         def loglikelihood(parameters):
387 |             stellar_mass = 10**parameters[-2]
388 |             redshift = parameters[-1]
389 |             systematic_flux_error = 0  # parameters[-1]
390 |             parameter_list_here = make_parameter_list(parameters)
391 |             assert module_list[-1] == 'redshifting'
392 |             parameter_list_here[-1] = dict(redshift=redshift)
393 |             
394 |             sed = gbl_warehouse.get_sed(module_list, parameter_list_here)
395 |             sed.cache_filters = cache_filters
396 | 
397 |             model_fluxes_full, model_variables = get_model_fluxes(sed)
398 | 
399 |             model_fluxes = model_fluxes_full[wobs] * stellar_mass
400 | 
401 |             # χ² of the comparison of each model to each observation.
402 |             # This mask selects the filter(s) for which measured fluxes are given
403 |             # i.e., when (obs_flux is >=0. and obs_errors>=0.) and lim_flag=True
404 |             mask_data = np.logical_and(obs_fluxes > TOLERANCE,
405 |                                        obs_errors > TOLERANCE)
406 |             # This mask selects the filter(s) for which upper limits are given
407 |             # i.e., when (obs_flux is >=0. (and obs_errors>=-9990., obs_errors<0.))
408 |             # and lim_flag=True
409 |             mask_lim = np.logical_and(obs_errors >= -9990., obs_errors < TOLERANCE)
410 |             chi2_ = np.sum(np.square(
411 |                 (obs_fluxes[mask_data]-model_fluxes[mask_data]) /
412 |                 (obs_errors[mask_data] * (1 + systematic_flux_error))))
413 | 
414 |             if mask_lim.any():
415 |                 chi2_ += -2. * log(
416 |                         np.sqrt(np.pi/2.)*(-obs_errors[mask_lim])*(
417 |                             1.+erf(
418 |                                 (obs_fluxes[mask_lim]-model_fluxes[mask_lim]) /
419 |                                 (np.sqrt(2)*(-obs_errors[mask_lim]))))).sum()
420 |             #print("chi2:", chi2_, parameters)
421 |             return -0.5 * chi2_
422 | 
423 |         sampler = ReactiveNestedSampler(
424 |             param_names, loglikelihood, prior_transform,
425 |             log_dir="analysis_%s" % obs['id'], resume=True)
426 |         sampler.run(frac_remain=0.5, max_num_improvement_loops=0)
427 |         sampler.print_results()
428 |         try:
429 |             sampler.plot()
430 |         except Exception:
431 |             pass
432 |         plot_results(sampler, obs, obs_fluxes, obs_errors, wobs, cache_filters)
433 |         
434 |         
435 | 
436 | if __name__ == '__main__':
437 |     main()
438 | 


--------------------------------------------------------------------------------
/dualsampler.py:
--------------------------------------------------------------------------------
   1 | """
   2 | GRAHSP (Genuine retrieval of the AGN host stellar population)
   3 | 
   4 | A broad-band SED fitting tool (X-ray to mid-infrared) for investigating host galaxies of
   5 | active galactic nuclei.
   6 | 
   7 | Features:
   8 | 
   9 | - Flexible empirical AGN model to avoid modelling bias
  10 | - broad and narrow line emission from AGN, in addition to accretion disk and obscurer.
  11 | - allows obscurer diversity (hotter and colder dust) and 12µm Si in emission or absorption
  12 | - allows different attenuation for AGN and host, with the more appropriate Prevot law.
  13 | - allows systematic modelling uncertainties, prevents biased overfits with overconfident errors
  14 | - allows AGN variability across assembled photometry points (L-dependent)
  15 | - allows redshift uncertainties (from photo-z)
  16 | - explores degeneracies with nested sampling Monte Carlo.
  17 | 
  18 | A configuration file called pcigale.ini is needed.
  19 | 
  20 | A data input file (typically called input.fits) is needed.
  21 | Columns:
  22 | - id
  23 | - redshift
  24 | - redshift_err (optional)
  25 | - FAGN: AGN 5100 Angstrom luminosity divided by luminosity distance
  26 | - FAGN_errlo and FAGN_errhi or FAGN_err for uncertainties
  27 | 
  28 | The L(2-10keV) luminosity is approximately the 5100 Angstrom luminosity,
  29 | with a systematic scatter of +-0.43 dex (Koss+2017).
  30 | 
  31 | """
  32 | 
  33 | import os
  34 | import sys
  35 | import argparse
  36 | import warnings
  37 | import itertools
  38 | from importlib import import_module
  39 | 
  40 | import numpy as np
  41 | from numpy import log, log10
  42 | import multiprocessing
  43 | import joblib
  44 | 
  45 | import scipy.stats
  46 | import scipy.special
  47 | from scipy.constants import c
  48 | import matplotlib.pyplot as plt
  49 | import matplotlib as mpl
  50 | import matplotlib.gridspec as gridspec
  51 | 
  52 | import pcigale
  53 | from pcigale.session.configuration import Configuration
  54 | from pcigale.analysis_modules import get_module as get_analysis_module
  55 | from pcigale.utils import read_table
  56 | from pcigale.analysis_modules import complete_obs_table
  57 | from pcigale.warehouse import SedWarehouse
  58 | from pcigale.analysis_modules.pdf_analysis import TOLERANCE
  59 | from pcigale.data import Database
  60 | from pcigale.creation_modules.biattenuation import BiAttenuationLaw
  61 | import astropy.cosmology
  62 | import astropy.units as units
  63 | from astropy.table import Table
  64 | import pcigale.creation_modules.redshifting
  65 | 
  66 | from ultranest import ReactiveNestedSampler
  67 | from ultranest.mlfriends import SimpleRegion, RobustEllipsoidRegion
  68 | from ultranest.plot import PredictionBand
  69 | import ultranest.stepsampler
  70 | import tqdm
  71 | from scipy.interpolate import interp1d
  72 | 
  73 | # some helper classes:
  74 | 
  75 | 
  76 | class DeltaDist(object):
  77 |     """Dirac Delta distribution."""
  78 | 
  79 |     # provides compatibility with scipy.stats distributions when a parameter is fixed
  80 |     def __init__(self, value):
  81 |         self.value = value
  82 | 
  83 |     def ppf(self, u):
  84 |         return self.value
  85 | 
  86 |     def mean(self):
  87 |         return self.value
  88 | 
  89 |     def std(self):
  90 |         return 0
  91 | 
  92 | class ExponentialDist(object):
  93 |     """Faster exponential distribution than the scipy implementation."""
  94 | 
  95 |     # provides compatibility with scipy.stats distributions when a parameter is fixed
  96 |     def __init__(self, scale):
  97 |         self.scale = scale
  98 | 
  99 |     def ppf(self, u):
 100 |         return self.scale * -np.log1p(-u)
 101 | 
 102 |     def mean(self):
 103 |         return self.scale
 104 | 
 105 |     def std(self):
 106 |         return self.scale
 107 | 
 108 | class NormalDist(object):
 109 |     """Faster Gaussian distribution than the scipy implementation."""
 110 | 
 111 |     # provides compatibility with scipy.stats distributions when a parameter is fixed
 112 |     def __init__(self, mean, scale):
 113 |         self.scale = scale
 114 |         self.mean = mean
 115 | 
 116 |     def ppf(self, u):
 117 |         return scipy.special.ndtri(u) * self.scale + self.mean
 118 | 
 119 |     def mean(self):
 120 |         return self.mean
 121 | 
 122 |     def std(self):
 123 |         return self.std
 124 | 
 125 | class PDFDist(object):
 126 |     """Probability distribution function."""
 127 |     
 128 |     # provides compatibility with scipy.stats distributions
 129 |     def __init__(self, grid, pdf_values):
 130 |         self.grid = np.linspace(grid[0], grid[-1], len(grid)*10)
 131 |         self.PDF_values = interp1d(grid, pdf_values)(self.grid)
 132 |         self.CDF = np.cumsum(self.PDF_values) / np.sum(self.PDF_values)
 133 |         self.ppf = interp1d(self.CDF, self.grid, bounds_error=False, fill_value=(0, 1))
 134 |         self.mean_val = self.ppf(0.5)
 135 |         self.std_val = np.sqrt((self.ppf(0.1585) - self.mean_val)**2 
 136 |                                + (self.ppf(0.8415) - self.mean_val)**2)
 137 | 
 138 |     def mean(self):
 139 |         return self.mean_val
 140 | 
 141 |     def std(self):
 142 |         return self.std_val 
 143 | 
 144 | 
 145 | class FastAttenuation(object):
 146 |     """Context within which the attenuation law reduces computation.
 147 | 
 148 |     Per-filter extinctions, and per-component extinction are skipped.
 149 |     """
 150 |     def __enter__(self):
 151 |         BiAttenuationLaw.store_filter_attenuation = False
 152 |         BiAttenuationLaw.store_component_attenuation = False
 153 | 
 154 |     def __exit__(self, type, value, traceback):
 155 |         BiAttenuationLaw.store_filter_attenuation = True
 156 |         BiAttenuationLaw.store_component_attenuation = True
 157 | 
 158 | # parse command line arguments
 159 | 
 160 | 
 161 | class HelpfulParser(argparse.ArgumentParser):
 162 |     def error(self, message):
 163 |         sys.stderr.write('error: %s\n' % message)
 164 |         self.print_help()
 165 |         sys.exit(2)
 166 | 
 167 | 
 168 | parser = HelpfulParser(
 169 |     description=__doc__,
 170 |     epilog="""
 171 | 
 172 | Environment variables (see README):
 173 |    MP_METHOD: parallelisation method
 174 |    DB_IN_MEMORY: if 1, copy database to memory to avoid mutual blocking of processes.
 175 |    CACHE_MAX: SED cache size
 176 |    CACHE_VERBOSE: if 1, print when cache reaches maximum.
 177 |    REPLOT: if 1, recompute existing plots even if resumed fit is unchanged
 178 |    PLOT_FILTERNAMES: if 1, show name of filters with filter curves
 179 |    PLOT_KEYSTATS: if 1, show key stats on the right of plot
 180 |    PLOT_SFH: if 1, plot star formation history
 181 |    PLOT_CORNER: if 1, create corner plots to investigate parameter degeneracies.
 182 |    PLOT_TRACE: if 1, create trace plots
 183 |    OMP_NUM_THREAD
 184 |    HDF5_USE_FILE_LOCKING
 185 | 
 186 | Johannes Buchner (C) 2013-2023 <johannes.buchner.acad@gmx.com>""",
 187 |     formatter_class=argparse.RawDescriptionHelpFormatter
 188 | )
 189 | 
 190 | 
 191 | parser.add_argument(
 192 |     '--offset', type=int, default=0,
 193 |     help='row in the input file to begin processing')
 194 | 
 195 | parser.add_argument(
 196 |     '--every', type=int, default=1,
 197 |     help='stride in the input file to process (every nth row)')
 198 | 
 199 | parser.add_argument(
 200 |     '--cores', type=int, default=1,
 201 |     help='number of processes to parallelise for (see joblib.Parallel)')
 202 | 
 203 | parser.add_argument(
 204 |     '--num-posterior-samples', type=int, default=50,
 205 |     help='number of posterior samples to analyse in post-processing')
 206 | 
 207 | parser.add_argument(
 208 |     '--num-live-points', type=int, default=50,
 209 |     help='number of live points for nested sampling')
 210 | 
 211 | parser.add_argument(
 212 |     '--plot', action='store_true',
 213 |     help='also make plots of the SED and parameter constraints')
 214 | 
 215 | parser.add_argument(
 216 |     '--randomize', action='store_true',
 217 |     help='Randomize order in which to analyse observations.')
 218 | 
 219 | parser.add_argument(
 220 |     'action', type=str, default='analyse', choices=('analyse', 'generate-from-prior', 'plot-model', 'list-filters'),
 221 |     help='''Mode.
 222 | generate-from-prior: Generate a file with fluxes from randomly drawn model instances.
 223 | plot-model: plot model SED variations.
 224 | list-filters: list the available photometric filters, or
 225 | analyse a photometry file.''')
 226 | 
 227 | parser.add_argument(
 228 |     '--sampler', type=str, default='nested-slice', choices=('nested-slice',),
 229 |     help='Parameter space sampling algorithm to use. Nested-slice is recommended.')
 230 | 
 231 | 
 232 | args = parser.parse_args()
 233 | 
 234 | # keeping it called pcigale.ini allows running pcigale with the same file
 235 | # if the user wants to run cigale
 236 | print("GRAHSP version %s | parsing pcigale.ini..." % pcigale.__version__)
 237 | config = Configuration("pcigale.ini")
 238 | 
 239 | cosmo_string = config.config.get('cosmology', 'concordance')
 240 | if cosmo_string != 'concordance':
 241 |     if not hasattr(astropy.cosmology, cosmo_string):
 242 |         print("ERROR: cosmology must be set to one of: concordance, " + ', '.join(astropy.cosmology.realizations.available))
 243 |     pcigale.creation_modules.redshifting.cosmology = getattr(astropy.cosmology, cosmo_string)
 244 | cosmology = pcigale.creation_modules.redshifting.cosmology
 245 | 
 246 | data_file = config.configuration['data_file']
 247 | assert os.path.exists(data_file), f'The file {data_file} does not exist.'
 248 | column_list = config.configuration['column_list']
 249 | module_list = config.configuration['creation_modules']
 250 | 
 251 | # receive number of cores from command line
 252 | # configuration describes the model, command line describes how to run
 253 | n_cores = args.cores
 254 | parameter_list = config.configuration['creation_modules_params']
 255 | # limit caching to the first few modules, rest is on-the-fly
 256 | cache_depth = module_list.index('biattenuation')
 257 | #cache_depth = module_list.index('triattenuation')
 258 | cache_depth_to_clear = cache_depth
 259 | if module_list[cache_depth_to_clear - 1] == 'activatebol':
 260 |     cache_depth_to_clear -= 1
 261 |     cache_depth -= 1
 262 | if module_list[cache_depth_to_clear - 1] == 'activatepl':
 263 |     cache_depth_to_clear -= 1
 264 | if module_list[cache_depth_to_clear - 1] == 'activategtorus':
 265 |     cache_depth_to_clear -= 1
 266 | cache_max = int(os.environ.get('CACHE_MAX', '10000'))
 267 | cache_print = os.environ.get('CACHE_VERBOSE', '0') == '1'
 268 | if cache_print:
 269 |     print("Caching modules:", module_list[:cache_depth])
 270 |     print("Caching SEDs:", module_list[:cache_depth_to_clear], "with %d entries" % cache_max)
 271 | analysis_module = get_analysis_module(config.configuration[
 272 |     'analysis_method'])
 273 | analysis_module_params = config.configuration['analysis_method_params']
 274 | 
 275 | analysed_variables = analysis_module_params["analysed_variables"]
 276 | n_variables = len(analysed_variables)
 277 | lim_flag = analysis_module_params["lim_flag"].lower() == "true"
 278 | mock_flag = analysis_module_params["mock_flag"].lower() == "true"
 279 | 
 280 | 
 281 | # get scaling parameters limits
 282 | scaling_limits = config.config['scaling_limits']
 283 | mass_min = float(scaling_limits['mass_min'])
 284 | mass_max = float(scaling_limits['mass_max'])
 285 | sfr_min = float(scaling_limits['sfr_min'])
 286 | sfr_max = float(scaling_limits['sfr_max'])
 287 | L_min = float(scaling_limits['L_min'])
 288 | L_max = float(scaling_limits['L_max'])
 289 | 
 290 | # get statistics configuration
 291 | statistics_config = config.config['statistics']
 292 | exponent = int(statistics_config.get('exponent', '2'))
 293 | with_attenuation_model_uncertainty = statistics_config.get('attenuation_model_uncertainty', 'false').lower() == 'true'
 294 | with_Ly_break_uncertainty = statistics_config.get('Ly_break_uncertainty', 'false').lower() == 'true'
 295 | variability_uncertainty = statistics_config.get('variability_uncertainty', 'true').lower() == 'true'
 296 | systematics_width = float(statistics_config.get('systematics_width', '0.01'))
 297 | 
 298 | gbl_warehouse = SedWarehouse(store_depth=cache_depth, reusable=('activategtorus', 'activatepl', 'activatebol', 'biattenuation'))
 299 | 
 300 | def list_filters():
 301 |     """List all known filters."""
 302 |     with Database() as base:
 303 |         keys, _ = base.get_filter_list()
 304 |         for filter_name in keys:
 305 |             f = base.get_filter(filter_name)
 306 |             print(f)
 307 | 
 308 | def get_filtergroup_name(filtername):
 309 |     if '_' in filtername:
 310 |         # take off band if _z or _Ks for example
 311 |         instrument_name = '_'.join(filtername.split('_')[:-1])
 312 |     else:
 313 |         # take off digit at the end, like WISE3 for example
 314 |         instrument_name = ''.join(character for character in filtername
 315 |             if not character.isdigit())
 316 |     if instrument_name != '':
 317 |         return instrument_name
 318 |     else:
 319 |         return filtername
 320 | 
 321 | # get list of user-selected filters
 322 | filters = [name for name in column_list if not name.endswith('_err')]
 323 | with Database() as base:
 324 |     filters_wl_orig = np.array([base.get_filter(name.rstrip('_')).effective_wavelength for name in filters])
 325 |     filters_colors = []
 326 |     default_color_cycle = itertools.cycle(plt.rcParams['axes.prop_cycle'].by_key()['color'])
 327 |     filtergroups_colors = {}
 328 |     for filtername in filters:
 329 |         filtergroup_name = get_filtergroup_name(filtername)
 330 |         if filtergroup_name not in filtergroups_colors:
 331 |             filtergroups_colors[filtergroup_name] = next(default_color_cycle)
 332 |         filters_colors.append(filtergroups_colors[filtergroup_name])
 333 |         del filtername, filtergroup_name
 334 |     del default_color_cycle
 335 | n_filters = len(filters)
 336 | 
 337 | # show chosen parameters to the user, in latex file and screen
 338 | latex_table = open('pcigale.ini.tex', 'w')
 339 | latex_table.write(r'  Parameter & Description & Values \\' + "\n")
 340 | latex_table.write(r'  \hline' + "\n")
 341 | latex_table.write(r'  \hline' + "\n")
 342 | latex_table.write(r'  Galaxy components: & & \\' + "\n")
 343 | latex_table.write(r'  \texttt{stellar\_mass} & & log-uniform between $10^{%d}$ and $10^{%d} M_\odot$ \\' % (mass_min, mass_max) + "\n")
 344 | param_names = []
 345 | is_log_param = []
 346 | print()
 347 | print("Parameters")
 348 | print("----------")
 349 | for module_name, module_parameters in zip(module_list, parameter_list):
 350 |     print("  [%s]" % module_name)
 351 |     latex_table.write("  \\texttt{[%s]} & & \\\\\n" % module_name)
 352 |     module = import_module("." + module_name, 'pcigale.creation_modules')
 353 |     for k, v in module_parameters.items():
 354 |         description = module.Module.parameter_list[k][1].split('.')[0]
 355 |         if len(v) > 1:
 356 |             if min(v) > 0 and max(v) > 0 and max(v) / min(v) > 40:
 357 |                 is_log_param.append(True)
 358 |                 param_names.append("log_%s_%s" % (module_name, k))
 359 |             else:
 360 |                 is_log_param.append(False)
 361 |                 param_names.append("%s.%s" % (module_name, k))
 362 |             print("    %20s : %s" % (k, v), '(log-uniform)' if is_log_param[-1] else '(uniform)')
 363 |             latex_table.write("  \\texttt{%s} & %s & %s %s \\\\\n" % (
 364 |                 k.replace('_', '\\_'), description.replace('&', '\\&').replace('_', '\\_'),
 365 |                 str(v).replace('[', '').replace(']', ''),
 366 |                 '(log-uniform)' if is_log_param[-1] else '(uniform)')
 367 |             )
 368 |         else:
 369 |             print("    %20s = %s" % (k, v[0]))
 370 |             latex_table.write("  \\texttt{%s} & %s & %s %s \\\\\n" % (
 371 |                 k.replace('_', '\\_'), description.replace('&', '\\&').replace('_', '\\_'),
 372 |                 v[0], '(fixed)'
 373 |             ))
 374 |         del k, v
 375 | 
 376 |     del module_name, module_parameters
 377 |     latex_table.write(r'  \hline' + "\n")
 378 | print()
 379 | latex_table.write(r'  \hline' + "\n")
 380 | latex_table.close()
 381 | param_names.append("log_stellar_mass")
 382 | param_names.append("log_L_AGN")
 383 | param_names.append("redshift")
 384 | param_names.append("systematics")
 385 | print('%d free model parameters' % len(param_names))
 386 | rv_systematics = ExponentialDist(scale=systematics_width)
 387 | 
 388 | print("Statistics")
 389 | print("----------")
 390 | print(" Exponent: %d (%s)" % (exponent, {1: 'L1', 2: 'Gaussian'}[exponent]))
 391 | print(" model uncertainty: ")
 392 | print("    white: exponential, scale=%s" % systematics_width)
 393 | print("    attenuation: %s" % with_attenuation_model_uncertainty)
 394 | print("    variability: %s" % variability_uncertainty)
 395 | print()
 396 | print("Cosmology:", cosmology)
 397 | print()
 398 | 
 399 | def make_parameter_list(parameters):
 400 |     """Make a parameter list given a array of values, which may be in log."""
 401 |     parameter_list_first = []
 402 |     i = 0
 403 |     for module_parameters in parameter_list:
 404 |         parameter_list_here = {}
 405 |         for k, v in module_parameters.items():
 406 |             if len(v) == 1:
 407 |                 parameter_list_here[k] = v[0]
 408 |             else:
 409 |                 if is_log_param[i]:
 410 |                     parameter_list_here[k] = 10**(parameters[i])
 411 |                 else:
 412 |                     parameter_list_here[k] = parameters[i]
 413 |                 i += 1
 414 | 
 415 |         parameter_list_first.append(parameter_list_here)
 416 |     return parameter_list_first
 417 | 
 418 | 
 419 | def compute_model_fluxes(sed, filters):
 420 |     """Compute fluxes and derived properties.
 421 | 
 422 |     Returns -99 values if the setup is unphysical (star formation before the age of the Universe).
 423 |     """
 424 |     if 'sfh.age' in sed.info and sed.info['sfh.age'] > sed.info['universe.age']:
 425 |         model_fluxes = -99. * np.ones(len(filters))
 426 |         model_variables = -99. * np.ones(len(analysed_variables))
 427 |     else:
 428 |         model_fluxes = np.array([sed.compute_fnu(filter_.rstrip('_')) for filter_ in filters])
 429 |         model_variables = np.array([sed.info[name] for name in analysed_variables])
 430 | 
 431 |     return model_fluxes, model_variables
 432 | 
 433 | 
 434 | def scale_sed_components(module_list, parameter_list_here, stellar_mass, L_AGN):
 435 |     """Create the hybrid galaxy & AGN SED.
 436 | 
 437 |     Runs the dual pipelines,
 438 |      - one for AGN components with mock galaxy properties
 439 |      - one for galaxy components with mock galaxy properties
 440 |     Then combines the result, with the SED scaled by stellar mass or AGN luminosity.
 441 | 
 442 |     Returns the scaled SED, and the unscaled galaxy and AGN SED
 443 |     """
 444 |     # get sed for galaxy
 445 |     parameter_list_gal = []
 446 |     parameter_list_agn = []
 447 |     for i, (module_name, module_parameters_available, module_parameters_selected) in enumerate(zip(module_list, parameter_list, parameter_list_here)):
 448 |         parameter_list_gal_here = {}
 449 |         parameter_list_agn_here = {}
 450 |         for k in sorted(module_parameters_available.keys()):
 451 |             selected_value = module_parameters_selected[k]
 452 |             mock_value = module_parameters_available[k][0]
 453 |             if i >= cache_depth:
 454 |                 # use the true value in both cases, because:
 455 |                 # module applies to both and is not cached
 456 |                 parameter_list_gal_here[k] = selected_value
 457 |                 parameter_list_agn_here[k] = selected_value
 458 |             elif 'activate' in module_name or 'AGN' in k:
 459 |                 # mock the value for galaxy sed
 460 |                 parameter_list_gal_here[k] = mock_value
 461 |                 parameter_list_agn_here[k] = selected_value
 462 |             else:
 463 |                 # mock the value for AGN sed
 464 |                 parameter_list_gal_here[k] = selected_value
 465 |                 parameter_list_agn_here[k] = mock_value
 466 |             del k
 467 |         parameter_list_gal.append(parameter_list_gal_here)
 468 |         parameter_list_agn.append(parameter_list_agn_here)
 469 | 
 470 |     # this clears the cache, if we are in danger of running out of memory (CACHE_MAX environment variable)
 471 |     if len(gbl_warehouse.storage.dictionary) > cache_max:
 472 |         if cache_print:
 473 |             sys.stderr.write("clearing cache (%d objects) ..." % len(gbl_warehouse.storage.dictionary))
 474 |         if np.random.uniform() < 0.01:
 475 |             # clear cache completely, occasionally, for speed
 476 |             gbl_warehouse.partial_clear_cache(0)
 477 |         else:
 478 |             gbl_warehouse.partial_clear_cache(cache_depth_to_clear)
 479 |         if cache_print:
 480 |             sys.stderr.write("cleared, %d remain.  \n" % len(gbl_warehouse.storage.dictionary))
 481 | 
 482 |     # compute galaxy and AGN SEDs, un-normalised
 483 |     sed = gbl_warehouse.get_sed(module_list[:cache_depth], parameter_list_gal[:cache_depth])
 484 |     agn_sed = gbl_warehouse.get_sed(module_list[:cache_depth], parameter_list_agn[:cache_depth]).copy()
 485 |     assert sed.contribution_names == agn_sed.contribution_names, (sed.contribution_names, agn_sed.contribution_names)
 486 | 
 487 |     # select AGN components
 488 |     agn_mask = np.array(['activate' in name for name in sed.contribution_names])
 489 | 
 490 |     # scale the AGN and galactic components as needed
 491 |     scaled_sed = sed.copy()
 492 |     scaled_sed.luminosities[~agn_mask] *= stellar_mass
 493 |     sed.luminosities[~agn_mask] *= stellar_mass
 494 |     sed.luminosities[agn_mask] *= 0.0
 495 |     assert sed.info['sfh.sfr'] > 0, sed.info['sfh.sfr']
 496 |     assert stellar_mass > 0, stellar_mass
 497 |     scaled_sed.info.update({k: v * stellar_mass for k, v in sed.info.items() if k in sed.mass_proportional_info and not ('activate' in k or 'agn' in k)})
 498 |     # convert from erg/s to W, the luminosity unit of cigale, with 1e7
 499 |     agn_sed.luminosities[agn_mask, :] *= L_AGN / 1e7
 500 |     agn_sed.luminosities[~agn_mask] *= 0.0
 501 |     scaled_sed.luminosities[agn_mask] = agn_sed.luminosities[agn_mask]
 502 |     agn_sed.luminosity = agn_sed.luminosities[agn_mask, :].sum(axis=0)
 503 |     # copy over AGN meta data
 504 |     scaled_sed.info.update({k: v * (L_AGN / 1e7 if 'agn.lum' in k else 1) for k, v in agn_sed.info.items() if 'activate' in k or 'agn' in k})
 505 |     agn_sed.info.update({k: v * (L_AGN / 1e7 if 'agn.lum' in k else 1) for k, v in agn_sed.info.items() if 'activate' in k or 'agn' in k})
 506 |     scaled_sed.luminosity = scaled_sed.luminosities.sum(0)
 507 | 
 508 |     # apply the remaining modules (post-caching)
 509 |     for module_name, module_parameters in zip(module_list[cache_depth:], parameter_list_here[cache_depth:]):
 510 |         module_instance = gbl_warehouse.get_module_cached(module_name, **module_parameters)
 511 |         module_instance.process(scaled_sed)
 512 | 
 513 |     return scaled_sed, sed, agn_sed
 514 | 
 515 | 
 516 | # Wavelength limits (restframe) when plotting the best SED.
 517 | PLOT_L_MIN = 0.1
 518 | PLOT_L_MAX = 50
 519 | 
 520 | 
 521 | def plot_posteriors(filename, prior_samples, param_names, samples):
 522 |     """Make plot of parameter posteriors compared to prior."""
 523 |     print("plotting posteriors ...")
 524 |     plt.figure(figsize=(12, 8))
 525 |     for i, (param_name, samples) in enumerate(zip(param_names, samples.transpose())):
 526 |         ax = plt.subplot(4, len(param_names) // 4 + 1, i + 1)
 527 |         bins = np.unique(list(set(prior_samples[:, i]).union(set(samples))))
 528 |         if not np.isfinite(bins).all():
 529 |             print("WARNING: parameter %s is bad, removing it from the analysis list" % param_name, bins[~np.isfinite(bins)])
 530 |         if len(bins) > 2 and bins[-1] > bins[-2]:
 531 |             bins = np.concatenate((bins, [bins[-1] + bins[-1] - bins[-2]]))
 532 |         elif len(bins) == 1:
 533 |             bins = [bins[0] - 0.02, bins[0], bins[0] + 0.02]
 534 |         if len(bins) > 40:
 535 |             bins = 20
 536 |         with np.errstate(invalid='ignore', divide='ignore'):
 537 |             plt.hist(samples, histtype='step', density=True, bins=bins, lw=2, alpha=0.5)
 538 |         xlo, xhi = plt.xlim()
 539 |         with np.errstate(invalid='ignore', divide='ignore'):
 540 |             plt.hist(
 541 |                 prior_samples[:, i], histtype='step',
 542 |                 density=True, bins=bins, color='gray', ls='-', lw=1, alpha=0.5)
 543 |         plt.xlim(xlo, xhi)
 544 |         plt.yticks([])
 545 |         ax.spines[['right', 'top', 'left']].set_visible(False)
 546 |         ax.get_xaxis().tick_bottom()
 547 |         plt.xlabel(('' if i % 2 == 0 else "\n") + param_name)
 548 |         xticks, _ = plt.xticks()
 549 |         if param_name.startswith('log_') and xhi < 5:
 550 |             if np.allclose(xticks, xticks.astype(int)):
 551 |                 plt.xticks(xticks, ['%g' % (10**xi) for xi in xticks])
 552 |                 plt.xlabel(('' if i % 2 == 0 else "\n") + param_name[4:])
 553 |             elif xhi - xlo < 2:
 554 |                 xspan = xhi - xlo
 555 |                 sigfig = int(np.ceil(xspan))
 556 |                 fmt = '%%.%df' % sigfig
 557 |                 xticklo = np.round(10**(xlo + 0.2 * xspan), sigfig)
 558 |                 xtickmid = np.round(10**(xlo + 0.5 * xspan), sigfig)
 559 |                 xtickhi = np.round(10**(xlo + 0.8 * xspan), sigfig)
 560 |                 if xticklo == 0:
 561 |                     xticklo = xtickmid / 100.0
 562 |                 with np.errstate(divide='ignore'):
 563 |                     plt.xticks(
 564 |                         [np.log10(xticklo), np.log10(xtickmid), np.log10(xtickhi)],
 565 |                         [fmt % xticklo, fmt % xtickmid, fmt % xtickhi])
 566 |                 plt.xlabel(('' if i % 2 == 0 else "\n") + param_name[4:])
 567 |         plt.xlim(xlo, xhi)
 568 | 
 569 |     plt.subplots_adjust(wspace=0.1, hspace=0.7)
 570 |     plt.savefig(filename, bbox_inches='tight')
 571 |     plt.close()
 572 | 
 573 | 
 574 | def _with_attenuation(keys):
 575 |     return keys + ['attenuation.' + key for key in keys]
 576 | 
 577 | # Plasma color scheme:
 578 | #plot_colors0 = plt.cm.plasma(np.linspace(0, 1, 10))
 579 | #plot_colors = plot_colors0[0], plot_colors0[2], plot_colors0[4], plot_colors0[-4], plot_colors0[-2], plot_colors0[-1]
 580 | # color brewer color schemes:
 581 | #plot_colors = '#c51b7d', '#e9a3c9', '#fde0ef', '#e6f5d0', '#a1d76a', '#4d9221'
 582 | #plot_colors = '#762a83', '#af8dc3', '#e7d4e8', '#d9f0d3', '#7fbf7b', '#1b7837'
 583 | # my color scheme:
 584 | plot_colors = ['#008fd5', '#fc4f30', '#e5ae38', '#6d904f', '#810f7c', '#8b8b8b']
 585 | 
 586 | # groups of SED contributions to include in the fit
 587 | plot_elements = [
 588 |     # if you also want the unattenuated emission:
 589 |     dict(keys=_with_attenuation(['agn.activate_Disk']),
 590 |          label="AGN disk", color=plot_colors[0], marker=None, linestyle='-', linewidth=1.5),
 591 |     dict(keys=_with_attenuation(['agn.activate_Torus', 'agn.activate_TorusSi']),
 592 |          label="AGN torus", color=plot_colors[2], marker=None, linestyle='-', linewidth=1.5),
 593 |     dict(keys=_with_attenuation(['agn.activate_EmLines_BL', 'agn.activate_EmLines_NL', 'agn.activate_FeLines', 'agn.activate_BC', 'agn.activate_EmLines_LINER']),
 594 |          label="AGN lines", color=plot_colors[1], marker=None, linestyle='-', linewidth=0.5, shading=False),
 595 |     dict(keys=_with_attenuation(['stellar.young', 'stellar.old']),
 596 |          label="Stellar (attenuated)", color=plot_colors[-2], marker=None, linestyle='-',),
 597 |     dict(keys=_with_attenuation(['nebular.lines_young', 'nebular.lines_old', 'nebular.continuum_young', 'nebular.continuum_old']),
 598 |          label="Nebular emission", color=plot_colors[-1], marker=None, linewidth=0.5, shading=False),
 599 |     dict(keys=['dust'], label="Dust", color=plot_colors[-3], marker=None, linestyle='-', linewidth=1),
 600 | ]
 601 | if os.environ.get('WITH_UNATTENUATED_COMPONENTS', '0') == '1':
 602 |     plot_elements += [
 603 |         dict(keys=['stellar.young', 'stellar.old'],
 604 |              label="Stellar (unattenuated)", color='orange', marker=None, linestyle=':',),
 605 |         # if you also want the unattenuated disk:
 606 |         dict(keys=['agn.activate_Disk'],
 607 |              label="AGN disk (unattenuated)", color=plot_colors[-2], marker=None, linestyle=':', linewidth=1.5),
 608 |     ]
 609 | 
 610 | def plot_results(sampler, prior_samples, obs, obs_fluxes, obs_errors, wobs, cache_filters, replot):
 611 |     """Make all the plots."""
 612 | 
 613 |     # only allow the main process to plot
 614 |     if not sampler.log:
 615 |         return
 616 | 
 617 |     results = sampler.results
 618 |     Z = results['logz']
 619 |     plot_dir = sampler.logs['plots']
 620 |     assert np.isfinite(results['samples']).all()
 621 | 
 622 |     # avoid replotting if nothing changed and all the files are there
 623 |     if not replot and all((os.path.exists('%s/%s.pdf' % (plot_dir, f)) for f in ('posteriors', 'derived', 'sed_mJy', 'sed_lum'))):
 624 |         print("not replotting.")
 625 |         return
 626 | 
 627 |     plot_posteriors('%s/posteriors.pdf' % plot_dir, prior_samples, param_names, results['samples'])
 628 | 
 629 |     if os.environ.get("PLOT_TRACE", "0") == "1":
 630 |         print("making trace plot ...")
 631 |         sampler.plot_trace()
 632 |     if os.environ.get("PLOT_CORNER", "0") == "1":
 633 |         print("making corner plot ...")
 634 |         import getdist, getdist.plots, logging
 635 |         smooth_samples = sampler.results['samples'].copy()
 636 |         for i in range(len(param_names)):
 637 |             bins = np.unique(prior_samples[:, i]).tolist()
 638 |             if len(bins) < 40:
 639 |                 if len(bins) == 1:
 640 |                     db = 1
 641 |                 else:
 642 |                     db = (bins[-1] - bins[-2])
 643 |                 for lo, hi in zip(bins, bins[1:] + [bins[-1] + db]):
 644 |                     mask = smooth_samples[:, i] == lo
 645 |                     smooth_samples[mask, i] = np.random.uniform(lo, hi, size=mask.sum())
 646 |                     #mask2 = prior_samples[:, i] == lo
 647 | 
 648 |         samples = getdist.MCSamples(
 649 |             samples=smooth_samples, names=param_names, sampler='nested',
 650 |             settings=dict(smooth_scale_2D=0.3, smooth_scale_1D=0.3))
 651 |         g = getdist.plots.get_subplot_plotter()
 652 |         g.triangle_plot([samples])
 653 |         for handler in logging.root.handlers[:]:
 654 |             logging.root.removeHandler(handler)
 655 |         plt.savefig('%s/corner.pdf' % plot_dir)
 656 |         plt.close()
 657 | 
 658 |     print("making SED instances for plotting ...")
 659 |     bands = {'lum': {}, 'mJy': {}}
 660 | 
 661 |     z = obs['redshift']
 662 |     chi2_best = 1e300
 663 | 
 664 |     posteriors_names = analysed_variables + ['chi2']
 665 |     logmask = np.array(['lum' in v or 'sfr' in v or 'age' in v for v in analysed_variables])
 666 |     def logfunc(x):
 667 |         with np.errstate(divide='ignore', invalid='ignore'):
 668 |             return np.where(logmask, np.log10(x), x)
 669 |     stellar_mass_column = []
 670 |     posteriors = []
 671 |     all_mod_fluxes = []
 672 |     agn_mod_fluxes = []
 673 |     gal_mod_fluxes = []
 674 |     obs_filter_wavelength = filters_wl_orig[wobs]
 675 |     sfhs = []
 676 | 
 677 |     for parameters in tqdm.tqdm(sampler.results['samples'][:args.num_posterior_samples, :]):
 678 |         stellar_mass = 10**parameters[-4]
 679 |         L_AGN = 10**parameters[-3]
 680 |         redshift = parameters[-2]
 681 |         sys_error = parameters[-1]
 682 |         parameter_list_here = make_parameter_list(parameters)
 683 |         parameter_list_here[-1] = dict(redshift=redshift)
 684 | 
 685 |         sed, gal_sed, agn_sed = scale_sed_components(module_list, parameter_list_here, stellar_mass, L_AGN)
 686 |         sed.cache_filters = cache_filters
 687 |         sfhs.append(gal_sed.sfh)
 688 |         agn_sed.cache_filters = cache_filters
 689 |         gal_sed.cache_filters = cache_filters
 690 | 
 691 |         model_fluxes_full, model_variables = compute_model_fluxes(sed, filters)
 692 |         for module_name, module_parameters in zip(module_list[cache_depth:], parameter_list_here[cache_depth:]):
 693 |             module_instance = gbl_warehouse.get_module_cached(module_name, **module_parameters)
 694 |             module_instance.process(agn_sed)
 695 | 
 696 |         agn_model_fluxes_full, _ = compute_model_fluxes(agn_sed, filters)
 697 |         agn_model_fluxes = agn_model_fluxes_full[wobs]
 698 | 
 699 |         mod_fluxes = model_fluxes_full[wobs]
 700 |         all_mod_fluxes.append(model_fluxes_full)
 701 |         agn_mod_fluxes.append(agn_model_fluxes_full)
 702 |         gal_mod_fluxes.append(model_fluxes_full - agn_model_fluxes_full)
 703 | 
 704 |         filter_wl_indices = np.searchsorted(sed.wavelength_grid, filters_wl_orig[wobs])
 705 |         filter_contrib = sed.luminosities[:, filter_wl_indices]
 706 |         filter_pos_contrib = np.where(filter_contrib > 0, filter_contrib, 0).sum(axis=0)
 707 |         transmitted_fraction = filter_contrib.sum(axis=0) / filter_pos_contrib
 708 | 
 709 |         _, chi2_0, _ = chi2_with_norm(
 710 |             mod_fluxes, agn_model_fluxes*0, obs_fluxes, obs_errors, obs_filter_wavelength * np.inf, redshift, sys_error * 0,
 711 |             NEV=sed.info.get('agn.NEV'), exponent=exponent, transmitted_fraction=transmitted_fraction * 0 + 1)
 712 |         chi2_best = min(chi2_0, chi2_best)
 713 |         norm, chi2_, total_variance = chi2_with_norm(
 714 |             mod_fluxes, agn_model_fluxes, obs_fluxes, obs_errors, obs_filter_wavelength, redshift, sys_error,
 715 |             NEV=sed.info.get('agn.NEV'), exponent=exponent, transmitted_fraction=transmitted_fraction)
 716 |         posteriors.append(np.concatenate((logfunc(model_variables), [chi2_])))
 717 |         stellar_mass_column.append(stellar_mass)
 718 | 
 719 |         wavelength_spec = sed.wavelength_grid
 720 |         DL = sed.info['universe.luminosity_distance']
 721 | 
 722 |         for sed_type in 'mJy', 'lum':
 723 |             wavelength_spec2 = wavelength_spec.copy()
 724 |             if sed_type == 'lum':
 725 |                 sed_multiplier = wavelength_spec2.copy() * (redshift + 1)
 726 |                 wavelength_spec2 /= 1. + redshift
 727 |             elif sed_type == 'mJy':
 728 |                 sed_multiplier = (wavelength_spec2 * 1e29 /
 729 |                                    (c / (wavelength_spec2 * 1e-9)) /
 730 |                                    (4. * np.pi * DL * DL))
 731 | 
 732 |             assert (sed_multiplier >= 0).all(), (stellar_mass, DL, wavelength_spec2)
 733 |             wavelength_spec2 /= 1000
 734 | 
 735 |             for j, plot_element in enumerate(plot_elements):
 736 |                 keys = plot_element['keys']
 737 |                 if not any(k in sed.contribution_names for k in keys):
 738 |                     continue
 739 |                 if j not in bands[sed_type]:
 740 |                     # print("  building", sed_type, plot_element['label'])
 741 |                     bands[sed_type][j] = PredictionBand(wavelength_spec2)
 742 |                 pred = sum(sed.get_lumin_contribution(k) * sed_multiplier for k in keys if k in sed.contribution_names)
 743 |                 assert np.isfinite(pred).all(), pred
 744 |                 assert bands[sed_type][j].x.shape == pred.shape, (bands[sed_type][j].x.shape, pred.shape)
 745 |                 # print(plot_element['label'], pred)
 746 |                 bands[sed_type][j].add(pred)
 747 |             if 'total' not in bands[sed_type]:
 748 |                 bands[sed_type]['total'] = PredictionBand(wavelength_spec2)
 749 |             bands[sed_type]['total'].add((sed.luminosity * sed_multiplier))
 750 | 
 751 |     posteriors = np.array(posteriors)
 752 |     posterior_summary_text = ''
 753 |     if 'log_stellar_mass' in param_names:
 754 |         posterior_summary_text += r'''
 755 | $m_\star$=
 756 | %.1f
 757 | $_{\pm%.1f}$
 758 | ''' % (
 759 |             np.log10(10**results['samples'][:,param_names.index('log_stellar_mass')].mean()),
 760 |             results['samples'][:,param_names.index('log_stellar_mass')].std(),)
 761 |     if 'sfh.sfr' in posteriors_names:
 762 |         posterior_summary_text += r'''
 763 | sfr=
 764 | %.1f
 765 | $_{\pm%.1f}$
 766 | ''' % (
 767 |             np.log10(10**posteriors[:,posteriors_names.index('sfh.sfr')].mean()),
 768 |             posteriors[:,posteriors_names.index('sfh.sfr')].std(),)
 769 |     if 'log_biattenuation_E(B-V)' in param_names:
 770 |         posterior_summary_text += r'''
 771 | E$_\mathrm{B-V}^\mathrm{gal}$=
 772 | %.1f$_{\pm%.1f}$
 773 | ''' % (
 774 |             (10**results['samples'][:,param_names.index('log_biattenuation_E(B-V)')]).mean(),
 775 |             (10**results['samples'][:,param_names.index('log_biattenuation_E(B-V)')]).std(),)
 776 |     elif 'biattenuation.E(B-V)' in param_names:
 777 |         posterior_summary_text += r'''
 778 | E$_\mathrm{B-V}^\mathrm{gal}$=
 779 | %.1f$_{\pm%.1f}$
 780 | ''' % (
 781 |             (results['samples'][:,param_names.index('biattenuation.E(B-V)')]).mean(),
 782 |             (results['samples'][:,param_names.index('biattenuation.E(B-V)')]).std(),)
 783 | 
 784 |     if 'log_L_AGN' in param_names:
 785 |         posterior_summary_text += r'''
 786 | $l_\mathrm{AGN}$=
 787 | %.1f
 788 | $_{\pm%.1f}$
 789 | ''' % (
 790 |             np.log10(10**results['samples'][:,param_names.index('log_L_AGN')]).mean(),
 791 |             results['samples'][:,param_names.index('log_L_AGN')].std(),)
 792 | 
 793 |     if 'log_biattenuation_E(B-V)-AGN' in param_names:
 794 |         posterior_summary_text += r'''
 795 | E$_\mathrm{B-V}^\mathrm{AGN}$=
 796 | %.1f$_{\pm%.1f}$''' % (
 797 |             (10**results['samples'][:,param_names.index('log_biattenuation_E(B-V)-AGN')]).mean(),
 798 |             (10**results['samples'][:,param_names.index('log_biattenuation_E(B-V)-AGN')]).std(),)
 799 |     elif 'biattenuation.E(B-V)-AGN' in param_names:
 800 |         posterior_summary_text += r'''
 801 | E$_\mathrm{B-V}^\mathrm{AGN}$=
 802 | %.1f$_{\pm%.1f}$''' % (
 803 |             (results['samples'][:,param_names.index('biattenuation.E(B-V)-AGN')]).mean(),
 804 |             (results['samples'][:,param_names.index('biattenuation.E(B-V)-AGN')]).std(),)
 805 |     
 806 |     if posterior_summary_text == '':
 807 |         print(posteriors_names)
 808 |         posterior_summary_text = 'N/A'
 809 | 
 810 |     # add specific (normalised by stellar mass) AGN luminosities
 811 |     for i, n in enumerate(list(posteriors_names)):
 812 |         if 'agn.lum' in n or 'Lbol' in n or 'sfh.sfr' in n:
 813 |             posteriors = np.hstack((posteriors, (posteriors[:,i] - np.log10(stellar_mass_column)).reshape((-1,1))))
 814 |             posteriors_names.append('s_' + n)
 815 |     # print("   +specific:", len(posteriors_names), posteriors_names)
 816 |     plot_posteriors('%s/derived.pdf' % plot_dir, np.zeros((0, len(posteriors_names))), posteriors_names, posteriors)
 817 |     # add model fluxes as output columns
 818 |     mod_fluxes = np.mean(all_mod_fluxes, axis=0)
 819 |     posteriors = np.hstack((posteriors, all_mod_fluxes, agn_mod_fluxes, gal_mod_fluxes))
 820 |     posteriors_names += ['totalflux_' + filtername for filtername in filters]
 821 |     posteriors_names += ['AGNflux_' + filtername for filtername in filters]
 822 |     posteriors_names += ['GALflux_' + filtername for filtername in filters]
 823 |     # print("   +fluxes:", len(posteriors_names), posteriors_names)
 824 |     np.savetxt('%s/derived.csv' % plot_dir, posteriors, header=','.join(posteriors_names), comments='', delimiter=',')
 825 | 
 826 |     for sed_type in 'mJy', 'lum':
 827 |         print("  plotting", sed_type, ', writing out as CSV file ...')
 828 |         filters_wl = filters_wl_orig / 1000
 829 |         # wsed = np.where((wavelength_spec2 > xmin) & (wavelength_spec2 < xmax))
 830 | 
 831 |         figure = plt.figure()
 832 |         gs = gridspec.GridSpec(3, 1, height_ratios=[0.2, 8, 2], hspace=0.0)
 833 | 
 834 |         ax1 = plt.subplot(gs[1])
 835 |         ax2 = plt.subplot(gs[2])
 836 |         ax3 = plt.subplot(gs[0])
 837 | 
 838 |         plt.sca(ax1)
 839 | 
 840 |         header = ['wavelength']
 841 |         seddata = [bands[sed_type]['total'].x]
 842 |         for j, plot_element in enumerate(plot_elements):
 843 |             if j not in bands[sed_type]:
 844 |                 continue
 845 |             k = plot_element['label']
 846 |             header += [k, k + '_errup', k + '_errlo']
 847 |             mid, up, lo = np.quantile(bands[sed_type][j].ys, [0.5, 0.5 + 0.341, 0.5 - 0.341], axis=0)
 848 |             assert mid.shape == bands[sed_type][j].x.shape
 849 |             line_kwargs = dict(plot_element)
 850 |             plot_shading = line_kwargs.pop('shading', True)
 851 |             del line_kwargs['keys']
 852 |             plt.plot(bands[sed_type][j].x, mid, **line_kwargs)
 853 |             if plot_shading:
 854 |                 plt.fill_between(bands[sed_type][j].x, lo, up, facecolor=plot_element['color'], edgecolor='none', alpha=0.2)
 855 |             seddata += [mid, up, lo]
 856 |             del j, plot_element
 857 |         k = 'total'
 858 |         header += [k, k + '_errup', k + '_errlo']
 859 |         mid, up, lo = np.quantile(bands[sed_type][k].ys, [0.5, 0.5 + 0.341, 0.5 - 0.341], axis=0)
 860 |         assert mid.shape == bands[sed_type][k].x.shape
 861 |         seddata += [mid, up, lo]
 862 |         np.savetxt('%s/sed_%s.csv.gz' % (plot_dir, sed_type), np.transpose(seddata), header=','.join(header), comments='', delimiter=',')
 863 |         plt.plot(bands[sed_type]['total'].x, mid, color='k', label='Model spectrum', linewidth=1.5)
 864 |         plt.fill_between(bands[sed_type]['total'].x, lo, up, color='k', alpha=0.2)
 865 | 
 866 |         if sed_type == 'lum':
 867 |             # shift back from observed-frame to rest-frame units:
 868 |             xmin = PLOT_L_MIN / (1 + z)
 869 |             xmax = PLOT_L_MAX / (1 + z)
 870 | 
 871 |             filters_wl /= 1. + z
 872 |             k_corr_SED = 1e-29 * (4. * np.pi * DL * DL) * c / (filters_wl * 1e-9) / 1000
 873 |             obs_fluxes = obs_fluxes * k_corr_SED[wobs]
 874 |             obs_fluxes_err = obs_errors * k_corr_SED[wobs]
 875 |             mod_fluxes = mod_fluxes * k_corr_SED
 876 |             mod_fluxes_err = total_variance**0.5 * k_corr_SED[wobs]
 877 |         elif sed_type == 'mJy':
 878 |             xmin = PLOT_L_MIN
 879 |             xmax = PLOT_L_MAX
 880 | 
 881 |             k_corr_SED = np.ones_like(filters_wl)
 882 |             obs_fluxes_err = obs_errors
 883 |             mod_fluxes_err = total_variance**0.5
 884 | 
 885 |         ax1.set_autoscale_on(False)
 886 |         ax1.scatter(filters_wl, mod_fluxes, marker='o', color='r', s=8,
 887 |                     zorder=3, label="Model fluxes")
 888 |         mask_ok = np.logical_and(obs_fluxes > 0., obs_errors > 0.)
 889 |         ax1.errorbar(filters_wl[wobs][mask_ok], obs_fluxes[mask_ok],
 890 |                      yerr=mod_fluxes_err[mask_ok] * 3, ls='',
 891 |                      markersize=6, color='b', capsize=2., elinewidth=1)
 892 |         ax1.errorbar(filters_wl[wobs][mask_ok], obs_fluxes[mask_ok],
 893 |                      yerr=obs_fluxes_err[mask_ok] * 3, ls='', marker='s',
 894 |                      label='Observed fluxes', markerfacecolor='None',
 895 |                      markersize=6, color='b', capsize=4., elinewidth=1)
 896 |         mask_uplim = np.logical_and(np.logical_and(obs_fluxes > 0.,
 897 |                                                obs_fluxes_err < 0.),
 898 |                                 obs_fluxes_err > -9990. * k_corr_SED[wobs])
 899 | 
 900 |         if not mask_uplim.any() == False:
 901 |             ax1.errorbar(x=filters_wl[wobs][mask_uplim], y=obs_fluxes[mask_uplim],
 902 |                          yerr=np.abs(obs_fluxes_err[mask_uplim]) * 3,
 903 |                          marker='v', label='Observed upper limits',
 904 |                          markerfacecolor='None', markersize=6,
 905 |                          color='navy',
 906 |                          capsize=4, linestyle=' ', elinewidth=1)
 907 |             ax1.errorbar(x=filters_wl[wobs][mask_uplim], y=obs_fluxes[mask_uplim],
 908 |                          yerr=np.abs(mod_fluxes_err[mask_uplim]) * 3,
 909 |                          marker=' ', color='navy',
 910 |                          capsize=2, linestyle=' ', elinewidth=1)
 911 |         mask_noerr = np.logical_and(obs_fluxes > 0.,
 912 |                                     obs_fluxes_err < -9990. * k_corr_SED[wobs])
 913 |         if not mask_noerr.any() == False:
 914 |             ax1.errorbar(filters_wl[wobs][mask_noerr], obs_fluxes[mask_noerr],
 915 |                          marker='s', markerfacecolor='None',
 916 |                          markersize=6, markeredgecolor='r',
 917 |                          label='Observed fluxes, no errors',
 918 |                          capsize=2, linestyle=' ', elinewidth=1)
 919 |         # residuals:
 920 |         ax2.errorbar(x=filters_wl[wobs][mask_ok],
 921 |                      y=(obs_fluxes[mask_ok]-mod_fluxes[wobs][mask_ok])/obs_fluxes[mask_ok],
 922 |                      yerr=obs_fluxes_err[mask_ok]/obs_fluxes[mask_ok],
 923 |                      marker='o', color='k', ms=2,
 924 |                      capsize=2, linestyle=' ', elinewidth=1)
 925 |         if mask_ok.any():
 926 |             maxresid = max(1.1, max(np.abs((obs_fluxes[mask_ok]-mod_fluxes[wobs][mask_ok])/obs_fluxes[mask_ok])))
 927 |         else:
 928 |             maxresid = 3.1
 929 |         ax2.plot([xmin, xmax], [0., 0.], ls='--', color='k', lw=0.2)
 930 |         ax2.set_xscale('log')
 931 |         ax3.set_xscale('log')
 932 |         ax1.set_xscale('log')
 933 |         ax2.minorticks_on()
 934 | 
 935 |         figure.subplots_adjust(hspace=0.2, wspace=0.)
 936 | 
 937 |         ax1.set_xlim(xmin, xmax)
 938 |         ax2.set_xlim(xmin, xmax)
 939 |         ax3.set_xlim(xmin, xmax)
 940 |         ax2.get_xaxis().set_major_formatter(mpl.ticker.ScalarFormatter())
 941 |         ax3.set_ylim(0, 1.0)
 942 |         ax3.axis('off')
 943 |         with Database() as base:
 944 |             legend_names_seen = set()
 945 |             #for filtername, used in zip([filters[i] for i in wobs], np.logical_or(mask_ok, np.logical_or(mask_noerr, mask_uplim))):
 946 |                 #if not used: continue
 947 |             for filtername in sorted(filters, key=lambda filtername: base.get_filter(filtername.rstrip('_')).effective_wavelength, reverse=True):
 948 |                 f = base.get_filter(filtername.rstrip('_'))
 949 |                 filtergroup_name = get_filtergroup_name(filtername)
 950 |                 color = filtergroups_colors.get(filtergroup_name)
 951 |                 legend_name = None if filtergroup_name in legend_names_seen else filtergroup_name
 952 |                 legend_names_seen.add(filtergroup_name)
 953 |                 if sed_type == 'lum':
 954 |                     wl = f.trans_table[0] / (1. + z) / 1000
 955 |                     wl_eff = f.effective_wavelength / (1. + z) / 1000
 956 |                 else:
 957 |                     wl = f.trans_table[0] / 1000
 958 |                     wl_eff = f.effective_wavelength / 1000
 959 |                 transmission = f.trans_table[1] / f.trans_table[1].max()
 960 | 
 961 |                 l, = ax3.plot(wl, transmission, color=color, lw=1)
 962 |                 if legend_name is not None and os.environ.get('PLOT_FILTERNAMES', '0') == '1':
 963 |                     ax3.text(
 964 |                         wl_eff, 1.0, legend_name, size=6,
 965 |                         color=color, va='bottom', ha='left')
 966 |             del legend_names_seen
 967 |         
 968 |         #ax3.legend()
 969 |         
 970 |         ax1.tick_params(which='both', labelbottom=False, labelright=False, top=True, right=True, direction="inout")
 971 |         # plot filter curves
 972 |         if mask_ok.any():
 973 |             ymin = min(np.nanmin(obs_fluxes[mask_ok]),
 974 |                        np.nanmin(mod_fluxes[wobs][mask_ok]))
 975 | 
 976 |             if not mask_uplim.any() == False:
 977 |                 ymax = max(max(np.nanmax(obs_fluxes[mask_ok]),
 978 |                                np.nanmax(obs_fluxes[mask_uplim])),
 979 |                            max(np.nanmax(mod_fluxes[wobs][mask_ok]),
 980 |                                np.nanmax(mod_fluxes[wobs][mask_uplim])))
 981 |             else:
 982 |                 ymax = max(np.nanmax(obs_fluxes[mask_ok]),
 983 |                            np.nanmax(mod_fluxes[wobs][mask_ok]))
 984 |             if np.isinf(ymax):
 985 |                 ymax = 100 * ymin
 986 |             if np.isinf(ymin) or ymin < 1e-6 * ymax:
 987 |                 ymin = ymax / 100
 988 |             ax1.set_ylim(1e-2*ymin, 1e1*ymax)
 989 |         ax1.set_yscale('log')
 990 |         ax2.set_ylim(-maxresid, maxresid)
 991 |         if sed_type == "mJy":
 992 |             # set right hand ticks in mag AB with round numbers
 993 |             ABmax = -2.5 * np.log10(ax1.get_ylim()[0] / 3631000)
 994 |             ABmin = -2.5 * np.log10(ax1.get_ylim()[1] / 3631000)
 995 |             ytick_ABs = np.arange(int(np.ceil(min(35,ABmin))), int(np.floor(max(10,min(30,ABmax)))) + 1)
 996 |             ytick_fluxes_mJy = 3631000 * 10**(ytick_ABs / -2.5)
 997 |             ax_r = ax1.secondary_yaxis('right')
 998 |             ax_r.set_ylim(ax1.get_ylim())
 999 |             ax_r.set_yscale('log')
1000 |             ax_r.set_yticks([1, 1.5], [1, 1.5], color="darkgrey")
1001 |             ax_r.set_yticks(ytick_fluxes_mJy, ytick_ABs, color="darkgrey")
1002 |             plt.sca(ax1)
1003 |         if sed_type == 'lum':
1004 |             ax2.set_xlabel("Rest-frame wavelength [$\\mu$m]")
1005 |             ax1.set_ylabel("Luminosity [W]")
1006 |             ax2.set_ylabel("(Obs-Mod)/Obs", size=8)
1007 |         else:
1008 |             ax2.set_xlabel("Observed wavelength [$\\mu$m]")
1009 |             ax1.set_ylabel("Flux [mJy]")
1010 |             ax2.set_ylabel("(Obs-Mod)/Obs", size=8)
1011 |         ax1.legend(fontsize=6, loc='best', fancybox=True, framealpha=0.5)
1012 |         figure.suptitle(
1013 |             "%s at z=%.3f, $\\chi^2_{/n}$=%.1f/%d Z=%.1f" %
1014 |             (obs['id'], obs['redshift'], chi2_best, len(obs_fluxes), Z),
1015 |             y=0.95)
1016 |         if sed_type == "lum":
1017 |             if os.environ.get("PLOT_KEYSTATS", "1") == "1":
1018 |                 plt.figtext(0.91, 0.28, posterior_summary_text, fontsize=10,
1019 |                     va='bottom', fontfamily="serif", fontvariant='small-caps')
1020 |             if os.environ.get("PLOT_SFH", "0") == "1":
1021 |                 ax_sfh = figure.add_axes([0.915, 0.11, 0.08, 0.1])
1022 |                 band = PredictionBand(np.arange(14000) / 1000.)
1023 |                 for sfh in sfhs:
1024 |                     y = np.zeros_like(band.x)
1025 |                     y[:len(sfh)] = sfh #[::-1]
1026 |                     plt.plot(band.x, y / y.max(), color='blue', lw=0.2, alpha=0.5)
1027 |                 ax_sfh.set_title('SFH', size=8, loc='left')
1028 |                 ax_sfh.yaxis.set_visible(False)
1029 |                 ax_sfh.xaxis.set_ticks_position('bottom')
1030 |                 ax_sfh.spines['top'].set_visible(False)
1031 |                 ax_sfh.spines['right'].set_visible(False)
1032 |                 ax_sfh.set_xticks([0, 1, 2, 10], [0, 1, 2, 10], fontsize=6)
1033 |                 ax_sfh.set_xlim(0, 3)
1034 |                 ax_sfh.set_xlabel('Age [Gyr]', size=8)
1035 |                 
1036 |                 plt.sca(ax1)
1037 |         figure.savefig("%s/sed_%s.pdf" % (plot_dir, sed_type))
1038 |         plt.close(figure)
1039 | 
1040 |     with np.errstate(over='ignore'):
1041 |         return (
1042 |             param_names + posteriors_names,
1043 |             np.concatenate((results['samples'].mean(axis=0), np.mean(posteriors, axis=0))),
1044 |             np.concatenate((results['samples'].std(axis=0), np.std(posteriors, axis=0))),
1045 |             np.concatenate((np.quantile(results['samples'], 0.02275, axis=0), np.quantile(posteriors, 0.02275, axis=0))),
1046 |             np.concatenate((np.quantile(results['samples'], 0.97725, axis=0), np.quantile(posteriors, 0.97725, axis=0))),
1047 |             np.concatenate((np.median(results['samples'], axis=0), np.median(posteriors, axis=0))),
1048 |             np.concatenate((np.log(np.exp(results['samples']).mean(axis=0)), np.log(np.exp(posteriors).mean(axis=0)))),
1049 |         )
1050 | 
1051 | 
1052 | def make_prior_transform(rv_redshift, Finfo=None, Linfo=None, num_redshift_points=40):
1053 |     """Create the prior transform given prior information about the flux or redshift.
1054 | 
1055 |     Parameters
1056 |     ----------
1057 |     rv_redshift: scipy.stats random variable
1058 |         prior for redshift
1059 |     Finfo: None or tuple
1060 |         If None, a flat prior on AGN luminosity is assumed
1061 |         If (FAGN, FAGN_errlo, FAGN_errhi) is provided,
1062 |         this describes the log10(erg/s/cm^2 flux) prior and its error bars.
1063 |     num_redshift_points: int
1064 |         Number of discrete points for the redshift parameter.
1065 | 
1066 |     Returns
1067 |     -------
1068 |     prior_transform: func
1069 |         Prior transform function
1070 |     """
1071 |     redshift_fixed = rv_redshift.std() == 0
1072 | 
1073 |     # consider flux prior on 5100A luminosity
1074 |     if Finfo is not None:
1075 |         FAGN, FAGN_errlo, FAGN_errhi = Finfo
1076 |         # Sides of the asymmetric gaussian prior on log(flux)
1077 |         rv_F_lo = NormalDist(FAGN, FAGN_errlo)
1078 |         rv_F_hi = NormalDist(FAGN, FAGN_errhi)
1079 | 
1080 |         def L_prior_transform(u, redshift):
1081 |             # pick the appropriate side
1082 |             rv = rv_F_lo if u < 0.5 else rv_F_hi
1083 |             # convert to flux
1084 |             logF = rv.ppf(u)
1085 |             # convert from erg/s/cm^2 to erg/s
1086 |             logL = logF + np.log10(4 * np.pi) + 2 * np.log10((cosmology.luminosity_distance(redshift) / units.cm).to(1))
1087 |             return logL
1088 |     # consider prior on 5100A luminosity
1089 |     elif Linfo is not None:
1090 |         LAGN, LAGN_errlo, LAGN_errhi = Linfo
1091 |         # Sides of the asymmetric gaussian prior on log(flux)
1092 |         rv_L_lo = NormalDist(LAGN, LAGN_errlo)
1093 |         rv_L_hi = NormalDist(LAGN, LAGN_errhi)
1094 | 
1095 |         def L_prior_transform(u, redshift):
1096 |             # pick the appropriate side
1097 |             rv = rv_L_lo if u < 0.5 else rv_L_hi
1098 |             return rv.ppf(u)
1099 |     else:
1100 |         def L_prior_transform(u, z):
1101 |             # AGN luminosity from 10^L_min to 10^L_max
1102 |             del z
1103 |             return u * (L_max - L_min) + L_min
1104 | 
1105 |     def prior_transform(cube):
1106 |         params = np.empty(len(cube) + (1 if redshift_fixed else 0)) + np.nan
1107 |         i = 0
1108 |         for module_parameters in parameter_list:
1109 |             for k, v in module_parameters.items():
1110 |                 if len(v) > 1:
1111 |                     params[i] = v[min(len(v) - 1, int(len(v) * cube[i]))]
1112 |                     if is_log_param[i]:
1113 |                         params[i] = log10(params[i])
1114 |                     i += 1
1115 | 
1116 |         # stellar mass from 10^5 to 10^15
1117 |         params[i] = cube[i] * (mass_max - mass_min) + mass_min
1118 | 
1119 |         # redshift.
1120 |         # Approximate redshift with points on the CDF
1121 |         params[i + 2] = rv_redshift.ppf((1 + np.round(cube[i + 2] * num_redshift_points)) / (num_redshift_points + 2))
1122 | 
1123 |         # AGN luminosity from 10^30 to 10^50
1124 |         params[i + 1] = L_prior_transform(cube[i + 1], params[i + 2])
1125 | 
1126 |         j = i + 2 if redshift_fixed else i + 3
1127 | 
1128 |         # systematic uncertainty
1129 |         params[i + 3] = rv_systematics.ppf(cube[j])
1130 |         return params
1131 |     return prior_transform
1132 | 
1133 | 
1134 | def plot_model():
1135 |     """Vary each model parameter and show its effect as plots.
1136 | 
1137 |     This also saves the plot elements as machine-readable csv files.
1138 |     """
1139 |     umid = np.array([1e-6 if 'E(B-V)' in p else 0.5 for p in param_names])
1140 |     redshift = 1.0
1141 |     for logL_AGN in [44, 38, 46, 42]:
1142 |         L_AGN = 10**logL_AGN
1143 |         rv_redshift = scipy.stats.uniform(redshift, redshift + 1e-3)
1144 |         prior_transform = make_prior_transform(rv_redshift)
1145 |         print("reference values:")
1146 |         for p, v in zip(param_names, prior_transform(umid)):
1147 |             print("   %-20s: %s" % (p, v))
1148 |         cache_filters = {}
1149 | 
1150 |         for i, p in enumerate(param_names):
1151 |             if p in ('redshift', 'log_L_AGN', 'activate_AGNtype', 'systematics'):
1152 |                 continue
1153 |             print("varying", p)
1154 | 
1155 |             u = umid.copy()
1156 |             for AGNtype in 1,: # 2, 3:
1157 |                 last_value = np.nan
1158 |                 plt.figure(figsize=(12, 6))
1159 |                 filename = 'modelspectrum_L%d_type%s_%s' % (logL_AGN, AGNtype, p.replace('(', '').replace(')', ''))
1160 |                 first_legend = None
1161 |                 for v in np.linspace(0.001, 0.999, 11):
1162 |                     u[i] = v
1163 |                     parameters = prior_transform(u)
1164 |                     if 'activate_AGNtype' in param_names:
1165 |                         parameters[param_names.index('activate_AGNtype')] = AGNtype
1166 |                     if parameters[i] == last_value:
1167 |                         continue
1168 |                     last_value = parameters[i]
1169 |                     stellar_mass = 10**parameters[-4]
1170 | 
1171 |                     parameter_list_here = make_parameter_list(parameters)
1172 |                     assert module_list[-1] == 'redshifting'
1173 |                     parameter_list_here[-1] = dict(redshift=redshift)
1174 | 
1175 |                     with np.errstate(invalid='ignore'):
1176 |                         sed, _, agn_sed = scale_sed_components(module_list, parameter_list_here, stellar_mass, L_AGN)
1177 |                     sed.cache_filters = cache_filters
1178 | 
1179 |                     _, model_variables = compute_model_fluxes(sed, filters)
1180 | 
1181 |                     wavelength_spec = sed.wavelength_grid.copy()
1182 |                     wavelength_spec /= 1. + redshift
1183 |                     DL = sed.info['universe.luminosity_distance']
1184 |                     sed_multiplier = wavelength_spec.copy() * (redshift + 1)
1185 |                     assert (sed_multiplier >= 0).all(), (stellar_mass, DL, wavelength_spec)
1186 |                     wavelength_spec /= 1000
1187 |                     mask = np.logical_and(wavelength_spec >= PLOT_L_MIN, wavelength_spec <= PLOT_L_MAX)
1188 |                     alpha = 1 - (v + 0.2) / 1.2
1189 | 
1190 |                     output_labels = ['wavelength', 'total']
1191 |                     outputs = [wavelength_spec, sed.luminosity * sed_multiplier]
1192 |                     # print(sed.contribution_names)
1193 |                     for j, plot_element in enumerate(plot_elements):
1194 |                         keys = plot_element['keys']
1195 |                         if not any(k in sed.contribution_names for k in keys):
1196 |                             # print("skipping", plot_element['label'], 'because need', keys, "have only some:", [k in sed.contribution_names for k in keys])
1197 |                             continue
1198 | 
1199 |                         pred = sum(sed.get_lumin_contribution(k) * sed_multiplier for k in keys if k in sed.contribution_names)
1200 | 
1201 |                         line_kwargs = dict(plot_element)
1202 |                         del line_kwargs['keys']
1203 |                         line_kwargs['alpha'] = alpha
1204 |                         line_kwargs.pop('shading', None)
1205 |                         plt.plot(wavelength_spec[mask], pred[mask], **line_kwargs)
1206 |                         outputs.append(pred)
1207 |                         output_labels.append(plot_element['label'])
1208 | 
1209 |                     line, = plt.plot(wavelength_spec[mask], (sed.luminosity * sed_multiplier)[mask], '-', color='k', alpha=alpha, label='total')
1210 |                     subfilename = filename + '_at%s' % (parameters[i])
1211 |                     np.savetxt(
1212 |                         subfilename + '.params',
1213 |                         [np.concatenate((parameters, model_variables))],
1214 |                         delimiter=',', header=','.join(param_names + analysed_variables)
1215 |                     )
1216 |                     np.savetxt(subfilename + '.csv', np.transpose(outputs), delimiter=',', header=','.join(output_labels), comments='')
1217 | 
1218 |                     if first_legend is None:
1219 |                         first_legend = plt.legend(title='Components', framealpha=0.5, loc='upper left')
1220 |                 plt.gca().add_artist(first_legend)
1221 | 
1222 |                 plt.xlabel("Wavelength [$\\mu$m]")
1223 |                 plt.ylabel("Luminosity [W]")
1224 |                 plt.xscale('log')
1225 |                 plt.yscale('log')
1226 |                 plt.xlim(PLOT_L_MIN, PLOT_L_MAX)
1227 |                 plt.title(p)
1228 |                 plt.ylim(1e34, 1e39)
1229 | 
1230 |                 plt.savefig(filename + '.png', bbox_inches='tight')
1231 |                 plt.close()
1232 | 
1233 |     print("SED model components:", ' '.join(sed.contribution_names))
1234 |     print("SED info available:", ' '.join(sed.info.keys()))
1235 |     print("Selected for analysis:", len(analysed_variables), analysed_variables)
1236 | 
1237 | 
1238 | 
1239 | def generate_fluxes(Ngen=100000):
1240 |     """Generate random SEDs from the configuration and save the fluxes."""
1241 |     # redshifts between 0 and 7, with a wide peak around 1-3.
1242 |     rv_redshift = scipy.stats.beta(2, 5, scale=7)
1243 |     prior_transform = make_prior_transform(rv_redshift, num_redshift_points=400)
1244 |     cache_filters = {}
1245 | 
1246 |     fluxdata = np.empty((Ngen, len(param_names + analysed_variables) + len(filters))) * np.nan
1247 |     u = np.random.uniform(size=len(param_names))
1248 |     assert module_list[-1] == 'redshifting'
1249 |     with FastAttenuation():
1250 |         for i in tqdm.trange(Ngen):
1251 |             while True:
1252 |                 u[i % len(param_names)] = np.random.uniform()
1253 |                 # last four are always updated, because they are not cached
1254 |                 u[-4:] = np.random.uniform(size=len(u[-4:]))
1255 |                 parameters = prior_transform(u)
1256 |                 stellar_mass = 10**parameters[-4]
1257 |                 L_AGN = 10**parameters[-3]
1258 |                 redshift = parameters[-2]
1259 |                 parameter_list_here = make_parameter_list(parameters)
1260 |                 parameter_list_here[-1] = dict(redshift=redshift)
1261 | 
1262 |                 sed, _, agn_sed = scale_sed_components(module_list, parameter_list_here, stellar_mass, L_AGN)
1263 |                 sed.cache_filters = cache_filters
1264 | 
1265 |                 model_fluxes_full, model_variables = compute_model_fluxes(sed, filters)
1266 |                 if (model_fluxes_full >= 0).all():
1267 |                     fluxdata[i][:len(param_names)] = parameters
1268 |                     fluxdata[i][len(param_names):len(param_names + analysed_variables)] = model_variables
1269 |                     fluxdata[i][len(param_names + analysed_variables):] = model_fluxes_full
1270 |                     break
1271 |                 else:
1272 |                     assert np.logical_or(model_fluxes_full > -1e-6, model_fluxes_full == -99).all(), ("some filters give negative fluxes:", filters, model_fluxes_full)
1273 |             assert np.isfinite(fluxdata[i]).all(), fluxdata[i]
1274 | 
1275 |     # save as fits file
1276 |     tout = Table(data=fluxdata, names=param_names + analysed_variables + filters)
1277 |     tout.write('model_fluxes.fits', overwrite=True)
1278 | 
1279 | def chi2_upper_limit(obs_fluxes, model_fluxes, total_variance):
1280 |     erf_result = scipy.special.erf((obs_fluxes-model_fluxes) / (np.sqrt(2) * (total_variance)))
1281 |     return -2.0 * log(0.5 * (1 + erf_result) + 1e-300)
1282 | 
1283 | def chi2_with_norm(model_fluxes, agn_model_fluxes, obs_fluxes, obs_errors, obs_filter_wavelength, redshift, sys_error, NEV, transmitted_fraction, exponent=2):
1284 |     """Likelihood considering all variance contributions.
1285 | 
1286 |     Parameters
1287 |     ----------
1288 |     model_fluxes: array
1289 |         list of SED fluxes
1290 |     agn_model_fluxes: array
1291 |         list of SED fluxes from the AGN components alone
1292 |     obs_fluxes: array
1293 |         list of measured fluxes
1294 |     obs_errors: array
1295 |         uncertainties for obs_fluxes
1296 |     obs_filter_wavelength: array
1297 |         observed-frame wavelength of the filters
1298 |     redshift: float
1299 |         Redshift
1300 |     sys_error: float
1301 |         fractional systematic error to apply
1302 |     NEV: float
1303 |         normalised excess variance to consider on AGN components due to variability
1304 |     transmitted_fraction: array
1305 |         for each filter, the fraction of flux transmitted through any attenuation.
1306 |         This is the ratio of flux to the flux if there was no attenuation.
1307 |     exponent: float
1308 |         2 for Gaussian statistics (L2), 1 for exponential statistics (L1) being more permissive to outliers
1309 | 
1310 |     Returns
1311 |     -------
1312 |     norm: float
1313 |         logarithm of Gaussian likelihood normalisation factor
1314 |     chi2: float
1315 |         "chi-square", i.e., the -0.5 times the Gaussian likelihood exponential factor
1316 |     total_variance: array
1317 |         total variance from all contributions (data, variability, model uncertainties)
1318 |     """
1319 |     # Some observations may not have flux values in some filter(s), but
1320 |     # they can have upper limit(s). To process upper limits, the user
1321 |     # is asked to put the upper limit as flux value and an error value with
1322 |     # (obs_errors>=-9990. and obs_errors<0.).
1323 |     # Next, the user has two options:
1324 |     # 1) s/he puts True in the boolean lim_flag
1325 |     # and the limits are processed as upper limits below.
1326 |     # 2) s/he puts False in the boolean lim_flag
1327 |     # and the limits are processed as no-data below.
1328 | 
1329 |     # χ² of the comparison of each model to each observation.
1330 |     # This mask selects the filter(s) for which measured fluxes are given
1331 |     # i.e., when (obs_flux is >=0. and obs_errors>=0.) and lim_flag=True
1332 |     mask_data = np.logical_and(obs_fluxes > TOLERANCE,
1333 |                                obs_errors > TOLERANCE)
1334 |     # This mask selects the filter(s) for which upper limits are given
1335 |     # i.e., when (obs_flux is >=0. (and obs_errors>=-9990., obs_errors<0.))
1336 |     # and lim_flag=True
1337 |     mask_lim = np.logical_and(obs_errors >= -9990., obs_errors < TOLERANCE)
1338 | 
1339 |     # (1) variance from observation, according to the reported errors
1340 |     obs_variance = obs_errors**2
1341 | 
1342 |     # (2) variance from year-to-year variability (Simm+ paper)
1343 |     if variability_uncertainty:
1344 |         var_variance = NEV * agn_model_fluxes**2
1345 |     else:
1346 |         var_variance = 0.0
1347 | 
1348 |     # (3) variance from model systematic uncertainties
1349 |     sys_variance = (sys_error * model_fluxes)**2
1350 | 
1351 |     if with_attenuation_model_uncertainty:
1352 |         # (3b) attenuation model error
1353 |         # map transmitted fraction (from 1..0.1..0.01 [0..1..2]) to standard deviation (0..0.01..0.01)
1354 |         transmitted_fraction[transmitted_fraction < 1e-4] = 1e-4
1355 |         transmitted_fraction[transmitted_fraction > 1.0] = 1.0
1356 |         neg_log_transmitted = -np.log10(transmitted_fraction + 1e-4)
1357 |         # powerlaw increase of uncertainty fraction
1358 |         # at transmitted_fraction     = 0, 0.1, 1
1359 |         # is neg_log_transmitted      = -4, -1, 0
1360 |         # is log_uncertainty_fraction = -1, -2, -4
1361 |         log_uncertainty_fraction = -4 + 2 * neg_log_transmitted
1362 |         # but threshold at 10%
1363 |         log_uncertainty_fraction[log_uncertainty_fraction > -1] = -1
1364 |         # this is the uncertainty relative to the unattenuated spectrum
1365 |         # since we apply to total fluxes, we need to correct it upwards.
1366 |         attenuation_uncertainty = 10**log_uncertainty_fraction / transmitted_fraction
1367 |         # print("attenuation model uncertainties:", transmitted_fraction, 10**log_uncertainty_fraction, attenuation_uncertainty)
1368 |         # apply as a fraction of total model fluxes
1369 |         sys_variance += (attenuation_uncertainty * model_fluxes)**2
1370 | 
1371 |     if with_Ly_break_uncertainty:
1372 |         # ignore measurements falling below the Lyman break, as the IGM model is simplistic
1373 |         Ly_break_uncertainty = np.where(obs_filter_wavelength / (1 + redshift) < 150, 10000, 0)
1374 |         sys_variance += (Ly_break_uncertainty * model_fluxes)**2
1375 |     del obs_filter_wavelength
1376 |     del redshift
1377 | 
1378 |     # combined variance in each filter
1379 |     total_variance = obs_variance + sys_variance + var_variance
1380 | 
1381 |     # compute chi^2 and the Gaussian likelihood normalisation
1382 |     chi2_ = np.sum(
1383 |         ((obs_fluxes[mask_data] - model_fluxes[mask_data])**2 / total_variance[mask_data])**(exponent/2.0))
1384 |     norm = 0.5 * np.log(2 * np.pi * total_variance**(exponent/2.0)).sum()
1385 | 
1386 |     if mask_lim.any():
1387 |         chi2_ += chi2_upper_limit(
1388 |             obs_fluxes[mask_lim],
1389 |             model_fluxes[mask_lim],
1390 |             total_variance[mask_lim]).sum()
1391 | 
1392 |     return norm, chi2_, total_variance
1393 | 
1394 | 
1395 | class ModelLikelihood(object):
1396 |     """
1397 |     Likelihood function, which also knows about the observational data.
1398 | 
1399 |     Parameters
1400 |     ----------
1401 |     wobs: list of bool
1402 |         which filters are active
1403 |     obs_fluxes: list
1404 |         observed flux values
1405 |     obs_errors: list
1406 |         uncertainties for obs_fluxes
1407 |     obs_filter_wavelength: list
1408 |         observed_frame wavelength of each filter
1409 |     additional_information: list
1410 |         list of SED info variable names and likelihood functions for that variable,
1411 |         [("sfh.sfr100Myrs", scipy.stats.norm(0, 1).logpdf), ...]
1412 |     """
1413 | 
1414 |     def __init__(self, wobs, obs_fluxes, obs_errors, obs_filter_wavelength, additional_likelihood_terms=[]):
1415 |         self.cache_filters = {}
1416 |         self.last_parameters = None
1417 |         self.wobs = wobs
1418 |         self.obs_fluxes = obs_fluxes
1419 |         self.obs_errors = obs_errors
1420 |         self.obs_filter_wavelength = obs_filter_wavelength
1421 |         self.additional_likelihood_terms = additional_likelihood_terms
1422 |         self.last_loglikelihood = None
1423 |         self.sampler = None
1424 |         self.counter_early_reject = 0
1425 |         self.counter_avoid_recompute_of_same = 0
1426 |         self.counter_unphysical_reject = 0
1427 |         self.counter_calls = 0
1428 |         self.agn_flux_filters = []
1429 |         self.gal_flux_filters = []
1430 |         for analysed_variable_key, vi, _ in additional_likelihood_terms:
1431 |             if analysed_variable_key == 'AGNflux':
1432 |                 self.agn_flux_filters.append(vi)
1433 |             elif analysed_variable_key == 'GALflux':
1434 |                 self.gal_flux_filters.append(vi)
1435 |             else:
1436 |                 assert analysed_variable_key == 'analysed_variables', 'GRAHSP bug: analysed_variable_key can only be AGNflux, GALflux or analysed_variables.'
1437 |             
1438 | 
1439 |     def stats(self):
1440 |         """Return statistics on computational effort."""
1441 |         return 'calls: %d, %d were repeat calls, %d had unphysical age, %d dismissed from first band fit alone' % (
1442 |             self.counter_calls,
1443 |             self.counter_avoid_recompute_of_same,
1444 |             self.counter_unphysical_reject,
1445 |             self.counter_early_reject,
1446 |         )
1447 | 
1448 | 
1449 |     def __call__(self, parameters):
1450 |         """Fitting likelihood function"""
1451 | 
1452 |         
1453 |         self.counter_calls += 1
1454 |         # if we are called with the same values again, return what we just computed
1455 |         # this can happen because the parameters are binned
1456 |         if self.last_parameters is not None and np.all(self.last_parameters == parameters):
1457 |             self.counter_avoid_recompute_of_same += 1
1458 |             return self.last_loglikelihood
1459 | 
1460 |         # get the normalisation parameters
1461 |         stellar_mass = 10**parameters[-4]
1462 |         L_AGN = 10**parameters[-3]
1463 |         redshift = parameters[-2]
1464 |         sys_error = parameters[-1]
1465 |         parameter_list_here = make_parameter_list(parameters)
1466 |         assert module_list[-1] == 'redshifting'
1467 |         parameter_list_here[-1] = dict(redshift=redshift)
1468 | 
1469 |         # compute SED
1470 |         sed, gal_sed, agn_sed = scale_sed_components(module_list, parameter_list_here, stellar_mass, L_AGN)
1471 |         sed.cache_filters = self.cache_filters
1472 |         agn_sed.cache_filters = self.cache_filters
1473 | 
1474 |         sfr = sed.info['sfh.sfr100Myrs']
1475 |         if sed.info['sfh.age'] > sed.info['universe.age'] or not sfr_min <= sfr <= sfr_max:
1476 |             # excluded by exceeding age of Universe
1477 |             # assign lower number for those further away from the constraints
1478 |             logl = -1e20 * (np.log10(stellar_mass) + abs(sfr) + max(0, sed.info['sfh.age'] - sed.info['universe.age']))
1479 |             #print("violation", (sfr_min, sfr, sfr_max), (sed.info['sfh.age'], sed.info['universe.age']), "-->", logl)
1480 |             self.last_parameters = parameters
1481 |             self.last_loglikelihood = logl
1482 |             self.counter_unphysical_reject += 1
1483 |             return logl
1484 | 
1485 |         # get fraction of non-attenuated flux at the filters
1486 |         filter_wl_indices = np.searchsorted(sed.wavelength_grid, filters_wl_orig[self.wobs])
1487 |         filter_contrib = sed.luminosities[:, filter_wl_indices]
1488 |         filter_pos_contrib = np.where(filter_contrib > 0, filter_contrib, 0).sum(axis=0)
1489 |         transmitted_fraction = filter_contrib.sum(axis=0) / filter_pos_contrib
1490 | 
1491 |         filters_observed = [filters[i] for i in self.wobs]
1492 | 
1493 |         model_fluxes, model_variables = compute_model_fluxes(sed, filters_observed)
1494 | 
1495 |         # do costly AGN flux filter computation only if needed
1496 |         if variability_uncertainty:
1497 |             for module_name, module_parameters in zip(module_list[cache_depth:], parameter_list_here[cache_depth:]):
1498 |                 module_instance = gbl_warehouse.get_module_cached(module_name, **module_parameters)
1499 |                 module_instance.process(agn_sed)
1500 |             agn_model_fluxes, _ = compute_model_fluxes(agn_sed, filters_observed)
1501 |         else:
1502 |             agn_model_fluxes = model_fluxes * np.nan
1503 | 
1504 |         assert self.agn_flux_filters == [], 'AGN fluxes as priors is currently not implemented'
1505 | 
1506 |         if len(self.gal_flux_filters) > 0:
1507 |             for module_name, module_parameters in zip(module_list[cache_depth:], parameter_list_here[cache_depth:]):
1508 |                 module_instance = gbl_warehouse.get_module_cached(module_name, **module_parameters)
1509 |                 module_instance.process(gal_sed)
1510 |             gal_model_fluxes = {filter_: gal_sed.compute_fnu(filter_) for filter_ in self.gal_flux_filters}
1511 |         else:
1512 |             gal_model_fluxes = {}
1513 | 
1514 |         # compute likelihood:
1515 |         norm, chi2_, _ = chi2_with_norm(
1516 |             model_fluxes, agn_model_fluxes, self.obs_fluxes, self.obs_errors,
1517 |             self.obs_filter_wavelength, redshift, sys_error, NEV=sed.info.get('agn.NEV'),
1518 |             exponent=exponent, transmitted_fraction=transmitted_fraction)
1519 | 
1520 |         analysed_variables_dict = dict(
1521 |             GALflux=gal_model_fluxes,
1522 |             AGNflux=agn_model_fluxes,
1523 |             analysed_variables=model_variables)
1524 | 
1525 |         logl = -0.5 * chi2_ - norm
1526 | 
1527 |         for analysed_variable_key, analysed_variable_index, likelihood_term in self.additional_likelihood_terms:
1528 |             # print('%.1f' % logl, analysed_variable_key, analysed_variable_index, analysed_variables_dict[analysed_variable_key][analysed_variable_index])
1529 |             logl += likelihood_term(analysed_variables_dict[analysed_variable_key][analysed_variable_index])
1530 |         self.last_parameters = parameters
1531 |         self.last_loglikelihood = logl
1532 |         return logl
1533 | 
1534 | 
1535 | def analyse_obs_wrapper(args):
1536 |     """Wrapper catching crashes to continue with the next source.
1537 | 
1538 |     When analysing large samples with large machines, it can be annoying if
1539 |     an analysis fails due to file locking or files being deleted etc.
1540 |     This allows continuing the run with the next source,
1541 |     and later reprocessing the entire sample.
1542 |     """
1543 |     i, N, samplername, obs, plot = args
1544 |     try:
1545 |         return analyse_obs(i, N, samplername, obs, plot=plot)
1546 |     except OSError as e:
1547 |         print("skipping '%s', probably analysed on another machine. error was: '%s'" % (obs['id'], e))
1548 |         return obs['id'], None, None
1549 |     except RuntimeError as e:
1550 |         print("skipping '%s', probably analysed on another machine. error was: '%s'" % (obs['id'], e))
1551 |         return obs['id'], None, None
1552 |     except BlockingIOError as e:
1553 |         print("skipping '%s', probably analysed on another machine. error was: '%s'" % (obs['id'], e))
1554 |         return obs['id'], None, None
1555 |     except np.linalg.LinAlgError as e:
1556 |         # print("skipping '%s', too few live points. error was: '%s'" % (obs['id'], e))
1557 |         raise Exception("Too few live points, caused a LinAlgError. use --num-live-points.") from e
1558 | 
1559 | 
1560 | def build_likelihood_term(v, mid, sigma_lo, sigma_hi):
1561 |     def likelihood_term(x):
1562 |         # print("likelihood term for", v, "value", x, mid, sigma_lo, sigma_hi)
1563 |         return -0.5 * np.where(x < mid, (x - mid) / sigma_lo, (x - mid) / sigma_hi)**2
1564 |     likelihood_term.__name__ = 'likelihood_term_%s' % v
1565 |     return likelihood_term
1566 | 
1567 | 
1568 | def analyse_obs(i, N, samplername, obs, plot=True):
1569 |     """Source fitting.
1570 | 
1571 |     Parameters
1572 |     ----------
1573 |     i: int
1574 |         source index out of N being analysed. Not used except for print.
1575 |     N: int
1576 |         total number of sources being analysed. Not used except for print.
1577 |     obs: dict
1578 |         Observation table row (id, redshift, etc)
1579 |     plot: bool
1580 |         whether to produce plots
1581 |     samplername: str
1582 |         which fitting algorithm to run.
1583 |         MCMC, optimization, MLFriends were supported in a previous version
1584 |         but are inefficient or get stuck.
1585 |         samplername='nested-slice' is the only currently supported and works well.
1586 |         It runs nested sampling with MLFriends initially, then switches over to
1587 |         slice sampling.
1588 | 
1589 |     Returns
1590 |     -------
1591 |     id: str
1592 |         source id
1593 |     results: array
1594 |         list of fitting result values. May be None if analysis failed.
1595 |     results_string: str
1596 |         same as results, but converted to a string. Can be present even if results is None.
1597 |     """
1598 |     assert samplername == 'nested-slice'
1599 | 
1600 |     redshift_mean = obs['redshift']
1601 |     num_redshift_points = 40
1602 | 
1603 |     if samplername.startswith('nested') and ('flag_zphot' in obs.colnames and obs['flag_zphot']):
1604 |         assert 'ZGRID' in obs.colnames and 'PZ' in obs.colnames
1605 |         rv_redshift = PDFDist(obs['ZGRID'], obs['PZ'])
1606 |         active_param_names = param_names
1607 |         derived_param_names = []
1608 |     elif samplername.startswith('nested') and ('redshift_err' not in obs.colnames or 0<=obs['redshift_err']<=0.001):
1609 |         rv_redshift = DeltaDist(redshift_mean)
1610 |         active_param_names = param_names[:-1]
1611 |         derived_param_names = [param_names[-1]]
1612 |     else:
1613 |         if 'redshift_err' in obs.colnames:
1614 |             redshift_err = obs['redshift_err']
1615 |         else:
1616 |             # put a 1% error on 1+z, at least
1617 |             redshift_err = 0.01 * redshift_mean
1618 |         if redshift_err < 0:
1619 |             num_redshift_points = 200
1620 |             print("unknown-z mode: flat redshift prior from 0 to 6")
1621 |             # flat redshift distribution, photo-z mode
1622 |             rv_redshift = scipy.stats.uniform(0.001, 6)
1623 |         else:
1624 |             rng = np.random.default_rng(42)
1625 |             redshift_samples = rng.normal(redshift_mean, redshift_err, size=1000)
1626 |             redshift_samples = redshift_samples[redshift_samples>0]
1627 |             redshift_shape, _, redshift_scale = scipy.stats.weibull_min.fit(
1628 |                 redshift_samples, floc=0,
1629 |             )
1630 |             rv_redshift = scipy.stats.weibull_min(redshift_shape, scale=redshift_scale)
1631 |             print("photo-z mode: redshift prior: ", redshift_shape, redshift_scale)
1632 |         active_param_names = param_names
1633 |         derived_param_names = []
1634 | 
1635 |     print()
1636 |     print("="*80)
1637 |     print()
1638 |     print("[%d/%d] Source:" % (i+1, N), obs['id'], "Redshift:", rv_redshift.mean(), rv_redshift.std())
1639 |     del i, N
1640 |     print()
1641 |     if 'FAGN' in obs.keys() and 'FAGN_errlo' in obs.keys() and 'FAGN_errhi' in obs.keys():
1642 |         Finfo = (obs['FAGN'], obs['FAGN_errlo'], obs['FAGN_errhi'])
1643 |         print("Including Gaussian AGN log-flux constraint:", Finfo)
1644 |     elif 'FAGN' in obs.keys() and 'FAGN_err' in obs.keys():
1645 |         Finfo = (obs['FAGN'], obs['FAGN_err'], obs['FAGN_err'])
1646 |         print("Including Gaussian AGN log-flux constraint:", Finfo)
1647 |     else:
1648 |         Finfo = None
1649 |     if 'LAGN' in obs.keys() and 'LAGN_errlo' in obs.keys() and 'LAGN_errhi' in obs.keys():
1650 |         Linfo = (obs['LAGN'], obs['LAGN_errlo'], obs['LAGN_errhi'])
1651 |         print("Including Gaussian AGN log-L constraint:", Linfo)
1652 |     else:
1653 |         Linfo = None
1654 |     additional_likelihood_terms = []
1655 |     potential_prior_names  = [('analysed_variables', v, vi) for vi, v in enumerate(analysed_variables)]
1656 |     potential_prior_names += [('GALflux', 'GALflux_' + filtername, filtername) for filtername in filters]
1657 |     potential_prior_names += [('AGNflux', 'AGNflux_' + filtername, filtername) for filtername in filters]
1658 | 
1659 |     for analysed_variable_key, v, vi in potential_prior_names:
1660 |         priorv = 'prior_' + v
1661 |         if priorv not in obs.keys():
1662 |             continue
1663 |         if not ((priorv + '_errlo') in obs.keys() and (priorv + '_errhi') in obs.keys()):
1664 |             assert False, ('error columns missing:', priorv + '_errlo', priorv + '_errhi')
1665 |             continue
1666 |         mid, sigma_lo, sigma_hi = float(obs[priorv]), float(obs[priorv + '_errlo']), float(obs[priorv + '_errhi'])
1667 |         if not (np.isfinite(mid) and sigma_lo > 0 and sigma_hi > 0):
1668 |             print("NOT including Gaussian constraint on '%s'" % v, (mid, sigma_lo, sigma_hi))
1669 |             continue
1670 |         print("Including Gaussian constraint on '%s'" % v, (mid, sigma_lo, sigma_hi))
1671 |         likelihood_term = build_likelihood_term(v, mid, sigma_lo, sigma_hi)
1672 |         additional_likelihood_terms.append((analysed_variable_key, vi, likelihood_term))
1673 |         del analysed_variable_key, v, vi
1674 | 
1675 |     prior_transform = make_prior_transform(rv_redshift, Finfo=Finfo, Linfo=Linfo, num_redshift_points=num_redshift_points)
1676 | 
1677 |     prior_samples = np.asarray([prior_transform(u) for u in np.random.uniform(size=(10000, len(active_param_names)))])
1678 |     assert np.isfinite(prior_samples).all()
1679 | 
1680 |     # select the filters from the list of active filters
1681 | 
1682 |     with warnings.catch_warnings():
1683 |         warnings.simplefilter("ignore", UserWarning)
1684 |         obs_fluxes_full = np.array([obs[name] for name in filters])
1685 |         obs_errors_full = np.array([obs[name + "_err"] for name in filters])
1686 | 
1687 |     wobs, = np.where(obs_fluxes_full > TOLERANCE)
1688 |     obs_fluxes = obs_fluxes_full[wobs]
1689 |     obs_errors = obs_errors_full[wobs]
1690 |     obs_filter_wavelength = filters_wl_orig[wobs]
1691 | 
1692 |     loglikelihood = ModelLikelihood(wobs, obs_fluxes, obs_errors, obs_filter_wavelength, additional_likelihood_terms=additional_likelihood_terms)
1693 | 
1694 |     outdir = "grahsp_%s_var%s%s%d" % (
1695 |         str(obs['id']).strip(),
1696 |         'V' if variability_uncertainty else '',
1697 |         'A' if with_attenuation_model_uncertainty else '',
1698 |         -int(np.log10(systematics_width)),
1699 |     )
1700 |     if mass_max != 15:
1701 |         outdir += "_maxgal%d" % mass_max
1702 | 
1703 |     replot = (os.environ.get('REPLOT', '0') == '1') or not os.path.exists(outdir + '/analysis_results.txt')
1704 |     results = None
1705 |     with FastAttenuation():
1706 |         try:
1707 |             sampler = ReactiveNestedSampler(
1708 |                 active_param_names, loglikelihood, prior_transform,
1709 |                 log_dir=outdir, resume='resume', derived_param_names=derived_param_names)
1710 |         except Exception as e:
1711 |             # check that there are not errors running the model ...
1712 |             loglikelihood(prior_samples[0])
1713 |             print("WARNING: could not resume because of %s. overwriting." % e)
1714 |             os.unlink(outdir + '/results/points.hdf5')
1715 |             # previous results are invalid, so start from scratch
1716 |             replot = True
1717 |             sampler = ReactiveNestedSampler(
1718 |                 active_param_names, loglikelihood, prior_transform,
1719 |                 log_dir=outdir, resume='overwrite', derived_param_names=derived_param_names)
1720 |         print("  running without step sampler ...")
1721 |         loglikelihood.sampler = sampler
1722 |         sampler_args = dict(
1723 |             frac_remain=0.5, max_num_improvement_loops=0, min_num_live_points=args.num_live_points,
1724 |             dlogz=10, min_ess=100, cluster_num_live_points=0, viz_callback=None
1725 |         )
1726 |         sampler.run(max_ncalls=10000, region_class=RobustEllipsoidRegion, **sampler_args)
1727 |         print("  running with step sampler ...")
1728 |         sampler.stepsampler = ultranest.stepsampler.SliceSampler(
1729 |             nsteps=20, generate_direction=ultranest.stepsampler.generate_mixture_random_direction)
1730 |         sampler.run(region_class=SimpleRegion, **sampler_args)
1731 |         print('loglikelihood stats:', loglikelihood.stats())
1732 |         sampler.print_results()
1733 |     if plot:
1734 |         results = plot_results(
1735 |             sampler, prior_samples, obs, obs_fluxes, obs_errors, wobs,
1736 |             loglikelihood.cache_filters, replot=replot)
1737 |     sampler.pointstore.close()
1738 |     if results is not None:
1739 |         names, means, stds, los, his, medians, lmeans = results
1740 |         with open(outdir + '/analysis_results.txt', 'w') as fout:
1741 |             fout.write("%s" % obs['id'])
1742 |             for name, mean, std, lo, hi, med, lmean in zip(names, means, stds, los, his, medians, lmeans):
1743 |                 fout.write("\t%g\t%g\t%g\t%g\t%g\t%g" % (mean, std, lo, hi, med, lmean))
1744 |             fout.write('\n')
1745 |     try:
1746 |         results_string = open(outdir + '/analysis_results.txt', 'r').read()
1747 |     except IOError:
1748 |         results_string = None
1749 | 
1750 |     print("  results stored in %s" % outdir)
1751 |     return obs['id'], results, results_string
1752 | 
1753 | 
1754 | def main():
1755 |     """Script entry point, calls function depending on the mode."""
1756 |     if args.action == 'generate-from-prior':
1757 |         generate_fluxes()
1758 |     elif args.action == 'list-filters':
1759 |         list_filters()
1760 |     elif args.action == 'plot-model':
1761 |         plot_model()
1762 |     else:
1763 |         plot = args.plot
1764 |         # Read the observation table and complete it by adding error where
1765 |         # none is provided and by adding the systematic deviation.
1766 |         obs_table = complete_obs_table(read_table(data_file), column_list,
1767 |                                        filters, TOLERANCE, lim_flag, 
1768 |                                        systematic_deviation=0.1)
1769 | 
1770 |         # pick observations to analyse in this process
1771 |         obs_table_here = obs_table[args.offset::args.every]
1772 |         indices = np.arange(len(obs_table_here))
1773 |         if args.randomize:
1774 |             np.random.shuffle(indices)
1775 |         fout = None
1776 |         # analyse observations in parallel
1777 |         MP_method = os.environ.get('MP_METHOD', 'forkserver')
1778 |         if n_cores == 1:
1779 |             # to preserve traceback for debugging run in here
1780 |             allresults = (analyse_obs_wrapper((i, len(indices), args.sampler, obs_table_here[i], plot)) for i in indices)
1781 |         elif MP_method == 'joblib':
1782 |             print(f"Parallelisation with MP_METHOD={MP_method} (if the process is stuck, change MP_METHOD)")
1783 |             try:
1784 |                 parallel = joblib.Parallel(n_jobs=n_cores, return_generator=True)  # joblib>1.2 will support this
1785 |             except TypeError:
1786 |                 parallel = joblib.Parallel(n_jobs=n_cores) # fall-back for joblib <= 1.2
1787 |             allresults = parallel(
1788 |                 joblib.delayed(analyse_obs_wrapper)(
1789 |                 (i, len(indices), args.sampler, obs_table_here[i], plot)) for i in indices)
1790 |         else:
1791 |             print(f"Parallelisation with MP_METHOD={MP_method} (if the process is stuck, change MP_METHOD)")
1792 |             mp_ctx = multiprocessing.get_context(MP_method)
1793 |             with mp_ctx.Pool(n_cores, maxtasksperchild=3) as pool:
1794 |                 # farm out to process pool
1795 |                 allresults = pool.imap_unordered(
1796 |                     analyse_obs_wrapper,
1797 |                     ((args.sampler, obs_table_here[i], plot) for i in indices))
1798 |         for id, result, results_string in allresults:
1799 |             if results_string is None:
1800 |                 print("no result to store for", id, ". Delete plots, otherwise results will not be reanalysed.")
1801 |                 continue
1802 |             derived_names = analysed_variables + ['chi2']
1803 |             names = param_names + derived_names
1804 |             names += ['s_' + n for n in derived_names if 'sfh.sfr' in n or 'agn.lum' in n or 'Lbol' in n]
1805 |             names += ['totalflux_' + filtername for filtername in filters]
1806 |             names += ['AGNflux_' + filtername for filtername in filters]
1807 |             names += ['GALflux_' + filtername for filtername in filters]
1808 |             if fout is None:
1809 |                 fout = open(data_file + '_analysis_results.txt', 'w')
1810 |                 fout.write('# id')
1811 |                 for name in names:
1812 |                     fout.write('\t%s_mean\t%s_std\t%s_lo\t%s_hi\t%s_med\t%s_lmean' % (name, name, name, name, name, name))
1813 |                 fout.write('\n')
1814 |             fout.write(results_string)
1815 |             fout.flush()
1816 | 
1817 |         print("analysing %d observations done." % len(obs_table_here))
1818 | 
1819 | 
1820 | if __name__ == '__main__':
1821 |     main()
1822 | 


--------------------------------------------------------------------------------