├── .gitignore ├── CITATION.cff ├── LICENSE ├── MANIFEST.in ├── README.rst ├── braincoder ├── __init__.py ├── barstimuli.py ├── data │ └── szinte2024 │ │ ├── dm_gazecenter.npz │ │ ├── gauss_parameters.tsv │ │ ├── grid_coordinates.tsv │ │ └── mask-v1_bold.tsv.gz ├── estimators.py ├── hrf.py ├── models.py ├── optimize.py ├── stimuli.py ├── tests │ └── test_gaussprf.py └── utils │ ├── __init__.py │ ├── data.py │ ├── formatting.py │ ├── math.py │ ├── mcmc.py │ ├── stats.py │ └── visualize.py ├── docs ├── Makefile ├── _static │ └── css │ │ └── custom.css ├── bibliography.rst ├── conf.py ├── index.rst ├── make.bat ├── references.bib └── tutorial │ ├── index.rst │ ├── lesson1.rst │ ├── lesson2.rst │ ├── lesson3.rst │ ├── lesson4.rst │ ├── lesson5.rst │ ├── lesson6.rst │ └── lesson7.rst ├── examples ├── 00_encodingdecoding │ ├── README.txt │ ├── decode.py │ ├── decode2d.py │ ├── decode_v1.py │ ├── decode_visual.py │ ├── encoding_model.py │ ├── fisher_information.ipynb │ ├── fisher_information.py │ ├── fit_prf.ipynb │ ├── fit_prf.py │ ├── fit_residuals.py │ ├── invert_model.py │ ├── linear_encoding_model.py │ └── masked_stimulus_decoding.py ├── README.txt └── masked_stimulus_decoding.ipynb ├── notebooks ├── Fit 2D Gaussian PRF.ipynb ├── Fit 2dPointPRF.ipynb ├── Gaussian PRF demo.ipynb ├── GazeCenterFS_vd.mat └── encoding_decoding.ipynb ├── setup.py └── tests ├── test_gauss_rf.py ├── test_gauss_to_stick.py ├── test_glm.py ├── test_glm_t_dist.py └── test_stickmodel.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | bin/ 10 | build/ 11 | develop-eggs/ 12 | dist/ 13 | eggs/ 14 | lib/ 15 | lib64/ 16 | parts/ 17 | sdist/ 18 | var/ 19 | *.egg-info/ 20 | .installed.cfg 21 | *.egg 22 | 23 | # Installer logs 24 | pip-log.txt 25 | pip-delete-this-directory.txt 26 | 27 | # Unit test / coverage reports 28 | .tox/ 29 | .coverage 30 | .cache 31 | nosetests.xml 32 | coverage.xml 33 | 34 | # Translations 35 | *.mo 36 | 37 | # Mr Developer 38 | .mr.developer.cfg 39 | .project 40 | .pydevproject 41 | 42 | # Rope 43 | .ropeproject 44 | 45 | # Django stuff: 46 | *.log 47 | *.pot 48 | 49 | # Sphinx documentation 50 | docs/_build/ 51 | docs/auto_examples/ 52 | sg_execution_times.rst 53 | 54 | .DS_store 55 | 56 | test*.ipynb -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | message: "If you use this software, please cite it using the metadata below." 3 | title: "braincoder" 4 | version: 0.3.0 5 | doi: 10.5281/zenodo.10778413 6 | authors: 7 | - name: "Gilles de Hollander" 8 | orcid: "0000-0002-XXXX-XXXX" 9 | - name: "Maike Renkert" 10 | orcid: "0000-0002-4809-4216" 11 | - name: "Christian C. Ruff" 12 | orcid: "0000-0002-3964-2364" 13 | - name: "Tomas H. Knapen" 14 | orcid: "0000-0001-5863-8689" 15 | repository-code: "https://github.com/Gilles86/braincoder" 16 | date-released: "2024-12-06" 17 | abstract: | 18 | braincoder is a Python package designed to help researchers fit encoding models to neural data 19 | and decode stimulus features from unseen neural data. It leverages computational modeling to 20 | bridge neuroscience and data analysis, providing robust tools for understanding neural representation. -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Tomas Knapen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS.rst 2 | include CONTRIBUTING.rst 3 | include HISTORY.rst 4 | include LICENSE 5 | include README.rst 6 | include braincoder/data/* 7 | 8 | recursive-include tests * 9 | recursive-exclude * __pycache__ 10 | recursive-exclude * *.py[co] 11 | 12 | recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif 13 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Welcome to Braincoder's documentation! 2 | ====================================== 3 | 4 | **Braincoder** is a package to fit encoding models to neural data (for now fMRI) and 5 | to then *invert* those models to decode stimulus information from neural data. 6 | 7 | Important links 8 | =============== 9 | 10 | - Official source code repo: https://github.com/Gilles86/braincoder/tree/main 11 | - HTML documentation (stable release): https://braincoder-devs.github.io/ 12 | 13 | Installation 14 | ============ 15 | 16 | Note that you need an environment with both `tensorflow-probability` and 17 | `tensorflow`. 18 | 19 | Set up miniforge 20 | ----------------- 21 | 22 | (Only do this if you don't have conda installed) 23 | I recommend using `miniforge `_, 24 | make sure you use the ``mamba``-solver and set ``channel-priority`` to ``strict``: 25 | 26 | .. code-block:: bash 27 | 28 | # Install mamba solver and set channel priority 29 | conda install mamba -n base -c conda-forge 30 | conda config --set channel_priority strict. 31 | 32 | Install braincoder 33 | ------------------ 34 | 35 | Here we create a new environment called `braincoder` with the required packages: 36 | 37 | .. code-block:: bash 38 | 39 | mamba create --name braincoder tensorflow-probability tensorflow -c conda-forge 40 | mamba activate braincoder 41 | pip install git+https://github.com/Gilles86/braincoder.git 42 | 43 | How to Cite 44 | =========== 45 | 46 | If you use **Braincoder** in your research, please cite it using the following information: 47 | 48 | > de Hollander, G., Renkert, M., Ruff, C. C., & Knapen, T. H. (2024). *Braincoder: A package for fitting encoding models to neural data and decoding stimulus features*. `Zenodo `_. DOI: `10.5281/zenodo.10778413 `_. 49 | 50 | Alternatively, use this BibTeX entry: 51 | 52 | .. code-block:: bibtex 53 | 54 | @software{deHollander2024braincoder, 55 | author = {Gilles de Hollander and Maike Renkert and Christian C. Ruff and Tomas H. Knapen}, 56 | title = {braincoder: A package for fitting encoding models to neural data and decoding stimulus features}, 57 | year = {2024}, 58 | publisher = {Zenodo}, 59 | doi = {10.5281/zenodo.10778413}, 60 | url = {https://github.com/Gilles86/braincoder} 61 | } 62 | 63 | By citing this software, you help support open-source development and proper crediting in academic research. 64 | 65 | Usage 66 | ===== 67 | 68 | Please have a look at the `tutorials `_ to get started. -------------------------------------------------------------------------------- /braincoder/__init__.py: -------------------------------------------------------------------------------- 1 | """Top-level package for braincoder.""" 2 | 3 | __author__ = """Gilles de Hollander""" 4 | __email__ = 'giles.de.hollander@gmail.com' 5 | __version__ = '0.1.0' -------------------------------------------------------------------------------- /braincoder/barstimuli.py: -------------------------------------------------------------------------------- 1 | from tqdm import tqdm 2 | import tensorflow as tf 3 | import numpy as np 4 | import pandas as pd 5 | from .optimize import StimulusFitter 6 | import logging 7 | import tensorflow_probability as tfp 8 | from tensorflow_probability import bijectors as tfb 9 | from .utils.mcmc import cleanup_chain, sample_hmc, Periodic 10 | 11 | 12 | class BarStimulusFitter(StimulusFitter): 13 | 14 | def __init__(self, data, model, grid_coordinates, omega, dof=None, 15 | max_radius=None, max_width=None, bar_intensity=1.0, 16 | max_intensity=None, 17 | baseline_image=None): 18 | 19 | self.data = data 20 | self.model = model 21 | self.grid_coordinates = pd.DataFrame( 22 | grid_coordinates, columns=['x', 'y']) 23 | self.model.omega = omega 24 | self.model.omega_chol = tf.linalg.cholesky(omega).numpy() 25 | self.model.dof = dof 26 | self.bar_intensity = bar_intensity 27 | 28 | if max_intensity is None: 29 | self.max_intensity = self.bar_intensity 30 | else: 31 | self.max_intensity = max_intensity 32 | 33 | 34 | if baseline_image is None: 35 | self.baseline_image = None 36 | else: 37 | self.baseline_image = baseline_image.reshape(len(self.grid_coordinates)) 38 | 39 | self.min_x, self.max_x = self.grid_coordinates['x'].min( 40 | ), self.grid_coordinates['x'].max() 41 | self.min_y, self.max_y = self.grid_coordinates['y'].min( 42 | ), self.grid_coordinates['y'].max() 43 | 44 | if max_radius is None: 45 | self.max_radius = np.sqrt( 46 | np.max(self.grid_coordinates['x'])**2 + np.max(self.grid_coordinates['y'])**2) 47 | else: 48 | self.max_radius = max_radius 49 | 50 | if max_width is None: 51 | self.max_width = .5 * self.max_radius 52 | else: 53 | self.max_width = max_width 54 | 55 | self.max_radius = np.float32(self.max_radius) 56 | self.max_width = np.float32(self.max_width) 57 | 58 | if self.model.weights is None: 59 | self.model.weights 60 | 61 | def fit_grid(self, x, y, width, parameterization='xy', include_xy=True): 62 | 63 | if parameterization == 'xy': 64 | 65 | grid = pd.MultiIndex.from_product( 66 | [x, y, width], names=['x', 'y', 'width']).to_frame(index=False).astype(np.float32) 67 | 68 | logging.info('Built grid of {len(par_grid)} bar settings...') 69 | 70 | bars = make_bar_stimuli(self.grid_coordinates.values, 71 | grid['x'].values[np.newaxis, ...], 72 | grid['y'].values[np.newaxis, ...], 73 | grid['width'].values[np.newaxis, ...], 74 | intensity=self.bar_intensity)[0] 75 | 76 | if self.baseline_image is not None: 77 | bars = bars + self.baseline_image[tf.newaxis, ...] 78 | bars = tf.clip_by_value(bars, 0, self.max_intensity) 79 | 80 | else: 81 | raise NotImplementedError 82 | 83 | return self._fit_grid(grid, bars) 84 | 85 | def _fit_grid(self, grid, bars): 86 | 87 | data = self.data.values 88 | model = self.model 89 | parameters = self.model.parameters.values[np.newaxis, ...] 90 | weights = None if model.weights is None else model.weights.values[np.newaxis, ...] 91 | 92 | if hasattr(self.model, 'hrf_model'): 93 | 94 | bars = tf.concat((tf.zeros((bars.shape[0], 95 | 1, 96 | bars.shape[1])), 97 | bars[:, tf.newaxis, :], 98 | tf.zeros((bars.shape[0], 99 | len(self.model.hrf_model.hrf)-1, 100 | bars.shape[1]))), 101 | 1) 102 | 103 | hrf_shift = np.argmax(model.hrf_model.hrf) 104 | 105 | ts_prediction = model._predict(bars, parameters, weights) 106 | 107 | baseline = ts_prediction[:, 0, :] 108 | 109 | ts_prediction_summed_over_hrf = tf.reduce_sum( 110 | ts_prediction - baseline[:, tf.newaxis, :], 1) / tf.reduce_sum(model.hrf_model.hrf) + baseline 111 | 112 | ll = self.model._likelihood_timeseries(data[tf.newaxis, ...], 113 | ts_prediction_summed_over_hrf[:, 114 | tf.newaxis, :], 115 | self.model.omega_chol, 116 | self.model.dof, 117 | logp=True, 118 | normalize=False) 119 | 120 | ll = tf.concat((tf.roll(ll, -hrf_shift, 1) 121 | [:, :-hrf_shift], tf.ones((len(bars), hrf_shift)) * -1e12), 1) 122 | 123 | else: 124 | bars = bars[:, tf.newaxis, :] 125 | 126 | ll = self.model._likelihood(bars, data[tf.newaxis, ...], parameters, weights, 127 | self.model.omega_chol, self.model.dof, logp=True) 128 | ll = pd.DataFrame(ll.numpy().T, columns=pd.MultiIndex.from_frame(grid)) 129 | 130 | best_pars = ll.columns.to_frame().iloc[ll.values.argmax(1)] 131 | best_pars.index = self.data.index 132 | 133 | if 'x' not in best_pars.columns: 134 | best_pars['x'] = np.cos(best_pars['angle']) * best_pars['radius'] 135 | best_pars['y'] = np.sin(best_pars['angle']) * best_pars['radius'] 136 | 137 | if 'angle' not in best_pars.columns: 138 | best_pars = get_angle_radius_from_xy(best_pars) 139 | 140 | return best_pars.astype(np.float32) 141 | 142 | def fit(self, init_pars, learning_rate=0.01, max_n_iterations=500, min_n_iterations=100, lag=100, 143 | relevant_frames=None, rtol=1e-7, parameterization='xy', include_xy=True): 144 | 145 | opt = tf.optimizers.Adam(learning_rate=learning_rate) 146 | 147 | # init_pars: x, y, width 148 | 149 | if parameterization == 'xy': 150 | if hasattr(init_pars, 'values'): 151 | init_pars = init_pars[['x', 'y', 'width']].values 152 | 153 | if np.any(init_pars[:, 0] < self.min_x): 154 | raise ValueError( 155 | f'All x-values should not be less than {self.min_x}') 156 | 157 | if np.any(init_pars[:, 0] > self.max_x): 158 | raise ValueError( 159 | f'All x-values should not be more than {self.max_x}') 160 | 161 | if np.any(init_pars[:, 1] < self.min_y): 162 | raise ValueError( 163 | f'All y-values should not be less than {self.min_y}') 164 | 165 | if np.any(init_pars[:, 1] > self.max_y): 166 | raise ValueError( 167 | f'All y-values should not be more than {self.max_y}') 168 | 169 | if np.any(np.abs(init_pars[:, 2]) < 0.0): 170 | raise ValueError('All widths should be positive') 171 | 172 | if np.any(np.abs(init_pars[:, 2]) > self.max_width): 173 | raise ValueError( 174 | f'All widths should be less than {self.max_width}') 175 | 176 | if (relevant_frames is not None) and (len(init_pars) > len(relevant_frames)): 177 | init_pars = init_pars[relevant_frames, :] 178 | 179 | x_bijector = Periodic(low=np.float32(self.min_x - self.max_width/2.), 180 | high=np.float32(self.max_x + self.max_width/2.)) 181 | 182 | y_bijector = Periodic(low=np.float32(self.min_y - self.max_width/2.), 183 | high=np.float32(self.max_y + self.max_width/2.)) 184 | 185 | x_ = tf.Variable(name='x', 186 | shape=len(init_pars), 187 | initial_value=init_pars[:, 0]) 188 | 189 | y_ = tf.Variable(name='y', 190 | shape=len(init_pars), 191 | initial_value=init_pars[:, 1]) 192 | 193 | elif parameterization == 'angle': 194 | 195 | if hasattr(init_pars, 'values'): 196 | init_pars = init_pars[['angle', 'radius', 'width']].values 197 | 198 | if np.any(init_pars[:, 0] < -.5*np.pi): 199 | raise ValueError( 200 | 'All angles should be more than -1/2 pi radians') 201 | 202 | if np.any(init_pars[:, 0] > .5*np.pi): 203 | raise ValueError( 204 | 'All angles should be less than 1/2 pi radians') 205 | 206 | if np.any(np.abs(init_pars[:, 1]) > self.max_radius): 207 | raise ValueError( 208 | f'All radiuses should be within (-{self.max_radius}, {self.max_radius})') 209 | 210 | if np.any(np.abs(init_pars[:, 2]) < 0.0): 211 | raise ValueError('All widths should be positive') 212 | 213 | if np.any(np.abs(init_pars[:, 2]) > self.max_width): 214 | raise ValueError( 215 | f'All widths should be less than {self.max_width}') 216 | 217 | if (relevant_frames is not None) and (len(init_pars) > len(relevant_frames)): 218 | init_pars = init_pars[relevant_frames, :] 219 | 220 | init_pars[:, 0] = tf.clip_by_value( 221 | init_pars[:, 0], -.5*np.pi + 1e-6, .5 * np.pi-1e-6) 222 | init_pars[:, 1] = tf.clip_by_value( 223 | init_pars[:, 1], -self.max_radius + 1e-6, self.max_radius - 1e-6) 224 | init_pars[:, 2] = tf.clip_by_value( 225 | init_pars[:, 2], 1e-6, self.max_width - 1e-6) 226 | 227 | orient_x = tf.Variable(name='orient_x', 228 | shape=len(init_pars), 229 | initial_value=np.cos(init_pars[:, 0])) 230 | 231 | orient_y = tf.Variable(name='orient_y', 232 | shape=len(init_pars), 233 | initial_value=np.sin(init_pars[:, 0])) 234 | 235 | radius_bijector = Periodic(low=np.float32(-self.max_radius), 236 | high=np.float32(self.max_radius)) 237 | 238 | radius_ = tf.Variable(name='radius', 239 | shape=len(init_pars), 240 | initial_value=radius_bijector.inverse(init_pars[:, 1])) 241 | 242 | else: 243 | raise NotImplementedError 244 | 245 | width_bijector = tfb.Sigmoid(low=np.float32(0.0), 246 | high=np.float32(self.max_width)) 247 | 248 | width_ = tf.Variable(name='width', 249 | shape=len(init_pars), 250 | initial_value=width_bijector.inverse(init_pars[:, 2])) 251 | 252 | if parameterization == 'xy': 253 | trainable_vars = [x_, y_, width_] 254 | bijectors = [x_bijector, y_bijector, width_bijector] 255 | par_names = ['x', 'y', 'width'] 256 | elif parameterization == 'angle': 257 | trainable_vars = [orient_x, orient_y, radius_, width_] 258 | bijectors = [tfb.Identity(), tfb.Identity(), 259 | radius_bijector, width_bijector] 260 | par_names = ['orient_x', 'orient_y', 'radius', 'width'] 261 | 262 | pbar = tqdm(range(max_n_iterations)) 263 | self.costs = np.ones(max_n_iterations) * 1e12 264 | 265 | likelihood = self.build_likelihood_function(relevant_frames, parameterization=parameterization) 266 | 267 | for step in pbar: 268 | with tf.GradientTape() as tape: 269 | 270 | untransformed_pars = [bijector.forward( 271 | par) for bijector, par in zip(bijectors, trainable_vars)] 272 | ll = likelihood(*untransformed_pars)[0] 273 | cost = -ll 274 | 275 | gradients = tape.gradient(cost, 276 | trainable_vars) 277 | 278 | opt.apply_gradients(zip(gradients, trainable_vars)) 279 | 280 | pbar.set_description(f'LL: {ll:6.4f}') 281 | 282 | self.costs[step] = cost.numpy() 283 | 284 | previous_cost = self.costs[np.max((step - lag, 0))] 285 | if step > min_n_iterations: 286 | if np.sign(previous_cost) == np.sign(cost): 287 | if np.sign(cost) == 1: 288 | if (cost / previous_cost) > 1 - rtol: 289 | break 290 | else: 291 | if (cost / previous_cost) < 1 - rtol: 292 | break 293 | 294 | fitted_pars_ = np.concatenate( 295 | [par.numpy()[:, np.newaxis] for par in untransformed_pars], axis=1) 296 | 297 | if relevant_frames is None: 298 | fitted_pars = pd.DataFrame(fitted_pars_, columns=par_names, 299 | index=self.data.index, 300 | dtype=np.float32) 301 | else: 302 | 303 | fitted_pars = pd.DataFrame(np.nan * np.zeros((self.data.shape[0], len(trainable_vars))), columns=par_names, 304 | index=self.data.index, 305 | dtype=np.float32) 306 | fitted_pars.iloc[relevant_frames, :] = fitted_pars_ 307 | 308 | print(fitted_pars) 309 | if parameterization == 'xy': 310 | fitted_pars = get_angle_radius_from_xy(fitted_pars) 311 | 312 | elif parameterization == 'angle': 313 | print(relevant_frames) 314 | fitted_pars.at[relevant_frames, 'angle'] = tf.math.atan(orient_y / orient_x).numpy() 315 | fitted_pars['x'] = np.cos( 316 | fitted_pars['angle']) * fitted_pars['radius'] 317 | fitted_pars['y'] = np.sin( 318 | fitted_pars['angle']) * fitted_pars['radius'] 319 | 320 | 321 | return fitted_pars 322 | 323 | def build_likelihood_function(self, relevant_frames=None, falloff_speed=1000., parameterization='xy', n_batches=1): 324 | 325 | data = self.data.values[tf.newaxis, ...] 326 | grid_coordinates = self.grid_coordinates 327 | model = self.model 328 | parameters = self.model.parameters.values[tf.newaxis, ...] 329 | weights = None if model.weights is None else model.weights.values[tf.newaxis, ...] 330 | 331 | if self.baseline_image is None: 332 | @tf.function 333 | def add_baseline(bars): 334 | return bars 335 | else: 336 | print('Including base image (e.g., fixation image) into estimation') 337 | @tf.function 338 | def add_baseline(bars): 339 | bars = bars + self.baseline_image[tf.newaxis, :] 340 | bars = tf.clip_by_value(bars, 0, self.max_intensity) 341 | return bars 342 | 343 | if relevant_frames is None: 344 | 345 | if parameterization == 'xy': 346 | @tf.function 347 | def likelihood(x, y, width): 348 | 349 | bars = make_bar_stimuli( 350 | grid_coordinates, 351 | x[tf.newaxis, ...], 352 | y[tf.newaxis, ...], 353 | width[tf.newaxis, ...], 354 | falloff_speed=falloff_speed, 355 | intensity=self.bar_intensity) 356 | 357 | bars = add_baseline(bars) 358 | 359 | ll = self.model._likelihood( 360 | bars, data, parameters, weights, self.model.omega_chol, dof=self.model.dof, logp=True) 361 | 362 | return tf.reduce_sum(ll, 1) 363 | 364 | elif parameterization == 'angle': 365 | @tf.function 366 | def likelihood(orient_x, orient_y, radius, width): 367 | 368 | angle = tf.math.atan(orient_y/orient_x) 369 | 370 | bars = make_bar_stimuli2( 371 | grid_coordinates, 372 | angle[tf.newaxis, ...], 373 | radius[tf.newaxis, ...], 374 | width[tf.newaxis, ...], 375 | falloff_speed=falloff_speed, 376 | intensity=self.bar_intensity) 377 | 378 | bars = add_baseline(bars) 379 | 380 | ll = self.model._likelihood( 381 | bars, data, parameters, weights, self.model.omega_chol, dof=self.model.dof, logp=True) 382 | 383 | return tf.reduce_sum(ll, 1) 384 | else: 385 | raise NotImplementedError 386 | 387 | else: 388 | relevant_frames = tf.constant(relevant_frames, tf.int32) 389 | 390 | size_ = (n_batches, data.shape[1], len(grid_coordinates)) 391 | size_ = tf.constant(size_, dtype=tf.int32) 392 | 393 | time_ix, batch_ix = np.meshgrid(relevant_frames, range(n_batches)) 394 | indices = np.zeros( 395 | (n_batches, len(relevant_frames), 2), dtype=np.int32) 396 | indices[..., 0] = batch_ix 397 | indices[..., 1] = time_ix 398 | 399 | if parameterization == 'xy': 400 | @tf.function 401 | def likelihood(x, 402 | y, 403 | width): 404 | 405 | bars = make_bar_stimuli( 406 | grid_coordinates.values, x, y, width, 407 | falloff_speed=falloff_speed, 408 | intensity=self.bar_intensity) 409 | 410 | bars = add_baseline(bars) 411 | 412 | stimulus = tf.scatter_nd(indices, 413 | bars, 414 | size_) 415 | 416 | ll = self.model._likelihood( 417 | stimulus, data, parameters, weights, self.model.omega_chol, dof=self.model.dof, logp=True) 418 | 419 | sll = tf.reduce_sum(ll, 1) 420 | 421 | return sll 422 | 423 | elif parameterization == 'angle': 424 | 425 | @tf.function 426 | def likelihood(orient_x, 427 | orient_y, 428 | radius, 429 | width): 430 | 431 | angle = tf.math.atan(orient_y/orient_x) 432 | 433 | bars = make_bar_stimuli2( 434 | grid_coordinates, 435 | angle[tf.newaxis, ...], 436 | radius[tf.newaxis, ...], 437 | width[tf.newaxis, ...], 438 | falloff_speed=falloff_speed, 439 | intensity=self.bar_intensity) 440 | 441 | bars = add_baseline(bars) 442 | 443 | stimulus = tf.scatter_nd(indices, 444 | bars, 445 | size_) 446 | 447 | ll = self.model._likelihood( 448 | stimulus, data, parameters, weights, self.model.omega_chol, dof=self.model.dof, logp=True) 449 | 450 | sll = tf.reduce_sum(ll, 1) 451 | 452 | return sll 453 | 454 | return likelihood 455 | 456 | def sample_posterior(self, 457 | init_pars, 458 | n_chains, 459 | relevant_frames=None, 460 | step_size=0.0001, 461 | n_burnin=10, 462 | n_samples=10, 463 | max_tree_depth=10, 464 | unrolled_leapfrog_steps=1, 465 | falloff_speed=1000., 466 | target_accept_prob=0.85, 467 | parameterization='xy'): 468 | 469 | init_pars = init_pars.astype(np.float32) 470 | 471 | if (relevant_frames is not None) and (len(init_pars) > len(relevant_frames)): 472 | init_pars = init_pars.iloc[relevant_frames, :] 473 | 474 | 475 | if parameterization == 'xy': 476 | 477 | init_pars = init_pars[['x', 'y', 'width']] 478 | 479 | bijectors = [Periodic(low=self.min_x - self.max_width/2., 480 | high=self.max_x + self.max_width/2.), # x 481 | Periodic(low=self.min_y - self.max_width/2., 482 | high=self.max_y + self.max_width/2.), 483 | tfb.Sigmoid(low=np.float32(0.0), high=self.max_width)] # width 484 | 485 | initial_state = list( 486 | np.repeat(init_pars.values.T[:, np.newaxis, :], n_chains, 1)) 487 | 488 | likelihood = self.build_likelihood_function( 489 | relevant_frames, falloff_speed=falloff_speed, n_batches=n_chains, parameterization=parameterization) 490 | 491 | step_size = [tf.fill([n_chains] + [1] * (len(s.shape) - 1), 492 | tf.constant(step_size, np.float32)) for s in initial_state] 493 | samples, stats = sample_hmc( 494 | initial_state, step_size, likelihood, bijectors, num_steps=n_samples, burnin=n_burnin, 495 | target_accept_prob=target_accept_prob, unrolled_leapfrog_steps=unrolled_leapfrog_steps, 496 | max_tree_depth=max_tree_depth) 497 | 498 | if relevant_frames is None: 499 | frame_index = self.data.index 500 | else: 501 | frame_index = self.data.index[relevant_frames] 502 | 503 | cleaned_up_chains = [cleanup_chain(chain.numpy(), init_pars.columns[ix], frame_index) for ix, chain in enumerate(samples)] 504 | 505 | samples = pd.concat(cleaned_up_chains, 1) 506 | 507 | return samples, stats 508 | 509 | 510 | @tf.function 511 | def make_bar_stimuli(grid_coordinates, x, y, width, falloff_speed=1000., intensity=1.0): 512 | 513 | x_ = grid_coordinates[:, 0] 514 | y_ = grid_coordinates[:, 1] 515 | 516 | radius2 = x**2 + y**2 517 | radius = tf.math.sqrt(radius2) 518 | a = x / radius 519 | b = y / radius 520 | c = -radius2 / radius 521 | 522 | distance = tf.abs(a[..., tf.newaxis] * x_[tf.newaxis, tf.newaxis, ...] + 523 | b[..., tf.newaxis] * y_[tf.newaxis, tf.newaxis, ...] + 524 | c[..., tf.newaxis]) / tf.sqrt(a[..., tf.newaxis]**2 + b[..., tf.newaxis]**2) 525 | 526 | bar = tf.math.sigmoid( 527 | (-distance + width[..., tf.newaxis] / 2) * falloff_speed) * intensity 528 | 529 | return bar 530 | 531 | 532 | @tf.function 533 | def make_bar_stimuli2(grid_coordinates, angle, radius, width, falloff_speed=1000., intensity=1.0): 534 | 535 | # batch x stimulus x stimulus_dimension 536 | 537 | x = grid_coordinates[:, 0] 538 | y = grid_coordinates[:, 1] 539 | 540 | a = tf.cos(angle) 541 | b = tf.sin(angle) 542 | c = tf.sqrt(a**2 + b**2) * - radius 543 | 544 | distance = tf.abs(a[..., tf.newaxis] * x[tf.newaxis, tf.newaxis, ...] + 545 | b[..., tf.newaxis] * y[tf.newaxis, tf.newaxis, ...] + 546 | c[..., tf.newaxis]) / tf.sqrt(a[..., tf.newaxis]**2 + b[..., tf.newaxis]**2) 547 | 548 | bar = tf.math.sigmoid( 549 | (-distance + width[..., tf.newaxis] / 2) * falloff_speed) * intensity 550 | 551 | return bar 552 | 553 | 554 | def get_angle_radius_from_xy(d): 555 | d = d.assign(angle=np.arctan2(d['y'], d['x'])) 556 | d = d.assign(radius=np.sqrt(d['y']**2 + d['x']**2)) 557 | d = d.assign(ecc=np.abs(d['radius'])) 558 | 559 | return constrain_angle(d) 560 | 561 | 562 | 563 | def constrain_angle(d): 564 | 565 | pi5 = .5 * np.pi 566 | d = d.assign(radius=d['radius'].where(d['angle'].abs() < pi5, -d['radius'])) 567 | d = d.assign(angle=d['angle'].where(d['angle'] < pi5, d['angle'] - np.pi)) 568 | d = d.assign(angle=d['angle'].where(d['angle'] > -pi5, d['angle'] + np.pi)) 569 | 570 | return d 571 | -------------------------------------------------------------------------------- /braincoder/data/szinte2024/dm_gazecenter.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gilles86/braincoder/7001c05a07ca7accf471b40b0822a7d1ac525b75/braincoder/data/szinte2024/dm_gazecenter.npz -------------------------------------------------------------------------------- /braincoder/data/szinte2024/mask-v1_bold.tsv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gilles86/braincoder/7001c05a07ca7accf471b40b0822a7d1ac525b75/braincoder/data/szinte2024/mask-v1_bold.tsv.gz -------------------------------------------------------------------------------- /braincoder/estimators.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from .utils.math import log2 3 | import tensorflow as tf 4 | 5 | 6 | class MutualInformationEstimator(object): 7 | 8 | def __init__(self, model, stimulus_range, omega=None, dof=None): 9 | self.stimulus_range = np.array(stimulus_range).astype(np.float32) 10 | 11 | if self.stimulus_range.ndim == 1: 12 | self.stimulus_range = stimulus_range[:, np.newaxis].astype( 13 | np.float32) 14 | 15 | if self.stimulus_range.shape[1] > 1: 16 | raise NotImplementedError() 17 | 18 | self.model = model 19 | self.n_units = self.model.parameters.shape[0] 20 | 21 | self.p_stimulus = 1. / len(self.stimulus_range) 22 | 23 | self.omega = omega 24 | self.omega_chol = tf.linalg.cholesky(omega).numpy() 25 | self.dof = dof 26 | 27 | if self.model.weights is None: 28 | self.weights_ = None 29 | else: 30 | self.weights_ = self.model.weights.values[np.newaxis, ...] 31 | 32 | def estimate_mi(self, n=1000, uselog=True): 33 | 34 | resid_dist = self.model.get_residual_dist( 35 | self.n_units, self.omega_chol, self.dof) 36 | 37 | noise = resid_dist.sample(n) 38 | 39 | 40 | predictions = self.model._predict(self.stimulus_range[np.newaxis, ...], 41 | self.model.parameters.values[np.newaxis, ...], 42 | self.weights_) 43 | 44 | # n samples x actual_stimuli x n_voxels 45 | neural_data = predictions + noise[:, tf.newaxis, :] 46 | 47 | p_stimulus = self.p_stimulus 48 | 49 | if uselog: 50 | 51 | # logp_joint = self.model._likelihood(self.stimulus_range[np.newaxis, :, :], 52 | # neural_data, 53 | # self.model.parameters.values[np.newaxis, ...], 54 | # self.weights_, 55 | # self.omega_chol, 56 | # self.dof, 57 | # logp=True) 58 | 59 | logp_joint = resid_dist.log_prob(noise)[:, tf.newaxis] 60 | 61 | # n x n_stimuli 62 | p_joint = np.exp(logp_joint, dtype=np.float128) * p_stimulus 63 | 64 | residuals = neural_data[:, :, tf.newaxis, :] - \ 65 | predictions[:, tf.newaxis, :, :] 66 | 67 | # Still needs to be summed over 2 dimension 68 | # n x n_stimuli x n_hypothetical stimuli 69 | logp_data = resid_dist.log_prob(residuals) 70 | p_data = np.exp(logp_data, dtype=np.float128) 71 | p_data = np.sum(p_data, 2) / p_data.shape[2] 72 | 73 | print(p_joint / (p_data * np.float128(p_stimulus))) 74 | 75 | mi = 1. / n * np.sum(p_joint * np.log2(p_joint / (p_data * np.float128(p_stimulus)))) 76 | 77 | return np.array([mi]) 78 | 79 | 80 | else: 81 | # p_joint = self.model._likelihood(self.stimulus_range[np.newaxis, :, :], 82 | # neural_data, 83 | # self.model.parameters.values[np.newaxis, ...], 84 | # self.weights_, 85 | # self.omega_chol, 86 | # self.dof) * p_stimulus 87 | 88 | 89 | p_joint = resid_dist.prob(noise)[:, tf.newaxis] * p_stimulus 90 | # n samples x n_simulated x n_hypothetical x n_voxels 91 | residuals = neural_data[:, :, tf.newaxis, :] - \ 92 | predictions[:, tf.newaxis, :, :] 93 | 94 | p_data = resid_dist.prob(residuals) 95 | p_data = tf.reduce_sum(p_data, 2) / p_data.shape[2] 96 | 97 | mi = 1. / n * tf.reduce_sum(p_joint * 98 | log2(p_joint / (p_data * p_stimulus))) 99 | 100 | return mi.numpy() 101 | -------------------------------------------------------------------------------- /braincoder/hrf.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import tensorflow_probability as tfp 4 | from tensorflow.math import sigmoid 5 | from .utils import logit 6 | 7 | class HRFModel(object): 8 | 9 | def __init__(self, unique_hrfs=False): 10 | self.set_unique_hrfs(unique_hrfs) 11 | 12 | def set_unique_hrfs(self, unique_hrfs): 13 | self.unique_hrfs = unique_hrfs 14 | 15 | if self.unique_hrfs: 16 | self._convolve = self._convolve_unique 17 | else: 18 | self._convolve = self._convolve_shared 19 | 20 | @tf.function 21 | def convolve(self, timeseries, **kwargs): 22 | 23 | hrf = self.get_hrf(**kwargs) 24 | 25 | if self.oversampling == 1: 26 | return self._convolve(timeseries, hrf=hrf) 27 | 28 | else: 29 | upsampled_timeseries = self._upsample(timeseries) 30 | cts = self._convolve(upsampled_timeseries, hrf=hrf) 31 | return self._downsample(cts) 32 | 33 | def _convolve_shared(self, timeseries, hrf): 34 | # timeseries: returns: n_batch x n_timepoints x n_units 35 | # hrf: n_hrf_timepoints 36 | pad_ = tf.tile(timeseries[:, :1, :], [1, len(hrf), 1]) 37 | timeseries_padded = tf.concat((pad_, timeseries), 1) 38 | 39 | cts = tf.nn.conv2d(timeseries_padded[:, :, :, tf.newaxis], 40 | # THIS IS WEIRD TENSORFLOW BEHAVIOR 41 | hrf[:, tf.newaxis, 42 | :, tf.newaxis][::-1], 43 | strides=[1, 1], 44 | padding='VALID')[:, :, :, 0] 45 | 46 | return cts[:, 1:, :] 47 | 48 | def _convolve_unique(self, timeseries, hrf): 49 | # timeseries: returns: n_batch x n_timepoints x n_units 50 | # hrf: n_hrf_timepoints x n_units 51 | print(timeseries.shape, hrf.shape) 52 | n, m = timeseries.shape[1], timeseries.shape[2] 53 | 54 | pad_ = tf.tile(timeseries[:, :1, :], [1, len(hrf), 1]) 55 | timeseries_padded = tf.concat((pad_, timeseries), 1) 56 | 57 | # Reshape the timeseries data to 4D 58 | timeseries_4d = tf.reshape(timeseries_padded, [1, 1, n+len(hrf), m]) # Shape: [batch, in_height, in_width, in_channels] 59 | 60 | # Reshape the HRF filters to 4D 61 | hrf_filters_4d = tf.reshape(hrf, [1, len(hrf), m, 1]) # Shape: [filter_height, filter_width, in_channels, channel_multiplier] 62 | 63 | # Perform depthwise convolution 64 | convolved = tf.nn.depthwise_conv2d( 65 | input=timeseries_4d, 66 | filter=tf.reverse(hrf_filters_4d, axis=[1]), 67 | strides=[1, 1, 1, 1], 68 | padding='VALID' 69 | ) 70 | 71 | convolved = convolved[:, :, 1:, :] 72 | 73 | return tf.reshape(convolved, [1, n, m]) 74 | 75 | def _upsample(self, timeseries): 76 | new_length = len(timeseries) * self.oversampling 77 | timeseries_upsampled = tf.image.resize(timeseries[tf.newaxis, :, :, tf.newaxis], 78 | [new_length, timeseries.shape[1]]) 79 | 80 | return tf.squeeze(timeseries_upsampled) 81 | 82 | def _downsample(self, upsampled_timeseries): 83 | new_length = len(upsampled_timeseries) // self.oversampling 84 | timeseries_downsampled = tf.image.resize(upsampled_timeseries[tf.newaxis, :, :, tf.newaxis], 85 | [new_length, upsampled_timeseries.shape[1]]) 86 | 87 | return tf.squeeze(timeseries_downsampled) 88 | 89 | def gamma_pdf(t, a, d): 90 | """Compute the gamma probability density function at t.""" 91 | return tf.pow(t, a - 1) * tf.exp(-t / d) / (tf.pow(d, a) * tf.exp(tf.math.lgamma(a))) 92 | 93 | def spm_hrf(t, a1=6., d1=1., a2=16., d2=1., c=1./6): 94 | """Compute the SPM canonical HRF at time points t.""" 95 | hrf = gamma_pdf(t, a1, d1) - c * gamma_pdf(t, a2, d2) 96 | return hrf 97 | 98 | class SPMHRFModel(HRFModel): 99 | 100 | parameter_labels = ['hrf_delay', 'hrf_dispersion'] 101 | n_parameters = 2 102 | 103 | def __init__(self, tr, unique_hrfs=False, oversampling=1, time_length=32., onset=0., 104 | delay=6., undershoot=16., dispersion=1., 105 | u_dispersion=1., ratio=0.167): 106 | 107 | self.tr = tr 108 | self.oversampling = oversampling 109 | self.time_length = time_length 110 | self.onset = onset 111 | self.hrf_delay = delay 112 | self.undershoot = undershoot 113 | self.hrf_dispersion = dispersion 114 | self.u_dispersion = u_dispersion 115 | self.ratio = ratio 116 | 117 | self.dt = self.tr / self.oversampling 118 | self.time_stamps = np.linspace(0, self.time_length, 119 | np.rint(float(self.time_length) / self.dt).astype(np.int32)).astype(np.float32) 120 | self.time_stamps -= self.onset 121 | 122 | # time x n_hrfs 123 | self.time_stamps = self.time_stamps[:, np.newaxis] 124 | 125 | super().__init__(unique_hrfs=unique_hrfs) 126 | 127 | def get_hrf(self, hrf_delay=None, hrf_dispersion=None): 128 | 129 | if hrf_delay is None: 130 | hrf_delay = self.hrf_delay 131 | 132 | if hrf_dispersion is None: 133 | hrf_dispersion = self.hrf_dispersion 134 | 135 | return spm_hrf(self.time_stamps, a1=hrf_delay, d1=hrf_dispersion, 136 | a2=self.undershoot, d2=self.u_dispersion, c=self.ratio) 137 | 138 | @tf.function 139 | def _transform_parameters_forward(self, parameters): 140 | delay = sigmoid(parameters[:, 0][:, tf.newaxis]) 141 | dispersion = sigmoid(parameters[:, 1][:, tf.newaxis]) 142 | 143 | # Scale delay to be between [1.0, 10.0] 144 | delay = 9.0 * delay + 1.0 145 | 146 | # Scale dispersion to be between [0.75, 3.0] 147 | dispersion = 2.25 * dispersion + 0.75 148 | 149 | return tf.concat([delay, dispersion], axis=1) 150 | 151 | @tf.function 152 | def _transform_parameters_backward(self, parameters): 153 | delay = parameters[:, 0][:, tf.newaxis] 154 | delay = logit((delay - 1.0) / 9.0) 155 | 156 | dispersion = parameters[:, 1][:, tf.newaxis] 157 | dispersion = logit((dispersion - 0.75) / 2.25) 158 | 159 | return tf.concat([delay, dispersion], axis=1) 160 | 161 | class CustomHRFModel(HRFModel): 162 | 163 | def __init__(self, hrf): 164 | self.hrf = hrf.astype(np.float32) 165 | self.oversampling = 1. 166 | -------------------------------------------------------------------------------- /braincoder/stimuli.py: -------------------------------------------------------------------------------- 1 | import tensorflow_probability as tfp 2 | from tqdm import tqdm 3 | import tensorflow as tf 4 | import numpy as np 5 | import pandas as pd 6 | import logging 7 | import tensorflow_probability as tfp 8 | from tensorflow_probability import bijectors as tfb 9 | from .utils.mcmc import cleanup_chain, sample_hmc, Periodic 10 | 11 | class Stimulus(object): 12 | 13 | dimension_labels = ['x'] 14 | 15 | def __init__(self, n_dimensions=1): 16 | if n_dimensions != 1: 17 | self.dimension_labels = [f'dim_{ix}' for ix in range(n_dimensions)] 18 | 19 | self.bijectors = [tfp.bijectors.Identity(name=label) for label in self.dimension_labels] 20 | 21 | def clean_paradigm(self, paradigm): 22 | 23 | if (not isinstance(paradigm, pd.DataFrame)) and (paradigm is not None): 24 | 25 | if isinstance(paradigm, pd.Series): 26 | paradigm = paradigm.to_frame() 27 | paradigm.columns = self.dimension_labels 28 | else: 29 | if paradigm.ndim == 1: 30 | paradigm = paradigm[:, np.newaxis] 31 | 32 | paradigm = pd.DataFrame(paradigm, columns=pd.Index(self.dimension_labels, name='stimulus dimensions'), 33 | index=pd.Index(np.arange(len(paradigm)), name='frame')).astype(np.float32) 34 | 35 | if isinstance(paradigm, pd.DataFrame): 36 | paradigm = paradigm.astype(np.float32) 37 | 38 | return paradigm 39 | 40 | def _clean_paradigm(self, paradigm): 41 | if paradigm is not None: 42 | if isinstance(paradigm, pd.DataFrame): 43 | return paradigm.values.astype(np.float32) 44 | elif isinstance(paradigm, np.ndarray): 45 | return paradigm.astype(np.float32) 46 | 47 | return paradigm 48 | 49 | def _generate_stimulus(self, paradigm): 50 | return paradigm 51 | 52 | def generate_stimulus(self, paradigm): 53 | return self.clean_paradigm(paradigm) 54 | 55 | def generate_empty_stimulus(self, size): 56 | stimulus = np.ones((size, len(self.dimension_labels)), dtype=np.float32) * 1e-6 57 | 58 | if self.bijectors is not None: 59 | 60 | if isinstance(self.bijectors, list): 61 | stimulus = np.stack([bijector.forward(stimulus[:, ix]).numpy() for ix, bijector in enumerate(self.bijectors)], axis=1) 62 | else: 63 | stimulus = self.bijectors.forward(stimulus).numpy() 64 | 65 | return stimulus.astype(np.float32) 66 | 67 | 68 | class OneDimensionalStimulusWithAmplitude(Stimulus): 69 | dimension_labels = ['x', 'amplitude'] 70 | 71 | def __init__(self, positive_only=True): 72 | 73 | return super().__init__() 74 | 75 | if positive_only: 76 | self.bijectors = [tfp.bijectors.Identity(name='x'), tfp.bijectors.Softplus(name='amplitude')] 77 | 78 | class TwoDimensionalStimulus(Stimulus): 79 | dimension_labels = ['x', 'y'] 80 | 81 | 82 | class OneDimensionalRadialStimulus(Stimulus): 83 | dimension_labels = ['x (radians)'] 84 | 85 | def __init__(self): 86 | 87 | super().__init__() 88 | self.bijectors = [Periodic(low=0.0, high=2*np.pi, name='x')] 89 | 90 | 91 | class OneDimensionalRadialStimulusWithAmplitude(OneDimensionalStimulusWithAmplitude): 92 | dimension_labels = ['x (radians)', 'amplitude'] 93 | 94 | def __init__(self, positive_only=True): 95 | 96 | super().__init__() 97 | 98 | if positive_only: 99 | self.bijectors = [Periodic(low=0.0, high=2*np.pi, name='x'), tfp.bijectors.Softplus(name='amplitude')] 100 | else: 101 | self.bijectors = [Periodic(low=0.0, high=2*np.pi, name='x'), tfp.bijectors.Identity(name='amplitude')] 102 | 103 | class OneDimensionalGaussianStimulus(Stimulus): 104 | dimension_labels = ['x', 'sd'] 105 | 106 | def __init__(self, positive_only=True): 107 | super().__init__() 108 | 109 | self.bijectors = [tfp.bijectors.Identity(name='x'), tfp.bijectors.Softplus(name='sd')] 110 | 111 | 112 | class OneDimensionalGaussianStimulusWithAmplitude(Stimulus): 113 | dimension_labels = ['x', 'sd', 'amplitude'] 114 | 115 | def __init__(self, positive_only=True): 116 | super().__init__() 117 | 118 | if positive_only: 119 | self.bijectors = [tfp.bijectors.Identity(name='x'), tfp.bijectors.Softplus(name='sd'), tfp.bijectors.Softplus(name='amplitude')] 120 | else: 121 | self.bijectors = [tfp.bijectors.Identity(name='x'), tfp.bijectors.Softplus(name='sd'), tfp.bijectors.Identity(name='amplitude')] 122 | 123 | 124 | class ImageStimulus(Stimulus): 125 | 126 | def __init__(self, grid_coordinates, positive_only=True): 127 | self.grid_coordinates = pd.DataFrame(grid_coordinates, columns=['x', 'y']) 128 | 129 | if positive_only: 130 | self.bijectors = tfp.bijectors.Softplus(name='intensity') 131 | else: 132 | self.bijectors = tfp.bijectors.Identity(name='intensity') 133 | 134 | self.dimension_labels = pd.MultiIndex.from_frame(self.grid_coordinates) 135 | 136 | def clean_paradigm(self, paradigm): 137 | 138 | if (not isinstance(paradigm, pd.DataFrame)) and (paradigm is not None): 139 | 140 | if paradigm.ndim == 3: 141 | paradigm = paradigm[:, np.newaxis] 142 | 143 | paradigm = paradigm.reshape((paradigm.shape[0], -1)) 144 | elif paradigm.ndim == 2: 145 | pass 146 | else: 147 | raise ValueError('Paradigm should be 2 or 3 dimensional') 148 | 149 | paradigm = pd.DataFrame(paradigm, columns=self.dimension_labels, index=pd.Index(np.arange(len(paradigm)), name='frame')).astype(np.float32) 150 | 151 | if isinstance(paradigm, pd.DataFrame): 152 | paradigm = paradigm.astype(np.float32) 153 | 154 | return paradigm 155 | 156 | def generate_empty_stimulus(self, size): 157 | return np.ones((size, len(self.dimension_labels)), dtype=np.float32) * 1e-6 158 | -------------------------------------------------------------------------------- /braincoder/tests/test_gaussprf.py: -------------------------------------------------------------------------------- 1 | from braincoder.models import GaussianPRF 2 | from braincoder.optimize import ParameterOptimizer 3 | import numpy as np 4 | import scipy.stats as ss 5 | 6 | def get_correlation_matrix(a, b): 7 | 8 | assert(a.ndim == 2) 9 | assert(b.ndim == 2) 10 | 11 | if hasattr(a, 'values'): 12 | a = a.values 13 | 14 | if hasattr(b, 'values'): 15 | b = b.values 16 | 17 | a = (a - a.mean(0)) / np.var(a) 18 | b = (b - b.mean(0)) / np.var(b) 19 | 20 | return np.mean(a*b, 0) 21 | 22 | 23 | def get_paradigm(): 24 | return np.linspace(-5, 5, dtype=np.float32)[:, np.newaxis] 25 | 26 | 27 | def get_parameters(n_pars=100): 28 | mus = np.random.rand(n_pars) * 3. - 1.5 29 | sds = np.random.rand(n_pars)*3 30 | amplitudes = np.random.rand(n_pars) * 2 31 | baselines = np.random.rand(n_pars) * 2 - 1 32 | 33 | parameters = np.concatenate((mus[:, np.newaxis], 34 | sds[:, np.newaxis], 35 | amplitudes[:, np.newaxis], 36 | baselines[:, np.newaxis]), 1) 37 | 38 | return parameters 39 | 40 | def test_gauss_prf(): 41 | 42 | paradigm = get_paradigm() 43 | parameters = get_parameters() 44 | 45 | model = GaussianPRF() 46 | 47 | data = model.simulate(paradigm, parameters, noise=.1) 48 | 49 | optimizer = ParameterOptimizer(model, data, paradigm) 50 | 51 | optimizer.fit() 52 | 53 | corr = get_correlation_matrix(parameters, optimizer.estimated_parameters) 54 | print(corr) 55 | 56 | assert(np.all(corr > 0.1)) 57 | 58 | 59 | -------------------------------------------------------------------------------- /braincoder/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .math import logit, norm, log2, restrict_radians, lognormalpdf_n, von_mises_pdf, lognormal_pdf_mode_fwhm, norm2d 2 | from .formatting import format_data, format_paradigm, format_weights, format_parameters 3 | from .stats import get_map, get_rsq, get_r 4 | from .visualize import show_animation 5 | -------------------------------------------------------------------------------- /braincoder/utils/data.py: -------------------------------------------------------------------------------- 1 | import pkg_resources 2 | import numpy as np 3 | import pandas as pd 4 | from scipy import ndimage, io 5 | import tqdm 6 | from nilearn.surface import load_surf_data 7 | from scipy.ndimage import zoom 8 | 9 | def load_szinte2024(resize_factor=1., best_voxels=None): 10 | 11 | data = {} 12 | 13 | stream = pkg_resources.resource_stream(__name__, '../data/szinte2024/dm_gazecenter.npz') 14 | data['stimulus'] = np.load(stream)['arr_0'].astype(np.float32) 15 | 16 | data['grid_coordinates'] = pd.read_csv(pkg_resources.resource_stream(__name__, '../data/szinte2024/grid_coordinates.tsv'), sep='\t').astype(np.float32) 17 | 18 | prf_pars = pd.read_csv(pkg_resources.resource_stream(__name__, '../data/szinte2024/gauss_parameters.tsv'), index_col='source', sep='\t').astype(np.float32) 19 | 20 | if resize_factor != 1.: 21 | data['stimulus'] = np.array([ndimage.zoom(d, 1./resize_factor) for d in data['stimulus']]).astype(np.float32) 22 | 23 | tmp_x = data['grid_coordinates'].set_index(pd.MultiIndex.from_frame(data['grid_coordinates'] ))['x'] 24 | tmp_y = data['grid_coordinates'].set_index(pd.MultiIndex.from_frame(data['grid_coordinates'] ))['y'] 25 | 26 | new_x = ndimage.zoom(tmp_x.unstack('y'), 1./resize_factor) 27 | new_y = ndimage.zoom(tmp_y.unstack('y'), 1./resize_factor) 28 | 29 | # y needs to be flipped because of how stack and ravel works... 30 | data['grid_coordinates'] = pd.DataFrame({'x':new_x.ravel(), 'y':new_y.ravel()[::-1]}).astype(np.float32) 31 | 32 | data['v1_timeseries'] = pd.read_csv(pkg_resources.resource_stream(__name__, '../data/szinte2024/mask-v1_bold.tsv.gz'), sep='\t', index_col=['time'], compression='gzip').astype(np.float32) 33 | data['v1_timeseries'].columns = data['v1_timeseries'].columns.astype(int) 34 | data['tr'] = 1.317025 35 | 36 | if best_voxels: 37 | best_voxels = prf_pars['r2'].sort_values(ascending=False).iloc[:best_voxels].index 38 | data['v1_timeseries'] = data['v1_timeseries'].loc[:, best_voxels] 39 | prf_pars = prf_pars.loc[best_voxels] 40 | 41 | data['prf_pars'] = prf_pars 42 | data['r2'] = prf_pars['r2'] 43 | data['prf_pars'].drop('r2', axis=1, inplace=True) 44 | 45 | return data 46 | 47 | import os 48 | import pathlib 49 | import requests 50 | import zipfile 51 | import shutil 52 | from tqdm import tqdm 53 | 54 | FIGSHARE_URL = "https://figshare.com/ndownloader/files/26577941" 55 | DATA_DIR = pathlib.Path.home() / ".braincoder" / "data" 56 | ZIP_PATH = DATA_DIR / "fmri_teaching_materials.zip" 57 | 58 | def ensure_directory_exists(directory): 59 | """Create directory if it doesn't exist.""" 60 | directory.mkdir(parents=True, exist_ok=True) 61 | 62 | def download_file(url, save_path): 63 | """Download a file from a URL with a progress bar.""" 64 | response = requests.get(url, stream=True) 65 | response.raise_for_status() # Raise error if download fails 66 | 67 | total_size = int(response.headers.get("content-length", 0)) 68 | block_size = 1024 # 1 KB 69 | progress_bar = tqdm(total=total_size, unit="B", unit_scale=True, desc="Downloading") 70 | 71 | with open(save_path, "wb") as f: 72 | for data in response.iter_content(block_size): 73 | progress_bar.update(len(data)) 74 | f.write(data) 75 | 76 | progress_bar.close() 77 | 78 | def extract_zip(zip_path, extract_to): 79 | """Extracts the given zip file.""" 80 | with zipfile.ZipFile(zip_path, "r") as zip_ref: 81 | zip_ref.extractall(extract_to) 82 | 83 | def load_vanes2019(raw_files=False, downsample_stimulus=5.): 84 | """ 85 | Ensures the fMRI van Es 2019 dataset is available in ~/.braincoder/data. 86 | Downloads and extracts it if necessary. 87 | Returns the path to the dataset. 88 | """ 89 | ensure_directory_exists(DATA_DIR) 90 | dataset_folder = DATA_DIR / "prf_vanes2019" 91 | 92 | if not dataset_folder.exists() or not any(dataset_folder.iterdir()): 93 | print(f"Dataset missing or incomplete in {dataset_folder}, redownloading...") 94 | if dataset_folder.exists(): 95 | shutil.rmtree(dataset_folder) # Remove partial data 96 | download_file(FIGSHARE_URL, ZIP_PATH) 97 | print("\nDownload complete, extracting...") 98 | extract_zip(ZIP_PATH, dataset_folder) 99 | print("\nExtraction complete, cleaning up...") 100 | ZIP_PATH.unlink() # Remove zip file after extraction 101 | 102 | data_lh = load_surf_data(dataset_folder / "sub-02_task-prf_space-59k_hemi-L_run-median_desc-bold.func.gii") 103 | data_rh = load_surf_data(dataset_folder / "sub-02_task-prf_space-59k_hemi-R_run-median_desc-bold.func.gii") 104 | 105 | if raw_files: 106 | # Return a list with all files in dataset_folder: 107 | return list(dataset_folder.iterdir()) 108 | 109 | data = {} 110 | data['ts'] = pd.concat((pd.DataFrame(data_lh, index=pd.Index(np.arange(len(data_lh)), name='vertex')), 111 | pd.DataFrame(data_rh, index=pd.Index(np.arange(len(data_rh)), name='vertex'))), keys=['L', 'R'], names=['hemisphere'], axis=0).T 112 | 113 | # Convert ts to percent signal change: 114 | data['ts'] = (data['ts'] - data['ts'].mean()) / data['ts'].mean() * 100 115 | 116 | data['stimulus'] = zoom(io.loadmat(dataset_folder / "vis_design.mat")['stim'], (1./downsample_stimulus, 1./downsample_stimulus, 1)).astype(np.float32) 117 | data['stimulus'] = np.clip(np.moveaxis(np.moveaxis(data['stimulus'], -1, 0), -1, 1) / 255., 0, 1) 118 | 119 | # Calculate the degree per pixel scaling factors 120 | width_pixels, height_pixels = data['stimulus'].shape[1:] 121 | width_degrees = 20 122 | dx = width_degrees / width_pixels 123 | dy = dx 124 | height_degrees = height_pixels * dy 125 | 126 | x_degrees = np.linspace(-width_degrees / 2 + dx / 2, width_degrees / 2 - dx / 2, width_pixels) 127 | y_degrees = np.linspace(-height_degrees / 2 + dy / 2, height_degrees / 2 - dy / 2, height_pixels) 128 | 129 | x_mesh, y_mesh = np.meshgrid(x_degrees, y_degrees) 130 | 131 | data['grid_coordinates'] = pd.DataFrame({'x': x_mesh.ravel(), 'y': y_mesh.ravel()}).astype(np.float32) 132 | data['tr'] = 1.5 133 | 134 | return data 135 | -------------------------------------------------------------------------------- /braincoder/utils/formatting.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | import tensorflow as tf 4 | 5 | 6 | def format_paradigm(paradigm): 7 | if paradigm is None: 8 | return None 9 | 10 | if isinstance(paradigm, pd.DataFrame): 11 | return paradigm 12 | 13 | if isinstance(paradigm, pd.Series): 14 | return paradigm.to_frame() 15 | 16 | if paradigm.ndim == 1: 17 | paradigm = paradigm[:, np.newaxis] 18 | elif paradigm.ndim > 2: 19 | paradigm = paradigm.reshape((paradigm.shape[0], -1)) 20 | 21 | return pd.DataFrame(paradigm, index=pd.Index(range(len(paradigm)), name='time'), 22 | columns=pd.Index(range(paradigm.shape[1]), name='stimulus dimension')).astype(np.float32) 23 | 24 | 25 | def format_parameters(parameters, parameter_labels=None): 26 | 27 | if isinstance(parameters, pd.DataFrame): 28 | return parameters.astype(np.float32) 29 | 30 | if parameters is None: 31 | return None 32 | 33 | if parameter_labels is None: 34 | parameter_labels = [ 35 | f'par{i+1}' for i in range(parameters.shape[1])] 36 | 37 | if type(parameter_labels) is list: 38 | parameter_labels = pd.Index(parameter_labels, name='parameter') 39 | 40 | return pd.DataFrame(parameters, 41 | columns=parameter_labels, 42 | index=pd.Index(range(len(parameters)), name='source')).astype(np.float32) 43 | 44 | 45 | def format_weights(weights): 46 | if weights is not None: 47 | if isinstance(weights, pd.DataFrame): 48 | return weights 49 | else: 50 | return pd.DataFrame(weights, 51 | index=pd.Index( 52 | range(1, len(weights) + 1), name='population'), 53 | columns=pd.Index(np.arange(weights.shape[1]), name='unit')).astype(np.float32) 54 | 55 | 56 | def format_data(data): 57 | 58 | if isinstance(data, pd.DataFrame): 59 | return data 60 | 61 | if isinstance(data, tf.Tensor): 62 | data = data.numpy() 63 | 64 | return pd.DataFrame(data, 65 | index=pd.Index( 66 | np.arange(len(data)), name='time'), 67 | columns=pd.Index(range(data.shape[1]), name='unit')).astype(np.float32) 68 | -------------------------------------------------------------------------------- /braincoder/utils/math.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import tensorflow_probability as tfp 3 | import pandas as pd 4 | import numpy as np 5 | 6 | 7 | def norm(x, mu, sigma): 8 | kernel = tf.math.exp(-.5 * (x - mu)**2. / sigma**2) 9 | return kernel 10 | 11 | def norm2d(x, y, mu_x, mu_y, sigma_x, sigma_y, rho=None): 12 | if rho is None: 13 | # Default to 0 covariance (independent x and y) 14 | rho = 0.0 15 | z = ((x - mu_x) ** 2 / sigma_x ** 2) + \ 16 | ((y - mu_y) ** 2 / sigma_y ** 2) - \ 17 | (2 * rho * (x - mu_x) * (y - mu_y) / (sigma_x * sigma_y)) 18 | kernel = tf.math.exp(-z / (2 * (1 - rho ** 2))) 19 | return kernel 20 | 21 | def logit(x): 22 | """ Computes the logit function, i.e. the logistic sigmoid inverse. """ 23 | return tf.clip_by_value(-tf.math.log(1. / x - 1.), -1e12, 1e12) 24 | 25 | def logistic_transfer(x, lower_bound, upper_bound): 26 | return lower_bound + (upper_bound - lower_bound) / (1 + tf.exp(-x)) 27 | 28 | @tf.function 29 | def log2(x): 30 | return tf.math.log(x) / tf.math.log(2.) 31 | 32 | @tf.function 33 | def restrict_radians(x): 34 | x = x+np.pi 35 | return x - tf.floor(x / (2*np.pi)) * 2*np.pi - np.pi 36 | 37 | def lognormalpdf_n(x, mu_n, sigma_n, normalize=False): 38 | 39 | denom = 1+sigma_n**2/mu_n**2 40 | 41 | part2 = tf.exp(-((tf.math.log(x)- tf.math.log(mu_n / tf.sqrt(denom)))**2 / (2*tf.math.log(denom)))) 42 | 43 | if normalize: 44 | part1 = 1. / (x*tf.sqrt(2*np.pi*tf.math.log(denom))) 45 | return part1*part2 46 | else: 47 | return part2 48 | 49 | def lognormal_pdf_mode_fwhm(x, mode, fwhm): 50 | 51 | sigma = 1./(tf.sqrt(2.*tf.math.log(2.))) * tf.math.asinh(fwhm/(mode*2.)) 52 | sigma2 = sigma**2 53 | p = (mode / x) * tf.math.exp(.5*sigma2 - .5 *((tf.math.log(x/mode) - sigma2)**2)/sigma2) 54 | 55 | return p 56 | 57 | def von_mises_pdf(x, mu, kappa): 58 | # Constants 59 | TWO_PI = tf.constant(2 * np.pi) 60 | 61 | # Calculate the PDF formula 62 | pdf = tf.exp(kappa * tf.cos(x - mu)) / (TWO_PI * tf.math.bessel_i0(kappa)) 63 | 64 | return pdf 65 | 66 | # Aggressive softplus with alpha=100 67 | alpha = 100 68 | aggressive_softplus = lambda x: (1./alpha) * tf.math.softplus(alpha*x) 69 | aggressive_softplus_inverse = lambda y: (1./alpha) * tfp.math.softplus_inverse(alpha * y) 70 | 71 | 72 | def get_expected_value(stimulus_pdf, normalize=True): 73 | 74 | x = stimulus_pdf.columns.astype(np.float32) 75 | 76 | if normalize: 77 | stimulus_pdf /= np.trapz(stimulus_pdf, x=x, axis=1)[:, np.newaxis] 78 | 79 | 80 | E = np.trapz(stimulus_pdf * x, x=x, axis=1) 81 | 82 | return pd.Series(E, name='E', index=stimulus_pdf.index) 83 | 84 | 85 | def get_sd_posterior(stimulus_pdf, E=None, normalize=True): 86 | 87 | x = stimulus_pdf.columns.astype(np.float32).values 88 | 89 | if normalize: 90 | stimulus_pdf /= np.trapz(stimulus_pdf, x=x, axis=1)[:, np.newaxis] 91 | 92 | if E is None: 93 | E = get_expected_value(stimulus_pdf, normalize=normalize).values 94 | else: 95 | if hasattr(E, 'values'): 96 | E = E.values 97 | 98 | sd = np.sqrt(np.trapz(stimulus_pdf * (x[np.newaxis, :] - E[:, np.newaxis]) ** 2, x=x, axis=1)) 99 | 100 | return pd.Series(sd, name='sd', index=stimulus_pdf.index) -------------------------------------------------------------------------------- /braincoder/utils/mcmc.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import tensorflow_probability as tfp 3 | from tensorflow_probability import bijectors as tfb 4 | import pandas as pd 5 | from timeit import default_timer as timer 6 | 7 | def cleanup_chain(chain, name, time_index): 8 | 9 | n_chains = chain.shape[1] 10 | n_samples = chain.shape[0] 11 | 12 | chain_index = pd.Index(range(n_chains), name='chain') 13 | 14 | if not issubclass(type(time_index), pd.core.indexes.base.Index): 15 | time_index = pd.Index(frames, name='frame') 16 | else: 17 | time_index = time_index 18 | 19 | sample_index = pd.Index(range(n_samples), name='sample') 20 | 21 | chain = [pd.DataFrame(chain[:, ix, :], index=sample_index, 22 | columns=time_index) for ix in range(n_chains)] 23 | chain = pd.concat(chain, keys=chain_index) 24 | 25 | return chain.stack(list(range(chain.columns.nlevels))).to_frame(name).sort_index() 26 | 27 | @tf.function 28 | def sample_hmc( 29 | init_state, 30 | step_size, 31 | target_log_prob_fn, 32 | unconstraining_bijectors, 33 | target_accept_prob=0.85, 34 | unrolled_leapfrog_steps=1, 35 | max_tree_depth=10, 36 | num_steps=50, 37 | burnin=50): 38 | 39 | 40 | def trace_fn(_, pkr): 41 | return { 42 | 'log_prob': pkr.inner_results.inner_results.target_log_prob, 43 | 'diverging': pkr.inner_results.inner_results.has_divergence, 44 | 'is_accepted': pkr.inner_results.inner_results.is_accepted, 45 | 'accept_ratio': tf.exp(pkr.inner_results.inner_results.log_accept_ratio), 46 | 'leapfrogs_taken': pkr.inner_results.inner_results.leapfrogs_taken, 47 | 'step_size': pkr.inner_results.inner_results.step_size} 48 | 49 | hmc = tfp.mcmc.NoUTurnSampler( 50 | target_log_prob_fn, 51 | unrolled_leapfrog_steps=unrolled_leapfrog_steps, 52 | max_tree_depth=max_tree_depth, 53 | step_size=step_size) 54 | 55 | hmc = tfp.mcmc.TransformedTransitionKernel( 56 | inner_kernel=hmc, 57 | bijector=unconstraining_bijectors) 58 | 59 | adaptive_sampler = tfp.mcmc.DualAveragingStepSizeAdaptation( 60 | inner_kernel=hmc, 61 | num_adaptation_steps=int(0.8 * burnin), 62 | target_accept_prob=target_accept_prob, 63 | # NUTS inside of a TTK requires custom getter/setter functions. 64 | step_size_setter_fn=lambda pkr, new_step_size: pkr._replace( 65 | inner_results=pkr.inner_results._replace( 66 | step_size=new_step_size) 67 | ), 68 | step_size_getter_fn=lambda pkr: pkr.inner_results.step_size, 69 | log_accept_prob_getter_fn=lambda pkr: pkr.inner_results.log_accept_ratio, 70 | ) 71 | 72 | start = timer() 73 | # Sampling from the chain. 74 | samples, stats = tfp.mcmc.sample_chain( 75 | num_results=burnin + num_steps, 76 | current_state=init_state, 77 | kernel=adaptive_sampler, 78 | trace_fn=trace_fn) 79 | 80 | duration = timer() - start 81 | stats['elapsed_time'] = duration 82 | 83 | return samples, stats 84 | 85 | 86 | 87 | class Periodic(tfb.Bijector): 88 | 89 | def __init__(self, low, high, validate_args=False, name='periodic'): 90 | 91 | self.low = low 92 | self.high = high 93 | self.width = high - low 94 | 95 | super(Periodic, self).__init__( 96 | is_constant_jacobian=True, 97 | validate_args=validate_args, 98 | forward_min_event_ndims=0, 99 | name=name) 100 | 101 | def _forward(self, x): 102 | return ((x - self.low) % self.width) + self.low 103 | 104 | def _inverse(self, y): 105 | return y 106 | 107 | # def _inverse_log_det_jacobian(self, y): 108 | # return -self._forward_log_det_jacobian(self._inverse(y)) 109 | 110 | # def _forward_log_det_jacobian(self, x): 111 | # # The full log jacobian determinant would be tf.zero_like(x). 112 | # # However, we circumvent materializing that, since the jacobian 113 | # # calculation is input independent, and we specify it for one input. 114 | # return tf.constant(0., x.dtype) 115 | -------------------------------------------------------------------------------- /braincoder/utils/stats.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def get_map(p): 5 | stimuli = p.columns.to_frame(index=False).T 6 | return stimuli.groupby(level=0).apply(lambda d: (p * d.values).sum(1) / p.sum(1)).T 7 | 8 | def get_rsq(data, predictions, zerovartonan=True, allow_biased_residuals=False): 9 | 10 | resid = data - predictions 11 | 12 | # ssq_data = np.clip(((data - data.mean(0))**2).sum(0), 1e-5, None) 13 | ssq_data = ((data - data.mean(0))**2).sum(0) 14 | if allow_biased_residuals: 15 | ssq_resid = ((resid - resid.mean(0))**2).sum(0) 16 | else: 17 | ssq_resid = (resid**2).sum(0) 18 | 19 | r2 = (1 - (ssq_resid / ssq_data)) 20 | 21 | if zerovartonan: 22 | r2[data.var() == 0] = np.nan 23 | 24 | r2.name = 'r2' 25 | 26 | return r2 27 | 28 | 29 | def get_r(data, predictions): 30 | 31 | data_ = data - data.mean(0) 32 | predictions_ = predictions - predictions.mean(0) 33 | 34 | r = (data_*predictions_ ).sum(0) 35 | r = r / (np.sqrt((data_**2).sum(0) * (predictions_**2).sum(0))) 36 | 37 | return r 38 | -------------------------------------------------------------------------------- /braincoder/utils/visualize.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def show_animation(images, vmin=None, vmax=None): 5 | import matplotlib.pyplot as plt 6 | import matplotlib.animation as animation 7 | from IPython.display import HTML 8 | 9 | fig = plt.figure(figsize=(6, 4)) 10 | ims = [] 11 | 12 | if vmin is None: 13 | vmin = np.percentile(images, 5) 14 | 15 | if vmax is None: 16 | vmax = np.percentile(images, 95) 17 | 18 | for t in range(len(images)): 19 | im = plt.imshow(images[t], animated=True, cmap='gray', vmin=vmin, vmax=vmax) 20 | plt.axis('off') 21 | ims.append([im]) 22 | 23 | ani = animation.ArtistAnimation( 24 | fig, ims, interval=150, blit=True, repeat_delay=1500) 25 | 26 | return HTML(ani.to_html5_video()) 27 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= -v 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/_static/css/custom.css: -------------------------------------------------------------------------------- 1 | div.sphx-glr-download-link-note { 2 | height: 0px; 3 | visibility: hidden; 4 | } -------------------------------------------------------------------------------- /docs/bibliography.rst: -------------------------------------------------------------------------------- 1 | :orphan: 2 | 3 | .. _general_bibliography: 4 | 5 | General bibliography 6 | ==================== 7 | 8 | The references below are arranged alphabetically by first author. You can download the bib file :download:`here <./references.bib>`. 9 | 10 | .. bibliography:: ./references.bib 11 | :all: 12 | :list: enumerated -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # For the full list of built-in configuration values, see the documentation: 4 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 5 | 6 | # -- Project information ----------------------------------------------------- 7 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information 8 | 9 | project = 'Braincoder' 10 | copyright = '2023, Gilles de Hollander' 11 | author = 'Gilles de Hollander' 12 | 13 | # -- General configuration --------------------------------------------------- 14 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration 15 | 16 | # These folders are copied to the documentation's HTML output 17 | html_static_path = ['_static'] 18 | 19 | # These paths are either relative to html_static_path 20 | # or fully qualified paths (eg. https://...) 21 | html_css_files = ['css/custom.css'] 22 | 23 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.duration', 24 | 'sphinx_gallery.gen_gallery', 'sphinxcontrib.bibtex', 'sphinx.ext.mathjax'] 25 | 26 | bibtex_bibfiles = ["./references.bib"] 27 | bibtex_style = "unsrt" 28 | bibtex_reference_style = "author_year" 29 | bibtex_footreference_style = "author_year" 30 | bibtex_footbibliography_header = "" 31 | 32 | autosummary_generate = True 33 | 34 | # Generate the plots for the gallery 35 | plot_gallery = "True" 36 | 37 | sphinx_gallery_conf = { 38 | "doc_module": "braincoder", 39 | "reference_url": {"braincoder": None}, 40 | "examples_dirs": "../examples", 41 | "gallery_dirs": "auto_examples", 42 | # Ignore the function signature leftover by joblib 43 | "ignore_pattern": r"func_code\.py", 44 | "filename_pattern": '.*', 45 | # "show_memory": not sys.platform.startswith("win"), 46 | "remove_config_comments": True, 47 | "nested_sections": True, 48 | } 49 | 50 | templates_path = ['_templates'] 51 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 52 | 53 | 54 | # -- Options for HTML output ------------------------------------------------- 55 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output 56 | font_awesome = "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/" 57 | html_css_files += [ 58 | "custom.css", 59 | ( 60 | "https://cdnjs.cloudflare.com/ajax/libs/" 61 | "font-awesome/5.15.4/css/all.min.css" 62 | ), 63 | f"{font_awesome}fontawesome.min.css", 64 | f"{font_awesome}solid.min.css", 65 | f"{font_awesome}brands.min.css", 66 | ] 67 | html_theme = 'furo' 68 | html_static_path = ['_static'] 69 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | =========== 2 | Tutorial 3 | =========== 4 | 5 | For an in-depth introduction to encoding/decoding models and 6 | the ``braincoder``-package follow the :doc:`tutorial/index`. 7 | 8 | =========== 9 | Quick start 10 | =========== 11 | 12 | .. include:: ../README.rst 13 | 14 | 15 | .. toctree:: 16 | :maxdepth: 1 17 | :hidden: 18 | 19 | 20 | tutorial/index.rst 21 | auto_examples/index.rst 22 | bibliography.rst 23 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | %SPHINXBUILD% >NUL 2>NUL 14 | if errorlevel 9009 ( 15 | echo. 16 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 17 | echo.installed, then set the SPHINXBUILD environment variable to point 18 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 19 | echo.may add the Sphinx directory to PATH. 20 | echo. 21 | echo.If you don't have Sphinx installed, grab it from 22 | echo.https://www.sphinx-doc.org/ 23 | exit /b 1 24 | ) 25 | 26 | if "%1" == "" goto help 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/references.bib: -------------------------------------------------------------------------------- 1 | @article{serences2007spatially, 2 | title={Spatially selective representations of voluntary and stimulus-driven attentional priority in human occipital, parietal, and frontal cortex}, 3 | author={Serences, John T and Yantis, Steven}, 4 | journal={Cerebral cortex}, 5 | volume={17}, 6 | number={2}, 7 | pages={284--293}, 8 | year={2007}, 9 | publisher={Oxford University Press}, 10 | url={https://drive.google.com/file/d/16-t8zlMJlHIK5Qt8MeJy6NK5r9WzjgT9/view?usp=sharing}, 11 | } 12 | 13 | @article{sawetsuttipan2023perceptual, 14 | title={Perceptual Difficulty Regulates Attentional Gain Modulations in Human Visual Cortex}, 15 | author={Sawetsuttipan, Prapasiri and Phunchongharn, Phond and Ounjai, Kajornvut and Salazar, Annalisa and Pongsuwan, Sarigga and Intrachooto, Singh and Serences, John T and Itthipuripat, Sirawaj}, 16 | journal={Journal of Neuroscience}, 17 | volume={43}, 18 | number={18}, 19 | pages={3312--3330}, 20 | year={2023}, 21 | publisher={Soc Neuroscience}, 22 | url={https://www.jneurosci.org/content/43/18/3312} 23 | } 24 | 25 | @article{sprague2013attention, 26 | title={Attention modulates spatial priority maps in the human occipital, parietal and frontal cortices}, 27 | author={Sprague, Thomas C and Serences, John T}, 28 | journal={Nature neuroscience}, 29 | volume={16}, 30 | number={12}, 31 | pages={1879--1887}, 32 | year={2013}, 33 | publisher={Nature Publishing Group US New York}, 34 | url={https://www.nature.com/articles/nn.3574} 35 | } 36 | 37 | @article{wei2015bayesian, 38 | title={A Bayesian observer model constrained by efficient coding can explain'anti-Bayesian'percepts}, 39 | author={Wei, Xue-Xin and Stocker, Alan A}, 40 | journal={Nature neuroscience}, 41 | volume={18}, 42 | number={10}, 43 | pages={1509--1517}, 44 | year={2015}, 45 | publisher={Nature Publishing Group US New York} 46 | } -------------------------------------------------------------------------------- /docs/tutorial/index.rst: -------------------------------------------------------------------------------- 1 | =========== 2 | Tutorial 3 | =========== 4 | 5 | Welcome to the tutorial on ``braincoder`` and the encoding-decoding framework! 6 | 7 | This tutorial will walk you through the basic steps of using ``braincoder`` to 8 | build encoding and decoding models. 9 | 10 | We will use the ``braincoder`` package to build a simple encoding model and 11 | then use it to predict the voxel responses to a new stimulus. We will then 12 | build a decoding model and use it to predict the stimulus from the voxel 13 | activations. 14 | 15 | ####### 16 | Lessons 17 | ####### 18 | 19 | .. toctree:: 20 | :maxdepth: 1 21 | 22 | lesson1.rst 23 | lesson2.rst 24 | lesson3.rst 25 | lesson4.rst 26 | lesson5.rst 27 | lesson6.rst 28 | lesson7.rst 29 | -------------------------------------------------------------------------------- /docs/tutorial/lesson1.rst: -------------------------------------------------------------------------------- 1 | .. _tutorial_lesson1: 2 | 3 | ============================ 4 | Lesson 1: Encoding models 5 | ============================ 6 | 7 | Encoding models 8 | ############### 9 | 10 | Standard encoding models, like 2D PRF models that are used 11 | in retinotopic mapping, but also models of numerosity 12 | and Gabor orientations define **determinsitic one-to-one mapping** from 13 | **stimulus space** to **BOLD response space**: 14 | 15 | .. math:: 16 | 17 | f(s; \theta): s \mapsto x 18 | 19 | 20 | Here: 21 | * The stimulus :math:`s` is some point in a n-dimensional feature space. For example, :math:`s` could be the *numerosity* of a stimulus array, the *orientation* of a Gabor patch, or a *2D image*. 22 | * :math:`x` is a BOLD activation pattern *in a single voxel*. This could both be a single-trial estimate, or the actual activity pattern on a given timepoint :math:`t` 23 | * :math:`\theta` is a set of parameters tha define the particular mapping of a particular voxel. For example, the center and dispersion of a 2D PRF, or the preferred orientation of a Gabor patch. In some cases, :math:`\theta` is fitted to data. In other cases, it is fixed. 24 | 25 | Concrete example 26 | **************** 27 | 28 | 29 | One standard encoding model is the **1D Gaussian PRF**. This is simply the probability density 30 | of a 1D Gaussian distribution, centered at :math:`\mu` and with dispersion :math:`\sigma`, evaluated at 31 | :math:`x`, multiplied by an amplitude :math:`a`: and added to a baseline :math:`b`: 32 | 33 | .. math:: 34 | f(s; \mu, \sigma, a, b) = a \cdot \mathcal{N}(s; \mu, \sigma) + b 35 | 36 | 37 | Simulate data 38 | ************* 39 | 40 | This model is implemented in the ``GaussianPRF`` class in ``braincoder.models``: 41 | 42 | .. literalinclude:: /../examples/00_encodingdecoding/encoding_model.py 43 | :start-after: # Import necessary libraries 44 | :end-before: # %% 45 | 46 | 47 | .. |prf_timeseries| image:: ../auto_examples/00_encodingdecoding/images/sphx_glr_encoding_model_001.png 48 | .. centered:: |prf_timeseries| 49 | 50 | 51 | We can also simulate noisy data and try to estimate back the generating parameters: 52 | 53 | .. literalinclude:: /../examples/00_encodingdecoding/encoding_model.py 54 | :start-after: # We simulate data with a bit of noise 55 | :end-before: # %% 56 | 57 | .. |noisy_prf_timeseries| image:: ../auto_examples/00_encodingdecoding/images/sphx_glr_encoding_model_002.png 58 | .. centered:: |noisy_prf_timeseries| 59 | 60 | Estimate parameters 61 | ******************* 62 | 63 | Then we can try to estimate back the generating parameters using a grid search. 64 | This code automatically picks, for each voxel, the parameters that maximize the 65 | **correlation** between the predicted and actual BOLD response: 66 | 67 | .. literalinclude:: /../examples/00_encodingdecoding/encoding_model.py 68 | :start-after: # Import and set up a parameter fitter with the (simulated) data, 69 | :end-before: # %% 70 | 71 | 72 | The grid search only optimized for the center and dispersion of the Gaussian, 73 | but we also want to optimize for the amplitude and baseline, 74 | using ordinary least squares (note that this is computationally much cheaper than adding 75 | ``amplitude`` and ``baseline`` to the grid search). 76 | Now we minimize the sum of squared errors between the predicted and actual BOLD response 77 | **R2**. 78 | 79 | .. literalinclude:: /../examples/00_encodingdecoding/encoding_model.py 80 | :start-after: # We can now fit the amplitude and baseline using OLS 81 | :end-before: # %% 82 | 83 | This grid-optimized parameters already fit the data pretty well. 84 | Note how we can use ``get_rsq`` to calculate the *fraction of explained variance R2* 85 | between the predicted and actual BOLD response. This is the exact same R2 that is used 86 | to optimize the parameters on. 87 | 88 | .. |grid_fit| image:: ../auto_examples/00_encodingdecoding/images/sphx_glr_encoding_model_003.png 89 | .. centered:: |grid_fit| 90 | 91 | We can do even better using *gradient descent* optimisation. Note that because `braincoder` uses 92 | `tensorflow`, it uses autodiff-calculated exact gradients, as well as the GPU to speed up the 93 | computation. This is especially useful for more complex models. `braincoder` is under some 94 | circumstances multiple orders of magnitude faster than other PRF libraries. 95 | 96 | .. literalinclude:: /../examples/00_encodingdecoding/encoding_model.py 97 | :start-after: # Final optimisation using gradient descent: 98 | :end-before: # %% 99 | 100 | .. |gd_fit| image:: ../auto_examples/00_encodingdecoding/images/sphx_glr_encoding_model_004.png 101 | .. centered:: |gd_fit| 102 | 103 | .. note:: 104 | 105 | The complete Python script its output can be found :ref:`here`. 106 | 107 | Summary 108 | ####### 109 | 110 | In this lesson we have seen: 111 | * Encoding models define a deterministic mapping from stimulus space to BOLD response space 112 | * `braincoder` allows us to define all kins of encoding models, and to fit them to data 113 | * `braincoder` uses tensorflow to speed up the computation, and to allow for gradient descent optimisation 114 | * Fitting encoding models usually take the following steps: 115 | * Set up a grid of possible non-linear parameter values, and find best-fitting ones (``optimizer.fit_grid()``) 116 | * Fit linear parameters using ordinary least squares (``optimizer.refine_amplitude_baseline()``) 117 | * Finialize fit using gradient descent (``optimizer.fit()`` 118 | 119 | In the :ref:`next lesson`, we will see how we can fit *linear* encoding models. 120 | -------------------------------------------------------------------------------- /docs/tutorial/lesson2.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _tutorial_lesson2: 3 | 4 | ================================ 5 | Lesson 2: Linear encoding models 6 | ================================ 7 | 8 | A linear encoding approach 9 | ########################### 10 | 11 | In the previous lesson, we saw how we can define (non-linear) encoding models 12 | and fit their parameter :math:`\theta` to predict voxel responses :math:`x` 13 | using non-linear descent. 14 | 15 | It is important to point out that in most of the literature, the parameters 16 | :math:`\theta` are **asssumed to be fixed**. In such work, researchers assume 17 | a limited set of :math:`m` neural populations, each with their own set of parameters 18 | :math:`\theta`. 19 | 20 | The responses in different :math:`n` neural measures (e.g., fMRI voxels) are then to 21 | assume to be a **linear combination** of these fixed neural populations. 22 | How much each neural population contributes to each voxel is then defined by a 23 | weight matrix :math:`W` of size :math:`m \times n`. 24 | 25 | .. math:: 26 | x_j = \sum W_{i, j} \cdot f_j(\theta_j) 27 | 28 | The big advantage of this approach is that it allows us fit the weight matrix 29 | :math:`W` using linear regression. This is a much, much faster approach than fitting 30 | the parameters :math:`\theta` using non-linear gradient descent. 31 | 32 | In this lesson, we will see how we can use the ``EncodingModel`` class to fit 33 | linear encoding models. 34 | 35 | Setting up a linear encoding model in ``braincoder`` 36 | #################################################### 37 | 38 | In braincoder, we can set up a linear encoding model by defining a 39 | **fixed number of neural encoding populations**, each with their own 40 | parameters set :math:`\theta_j`. 41 | 42 | Here we use a Von Mises tuning curve to define the neural populations 43 | that are senstive to the orientation of a grating stimulus. 44 | Note that orientations are given in radians, so lie between -pi and pi. 45 | 46 | Set up a von Mises model 47 | ************************ 48 | 49 | .. literalinclude:: /../examples/00_encodingdecoding/linear_encoding_model.py 50 | :start-after: # Import necessary libraries 51 | :end-before: # %% 52 | 53 | Once we have set up the model, we can first have a look at the predictions for the 54 | 6 different basis functions: 55 | 56 | .. literalinclude:: /../examples/00_encodingdecoding/linear_encoding_model.py 57 | :start-after: # Plot the basis functions 58 | :end-before: # %% 59 | 60 | .. |basis_functions| image:: ../auto_examples/00_encodingdecoding/images/sphx_glr_linear_encoding_model_001.png 61 | .. centered:: |basis_functions| 62 | 63 | Because the model also has a :math:`n{\times}m` weight matrix (number of voxels x number of neural populations), 64 | we can also use the model to predict the responses of different voxels to the same orientation stimuli: 65 | 66 | .. literalinclude:: /../examples/00_encodingdecoding/linear_encoding_model.py 67 | :start-after: # Plot the predicted responses for the 3 voxels 68 | :end-before: # %% 69 | 70 | .. |voxel_predictions1| image:: ../auto_examples/00_encodingdecoding/images/sphx_glr_linear_encoding_model_002.png 71 | .. centered:: |voxel_predictions1| 72 | 73 | Fit a linear encoding model using (regularised) OLS 74 | **************************************************** 75 | 76 | To fit linear encoding models we can use the ``braincoder.optimize.WeightFitter``. 77 | This fits weights to the model using linear regression. Note that one can 78 | also provide an ``alpha``-parameter to the ``WeightFitter`` to regularize the 79 | weights (pull them to 0; equivalent to putting a Gaussian prior on the weights). 80 | This is often a very good idea in real data! 81 | 82 | .. literalinclude:: /../examples/00_encodingdecoding/linear_encoding_model.py 83 | :start-after: # Import the weight fitter 84 | :end-before: # %% 85 | 86 | .. |voxel_predictions2| image:: ../auto_examples/00_encodingdecoding/images/sphx_glr_linear_encoding_model_003.png 87 | .. centered:: |voxel_predictions2| 88 | 89 | .. note:: 90 | 91 | The complete Python script and its output can be found :ref:`here`. 92 | 93 | Summary 94 | ####### 95 | In this lesson, we had a look at *linear* encoding models. 96 | These models 97 | * Assume a fixed number of neural populations, each with their own parameters :math:`\theta_j` 98 | * Every voxel then is assumed to be a linear combination of these neural populations 99 | * The weights of this linear combination can be fit using linear regression 100 | * This is much faster than fitting the parameters :math:`\theta_j` using non-linear gradient descent 101 | 102 | In the :ref:`next lesson`, we will see how we can add a *noise model* 103 | to the encoding models, which yields a likelihood function which we can invert in 104 | a principled manner. -------------------------------------------------------------------------------- /docs/tutorial/lesson3.rst: -------------------------------------------------------------------------------- 1 | .. _tutorial_lesson3: 2 | 3 | ==================================================================== 4 | Lesson 3: Building the likelihood (by fitting the covariance matrix) 5 | ==================================================================== 6 | 7 | For many neuroscientific questions, we are interested in the relationship between neural codes 8 | and objective stimulus features. For example, we might want to know how the brain represents 9 | numbers, orientations, or spatial positions, and how these representations change as a function 10 | of task demands, attention, or prior expectations. 11 | 12 | One particular powerful approach is to *decode* stimulus features from neural data 13 | in a Bayesian fashion (e.g., van Bergen et al., 2015; Baretto-Garcia et al, 2023). 14 | 15 | Inverting the encoding models 16 | ############################# 17 | 18 | Here, we show how we go from a **deterministic** forward model (i.e., a model that predicts neural 19 | responses from stimulus features) to a **probabilistic** inverse model (i.e., a model that 20 | predicts stimulus features from neural responses). We will do so using a **Bayesian inversion scheme**: 21 | 22 | .. math:: 23 | p(s | x; \theta) = \frac{p(x | s; \theta) p(s)}{p(x; \theta)} 24 | 25 | where :math:`s` is a n-dimensional point in stimulus space, and :math:`x` is a n-dimensional 26 | activation pattern in neural space, and :math:`p(s | x; \theta)` is the posterior probability of 27 | stimulus :math:`s` given neural response :math:`x` and model parameters :math:`\theta`. 28 | 29 | A multivariate likelihood function 30 | ################################## 31 | 32 | The crucial element that is still lacking for this Bayesian inversion scheme is a **likelihood function**. 33 | Note standard encoding models do not have a likelihood function, because they are deterministic 34 | (:math:`f(s;\theta): s \mapsto x`). 35 | They give us the average neural response to a certain stimulus, but they don't tell us how likely a certain 36 | neural response is, given a certain stimulus. 37 | However, we can easily derive a likelihood function from the forward model by adding Gaussian noise: 38 | 39 | .. math:: 40 | x = f(s; \theta) + \epsilon 41 | .. 42 | p(x | s; \theta) = f(s; \theta) + \epsilon 43 | 44 | where :math:`\epsilon` is a *multivariate Normal distribution* 45 | 46 | .. math:: 47 | \epsilon \sim \mathcal{N}(0, \Sigma) 48 | 49 | 50 | Estimating :math:`\Sigma` 51 | ######################### 52 | 53 | The problem with high-dimensional covariance matrices 54 | ***************************************************** 55 | 56 | How to set the covariance matrix :math:`\Sigma`? 57 | One approach would be to assume independent noise across neural dimensions (e.g., 58 | fMRI voxels), and use a spherical covariance matrix :math:`\Sigma = \tau^T\tau I`, where 59 | :math:`\tau` is a vector containing the standard deviation of the residuals of the encoding models 60 | and I is the identity matrix. 61 | However, if there *is* substantial covariance between the noise terms of different neural 62 | dimensions (i.e., voxels), this could have severe consequences for the decoding performance. 63 | In particular, the posterior might be overly confident in its predictions, assuming 64 | independent sources of information that are not. Under some circumstances, the mean posterior, 65 | the point estimate of the decoded stimulus, can also be affected. 66 | Van Bergen et al. (2015, 2017) showed that modeling some of the covariance is indeed crucial 67 | for making correct inferences about neural data. 68 | However, we generally have a large number of voxels and very limited data. 69 | Therefore, estimating the full covariance 70 | matrix is not feasible (note that the number of free parameters scales quadratically with 71 | the number of voxels :math:`p= n \times \frac{n-1}{2}`). Note that this is a general 72 | problem of estimating covariance, and not specific to our use case (e.g., Ledoit, ...). 73 | 74 | Van Bergen et al. (2015, 2017) proposed a two-part solution. 75 | 76 | Regularizing the covariance matrix 77 | ********************************** 78 | 79 | The first proposal of van Bergen et al. (2015, 2017), based on the work of 80 | Ledoit (...), is to use a *shrinkage estimator* to estimate the covariance matrix. 81 | Specifically, the free parameter :math:`\rho` scales between a perfectly-correlated 82 | covariance matrix (:math:`\rho = 1`) and a diagonal covariance matrix (:math:`\rho = 0`): 83 | 84 | .. math:: 85 | \Sigma = \rho \tau^T\tau + (1-\rho) \tau^T\tau I 86 | 87 | Accounting for shared noise due to shared neural populations 88 | ************************************************************ 89 | 90 | Van Bergen et al. (2015) also note that *voxels that share more tuned neural populations* 91 | should also be more highly correlated. They do so by adding a second term to the covariance 92 | matrix, which is based on the similarity of the tuning curves of the voxels, 93 | given by the weight matrix :math:`W`. 94 | Recall that W is a :math:`m \times n` matrix, where :math:`n` is the number of voxels, 95 | and :math:`m` is the number of neural populations. So :math:`WW^T` is a :math:`n \times n` 96 | matrix that contains the similarity between all pairs of voxels. We scale this 97 | matrix by the free parameters :math:`\sigma^2`: 98 | 99 | .. math:: 100 | \sigma^2 WW^T 101 | 102 | .. note:: 103 | You might realize now that non-linear encoding models, like a standard 104 | ``GaussianPRFModel`` does not have a weight matrix :math:`W`. There are 105 | two work arounds here: 106 | * Use the identity matrix :math:`I` as weight-matrix. This corresponds to saying that all neural populations in all the voxels are unique and share no noise sources. 107 | * Rewrite the individual encoding functions :math:`f(\theta_{1..i})` to linear functions that can be interpreted as a weight matrix :math:`W`. For PRF models, we can just set up a grid of plausible stimulus values and set the height of the PRF as the weight of those stimulus values. The weight matrix :math:`W` effectively then describes the amount of overlap between the receptive fields of different voxels. The noise covariance is then assumed to scale with the amount of overlap in recpeptive fields, which seems quite plausible and has worked well in earlier work (e.g., Baretto-Garcia et al., 2023). 108 | 109 | The complete formula for the covariance matrix is thus: 110 | 111 | .. math:: 112 | \Sigma = \rho \tau^T\tau + (1-\rho) \tau^T\tau I + \sigma^2 W^TW 113 | 114 | Thus, the :math:`n \times n` covariance-matrix :math:`\Sigma` is now described by 115 | :math:`n + 2` parameters (the :math:`\tau` noise vector of length 116 | :math:`n` plus :math:`\rho` and :math:`\sigma`. 117 | 118 | Using the sample covariance 119 | *************************** 120 | 121 | Note that additional elements can be added to the covariance matrix. 122 | For one, we can add a proportion :math:`\lambda` of the empirical 123 | noise covariance matrix :math:`S` to the regularized covariance matrix, to allow for 124 | a more sophisticated noise model: 125 | 126 | .. math:: 127 | \Sigma' = (1-\lambda) \cdot \Sigma + \lambda S 128 | 129 | Using the anatomical distance 130 | ***************************** 131 | Similarly, one could add a term that accounts for the physical 132 | distance between different neural sources (i.e., voxels). 133 | 134 | 135 | Fitting :math:`\Sigma` 136 | ########################## 137 | 138 | ``braincoder`` contains the ``ResidualFitter``-class that can be used 139 | to fit a noise covariance matrix, and thereby a likelihood function 140 | to a fitted ``EncodingModel``. 141 | 142 | Here we first set up a simple ``VonmisesPRF``-model and simulate some data 143 | with covarying noise. 144 | 145 | .. literalinclude:: /../examples/00_encodingdecoding/fit_residuals.py 146 | :start-after: # We set up a simple VonMisesPRF model 147 | :end-before: # %% 148 | 149 | Now we can import the `ResidualFitter` and estimate :math:`\Sigma` (here called 150 | `omega` for legacy reasons): 151 | 152 | .. include:: ../auto_examples/00_encodingdecoding/fit_residuals.rst 153 | :start-after: .. GENERATED FROM PYTHON SOURCE LINES 41-53 154 | :end-before: .. rst-class:: sphx-glr-timing 155 | 156 | 157 | .. note:: 158 | Here we have used the *generating* parameters and weights to fit the 159 | ``omega``-matrix. Note that in real data, we would use estimated parameters 160 | and/or weights. 161 | 162 | .. note:: 163 | 164 | The complete Python script and its output can be found :ref:`here`. 165 | 166 | Summary 167 | ####### 168 | 169 | In this lesson we have seen: 170 | * We need to add a noise model to the classical encoding models :math:`f(s, \theta): s \mapsto x` to get a **likelihood function** which we can invert. 171 | * Concretely, we add a multivariate Gaussian noise model to the deterministic predictions of the encoding models 172 | * We need a noise covariance matrix :math:`\Sigma` (or ``omega``) for this to work. 173 | * We use a regularised estimate of the covariance matrix. 174 | 175 | In the :ref:`next lesson`, we will further explore how we can use a likelihood 176 | function to map from neural responses to stimulus features. 177 | -------------------------------------------------------------------------------- /docs/tutorial/lesson4.rst: -------------------------------------------------------------------------------- 1 | .. _tutorial_lesson4: 2 | 3 | ==================================================== 4 | Lesson 4: From neural responses to stimulus features 5 | ==================================================== 6 | 7 | Now we have a likelihood function :math:`p(x|s;\theta)`, we know, 8 | **for a given stimulus**, how likely different multivariate neural responses 9 | are. What we might be interested in the inverse, :math:`p(s|x;\theta)`, 10 | how likely different stimuli are *for a given BOLD pattern*. 11 | 12 | To get here, we can use Bayes rule: 13 | 14 | .. math:: 15 | p(s|x, \theta) = \frac{p(x|s, \theta) p(s)}{p(x)} 16 | 17 | Note that 18 | * We need to define a prior :math:`p(s)` 19 | * To be able to integrate over :math:`p(s|x, \theta)` we need to approximate :math:`p(x)` by normalisation. 20 | .. * For a given :math:`x`, we want to evaluate the likelihood at all possible :math:`s` 21 | 22 | In practice, for simple one or two-dimensional stimulus space, 23 | we can use a uniform prior and evaluate likelihoods at a grid of useful 24 | points within that prior. 25 | 26 | The crucial insight here is that we, for a given neural response pattern :math:`x` 27 | **try out which of a large set of possible stimuli are actually consistent with this 28 | neural response pattern** 29 | 30 | Let's say we observed the following orientation receptive field, centered around 31 | a half :math:`pi` and we observe, in unseen data, an activation of slightly less 32 | than 0.2: 33 | 34 | .. |rf_activation| image:: ../auto_examples/00_encodingdecoding/images/sphx_glr_invert_model_001.png 35 | .. centered:: |rf_activation| 36 | 37 | We can now set up a likelihood function and find which stimuli are consistent with 38 | this activation pattern as follows. 39 | 40 | First set up the model.. 41 | 42 | .. literalinclude:: /../examples/00_encodingdecoding/invert_model.py 43 | :start-after: # Set up six evenly spaced von Mises PRFs 44 | :end-before: # %% 45 | 46 | Then evaluate the likelihood for different orientations 47 | 48 | .. literalinclude:: /../examples/00_encodingdecoding/invert_model.py 49 | :start-after: # Evaluate the likelihood of different possible orientations 50 | :end-before: # %% 51 | 52 | .. |likelihood1| image:: ../auto_examples/00_encodingdecoding/images/sphx_glr_invert_model_002.png 53 | .. centered:: |likelihood1| 54 | 55 | You can see that the likelihood is highest around :math:`\frac{1}{8}\pi` and 56 | :math:`\frac{3}{8}\pi`! With only one receptive field the predictions for these 57 | two points in stimulus space are identical! 58 | 59 | 60 | If we have two RFs, the situation becomes unambiguous: 61 | 62 | .. |rf_activation2| image:: ../auto_examples/00_encodingdecoding/images/sphx_glr_invert_model_003.png 63 | .. centered:: |rf_activation2| 64 | 65 | 66 | .. literalinclude:: /../examples/00_encodingdecoding/invert_model.py 67 | :start-after: # Set up 2-dimensional model to invert 68 | :end-before: # %% 69 | 70 | .. |likelihood2| image:: ../auto_examples/00_encodingdecoding/images/sphx_glr_invert_model_004.png 71 | .. centered:: |likelihood2| 72 | 73 | .. note:: 74 | 75 | The complete Python script and its output can be found :ref:`here`. 76 | Summary 77 | ####### 78 | 79 | We have seen how to set up a simple encoding model and how to invert it to 80 | see which stimulis :math:`s` are consistent with a given neural response :math:`x`. 81 | 82 | For real data we of course have hundreds of voxels and thousands of stimuli. 83 | In the :ref:`next tutorial ` we will see how we can 84 | decode data in a more natural setting. 85 | -------------------------------------------------------------------------------- /docs/tutorial/lesson5.rst: -------------------------------------------------------------------------------- 1 | .. _tutorial_lesson5: 2 | 3 | ==================================================== 4 | Lesson 5: Decoding noisy neural data 5 | ==================================================== 6 | 7 | In this lesson, we will see a more elaborate example of how we 8 | can decode stimulus features from neural data and check how well it worked. 9 | 10 | Simulate data 11 | ############# 12 | 13 | 14 | First we simulate data from a ``GaussianPRF`` with 20 PRFs 15 | a 50 time points. The parameters are randomly generated 16 | within a plausible range. 17 | 18 | .. literalinclude:: /../examples/00_encodingdecoding/decode.py 19 | :start-after: # Set up a neural model 20 | :end-before: # %% 21 | 22 | 23 | Estimate parameters of encoding model :math:`\theta` 24 | #################################################### 25 | 26 | Now we can use ``ParameterFitter`` to estimate back the parameters. 27 | We also print out the average correlation between the true and estimated 28 | parameters, for each parameter seperately. 29 | 30 | .. include:: ../auto_examples/00_encodingdecoding/decode.rst 31 | :start-after: .. GENERATED FROM PYTHON SOURCE LINES 34-53 32 | :end-before: .. GENERATED FROM PYTHON SOURCE LINES 54-62 33 | 34 | Note how some parameters (e.g., ``mu`` and ``amplitude``) are easier 35 | to recover than others (e.g., ``sd``). 36 | 37 | Estimate the covariance matrix :math:`\Sigma` 38 | ############################################# 39 | 40 | To convert the **encoding model** into a **likelihood model**, 41 | we need to add a **noise model**. More specifically, a 42 | Gaussian noise model. This requires us to estimate 43 | the covariance matrix :math:`\Sigma` of the noise. 44 | 45 | Note how we use ``init_pseudoWWT`` and a plausible ``stimulus_range`` 46 | to approximate the ``WWT`` matrix. 47 | 48 | .. literalinclude:: /../examples/00_encodingdecoding/decode.py 49 | :start-after: # Now we fit the covariance matrix 50 | :end-before: # %% 51 | 52 | Decode stimuli 53 | ############## 54 | 55 | Simulate "unseen" data 56 | ********************** 57 | 58 | Now we have estimated the **encoding model** and the **noise model**, 59 | we can use them to decode the stimuli from the neural data. 60 | For that, we simulate a new set of neural data and decode the stimuli 61 | from. 62 | 63 | .. literalinclude:: /../examples/00_encodingdecoding/decode.py 64 | :start-after: # Now we simulate unseen test data: 65 | :end-before: # And decode the test paradigm 66 | 67 | Calculate likelihood density for a range of stimuli 68 | *************************************************** 69 | 70 | We use ``model.get_stimulus_pdf`` to get the likelihood 71 | for a plausible range of stimuli. 72 | 73 | .. literalinclude:: /../examples/00_encodingdecoding/decode.py 74 | :start-after: # And decode the test paradigm 75 | :end-before: # %% 76 | 77 | Plot posterior distributions 78 | **************************** 79 | 80 | Here, we plot some of the posterior distributions for the stimuli. 81 | 82 | .. literalinclude:: /../examples/00_encodingdecoding/decode.py 83 | :start-after: # Finally, we make some plots to see how well the decoder did 84 | :end-before: # %% 85 | 86 | .. |posteriors| image:: ../auto_examples/00_encodingdecoding/images/sphx_glr_decode_001.png 87 | .. centered:: |posteriors| 88 | 89 | Estimate the mean posterior 90 | ******************************************** 91 | 92 | We can also estimate the mean posterior. 93 | To do this we should just take the expectation of the posterior, 94 | which is an integral: 95 | 96 | .. math:: 97 | \mathbb{E}[s] = \int s \cdot p(s|x) ds 98 | 99 | similarly, we can also calculate the expected distance of the 100 | expected stimulus :math:``E[s]`` to the true stimulus: 101 | 102 | .. math:: 103 | \mathbb{E}[d] = \int \|s - \mathbb{E}[s]\| \cdot p(s|x) ds 104 | 105 | .. literalinclude:: /../examples/00_encodingdecoding/decode.py 106 | :start-after: # Let's look at the summary statistics of the posteriors posteriors 107 | :end-before: # Let's see how far the posterior mean is from the ground truth 108 | 109 | Once we have these two summary statistics of the posterior, we can compare 110 | them to the ground truth. 111 | 112 | First of all, how close is the MAP stimulus to the true stimulus? 113 | 114 | .. literalinclude:: /../examples/00_encodingdecoding/decode.py 115 | :start-after: # Let's see how far the posterior mean is from the ground truth 116 | :end-before: # Let's see how the error depends on the standard deviation of the posterior 117 | 118 | 119 | .. |decoding_error| image:: ../auto_examples/00_encodingdecoding/images/sphx_glr_decode_002.png 120 | .. centered:: |decoding_error| 121 | 122 | That looks pretty good! 123 | 124 | Finally, is the standard deviation of the posterior a good estimate 125 | of the true noise level? 126 | 127 | .. literalinclude:: /../examples/00_encodingdecoding/decode.py 128 | :start-after: # Let's see how the error depends on the standard deviation of the posterior 129 | :end-before: # %% 130 | 131 | .. |decoding_error_error| image:: ../auto_examples/00_encodingdecoding/images/sphx_glr_decode_003.png 132 | .. centered:: |decoding_error_error| 133 | 134 | That looks pretty good too! The standard deviation of the posterior is actually 135 | predictive of the true noise level. 136 | 137 | Maximum a posteriori (MAP) estimate 138 | #################################### 139 | 140 | Alternatively, instead of the mean of the posterior, we can also try to 141 | find the maximum a posteriori (MAP) estimate. This is the stimulus that 142 | has the highest probability of being the true stimulus, given the neural data. 143 | 144 | The MAP estimate is the stimulus that maximizes the posterior probability: 145 | 146 | .. math:: 147 | \hat{s} = \arg\max_s p(s|x) 148 | 149 | Note that there are theoretical advantages to the **mean posterior** over the **MAP estimate**. 150 | However, when the stimulus is high-dimensional, it is often easier to find the MAP estimate. 151 | 152 | ``braincoder`` has a ``StimulusFitter``, that can findo MAP estimates of the stimulus for you. 153 | Stimulus optimisation tends to be very sensitive to local minima, so we first 154 | use a grid search to find a good starting point for the optimisation. 155 | 156 | .. literalinclude:: /../examples/00_encodingdecoding/decode.py 157 | :start-after: # Now, let's try to find the MAP estimate using gradient descent 158 | :end-before: # %% 159 | 160 | Now we apply gradient descent. 161 | 162 | .. literalinclude:: /../examples/00_encodingdecoding/decode.py 163 | :start-after: # We can then refine the estimate using gradient descent 164 | :end-before: # Let's see how well we did 165 | 166 | And then plot how well we did. 167 | 168 | .. literalinclude:: /../examples/00_encodingdecoding/decode.py 169 | :start-after: # Let's see how well we did 170 | :end-before: # %% 171 | 172 | .. |estimates_vs_groundtruth| image:: ../auto_examples/00_encodingdecoding/images/sphx_glr_decode_004.png 173 | .. centered:: |estimates_vs_groundtruth| 174 | 175 | 176 | Note how in this example, the MAP and mean posterior estimates are almost identical. 177 | However, the mean posterior, where we take into account uncertainty in all stimulus 178 | posterior dimensions, has theoretical advantages, in particular when the stimulus dimensionality 179 | is high (here it is only 1-dimensional). 180 | 181 | .. note:: 182 | 183 | The complete Python script and its output can be found :ref:`here`. 184 | 185 | Summary 186 | ####### 187 | In this tutorial, we have seen how to decode stimuli from neural data 188 | in a more realistic setting. 189 | 190 | The concrete steps were: 191 | * Fit the parameters of the encoding model 192 | * Estimate the covariance matrix of the noise 193 | * Apply the likelihood model to get a posterior over stimuli 194 | * Use numerical integration to get the expected stimulus :math:`E[s]` 195 | * Use numerical integration to get the expected distance between the real and expected stimulus (the standard deviation of the posterior) 196 | * Use grid+gradient descent optimisation to get the most likely stimulus 197 | 198 | In the :ref:`next tutorial `, we will see how to do the same thing, but in a two-dimensional 199 | stimulus space! -------------------------------------------------------------------------------- /docs/tutorial/lesson6.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _tutorial_lesson6: 3 | 4 | ==================================================== 5 | Lesson 6: Decoding two-dimensional stimulus spaces 6 | ==================================================== 7 | 8 | In this lesson, we will decode a **two-dimensional** stimulus 9 | space. Specifically, we are going to assume that the 10 | *stimulus drive* (`amplitude`) of specific stimuli can be modulated, 11 | for example by the *attentional state* of the subject 12 | (see, e.g., :footcite:t:`serences2007spatially`, :footcite:t:`sprague2013attention`, and 13 | :footcite:t:`sawetsuttipan2023perceptual`). 14 | 15 | Hence, every stimulus is now characterized by two parameters: 16 | its *orientation* (``x (radians)``) and its *amplitude* (``amplitude``). 17 | 18 | Let's set up a virtual **mapping experiment**, where we are simulating data, 19 | estimate parameters, and decode the stimulus space. 20 | 21 | *Crucially*, in the mapping experiment, we are ('rightfully') going 22 | to assume that the stimulus drive (``amplitude``) is not modulated 23 | and is always ``1.0``. 24 | 25 | We use the argument ``model_stimulus_amplitude`` of the ``VonMisesPRF``-model 26 | to indicate we want to model both orientation and amplitudes. 27 | 28 | .. include:: ../auto_examples/00_encodingdecoding/decode2d.rst 29 | :start-after: .. GENERATED FROM PYTHON SOURCE LINES 12-36 30 | :end-before: .. GENERATED FROM PYTHON SOURCE LINES 37-52 31 | 32 | Now we simulate data, estimate parameters, and the noise model: 33 | 34 | .. literalinclude:: /../examples/00_encodingdecoding/decode2d.py 35 | :start-after: # Now we can simulate some data and estimate parameters+noise 36 | :end-before: # %% 37 | 38 | After fitting the mapping paradigm, we are now going to simulate an 39 | experiment with two conditions, where the stimulus drive is modulated 40 | in the **attended** condition, the stimulus drive is modulated by a factor of **``1.5``**, 41 | whereas in the **unattended** condition, the stimulus drive is modulated by a factor of **``0.5``**. 42 | 43 | .. literalinclude:: /../examples/00_encodingdecoding/decode2d.py 44 | :start-at: # Now we set up an experimental paradigm with two conditions 45 | :end-before: # %% 46 | 47 | If we plot the simulated responses as a function of the ground truth orientation, 48 | we can clearly see both the preferred orientation and the modulation of the stimulus drive 49 | in the attended condition: 50 | 51 | .. literalinclude:: /../examples/00_encodingdecoding/decode2d.py 52 | :start-at: # Plot the data 53 | :end-before: # %% 54 | 55 | .. |data| image:: ../auto_examples/00_encodingdecoding/images/sphx_glr_decode2d_001.png 56 | .. centered:: |data| 57 | 58 | Now we have the data and the noise model we can, for each data point, calculate its likelihood 59 | given different plausible stimuli. Note that this set of plausible stimuli 60 | consistutes a (flat) prior, so after normalizing likelihoods, we can interpret 61 | the likelihood as a posterior distribution. 62 | Also, remember that we use the PRF and noise parameters that we estimated 63 | in the mapping experiment! 64 | 65 | Note how we use the ``pandas`` library to flexibly manipulate the multidimensional likelihood; 66 | 67 | .. literalinclude:: /../examples/00_encodingdecoding/decode2d.py 68 | :start-at: # Now we can calculate the 2D likelihood/posterior of different orientations+amplitudes for the data 69 | :end-before: # %% 70 | 71 | Here we plot the posterior distribution for the first 9 data points of the **attended** 72 | and **unattended** conditions: 73 | 74 | .. literalinclude:: /../examples/00_encodingdecoding/decode2d.py 75 | :start-at: # Plot 2D posteriors for first 9 trials 76 | :end-before: # %% 77 | 78 | .. |attended_posterior| image:: ../auto_examples/00_encodingdecoding/images/sphx_glr_decode2d_002.png 79 | .. centered:: |attended_posterior| 80 | 81 | .. |unattended_posterior| image:: ../auto_examples/00_encodingdecoding/images/sphx_glr_decode2d_003.png 82 | .. centered:: |unattended_posterior| 83 | 84 | These 2D posterior distributions are very insightful! Notice, for example: 85 | * The orientation of the attended stimulus is more precisely defined in the posterior than the unattended stimulus. 86 | * The amplitude of the attended stimulus is more precisely defined in the posterior than the unattended stimulus. 87 | * The MAP estimate/mean posterior (red ``+``) are closer to the ground truth for the attended stimulus than the unattended stimulus. 88 | 89 | ============================================ 90 | Decode the marginal probability distribution 91 | ============================================ 92 | 93 | When we now want to decode the marginal probability distribution of ``orientation`` 94 | we need to take into the account of the uncertainty of both dimensions. Hence, 95 | we take the expectation, integrating the other dimension away: 96 | 97 | .. math:: 98 | p(\text{orientation}) = \int_{\text{amplitude}} p(\text{orientation}, \text{amplitude}) da 99 | 100 | .. literalinclude:: /../examples/00_encodingdecoding/decode2d.py 101 | :start-at: # Now we can calculate the 1D posterior for specific orientations _or_ amplitudes 102 | :end-before: # %% 103 | 104 | .. |orientation_posterior| image:: ../auto_examples/00_encodingdecoding/images/sphx_glr_decode2d_004.png 105 | .. centered:: |orientation_posterior| 106 | 107 | =================================================== 108 | Decode the conditional probability distribution 109 | =================================================== 110 | 111 | We see that especially the orientation posterior of the unattended condition is very broad. 112 | Part of the problem is that we have to take into account the uncertainty surrounding the 113 | generating amplitude. 114 | We can make the posterior more precise by using a ground truth amplitude, which we can 115 | use to condition the posterior on: 116 | 117 | .. math:: 118 | p(\text{orientation} | \text{amplitude} = a) = p(\text{orientation}, \text{amplitude} = a) 119 | 120 | (With :math:`a` being the ground truth amplitude). 121 | 122 | .. literalinclude:: /../examples/00_encodingdecoding/decode2d.py 123 | :start-at: # Use the ground truth amplitude to improve the orientation posterior 124 | :end-before: # %% 125 | 126 | .. |orientation_posterior_conditional| image:: ../auto_examples/00_encodingdecoding/images/sphx_glr_decode2d_005.png 127 | .. centered:: |orientation_posterior_conditional| 128 | 129 | Note how especially for the unattended condition, the orientation posterior is now much more precise. 130 | 131 | ================= 132 | The complex plane 133 | ================= 134 | 135 | If we want to have calculate the *mean posterior* for ``orientation`` and ``amplitude``, 136 | we can use numerical integration. Notice, however, that ``orientation`` is in polar space, 137 | which means we can not just integrate over raw angles (e.g., what is the mean angle of ``[0.1*pi, 1.9*pi]``?). 138 | 139 | The problem is solved by using integration over ``complex`r numbers. 140 | Briefly, complex numbers represent a point in the complex plane, where the real part is the ``x``-coordinate 141 | and the imaginary part is the ``y``-coordinate. 142 | 143 | This is what the complex plane looks like for the first 10 data points of the **attended** condition: 144 | 145 | .. literalinclude:: /../examples/00_encodingdecoding/decode2d.py 146 | :start-at: # Let's plot the firs 10 trials in the complex plane 147 | :end-before: # %% 148 | 149 | .. |10trials_complex_plane| image:: ../auto_examples/00_encodingdecoding/images/sphx_glr_decode2d_006.png 150 | .. centered:: |10trials_complex_plane| 151 | 152 | If we now want to take integrals/expectations/averages over angles, we can simply 153 | take the average over the complex numbers that represent the angles. which 154 | is equivalent to taking the average over the unit circle in the complex plane. 155 | 156 | .. include:: ../auto_examples/00_encodingdecoding/decode2d.rst 157 | :start-after: .. GENERATED FROM PYTHON SOURCE LINES 217-254 158 | :end-before: .. GENERATED FROM PYTHON SOURCE LINES 255-289 159 | 160 | Note how the correlation between the ground truth and posterior mean 161 | is much higher for the attended condition than the unattended condition. 162 | 163 | Also note that you should never use normal (e.g., Pearson's) correlation 164 | on angles! 165 | 166 | We can now also plot the mean posteriors in the complex plane: 167 | 168 | .. literalinclude:: /../examples/00_encodingdecoding/decode2d.py 169 | :start-at: # Let's see how far the posterior mean is from the ground truth 170 | :end-before: # %% 171 | 172 | .. |estimates_complex_plane| image:: ../auto_examples/00_encodingdecoding/images/sphx_glr_decode2d_007.png 173 | .. centered:: |estimates_complex_plane| 174 | 175 | Note how the unattended condition is, indeed, generally much further away from the ground truth 176 | than the attended condition. 177 | 178 | Note, also that even **within** conditions we can predict the error of the classifier 179 | by using the variance of the posterior distribution: 180 | 181 | .. literalinclude:: /../examples/00_encodingdecoding/decode2d.py 182 | :start-at: # Plot the error as a function of the standard deviation of the posterior 183 | :end-before: # %% 184 | 185 | .. |error_vs_posterior_sd| image:: ../auto_examples/00_encodingdecoding/images/sphx_glr_decode2d_008.png 186 | .. centered:: |error_vs_posterior_sd| 187 | 188 | References 189 | ========== 190 | .. footbibliography:: 191 | -------------------------------------------------------------------------------- /docs/tutorial/lesson7.rst: -------------------------------------------------------------------------------- 1 | .. _tutorial_lesson7: 2 | 3 | ==================================================== 4 | Lesson 7: Fitting PRF models to visual space 5 | ==================================================== 6 | 7 | 8 | .. literalinclude:: /../examples/00_encodingdecoding/fit_prf.py -------------------------------------------------------------------------------- /examples/00_encodingdecoding/README.txt: -------------------------------------------------------------------------------- 1 | ######## 2 | Examples 3 | ######## 4 | 5 | Some introduction to the encoding/decoding framework. -------------------------------------------------------------------------------- /examples/00_encodingdecoding/decode.py: -------------------------------------------------------------------------------- 1 | """ 2 | Decoding of stimuli from neural data 3 | ============================================= 4 | 5 | Here we will simulate neural data given a ground truth encoding model 6 | and try to decode the stimulus from the data. 7 | """ 8 | 9 | # Set up a neural model 10 | from braincoder.models import GaussianPRF 11 | import numpy as np 12 | import pandas as pd 13 | import scipy.stats as ss 14 | 15 | # Set up 100 random of PRF parameters 16 | n = 20 17 | n_trials = 50 18 | noise = 1. 19 | 20 | mu = np.random.rand(n) * 100 21 | sd = np.random.rand(n) * 45 + 5 22 | amplitude = np.random.rand(n) * 5 23 | baseline = np.random.rand(n) * 2 - 1 24 | 25 | parameters = pd.DataFrame({'mu':mu, 'sd':sd, 'amplitude':amplitude, 'baseline':baseline}) 26 | 27 | # We have a paradigm of random numbers between 0 and 100 28 | paradigm = np.ceil(np.random.rand(n_trials) * 100) 29 | 30 | model = GaussianPRF(parameters=parameters) 31 | data = model.simulate(paradigm=paradigm, noise=noise) 32 | 33 | # %% 34 | 35 | # Now we fit back the PRF parameters 36 | from braincoder.optimize import ParameterFitter, ResidualFitter 37 | fitter = ParameterFitter(model, data, paradigm) 38 | mu_grid = np.arange(0, 100, 5) 39 | sd_grid = np.arange(5, 50, 5) 40 | 41 | grid_pars = fitter.fit_grid(mu_grid, sd_grid, [1.0], [0.0], use_correlation_cost=True, progressbar=False) 42 | grid_pars = fitter.refine_baseline_and_amplitude(grid_pars) 43 | 44 | for par in ['mu', 'sd', 'amplitude', 'baseline']: 45 | print(f'Correlation grid-fitted parameter and ground truth for *{par}*: {ss.pearsonr(grid_pars[par], parameters[par])[0]:0.2f}') 46 | 47 | gd_pars = fitter.fit(init_pars=grid_pars, progressbar=False) 48 | 49 | for par in ['mu', 'sd', 'amplitude', 'baseline']: 50 | print(f'Correlation gradient descent-fitted parameter and ground truth for *{par}*: {ss.pearsonr(grid_pars[par], parameters[par])[0]:0.2f}') 51 | 52 | 53 | # %% 54 | 55 | # Now we fit the covariance matrix 56 | stimulus_range = np.arange(1, 100).astype(np.float32) 57 | 58 | model.init_pseudoWWT(stimulus_range=stimulus_range, parameters=gd_pars) 59 | resid_fitter = ResidualFitter(model, data, paradigm, gd_pars) 60 | omega, dof = resid_fitter.fit(progressbar=False) 61 | 62 | # %% 63 | 64 | # Now we simulate unseen test data: 65 | test_paradigm = np.ceil(np.random.rand(n_trials) * 100) 66 | test_data = model.simulate(paradigm=test_paradigm, noise=noise) 67 | 68 | # And decode the test paradigm 69 | posterior = model.get_stimulus_pdf(test_data, stimulus_range, model.parameters, omega=omega, dof=dof) 70 | 71 | # %% 72 | 73 | # Finally, we make some plots to see how well the decoder did 74 | import matplotlib.pyplot as plt 75 | import seaborn as sns 76 | 77 | tmp = posterior.set_index(pd.Series(test_paradigm, name='ground truth'), append=True).loc[:8].stack().to_frame('p') 78 | 79 | g = sns.FacetGrid(tmp.reset_index(), col='frame', col_wrap=3) 80 | 81 | g.map(plt.plot, 'stimulus', 'p', color='k') 82 | 83 | def test(data, **kwargs): 84 | plt.axvline(data.mean(), c='k', ls='--', **kwargs) 85 | g.map(test, 'ground truth') 86 | g.set(xlabel='Stimulus value', ylabel='Posterior probability density') 87 | 88 | # %% 89 | 90 | # Let's look at the summary statistics of the posteriors posteriors 91 | def get_posterior_stats(posterior, normalize=True): 92 | posterior = posterior.copy() 93 | posterior = posterior.div(np.trapz(posterior, posterior.columns,axis=1), axis=0) 94 | 95 | # Take integral over the posterior to get to the expectation (mean posterior) 96 | E = np.trapz(posterior*posterior.columns.values[np.newaxis,:], posterior.columns, axis=1) 97 | 98 | # Take the integral over the posterior to get the expectation of the distance to the 99 | # mean posterior (i.e., standard deviation) 100 | sd = np.trapz(np.abs(E[:, np.newaxis] - posterior.columns.astype(float).values[np.newaxis, :]) * posterior, posterior.columns, axis=1) 101 | 102 | stats = pd.DataFrame({'E':E, 'sd':sd}, index=posterior.index) 103 | return stats 104 | 105 | posterior_stats = get_posterior_stats(posterior) 106 | 107 | # Let's see how far the posterior mean is from the ground truth 108 | plt.errorbar(test_paradigm, posterior_stats['E'],posterior_stats['sd'], fmt='o',) 109 | plt.plot([0, 100], [0,100], c='k', ls='--') 110 | 111 | plt.xlabel('Ground truth') 112 | plt.ylabel('Mean posterior') 113 | 114 | # Let's see how the error depends on the standard deviation of the posterior 115 | error = test_paradigm - posterior_stats['E'] 116 | error_abs = np.abs(error) 117 | error_abs.name = 'error' 118 | 119 | sns.lmplot(x='sd', y='error', data=posterior_stats.join(error_abs)) 120 | 121 | plt.xlabel('Standard deviation of posterior') 122 | plt.ylabel('Objective error') 123 | 124 | # %% 125 | 126 | 127 | # Now, let's try to find the MAP estimate using gradient descent 128 | from braincoder.optimize import StimulusFitter 129 | stimulus_fitter = StimulusFitter(model=model, data=test_data, omega=omega) 130 | 131 | # We start with a very coarse grid search, so we are sure we are in the right ballpark 132 | estimated_stimuli_grid = stimulus_fitter.fit_grid(np.arange(1, 100, 5)) 133 | 134 | # %% 135 | 136 | 137 | # We can then refine the estimate using gradient descent 138 | estimated_stimuli_gd = stimulus_fitter.fit(init_pars=estimated_stimuli_grid, progressbar=False) 139 | 140 | # Let's see how well we did 141 | plt.scatter(test_paradigm, estimated_stimuli_grid, alpha=.5, label='MAP (grid search)') 142 | plt.scatter(test_paradigm, estimated_stimuli_gd, alpha=.5, label='MAP (gradient descent)') 143 | plt.scatter(test_paradigm, posterior_stats['E'], alpha=.5, label='Mean posterior') 144 | plt.plot([0, 100], [0,100], c='k', ls='--', label='Identity line') 145 | plt.legend() 146 | # %% -------------------------------------------------------------------------------- /examples/00_encodingdecoding/decode2d.py: -------------------------------------------------------------------------------- 1 | """ 2 | Decoding 2D stimuli from neural data 3 | ============================================= 4 | 5 | Here, we are decoding oriented stimuli from neural 6 | data that have an extra dimension: amplitude. 7 | 8 | Amplitude can be interpreted as the stimulus drive 9 | and has been shown to be modulated by cognitive effects 10 | such as expectation and attention. 11 | """ 12 | 13 | 14 | # Set up a neural model 15 | from braincoder.models import VonMisesPRF 16 | import numpy as np 17 | import pandas as pd 18 | import scipy.stats as ss 19 | noise = 0.5 20 | 21 | # We are setting up a VonMisesPRF model with 8 orientations, 22 | # We have 8 voxels, each with a linear combination of the 8 von Mises functions 23 | # We use the identity matrix with some noice, so that each voxel is driven by 24 | # largely by a single PRF 25 | parameters = pd.DataFrame({'mu':np.linspace(0, 2*np.pi, 8, False), 'kappa':1.0, 'amplitude':1.0, 'baseline':0.0}) 26 | 27 | weights = np.identity(8) * 5.0 28 | weights += np.random.rand(8, 8) 29 | model = VonMisesPRF(parameters=parameters, model_stimulus_amplitude=True, weights=weights) 30 | 31 | # Note how the stimulus type is now `OneDimensionalRadialStimulusWithAmplitude` 32 | # which means that the stimulus is two-dimensional 33 | print(model.stimulus) 34 | print(model.stimulus.dimension_labels) 35 | 36 | # %% 37 | 38 | # Now we can simulate some data and estimate parameters+noise 39 | mapper_paradigm = pd.DataFrame({'x (radians)':np.random.rand(100)*2*np.pi, 'amplitude':1.}) 40 | data = model.simulate(paradigm=mapper_paradigm, noise=noise) 41 | 42 | # Set up parameter fitter 43 | from braincoder.optimize import WeightFitter, ResidualFitter 44 | fitter = WeightFitter(model, parameters, data, mapper_paradigm) 45 | # With 8 overlapping Von Mises functions, we already need some regularisation, hence alpha=1.0 46 | fitted_weights = fitter.fit(alpha=1.0) 47 | 48 | # Now we fit the covariance matrix on the residuals 49 | resid_fitter = ResidualFitter(model, data, mapper_paradigm, parameters, fitted_weights) 50 | omega, dof = resid_fitter.fit(progressbar=False) 51 | 52 | # %% 53 | 54 | # Now we set up an experimental paradigm with two conditions 55 | # An `attended` and an `unattended` condition. 56 | # In the attended condition, the stimulus will have more drive (1.5), 57 | # in the unattended condition, the stimulus will have less drive (0.5). 58 | 59 | n = 200 60 | experimental_paradigm = pd.DataFrame(index=pd.MultiIndex.from_product([np.arange(n/2.), ['attended', 'unattended']], names=['frame', 'condition'])) 61 | 62 | # Random orientations 63 | experimental_paradigm['x (radians)'] = np.random.rand(n)*2*np.pi 64 | 65 | # Amplitudes have some noise 66 | experimental_paradigm['amplitude'] = np.where(experimental_paradigm.index.get_level_values('condition') == 'attended', ss.norm(1.5, 0.1).rvs(n), ss.norm(.5, 0.1).rvs(n)) 67 | experimental_data = model.simulate(paradigm=experimental_paradigm, noise=noise) 68 | 69 | 70 | # %% 71 | 72 | # Plot the data 73 | import seaborn as sns 74 | import matplotlib.pyplot as plt 75 | tmp = experimental_data.set_index(experimental_paradigm['x (radians)'], append=True).stack().to_frame('activity') 76 | g = sns.FacetGrid(tmp.reset_index(), col='unit', col_wrap=4, hue='condition', palette='coolwarm_r') 77 | g.map(plt.scatter, 'x (radians)', 'activity', alpha=0.85) 78 | g.add_legend() 79 | 80 | # %% 81 | 82 | # Now we can calculate the 2D likelihood/posterior of different orientations+amplitudes for the data 83 | lower_amplitude, higher_amplitude = 0.0, 4.5 84 | potential_amplitudes = np.linspace(lower_amplitude, higher_amplitude, 50) 85 | potential_orientations = np.linspace(0, 2*np.pi, 50, False) 86 | 87 | # Make sure ground truth is the potential stimuli 88 | potential_amplitudes = np.sort(np.append(potential_amplitudes, [0.5, 1.5])) 89 | 90 | # We use the `pd.MultiIndex.from_product` function to create a grid of possible stimuli 91 | potential_stimuli = pd.MultiIndex.from_product([potential_orientations, potential_amplitudes], names=['x (radians)', 'amplitude']).to_frame(index=False) 92 | 93 | # Now we get, for each data point, the likelihood of each possible stimulus 94 | ll = model.get_stimulus_pdf(experimental_data, potential_stimuli) 95 | 96 | # %% 97 | 98 | # Plot 2D posteriors for first 9 trials 99 | 100 | # Once we have these 2D likelihoods, now we want to be able to plot them. 101 | def plot_trial(key, ll=ll, paradigm=experimental_paradigm, xlabel=False, ylabel=False): 102 | # We use the `stack` method to turn the `amplitude` dimension into a column 103 | ll = ll.loc[key].unstack('amplitude') 104 | 105 | # Use imshow to show a 2D image of the likelihood 106 | vmin, vmax = ll.min().min(), ll.max().max() 107 | plt.imshow(ll, origin='lower', aspect='auto', extent=[lower_amplitude, higher_amplitude, 0, 2*np.pi], vmin=vmin, vmax=vmax) 108 | 109 | # Plot the _actual_ ground truth amplitude and orientation 110 | plt.scatter(paradigm.loc[key]['amplitude'], paradigm.loc[key]['x (radians)'], c='r', s=25, marker='+') 111 | 112 | # Some housekeeping for the subplots 113 | plt.title(f'Trial {key[0]}') 114 | if xlabel: 115 | plt.xticks() 116 | plt.xlabel('Amplitude') 117 | else: 118 | plt.xticks([]) 119 | 120 | if ylabel: 121 | plt.yticks() 122 | plt.ylabel('Orientation') 123 | else: 124 | plt.yticks([]) 125 | 126 | def plot_condition(condition): 127 | """ 128 | Plot the 2D posterior for a given condition for the first 9 trials. 129 | 130 | Parameters: 131 | condition (str): The condition for which to plot the posterior. 132 | 133 | Returns: 134 | None 135 | """ 136 | plt.figure(figsize=(7, 7)) 137 | for ix in range(9): 138 | plt.subplot(3, 3, ix+1) 139 | 140 | xlabel = ix in [6, 7, 8] 141 | ylabel = ix in [0, 3, 6] 142 | 143 | plot_trial((ix, condition), xlabel=xlabel, ylabel=ylabel) 144 | 145 | plt.suptitle(f'2D posterior ({condition} trials)') 146 | plt.tight_layout() 147 | 148 | plot_condition('attended') 149 | plot_condition('unattended') 150 | 151 | # %% 152 | 153 | # Now we can calculate the 1D posterior for specific orientations _or_ amplitudes 154 | 155 | # Marginalize out orientations 156 | amplitudes_posterior = ll.groupby('amplitude', axis=1).sum() 157 | amplitudes_posterior = amplitudes_posterior.div(np.trapz(amplitudes_posterior, amplitudes_posterior.columns, axis=1), axis=0) # This is the same as normalizing the posterior 158 | 159 | # Marginalize out amplitudes 160 | orientations_posterior = ll.groupby('x (radians)', axis=1).sum() 161 | orientations_posterior = orientations_posterior.div(np.trapz(orientations_posterior, orientations_posterior.columns, axis=1), axis=0) 162 | 163 | # Plot orientation posteriors 164 | tmp = orientations_posterior.stack().loc[:8].to_frame('p') 165 | 166 | g = sns.FacetGrid(tmp.reset_index(), col='frame', col_wrap=3, hue='condition', palette='coolwarm_r') 167 | g.map(plt.plot, 'x (radians)', 'p', alpha=0.85) 168 | 169 | g.map(plt.axvline, x=1.5, c=sns.color_palette('coolwarm_r', 2)[0], ls='--') 170 | g.map(plt.axvline, x=0.5, c=sns.color_palette('coolwarm_r', 2)[1], ls='--') 171 | 172 | 173 | # %% 174 | 175 | # Use the ground truth amplitude to improve the orientation posterior 176 | # so p(orientation|true_amplitude) 177 | conditional_orientation_ll = pd.concat((ll.stack().xs('attended', 0, 'condition').xs(1.5, 0, 'amplitude'), 178 | ll.stack().xs('unattended', 0, 'condition').xs(0.5, 0, 'amplitude')), 179 | axis=0, 180 | keys=['attended', 'unattended'], 181 | names=['condition']).swaplevel(0, 1).sort_index() 182 | 183 | # Normalize! 184 | conditional_orientation_ll = conditional_orientation_ll.div(np.trapz(conditional_orientation_ll, conditional_orientation_ll.columns, axis=1), axis=0) 185 | tmp = conditional_orientation_ll.stack().loc[:8].to_frame('p') 186 | 187 | g = sns.FacetGrid(tmp.reset_index(), col='frame', col_wrap=3, hue='condition', palette='coolwarm_r') 188 | g.map(plt.plot, 'x (radians)', 'p', alpha=0.85) 189 | 190 | g.map(plt.axvline, x=1.5, c=sns.color_palette('coolwarm_r', 2)[0], ls='--') 191 | g.map(plt.axvline, x=0.5, c=sns.color_palette('coolwarm_r', 2)[1], ls='--') 192 | 193 | # %% 194 | 195 | # Intro to complex numbers 196 | 197 | def to_complex(x): 198 | return np.exp(1j*x) 199 | 200 | def from_complex(x): 201 | x = np.angle(x) 202 | return np.where(x < 0, x + 2*np.pi, x) 203 | 204 | # Let's plot the firs 10 trials in the complex plane 205 | first_10_trials = experimental_paradigm.xs('attended', 0, 'condition')['x (radians)'].iloc[:10] 206 | 207 | orientations_complex = to_complex(first_10_trials.values) 208 | 209 | plt.figure() 210 | plt.scatter(orientations_complex.real, orientations_complex.imag, c=first_10_trials.index) 211 | plt.gca().set_aspect('equal') 212 | plt.xlabel('Real') 213 | plt.ylabel('Imaginary') 214 | sns.despine() 215 | 216 | # %% 217 | 218 | # Get posterior means by integrating over complex numbers 219 | def wrap_angle(x): 220 | return np.mod(x + np.pi, 2*np.pi) - np.pi 221 | 222 | def get_posterior_stats(posterior, ground_truth=None): 223 | posterior = posterior.copy() 224 | complex_grid = np.asarray(to_complex(posterior.columns)) 225 | 226 | # Take integral over the posterior to get to the expectation (mean posterior) 227 | # In this case a complex number that we convert back to an angle between 0 and 2pi 228 | E = from_complex(np.trapz(posterior*complex_grid[np.newaxis,:], axis=1)) 229 | 230 | # Take the integral over the posterior to get the expectation of the distance to the 231 | # mean posterior (i.e., standard deviation) 232 | relative_error = E[:, np.newaxis] - posterior.columns.values[np.newaxis,:] 233 | 234 | # Wrap the angle to be between 0 and pi, the error can never be larger than pi (180 degrees) 235 | relative_error = wrap_angle(relative_error) 236 | absolute_error = np.abs(relative_error) 237 | sd = np.trapz(absolute_error * posterior, posterior.columns, axis=1) 238 | 239 | stats = pd.DataFrame({'E':E, 'sd':sd}, index=posterior.index) 240 | 241 | if ground_truth is not None: 242 | stats['E_error'] = wrap_angle(stats['E'] - ground_truth) 243 | stats['E_error_abs'] = np.abs(stats['E_error']) 244 | stats['ground_truth'] = ground_truth 245 | 246 | return stats 247 | 248 | posterior_stats = get_posterior_stats(conditional_orientation_ll, ground_truth=experimental_paradigm['x (radians)'].values) 249 | 250 | # Circular correlations: 251 | import pingouin as pg 252 | posterior_stats.groupby('condition').apply(lambda d: pd.Series(pg.circ_corrcc(d['E'], d['ground_truth'], True), index=['rho', 'p'])) 253 | 254 | # %% 255 | 256 | # Let's see how far the posterior mean is from the ground truth 257 | # by plotting the estimates and groun truth in the complex plane 258 | palette = sns.color_palette('coolwarm_r', 2) 259 | 260 | # Create custom legend 261 | legend_elements = [ 262 | plt.Line2D([0], [0], marker='x', color='k', label='Estimate', markersize=8, linewidth=0), 263 | plt.Line2D([0], [0], marker='o', color='k', label='Truth', markersize=8, linewidth=0), 264 | plt.Line2D([0], [0], marker='s', color=palette[0], label='Attended', markersize=8, linewidth=0), 265 | plt.Line2D([0], [0], marker='s', color=palette[1], label='Unattended', markersize=8, linewidth=0) 266 | ] 267 | 268 | # Plot the data 269 | for ix, row in posterior_stats.iloc[:10].iterrows(): 270 | hue = sns.color_palette('coolwarm_r', 2)[['attended', 'unattended'].index(ix[1])] 271 | 272 | estimate_complex = to_complex(row['E']) 273 | ground_truth_complex = to_complex(row['ground_truth']) 274 | 275 | plt.plot([estimate_complex.real, ground_truth_complex.real], [estimate_complex.imag, ground_truth_complex.imag], color=hue) 276 | plt.scatter(estimate_complex.real, estimate_complex.imag, color=hue, s=50, marker='x') 277 | plt.scatter(ground_truth_complex.real, ground_truth_complex.imag, color=hue, s=50, marker='o') 278 | 279 | # Set aspect ratio and remove spines 280 | plt.gca().set_aspect('equal') 281 | sns.despine() 282 | 283 | # Add legend 284 | plt.legend(handles=legend_elements) 285 | 286 | # Show the plot 287 | plt.show() 288 | 289 | # %% 290 | 291 | # Plot the error as a function of the standard deviation of the posterior 292 | sns.lmplot(x='sd', y='E_error_abs', data=posterior_stats.reset_index(), hue='condition') 293 | 294 | # %% -------------------------------------------------------------------------------- /examples/00_encodingdecoding/decode_v1.py: -------------------------------------------------------------------------------- 1 | """ 2 | Recontruct a 2D visual stimulus from real fMRI data 3 | =================================================== 4 | Bla. 5 | Here we decode a 2D stimulus from te Szinte (2024)-dataset. 6 | 7 | """ 8 | 9 | import matplotlib.pyplot as plt 10 | from matplotlib.animation import FuncAnimation 11 | from IPython.display import HTML 12 | import seaborn as sns 13 | from braincoder.utils.data import load_szinte2024 14 | import numpy as np 15 | import pandas as pd 16 | 17 | # Load data (high-res for PRF fitting) 18 | data = load_szinte2024() 19 | stimulus = data['stimulus'] 20 | grid_coordinates = data['grid_coordinates'] 21 | tr = data['tr'] 22 | data = data['v1_timeseries'] 23 | 24 | from braincoder.models import GaussianPRF2DWithHRF 25 | from braincoder.hrf import SPMHRFModel 26 | 27 | model = GaussianPRF2DWithHRF(grid_coordinates=grid_coordinates, 28 | paradigm=stimulus, 29 | hrf_model=SPMHRFModel(tr=tr)) 30 | # %% 31 | 32 | """ 33 | Fit parameters 34 | -------------- 35 | """ 36 | # Fit PRF parameters (encoding model) 37 | from braincoder.optimize import ParameterFitter 38 | 39 | # We set up a parameter fitter 40 | par_fitter = ParameterFitter(model, data, stimulus) 41 | 42 | # We set up a grid of parameters to search over 43 | x = np.linspace(-8, 8, 20) 44 | y = np.linspace(-4, 4, 20) 45 | sd = np.linspace(1, 5, 10) 46 | 47 | # For now, we only use one amplitude and baseline, because we 48 | # use a correlation cost function, which is indifferent to 49 | # the overall scaling of the model 50 | # We can easily estimate these later using OLS 51 | amplitudes = [1.0] 52 | baseline = [0.0] 53 | 54 | # Note that the grids should be given in the correct order (can be found back in 55 | # model.parameter_labels) 56 | grid_pars = par_fitter.fit_grid(x, y, sd, baseline, amplitudes, use_correlation_cost=True) 57 | 58 | # Once we have the best parameters from the grid, we can optimize the baseline 59 | # and amplitude 60 | refined_grid_pars = par_fitter.refine_baseline_and_amplitude(grid_pars) 61 | 62 | # Now we use gradient descent to further optimize the parameters 63 | pars = par_fitter.fit(init_pars=refined_grid_pars, learning_rate=1e-2, max_n_iterations=5000, 64 | min_n_iterations=100, 65 | r2_atol=0.0001) 66 | 67 | from braincoder.utils import get_rsq 68 | fitted_r2 = get_rsq(data, model.predict(parameters=pars)) 69 | 70 | # %% 71 | 72 | """ 73 | Analyze PRF locations 74 | --------------------- 75 | """ 76 | # Let's plot the location of all these PRFs: 77 | sns.relplot(x='x', y='y', hue='r2', data=pars.join(fitted_r2.to_frame('r2')), size='sd', sizes=(10, 100), palette='viridis') 78 | plt.title('PRF locations') 79 | 80 | # Now we get the 250 best voxels with reasonable parameters 81 | best_voxels = fitted_r2.loc[pars['sd'] > 0.5].sort_values(ascending=False).iloc[:250].index 82 | 83 | plt.figure() 84 | sns.relplot(x='x', y='y', hue='r2', data=pars.loc[best_voxels].join(fitted_r2.to_frame('r2')), size='sd', sizes=(10, 100), palette='viridis') 85 | plt.title('PRF locations (best 250 voxels)') 86 | 87 | """ 88 | Fit noise model on residuals 89 | ---------------------------- 90 | """ 91 | from braincoder.optimize import ResidualFitter 92 | resid_fitter = ResidualFitter(model, data.loc[:, best_voxels], stimulus, parameters=pars.loc[best_voxels]) 93 | omega, dof = resid_fitter.fit() 94 | 95 | """ 96 | Reconstruct stimulus 97 | -------------------- 98 | 99 | For stimulus reconstruction, we slightly downsample the stimulus space 100 | otherwise the optimization takes too long on a CPU 101 | we can do that by simply setting up a new model with a different grid 102 | 103 | """ 104 | data = load_szinte2024(resize_factor=2.5) 105 | grid_coordinates = data['grid_coordinates'] 106 | stimulus = data['stimulus'] 107 | tr = data['tr'] 108 | data = data['v1_timeseries'] 109 | 110 | model = GaussianPRF2DWithHRF(grid_coordinates=grid_coordinates, 111 | parameters=pars.loc[best_voxels], 112 | hrf_model=SPMHRFModel(tr=tr)) 113 | 114 | # We set up a stimulus fitter 115 | from braincoder.optimize import StimulusFitter 116 | stim_fitter = StimulusFitter(data.loc[:, best_voxels], model, omega) 117 | 118 | # Legacy Adam is a bit faster than the default Adam optimizer on M1 119 | # Learning rate of 1.0 is a bit high, but works well here 120 | reconstructed_stimulus = stim_fitter.fit(legacy_adam=True, min_n_iterations=200, max_n_iterations=1000, learning_rate=.1) 121 | 122 | def play_reconstruction(reconstructed_stimulus): 123 | 124 | # Here we make a movie of the decoded stimulus 125 | # Set up a function to draw a single frame 126 | vmin, vmax = 0.0, np.quantile(reconstructed_stimulus.values.ravel(), 0.99) 127 | 128 | def update(frame): 129 | plt.clf() # Clear the current figure 130 | plt.imshow(reconstructed_stimulus.stack('y').loc[frame], cmap='viridis', vmin=vmin, vmax=vmax) 131 | plt.axis('off') 132 | plt.title(f"Frame {frame}") 133 | 134 | # Create the animation 135 | fig = plt.figure() 136 | ani = FuncAnimation(fig, update, frames=range(stimulus.shape[0]), interval=100) 137 | 138 | return HTML(ani.to_html5_video()) 139 | 140 | play_reconstruction(reconstructed_stimulus) 141 | 142 | # %% 143 | 144 | """ 145 | Reconstruct stimulus with L2-norm 146 | --------------------------------- 147 | Note how this reconstructed stimulus is very sparse and doesn't look a lot like 148 | the actual image. Part of the problem is that 149 | the optimisation is very unconstrained: we have 250 voxels times 150 | 150 (correlated!) datapoints, but ~800 time 150 stimulus pixels 151 | We can induce less extreme intensities, and thereby less 152 | sparseness, by inducing a L2-penalty on the stimulus intensities 153 | """ 154 | 155 | reconstructed_stimulus = stim_fitter.fit(legacy_adam=True, min_n_iterations=200, max_n_iterations=1000, learning_rate=0.1, l2_norm=0.01) 156 | 157 | play_reconstruction(reconstructed_stimulus) 158 | 159 | # %% 160 | 161 | 162 | """ 163 | Reconstruct stimulus with L1-norm 164 | --------------------------------- 165 | For completeness, one can also use a sparse-inducing L1-norm 166 | """ 167 | reconstructed_stimulus = stim_fitter.fit(legacy_adam=True, min_n_iterations=200, max_n_iterations=1000, learning_rate=.1, l1_norm=0.01) 168 | play_reconstruction(reconstructed_stimulus) -------------------------------------------------------------------------------- /examples/00_encodingdecoding/decode_visual.py: -------------------------------------------------------------------------------- 1 | """ 2 | Fit a 2D PRF model 3 | ============================================= 4 | 5 | Here we fit a 2D PRF model to data from the Szinte (2024)-dataset. 6 | 7 | """ 8 | 9 | import matplotlib.pyplot as plt 10 | from matplotlib.animation import FuncAnimation 11 | from IPython.display import HTML, display 12 | 13 | from braincoder.utils.data import load_szinte2024 14 | import numpy as np 15 | import pandas as pd 16 | 17 | # Load data 18 | data = load_szinte2024() 19 | stimulus = data['stimulus'] 20 | grid_coordinates = data['grid_coordinates'] 21 | 22 | # Set up a function to draw a single frame 23 | def update(frame): 24 | plt.clf() # Clear the current figure 25 | plt.imshow(stimulus[frame, :, :].T, cmap='viridis') 26 | plt.title(f"Frame {frame}") 27 | 28 | # Create the animation 29 | fig = plt.figure() 30 | ani = FuncAnimation(fig, update, frames=range(stimulus.shape[0]), interval=100) 31 | 32 | # Convert to HTML for easy display 33 | HTML(ani.to_html5_video()) 34 | 35 | # %% 36 | 37 | # Set up a PRF model 38 | # Now we will set up fake PRFs just to show how the data is structured 39 | # We make a 9-by-9 grid of simulated PRFs 40 | x, y = np.meshgrid(np.linspace(-6, 6, 3), np.linspace(-4, 4, 3)) 41 | 42 | # Set them up in a parameter table 43 | # All PRFs have the same baseline and amplitude 44 | from braincoder.models import GaussianPRF2DWithHRF 45 | from braincoder.hrf import SPMHRFModel 46 | 47 | parameters = pd.DataFrame({'x':x.ravel(), 48 | 'y':y.ravel(), 49 | 'sd':2.5, 50 | 'baseline':0.0, 51 | 'amplitude':1.}).astype(np.float32) 52 | model = GaussianPRF2DWithHRF(grid_coordinates=grid_coordinates, 53 | paradigm=stimulus, 54 | parameters=parameters, 55 | hrf_model=SPMHRFModel(tr=data['tr'])) 56 | # %% 57 | 58 | # Let's plot all the RFs 59 | rfs = model.get_rf(as_frame=True) 60 | 61 | for i, rf in rfs.groupby(level=0): 62 | plt.subplot(3, 3, i+1) 63 | plt.title(f'RF {i+1}') 64 | plt.imshow(rf.unstack('y').loc[i].T) 65 | plt.axis('off') 66 | 67 | # %% 68 | 69 | # We simulate data for the given paradigm and parameters and plot the resulting time series 70 | import seaborn as sns 71 | data = model.simulate(noise=1.) 72 | data.columns.set_names('voxel', inplace=True) 73 | tmp = data.stack().to_frame('activity') 74 | sns.relplot(x='frame', y='activity', data=tmp.reset_index(), hue='voxel', kind='line', palette=sns.color_palette('tab10', n_colors=parameters.shape[0]), aspect=2.) 75 | 76 | # %% 77 | 78 | # We can also fit parameters back to data 79 | from braincoder.optimize import ParameterFitter 80 | 81 | # We set up a parameter fitter 82 | par_fitter = ParameterFitter(model, data, stimulus) 83 | 84 | # We set up a grid of parameters to search over 85 | x = np.linspace(-8, 8, 20) 86 | y = np.linspace(-4, 4, 20) 87 | sd = np.linspace(1, 5, 10) 88 | 89 | # For now, we only use one amplitude and baseline, because we 90 | # use a correlation cost function, which is indifferent to 91 | # the overall scaling of the model 92 | # We can easily estimate these later using OLS 93 | amplitudes = [1.0] 94 | baseline = [0.0] 95 | 96 | # Note that the grids should be given in the correct order (can be found back in 97 | # model.parameter_labels) 98 | grid_pars = par_fitter.fit_grid(x, y, sd, baseline, amplitudes, use_correlation_cost=True) 99 | 100 | # Once we have the best parameters from the grid, we can optimize the baseline 101 | # and amplitude 102 | refined_grid_pars = par_fitter.refine_baseline_and_amplitude(grid_pars) 103 | 104 | # We get the explained variance of these parameters 105 | from braincoder.utils import get_rsq 106 | refined_grid_r2 = get_rsq(data, model.predict(parameters=refined_grid_pars)) 107 | 108 | # Now we use gradient descent to further optimize the parameters 109 | pars = par_fitter.fit(init_pars=refined_grid_pars, learning_rate=1e-2, max_n_iterations=5000, 110 | min_n_iterations=100, 111 | r2_atol=0.0001) 112 | 113 | fitted_r2 = get_rsq(data, model.predict(parameters=pars)) 114 | 115 | # The fitted R2s tend to be a bit better than the grid R2s 116 | display(refined_grid_r2.to_frame('r2').join(fitted_r2.to_frame('r2'), lsuffix='_grid', rsuffix='_fitted')) 117 | 118 | # The real parameters are very similar to the estimated parameters 119 | display(pars.join(parameters, lsuffix='_fit', rsuffix='_true')) 120 | 121 | # %% 122 | 123 | # Decode the *stimulus* from "unseen" data: 124 | # First we need to fit a noise model 125 | from braincoder.optimize import ResidualFitter 126 | resid_fitter = ResidualFitter(model, data, stimulus, parameters=pars) 127 | omega, dof = resid_fitter.fit() 128 | 129 | # Simulate new "unseen" data 130 | unseen_data = model.simulate(noise=1.) 131 | 132 | # For stimulus reconstruction, we slightly downsample the stimulus space 133 | # otherwise the optimization takes too long on a CPU 134 | # we can do that by simply setting up a new model with a different grid 135 | data = load_szinte2024(resize_factor=2.5) 136 | grid_coordinates = data['grid_coordinates'] 137 | stimulus = data['stimulus'] 138 | 139 | model = GaussianPRF2DWithHRF(grid_coordinates=grid_coordinates, 140 | parameters=parameters, 141 | hrf_model=SPMHRFModel(tr=data['tr'])) 142 | 143 | # We set up a stimulus fitter 144 | from braincoder.optimize import StimulusFitter 145 | stim_fitter = StimulusFitter(unseen_data, model, omega) 146 | 147 | # Legacy Adam is a bit faster than the default Adam optimizer on M1 148 | # Learning rate of 1.0 is a bit high, but works well here 149 | reconstructed_stimulus = stim_fitter.fit(legacy_adam=True, min_n_iterations=200, max_n_iterations=200, learning_rate=.1) 150 | 151 | # Here we make a movie of the decoded stimulus 152 | # Set up a function to draw a single frame 153 | vmin, vmax = 0.0, np.quantile(reconstructed_stimulus.values.ravel(), 0.95) 154 | 155 | def update(frame): 156 | plt.clf() # Clear the current figure 157 | plt.imshow(reconstructed_stimulus.stack('y').loc[frame], cmap='viridis', vmin=vmin, vmax=vmax) 158 | plt.axis('off') 159 | plt.title(f"Frame {frame}") 160 | 161 | # Create the animation 162 | fig = plt.figure() 163 | ani = FuncAnimation(fig, update, frames=range(stimulus.shape[0]), interval=100) 164 | 165 | HTML(ani.to_html5_video()) -------------------------------------------------------------------------------- /examples/00_encodingdecoding/encoding_model.py: -------------------------------------------------------------------------------- 1 | """ 2 | Create a simple Gaussian Prf encoding model 3 | ============================================= 4 | 5 | In this example, we create a Gaussian PRF model and plot the predictions. 6 | We also simulate data and then estimate back the generating parameters. 7 | 8 | """ 9 | 10 | # Import necessary libraries 11 | from braincoder.models import GaussianPRF 12 | import pandas as pd 13 | import numpy as np 14 | 15 | # We set up two PRFS, one centered at 1 and one centered at -2 16 | # The first one has a sd of 1 and the second one has a sd of 1.5 17 | parameters = [{'mu':1.0, 'sd':1.0, 'amplitude':1.0, 'baseline':0.0}, 18 | {'mu':-2., 'sd':1.5, 'amplitude':1.0, 'baseline':0.0} 19 | ] 20 | parameters = pd.DataFrame(parameters, index=['voxel 1', 'voxel 2']) 21 | 22 | # We have a virtual experimental paradigm where we go from -5 to 5 23 | paradigm = np.linspace(-5, 5, 100) 24 | 25 | # Set up the model. 26 | model = GaussianPRF(paradigm=paradigm, parameters=parameters) 27 | 28 | # Extract and plot the predictions 29 | predictions = model.predict() 30 | predictions.index = pd.Index(paradigm, name='Stimulus value') 31 | ax = predictions.plot() 32 | 33 | # %% 34 | 35 | # We simulate data with a bit of noise 36 | data = model.simulate(noise=0.2) 37 | data.plot() 38 | # %% 39 | 40 | # Import and set up a parameter fitter with the (simulated) data, 41 | # the paradigm, and the model 42 | from braincoder.optimize import ParameterFitter 43 | from braincoder.utils import get_rsq 44 | 45 | optimizer = ParameterFitter(model, data=data, paradigm=paradigm) 46 | 47 | # Set up a grid search over the parameters 48 | possible_mus = np.linspace(-5, 5, 10) 49 | possible_sds = np.linspace(0.1, 5, 10) 50 | 51 | # For the grid search we use a correlation cost function, so we can fit 52 | # the amplitude an baseline later using OLS 53 | possible_amplitudes = [1.0] 54 | possible_baselines = [0.0] 55 | 56 | # Fit the grid 57 | grid_pars = optimizer.fit_grid(possible_mus, possible_sds, possible_amplitudes, possible_baselines, use_correlation_cost=True, progressbar=False) 58 | 59 | # Show the results 60 | grid_pars 61 | # %% 62 | 63 | # We can now fit the amplitude and baseline using OLS 64 | grid_pars = optimizer.refine_baseline_and_amplitude(grid_pars) 65 | 66 | # Show the fitted timeseries 67 | import matplotlib.pyplot as plt 68 | import seaborn as sns 69 | palette = sns.color_palette() 70 | grid_pred = model.predict(parameters=grid_pars) 71 | 72 | # See how well the predictions align with the data 73 | # using the explained variance statistic 74 | r2_grid = get_rsq(data, grid_pred) 75 | 76 | plt.plot(paradigm, data) 77 | plt.plot(paradigm, grid_pred, ls='--', c='k', label='Grid prediction') 78 | plt.title(f'R2 = {r2_grid.mean():.2f}') 79 | 80 | # %% 81 | 82 | # Final optimisation using gradient descent: 83 | gd_pars = optimizer.fit(init_pars=grid_pars, progressbar=False) 84 | gd_pred = model.predict(parameters=gd_pars) 85 | 86 | r2_gd = get_rsq(data, gd_pred) 87 | 88 | plt.plot(paradigm, data) 89 | plt.plot(paradigm, grid_pred, ls='--', c='k', alpha=0.5, label='Grid prediction') 90 | plt.plot(paradigm, gd_pred, ls='--', c='k', label='Gradient descent prediction') 91 | plt.title(f'R2 = {r2_gd.mean():.2f}') 92 | 93 | # %% 94 | -------------------------------------------------------------------------------- /examples/00_encodingdecoding/fisher_information.py: -------------------------------------------------------------------------------- 1 | """ 2 | Fisher information to estimate precision of encoding parameters across stimulus space 3 | ===================================================================================== 4 | 5 | Fisher information is defined as the expected value of the square of the derivative of the log-likelihood 6 | with respect to some parameter. 7 | 8 | 9 | .. math:: 10 | I(\\theta) = E[ (\\frac{\\partial}{\\partial \\theta} \\log f(X; \\theta) )^2 ] 11 | 12 | In our case, the parameter $\\theta$ might be the stimulus (dimension) we are interested in. What the Fisher information 13 | now gives us, \_given a fitted encoding *and* noise model\_, is the expected precision of the estimated 14 | parameter across the stimulus space. 15 | 16 | In other words, using the Fisher information, we can already estimate how well we can estimate the stimulus 17 | at different locations in stimulus space without actually doing any decoding. 18 | 19 | Notably, Fisher information also plays a crucial role in efficient coding theory. In general, the expected 20 | squared error of a likelihood function, given a limited 'information budget', 21 | can be minimized by using a likelihood function where the Fisher information is proportional 22 | to the derivative of the cummulative prior distribution (:cite:t:`wei2015bayesian`). 23 | 24 | """ 25 | 26 | # Import necessary libraries 27 | from braincoder.models import GaussianPRF 28 | import pandas as pd 29 | import numpy as np 30 | import scipy.stats as ss 31 | import seaborn as sns 32 | 33 | 34 | # %% 35 | # Set up generating model 36 | # ------------------------------------- 37 | # Let's set up a Gaussian PRF on the line that generates some data. 38 | # We put the centers of the PRFs equally across the line, using a 39 | # the quantiles of a *uniform prior distribution* 40 | # (i.e. the quantiles of a uniform distribution are equally spaced). 41 | # % 42 | 43 | # 8 points on the uniform distribution between 0 and 10 44 | mus = ss.uniform(0, 10.).ppf(np.linspace(0.0, 1.0, 10)[1:-1]) 45 | parameters = pd.DataFrame({'mu':mus, 'sd':1., 'baseline':0.0, 'amplitude':1.0})[['mu', 'sd', 'amplitude', 'baseline']] 46 | 47 | # Let's go over the entire line from 0 to 10 48 | paradigm = pd.Series(np.linspace(-2, 12, 100, dtype=np.float32)) 49 | 50 | # For now let's assume the noise is spherical 51 | omega = np.identity(len(parameters)).astype(np.float32) 52 | 53 | model = GaussianPRF(parameters=parameters, paradigm=paradigm) 54 | data = model.predict(paradigm=paradigm).set_index(pd.Index(paradigm, name='stimulus')) 55 | data = data.stack().to_frame('strength') 56 | data.index.set_names('rf', level=1, inplace=True) 57 | 58 | sns.lineplot(data=data.reset_index(), x='stimulus', y='strength', hue='rf', palette=['k'], alpha=0.5, legend=False) 59 | sns.despine() 60 | 61 | 62 | # %% 63 | # Calculate Fisher Information 64 | # ------------------------------ 65 | # Now that we have the generating model, we can calculate the Fisher information 66 | # for each stimulus point. We can do this by taking the expectation of the 67 | # second derivative of the log-likelihood with respect to the stimulus. 68 | # the fucntion `Model.get_fisher_information` does this for us by sampling 69 | # from the noise distribution defined by `omega` and calculating the second 70 | # derivative of the log-likelihood with respect to the stimulus using autodiff. 71 | # % 72 | 73 | fisher_information = model.get_fisher_information(stimuli=np.linspace(-5, 15, 1000).astype(np.float32), omega=omega, dof=None, n=5000).to_frame() 74 | 75 | sns.lineplot(x='stimulus', y='Fisher information', data=fisher_information.reset_index(), color='k') 76 | sns.despine() 77 | 78 | 79 | # %% 80 | # We can see that the Fisher information is largest at the center of the PRFs and 81 | # decreases towards the edges. This is because the PRFs are more overlapping at the 82 | # center and less overlapping at the edges. This is also reflected in the Fisher 83 | # information, which is a measure of the expected precision of the estimated stimulus 84 | # at each point in stimulus space. 85 | # Outside of the receptive fields, the Fisher information is 0: when stimuli occur 86 | # outside of the receptive fields, the likelihood function is flat and the Fisher 87 | # information is 0. 88 | # %% 89 | 90 | # %% 91 | # Note that if we include some correlation in the errors 92 | # of the receptive fields, under some conditions, 93 | # the Fisher information is higher. 94 | 95 | omega_correlated = omega + .5 - .5*np.identity(len(parameters)) 96 | 97 | fisher_information_uncorrelated = model.get_fisher_information(stimuli=np.linspace(-5, 15, 1000).astype(np.float32), omega=omega.astype(np.float32), dof=None, n=5000).to_frame() 98 | fisher_information_correlated = model.get_fisher_information(stimuli=np.linspace(-5, 15, 1000).astype(np.float32), omega=omega_correlated.astype(np.float32), dof=None, n=5000).to_frame() 99 | 100 | fisher_information = pd.concat((fisher_information_uncorrelated, fisher_information_correlated), axis=0, keys=['uncorrelated', 'correlated'], names=['correlation']) 101 | 102 | sns.lineplot(x='stimulus', y='Fisher information', data=fisher_information.reset_index(), color='k', style='correlation') 103 | sns.despine() 104 | 105 | # %% 106 | # We can also validate the Fisher information by decoding and looking 107 | # at the errors of the decoded stimuli. The Fisher information should 108 | # be inversely proportional to the expected squared error of the decoded 109 | # stimuli. 110 | # %% 111 | 112 | import matplotlib.pyplot as plt 113 | 114 | paradigm = np.repeat(np.linspace(-2, 12, 50, dtype=np.float32), 100) 115 | simulated_data = model.simulate(paradigm=paradigm, noise=omega) 116 | omega = np.identity(len(parameters)).astype(np.float32) 117 | 118 | 119 | 120 | pdf = model.get_stimulus_pdf(simulated_data, np.linspace(-5, 15, 100), omega=omega) 121 | E = np.trapz(pdf*pdf.columns.values[np.newaxis, :], x=pdf.columns, axis=1) 122 | error = np.sqrt((paradigm - E)**2) 123 | error = pd.Series(error, index=pd.Index(paradigm, name='stimulus')).to_frame('error') 124 | sns.lineplot(x='stimulus', y='error', data=error) 125 | sns.despine() 126 | plt.title('Objective decoding error') 127 | 128 | 129 | 130 | # %% 131 | # same goes for the variance of the decoded posterior 132 | posterior_variance = np.trapz(pdf*(pdf.columns.values[np.newaxis, :] - E[:, np.newaxis])**2, x=pdf.columns, axis=1) 133 | 134 | error['posterior sd'] = np.sqrt(posterior_variance) 135 | plt.figure() 136 | sns.lineplot(x='stimulus', y='posterior sd', data=error.reset_index()) 137 | sns.despine() 138 | plt.title('Posterior std.') 139 | 140 | # %% 141 | # Fisher information for different RF distributions 142 | # ------------------------------------------------- 143 | # Let's calculate the Fisher information for different RF structures 144 | # % 145 | 146 | # Centered distribution around 5. 147 | mus1 = ss.norm(5, 1.).ppf(np.linspace(1e-4, 1-1e-4, 10)[1:-1]) 148 | 149 | # 8 points on the uniform distribution between 0 and 10 150 | mus2 = ss.uniform(0, 10).ppf(np.linspace(1e-4, 1-1e-4, 10)[1:-1]) 151 | parameters_sets = [{'mu':mus1, 'sd':1.5, 'baseline':0.0, 'amplitude':1.0}, 152 | {'mu':mus2, 'sd':1.5, 'baseline':0.0, 'amplitude':1.0}, 153 | {'mu':mus2, 'sd':.75, 'baseline':0.0, 'amplitude':1.0}, 154 | {'mu':mus2, 'sd':1.5, 'baseline':0.0, 'amplitude':.5}] 155 | 156 | parameter_sets = pd.concat([pd.DataFrame(p)[['mu', 'sd', 'amplitude', 'baseline']] for p in parameters_sets], keys=range(4), names=['set', 'parameter'], axis=0) 157 | 158 | 159 | set_labels = ['Unequally distributed mus', 'Equally distributed mus', 'Smaller dispersion', 'Smaller amplitude (vs noise)' ] 160 | 161 | 162 | # Make predictions 163 | preds = [] 164 | for set, pars in parameter_sets.groupby('set'): 165 | model = GaussianPRF(parameters=pars.droplevel(0), paradigm=paradigm) 166 | pred = model.predict(paradigm=paradigm).set_index(pd.Index(paradigm, name='stimulus')) 167 | preds.append(pred) 168 | 169 | preds = pd.concat(preds, keys=set_labels, axis=0, names=['set']) 170 | 171 | # Calculate Fisher information 172 | fis = [] 173 | 174 | stimuli = np.linspace(-5, 15, 200, dtype=np.float32) 175 | for set, pars in parameter_sets.groupby('set'): 176 | model = GaussianPRF(parameters=pars.droplevel(0), paradigm=paradigm) 177 | omega = np.identity(len(pars)).astype(np.float32) 178 | fi = model.get_fisher_information(stimuli=stimuli, omega=omega, n=10000).to_frame("fisher information").set_index(pd.Index(stimuli, name='stimulus')) 179 | fis.append(fi) 180 | 181 | fis = pd.concat(fis, keys=set_labels, axis=0, names=['set']) 182 | 183 | 184 | # Plot the receptive fields and the Fisher information 185 | tmp = preds.stack().to_frame('strength').join(fis) 186 | g = sns.FacetGrid(col='set', col_wrap=2, data=tmp.reset_index(), sharey=True, col_order=set_labels) 187 | g.map_dataframe(sns.lineplot, 'stimulus', 'strength', hue='parameter', palette=['k'], alpha=0.5) 188 | g.set_titles('{col_name}') 189 | g.figure.suptitle('Receptive field distributions') 190 | g.figure.subplots_adjust(top=.9) 191 | 192 | g2 = sns.FacetGrid(col='set', col_wrap=2, data=fis.reset_index(), sharey=True, col_order=set_labels) 193 | g2.map(sns.lineplot, 'stimulus', 'fisher information', color='r') 194 | g2.add_legend() 195 | g2.set_titles('{col_name}') 196 | g2.figure.suptitle('Fisher information') 197 | g2.figure.subplots_adjust(top=.9) 198 | 199 | g3 = sns.FacetGrid(hue='set', data=fis.reset_index(), sharey=True) 200 | g3.map(sns.lineplot, 'stimulus', 'fisher information') 201 | g3.add_legend() 202 | 203 | 204 | # %% 205 | # Another intersting phenomenon we can now study is how different 206 | # basis functions/receptive fields modulate the Fisher information. 207 | # For example, it is well-known that numerical receptive fields in 208 | # parietal cortex are shaped as a log-normal distribution. We can 209 | # now study how the Fisher information is modulated by the shape of 210 | # the receptive fields. 211 | # 212 | 213 | from braincoder.models import LogGaussianPRF 214 | mus = np.linspace(5, 25, 8) 215 | 216 | parameters = pd.DataFrame({'mu':mus, 'sd':25., 'baseline':0.0, 'amplitude':1.0})[['mu', 'sd', 'amplitude', 'baseline']] 217 | 218 | # Set up model and paradigm and plot the receptive fields 219 | paradigm = np.linspace(0, 100, 100, dtype=np.float32) 220 | model = LogGaussianPRF(parameters=parameters, paradigm=paradigm) 221 | y = model.predict(paradigm=paradigm) 222 | 223 | y.plot(c='k', legend=False, alpha=0.5) 224 | sns.despine() 225 | plt.title('Receptive fields') 226 | 227 | # Calculate Fisher information 228 | omega = np.identity(len(parameters)).astype(np.float32) 229 | fisher_information = model.get_fisher_information(stimuli=np.linspace(5, 100, 1000).astype(np.float32), omega=omega, n=5000).to_frame() 230 | 231 | fisher_information['sqrt(fi)'] = np.sqrt(fisher_information['Fisher information']) 232 | 233 | plt.figure() 234 | sns.lineplot(x='stimulus', y='sqrt(fi)', data=fisher_information.reset_index(), color='k') 235 | plt.title('Square root of Fisher information') 236 | 237 | 238 | # %% 239 | # Somewhat consistent with the literature, the Fisher information 240 | # behaves roughly like 1/x now, where x is the stimulus. This means that the 241 | # precision of the estimated stimulus decreases with increasing 242 | # stimulus value. 243 | # 244 | 245 | # % 246 | # Another interesting corner case is a single receptive field with a log-normal 247 | # kernel. 248 | from braincoder.models import LogGaussianPRF 249 | mus = [25.] 250 | 251 | parameters = pd.DataFrame({'mu':mus, 'sd':25., 'baseline':0.0, 'amplitude':1.0})[['mu', 'sd', 'amplitude', 'baseline']] 252 | 253 | # Set up model and paradigm and plot the receptive fields 254 | paradigm = np.linspace(0, 100, 100, dtype=np.float32) 255 | model = LogGaussianPRF(parameters=parameters, paradigm=paradigm) 256 | y = model.predict(paradigm=paradigm) 257 | 258 | y.plot(c='k', legend=False) 259 | 260 | # Calculate Fisher information 261 | omega = np.identity(len(parameters)).astype(np.float32) 262 | fisher_information = model.get_fisher_information(stimuli=np.linspace(5, 100, 1000).astype(np.float32), omega=omega, n=5000).to_frame() 263 | 264 | fisher_information['sqrt(fi)'] = np.sqrt(fisher_information['Fisher information']) 265 | 266 | plt.figure() 267 | sns.lineplot(x='stimulus', y='sqrt(fi)', data=fisher_information.reset_index(), color='k') -------------------------------------------------------------------------------- /examples/00_encodingdecoding/fit_prf.py: -------------------------------------------------------------------------------- 1 | """ 2 | Different flavors of visual population receptive field models 3 | ============================================================== 4 | 5 | In this example script we will try out increasingly complex models for 6 | visual population receptive fields (PRFs). We will start with a simple 7 | Gaussian PRF model, and then add more complexity step by step. 8 | 9 | """ 10 | 11 | # %% 12 | # Load data 13 | # --------- 14 | # First we load in the data. We will use the Szinte (2024)-dataset. 15 | from braincoder.utils.data import load_szinte2024 16 | import matplotlib.pyplot as plt 17 | 18 | data = load_szinte2024() 19 | 20 | # This is the visual stimulus ("design matrix") 21 | paradigm = data['stimulus'] 22 | grid_coordinates = data['grid_coordinates'] 23 | 24 | # This is the fMRI response data 25 | d = data['v1_timeseries'] 26 | d.index.name = 'frame' 27 | tr = data['tr'] 28 | 29 | 30 | # %% 31 | # Simple 2D Gaussian Recetive Field model 32 | # ------------------------------------- 33 | # Now we set up a simple Gaussian PRF model 34 | from braincoder.models import GaussianPRF2DWithHRF 35 | from braincoder.hrf import SPMHRFModel 36 | hrf_model = SPMHRFModel(tr=tr) 37 | model_gauss = GaussianPRF2DWithHRF(data=d, paradigm=paradigm, hrf_model=hrf_model, grid_coordinates=grid_coordinates) 38 | 39 | # %% 40 | # And a parameter fitter... 41 | from braincoder.optimize import ParameterFitter 42 | par_fitter = ParameterFitter(model=model_gauss, data=d, paradigm=paradigm) 43 | 44 | 45 | # %% 46 | # Now we try out a relatively coarse grid search to find the some 47 | # parameters to start the gradient descent from. 48 | 49 | import numpy as np 50 | x = np.linspace(-8, 8, 10) 51 | y = np.linspace(-4, 4, 10) 52 | sd = np.linspace(0.1, 4, 10) 53 | 54 | # We start the grid search using a correlation cost, so ampltiude 55 | # and baseline do not influence those results. 56 | # We will optimize them later using OLS. 57 | baseline = [0.0] 58 | amplitude = [1.0] 59 | 60 | # Now we can do the grid search 61 | pars_gauss_grid = par_fitter.fit_grid(x, y, sd, baseline, amplitude, correlation_cost=True) 62 | 63 | # And refine the baseline and amplitude parameters using OLS 64 | pars_gauss_ols = par_fitter.refine_baseline_and_amplitude(pars_gauss_grid) 65 | 66 | 67 | # %% 68 | # Here we can plot the resulting r2s of the grid search 69 | r2_gauss_ols = par_fitter.get_rsq(pars_gauss_ols) 70 | 71 | import seaborn as sns 72 | sns.kdeplot(r2_gauss_ols, shade=True) 73 | sns.despine() 74 | # %% 75 | 76 | # %% 77 | # We can substantially improve the fit by using gradient descent optimisation 78 | pars_gauss_gd = par_fitter.fit(init_pars=pars_gauss_ols, max_n_iterations=1000) 79 | 80 | # %% 81 | r2_gauss_gd = par_fitter.get_rsq(pars_gauss_gd) 82 | sns.kdeplot(r2_gauss_gd, shade=True) 83 | 84 | import pandas as pd 85 | r2 = pd.concat((r2_gauss_ols, r2_gauss_gd), keys=['r2_ols', 'r2_gd'], axis=1) 86 | 87 | # %% 88 | # Clearly, the gradient descent optimization improves the fit substantially. 89 | sns.relplot(x='r2_ols', y='r2_gd', data=r2.reset_index(), kind='scatter') 90 | plt.plot([0, 1], [0, 1], 'k--') 91 | # 92 | # %% 93 | 94 | # %% 95 | # Fit HRFs 96 | # -------- 97 | # The standard canonical (SPM) HRF we use is often not a great fit to actual 98 | # data. To better account for the HRF. We can optimize the HRFs per voxel. 99 | # We first initialize a GaussianPRF-model with a flexible HRF. 100 | model_hrf = GaussianPRF2DWithHRF(data=d, paradigm=paradigm, hrf_model=hrf_model, 101 | grid_coordinates=grid_coordinates, flexible_hrf_parameters=True) 102 | 103 | par_fitter_hrf = ParameterFitter(model=model_hrf, data=d, paradigm=paradigm) 104 | 105 | # We set hrf_delay and hrf_dispersion to standard values 106 | pars_gauss_gd['hrf_delay'] = 6 107 | pars_gauss_gd['hrf_dispersion'] = 1 108 | 109 | pars_gauss_hrf = par_fitter_hrf.fit(init_pars=pars_gauss_gd, max_n_iterations=1000) 110 | 111 | # %% 112 | r2_gauss_hrf = par_fitter_hrf.get_rsq(pars_gauss_hrf) 113 | 114 | r2 = pd.concat((r2_gauss_gd, r2_gauss_hrf), keys=['r2_gd', 'r2_hrf'], axis=1) 115 | sns.relplot(x='r2_gd', y='r2_hrf', data=r2.reset_index(), kind='scatter') 116 | plt.plot([0, 1], [0, 1], 'k--') 117 | 118 | # %% 119 | # Here we plot the predicted time courses of the original model 120 | # and the model with the optimized HRFs for 9 voxels where the fit 121 | # improved the most. You can clearly see that, in general, the 122 | # HRFs have shorter delays than the default setting. 123 | improvement = r2_gauss_hrf - r2_gauss_gd 124 | largest_improvements = improvement.sort_values(ascending=False).index[:9] 125 | pred_gauss_gd = model_gauss.predict(parameters=pars_gauss_gd) 126 | pred_gauss_hrf = model_hrf.predict(parameters=pars_gauss_hrf) 127 | pred = pd.concat((d.loc[:, largest_improvements], pred_gauss_gd.loc[:, largest_improvements], pred_gauss_hrf.loc[:, largest_improvements]), axis=1, keys=['data', 'gauss', 'gauss+hrf'], names=['model']) 128 | 129 | # 130 | tmp = pred.stack(['model', 'source']).to_frame('value') 131 | sns.relplot(x='frame', y='value', hue='model', col='source', data=tmp.reset_index(), kind='line', col_wrap=3) 132 | 133 | 134 | # %% 135 | 136 | # %% 137 | # Fit a Difference of Gaussians model 138 | # ----------------------------------- 139 | # Now we will try to fit a Difference of Gaussians model. This model 140 | # has two Gaussian receptive fields, one excitatory and one inhibitory. 141 | # The inhibitory receptive field is subtracted from the excitatory one. 142 | # The resulting receptive field is then convolved with the HRF. 143 | from braincoder.models import DifferenceOfGaussiansPRF2DWithHRF 144 | model_dog = DifferenceOfGaussiansPRF2DWithHRF(data=d, paradigm=paradigm, hrf_model=hrf_model, 145 | grid_coordinates=grid_coordinates, flexible_hrf_parameters=True) 146 | 147 | pars_dog_init = pars_gauss_hrf.copy() 148 | # This is the relative amplitude of the inhibitory receptive field 149 | # compared to the excitatory one. 150 | pars_dog_init['srf_amplitude'] = 0.1 151 | 152 | # This is the relative size of the inhibitory receptive field 153 | # compared to the excitatory one. 154 | pars_dog_init['srf_size'] = 2. 155 | 156 | # Let's set up a new parameterfitter 157 | par_fitter_dog = ParameterFitter(model=model_dog, data=d, paradigm=paradigm) 158 | 159 | # Note how, for now, we are not optimizing the HRF parameters. 160 | pars_dog = par_fitter_dog.fit(init_pars=pars_dog_init, max_n_iterations=1000, 161 | fixed_pars=['hrf_delay', 'hrf_dispersion']) 162 | 163 | # Now we optimize _with_ the HRF parameters 164 | pars_dog_hrf = par_fitter_dog.fit(init_pars=pars_dog, max_n_iterations=1000) 165 | 166 | r2_dog_hrf = par_fitter_dog.get_rsq(pars_dog_hrf) 167 | 168 | sns.relplot(x='r2_hrf', y='r2_dog_hrf', data=pd.concat((r2_gauss_hrf, r2_dog_hrf), axis=1, 169 | keys=['r2_hrf', 'r2_dog_hrf']).reset_index(), kind='scatter') 170 | # %% 171 | 172 | 173 | # %% 174 | # Here, we plot the predicted time courses of the difference-of-gaussians 175 | # model versus the original Gaussian model for the 9 voxels where the fit 176 | # imoproved the most. 177 | improvement = r2_dog_hrf - r2_gauss_hrf 178 | largest_improvements = improvement.sort_values(ascending=False).index[:9] 179 | pred_dog_hrf = model_dog.predict(parameters=pars_dog_hrf) 180 | pred = pd.concat((d.loc[:, largest_improvements], pred_gauss_hrf.loc[:, largest_improvements], pred_dog_hrf.loc[:, largest_improvements]), axis=1, keys=['data', 'gauss+hrf', 'dog+hrf'], names=['model']) 181 | 182 | tmp = pred.stack(['model', 'source']).to_frame('value') 183 | sns.relplot(x='frame', y='value', hue='model', col='source', data=tmp.reset_index(), kind='line', col_wrap=3, 184 | palette=['k'] + sns.color_palette(), 185 | hue_order=['data', 'gauss+hrf', 'dog+hrf']) 186 | 187 | 188 | # %% 189 | # Divisve Normalization PRF model 190 | # ------------------------------- 191 | # The most complex model we have is the DN-PRF model (Aqil et al., 2021). 192 | # This model has a Gaussian excitatory receptive field, and a Gaussian 193 | # inhibitory receptive field. The excitatory receptive field is divided 194 | # by the sum of the excitatory and inhibitory receptive fields. 195 | # The resulting receptive field is then convolved with the HRF. 196 | from braincoder.models import DivisiveNormalizationGaussianPRF2DWithHRF 197 | model_dn = DivisiveNormalizationGaussianPRF2DWithHRF(data=d, 198 | paradigm=paradigm, 199 | hrf_model=hrf_model, 200 | grid_coordinates=grid_coordinates, 201 | flexible_hrf_parameters=True) 202 | 203 | pars_dn_init = pars_dog_hrf.copy() 204 | pars_dn_init['srf_amplitude'] = 0.01 205 | pars_dn_init['srf_size'] = 2. 206 | pars_dn_init['baseline'] = 0.0 207 | pars_dn_init['neural_baseline'] = 1.0 208 | pars_dn_init['surround_baseline'] = 1.0 209 | 210 | par_fitter_dn = ParameterFitter(model=model_dn, data=d, paradigm=paradigm) 211 | # Without HRF 212 | pars_dn = par_fitter_dn.fit(init_pars=pars_dn_init, max_n_iterations=1000, fixed_pars=['hrf_delay', 'hrf_dispersion']) 213 | 214 | # With HRF 215 | pars_dn = par_fitter_dn.fit(init_pars=pars_dn, max_n_iterations=1000) 216 | 217 | # %% 218 | # Again, let's plot the R2 improvements 219 | r2_dn = par_fitter_dn.get_rsq(pars_dn) 220 | sns.relplot(x='r2_dog_hrf', y='r2_dn', data=pd.concat((r2_dog_hrf, r2_dn), axis=1, 221 | keys=['r2_dog_hrf', 'r2_dn']).reset_index(), kind='scatter') 222 | 223 | plt.plot([0, 1], [0, 1], 'k--') 224 | 225 | # %% 226 | improvement = r2_dn - r2_dog_hrf 227 | largest_improvements = improvement.sort_values(ascending=False).index[:9] 228 | 229 | pred_dn = model_dn.predict(parameters=pars_dn) 230 | pred = pd.concat((d.loc[:, largest_improvements], pred_dog_hrf.loc[:, largest_improvements], pred_dn.loc[:, largest_improvements]), axis=1, keys=['data', 'dog+hrf', 'dn+hrf'], names=['model']) 231 | 232 | tmp = pred.stack(['model', 'source']).to_frame('value') 233 | sns.relplot(x='frame', y='value', hue='model', col='source', data=tmp.reset_index(), kind='line', col_wrap=3, 234 | palette=['k'] + sns.color_palette(), 235 | hue_order=['data', 'dog+hrf', 'dn+hrf']) 236 | # %% 237 | 238 | 239 | # Decoding 240 | # -------- 241 | # We can also use the fitted models to decode the stimulus from the 242 | # fMRI response. Let's compare our simplest model versus our most 243 | # complex model. 244 | 245 | # First we fit the noise models 246 | from braincoder.optimize import ResidualFitter, StimulusFitter 247 | 248 | # Let's first get grid coordinates and paradigm at a slightly lower resolution 249 | data = load_szinte2024(resize_factor=2.5) 250 | grid_coordinates = data['grid_coordinates'] 251 | paradigm = data['stimulus'] 252 | 253 | best_voxels_gauss = r2_gauss_gd[pars_gauss_gd['sd'] > 0.5].sort_values(ascending=False).index[:200] 254 | 255 | model_gauss = GaussianPRF2DWithHRF(data=d[best_voxels_gauss], 256 | hrf_model=hrf_model, 257 | grid_coordinates=grid_coordinates.astype(np.float32), 258 | parameters=pars_gauss_gd.loc[best_voxels_gauss].astype(np.float32)) 259 | 260 | resid_fitter_gauss = ResidualFitter(model=model_gauss, data=d.loc[:, best_voxels_gauss], 261 | paradigm=paradigm.astype(np.float32), parameters=pars_gauss_gd.loc[best_voxels_gauss].astype(np.float32)) 262 | omega_gauss, _ = resid_fitter_gauss.fit() 263 | 264 | 265 | 266 | 267 | # %% 268 | 269 | 270 | # %% 271 | best_voxels_dn = r2_dn[pars_dn['sd'] > 0.5].sort_values(ascending=False).index[:200] 272 | 273 | model_dn = DivisiveNormalizationGaussianPRF2DWithHRF(data=d[best_voxels_dn], 274 | hrf_model=hrf_model, 275 | grid_coordinates=grid_coordinates.astype(np.float32), 276 | parameters=pars_dn.loc[best_voxels_dn].astype(np.float32)) 277 | 278 | resid_fitter_dn = ResidualFitter(model=model_dn, data=d.loc[:, best_voxels_dn], 279 | paradigm=paradigm, parameters=pars_dn.loc[best_voxels_dn]) 280 | 281 | omega_dn, _ = resid_fitter_dn.fit() 282 | 283 | # %% 284 | # Decoded stimulus: Gaussian model 285 | # =============================== 286 | # Now we can decode the stimulus from the fMRI responses 287 | stim_fitter_gauss = StimulusFitter(model=model_gauss, data=d.loc[:, best_voxels_gauss], omega=omega_gauss) 288 | stim_gauss = stim_fitter_gauss.fit(l2_norm=0.01, learning_rate=0.01, max_n_iterations=1000) 289 | 290 | # %% 291 | from matplotlib.animation import FuncAnimation 292 | from IPython.display import HTML 293 | 294 | def play_reconstruction(reconstructed_stimulus): 295 | 296 | # Here we make a movie of the decoded stimulus 297 | # Set up a function to draw a single frame 298 | vmin, vmax = 0.0, np.quantile(reconstructed_stimulus.values.ravel(), 0.99) 299 | 300 | def update(frame): 301 | plt.clf() # Clear the current figure 302 | plt.imshow(reconstructed_stimulus.stack('y').loc[frame].iloc[::-1, :], cmap='viridis', vmin=vmin, vmax=vmax) 303 | plt.axis('off') 304 | plt.title(f"Frame {frame}") 305 | 306 | # Create the animation 307 | fig = plt.figure() 308 | ani = FuncAnimation(fig, update, frames=range(paradigm.shape[0]), interval=100) 309 | 310 | return HTML(ani.to_html5_video()) 311 | 312 | play_reconstruction(stim_gauss) 313 | 314 | 315 | # %% 316 | # Decoded stimulus: DN model 317 | # ========================== 318 | stim_fitter_dn = StimulusFitter(model=model_dn, data=d.loc[:, best_voxels_dn], omega=omega_dn) 319 | stim_dn = stim_fitter_dn.fit(l2_norm=0.01, learning_rate=0.01, max_n_iterations=1000) 320 | 321 | # %% 322 | play_reconstruction(stim_dn) 323 | # As you can see, the DN model works a lot better than the Gaussian model. ;) 324 | 325 | # %% 326 | -------------------------------------------------------------------------------- /examples/00_encodingdecoding/fit_residuals.py: -------------------------------------------------------------------------------- 1 | """ 2 | Fit the residual covariance matrix 3 | ============================================= 4 | 5 | In this example, we fit a residual noise covariance matrix to 6 | simulated data from a ``VonMisesPRF``-model. 7 | 8 | """ 9 | 10 | # We set up a simple VonMisesPRF model 11 | from braincoder.models import VonMisesPRF 12 | import numpy as np 13 | import pandas as pd 14 | 15 | # Set up six evenly spaced von Mises PRFs 16 | centers = np.linspace(0.0, 2*np.pi, 6, endpoint=False) 17 | parameters = pd.DataFrame({'mu':centers, 'kappa':1., 'amplitude':1.0, 'baseline':0.0}, 18 | index=pd.Index([f'Voxel {i+1}' for i in range(6)], name='voxel')).astype(np.float32) 19 | 20 | # We have only 3 voxels, each with a linear combination of the 6 von Mises functions: 21 | weights = np.array([[1, 0, 1], 22 | [1, .5, 1], 23 | [0, 1, 0], 24 | [0, .5, 0], 25 | [0, 0, 1], 26 | [0, 0, 1]]).astype(np.float32) 27 | 28 | model = VonMisesPRF(parameters=parameters, weights=weights) 29 | 30 | # 50 random orientations 31 | paradigm = np.random.rand(50) * np.pi*2 32 | 33 | # Arbitrary covariance matrix 34 | cov = np.array([[.5, 0.0, 0.0], 35 | [.25, .75, .25], 36 | [.25, .25, .75]]) 37 | 38 | data = model.simulate(noise=cov, weights=weights, paradigm=paradigm) 39 | 40 | # %% 41 | 42 | # Import ResidualFitter 43 | from braincoder.optimize import ResidualFitter 44 | 45 | fitter = ResidualFitter(model, data, paradigm, parameters, weights) 46 | 47 | # omega is the covariance matrix, dof can be estimated when a 48 | # multivariate t-distribution (rather than a normal distribution) 49 | # is used 50 | omega, dof = fitter.fit(progressbar=False) 51 | print(omega) 52 | 53 | # %% -------------------------------------------------------------------------------- /examples/00_encodingdecoding/invert_model.py: -------------------------------------------------------------------------------- 1 | """ 2 | Invert encoding model 3 | ============================================= 4 | 5 | Here we invert different encoding models 6 | 7 | """ 8 | 9 | import scipy.stats as ss 10 | import matplotlib.pyplot as plt 11 | import seaborn as sns 12 | import numpy as np 13 | 14 | plt.title('Hypothetical RF + response') 15 | x = np.linspace(0, 2*np.pi) 16 | dist = ss.vonmises(loc=.5*np.pi, kappa=1.) 17 | 18 | plt.plot(x, dist.pdf(x), c='k', ls='--', label='Receptive field') 19 | y =dist.pdf(7./8.*np.pi) 20 | plt.plot(x, np.ones_like(x)*y, c='k', ls='-') 21 | plt.fill_between(x, y-0.025, y+0.025, alpha=.25, color='k', label='Measured activity') 22 | 23 | plt.scatter(1./8.*np.pi, y, c='k') 24 | plt.scatter(7./8.*np.pi, y, c='k') 25 | 26 | plt.xticks([0.0, .5*np.pi, np.pi, 1.5*np.pi, 2*np.pi], ['0', '1/2 $\pi$', '$\pi$', '1.5 $\pi$', '2 $\pi$']) 27 | sns.despine() 28 | 29 | plt.legend() 30 | 31 | # %% 32 | 33 | # We set up a simple VonMisesPRF model 34 | from braincoder.models import VonMisesPRF 35 | import pandas as pd 36 | import numpy as np 37 | 38 | # Set up six evenly spaced von Mises PRFs 39 | parameters = pd.DataFrame([{'mu':0.5*np.pi, 'kappa':1., 'amplitude':1.0, 'baseline':0.0}]).astype(np.float32) 40 | weights = np.array([[1]]).astype(np.float32) 41 | 42 | model = VonMisesPRF(parameters=parameters, weights=weights) 43 | omega = np.array([[0.1]]).astype(np.float32) 44 | 45 | data = pd.DataFrame([y]).astype(np.float32) 46 | # %% 47 | 48 | # Evaluate the likelihood of different possible orientations 49 | orientations = np.linspace(0.0, 2*np.pi).astype(np.float32) 50 | likelihood = model.likelihood(orientations, data, parameters, weights, omega) 51 | 52 | # And plot it.. 53 | plt.figure() 54 | plt.plot(orientations, likelihood.T, c='k') 55 | plt.xticks([0.0, .5*np.pi, np.pi, 1.5*np.pi, 2*np.pi], ['0', '1/2 $\pi$', '$\pi$', '1.5 $\pi$', '2 $\pi$']) 56 | 57 | sns.despine() 58 | plt.xlabel('Orientation') 59 | plt.ylabel('Likelihood') 60 | # %% 61 | 62 | # Simulate two-RF model 63 | palette = sns.color_palette() 64 | 65 | plt.title('Hypothetical RF + response') 66 | x = np.linspace(0, 2*np.pi) 67 | dist1 = ss.vonmises(loc=.5*np.pi, kappa=.5) 68 | dist2 = ss.vonmises(loc=1.*np.pi, kappa=.5) 69 | 70 | plt.plot(x, dist1.pdf(x), ls='--', label='Receptive field 1', color=palette[0]) 71 | plt.plot(x, dist2.pdf(x), ls='--', label='Receptive field 2', color=palette[1]) 72 | 73 | 74 | y1 =dist1.pdf(7./8.*np.pi) 75 | y2 =dist2.pdf(7./8.*np.pi) 76 | 77 | plt.plot(x, np.ones_like(x)*y1, c=palette[0], ls='-') 78 | plt.plot(x, np.ones_like(x)*y2, c=palette[1], ls='-') 79 | 80 | plt.fill_between(x, y1-0.025, y1+0.025, alpha=.25, color=palette[0], label='Measured activity RF1') 81 | plt.fill_between(x, y2-0.025, y2+0.025, alpha=.25, color=palette[1], label='Measured activity RF2') 82 | 83 | plt.xticks([0.0, .5*np.pi, np.pi, 1.5*np.pi, 2*np.pi], ['0', '1/2 $\pi$', '$\pi$', '1.5 $\pi$', '2 $\pi$']) 84 | sns.despine() 85 | plt.legend() 86 | 87 | # %% 88 | 89 | # Set up 2-dimensional model to invert 90 | parameters = pd.DataFrame([{'mu':0.5*np.pi, 'kappa':.5, 'amplitude':1.0, 'baseline':0.0}, 91 | {'mu':1.*np.pi, 'kappa':.5, 'amplitude':1.0, 'baseline':0.0}]).astype(np.float32) 92 | 93 | model = VonMisesPRF(parameters=parameters) 94 | omega = np.array([[0.05, 0.0], [0.0, 0.05]]).astype(np.float32) 95 | 96 | dist1 = ss.vonmises(loc=.5*np.pi, kappa=.5) 97 | dist2 = ss.vonmises(loc=1.*np.pi, kappa=.5) 98 | x = 7./8.*np.pi 99 | y1 =dist1.pdf(x) 100 | y2 =dist2.pdf(x) 101 | 102 | data = pd.DataFrame([[y1, y2]]).astype(np.float32) 103 | 104 | likelihood = model.likelihood(orientations, data, parameters, None, omega) 105 | 106 | plt.plot(orientations, likelihood.T, c='k') 107 | plt.xticks([0.0, .5*np.pi, np.pi, 1.5*np.pi, 2*np.pi], ['0', '1/2 $\pi$', '$\pi$', '1.5 $\pi$', '2 $\pi$']) 108 | 109 | sns.despine() 110 | plt.xlabel('Orientation') 111 | plt.ylabel('Likelihood') 112 | # %% -------------------------------------------------------------------------------- /examples/00_encodingdecoding/linear_encoding_model.py: -------------------------------------------------------------------------------- 1 | """ 2 | Linear encoding model 3 | ============================================= 4 | 5 | When it comes to fitting encoding models to fMRI data, 6 | the most common approach is to use a linear encoding model, 7 | were the predicted BOLD response is a linear combination of 8 | different neural populations with predefined tuning properties. 9 | 10 | Here we explore such an approach. 11 | 12 | """ 13 | 14 | # Import necessary libraries 15 | from braincoder.models import VonMisesPRF 16 | import numpy as np 17 | import pandas as pd 18 | import matplotlib.pyplot as plt 19 | 20 | # Set up six evenly spaced von Mises PRFs 21 | centers = np.linspace(0.0, 2*np.pi, 6, endpoint=False) 22 | parameters = pd.DataFrame({'mu':centers, 'kappa':1., 'amplitude':1.0, 'baseline':0.0}, 23 | index=pd.Index([f'Voxel {i+1}' for i in range(6)], name='voxel')) 24 | 25 | # We have 3 voxels, each with a linear combination of the 6 von Mises functions: 26 | weights = np.array([[1, 0, 1], 27 | [1, .5, 1], 28 | [0, 1, 0], 29 | [0, .5, 0], 30 | [0, 0, 1], 31 | [0, 0, 1]]).astype(np.float32) 32 | 33 | model = VonMisesPRF(parameters=parameters, weights=weights) 34 | # %% 35 | 36 | # Plot the basis functions 37 | # Note that the function `basis_functions` returns a `tensorflow` `Tensor`, 38 | # which has to be converted to a numpy array: 39 | orientations = np.linspace(0, np.pi*2, 100) 40 | basis_responses = model.basis_predictions(orientations, parameters).numpy() 41 | 42 | _ = plt.plot(orientations, basis_responses) 43 | 44 | # %% 45 | 46 | # Plot the predicted responses for the 3 voxels 47 | # Each voxel timeseries is a weighted sum of the six basis functions 48 | pred = model.predict(paradigm=orientations) 49 | _ = plt.plot(orientations, pred) 50 | 51 | # %% 52 | 53 | # Import the weight fitter 54 | from braincoder.optimize import WeightFitter 55 | from braincoder.utils import get_rsq 56 | 57 | # Simulate data 58 | data = model.simulate(paradigm=orientations, noise=0.1) 59 | 60 | # Fit the weights 61 | weight_fitter = WeightFitter(model, parameters, data, orientations) 62 | estimated_weights = weight_fitter.fit(alpha=0.1) 63 | 64 | # Get predictions for the fitted weights 65 | pred = model.predict(paradigm=orientations, weights=estimated_weights) 66 | r2 = get_rsq(data, pred) 67 | 68 | # Plot the data and the predictions 69 | plt.figure() 70 | plt.plot(orientations, data, c='k') 71 | plt.plot(orientations, pred.values, c='k', ls='--') 72 | plt.title(f'R2 = {r2.mean():.2f}') 73 | # %% -------------------------------------------------------------------------------- /examples/00_encodingdecoding/masked_stimulus_decoding.py: -------------------------------------------------------------------------------- 1 | """ 2 | Stimulus decoding using stimulus mask 3 | ============================================= 4 | 5 | It can be helpful to _constrain_ the decoding of a stimulus. 6 | For example, we might no be interested in decoding the location 7 | of the stimulus, but, rather, want to know which part of the stimulus 8 | show most drive. 9 | 10 | Here we give a small example of that. 11 | 12 | """ 13 | from braincoder.utils.data import load_szinte2024 14 | from braincoder.models import GaussianPRF2DWithHRF 15 | from braincoder.optimize import ResidualFitter 16 | from braincoder.hrf import SPMHRFModel 17 | 18 | ds = load_szinte2024(best_voxels=200) 19 | 20 | stimulus = ds['stimulus'] 21 | grid_coordinates = ds['grid_coordinates'] 22 | tr = ds['tr'] 23 | prf_pars = ds['prf_pars'] 24 | data = ds['v1_timeseries'] 25 | 26 | 27 | model = GaussianPRF2DWithHRF(grid_coordinates=grid_coordinates, 28 | paradigm=stimulus, 29 | parameters=prf_pars, 30 | hrf_model=SPMHRFModel(tr=tr)) 31 | 32 | 33 | resid_fitter = ResidualFitter(model, data, stimulus, parameters=prf_pars) 34 | omega, dof = resid_fitter.fit() 35 | 36 | from braincoder.optimize import StimulusFitter 37 | stim_fitter = StimulusFitter(data, model, omega) 38 | 39 | 40 | # Note how we use the original stimulus as a mask to only fit the stimulus at the locations where it was actually presented 41 | reconstructed_stimulus = stim_fitter.fit(legacy_adam=True, min_n_iterations=1000, max_n_iterations=2000, learning_rate=.05, mask=(stimulus > 0.1)) 42 | 43 | import matplotlib.pyplot as plt 44 | from matplotlib.animation import FuncAnimation 45 | from IPython.display import HTML 46 | import numpy as np 47 | 48 | def play_reconstruction(reconstructed_stimulus): 49 | 50 | # Here we make a movie of the decoded stimulus 51 | # Set up a function to draw a single frame 52 | vmin, vmax = 0.0, np.quantile(reconstructed_stimulus.values.ravel(), 0.99) 53 | 54 | def update(frame): 55 | plt.clf() # Clear the current figure 56 | plt.imshow(reconstructed_stimulus.loc[frame].values.reshape((stimulus.shape[1:])).T, cmap='viridis', vmin=vmin, vmax=vmax, origin='upper') 57 | plt.axis('off') 58 | plt.title(f"Frame {frame}") 59 | 60 | # Create the animation 61 | fig = plt.figure() 62 | ani = FuncAnimation(fig, update, frames=range(stimulus.shape[0]), interval=100) 63 | 64 | return HTML(ani.to_html5_video()) 65 | 66 | play_reconstruction(reconstructed_stimulus) 67 | 68 | # %% -------------------------------------------------------------------------------- /examples/README.txt: -------------------------------------------------------------------------------- 1 | ============================================= 2 | Examples 3 | ============================================= 4 | -------------------------------------------------------------------------------- /notebooks/GazeCenterFS_vd.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gilles86/braincoder/7001c05a07ca7accf471b40b0822a7d1ac525b75/notebooks/GazeCenterFS_vd.mat -------------------------------------------------------------------------------- /notebooks/encoding_decoding.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gilles86/braincoder/7001c05a07ca7accf471b40b0822a7d1ac525b75/notebooks/encoding_decoding.ipynb -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """The setup script.""" 4 | 5 | from setuptools import setup, find_packages 6 | 7 | with open('README.rst') as readme_file: 8 | readme = readme_file.read() 9 | 10 | 11 | requirements = ['tqdm', 'pandas', 'matplotlib', 'seaborn', 'pingouin'] 12 | 13 | test_requirements = [] 14 | 15 | setup( 16 | author="Gilles de Hollander", 17 | author_email='giles.de.hollander@gmail.com', 18 | python_requires='>=3.6', 19 | classifiers=[ 20 | 'Development Status :: 2 - Pre-Alpha', 21 | 'Intended Audience :: Developers', 22 | 'License :: OSI Approved :: MIT License', 23 | 'Natural Language :: English', 24 | 'Programming Language :: Python :: 3', 25 | 'Programming Language :: Python :: 3.6', 26 | 'Programming Language :: Python :: 3.7', 27 | 'Programming Language :: Python :: 3.8', 28 | ], 29 | description="Bayesian encoding/decoding of fMRI data", 30 | install_requires=requirements, 31 | license="MIT license", 32 | long_description=readme, 33 | keywords='braincoder', 34 | name='braincoder', 35 | packages=find_packages(include=['braincoder', 'braincoder.*']), 36 | package_data={'braincoder': ['data/szinte2024/*']}, 37 | include_package_data=True, 38 | test_suite='tests', 39 | tests_require=test_requirements, 40 | url='https://github.com/Gilles86/braincoder', 41 | version='0.1.0', 42 | zip_safe=False, 43 | ) 44 | -------------------------------------------------------------------------------- /tests/test_gauss_rf.py: -------------------------------------------------------------------------------- 1 | from braincoder.models import VoxelwiseGaussianReceptiveFieldModel 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | import seaborn as sns 5 | 6 | model = VoxelwiseGaussianReceptiveFieldModel() 7 | 8 | 9 | n_voxels = 15 10 | n_timepoints = 50 11 | 12 | paradigm = np.arange(0, 20) 13 | 14 | parameters = np.ones((n_voxels, 4)) 15 | parameters[:, 0] = np.linspace(5, 15, n_voxels) 16 | parameters[:, 3] = 0 17 | 18 | data = model.simulate(paradigm, parameters, noise=0.1) 19 | 20 | costs, pars_, pred_ = model.fit_parameters(paradigm, data, progressbar=True) 21 | 22 | predictions = model.get_predictions() 23 | r2 = model.get_rsq(data) 24 | r = model.get_r(data) 25 | 26 | -------------------------------------------------------------------------------- /tests/test_gauss_to_stick.py: -------------------------------------------------------------------------------- 1 | from braincoder.models import VoxelwiseGaussianReceptiveFieldModel 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | import seaborn as sns 5 | import scipy.stats as ss 6 | 7 | model = VoxelwiseGaussianReceptiveFieldModel() 8 | palette = sns.color_palette() 9 | 10 | n_voxels = 25 11 | n_timepoints = 150 12 | 13 | noise = 1.0 14 | 15 | paradigm = np.tile(np.arange(0, 20), int(n_timepoints / 20 + 1)) 16 | paradigm = paradigm[:n_timepoints] 17 | 18 | parameters = np.ones((n_voxels, 4)) 19 | parameters[:, 0] = np.linspace(0, 20, n_voxels) 20 | parameters[:, 1] = np.abs(np.random.randn(n_voxels)) * 3 21 | # parameters[:, 3] = np.random.randn(n_voxels) 22 | 23 | data = model.simulate(paradigm, parameters, noise=noise) 24 | 25 | 26 | costs, pars_, pred_ = model.fit_parameters(paradigm, data, progressbar=True) 27 | stimuli = np.linspace(-20, 40, 1000) 28 | sm = model.to_stickmodel(basis_stimuli=stimuli) 29 | 30 | sm.fit_residuals(data=data) 31 | 32 | data2 = model.simulate(paradigm, parameters, noise=noise) 33 | s, map, sd, ci = sm.get_stimulus_posterior(data2, stimulus_range=stimuli, normalize=True) 34 | plt.plot(paradigm, color=palette[0]) 35 | plt.plot(map, ls='--', color=palette[1]) 36 | plt.title('r = {:.2f}'.format(ss.pearsonr(map.ravel(), paradigm)[0])) 37 | plt.fill_between(range(len(map)), ci[0][:, 0], ci[1][:, 0], 38 | alpha=0.2, color=palette[1]) 39 | 40 | plt.figure() 41 | # s = np.clip(s, np.percentile(s, 1), np.percentile(s, 99)) 42 | sns.heatmap(s) 43 | plt.show() 44 | plt.figure() 45 | plt.plot(stimuli, s[:5].T) 46 | plt.show() 47 | -------------------------------------------------------------------------------- /tests/test_glm.py: -------------------------------------------------------------------------------- 1 | from braincoder.models import EncodingModel, GLMModel 2 | 3 | # from braincoder.decoders import WeightedEncodingModel 4 | import numpy as np 5 | from braincoder.utils import get_rsq, get_r 6 | import matplotlib.pyplot as plt 7 | import seaborn as sns 8 | 9 | import scipy.stats as ss 10 | 11 | n_timepoints = 100 12 | n_dimensions = 2 13 | n_voxels = 25 14 | 15 | paradigm = np.random.randn(n_timepoints, n_dimensions) 16 | weights = np.random.randn(n_dimensions, n_voxels) 17 | 18 | model = GLMModel() 19 | 20 | data = model.simulate(paradigm, weights=weights, noise=0.0) 21 | data2 = model.simulate(paradigm, weights=weights, noise=0.0) 22 | 23 | data += ss.t(3).rvs(data.shape) 24 | data2 += ss.t(3).rvs(data2.shape) 25 | 26 | test = model.fit_weights(paradigm, data, l2_cost=0.0) 27 | 28 | predictions = model.get_predictions() 29 | r2 = model.get_rsq(data) 30 | r22 = model.get_rsq(data2) 31 | r = model.get_r(data) 32 | 33 | model.fit_residuals(paradigm, data, residual_dist='t') 34 | stimulus_range = np.linspace(-5, 5, 100, dtype=np.float32)[:, np.newaxis] 35 | # stimulus_range = np.repeat(stimulus_range, n_dimensions, 1) 36 | 37 | stimulus_range = np.array(np.meshgrid(*[np.linspace(-3, 3, 20) for i in range(n_dimensions)])) 38 | 39 | stimulus_range = stimulus_range.reshape((n_dimensions, np.prod(stimulus_range.shape[1:]))).T 40 | 41 | p, map_, sd, ci = model.get_stimulus_posterior(data, stimulus_range) 42 | r_decode = get_r(map_, paradigm) 43 | 44 | p, map_, sd, ci = model.get_stimulus_posterior(data2, stimulus_range) 45 | r_decode2 = get_r(map_, paradigm) 46 | -------------------------------------------------------------------------------- /tests/test_glm_t_dist.py: -------------------------------------------------------------------------------- 1 | from braincoder.models import EncodingModel, GLMModel 2 | 3 | # from braincoder.decoders import WeightedEncodingModel 4 | import numpy as np 5 | from braincoder.utils import get_rsq, get_r 6 | import matplotlib.pyplot as plt 7 | import seaborn as sns 8 | 9 | import scipy.stats as ss 10 | 11 | n_timepoints = 100 12 | n_dimensions = 1 13 | n_voxels = 15 14 | 15 | paradigm = np.random.randn(n_timepoints, n_dimensions) 16 | weights = np.random.randn(n_dimensions, n_voxels) 17 | 18 | model = GLMModel() 19 | model_t = GLMModel() 20 | 21 | data = model.simulate(paradigm, weights=weights, noise=0.0) 22 | 23 | data += ss.t(3).rvs(data.shape) * 3 24 | 25 | model.fit_weights(paradigm, data, l2_cost=0.0) 26 | model_t.fit_weights(paradigm, data, l2_cost=0.0) 27 | 28 | predictions = model.get_predictions() 29 | r2 = model.get_rsq(data) 30 | r = model.get_r(data) 31 | 32 | model.fit_residuals(paradigm, data, residual_dist='gaussian', also_fit_weights=True) 33 | model_t.fit_residuals(paradigm, data, residual_dist='t', also_fit_weights=True) 34 | 35 | stimulus_range = np.array(np.meshgrid(*[np.linspace(-3, 3, 200) for i in range(n_dimensions)])) 36 | stimulus_range = stimulus_range.reshape((n_dimensions, np.prod(stimulus_range.shape[1:]))).T 37 | 38 | p, map_, sd, ci = model.get_stimulus_posterior(data, stimulus_range) 39 | p_t, map_t, sd_t, ci_t = model_t.get_stimulus_posterior(data, stimulus_range) 40 | 41 | 42 | in_ci = (paradigm > ci[0]) & (paradigm < ci[1]) 43 | in_ci_t = (paradigm > ci_t[0]) & (paradigm < ci_t[1]) 44 | 45 | -------------------------------------------------------------------------------- /tests/test_stickmodel.py: -------------------------------------------------------------------------------- 1 | from braincoder.models import StickModel 2 | import numpy as np 3 | 4 | n_voxels = 10 5 | n_timepoints = 30 6 | n_populations = 11 7 | noise = 0.1 8 | 9 | parameters = np.arange(n_populations)[:, np.newaxis] 10 | 11 | weights = np.zeros((n_populations, n_voxels)) 12 | populations = np.random.choice(np.arange(1, 11), n_voxels) 13 | intercepts = np.random.randn(n_voxels) 14 | 15 | weights[0] = intercepts 16 | weights[populations, np.arange(n_voxels)] = 1 17 | 18 | paradigm = np.tile(np.arange(-1, 12), int(n_timepoints / 10))[:n_timepoints] 19 | 20 | model = StickModel(parameters) 21 | data = model.simulate(paradigm, weights=weights, noise=noise) 22 | model.fit_weights(paradigm=paradigm, data=data) 23 | 24 | r2 = model.get_rsq(data) 25 | 26 | --------------------------------------------------------------------------------