├── .github └── workflows │ └── python-publish.yml ├── .gitignore ├── LICENSE.txt ├── README.md ├── gridfix ├── __init__.py ├── features.py ├── model.py └── regionset.py ├── pyproject.toml └── setup.py /.github/workflows/python-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflow will upload a Python Package using Twine when a release is created 2 | # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries 3 | 4 | # This workflow uses actions that are not certified by GitHub. 5 | # They are provided by a third-party and are governed by 6 | # separate terms of service, privacy policy, and support 7 | # documentation. 8 | 9 | name: Upload Python Package 10 | 11 | on: 12 | release: 13 | types: [published] 14 | 15 | jobs: 16 | deploy: 17 | 18 | runs-on: ubuntu-latest 19 | 20 | steps: 21 | - uses: actions/checkout@v2 22 | - name: Set up Python 23 | uses: actions/setup-python@v2 24 | with: 25 | python-version: '3.x' 26 | - name: Install dependencies 27 | run: | 28 | python -m pip install --upgrade pip 29 | pip install build 30 | - name: Build package 31 | run: python -m build 32 | - name: Publish package 33 | uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 34 | with: 35 | user: __token__ 36 | password: ${{ secrets.PYPI_API_TOKEN }} 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 98 | __pypackages__/ 99 | 100 | # Celery stuff 101 | celerybeat-schedule 102 | celerybeat.pid 103 | 104 | # SageMath parsed files 105 | *.sage.py 106 | 107 | # Environments 108 | .env 109 | .venv 110 | env/ 111 | venv/ 112 | ENV/ 113 | env.bak/ 114 | venv.bak/ 115 | 116 | # Spyder project settings 117 | .spyderproject 118 | .spyproject 119 | 120 | # Rope project settings 121 | .ropeproject 122 | 123 | # mkdocs documentation 124 | /site 125 | 126 | # mypy 127 | .mypy_cache/ 128 | .dmypy.json 129 | dmypy.json 130 | 131 | # Pyre type checker 132 | .pyre/ 133 | 134 | # pytype static type analyzer 135 | .pytype/ 136 | 137 | # Cython debug symbols 138 | cython_debug/ 139 | 140 | docs/ -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Immo Schuetz 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GridFix 2 | 3 | 4 | [![PyPI version](https://badge.fury.io/py/gridfix.svg)](https://badge.fury.io/py/gridfix) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.998553.svg)](https://doi.org/10.5281/zenodo.998553) 5 | 6 | 7 | GridFix is a Python 3 toolbox to facilitate preprocessing of scene fixation data for region-based analysis using Generalized Linear Mixed Models (GLMM). 8 | 9 | [Our recently published manuscript](https://www.frontiersin.org/articles/10.3389/fnhum.2017.00491) [1] describes how this approach can be used to evaluate models of visual saliency above and beyond content-independent biases. Please also see [3] for a previous description of the approach and [our ECVP 2016 poster](http://doi.org/10.5281/zenodo.571067) [2] for an overview about the structure and workflow of the GridFix toolbox. 10 | 11 | ![Example Image](https://ischtz.github.io/gridfix/_images/example_grid.png) 12 | 13 | 14 | ## Features 15 | - Define image parcellations (region masks) based on a grid or bounding box coordinates (RegionSet) 16 | - Apply these parcellations to collections of images or saliency maps (ImageSet) 17 | - Calculate per-region fixation status (fixated/not fixated), count, and duration-based dependent measures 18 | - Define features to assign a value to each region X image (e.g., mean saliency of each region) 19 | - Explicitly model central viewer bias using different approaches (e.g. Euclidean distance, Gaussian) 20 | - Output the resulting feature vectors for GLMM-based analysis [1,2] 21 | - Create initial R source code to facilitate GLMM analysis using lme4 22 | 23 | 24 | ## Installation 25 | 26 | GridFix requires Python 3. The following packages need to be installed: 27 | - numpy 28 | - scipy 29 | - matplotlib 30 | - pandas 31 | - PIL or Pillow (image library) 32 | 33 | If you need to first install a Python environment, we recommend the Anaconda distribution which includes all of the listed prerequisites by default, is available for all major platforms (Windows, Mac, Linux) and can be downloaded at https://www.anaconda.com/products/individual#Downloads. 34 | 35 | To install GridFix from the Python Package Index (PyPI), simply run: 36 | 37 | ``` 38 | pip install gridfix 39 | ``` 40 | 41 | 42 | Alternatively, you can clone this repository using git or download a ZIP file using the "Clone or download" button, then run the following: 43 | 44 | ``` 45 | python3 setup.py install 46 | ``` 47 | or, in case you want to only install for your user or receive an error regarding permissions: 48 | 49 | ``` 50 | python3 setup.py install --user 51 | ``` 52 | 53 | GridFix was installed correctly if you can execute 54 | 55 | ``` 56 | from gridfix import * 57 | ``` 58 | 59 | in an interactive Python 3 session without errors. 60 | 61 | 62 | ## Documentation 63 | 64 | See [GridFix documentation](https://ischtz.github.io/gridfix/). 65 | Tutorial Jupyter Notebooks and example data are [available as a separate download](https://github.com/ischtz/gridfix-tutorial/releases). 66 | 67 | 68 | ## References 69 | [1] Nuthmann, A., Einhäuser, W., & Schütz, I. (2017). How well can saliency models predict fixation selection in scenes beyond central bias? A new approach to model evaluation using generalized linear mixed models. Frontiers in Human Neuroscience. http://doi.org/10.3389/fnhum.2017.00491 70 | 71 | [2] Schütz, I., Einhäuser, W., & Nuthmann, A. (2016). GridFix: A Python toolbox to facilitate fixation analysis and evaluation of saliency algorithms using Generalized linear mixed models (GLMM). Poster presented at the European Conference on Visual Perception (ECVP), Barcelona. http://doi.org/10.5281/zenodo.571067 72 | 73 | [3] Nuthmann, A., & Einhäuser, W. (2015). A new approach to modeling the influence of image features on fixation selection in scenes. Annals of the New York Academy of Sciences, 1339(1), 82-96. http://dx.doi.org/10.1111/nyas.12705 74 | 75 | -------------------------------------------------------------------------------- /gridfix/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3.4 2 | # -*- coding: utf-8 -*- 3 | 4 | __version__ = '0.3.2' 5 | 6 | from .model import * 7 | from .regionset import * 8 | from .features import * -------------------------------------------------------------------------------- /gridfix/features.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | import types 5 | 6 | import numpy as np 7 | import matplotlib as mp 8 | import matplotlib.pyplot as plt 9 | 10 | from pandas import DataFrame, read_table 11 | from scipy.ndimage import center_of_mass # for CentralBiasFeature 12 | from scipy.ndimage.filters import sobel # for SobelEdgeFeature 13 | 14 | from .regionset import RegionSet 15 | from .model import ImageSet 16 | 17 | 18 | class Feature(object): 19 | """ Base class for image features defined for each region in a set 20 | 21 | A Feature can be described as a two-step "recipe" on a given ImageSet and 22 | RegionSet: First, a transform() is executed upon each image array, e.g. a 23 | filter operation. Second, a combine() operation yields a single value per region. 24 | The result is combined into a feature vector of len(regionset). 25 | 26 | Attributes: 27 | regionset (RegionSet): the associated regionset object 28 | imageset (ImageSet): a set of images or feature maps to process 29 | length (int): length of feature vector, i.e. number of regions 30 | """ 31 | 32 | def __init__(self, regionset, imageset, trans_fun=None, comb_fun=None, label=None, normalize_output=False): 33 | """ Create a new basic Feature object. 34 | 35 | Args: 36 | regionset (RegionSet): a RegionSet object 37 | trans_fun (function): Function to use for the transformation step instead 38 | of default. Must accept PIL.Image and return Image or np.ndarray of same size. 39 | comb_fun (function): Function to use for the reduction step instead of default. 40 | Must accept a np.ndarray and return a scalar value. 41 | label (string): optional label to distinguish between Features of the same type 42 | normalize_output (bool): if True, always normalize output values of this feature to 0..1 43 | """ 44 | self.regionset = regionset 45 | self.imageset = imageset 46 | self.label = label 47 | self.normalize_output = normalize_output 48 | 49 | self._fvalues = {} 50 | 51 | # Replace default functions only if the specified ones make sense 52 | try: 53 | tmp = trans_fun(self, np.array([[0,1],[0,0]])) 54 | self._transform = self.transform # keep default around 55 | self.transform = types.MethodType(trans_fun, self) 56 | except TypeError: 57 | if trans_fun is not None: 58 | print('Warning: trans_fun seems to return invalid values, using default!') 59 | 60 | try: 61 | tmp = comb_fun(self, np.array([[0,1],[0,0]]), np.array([[0,1],[0,0]])) 62 | self._combine = self.combine 63 | self.combine = types.MethodType(comb_fun, self) 64 | except TypeError: 65 | if comb_fun is not None: 66 | print('Warning: comb_fun seems to return invalid values, using default!') 67 | 68 | 69 | def __repr__(self): 70 | """ String representation for print summary. """ 71 | desc = '' 72 | if self.label is not None: 73 | desc = ' "{:s}"'.format(str(self.label)) 74 | 75 | norm = '' 76 | if self.normalize_output: 77 | norm = ', normalized' 78 | r = ''.format(self.__class__.__name__, desc, len(self.regionset), norm) 79 | r += '\nRegions:\n\t{:s}'.format(str(self.regionset)) 80 | r += '\nImages:\n\t{:s}'.format(str(self.imageset)) 81 | 82 | return(r) 83 | 84 | 85 | def __len__(self): 86 | """ Overload len() to report length of feature vector. """ 87 | return len(self.regionset) 88 | 89 | 90 | def __getitem__(self, imageid): 91 | """ Bracket indexing using an imageid returns the feature vector. """ 92 | return self.apply(imageid, normalize=False) 93 | 94 | 95 | def transform(self, image): 96 | """ Apply image transform to specified image. 97 | 98 | In the base Feature class, this simply returns the input image as-is. 99 | This function may be overloaded in subclasses or replaced by the 100 | trans_fun= argument on construction time. 101 | 102 | Args: 103 | image (ndarray): image / feature map array to transform 104 | 105 | Returns: 106 | np.ndarray of image data 107 | """ 108 | return np.asarray(image, dtype=float) 109 | 110 | 111 | def combine(self, image, region, fun=np.mean): 112 | """ Combine all selected pixel values in selection using specified function. 113 | 114 | In the base Feature class, this simply returns the mean of all pixels. 115 | This function may be overloaded in subclasses or replaced by the comb_fun= argument. 116 | 117 | Args: 118 | image (np.ndarray): 2D feature image 119 | region (np.ndarray): binary mask array defining a region 120 | fun (function): function to apply to selection. Must return a scalar. 121 | 122 | Returns: 123 | Scalar value depending on the specified function. 124 | """ 125 | return fun(np.asarray(image[region], dtype=float)) 126 | 127 | 128 | def apply(self, imageid, normalize=None): 129 | """ Apply feature to a single image from associated ImageSet. 130 | 131 | Args: 132 | imageid (str): valid ID from associated ImageSet 133 | normalize (bool): if True, scale output to range 0...1 (default: False) 134 | Returns: 135 | 1D numpy.ndarray of feature values, same length as regionset 136 | """ 137 | if imageid not in self._fvalues.keys(): 138 | fv = [] 139 | img_in = self.imageset[imageid] 140 | 141 | # Transformation step 142 | img_trans = self.transform(img_in) 143 | 144 | # Combination step 145 | for region in self.regionset[imageid]: 146 | f = self.combine(np.asarray(img_trans), # make sure combine gets arrays 147 | np.asarray(region, dtype=bool)) 148 | fv.append(f) 149 | 150 | # cache for later use 151 | self._fvalues[imageid] = np.array(fv) 152 | 153 | 154 | f = self._fvalues[imageid] 155 | if self.normalize_output: 156 | # Feature set to always normalize 157 | return((f - f.min()) / (f.max() - f.min())) 158 | elif normalize is not None: 159 | if normalize: 160 | # Upstream Model requested normalized values 161 | return((f - f.min()) / (f.max() - f.min())) 162 | else: 163 | # Default is to return unmodified values 164 | return f 165 | 166 | 167 | def apply_all(self, normalize=None): 168 | """ Apply feature to every image in the ImageSet and return a DataFrame. 169 | 170 | Args: 171 | normalize (bool): if True, scale output to range 0...1 (default: False) 172 | 173 | Returns: DataFrame similar to RegionSet.info with region feature values 174 | """ 175 | rmeta = self.regionset.info.copy() 176 | if self.regionset.is_global: 177 | # If global RegionSet, copy region data once per image 178 | df = DataFrame(columns=rmeta.columns) 179 | for img in self.imageset.imageids: 180 | tmp = rmeta.copy() 181 | tmp['imageid'] = img 182 | df = df.append(tmp) 183 | else: 184 | df = rmeta 185 | df['value'] = np.nan 186 | 187 | for img in self.imageset.imageids: 188 | if img not in self.regionset.imageids and not self.regionset.is_global: 189 | continue 190 | 191 | img_trans = self.transform(self.imageset[img]) 192 | l = self.regionset._select_labels(img) 193 | 194 | for idx,region in enumerate(self.regionset[img]): 195 | f = self.combine(np.asarray(img_trans), np.asarray(region, dtype=bool)) 196 | df.loc[(df['imageid'] == img) & (df['region'] == str(l[idx])), 'value'] = f 197 | 198 | # Normalize value column if requested 199 | if self.normalize_output: 200 | val = df.loc[:, 'value'] 201 | df.loc[:, 'value'] = (val - val.min()) / (val.max() - val.min()) 202 | 203 | elif normalize is not None: 204 | if normalize: 205 | val = df.loc[:, 'value'] 206 | df.loc[:, 'value'] = (val - val.min()) / (val.max() - val.min()) 207 | 208 | return df 209 | 210 | 211 | def plot(self, imageid, what='both', cmap='gray', image_only=False, ax=None): 212 | """ Display feature map and/or feature values. 213 | 214 | Args: 215 | imageid (str): valid ID from associated ImageSet 216 | what (str): show only feature 'values', 'map' or 'both' 217 | cmap (str): name of matplotlib colormap to use 218 | image_only (boolean): if True, return only image content without labels 219 | ax (Axes): axes object to draw to (only if what!='both') 220 | 221 | Returns: 222 | matplotlib figure object, or None if passed an axis to draw on 223 | """ 224 | lbl_text = '' 225 | if self.label is not None: 226 | lbl_text = ': {:s}'.format(str(self.label)) 227 | 228 | if what == 'map': 229 | if ax is not None: 230 | ax1 = ax 231 | else: 232 | fig = plt.figure() 233 | ax1 = fig.add_subplot(1,1,1) 234 | 235 | imap = self.transform(self.imageset[imageid]) 236 | ax1.imshow(imap, cmap=plt.get_cmap(cmap), interpolation='none') 237 | if image_only: 238 | ax1.axis('off') 239 | else: 240 | ax1.set_title('{:s}{:s} (img: {:s})'.format(self.__class__.__name__, lbl_text, imageid)) 241 | 242 | elif what == 'values': 243 | if ax is not None: 244 | ax1 = ax 245 | else: 246 | fig = plt.figure() 247 | ax1 = fig.add_subplot(1,1,1) 248 | 249 | self.regionset.plot(values=self.apply(imageid), imageid=imageid, cmap=cmap, ax=ax1, image_only=image_only) 250 | if image_only: 251 | ax1.axis('off') 252 | else: 253 | ax1.set_title('{:s}{:s} (img: {:s})'.format(self.__class__.__name__, lbl_text, imageid)) 254 | 255 | else: 256 | if ax is not None: 257 | print('Warning: ax argument is only valid if a single plot type is specified! Returning full figure instead.') 258 | 259 | fig = plt.figure() 260 | ax1 = fig.add_subplot(1,2,1) 261 | imap = self.transform(self.imageset[imageid]) 262 | ax1.imshow(imap, cmap=plt.get_cmap(cmap), interpolation='none') 263 | ax2 = fig.add_subplot(1,2,2) 264 | self.regionset.plot(values=self.apply(imageid), imageid=imageid, cmap=cmap, ax=ax2, image_only=image_only) 265 | if image_only: 266 | ax1.axis('off') 267 | ax2.axis('off') 268 | else: 269 | ax1.set_title('map') 270 | ax2.set_title('values') 271 | fig.suptitle('{:s}{:s} (img: {:s})'.format(self.__class__.__name__, lbl_text, imageid)) 272 | 273 | if ax is None and not plt.isinteractive(): # see ImageSet.plot() 274 | return fig 275 | 276 | 277 | 278 | class CentralBiasFeature(Feature): 279 | """ Models central viewer bias as the distance to image center for each region. 280 | 281 | The exact model of distance depends on the "measure" argument: if set to 'distance' 282 | (the default), the image transformation step does nothing and euclidean distance is 283 | returned. If set to 'gaussian', anisotropic Gaussian distance based on Clarke & Tatler, 284 | 2014, Vis Res is used and transform() returns the corresponding Gaussian map. 285 | """ 286 | 287 | def __init__(self, regionset, imageset, measure='gaussian', sig2=0.23, nu=None, label=None, normalize_output=False): 288 | """ Create a new CentralBiasFeature object. 289 | 290 | Args: 291 | regionset (RegionSet): region set to apply feature to 292 | imageset (ImageSet): a set of images or feature maps (optional for this Feature) 293 | measure (string): distance measure to use ('euclidean', 'gaussian', 'taxicab') 294 | sig2 (float): variance value for type='gaussian' 295 | nu (float): anisotropy value for type='gaussian' 296 | label (string): optional label to distinguish between Features 297 | normalize_output (bool): if True, always normalize output values of this feature to 0..1 298 | """ 299 | if measure not in ['gaussian', 'euclidean', 'taxicab']: 300 | print('Warning: unknown central bias measure "{:s}" specified, falling back to euclidean distance!'.format(measure)) 301 | measure = 'euclidean' 302 | 303 | self.measure = measure 304 | self._map = None 305 | self._values = {} 306 | 307 | self.sig2 = sig2 308 | self.nu = nu 309 | 310 | if nu is None: 311 | if measure == 'gaussian': 312 | self.nu = 0.45 313 | else: 314 | self.nu = 1 315 | 316 | 317 | def _transform(self, image=None): 318 | """ For 'gaussian': Gauss distance map, else return empty array. """ 319 | 320 | if self._map is None: 321 | # compute feature map on first call 322 | 323 | mapsize = self.imageset.size[1], self.imageset.size[0] 324 | 325 | if measure == 'gaussian': 326 | self._map = self._aniso_gauss(mapsize, self.sig2, self.nu) 327 | else: 328 | self._map = np.zeros(mapsize, dtype=float) 329 | 330 | return self._map 331 | 332 | 333 | def _combine(self, image, region): 334 | """ Calculate distance from region center of mass to image center. """ 335 | 336 | mapsize = self.imageset.size 337 | 338 | com = np.round(center_of_mass(region)) 339 | img_center = (int(round(mapsize[1] / 2)), int(round(mapsize[0] / 2))) 340 | 341 | if self.measure == 'gaussian': 342 | return image[int(com[0]), int(com[1])] 343 | 344 | elif self.measure == 'euclidean': 345 | dist = np.hypot((com[0] - img_center[0]) / self.nu, com[1] - img_center[1]) 346 | return np.round(dist, 0) 347 | 348 | elif measure == 'taxicab': 349 | dist = abs(com[0] - img_center[0]) + abs(com[1] - img_center[1]) 350 | return np.round(dist, 0) 351 | 352 | self._trans_fun = _transform 353 | self._comb_fun = _combine 354 | 355 | Feature.__init__(self, regionset, imageset, trans_fun=_transform, comb_fun=_combine, label=label, 356 | normalize_output=normalize_output) 357 | 358 | 359 | def apply(self, imageid=None, normalize=None): 360 | """ Apply central bias to image, returning region distance values. 361 | 362 | Args: 363 | imageid (str): for consistency, ignored for central bias (same for all images) 364 | normalize (boolean): if True, scale output to range 0...1 (default: False) 365 | 366 | Returns: 367 | 1D numpy.ndarray of feature values, same length as regionset 368 | """ 369 | if imageid not in self._values.keys(): 370 | fv = [] 371 | 372 | img_trans = self.transform() 373 | 374 | for region in self.regionset[imageid]: 375 | f = self.combine(img_trans, np.asarray(region, dtype=bool)) 376 | fv.append(f) 377 | 378 | self._values[imageid] = np.array(fv) 379 | 380 | f = self._values[imageid] 381 | if self.normalize_output: 382 | # Feature set to always normalize 383 | return((f - f.min()) / (f.max() - f.min())) 384 | elif normalize is not None: 385 | if normalize: 386 | # Upstream Model requested normalized values 387 | return((f - f.min()) / (f.max() - f.min())) 388 | else: 389 | # Default is to return unmodified values 390 | return f 391 | 392 | 393 | def __repr__(self): 394 | """ String representation for printing """ 395 | desc = '' 396 | if self.label is not None: 397 | desc = ' "{:s}"'.format(str(self.label)) 398 | 399 | r = ' 2 and image.shape[2] == 3: 454 | R, G, B = image[:, :, 0], image[:, :, 1], image[:, :, 2] 455 | lum = 0.2989 * R + 0.5870 * G + 0.1140 * B 456 | return lum 457 | else: 458 | return image 459 | 460 | def _combine(self, image, region): 461 | """ Return mean region luminance. """ 462 | return(image[region].mean()) 463 | 464 | self.trans_fun = _transform 465 | self.comb_fun = _combine 466 | 467 | Feature.__init__(self, regionset, imageset, trans_fun=_transform, comb_fun=_combine, label=label, 468 | normalize_output=normalize_output) 469 | 470 | 471 | 472 | class LumContrastFeature(Feature): 473 | """ Feature based on local luminance contrast in each region """ 474 | 475 | def __init__(self, regionset, imageset, label=None, normalize_output=False): 476 | """ Create a new LuminanceContrastFeature 477 | 478 | Args: 479 | regionset: a RegionSet to be evaluated 480 | imageset: ImageSet containing images or feature maps to process 481 | label (str): optional label to distinguish between Features 482 | normalize_output (bool): if True, always normalize output values of this feature to 0..1 483 | """ 484 | 485 | def _transform(self, image): 486 | """ Convert 3D-RGB image to 2D-intensity (like rgb2gray.m). """ 487 | if image.ndim > 2 and image.shape[2] == 3: 488 | R, G, B = image[:, :, 0], image[:, :, 1], image[:, :, 2] 489 | lum = 0.2989 * R + 0.5870 * G + 0.1140 * B 490 | return lum 491 | else: 492 | return image 493 | 494 | def _combine(self, image, region): 495 | """ Return local contrast of luminance image. """ 496 | return(image[region].std() / image.mean()) 497 | 498 | self.trans_fun = _transform 499 | self.comb_fun = _combine 500 | 501 | Feature.__init__(self, regionset, imageset, trans_fun=_transform, comb_fun=_combine, label=label, 502 | normalize_output=normalize_output) 503 | 504 | 505 | 506 | class SobelEdgeFeature(Feature): 507 | """ Feature based on relative prevalence of edges within each region """ 508 | 509 | def __init__(self, regionset, imageset, label=None, normalize_output=False): 510 | """ Create a new SobelEdgeFeature 511 | 512 | Args: 513 | regionset: a RegionSet to be evaluated 514 | imageset: ImageSet containing images or feature maps to process 515 | label (str): optional label to distinguish between Features 516 | normalize_output (bool): if True, always normalize output values of this feature to 0..1 517 | """ 518 | 519 | def _transform(self, image): 520 | """ Run sobel filter on luminance image, then binarize. """ 521 | if image.ndim > 2 and image.shape[2] == 3: 522 | R, G, B = image[:, :, 0], image[:, :, 1], image[:, :, 2] 523 | lum = 0.2989 * R + 0.5870 * G + 0.1140 * B 524 | else: 525 | lum = image 526 | 527 | sx = sobel(lum, 0) 528 | sy = sobel(lum, 1) 529 | si = np.hypot(sx, sy) 530 | 531 | # Determine threshold for edge detection (adapted from MATLAB edge.m) 532 | scale = 4.0 533 | cutoff = scale * si.mean() 534 | thresh = np.sqrt(cutoff) 535 | 536 | return np.asarray(si >= thresh, dtype=np.uint8) 537 | 538 | 539 | def _combine(self, image, region): 540 | """ Mean of binary image yields fraction of edges. """ 541 | return(image[region].mean()) 542 | 543 | self.trans_fun = _transform 544 | self.comb_fun = _combine 545 | 546 | Feature.__init__(self, regionset, imageset, trans_fun=_transform, comb_fun=_combine, label=label, 547 | normalize_output=normalize_output) 548 | 549 | 550 | 551 | class MapFeature(Feature): 552 | """ Feature to apply a statistical function to each region in feature maps 553 | 554 | Attributes: 555 | stat (function): the statistics function to apply to each region 556 | """ 557 | 558 | def __init__(self, regionset, imageset, stat=np.mean, label=None, normalize_output=False): 559 | """ Create a new MapFeature 560 | 561 | Args: 562 | regionset: a RegionSet to be evaluated 563 | imageset: ImageSet containing images or feature maps to process 564 | stat (function): the statistics function to apply to each region 565 | label (str): optional label to distinguish between Features 566 | normalize_output (bool): if True, always normalize output values of this feature to 0..1 567 | """ 568 | self.stat = stat 569 | 570 | def _transform(self, image): 571 | """ Does nothing, expects predefined input map! """ 572 | return image 573 | 574 | def _combine(self, image, region): 575 | """ Applies stat function to region. """ 576 | return(self.stat(image[region])) 577 | 578 | self.trans_fun = _transform 579 | self.comb_fun = _combine 580 | 581 | Feature.__init__(self, regionset, imageset, trans_fun=_transform, comb_fun=_combine, label=label, 582 | normalize_output=normalize_output) 583 | 584 | 585 | 586 | class RegionFeature(Feature): 587 | """ Feature based on properties of the regions in a RegionSet. Allows to use 588 | e.g. region area or fraction of image pixels as a model predictor. 589 | 590 | Attributes: 591 | region_property (str): column from DataFrame to return as region feature 592 | """ 593 | 594 | def __init__(self, regionset, imageset, region_property='area', label=None, normalize_output=False): 595 | """ Create a new RegionFeature 596 | 597 | Args: 598 | regionset: a RegionSet to be evaluated 599 | imageset: ImageSet containing images or feature maps to process 600 | region_property (str): column from DataFrame to return as region feature 601 | label (str): optional label to distinguish between Features 602 | normalize_output (bool): if True, always normalize output values of this feature to 0..1 603 | """ 604 | if region_property not in regionset.info.columns: 605 | raise ValueError('Specified region property is not a column in RegionSet.info DataFrame! Example: "area"') 606 | self.region_property = region_property 607 | 608 | def _transform(self, image): 609 | """ Does nothing """ 610 | return image 611 | 612 | def _combine(self, image, region): 613 | """ Does nothing """ 614 | return None 615 | 616 | self.trans_fun = _transform 617 | self.comb_fun = _combine 618 | 619 | Feature.__init__(self, regionset, imageset, trans_fun=_transform, comb_fun=_combine, label=label, 620 | normalize_output=normalize_output) 621 | 622 | 623 | def apply(self, imageid, normalize=None): 624 | """ Return selected region property of each region for specified imageid. 625 | 626 | Args: 627 | imageid (str): valid ID from associated ImageSet 628 | normalize (bool): if True, scale output to range 0...1 (default: False) 629 | 630 | Returns: 631 | 1D numpy.ndarray of feature values, same length as regionset 632 | """ 633 | if self.regionset.is_global: 634 | imageid = '*' 635 | if imageid not in self.regionset._regions.keys(): 636 | raise ValueError('The imageid specified for RegionFeature was not found in the associated Imageset!') 637 | 638 | sel_df = self.regionset.info[self.regionset.info.imageid == imageid] 639 | 640 | f = np.array(sel_df[self.region_property]) 641 | if self.normalize_output: 642 | # Feature set to always normalize 643 | return((f - f.min()) / (f.max() - f.min())) 644 | elif normalize is not None: 645 | if normalize: 646 | # Upstream Model requested normalized values 647 | return((f - f.min()) / (f.max() - f.min())) 648 | else: 649 | # Default is to return unmodified values 650 | return f 651 | 652 | 653 | 654 | -------------------------------------------------------------------------------- /gridfix/model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | import os 5 | import sys 6 | import time 7 | import types 8 | import itertools 9 | 10 | import numpy as np 11 | import matplotlib as mp 12 | import matplotlib.pyplot as plt 13 | 14 | from PIL import Image 15 | from pandas import DataFrame, read_table 16 | from pandas import __version__ as pandas_version 17 | from distutils.version import LooseVersion 18 | 19 | from scipy.io import whosmat, loadmat 20 | from scipy.signal import convolve 21 | 22 | 23 | # Constants to facilitate specifying groups of Dependent Variables 24 | DV_FIXATED = ['fixated', 'count'] 25 | DV_FIXDUR = ['first', 'single', 'gaze', 'tofirst', 'total'] 26 | DV_ALL = ['fixated', 'count', 'fixid', 'first', 'single', 'gaze', 'tofirst', 'total', 'passes', 'refix'] 27 | 28 | 29 | class ImageSet(object): 30 | """ Set of images of equal size for masking and Feature creation. 31 | 32 | Attributes: 33 | info (DataFrame): table of image metadata (filenames, size, type...) 34 | imageids (list): All unique image IDs in the set 35 | label (str): optional label to distinguish between ImageSets 36 | mat_var (str): name of MATLAB variable name if imported from .mat 37 | mat_contents (list): list of all variable names in .mat if applicable 38 | normalize (boolean): True if image data was normalized to 0..1 range 39 | preload (boolean): True if images were preloaded into memory 40 | size (tuple): image dimensions, specified as (width, height) 41 | """ 42 | 43 | def __init__(self, images, mat_var=None, size=None, imageids=None, 44 | sep='\t', label=None, normalize=None, norm_range=None, preload=False): 45 | """ Create a new ImageSet and add specified images 46 | 47 | Args: 48 | images: one of the following: 49 | - path to a single image or .mat file 50 | - path to a folder containing image or .mat files 51 | - list of image or .mat file names 52 | - simple text file, one filename per line 53 | - text / CSV file containing columns 'filename' and 'imageid' 54 | mat_var (str): variable to use if _images_ is a MATLAB file. If not specified, 55 | gridfix attempts to load the first variable in alphabetical order. 56 | size (tuple): image dimensions, specified as (width, height) in pixels 57 | imageids (list): string ID labels, one for each image. If not specified, 58 | file names without extension or a numerical label 0..n will be used 59 | sep (str): if _images_ is a text file, use this separator 60 | label (string): optional descriptive label 61 | normalize (boolean): normalize image color / luminance values to 0...1 62 | (defaults to True for images, False for MATLAB data) 63 | norm_range (tuple): normalization range. Defaults to (0, 255) for image 64 | files and per-image (min, max) for data read from MATLAB files 65 | preload (boolean): if True, load all images at creation time 66 | (faster, but uses a lot of memory) 67 | """ 68 | self.imageids = [] 69 | self._images = {} 70 | 71 | # DF to hold image metadata 72 | df_col = ['imageid', 'filename', 'width', 'height', 'channels', 'type', 'mat_var'] 73 | self.info = DataFrame(columns=df_col) 74 | 75 | self.size = None 76 | self.label = label 77 | self.normalize = normalize 78 | self.norm_range = norm_range 79 | self.preload = preload 80 | self._last_image = None 81 | self._last_imageid = None 82 | 83 | self.mat_var = None 84 | self.mat_contents = None 85 | 86 | if size is not None: 87 | self.size = tuple(size) 88 | 89 | self._add_images(images, mat_var, imageids, sep) 90 | 91 | 92 | def __repr__(self): 93 | """ Short string representation for printing """ 94 | # Image size 95 | if self.size is None: 96 | size = 'undefined' 97 | else: 98 | size = str(self.size) 99 | 100 | # Number of images 101 | s = '' 102 | if len(self.imageids) != 1: 103 | s = 's' 104 | 105 | desc = '' 106 | if self.label is not None: 107 | desc = ' "{:s}"'.format(str(self.label)) 108 | 109 | mat = '' 110 | if self.mat_var is not None: 111 | mat = ', mat_var={:s}'.format(self.mat_var) 112 | norm = '' 113 | if self.normalize: 114 | norm = ', normalized' 115 | r = '' 116 | return r.format(desc, len(self.imageids), s, size, mat, norm) 117 | 118 | 119 | def __len__(self): 120 | """ Overload len(ImageSet) to report number of images. """ 121 | return len(self.imageids) 122 | 123 | 124 | def __iter__(self): 125 | """ Overload iteration to step through the ndarray representations of images. """ 126 | return iter([np.array(i) for i in self.images.keys()]) 127 | 128 | 129 | def __getitem__(self, imageid): 130 | """ Allow to retrieve image by bracket indexing """ 131 | return self.image(imageid) 132 | 133 | 134 | def _add_images(self, images, mat_var=None, imageids=None, sep='\t'): 135 | """ Add one or more image(s) to set. 136 | 137 | Args: 138 | images: one of the following: 139 | - path to a single image file 140 | - path to a folder containing image files 141 | - list of image file names 142 | - simple text file, one image filename per line 143 | - text / CSV file containing columns 'filename' and 'imageid' 144 | mat_var (str): variable to use if _images_ is a MATLAB file 145 | imageids (list): string ID labels, one for each image. If not specified, 146 | file names without extension or a numerical label 0..n will be used 147 | sep (str): if _images_ is a text file, use this separator 148 | """ 149 | filelist = [] 150 | if imageids is None: 151 | imageids = [] 152 | 153 | img_root = os.getcwd() 154 | 155 | # Build file list 156 | if type(images) == list: 157 | filelist = images 158 | 159 | elif type(images) == str: 160 | 161 | if os.path.isdir(images): 162 | # Directory 163 | filelist = [os.path.join(images, b) for b in sorted(os.listdir(images))] 164 | 165 | elif os.path.isfile(images): 166 | # Single file - check if it is a text list of images! 167 | (ifname, ifext) = os.path.splitext(images) 168 | 169 | if ifext.lower() in ['.txt', '.tsv', '.csv', '.dat']: 170 | # assume image list 171 | try: 172 | imgfiles = read_table(images, header=None, index_col=False, sep=sep) 173 | 174 | if imgfiles.shape[1] == 1: 175 | # Only one column: assume no headers and load as list of filenames 176 | filelist = list(imgfiles.iloc[:, 0]) 177 | 178 | elif imgfiles.shape[1] > 1: 179 | # More than one column: look for 'imageid' and 'filename' columns 180 | if 'imageid' in list(imgfiles.iloc[0, :]) and 'filename' in list(imgfiles.iloc[0, :]): 181 | imgfiles.columns = imgfiles.iloc[0, :] 182 | imgfiles = imgfiles.iloc[1:, :] 183 | filelist = list(imgfiles['filename']) 184 | imageids = list(imgfiles['imageid']) 185 | 186 | lfolder, lname = os.path.split(images) 187 | if len(lfolder) > 0: 188 | img_root = os.path.abspath(lfolder) 189 | 190 | except: 191 | raise ValueError('could not read image list file, check format!') 192 | else: 193 | # assume single image file 194 | filelist = [images] 195 | else: 196 | raise ValueError('first argument must be list or a string containing a file or directory!') 197 | 198 | # Verify image files 199 | filetable = [] 200 | for (idx, ifile) in enumerate(filelist): 201 | try: 202 | imeta = self._verify_image(ifile, mat_var, img_root) 203 | 204 | # assign imageid 205 | (ifdir, iffile) = os.path.split(ifile) 206 | (ifbase, ifext) = os.path.splitext(iffile) 207 | if imageids is not None and len(imageids) > 0: 208 | imageid = imageids.pop(0) 209 | else: 210 | imageid = ifbase 211 | imeta['imageid'] = imageid 212 | 213 | if imageid in self.imageids: 214 | print('Warning: replacing existing image with ID <{:s}>!'.format(imageid)) 215 | else: 216 | self.imageids.append(imageid) 217 | 218 | filetable.append(imeta) 219 | 220 | except ValueError as err: 221 | print('Warning: file {:s} could not be added, error: {:s}!'.format(ifile, str(err))) 222 | 223 | df_col = ['imageid', 'filename', 'width', 'height', 'channels', 'type', 'mat_var'] 224 | self.info = DataFrame(filetable, columns=df_col) 225 | self.imageids = list(self.info.imageid) 226 | 227 | # Preload files if requested 228 | for imid in self.imageids: 229 | if self.preload: 230 | self._images[imid] = self._load_image(imid) 231 | else: 232 | self._images[imid] = None 233 | 234 | 235 | def image(self, imageid): 236 | """ Get image by imageid, loading it if not preloaded 237 | 238 | Args: 239 | imageid (str): valid imageid from set 240 | 241 | Returns: 242 | ndarray of raw image data 243 | """ 244 | if imageid not in self.imageids: 245 | raise ValueError('Specified _imageid_ does not exist!') 246 | 247 | if self._images[imageid] is not None: 248 | return self._images[imageid] 249 | else: 250 | if self._last_imageid == imageid: 251 | return self._last_image 252 | else: 253 | return self._load_image(imageid) 254 | 255 | 256 | def _load_image(self, imageid): 257 | """ Load and return image data for imageid. 258 | 259 | Args: 260 | imageid (str): valid imageid from set 261 | 262 | Returns: 263 | ndarray of raw image data 264 | """ 265 | if imageid not in self.imageids: 266 | raise ValueError('Specified _imageid_ does not exist!') 267 | 268 | if self._last_imageid is not None and self._last_imageid == imageid: 269 | return self._last_image 270 | 271 | imdata = None 272 | imeta = self.info[self.info.imageid == imageid] 273 | 274 | if len(imeta) > 0: 275 | 276 | if imeta.type.iloc[0] == 'MAT': 277 | mat = loadmat(imeta.filename.iloc[0], variable_names=imeta.mat_var.iloc[0]) 278 | imdata = np.asarray(mat[imeta.mat_var.iloc[0]], dtype=float) 279 | 280 | else: 281 | i = Image.open(imeta.filename.iloc[0]) 282 | if i.mode == 'RGBA': 283 | i = i.convert('RGB') # Drop alpha channel 284 | imdata = np.asarray(i, dtype=float) 285 | 286 | if self.normalize is None and imeta.type.iloc[0] != 'MAT': 287 | if self.norm_range is None: 288 | imdata = imdata / 255.0 289 | else: 290 | imdata = (imdata - self.norm_range[0]) / (self.norm_range[1] - self.norm_range[0]) 291 | self.normalize = True 292 | 293 | elif self.normalize: 294 | if self.norm_range is None: 295 | imdata = (imdata - imdata.min()) / (imdata.max() - imdata.min()) 296 | else: 297 | imdata = (imdata - self.norm_range[0]) / (self.norm_range[1] - self.norm_range[0]) 298 | 299 | self._last_imageid = imageid 300 | self._last_image = imdata 301 | 302 | return imdata 303 | 304 | 305 | def _verify_image(self, image_file, mat_var=None, img_root=None): 306 | """ Verify type and size of image without actually loading data 307 | 308 | Args: 309 | image_file (str): path to image file to verify 310 | mat_var (str): optional variable name for MATLAB files 311 | img_root (str): folder containing image list in case file paths are relative 312 | 313 | Returns: 314 | dict of metadata, column names as in imageset.info DataFrame 315 | """ 316 | if not os.path.isfile(image_file): 317 | if img_root is not None and os.path.isfile(os.path.join(img_root, image_file)): 318 | image_file = os.path.join(os.path.join(img_root, image_file)) 319 | else: 320 | raise ValueError('file not found') 321 | 322 | (ifbase, ifext) = os.path.splitext(image_file) 323 | 324 | imeta = {'imageid': None, 325 | 'filename': '', 326 | 'width': -1, 327 | 'height': -1, 328 | 'channels': -1, 329 | 'type': None, 330 | 'mat_var': None} 331 | 332 | # Matlab files 333 | if ifext.lower() == '.mat': 334 | try: 335 | # Load .mat header and identify variables 336 | tmp = whosmat(image_file) 337 | tmpvars = [m[0] for m in tmp] 338 | 339 | if self.mat_contents is None: 340 | self.mat_contents = tmpvars 341 | 342 | if mat_var is None: 343 | mat_var = tmpvars[0] 344 | 345 | if mat_var in tmpvars: 346 | 347 | if self.mat_var is None: 348 | self.mat_var = mat_var 349 | 350 | # check image size 351 | imshape = [m[1] for m in tmp if m[0] == mat_var][0] 352 | if self.size is None: 353 | self.size = (imshape[1], imshape[0]) 354 | 355 | if (imshape[1], imshape[0]) == self.size: 356 | imeta['filename'] = image_file 357 | imeta['width'] = imshape[1] 358 | imeta['height'] = imshape[0] 359 | if len(imshape) > 2: 360 | imeta['channels'] = imshape[2] 361 | else: 362 | imeta['channels'] = 1 363 | imeta['type'] = 'MAT' 364 | imeta['mat_var'] = mat_var 365 | 366 | else: 367 | w = 'Warning: skipping {:s} due to image size ({:d}x{:d} instead of {:d}x{:d}).' 368 | print(w.format(iffile, imshape[1], imshape[0], self.size[0], self.size[1])) 369 | 370 | else: 371 | raise ValueError('specified MATLAB variable not in file') 372 | 373 | except: 374 | raise ValueError('error loading MATLAB data') 375 | 376 | # Image files 377 | else: 378 | try: 379 | i = Image.open(image_file) 380 | imsize = i.size 381 | 382 | if self.size is None: 383 | self.size = imsize 384 | 385 | # check image size 386 | if imsize == self.size: 387 | imeta['filename'] = image_file 388 | imeta['width'] = imsize[0] 389 | imeta['height'] = imsize[1] 390 | if i.mode in ['RGB', 'RGBA']: 391 | imeta['channels'] = 3 392 | else: 393 | imeta['channels'] = 1 394 | imeta['type'] = i.format 395 | imeta['mat_var'] = '' 396 | 397 | else: 398 | w = 'Warning: skipping {:s} due to image size ({:d}x{:d} instead of {:d}x{:d}).' 399 | print(w.format(image_file, imsize[0], imsize[1], self.size[0], self.size[1])) 400 | 401 | except OSError: 402 | raise ValueError('not an image or file could not be opened.') 403 | 404 | return imeta 405 | 406 | 407 | def plot(self, imageid, cmap='gray', image_only=False, ax=None): 408 | """ Display one of the contained images by imageid using matplotlib 409 | 410 | Args: 411 | imageid (str): valid imageid of image to show 412 | cmap (str): name of a matplotlib colormap to use 413 | image_only (boolean): if True, return only image content without labels 414 | ax (Axes): axes object to draw on, to include result in other figure 415 | 416 | Returns: 417 | matplotlib figure object, or None if passed an axis to draw on 418 | """ 419 | try: 420 | if ax is not None: 421 | ax1 = ax 422 | else: 423 | fig = plt.figure() 424 | ax1 = fig.add_subplot(1,1,1) 425 | 426 | ax1.imshow(self.image(imageid), cmap=plt.get_cmap(cmap)) 427 | if image_only: 428 | ax1.axis('off') 429 | else: 430 | ax1.set_title(imageid) 431 | 432 | if ax is None and not plt.isinteractive(): 433 | # Only return figure object in non-interactive mode, otherwise 434 | # IPython/Jupyter will display the figure twice (once while plotting 435 | # and once as the cell result)! 436 | return fig 437 | 438 | except KeyError: 439 | raise ValueError('No image with ID "{:s}" in set'.format(imageid)) 440 | 441 | 442 | class Fixations(object): 443 | """ Dataset of fixation locations. 444 | 445 | Fixation locations are assumed to be one-indexed in input, e.g. 1..800 if 446 | the image is 800 pixels wide, and converted internally to Pythons zero-indexed 447 | array convention. 448 | 449 | Attributes: 450 | data (DataFrame): DataFrame of raw fixation data 451 | has_times (boolean): True if fixation times have been loaded 452 | imageids (list): all unique image IDs represented in the dataset 453 | imageset (ImageSet): if present, the associated ImageSet 454 | input_file (str): file name of fixation data file 455 | num_samples (int): number of fixation samples 456 | num_vars (int): number of columns / variables in dataset 457 | offset (tuple): optional offset from raw (x,y) positions 458 | shape (tuple): dimensions of .data, i.e. same as Fixations.data.shape 459 | variables (list): list of all variables loaded from input file 460 | """ 461 | 462 | def __init__(self, data, imageset=None, offset=(0, 0), imageid='imageid', 463 | fixid='fixid', x='x', y='y', fixstart=None, fixend=None, 464 | fixdur=None, numericid=False, round_coords=True): 465 | """ Create new Fixations dataset and calculate defaults. 466 | 467 | Args: 468 | data: a predefined DataFrame or name of a file containing fixation report 469 | imageset (ImageSet): if present, verify imageids against this ImageSet 470 | offset (tuple): an (x, y) tuple of pixel values to offset fixations. E.g., 471 | if fixations have their origin at image center, use (-width/2, -height/2) 472 | imageid (str): name of data file column containing imageids 473 | fixid (str): name of data file column with unique fixation ID / index 474 | x (str): name of data file column for horizontal fixation locations 475 | y (str): name of data file column for vertical fixation locations 476 | fixstart (str): name of data file column containing fixation start time 477 | fixend (str): name of data file column containing fixation end time 478 | fixdur (start): name of column containing fixation durations. If start/end 479 | are given but this is None, durations will be calculated by gridfix 480 | numericid (boolean): if True, try to force parsing imageid as numeric 481 | round_coords (boolean): How to treat float values in fixation coordinates: 482 | if True, round values to closest pixel (e.g., x=4.4 -> pixel 4 / index 3) 483 | if False, round values up to next integer (x=4.4 -> pixel 5 / index 4) 484 | """ 485 | self.data = DataFrame() 486 | 487 | if isinstance(data, DataFrame): 488 | # data is already a DataFrame 489 | self.data = data 490 | self.input_file = None 491 | 492 | else: 493 | try: 494 | self.data = read_table(data, index_col=False) 495 | self.input_file = data 496 | 497 | except IOError: 498 | raise IOError('Could not load fixation data, check file name and type!') 499 | 500 | # Internal column names 501 | self._imageid = imageid 502 | self._fixid = fixid 503 | self._x = x 504 | self._y = y 505 | self._xpx = self._x + '_PX' 506 | self._ypx = self._y + '_PX' 507 | self._fixstart = fixstart 508 | self._fixend = fixend 509 | self._fixdur = fixdur 510 | 511 | # Verify that all required columns are present 512 | cols = [imageid, fixid, x, y] 513 | miss_cols = [] 514 | for c in cols: 515 | if c not in self.data.columns.values: 516 | miss_cols.append(c) 517 | 518 | if len(miss_cols) > 0: 519 | raise ValueError('Missing columns ({:s}), please specify column names!'.format(str(miss_cols))) 520 | 521 | # Image ID column should always be converted to strings 522 | if numericid: 523 | self.data[self._imageid] = self.data[self._imageid].astype(int).astype(str) 524 | else: 525 | self.data[self._imageid] = self.data[self._imageid].astype(str) 526 | 527 | self.imageids = list(self.data[self._imageid].unique()) 528 | self.shape = self.data.shape 529 | self.num_samples = self.shape[0] 530 | self.num_vars = self.shape[1] 531 | self.variables = list(self.data.columns.values) 532 | 533 | # Fixation timing columns (optional, default: not specified) 534 | self.has_times = False 535 | if fixstart is not None and fixend is not None: 536 | if fixstart not in self.data.columns.values: 537 | raise ValueError('Unknown column specified for fixation start time: "{:s}"'.format(fixstart)) 538 | if fixend not in self.data.columns.values: 539 | raise ValueError('Unknown column specified for fixation end time: "{:s}"'.format(fixend)) 540 | self.has_times = True 541 | 542 | # Calculate fixation durations if no pre-computed column was specified 543 | if self._fixdur is None: 544 | self._fixdur = '__FIXDUR' 545 | self.data[self._fixdur] = self.data[self._fixend] - self.data[self._fixstart] 546 | else: 547 | if (fixstart is None) - (fixend is None) != 0: 548 | raise ValueError('Optional timing columns (fixstart, fixend) must be specified together!') 549 | 550 | # If ImageSet provided, check if all images are present 551 | self.imageset = None 552 | if imageset is not None: 553 | self.imageset = imageset 554 | missing_imgs = [] 555 | for im in self.imageids: 556 | if im not in imageset.imageids: 557 | missing_imgs.append(im) 558 | if len(missing_imgs) > 0: 559 | print('Warning: the following images appear in the fixation data but not the speficied ImageSet: {:s}'.format(', '.join(missing_imgs))) 560 | 561 | # Set offset and calculate pixel indices (_xpx, _ypx) 562 | self.offset = (0, 0) 563 | self.round_coords = round_coords 564 | self.set_offset(offset) 565 | 566 | 567 | def __repr__(self): 568 | """ String representation """ 569 | r = ''.format(self.num_samples, len(self.imageids)) 570 | if self.imageset is not None: 571 | r += '\nImages:\n\t{:s}'.format(str(self.imageset)) 572 | return r 573 | 574 | 575 | def __len__(self): 576 | """ Overload len() to report the number of samples """ 577 | return self.data.shape[0] 578 | 579 | 580 | def __getitem__(self, imageid): 581 | """ Bracket indexing returns all fixations for a specified image """ 582 | return self.select_fix(select={self._imageid: imageid}) 583 | 584 | 585 | def set_offset(self, offset): 586 | """ Set a constant offset for eye x/y coordinates. 587 | 588 | If image coordinates are relative to image center, use (-width/2, -height/2) 589 | (GridFix uses a coordinate origin at the top left). 590 | 591 | Args: 592 | offset (tuple): 2-tuple of (hor, ver) offset values in pixels 593 | """ 594 | # Reset previous offset 595 | prevoffset = self.offset 596 | if prevoffset[0] != 0.0 or prevoffset[1] != 0.0: 597 | self.data[self._x] = self.data[self._x] - prevoffset[0] 598 | self.data[self._y] = self.data[self._y] - prevoffset[1] 599 | 600 | self.data[self._x] = self.data[self._x] + offset[0] 601 | self.data[self._y] = self.data[self._y] + offset[1] 602 | self.offset = (offset[0], offset[1]) 603 | 604 | # Convert x/y fixation positions to integers (pixels) while keeping original data 605 | # Note: we're going to use these as indices and Python is 0-indexed, so subtract 1! 606 | if self.round_coords: 607 | self.data[self._xpx] = np.asarray(np.round(self.data[self._x]), dtype=int) - 1 608 | self.data[self._ypx] = np.asarray(np.round(self.data[self._y]), dtype=int) - 1 609 | else: 610 | self.data[self._xpx] = np.asarray(np.ceil(self.data[self._x]), dtype=int) - 1 611 | self.data[self._ypx] = np.asarray(np.ceil(self.data[self._y]), dtype=int) - 1 612 | 613 | 614 | def select_fix(self, select={}): 615 | """ Return a subset of fixation data for specified imageid. 616 | 617 | Args:. 618 | select (dict): dict of filter variables, as {column: value} 619 | 620 | Returns: 621 | New Fixations object containing selected fixations only 622 | """ 623 | if self._imageid not in select.keys(): 624 | print('Warning: no image ID in filter variables, selection will yield fixations from multiple images! Proceed at own risk.') 625 | 626 | if select[self._imageid] not in self.data[self._imageid].values: 627 | print('Warning: zero fixations selected for specified imageid ({:s})'.format(select[self._imageid])) 628 | 629 | selection = self.data 630 | if len(select) > 0: 631 | for col, target in select.items(): 632 | if col not in selection.columns.values: 633 | print('Warning: filter variable {:s} not found in Fixations dataset!'.format(col)) 634 | else: 635 | # Make sure dict value is list-like 636 | if type(target) not in (tuple, list): 637 | target = [target] 638 | selection = selection[selection[col].isin(target)] 639 | 640 | result = Fixations(selection.copy(), imageid=self._imageid, fixid=self._fixid, 641 | x=self._x, y=self._y, imageset=self.imageset, 642 | fixstart=self._fixstart, fixend=self._fixend, fixdur=self._fixdur, 643 | offset=self.offset, round_coords=self.round_coords) 644 | return result 645 | 646 | 647 | def plot(self, imageid=None, select={}, on_image=True, oob=False, 648 | plotformat='o', plotsize=5.0, plotcolor=[1, 1, 1], 649 | durations=False, image_only=False, ax=None): 650 | """ Plot fixations for selected imageid, either alone or on image 651 | 652 | Args: 653 | imageid (str): optional image ID to plot fixations for 654 | select (dict): dict of additional filter variables (see select_fix()) 655 | image (bool): if True, superimpose fixations onto image (if ImageSet present) 656 | oob (bool): if True, include out-of-bounds fixations when plotting 657 | plotformat (str): format string for plt.pyplot.plot(), default: circles 658 | plotsize (float): fixation marker size 659 | plotcolor (color): fixation marker color, default: white 660 | durations (bool): if True, plot duration of each fixation next to marker 661 | image_only (boolean): if True, return only image content without labels 662 | ax (Axes): axes object to draw to, to include result in other figure 663 | 664 | Returns: 665 | matplotlib figure object, or None if passed an axis to draw on 666 | """ 667 | if imageid is not None: 668 | if imageid not in select.keys(): 669 | select[self._imageid] = imageid 670 | plotfix = self.select_fix(select) 671 | else: 672 | plotfix = self 673 | 674 | if ax is not None: 675 | ax1 = ax 676 | else: 677 | fig = plt.figure() 678 | ax1 = fig.add_subplot(1,1,1) 679 | 680 | try: 681 | if on_image: 682 | ax1.imshow(self.imageset.image(imageid), origin='upper') 683 | 684 | except AttributeError: 685 | print('Warning: cannot view fixations on image due to missing ImageSet!') 686 | 687 | if oob: 688 | ax1.plot(plotfix.data[self._xpx], plotfix.data[self._ypx], plotformat, 689 | markersize=plotsize, color=plotcolor) 690 | if durations and self._fixdur in plotfix.data.columns.to_list(): 691 | for r in plotfix.data.iterrows(): 692 | x = r[1][self._xpx] 693 | if r[1][self._ypx] > 15: 694 | y = r[1][self._ypx] - 15 695 | else: 696 | y = r[1][self._ypx] + 5 697 | d = r[1][self._fixdur] 698 | ax1.text(x, y, str(d), horizontalalignment='center') 699 | 700 | else: 701 | try: 702 | if self.imageset is not None: 703 | size = self.imageset.size 704 | else: 705 | size = (max(plotfix.data[self._xpx]), max(plotfix.data[self._ypx])) 706 | fix = plotfix.data[(plotfix.data[self._xpx] >= 0) & 707 | (plotfix.data[self._xpx] < size[0]) & 708 | (plotfix.data[self._ypx] >= 0) & 709 | (plotfix.data[self._ypx] < size[1])] 710 | ax1.plot(fix[self._xpx], fix[self._ypx], plotformat, 711 | markersize=plotsize, color=plotcolor) 712 | if durations and self._fixdur in plotfix.data.columns.to_list(): 713 | for r in fix.iterrows(): 714 | x = r[1][self._xpx] 715 | if r[1][self._ypx] > 15: 716 | y = r[1][self._ypx] - 15 717 | else: 718 | y = r[1][self._ypx] + 5 719 | d = r[1][self._fixdur] 720 | ax1.text(x, y, str(d), horizontalalignment='center') 721 | 722 | ax1.set_xlim((0, size[0])) 723 | ax1.set_ylim((0,size[1])) 724 | ax1.invert_yaxis() 725 | if image_only: 726 | ax1.axis('off') 727 | else: 728 | ax1.set_title(imageid) 729 | 730 | except AttributeError: 731 | print('Warning: cannot filter fixations for image boundaries due to missing ImageSet!') 732 | ax1.plot(plotfix[self._xpx], plotfix[self._ypx], plotformat, 733 | markersize=plotsize, color=plotcolor) 734 | ax1.invert_yaxis() 735 | 736 | if ax is None and not plt.isinteractive(): # see ImageSet.plot() 737 | return fig 738 | 739 | 740 | def location_map(self, imageid=None, size=None): 741 | """ Binary ndarray of fixated and non-fixated pixels within image area 742 | 743 | Args: 744 | imageid (str): optional image ID to create map for one image only 745 | size (tuple): image dimensions, specified as (width, height). 746 | 747 | Returns: 748 | 2d boolean ndarray, True where fixated, otherwise False 749 | """ 750 | if imageid is not None: 751 | mapfix = self.select_fix({self._imageid: imageid}) 752 | else: 753 | mapfix = self 754 | 755 | if size is None: 756 | if self.imageset is not None: 757 | size = self.imageset.size 758 | else: 759 | raise ValueError('Image size or attached ImageSet are necessary for location mapping!') 760 | 761 | fixloc = np.zeros((size[1], size[0]), dtype=bool) 762 | fix = mapfix.data[(self.data[self._xpx] >= 0) & (self.data[self._xpx] < size[0]) & 763 | (self.data[self._ypx] >= 0) & (self.data[self._ypx] < size[1])] 764 | fixloc[fix[self._ypx], fix[self._xpx]] = True 765 | return fixloc 766 | 767 | 768 | def count_map(self, imageid=None, size=None): 769 | """ Map of fixation counts for each image pixel 770 | 771 | Args: 772 | imageid (str): optional image ID to create map for one image only 773 | size (tuple): image dimensions, specified as (width, height). 774 | 775 | Returns: 776 | 2d ndarray of pixel fixation counts 777 | """ 778 | if imageid is not None: 779 | mapfix = self.select_fix({self._imageid: imageid}) 780 | else: 781 | mapfix = self 782 | 783 | if size is None: 784 | if self.imageset is not None: 785 | size = self.imageset.size 786 | else: 787 | raise ValueError('Image size or attached ImageSet are necessary for location mapping!') 788 | 789 | fixcount = np.zeros((size[1], size[0]), dtype=int) 790 | fix = mapfix.data[(self.data[self._xpx] >= 0) & (self.data[self._xpx] < size[0]) & 791 | (self.data[self._ypx] >= 0) & (self.data[self._ypx] < size[1])] 792 | fixloc = fix[[self._ypx, self._xpx]].to_numpy() 793 | for pos in fixloc: 794 | fixcount[pos[0], pos[1]] += 1 795 | return fixcount 796 | 797 | 798 | 799 | def dur_map(self, imageid=None, size=None): 800 | """ Map of total fixation durations for each image pixel 801 | 802 | Args: 803 | imageid (str): optional image ID to create map for one image only 804 | size (tuple): image dimensions, specified as (width, height). 805 | 806 | Returns: 807 | 2d ndarray of fixation durations at each pixel 808 | """ 809 | 810 | if imageid is not None: 811 | mapfix = self.select_fix({self._imageid: imageid}) 812 | else: 813 | mapfix = self 814 | 815 | if not self._fixdur in mapfix.data.columns.to_list(): 816 | raise ValueError('Duration map is only available if dataset contains fixation durations!') 817 | 818 | 819 | if size is None: 820 | if self.imageset is not None: 821 | size = self.imageset.size 822 | else: 823 | raise ValueError('Image size or attached ImageSet are necessary for location mapping!') 824 | 825 | durmap = np.zeros((size[1], size[0]), dtype=float) 826 | fix = mapfix.data[(self.data[self._xpx] >= 0) & (self.data[self._xpx] < size[0]) & 827 | (self.data[self._ypx] >= 0) & (self.data[self._ypx] < size[1])] 828 | for r in fix.iterrows(): 829 | durmap[r[1][self._ypx], r[1][self._xpx]] += r[1][self._fixdur] 830 | return durmap 831 | 832 | 833 | 834 | def _gauss2d(self, size_x=100, sigma_x=10, size_y=None, sigma_y=None): 835 | """ Create 2D Gaussian kernel for heat map visualizations 836 | 837 | Args: 838 | size_x, size_y (int): horizontal/vertical kernel size, in pixels 839 | sigma_x, sigma_y (float): hor/vert kernel standard deviaton, in pixels 840 | 841 | Returns: 842 | 2d ndarray containing Gaussian kernel 843 | """ 844 | if size_y is None: 845 | size_y = size_x 846 | if sigma_y is None: 847 | sigma_y = sigma_x 848 | 849 | (xx, yy) = np.meshgrid(np.linspace(-size_x/2, size_x/2, size_x), np.linspace(-size_y/2, size_y/2, size_y)) 850 | G = np.exp(-1.0*(((xx*xx) / (2*sigma_x*sigma_x)) + ((yy*yy) / (2*sigma_y*sigma_y)))) 851 | return G 852 | 853 | 854 | 855 | def heat_map(self, imageid=None, size=None, dur=False, convolution=True, 856 | size_x=100, sigma_x=10, size_y=None, sigma_y=None, normalize=False, threshold=None): 857 | """ 2D heat map using convolution with a Gaussian kernel. 858 | Best for fixations or samples in screen pixel coordinates. 859 | 860 | Args: 861 | imageid (str): optional image ID to create map for one image only 862 | size (tuple): image dimensions, specified as (width, height). 863 | convolution (bool): if False, use a sparse addition approch to calculate 864 | heat map. Works if scipy is unavailable, faster if very few fixations 865 | dur (bool): if True, weight heat map by fixation durations if available 866 | size_x, size_y (int): horizontal/vertical kernel size, in pixels 867 | sigma_x, sigma_y (float): hor/vert kernel standard deviaton, in pixels 868 | normalize (bool): if True, return values in range 0..1 869 | threshold (float): threshold map by masking output array at a given value 870 | """ 871 | if size_y is None: 872 | size_y = size_x 873 | if sigma_y is None: 874 | sigma_y = sigma_x 875 | 876 | if dur and not self._fixdur in self.data.columns.to_list(): 877 | print('Warning: no fixation durations in dataset, cannot apply duration weighting (dur=True)') 878 | dur = False 879 | 880 | kernel = self._gauss2d(size_x, sigma_x, size_y, sigma_y) 881 | 882 | if not convolution: 883 | # Use manual sparse addition of kernels 884 | if imageid is not None: 885 | mapfix = self.select_fix({self._imageid: imageid}) 886 | else: 887 | mapfix = self 888 | 889 | if size is None: 890 | if self.imageset is not None: 891 | size = self.imageset.size 892 | else: 893 | raise ValueError('Image size or attached ImageSet are necessary for location mapping!') 894 | 895 | hmap = np.zeros((size[1]+2*size_x, size[0]+2*size_y)) 896 | hwx = int(round(size_x/2)) 897 | hwy = int(round(size_y/2)) 898 | 899 | fix = mapfix.data[(self.data[self._xpx] >= 0) & (self.data[self._xpx] < size[0]) & 900 | (self.data[self._ypx] >= 0) & (self.data[self._ypx] < size[1])] 901 | for r in fix.iterrows(): 902 | pos = (r[1][self._ypx], r[1][self._xpx]) 903 | xl = pos[1]+size_x-hwx 904 | xr = pos[1]+size_x+hwx 905 | yl = pos[0]+size_y-hwy 906 | yr = pos[0]+size_y+hwy 907 | if dur: 908 | hmap[yl:yr, xl:xr] += kernel * r[1][self._fixdur] 909 | else: 910 | hmap[yl:yr, xl:xr] += kernel 911 | heatmap = hmap[size_y:size[1]+size_y, size_x:size[0]+size_x] 912 | 913 | else: 914 | # Use scipy.signal.convolve (default) 915 | if dur: 916 | hmap = self.dur_map(imageid, size).astype(np.float64) 917 | else: 918 | hmap = self.count_map(imageid, size).astype(np.float64) 919 | heatmap = convolve(hmap, kernel, mode='same') 920 | 921 | if normalize: 922 | heatmap = (heatmap-heatmap.min()) / (heatmap.max()-heatmap.min()) 923 | if threshold is not None: 924 | heatmap = np.ma.masked_less_equal(heatmap, threshold) 925 | 926 | return heatmap 927 | 928 | 929 | 930 | def assign_regions(self, regionset, regionno=True, col_prefix=None): 931 | """ Assign each fixation the corresponding region from a RegionSet. New columns 932 | are added directly to the Fixation object's underlying DataFrame. 933 | 934 | Note: in case of overlapping regions, fixatiońs will be labeled as the higher 935 | region number (likely the region which appeared later in the RegionSet input file)! 936 | 937 | Args: 938 | regionset: RegionSet object to match against (must contain same imageids) 939 | regionno (bool): if True, add column for region number in addition to regionid string 940 | col_prefix (str): name prefix for new column names 941 | """ 942 | 943 | if not regionset.is_global and not all(iid in regionset.imageids for iid in self.imageids): 944 | raise ValueError('At least one imageid was not found in the specified RegionSet!') 945 | 946 | # Generate new unique column names using RegionSet label if available 947 | idcol = 'regionid' 948 | rcol = 'regionno' 949 | 950 | if col_prefix is not None: 951 | idcol = '{:s}_{:s}'.format(col_prefix, idcol) 952 | rcol = '{:s}_{:s}'.format(col_prefix, rcol) 953 | else: 954 | if regionset.label is not None: 955 | idcol = '{:s}_{:s}'.format(regionset.label, idcol) 956 | rcol = '{:s}_{:s}'.format(regionset.label, rcol) 957 | 958 | if idcol in self.data.columns: 959 | idcol_num = idcol 960 | num = 1 961 | while idcol_num in self.data.columns: 962 | idcol_num = '{:s}{:d}'.format(idcol, num) 963 | num += 1 964 | idcol = idcol_num 965 | 966 | if regionno and rcol in self.data.columns: 967 | rcol_num = rcol 968 | num = 1 969 | while rcol_num in self.data.columns: 970 | rcol_num = '{:s}{:d}'.format(rcol, num) 971 | num += 1 972 | rcol = rcol_num 973 | 974 | self.data.loc[:, idcol] = 'none' 975 | if regionno: 976 | self.data[rcol] = np.nan 977 | 978 | # Filter for out-of-bounds fixations 979 | valid_fix = ((self.data[self._xpx] >= 0) & (self.data[self._xpx] < regionset.size[0]) & 980 | (self.data[self._ypx] >= 0) & (self.data[self._ypx] < regionset.size[1])) 981 | 982 | if regionset.is_global: 983 | # Apply global region map to all fixation coordinates 984 | rmap = regionset.region_map(imageid='*', ignore_background=False) 985 | 986 | rnos = np.ones(self.data.shape[0]) * np.nan 987 | rnos[valid_fix] = rmap[self.data.loc[valid_fix, self._ypx], self.data.loc[valid_fix,self._xpx]] 988 | if regionno: 989 | self.data.loc[:, rcol] = rnos 990 | 991 | # Resolve regionids from DataFrame (not guaranteed to be sorted by regionno) 992 | for rn in np.unique(rnos[~np.isnan(rnos)]): 993 | self.data.loc[rnos == rn, idcol] = regionset.info.regionid[regionset.info.regionno == rn].values 994 | 995 | else: 996 | # Resolve regions for each imageid 997 | for imid in self.imageids: 998 | img_fix = valid_fix & (self.data[self._imageid] == imid) 999 | rmap = regionset.region_map(imageid=imid, ignore_background=False) 1000 | rnos = np.ones(self.data.shape[0]) * np.nan 1001 | rnos[img_fix] = rmap[self.data.loc[img_fix, self._ypx], self.data.loc[img_fix,self._xpx]] 1002 | 1003 | region_info = regionset.info[regionset.info.imageid == imid] 1004 | if regionno: 1005 | self.data.loc[img_fix, rcol] = rnos[img_fix] 1006 | for rn in np.unique(rnos[~np.isnan(rnos)]): 1007 | self.data.loc[img_fix & (rnos == rn), idcol] = region_info.regionid[region_info.regionno == rn].values 1008 | 1009 | 1010 | 1011 | class FixationModel(object): 1012 | """ Combines Features and Fixations to create predictors and R source for GLMM 1013 | 1014 | Attributes: 1015 | chunks (list): list of data columns that define chunks (e.g., subjects or sessions) 1016 | comp_features (dict): dict of labelled feature comparisons. Predictors will be replicated 1017 | for each feature in a comparison so that features can serve as fixed or random factor 1018 | exclude_first_fix (boolean): if True, first fixation index was set NaN (e.g., fixation cross) 1019 | exclude_last_fix (boolean): if True, last fixation index was set NaN (e.g., image offset) 1020 | features (dict): dictionary of feature objects and feature groups 1021 | predictors (DataFrame): model predictors for GLMM 1022 | regionset (RegionSet): attached RegionSet 1023 | runtime (float): time in seconds for most recent update of predictor matrix 1024 | normalize_features (bool): if True, all feature values are normalized to 0..1 range 1025 | """ 1026 | 1027 | def __init__(self, fixations, regionset, dv_type='fixated', features=None, feature_labels=None, 1028 | chunks=[], progress=False, exclude_first_fix=False, exclude_last_fix=False, 1029 | normalize_features=None): 1030 | """ Create a new FixationModel. 1031 | 1032 | Args: 1033 | fixations (Fixations): fixation data to use as model DV (column 'fixation') 1034 | regionset (RegionSet): a RegionSet object defining length of all features 1035 | dv_type (str): type of DV to generate, or list of multiple options: 1036 | 'fixated': binary coding of fixated (1) and unfixated (0) regions 1037 | 'count': absolute fixation count for each region 1038 | 'fixid': fixation ID of the first valid fixation within each region (NaN if not fixated) 1039 | 'passes': total number of viewing passes on this object (NaN if not fixated) 1040 | 'refix': total number of refixations (viewing passes after first pass; NaN if not fixated) 1041 | 'total': total fixation time for each region (NaN if not fixated) 1042 | 'first': first fixation duration per region (NaN if not fixated) 1043 | 'gaze': first-pass gaze duration per region (all initial fixations without refixations) 1044 | 'single': fixation duration if region was fixated exactly once (NaN if not fixated) 1045 | 'tofirst': start time of the first fixation on each region (NaN if not fixated) 1046 | features (list): list of Feature objects to add (use add_comparison for feature groups) 1047 | feature_labels (list): string labels to apply to features defined using features= attribute 1048 | chunks (list): list of fixation data columns that define chunks (e.g., subjects or sessions) 1049 | progress (bool): print current image and group variables to indicate model build progress 1050 | exclude_first_fix (bool): if True, set first fixated region per image to NaN for GLMM 1051 | exclude_last_fix (str): controls how to deal with regions receiving the last image fixation: 1052 | 'never' or False: do not handle the last fixation specially 1053 | 'always' or True: drop the entire region if it received the last fixation at any time 1054 | 'pass': exclude viewing pass (one or multiple fixations) that received the last fixation 1055 | normalize_features (bool): if True, normalize all feature values to 0..1 range 1056 | """ 1057 | self.regionset = regionset 1058 | 1059 | self._fix = fixations 1060 | self._pred = DataFrame() 1061 | self._consistent = False # Flags whether we need to rebuild predictor matrix 1062 | 1063 | self.features = {} 1064 | self.comp_features = {} 1065 | self.normalize_features = normalize_features 1066 | self.exclude_first_fix = exclude_first_fix 1067 | self.exclude_last_fix = exclude_last_fix 1068 | 1069 | # Supported DVs: labels and other model parameters 1070 | self._dvs = {'fixated': {'f':'c', 'rvar': 'dvFix', 'fun': 'glmer', 'family': 'binomial'}, 1071 | 'count': {'f':'c', 'rvar': 'dvCount', 'fun': 'glmer', 'family': 'poisson'}, 1072 | 'fixid': {'f':'c', 'rvar': 'fixID', 'fun': None, 'family': None}, 1073 | 'passes': {'f':'c', 'rvar': 'dvPasses', 'fun': 'glmer', 'family': 'poisson'}, 1074 | 'refix': {'f':'c', 'rvar': 'dvRefix', 'fun': 'glmer', 'family': 'poisson'}, 1075 | 'first': {'f':'t', 'rvar': 'dvFirst', 'fun': 'lmer', 'family': None}, 1076 | 'gaze': {'f':'t', 'rvar': 'dvGaze', 'fun': 'lmer', 'family': None}, 1077 | 'tofirst': {'f':'t', 'rvar': 'dvToFirst','fun': 'lmer', 'family': None}, 1078 | 'total': {'f':'t', 'rvar': 'dvTotal', 'fun': 'lmer', 'family': None}, 1079 | 'single': {'f':'t', 'rvar': 'dvSingle', 'fun': 'lmer', 'family': None} 1080 | } 1081 | 1082 | if type(dv_type) != list: 1083 | dv_type = [dv_type,] 1084 | for dvt in dv_type: 1085 | if dvt not in self._dvs.keys(): 1086 | raise ValueError('Error: unknown DV type specified: "{:s}"!'.format(dvt)) 1087 | self.dv_type = dv_type 1088 | 1089 | # Make sure imageid is always a chunking variable and all chunk vars exist 1090 | if self._fix._imageid not in chunks: 1091 | chunks.append(self._fix._imageid) 1092 | for var in chunks: 1093 | if var not in self._fix.variables: 1094 | raise ValueError('Error: chunking variable "{:s}" does not exist in dataset!'.format(var)) 1095 | 1096 | self.chunks = chunks 1097 | self.progress = progress 1098 | 1099 | # Add any specified features to model 1100 | if features: 1101 | if type(features) != list and type(features) != tuple: 1102 | features = [features,] # force list of features 1103 | for f in features: 1104 | if feature_labels is not None and len(feature_labels) > 0: 1105 | label = feature_labels.pop(0) 1106 | elif f.label is not None: 1107 | label = f.label 1108 | else: 1109 | label = self._feat_label(f) 1110 | self.add_feature(f, label=label) 1111 | 1112 | self.update(progress=self.progress) 1113 | 1114 | 1115 | def _feat_label(self, feature): 1116 | """ Create label for unlabeled Feature object.""" 1117 | cls = feature.__class__.__name__ 1118 | label = 'f' + cls[0:5] 1119 | 1120 | f_label = label 1121 | suffix = 1 1122 | 1123 | while f_label in self.features.keys(): 1124 | f_label = label + str(suffix) 1125 | suffix += 1 1126 | return f_label 1127 | 1128 | 1129 | @property 1130 | def predictors(self): 1131 | """ Model predictor matrix, updated if necessary """ 1132 | if not self._consistent: 1133 | self.update() 1134 | return self._pred 1135 | 1136 | 1137 | def __repr__(self): 1138 | """ String representation for print summary. """ 1139 | if not self._consistent: 1140 | self.update() 1141 | 1142 | r = '\n'.format(self.predictors.shape[0], str(self.dv_type), str(self.chunks)) 1143 | r += 'Fixations:\n\t{:s}\n'.format(str(self._fix)) 1144 | r += 'Regions:\n\t{:s}\n'.format(str(self.regionset)) 1145 | 1146 | if len(self.features) > 0: 1147 | r += '\nFeatures:\n' 1148 | for l,f in self.features.items(): 1149 | r += '\t{:s}\t{:s}\n'.format(l, f.__class__.__name__) 1150 | 1151 | if len(self.comp_features) > 0: 1152 | r += '\nFeature Comparisons:\n' 1153 | for l,f in self.comp_features.items(): 1154 | r += '{:s}:\n'.format(l) 1155 | for code, feat in f.items(): 1156 | r += '\t{:s}\t{:s}\n'.format(str(code), feat.__class__.__name__) 1157 | return(r) 1158 | 1159 | 1160 | def r_source(self, datafile='gridfix.csv', comments=True, scale=True, center=True, 1161 | optimizer=None, fixed=None, random=None, random_slopes=False): 1162 | """ Generate R source code from current feature settings. 1163 | 1164 | Args: 1165 | datafile (str): predictor matrix file name (for R import via read.table()) 1166 | comments (boolean): if True, add explanatory comments and headers to source 1167 | scale (boolean): if True, add code to scale (normalize) feature predictors 1168 | center (boolean): if True, add code to center (demean) feature predictors 1169 | optimizer (str): optional optimizer to pass to R glmer() 1170 | fixed (list): list of column names (strings) to add as fixed factors 1171 | random (list): list of column names (strings) to add as random factors 1172 | random_slopes (boolean): also add random slopes to generated R code 1173 | 1174 | Returns: 1175 | R source code as string 1176 | """ 1177 | r_libs = ['lme4'] 1178 | 1179 | src = '' 1180 | if comments: 1181 | d = time.strftime('%d.%m.%y, %H:%M:%S', time.localtime()) 1182 | src = '# GridFix GLMM R source, generated on {:s}\n# \n'.format(d) 1183 | src += '# Predictor file:\t{:s}\n'.format(datafile) 1184 | src += '# Fixations file:\t{:s}\n'.format(str(self._fix.input_file)) 1185 | src += '# RegionSet:\t\t{:s}\n'.format(str(self.regionset)) 1186 | src += '# DV type(s):\t\t{:s}\n'.format(str(self.dv_type)) 1187 | src += '\n' 1188 | 1189 | # Libraries 1190 | for lib in r_libs: 1191 | src += 'library({:s})\n'.format(lib) 1192 | src += '\n' 1193 | 1194 | # Predictor file 1195 | src += 'gridfixdata <- read.table("{:s}", header=T, sep="\\t", row.names=NULL)\n\n'.format(datafile) 1196 | 1197 | # Factors 1198 | if comments: 1199 | src += '# Define R factors for all chunking variables and group dummy codes\n' 1200 | for chunk in self.chunks: 1201 | src += 'gridfixdata${:s} <- as.factor(gridfixdata${:s})\n'.format(chunk, chunk) 1202 | if len(self.comp_features) > 0: 1203 | for cf in self.comp_features.keys(): 1204 | src += 'gridfixdata${:s} <- as.factor(gridfixdata${:s})\n'.format(cf, cf) 1205 | src += '\n' 1206 | 1207 | # Center and scale 1208 | if scale or center: 1209 | if len(self.features) > 0: 1210 | r_cent = 'FALSE' 1211 | r_scal = 'FALSE' 1212 | if center: 1213 | r_cent = 'TRUE' 1214 | if scale: 1215 | r_scal = 'TRUE' 1216 | if comments: 1217 | src += '# Center and scale predictors\n' 1218 | for f in self.features.keys(): 1219 | src += 'gridfixdata${:s}_C <- scale(gridfixdata${:s}, center={:s}, scale={:s})\n'.format(f, f, r_cent, r_scal) 1220 | src += '\n' 1221 | 1222 | # GLMM model formula (DV is set later) 1223 | formula = '{:s} ~ 1' 1224 | 1225 | if fixed is None: 1226 | # Best guess: all simple features should be fixed factors! 1227 | if len(self.features) > 0: 1228 | fixed = self.features.keys() 1229 | else: 1230 | fixed = [] 1231 | 1232 | fixed_vars = '' 1233 | for f in fixed: 1234 | try: 1235 | # likely a feature object 1236 | fl = f.label 1237 | except AttributeError: 1238 | # text label specified 1239 | fl = f 1240 | if scale or center: 1241 | fixed_vars += ' + {:s}_C '.format(fl) 1242 | else: 1243 | fixed_vars += ' + {:s} '.format(fl) 1244 | formula += fixed_vars 1245 | 1246 | if random is None: 1247 | # imageid should be a random factor by default 1248 | random = [self._fix._imageid] 1249 | 1250 | if len(fixed) > 0: 1251 | for r in random: 1252 | try: 1253 | # likely a feature object 1254 | rl = r.label 1255 | except AttributeError: 1256 | rl = r 1257 | if random_slopes: 1258 | formula += ' + (1{:s} | {:s})'.format(fixed_vars, rl) 1259 | else: 1260 | formula += ' + (1 | {:s})'.format(rl) 1261 | 1262 | # Optimizer parameter 1263 | opt_call = '' 1264 | if optimizer is not None: 1265 | opt_call = ', control=glmerControl(optimizer="{:s}")'.format(optimizer) 1266 | 1267 | if comments: 1268 | src += '# NOTE: this source code can only serve as a scaffolding for your own analysis!\n' 1269 | src += '# You MUST adapt the GLMM model formula below to your model, then uncomment the corresponding line!\n' 1270 | 1271 | # GLMM model call(s) - one per requested DV 1272 | models = [] 1273 | for current_dv in self.dv_type: 1274 | if self._dvs[current_dv]['fun'] is None: 1275 | # Don't generate R code for a DV with no model function specified 1276 | continue 1277 | 1278 | models.append('model.{:s}'.format(current_dv)) 1279 | 1280 | if comments: 1281 | src += '# DV: {:s}\n#'.format(current_dv) 1282 | 1283 | src += 'model.{:s} <- {:s}({:s}, data=gridfixdata{:s}'.format(current_dv, self._dvs[current_dv]['fun'], formula.format(self._dvs[current_dv]['rvar']), opt_call) 1284 | if self._dvs[current_dv]['family'] is not None: 1285 | src += ', family={:s}'.format(self._dvs[current_dv]['family']) 1286 | src += ')\n\n' 1287 | 1288 | out_f, ext = os.path.splitext(datafile) 1289 | r_objlist = ','.join(['"{:s}"'.format(a) for a in models]) 1290 | src += 'save(file="{}_GLMM.Rdata", list = c({:s}))\n\n'.format(out_f, r_objlist) 1291 | src += 'print(summary(model))\n' 1292 | return src 1293 | 1294 | 1295 | def _process_chunk(self, chunk_vals, data, pred_columns, group_levels): 1296 | """ Process a single data chunk. """ 1297 | if data is not None: 1298 | sel_labels = dict(zip(self.chunks, chunk_vals)) 1299 | imageid = str(sel_labels[self._fix._imageid]) 1300 | tmpdf = DataFrame(columns=pred_columns, index=range(len(self.regionset[imageid]))) 1301 | 1302 | # groupby returns a single string if only one chunk columns is selected 1303 | if type(chunk_vals) != tuple: 1304 | chunk_vals = (chunk_vals,) 1305 | 1306 | # Fixations 1307 | subset = self._fix.select_fix(sel_labels) 1308 | 1309 | # Chunk and region values 1310 | for col in self.chunks: 1311 | tmpdf[col] = data[col].iloc[0] 1312 | 1313 | # Region ID and numbering 1314 | if self.regionset.is_global: 1315 | tmpdf.regionid = np.array(self.regionset.info[self.regionset.info.imageid == '*'].regionid, dtype=str) 1316 | tmpdf.regionno = np.array(self.regionset.info[self.regionset.info.imageid == '*'].regionno, dtype=int) 1317 | else: 1318 | tmpdf.regionid = np.array(self.regionset.info[self.regionset.info.imageid == imageid].regionid, dtype=str) 1319 | tmpdf.regionno = np.array(self.regionset.info[self.regionset.info.imageid == imageid].regionno, dtype=int) 1320 | 1321 | # Dependent Variables (DVs) 1322 | for dv in self._dvs.keys(): 1323 | if dv in self.dv_type: 1324 | if self._dvs[dv]['f'] == 'c': 1325 | # Fixation count based measures 1326 | tmpdf[self._dvs[dv]['rvar']] = self.regionset.fixated(subset, var=dv, imageid=imageid, exclude_first=self.exclude_first_fix, 1327 | exclude_last=self.exclude_last_fix) 1328 | 1329 | if self._dvs[dv]['f'] == 't' and subset.has_times: 1330 | # Fixation time based measures 1331 | tmpdf[self._dvs[dv]['rvar']] = self.regionset.fixtimes(subset, var=dv, imageid=imageid, exclude_first=self.exclude_first_fix, 1332 | exclude_last=self.exclude_last_fix) 1333 | 1334 | # Simple per-image features 1335 | for feat_col, feat in self.features.items(): 1336 | tmpdf[feat_col] = feat.apply(imageid, normalize=self.normalize_features) 1337 | 1338 | # Feature group comparisons 1339 | if len(self.comp_features) > 0: 1340 | for levels in group_levels: 1341 | for idx, gc in enumerate(self.comp_features.keys()): 1342 | tmpdf[gc] = levels[idx] 1343 | tmpdf['{:s}_val'.format(gc)] = self.comp_features[gc][levels[idx]].apply(imageid, normalize=self.normalize_features) 1344 | 1345 | return tmpdf 1346 | 1347 | 1348 | def update(self, progress=False): 1349 | """ Update predictor matrix from features (this may take a while). 1350 | 1351 | Args: 1352 | progress (boolean): if True, print model creation progress 1353 | """ 1354 | ts = time.time() 1355 | 1356 | # Output DF columns 1357 | pred_columns = self.chunks + ['regionid', 'regionno', 'dvFix'] 1358 | pred_columns += list(self.features.keys()) 1359 | for cf in self.comp_features.keys(): 1360 | pred_columns += [cf, '{:s}_val'.format(cf)] 1361 | 1362 | pred_new = DataFrame(columns=pred_columns) 1363 | 1364 | splitdata = self._fix.data.groupby(self.chunks) 1365 | 1366 | group_levels = [] 1367 | if len(self.comp_features) > 0: 1368 | groups = [list(f.keys()) for i,f in self.comp_features.items()] 1369 | group_levels = list(itertools.product(*groups)) 1370 | 1371 | # Process individual chunks 1372 | for chunk_vals, data in splitdata: 1373 | if progress: 1374 | print(chunk_vals) 1375 | results = self._process_chunk(chunk_vals, data, pred_columns, group_levels) 1376 | pred_new = pred_new.append(results) 1377 | 1378 | self._pred = pred_new 1379 | self._consistent = True 1380 | self.runtime = time.time() - ts # rebuild duration 1381 | 1382 | 1383 | def add_feature(self, feature, label=None): 1384 | """ Add a feature to the model. 1385 | 1386 | Args: 1387 | feature (Feature): Feature object to add 1388 | label (str): label and output column name for this feature 1389 | """ 1390 | # Generate unique feature label 1391 | if label is None: 1392 | label = self._feat_label(feature) 1393 | 1394 | # Check feature length 1395 | if len(feature) != len(self.regionset): 1396 | w = 'Could not add feature "{:s}": invalid length ({:d} instead of {:d})!' 1397 | raise ValueError(label, len(feature), len(self.regionset)) 1398 | 1399 | self.features[label] = feature 1400 | self._consistent = False 1401 | 1402 | 1403 | def add_comparison(self, features, codes=None, label=None): 1404 | """ Add a feature comparison group to the model. 1405 | 1406 | This generates a long-style predictor matrix for the specified features, 1407 | needed to compare e.g. different saliency maps in their relative effects. 1408 | 1409 | Args: 1410 | features (list): list of Feature objects to combine into a group 1411 | codes (list): numeric codes to use in "dummy coding", e.g. [0, 1, 2] 1412 | label (str): label and output column name for this feature group 1413 | """ 1414 | # Generate unique group label 1415 | if label is None: 1416 | suffix = 1 1417 | g_label = 'fC' + str(suffix) 1418 | 1419 | while g_label in self.comp_features.keys(): 1420 | suffix += 1 1421 | g_label = 'fC' + str(suffix) 1422 | 1423 | # Generate dummy codes if necessary 1424 | if codes is None: 1425 | codes = range(0, len(features)) 1426 | else: 1427 | codes = [int(c) for c in codes] 1428 | 1429 | comp = {codes[c]: f for c,f in enumerate(features)} 1430 | self.comp_features[g_label] = comp 1431 | self._consistent = False 1432 | 1433 | 1434 | def save(self, basename, sep='\t', pred=True, pred_pickle=False, src=True, src_comments=True, precision=10, 1435 | optimizer=None, fixed=None, random=None, random_slopes=False): 1436 | """ Saves the predictor matrix to a CSV text file. 1437 | 1438 | Args: 1439 | basename (str): base filename to save to, without extension 1440 | sep (str): item separator, default TAB 1441 | pred (boolean): if True, output predictor matrix as CSV 1442 | pred_pickle (boolean): if True, also save predictors to Pickle object 1443 | src (boolean): if True, output r source code file for lme4 1444 | src_comments (boolean): if True, add comments to source code 1445 | precision (int): number of decimal places for CSV (default: 10) 1446 | optimizer (str): optional optimizer to pass to R glmer() 1447 | fixed (list): list of column names (strings) to add as fixed factors 1448 | random (list): list of column names (strings) to add as random factors 1449 | random_slopes (boolean): also add random slopes to generated R code 1450 | 1451 | """ 1452 | if not self._consistent: 1453 | self.update() 1454 | 1455 | if pred: 1456 | if LooseVersion(pandas_version) >= LooseVersion('0.17.1'): 1457 | # compression supported from 0.17.1 1458 | f_pred = '{:s}.csv.gz'.format(basename) 1459 | self.predictors.to_csv(f_pred, sep, index=False, float_format='%.{:d}f'.format(precision), compression='gzip') 1460 | else: 1461 | f_pred = '{:s}.csv'.format(basename) 1462 | self.predictors.to_csv(f_pred, sep, index=False, float_format='%.{:d}f'.format(precision)) 1463 | 1464 | if pred_pickle: 1465 | f_pred = '{:s}.pkl'.format(basename) 1466 | self.predictors.to_pickle(f_pred) 1467 | 1468 | if src: 1469 | f_src = '{:s}.R'.format(basename) 1470 | src = self.r_source(comments=src_comments, datafile=f_pred, optimizer=optimizer, 1471 | fixed=fixed, random=random, random_slopes=random_slopes) 1472 | with open(f_src, 'w') as sf: 1473 | sf.write(src) 1474 | 1475 | 1476 | if __name__ == '__main__': 1477 | print('The gridfix modules cannot be called directly. Please use one of the included tools, e.g. gridmap.') 1478 | -------------------------------------------------------------------------------- /gridfix/regionset.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | import os 5 | 6 | import numpy as np 7 | import matplotlib as mp 8 | import matplotlib.pyplot as plt 9 | from matplotlib.patches import Rectangle 10 | 11 | from PIL import Image 12 | 13 | from fractions import Fraction 14 | from pandas import DataFrame, read_table 15 | 16 | from .model import ImageSet 17 | 18 | 19 | class RegionSet(object): 20 | """ Base class for sets of image regions of interest. 21 | 22 | RegionSets can be used to spatially group fixations, create Feature objects 23 | for a FixationModel and split an image into parts. Classes inheriting from 24 | RegionSet may specify functions to create regions. 25 | 26 | Attributes: 27 | info (DataFrame): table of region metadata (labels, bboxes, number of pixels...) 28 | imageids (list): list of all imageids associated with this RegionSet 29 | is_global (bool): True if regions are global (non-image-specific) 30 | label (str): optional label to distinguish between RegionSets. 31 | memory_usage (float): memory usage of all binary masks (kiB) 32 | size (tuple): image dimensions, specified as (width, height). 33 | """ 34 | 35 | def __init__(self, size, regions, region_labels=None, label=None, add_background=False): 36 | """ Create a new RegionSet from existing region masks. 37 | 38 | Args: 39 | size (tuple): image dimensions, specified as (width, height) 40 | regions: 3d ndarray (bool) with global set of masks 41 | OR dict of multiple such ndarrays, with imageids as keys 42 | region_labels: list of region labels IF _regions_ is a single array 43 | OR dict of such lists, with imageids as keys 44 | label (str): optional descriptive label for this RegionSet 45 | add_background (bool): if True, this creates a special region to capture all 46 | fixations that don't fall on an explicit region ("background" fixations) 47 | 48 | Raises: 49 | ValueError if incorrectly formatted regions/region_labels provided 50 | """ 51 | self._regions = {'*': np.ndarray((0,0,0))} 52 | self._labels = {'*': []} 53 | 54 | self.size = size 55 | self.label = label 56 | self._msize = (size[1], size[0]) # matrix convention 57 | self.has_background = False 58 | 59 | if isinstance(regions, dict): 60 | # Dict with image-specific region ndarrays 61 | self._regions = regions 62 | if region_labels is not None and isinstance(region_labels, dict) and len(regions) == len(region_labels): 63 | # Check imageids for consistency 64 | for r in regions.keys(): 65 | if r not in region_labels.keys(): 66 | raise ValueError('Labels not consistent: {:s} not in region_labels'.format(r)) 67 | for r in region_labels.keys(): 68 | if r not in regions.keys(): 69 | raise ValueError('Labels not consistent: {:s} not in regions'.format(r)) 70 | self._labels = region_labels 71 | else: 72 | self._labels = {} 73 | for imid in regions: 74 | self._labels[imid] = [str(x+1) for x in range(len(regions[imid]))] 75 | 76 | elif isinstance(regions, np.ndarray): 77 | # Single array of regions - assume global region set ('*') 78 | if regions.shape[1:] == self._msize: 79 | self._regions['*'] = regions.astype(bool) 80 | if region_labels is not None and len(region_labels) == regions.shape[0]: 81 | self._labels['*'] = region_labels 82 | else: 83 | self._labels['*'] = [str(x+1) for x in range(regions.shape[0])] 84 | 85 | else: 86 | raise ValueError('First argument for RegionSet creation must be ndarray ' + 87 | '(global regions) or dict of ndarrays (image-specific regions)!') 88 | 89 | if add_background: 90 | for iid in self._regions.keys(): 91 | bgmask = ~self.mask(iid).reshape(1, size[1], size[0]) 92 | self._regions[iid] = np.concatenate([self._regions[iid], bgmask], axis=0) 93 | self._labels[iid].append('__BG__') 94 | self.has_background = True 95 | 96 | self.info = self._region_metadata() 97 | 98 | 99 | def __repr__(self): 100 | """ String representation """ 101 | r = 'gridfix.RegionSet(label={:s}, size=({:d}, {:d}),\nregions={:s},\nregion_labels={:s})' 102 | return r.format(str(self.label), self.size[0], self.size[1], str(self._regions), str(self._labels)) 103 | 104 | 105 | def __str__(self): 106 | """ Short string representation for printing """ 107 | r = '<{:s}{:s}, size={:s}, {:d} region{:s}{:s}, memory={:.1f} kB>' 108 | 109 | myclass = str(self.__class__.__name__) 110 | if self.label is not None: 111 | lab = ' ({:s})'.format(self.label) 112 | else: 113 | lab = '' 114 | 115 | num_s = '' 116 | num_r = len(self) 117 | if num_r > 1: 118 | num_s = 's' 119 | 120 | imid_s = '' 121 | if len(self._regions) > 1 and not self.is_global: 122 | imid_s = ' in {:d} images'.format(len(self._regions)) 123 | 124 | return r.format(myclass, lab, str(self.size), num_r, num_s, imid_s, self.memory_usage) 125 | 126 | 127 | def __len__(self): 128 | """ Overload len(RegionSet) to report total number of regions. """ 129 | if self.is_global: 130 | return len(self._regions['*']) 131 | else: 132 | num_r = 0 133 | for imid in self._regions: 134 | num_r += len(self._regions[imid]) 135 | return num_r 136 | 137 | 138 | def __getitem__(self, imageid): 139 | """ Bracket indexing returns all region masks for a specified imageid. 140 | If global regions are set ('*'), always return global region set. 141 | """ 142 | return self._select_region(imageid) 143 | 144 | 145 | def _region_metadata(self): 146 | """ Return DataFrame of region metadata """ 147 | info_cols = ['imageid', 'regionid', 'regionno', 'left', 'top', 'right', 'bottom', 'width', 'height', 'area', 'imgfrac'] 148 | info = [] 149 | 150 | if self.is_global: 151 | imageids = ['*'] 152 | else: 153 | imageids = self.imageids 154 | 155 | for imid in imageids: 156 | reg = self._select_region(imid) 157 | lab = self._select_labels(imid) 158 | 159 | for i,l in enumerate(lab): 160 | a = np.argwhere(reg[i]) 161 | if a.shape[0] > 0: 162 | (top, left) = a.min(0)[0:2] 163 | (bottom, right) = a.max(0)[0:2] 164 | (width, height) = (right-left+1, bottom-top+1) 165 | area = reg[i][reg[i] > 0].sum() 166 | imgfrac = round(area / (reg[i].shape[0] * reg[i].shape[1]), 4) 167 | else: 168 | # Region is empty - shouldn't, but can happen with add_background at full coverage 169 | (top, left, bottom, right, width, height, area, imgfrac) = (0,) * 8 170 | 171 | rmeta = [imid, l, i+1, left, top, right, bottom, width, height, area, imgfrac] 172 | info.append(rmeta) 173 | 174 | return DataFrame(info, columns=info_cols) 175 | 176 | 177 | def _select_region(self, imageid=None): 178 | """ Select region by imageid with consistency check """ 179 | if self.is_global: 180 | return(self._regions['*']) 181 | 182 | if imageid is not None and imageid in self._regions.keys(): 183 | return(self._regions[imageid]) 184 | else: 185 | raise ValueError('RegionSet contains image-specific regions, but no valid imageid was specified!') 186 | 187 | 188 | def _select_labels(self, imageid=None): 189 | """ Select region labels corresponding to _select_region """ 190 | if self.is_global: 191 | return(self._labels['*']) 192 | 193 | if imageid is not None and imageid in self._regions.keys(): 194 | return(self._labels[imageid]) 195 | else: 196 | raise ValueError('RegionSet contains image-specific regions, but no valid imageid was specified!') 197 | 198 | 199 | @property 200 | def is_global(self): 201 | """ Return True if a global map is defined (key '*') """ 202 | if '*' in self._regions.keys(): 203 | return True 204 | else: 205 | return False 206 | 207 | 208 | @property 209 | def imageids(self): 210 | """ Return list of imageids for which region maps exist """ 211 | if self.is_global: 212 | return [] 213 | 214 | imids = [] 215 | for imid in self._regions.keys(): 216 | imids.append(imid) 217 | return imids 218 | 219 | 220 | @property 221 | def memory_usage(self): 222 | """ Calculate size in memory of all regions combined """ 223 | msize = 0.0 224 | for reg in self._regions.keys(): 225 | msize += float(self._regions[reg].nbytes) / 1024.0 226 | return msize 227 | 228 | 229 | def count_map(self, imageid=None, ignore_background=True): 230 | """ Return the number of regions referencing each pixel. 231 | 232 | Args: 233 | imageid (str): if set, return map for specified image only 234 | ignore_background (bool): if True, ignore auto-generated background region 235 | 236 | Returns: 237 | 2d ndarray of image size, counting number of regions for each pixel 238 | """ 239 | 240 | cm = np.zeros(self._msize, dtype=int) 241 | 242 | if self.is_global: 243 | for reidx, re in enumerate(self._regions['*'][:, ...]): 244 | if ignore_background and self._labels['*'][reidx] == '__BG__': 245 | continue 246 | cm += re.astype(int) 247 | return cm 248 | 249 | elif imageid is None: 250 | for imid in self._regions: 251 | if imid == '*': 252 | continue 253 | for reidx, re in enumerate(self._regions[imid][:, ...]): 254 | if ignore_background and self._labels[imid][reidx] == '__BG__': 255 | continue 256 | cm += re.astype(int) 257 | 258 | else: 259 | r = self._select_region(imageid) 260 | l = self._select_labels(imageid) 261 | for reidx, re in enumerate(r[:, ...]): 262 | if ignore_background and l[reidx] == '__BG__': 263 | continue 264 | cm += re.astype(int) 265 | 266 | return cm 267 | 268 | 269 | def mask(self, imageid=None, ignore_background=True): 270 | """ Return union mask of all regions or regions for specified image. 271 | 272 | Args: 273 | imageid (str): if set, return mask for specified image only 274 | ignore_background (bool): if True, ignore auto-generated background region 275 | 276 | Returns: 277 | 2d ndarray of image size (bool), True where at least one region 278 | references the corresponding pixel. 279 | """ 280 | return self.count_map(imageid, ignore_background).astype(bool) 281 | 282 | 283 | def region_map(self, imageid=None, ignore_background=True): 284 | """ Return map of region numbers, global or image-specifid. 285 | 286 | Args: 287 | imageid (str): if set, return map for specified image only 288 | ignore_background (bool): if True, ignore auto-generated background region 289 | 290 | Returns: 291 | 2d ndarray (int), containing the number (ascending) of the last 292 | region referencing the corresponding pixel. 293 | """ 294 | apply_regions = self._select_region(imageid) 295 | apply_labels = self._select_labels(imageid) 296 | tmpmap = np.zeros(self._msize) 297 | for idx, region in enumerate(apply_regions): 298 | if ignore_background and apply_labels[idx] == '__BG__': 299 | continue 300 | tmpmap[region] = (idx + 1) 301 | return tmpmap 302 | 303 | 304 | def coverage(self, imageid=None, normalize=False, ignore_background=True): 305 | """ Calculates coverage of the total image size as a scalar. 306 | 307 | Args: 308 | imageid (str): if set, return coverage for specified image only 309 | normalize (bool): if True, divide global result by number of imageids in set. 310 | ignore_background (bool): if True, ignore auto-generated background region 311 | 312 | Returns: 313 | Total coverage as a floating point number. 314 | """ 315 | if imageid is not None: 316 | counts = self.count_map(imageid, ignore_background) 317 | cov = float(counts.sum()) / float(self.size[0] * self.size[1]) 318 | return cov 319 | else: 320 | # Global coverage for all imageids 321 | cm = np.zeros(self._msize, dtype=int) 322 | for re in self._regions.keys(): 323 | if re == '*': 324 | cm += self.count_map('*', ignore_background) 325 | break 326 | cm += self.count_map(re, ignore_background) 327 | 328 | cov = float(cm.sum()) / float(self.size[0] * self.size[1]) 329 | if normalize: 330 | cov = cov / len(self) 331 | return cov 332 | 333 | 334 | def plot(self, imageid=None, values=None, cmap=None, image_only=False, ax=None, alpha=1.0): 335 | """ Plot regions as map of shaded areas with/without corresponding feature values 336 | 337 | Args: 338 | imageid (str): if set, plot regions for specified image 339 | values (array-like): one feature value per region 340 | cmap (str): name of matplotlib colormap to use to distinguish regions 341 | image_only (boolean): if True, return only image content without axes 342 | ax (Axes): axes object to draw to, to include result in other figure 343 | alpha (float): opacity of plotted regions (set < 1 to visualize overlap) 344 | 345 | Returns: 346 | matplotlib figure object, or None if passed an axis to draw on 347 | """ 348 | apply_regions = self._select_region(imageid) 349 | tmpmap = np.zeros(self._msize) 350 | 351 | if ax is not None: 352 | ax1 = ax 353 | else: 354 | fig = plt.figure() 355 | ax1 = fig.add_subplot(1,1,1) 356 | 357 | if cmap is None: 358 | if values is None and 'viridis' in plt.colormaps(): 359 | cmap = 'viridis' 360 | else: 361 | cmap = 'gray' 362 | 363 | if type(cmap) == str: 364 | cmap = plt.get_cmap(cmap) 365 | 366 | if alpha < 1.0: 367 | # allow stacking by setting masked values transparent 368 | alpha_cmap = cmap 369 | alpha_cmap.set_bad(alpha=0) 370 | 371 | ax1.imshow(tmpmap, cmap=plt.get_cmap('gray'), interpolation='none') 372 | 373 | for idx, region in enumerate(apply_regions): 374 | rmap = np.zeros(self._msize) 375 | if values is not None and len(values) == apply_regions.shape[0]: 376 | rmap[region] = values[idx] 377 | ax1.imshow(np.ma.masked_equal(rmap, 0), cmap=alpha_cmap, interpolation='none', alpha=alpha, 378 | vmin=0, vmax=np.nanmax(values)) 379 | 380 | else: 381 | rmap[region] = idx + 1 382 | ax1.imshow(np.ma.masked_equal(rmap, 0), cmap=alpha_cmap, interpolation='none', alpha=alpha, 383 | vmin=0, vmax=apply_regions.shape[0]) 384 | 385 | else: 386 | # If no alpha requested, this is much faster but doesn't show overlap 387 | ax1.imshow(tmpmap, cmap=plt.get_cmap('gray'), interpolation='none') 388 | 389 | if values is not None and len(values) == apply_regions.shape[0]: 390 | rmap = np.zeros(self._msize) 391 | for idx, region in enumerate(apply_regions): 392 | rmap[region] = values[idx] 393 | ax1.imshow(np.ma.masked_equal(rmap, 0), cmap=cmap, interpolation='none', vmin=0, vmax=np.nanmax(values)) 394 | else: 395 | ax1.imshow(np.ma.masked_equal(self.region_map(imageid), 0), cmap=cmap, interpolation='none', 396 | vmin=0, vmax=apply_regions.shape[0]) 397 | 398 | if image_only: 399 | ax1.axis('off') 400 | 401 | else: 402 | t = '{:s}'.format(self.__class__.__name__) 403 | if self.label is not None: 404 | t += ': {:s}'.format(self.label) 405 | if imageid is not None: 406 | t += ' (img: {:s})'.format(imageid) 407 | ax1.set_title(t) 408 | 409 | if ax is None and not plt.isinteractive(): # see ImageSet.plot() 410 | return fig 411 | 412 | 413 | def plot_regions_on_image(self, imageid=None, imageset=None, image_cmap=None, cmap=None, plotcolor=None, 414 | fill=False, alpha=0.4, labels=False, image_only=False, ax=None): 415 | """ Plot region bounding boxes on corresponding image 416 | 417 | Args: 418 | imageid (str): if set, plot regions for specified image 419 | imageset (ImageSet): ImageSet object containing background image/map 420 | image_cmap (str): name of matplotlib colormap to use for image 421 | cmap (str): name of matplotlib colormap to use for bounding boxes 422 | plotcolor (color): matplotlib color for bboxes (overrides colormap) 423 | fill (boolean): draw shaded filled rectangles instead of boxes 424 | alpha (float): rectangle opacity (only when fill=True) 425 | labels (boolean): if True, draw text labels next to regions 426 | image_only (boolean): if True, return only image content without axes 427 | ax (Axes): axes object to draw to, to include result in other figure 428 | 429 | Returns: 430 | matplotlib figure object, or None if passed an axis to draw on 431 | """ 432 | if imageset is None or imageid not in imageset.imageids: 433 | raise ValueError('To plot regions on top of image, specify ImageSet containing corresponding background image!') 434 | 435 | if ax is not None: 436 | ax1 = ax 437 | else: 438 | fig = plt.figure() 439 | ax1 = fig.add_subplot(1,1,1) 440 | 441 | if image_cmap is not None: 442 | if type(image_cmap) == str: 443 | image_cmap = plt.get_cmap(image_cmap) 444 | ax1.imshow(imageset[imageid], cmap=image_cmap, interpolation='none') 445 | else: 446 | ax1.imshow(imageset[imageid], interpolation='none') 447 | 448 | if cmap is None: 449 | if 'viridis' in plt.colormaps(): 450 | cmap = 'viridis' 451 | else: 452 | cmap = 'hsv' 453 | 454 | if type(cmap) == str: 455 | boxcolors = plt.get_cmap(cmap) 456 | else: 457 | boxcolors = cmap 458 | cstep = 0 459 | 460 | if self.is_global: 461 | rmeta = self.info[self.info.imageid == '*'] 462 | else: 463 | rmeta = self.info[self.info.imageid == imageid] 464 | 465 | for idx, region in rmeta.iterrows(): 466 | if self.has_background and region.regionid == '__BG__': 467 | # Always skip background region when drawing bboxes 468 | continue 469 | if plotcolor is None: 470 | c = boxcolors(cstep/len(rmeta)) 471 | else: 472 | c = plotcolor 473 | cstep += 1 474 | if not fill: 475 | ax1.add_patch(Rectangle((region.left, region.top), region.width, region.height, color=c, fill=False, linewidth=2)) 476 | else: 477 | ax1.add_patch(Rectangle((region.left, region.top), region.width, region.height, color=c, linewidth=0, alpha=0.7)) 478 | if labels: 479 | # Draw text labels with sensible default positions 480 | if region.right > (self.size[0] * .95): 481 | tx = region.right 482 | ha = 'right' 483 | else: 484 | tx = region.left 485 | ha = 'left' 486 | if region.bottom > (self.size[1] * .95): 487 | ty = region.top - 5 488 | else: 489 | ty = region.bottom + 20 490 | ax1.text(tx, ty, region.regionid, horizontalalignment=ha) 491 | 492 | if image_only: 493 | ax1.axis('off') 494 | 495 | else: 496 | t = '{:s}'.format(self.__class__.__name__) 497 | if self.label is not None: 498 | t += ': {:s}'.format(self.label) 499 | t += ' (img: {:s})'.format(imageid) 500 | ax1.set_title(t) 501 | 502 | if ax is None and not plt.isinteractive(): # see ImageSet.plot() 503 | return fig 504 | 505 | 506 | def apply(self, image, imageid=None, crop=False, ignore_background=True): 507 | """ Apply this RegionSet to a specified image. 508 | 509 | Returns a list of the image arrays "cut out" by each region mask, with 510 | non-selected image areas in black. If regionset is not global, _imageid_ needs 511 | to be specified! 512 | 513 | Args: 514 | image (ndarray): image array to be segmented. 515 | imageid (str): valid imageid (to select image-specific regions if not a global regionset) 516 | crop (bool): if True, return image cropped to bounding box of selected area 517 | ignore_background (bool): if True, ignore auto-generated background region 518 | 519 | Returns: 520 | If crop=False, a list of ndarrays of same size as image, with non-selected areas 521 | zeroed. Else a list of image patch arrays cropped to bounding box size. 522 | """ 523 | slices = [] 524 | apply_regions = self._select_region(imageid) 525 | apply_labels = self._select_labels(imageid) 526 | 527 | for idx, region in enumerate(apply_regions): 528 | if ignore_background and apply_labels[idx] == '__BG__': 529 | continue 530 | mask = (region == True) 531 | out = np.zeros(image.shape) 532 | out[mask] = image[mask] 533 | 534 | if crop: 535 | a = np.argwhere(out) 536 | if a.shape[0] > 0: 537 | (ul_x, ul_y) = a.min(0)[0:2] 538 | (br_x, br_y) = a.max(0)[0:2] 539 | out = out[ul_x:br_x+1, ul_y:br_y+1] 540 | slices.append(out) 541 | 542 | return slices 543 | 544 | 545 | def export_patches(self, image, imageid=None, crop=True, image_format='png', 546 | rescale=False, ignore_background=True): 547 | """ Apply this RegionSet to an image array and save the resulting image patches as files. 548 | 549 | Saves an image of each image part "cut out" by each region mask, cropped by default. 550 | If the RegionSet is not global, imageid needs to be specified! 551 | 552 | Args: 553 | image (ndarray): image array to be segmented. 554 | imageid (str): imageid (to select image-specific regions if not a global regionset) 555 | crop (bool): if True, return image cropped to bounding box of selected area 556 | image_format (str): image format that PIL understands (will also be used for extension) 557 | rescale (bool): if True, scale pixel values to full 0..255 range 558 | before saving (e.g., for saliency maps) 559 | ignore_background (bool): if True, ignore auto-generated background region 560 | 561 | """ 562 | apply_regions = self._select_region(imageid) 563 | apply_labels = self._select_labels(imageid) 564 | imstr = '{:s}_{:s}.{:s}' 565 | 566 | for idx, region in enumerate(apply_regions): 567 | if ignore_background and apply_labels[idx] == '__BG__': 568 | continue 569 | 570 | mask = (region == True) 571 | out = np.zeros(image.shape) 572 | out[mask] = image[mask] 573 | 574 | if crop: 575 | a = np.argwhere(out) 576 | if a.shape[0] > 0: 577 | (ul_x, ul_y) = a.min(0)[0:2] 578 | (br_x, br_y) = a.max(0)[0:2] 579 | out = out[ul_x:br_x+1, ul_y:br_y+1] 580 | 581 | if imageid is None or imageid == '*': 582 | imageid = 'image' 583 | 584 | if rescale: 585 | out = (out - out.min()) / out.max() * 255.0 586 | else: 587 | out *= 255.0 588 | 589 | rimg = Image.fromarray(np.array(out, np.uint8)) 590 | rimg.save(imstr.format(str(imageid), str(apply_labels[idx]), image_format), image_format) 591 | 592 | 593 | def export_patches_from_set(self, imageset, crop=True, image_format='png', rescale=False, ignore_background=True): 594 | """ Save all sliced image patches from an ImageSet as image files. 595 | 596 | Saves an image of each image part "cut out" by each region mask, cropped by default. 597 | If the RegionSet is not global, only images with valid region masks will be processed. 598 | 599 | Args: 600 | imageset (ImageSet): a valid ImageSet containing images to slice 601 | imageid (str): imageid (to select image-specific regions if not a global regionset) 602 | crop (bool): if True, return image cropped to bounding box of selected area 603 | image_format (str): image format that PIL understands (will also be used for extension) 604 | rescale (bool): if True, scale pixel values to full 0..255 range 605 | before saving (e.g., for saliency maps) 606 | ignore_background (bool): if True, ignore auto-generated background region 607 | 608 | """ 609 | if not isinstance(imageset, ImageSet): 610 | raise TypeError('First argument must be an ImageSet! To slice a single image, use export_patches().') 611 | 612 | for cimg in imageset.imageids: 613 | if not self.is_global and cimg not in self.imageids: 614 | print('Warning: RegionSet contains image-specific regions, but no regions available for {:s}. Skipped.'.format(cimg)) 615 | else: 616 | self.export_patches(imageset[cimg], imageid=cimg, crop=crop, image_format=image_format, 617 | rescale=rescale, ignore_background=ignore_background) 618 | 619 | 620 | def fixated(self, fixations, var='fixated', imageid=None, exclude_first=False, exclude_last=False): 621 | """ Returns visited / fixated regions using data from a Fixations object. 622 | 623 | Args: 624 | fixations (Fixations/DataFrame): fixation data to test against regions 625 | var (str): type of fixation mapping variable to calculate (default: 'fixated'): 626 | 'fixated': fixation status: 0 - region was not fixated, 1 - fixated (default) 627 | 'count': total number of fixations on each region 628 | 'fixid': fixation ID (from input dataset) for first fixation in each region 629 | imageid (str): imageid (to select image-specific regions if not a global regionset) 630 | exclude_first (bool): if True, first fixated region will always be returned as NaN 631 | exclude_last (str): controls how to deal with regions receiving the last image fixation: 632 | 'never' or False: do not handle the last fixation specially 633 | 'always' or True: drop the entire region if it received the last fixation at any time 634 | 'pass': exclude viewing pass (one or multiple fixations) that received the last fixation 635 | 636 | Returns: 637 | 1D ndarray (float) containing number of fixations per region (if count=True) 638 | or the values 0.0 (region was not fixated) or 1.0 (region was fixated) 639 | """ 640 | if type(exclude_last) == bool: 641 | if exclude_last: 642 | exclude_last = 'always' 643 | elif not exclude_last: 644 | exclude_last = 'never' 645 | 646 | apply_regions = self._select_region(imageid) 647 | vis = np.zeros(apply_regions.shape[0], dtype=float) 648 | 649 | # Drop out-of-bounds fixations 650 | fix = fixations.data[(fixations.data[fixations._xpx] >= 0) & 651 | (fixations.data[fixations._xpx] < self.size[0]) & 652 | (fixations.data[fixations._ypx] >= 0) & 653 | (fixations.data[fixations._ypx] < self.size[1])] 654 | 655 | if len(fix) > 0: 656 | if exclude_first: 657 | first_fix = fixations.data[fixations.data[fixations._fixid] == min(fixations.data[fixations._fixid])] 658 | if len(first_fix) > 1: 659 | print('Warning: you have requested to drop the first fixated region, but more than one ' + 660 | 'location ({:d}) matches the lowest fixation ID! Either your fixation ' .format(len(first_fix)) + 661 | 'IDs are not unique or the passed dataset contains data from multiple images or conditions.') 662 | if exclude_last != 'never': 663 | last_fix = fixations.data[fixations.data[fixations._fixid] == max(fixations.data[fixations._fixid])] 664 | if len(last_fix) > 1: 665 | print('Warning: you have requested to drop the last fixated region, but more than one ' + 666 | 'location ({:d}) matches the highest fixation ID! Either your fixation ' .format(len(last_fix)) + 667 | 'IDs are not unique or the passed dataset contains data from multiple images or conditions.') 668 | 669 | for (idx, roi) in enumerate(apply_regions): 670 | if exclude_first: 671 | try: 672 | is_first = roi[first_fix[fixations._ypx], first_fix[fixations._xpx]] 673 | if isinstance(is_first, np.ndarray) and np.any(is_first): 674 | vis[idx] = np.nan 675 | continue 676 | elif is_first: 677 | vis[idx] = np.nan 678 | continue 679 | except IndexError: 680 | pass # last fixation is out of bounds for image! 681 | 682 | if exclude_last == 'always': 683 | try: 684 | is_last = roi[last_fix[fixations._ypx], last_fix[fixations._xpx]] 685 | if isinstance(is_last, np.ndarray) and np.any(is_last): 686 | vis[idx] = np.nan 687 | continue 688 | elif is_last: 689 | vis[idx] = np.nan 690 | continue 691 | except IndexError: 692 | pass # last fixation is out of bounds for image! 693 | 694 | fv = roi[fix[fixations._ypx], fix[fixations._xpx]] 695 | if np.any(fv): 696 | rfix = fix[fv] # All fixations on region 697 | if fixations.has_times: 698 | # If fixation data has timing information, ensure to drop fixations 699 | # that began before the current fixation report 700 | bystart = rfix[rfix[fixations._fixstart] >= 0].sort_values(fixations._fixid) 701 | else: 702 | bystart = rfix.sort_values(fixations._fixid) 703 | 704 | if len(bystart) > 0: 705 | # Find viewing passes (sets of in-region fixations without leaving region) 706 | idxvalid = np.ones(bystart.shape[0], dtype=np.bool) # fix indices to keep 707 | idxdiff = bystart[fixations._fixid].diff().reset_index(drop=True) 708 | pass_onsets = idxdiff.index.values[(idxdiff > 1)].tolist() 709 | num_refix = len(pass_onsets) 710 | num_passes = num_refix + 1 711 | if len(pass_onsets) >= 1: 712 | end_first_pass = pass_onsets[0] 713 | else: 714 | end_first_pass = bystart.shape[0] 715 | 716 | # If requested, remove pass containing the last fixation 717 | if exclude_last == 'pass': 718 | passes = [0,] + pass_onsets + [len(bystart)+1,] 719 | for pidx in range(0, len(passes)-1): 720 | passfix = bystart.iloc[passes[pidx]:passes[pidx+1], :] 721 | if last_fix.index.values[0] in passfix.index: 722 | # Exclude this and all following passes. Note that no later passes 723 | # should exist unless there is an index error in the fixation data! 724 | idxvalid[passes[pidx]:] = False 725 | break 726 | 727 | if np.all(idxvalid == False): 728 | # If no valid fixations remain, drop the whole region (NA) 729 | vis[idx] = np.nan 730 | continue 731 | else: 732 | # Keep only valid fixations for fixation count measures 733 | bystart = bystart.loc[idxvalid, :] 734 | num_refix = np.sum(idxvalid[pass_onsets] == True) 735 | num_passes = num_refix + 1 736 | 737 | # Calculate fixation status measures 738 | if var == 'count': 739 | # Number of fixations in region 740 | vis[idx] = bystart.shape[0] 741 | 742 | elif var == 'fixated': 743 | # Binary coding of fixation status 744 | vis[idx] = (bystart.shape[0] >= 1.0) 745 | 746 | elif var == 'fixid': 747 | # Return first valid fixation ID in region 748 | if bystart.shape[0] >= 1.0: 749 | vis[idx] = bystart.loc[bystart.index[0], fixations._fixid] 750 | else: 751 | vis[idx] = np.nan 752 | 753 | elif var == 'passes': 754 | # Total number of fixation passes 755 | vis[idx] = num_passes 756 | 757 | elif var == 'refix': 758 | # Total number of fixation passes 759 | vis[idx] = num_refix 760 | 761 | else: 762 | # No fixations in region -> fixID should be NA 763 | if var == 'fixid': 764 | vis[idx] = np.nan 765 | 766 | return vis 767 | 768 | 769 | def fixtimes(self, fixations, var='total', imageid=None, exclude_first=False, exclude_last=False): 770 | """ Returns fixation-timing based variable for each region. Default is total viewing time. 771 | 772 | Args: 773 | fixations (Fixations/DataFrame): fixation data to test against regions 774 | var (str): type of fixation time variable to calculate (default: 'total'): 775 | 'total': total fixation time for each region 776 | 'gaze': gaze duration, i.e. total fixation time in first pass 777 | 'first': first fixation duration per region 778 | 'single': fixation duration if region was fixated exactly once 779 | 'tofirst': start time of the first fixation on each region 780 | imageid (str): imageid (to select image-specific regions if not a global regionset) 781 | exclude_first (bool): if True, first fixated region will always be returned as NaN 782 | exclude_last (str): controls how to deal with regions receiving the last image fixation: 783 | 'never' or False: do not handle the last fixation specially 784 | 'always' or True: drop the entire region if it received the last fixation at any time 785 | 'pass': exclude viewing pass (one or multiple fixations) that received the last fixation 786 | 787 | Returns: 788 | 1D ndarray (float) containing fixation time based dependent variable for each region. 789 | Regions that were never fixated according to criteria will be returned as NaN. 790 | """ 791 | if var not in ['total', 'gaze', 'first', 'single', 'tofirst']: 792 | raise ValueError('Unknown fixation time variable specified: {:s}'.format(var)) 793 | 794 | if not fixations.has_times: 795 | raise AttributeError('Trying to extract a time-based DV from a dataset without fixation timing information! Specify fixstart=/fixend= when loading fixation data!') 796 | 797 | if type(exclude_last) == bool: 798 | if exclude_last: 799 | exclude_last = 'always' 800 | elif not exclude_last: 801 | exclude_last = 'never' 802 | 803 | apply_regions = self._select_region(imageid) 804 | ft = np.ones(apply_regions.shape[0], dtype=float) * np.nan 805 | 806 | # Drop out-of-bounds fixations 807 | fix = fixations.data[(fixations.data[fixations._xpx] >= 0) & 808 | (fixations.data[fixations._xpx] < self.size[0]) & 809 | (fixations.data[fixations._ypx] >= 0) & 810 | (fixations.data[fixations._ypx] < self.size[1])] 811 | 812 | if len(fix) > 0: 813 | if exclude_first: 814 | first_fix = fixations.data[fixations.data[fixations._fixid] == min(fixations.data[fixations._fixid])] 815 | if len(first_fix) > 1: 816 | print('Warning: you have requested to drop the first fixated region, but more than one ' + 817 | 'location ({:d}) matches the lowest fixation ID! Either your fixation ' .format(len(first_fix)) + 818 | 'IDs are not unique or the passed dataset contains data from multiple images or conditions.') 819 | 820 | if exclude_last != 'never': 821 | last_fix = fixations.data[fixations.data[fixations._fixid] == max(fixations.data[fixations._fixid])] 822 | if len(last_fix) > 1: 823 | print('Warning: you have requested to drop the last fixated region, but more than one ' + 824 | 'location ({:d}) matches the highest fixation ID! Either your fixation ' .format(len(last_fix)) + 825 | 'IDs are not unique or the passed dataset contains data from multiple images or conditions.') 826 | 827 | for (idx, roi) in enumerate(apply_regions): 828 | if exclude_first: 829 | try: 830 | is_first = roi[first_fix[fixations._ypx], first_fix[fixations._xpx]] 831 | if isinstance(is_first, np.ndarray) and np.any(is_first): 832 | ft[idx] = np.nan 833 | continue 834 | elif is_first: 835 | ft[idx] = np.nan 836 | continue 837 | except IndexError: 838 | pass # first fixation is out of bounds for image! 839 | 840 | if exclude_last == 'always': 841 | # If this region has the last fixation, drop it here (NaN) and move on 842 | try: 843 | is_last = roi[last_fix[fixations._ypx], last_fix[fixations._xpx]] 844 | if isinstance(is_last, np.ndarray) and np.any(is_last): 845 | ft[idx] = np.nan 846 | continue 847 | elif is_last: 848 | ft[idx] = np.nan 849 | continue 850 | except IndexError: 851 | pass # last fixation is out of bounds for image! 852 | 853 | fidx = roi[fix[fixations._ypx], fix[fixations._xpx]] 854 | if np.any(fidx): 855 | rfix = fix[fidx] # all fixations in this region 856 | bystart = rfix[rfix[fixations._fixstart] >= 0].sort_values(fixations._fixid) 857 | 858 | if len(bystart) > 0: 859 | # Find viewing passes (sets of in-region fixations without leaving region) 860 | idxvalid = np.ones(bystart.shape[0], dtype=np.bool) # fix indices to keep 861 | idxdiff = bystart[fixations._fixid].diff().reset_index(drop=True) 862 | pass_onsets = idxdiff.index.values[(idxdiff > 1)].tolist() 863 | num_refix = len(pass_onsets) 864 | num_passes = num_refix + 1 865 | if len(pass_onsets) >= 1: 866 | end_first_pass = pass_onsets[0] 867 | else: 868 | end_first_pass = bystart.shape[0] 869 | 870 | # If requested, remove pass containing the last fixation 871 | if exclude_last == 'pass': 872 | passes = [0,] + pass_onsets + [len(bystart)+1,] 873 | for pidx in range(0, len(passes)-1): 874 | passfix = bystart.iloc[passes[pidx]:passes[pidx+1], :] 875 | if last_fix.index.values[0] in passfix.index: 876 | # Exclude this and all following passes. Note that no later passes 877 | # should exist unless there is an index error in the fixation data! 878 | idxvalid[passes[pidx]:] = False 879 | break 880 | 881 | if np.all(idxvalid == False): 882 | # If no valid fixations remain, drop the whole region (NA) 883 | ft[idx] = np.nan 884 | continue 885 | else: 886 | # Keep only valid fixations for fixation count measures 887 | bystart = bystart.loc[idxvalid, :] 888 | num_refix = np.sum(idxvalid[pass_onsets] == True) 889 | num_passes = num_refix + 1 890 | 891 | # Calculate fixation timing measures 892 | if var == 'gaze': 893 | # Gaze duration: total viewing time of first pass only 894 | ft[idx] = sum(bystart.loc[bystart.index[0:end_first_pass], fixations._fixdur]) 895 | 896 | elif var == 'first': 897 | # First fixation duration 898 | ft[idx] = bystart.loc[bystart.index[0], fixations._fixdur] 899 | 900 | elif var == 'single': 901 | # Single fixation duration (=first fixation duration if not refixated in first pass) 902 | ft[idx] = bystart.loc[bystart.index[0], fixations._fixdur] 903 | 904 | # If refixated on first pass, set to NaN instead 905 | if end_first_pass > 1: 906 | ft[idx] = np.nan 907 | 908 | elif var == 'tofirst': 909 | # Time until first fixation / first fixation onset 910 | ft[idx] = bystart.loc[bystart.index[0], fixations._fixstart] 911 | 912 | elif var == 'total': 913 | # Total viewing time of valid fixations 914 | ft[idx] = sum(bystart.loc[:, fixations._fixdur]) 915 | 916 | return ft 917 | 918 | 919 | 920 | class GridRegionSet(RegionSet): 921 | """ RegionSet defining an n-by-m regular grid covering the full image size. 922 | 923 | Attributes: 924 | cells (list): list of bounding box tuples for each cell, 925 | each formatted as (left, top, right, bottom) 926 | gridsize (tuple): grid dimensions as (width, height). If unspecified, 927 | gridfix will try to choose a sensible default. 928 | label (string): optional label to distinguish between RegionSets 929 | """ 930 | 931 | def __init__(self, size, gridsize=None, label=None, region_labels=None): 932 | """ Create a new grid RegionSet 933 | 934 | Args: 935 | size (tuple): image dimensions, specified as (width, height). 936 | gridsize(tuple): grid dimensions, specified as (width, height). 937 | region_labels (string): list of optional region labels (default: cell#) 938 | """ 939 | 940 | if gridsize is None: 941 | gridsize = self._suggest_grid(size) 942 | print('Note: no grid size was specified. Using {:d}x{:d} based on image size.'.format(gridsize[0], gridsize[1])) 943 | 944 | (regions, cells) = self._grid(size, gridsize) 945 | RegionSet.__init__(self, size=size, regions=regions, label=label, region_labels=region_labels, 946 | add_background=False) # GridRegionSets are exhaustive, so the 'background' is empty. 947 | 948 | self.gridsize = gridsize 949 | 950 | # List of region bboxes 951 | self.cells = cells 952 | 953 | 954 | def __str__(self): 955 | """ Short string representation for printing """ 956 | r = '' 957 | 958 | if self.label is not None: 959 | lab = ' ({:s})'.format(self.label) 960 | else: 961 | lab = '' 962 | 963 | num_s = '' 964 | num_r = len(self) 965 | if num_r > 1: 966 | num_s = 's' 967 | return r.format(lab, str(self.size), self.gridsize[0], self.gridsize[1], num_r, 968 | num_s, self.memory_usage) 969 | 970 | 971 | def _suggest_grid(self, size): 972 | """ Suggest grid dimensions based on image size. 973 | 974 | Args: 975 | size (tuple): image dimensions, specified as (width, height). 976 | 977 | Returns: 978 | Suggested grid size tuple as (width, height). 979 | """ 980 | aspect = Fraction(size[0], size[1]) 981 | s_width = aspect.numerator 982 | s_height = aspect.denominator 983 | if s_width < 6: 984 | s_width *= 2 985 | s_height *= 2 986 | return (s_width, s_height) 987 | 988 | 989 | def _grid(self, size, gridsize): 990 | """ Build m-by-n (width,height) grid as 3D nparray. 991 | 992 | Args: 993 | size (tuple): image dimensions, specified as (width, height). 994 | gridsize(tuple): grid dimensions, specified as (width, height). 995 | 996 | Returns: 997 | tuple containing the grid regions and their bounding box coordinates 998 | as (grid, cells): 999 | 1000 | grid (numpy.ndarray): regions for RegionSet creation 1001 | cells (list): list of bounding box tuples for each cell, 1002 | each formatted as (left, top, right, bottom) 1003 | 1004 | """ 1005 | (width, height) = size 1006 | _msize = (size[1], size[0]) 1007 | cell_x = int(width / gridsize[0]) 1008 | cell_y = int(height / gridsize[1]) 1009 | n_cells = int(gridsize[0] * gridsize[1]) 1010 | 1011 | grid = np.zeros((n_cells,) + _msize, dtype=bool) 1012 | cells = [] 1013 | 1014 | # Sanity check: do nothing if image dimensions not cleanly divisible by grid 1015 | if width % gridsize[0] > 0 or height % gridsize[1] > 0: 1016 | e = 'Error: image dimensions not cleanly divisible by grid! image=({:d}x{:d}), grid=({:d}x{:d})' 1017 | raise ValueError(e.format(width, height, gridsize[0], gridsize[1])) 1018 | 1019 | # Create a mask of 1s/True for each cell 1020 | cellno = 0 1021 | for y_es in range(0, height, cell_y): 1022 | for x_es in range(0, width, cell_x): 1023 | mask = np.zeros(_msize, dtype=bool) 1024 | mask[y_es:y_es + cell_y, x_es:x_es + cell_x] = True 1025 | grid[cellno,...] = mask 1026 | cells.append((x_es, y_es, x_es + cell_x, y_es + cell_y)) 1027 | cellno += 1 1028 | 1029 | return (grid, cells) 1030 | 1031 | 1032 | 1033 | class BBoxRegionSet(RegionSet): 1034 | """ RegionSet based on rectangular bounding boxes. 1035 | 1036 | Attributes: 1037 | cells (list): list of bounding box tuples for each cell, 1038 | each formatted as (left, top, right, bottom) 1039 | label (string): optional label to distinguish between RegionSets 1040 | from_file (string): filename in case regions were loaded from file 1041 | padding (tuple): padding in pixels as ('left', 'top', 'right', 'bottom') 1042 | """ 1043 | 1044 | def __init__(self, size, bounding_boxes, label=None, region_labels=None, sep='\t', 1045 | imageid='imageid', regionid='regionid', bbox_cols=('x1', 'y1', 'x2', 'y2'), 1046 | padding=0, add_background=False, coord_format=None): 1047 | """ Create new BBoxRegionSet 1048 | 1049 | Args: 1050 | size (tuple): image dimensions, specified as (width, height). 1051 | bounding_boxes: one of the following: 1052 | name of a text/CSV file with columns ([imageid], [regionid], x1, y1, x2, y2) 1053 | list of 4-tuples OR 2D ndarray with columns (x1, y1, x2, y2) for global bboxes 1054 | region_labels (str): list of optional region labels if bounding_boxes is a global array/list 1055 | imageid (str): name of imageid column in input file (if not present, bboxes will be treated as global) 1056 | regionid (str): name of regionid column in input file 1057 | sep (str): separator to use when reading files 1058 | bbox_cols: tuple of column names for ('left', 'top', 'right', 'bottom') 1059 | padding (int): optional bbox padding in pixels as ('left', 'top', 'right', 'bottom'), 1060 | or a single integer to specify equal padding on all sides 1061 | add_background (bool): if True, this creates a special region to capture all 1062 | fixations that don't fall on an explicit region ("background" fixations) 1063 | coord_format (str): Defines how input x and y coordinates are interpreted: 1064 | 'oneindexed': coordinates start at 1, e.g. 1..100 for a 100px box 1065 | 'zeroindexed': coordinates start at 0, e.g. 0..99 for a 100px box 1066 | 'apple': coordinates start at 0, but end at , e.g. 0..100 for a 100px box, 1067 | in this convention, the pixels sit "between" coordinate values 1068 | """ 1069 | self.input_file = None 1070 | self.input_df = None 1071 | 1072 | self._imageid = imageid 1073 | self._regionid = regionid 1074 | self._cols = bbox_cols 1075 | 1076 | if coord_format is None: 1077 | err = 'No coordinate format specified! Please provide coord_format argument:\n' 1078 | err += '"oneindexed": coordinates start at 1, e.g. 1..100 for a 100px box\n' 1079 | err += '"zeroindexed": coordinates start at 0, e.g. 0..99 for a 100px box\n' 1080 | err += '"apple": coordinates start at 0, but end at , e.g. 0..100 for a 100px box.' 1081 | raise ValueError(err) 1082 | 1083 | if type(padding) == int: 1084 | self.padding = (padding,) * 4 1085 | else: 1086 | self.padding = padding 1087 | 1088 | if isinstance(bounding_boxes, DataFrame): 1089 | # Passed a DataFrame 1090 | bbox = bounding_boxes 1091 | 1092 | elif type(bounding_boxes) == str: 1093 | # Passed a file name 1094 | try: 1095 | bbox = read_table(bounding_boxes, sep=sep) 1096 | self.input_file = bounding_boxes 1097 | 1098 | except: 1099 | raise ValueError('String argument supplied to BBoxRegionSet, but not a valid CSV file!') 1100 | 1101 | else: 1102 | # Try array type 1103 | try: 1104 | bbox = DataFrame(bounding_boxes, columns=['x1', 'y1', 'x2', 'y2']) 1105 | except: 1106 | raise ValueError('Supplied argument to BBoxRegionSet not in the form (x1, y1, x2, y2)') 1107 | 1108 | (regions, labels) = self._parse_bbox_df(bbox, size, padding, coord_format=coord_format) 1109 | if region_labels is not None: 1110 | labels = region_labels 1111 | 1112 | RegionSet.__init__(self, size=size, regions=regions, label=label, region_labels=labels, add_background=add_background) 1113 | 1114 | self.input_df = bbox 1115 | 1116 | 1117 | def _parse_bbox_df(self, df, size, padding, coord_format='oneindexed'): 1118 | """ Parse a DataFrame of bounding boxes into a region dict. 1119 | 1120 | Args: 1121 | df (DataFrame): DataFrame of bounding box coordinates 1122 | size (tuple): image size as (width, height), to check for out-of-bounds coordinates 1123 | padding (int): padding in pixels as ('left', 'top', 'right', 'bottom'), default (0,0,0,0) 1124 | coord_format (str): Defines how input x and y coordinates are interpreted (see __init__) 1125 | 1126 | Returns: 1127 | tuple of dicts as (regions, labels), using imageids as keys. Resulting dicts can be 1128 | passed to RegionSet.__init__ directly. 1129 | """ 1130 | regions = {} 1131 | labels = {} 1132 | _msize = (size[1], size[0]) 1133 | 1134 | if self._imageid in df.columns: 1135 | # Image-specific bounding boxes 1136 | 1137 | # Force imageid to strings 1138 | df[self._imageid] = df[self._imageid].astype(str) 1139 | 1140 | for imid, block in df.groupby(self._imageid): 1141 | N = block.shape[0] 1142 | reg = np.zeros((N,) + _msize, dtype=bool) 1143 | lab = [] 1144 | bidx = 0 1145 | 1146 | for idx,row in block.iterrows(): 1147 | # left, top, right, bottom 1148 | c = [row[self._cols[0]], row[self._cols[1]], row[self._cols[2]], row[self._cols[3]]] 1149 | 1150 | if self._regionid in df.columns: 1151 | l = row[self._regionid] 1152 | else: 1153 | l = str(bidx + 1) 1154 | 1155 | # Convert coordinates to Python indices 1156 | if coord_format.lower() == 'oneindexed': 1157 | c = [round(c[0]) - 1, # Correct lower bounds 1158 | round(c[1]) - 1, 1159 | round(c[2]), 1160 | round(c[3])] 1161 | elif coord_format.lower() == 'zeroindexed': 1162 | c = [round(c[0]), 1163 | round(c[1]), 1164 | round(c[2]) + 1, # Python slices are end-excluding, 1165 | round(c[3]) + 1] # i.e. [a, b] does not include b 1166 | elif coord_format.lower() == 'apple': 1167 | c = [round(c[0]), # Just round, format is already fine 1168 | round(c[1]), 1169 | round(c[2]), 1170 | round(c[3])] 1171 | 1172 | if c[0] > size[0] or c[2] > size[0] or c[1] > size[1] or c[3] > size[1]: 1173 | err = 'At least one coordinate of region {:s}/{:s} exceeds the specified image size!' 1174 | raise ValueError(err.format(imid, l)) 1175 | 1176 | if c[2]-c[0] < 1 or c[3]-c[1] < 1: 1177 | err = 'At least one dimension of {:s}/{:s} has a negative length! Columns specified in the wrong order?' 1178 | raise ValueError(err.format(imid, l)) 1179 | 1180 | # Add padding, ensuring padded bboxes are cropped to image size 1181 | c = [c[0] - self.padding[0], c[1] - self.padding[1], c[2] + self.padding[2], c[3] + self.padding[3]] 1182 | if c[0] < 0: 1183 | c[0] = 0 1184 | if c[1] < 0: 1185 | c[1] = 0 1186 | if c[2] > size[0]: 1187 | c[2] = size[0] 1188 | if c[3] > size[1]: 1189 | c[3] = size[1] 1190 | 1191 | mask = np.zeros(_msize, dtype=bool) 1192 | mask[c[1]:c[3], c[0]:c[2]] = True 1193 | reg[bidx,...] = mask 1194 | lab.append(l) 1195 | bidx += 1 1196 | 1197 | regions[imid] = reg 1198 | labels[imid] = lab 1199 | 1200 | else: 1201 | # Global bounding boxes 1202 | reg = reg = np.zeros((df.shape[0], ) + _msize, dtype=bool) 1203 | lab = [] 1204 | 1205 | for idx,row in df.iterrows(): 1206 | c = (row[self._cols[0]], row[self._cols[1]], row[self._cols[2]], row[self._cols[3]]) 1207 | 1208 | if self._regionid in df.columns: 1209 | l = row[self._regionid] 1210 | else: 1211 | l = str(idx + 1) 1212 | 1213 | # Convert coordinates to Python indices 1214 | if coord_format.lower() == 'oneindexed': 1215 | c = [round(c[0]) - 1, # Correct lower bounds 1216 | round(c[1]) - 1, 1217 | round(c[2]), 1218 | round(c[3])] 1219 | elif coord_format.lower() == 'zeroindexed': 1220 | c = [round(c[0]), 1221 | round(c[1]), 1222 | round(c[2]) + 1, # Python slices are end-excluding, 1223 | round(c[3]) + 1] # i.e. [a, b] does not include b 1224 | elif coord_format.lower() == 'apple': 1225 | c = [round(c[0]), # Just round, format is already fine 1226 | round(c[1]), 1227 | round(c[2]), 1228 | round(c[3])] 1229 | 1230 | if c[0] > size[0] or c[2] > size[0] or c[1] > size[1] or c[3] > size[1]: 1231 | err = 'At least one coordinate of region {:s} exceeds the specified image size!' 1232 | raise ValueError(err.format(l)) 1233 | 1234 | mask = np.zeros(_msize, dtype=bool) 1235 | mask[c[1]:c[3], c[0]:c[2]] = True 1236 | reg[idx,...] = mask 1237 | lab.append(l) 1238 | 1239 | regions = {'*': reg} 1240 | labels = {'*': lab} 1241 | 1242 | return (regions, labels) -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools>=42", 4 | "wheel" 5 | ] 6 | build-backend = "setuptools.build_meta" 7 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open('README.md', 'r') as rf: 4 | readme = rf.read() 5 | 6 | setuptools.setup( 7 | name='gridfix', 8 | version='0.3.2', 9 | description='Parcellation-based gaze fixation preprocessing for GLMM analysis', 10 | long_description=readme, 11 | long_description_content_type='text/markdown', 12 | url='https://github.com/ischtz/gridfix', 13 | author='Immo Schuetz', 14 | author_email='schuetz.immo@gmail.com', 15 | license='MIT', 16 | packages=setuptools.find_packages(), 17 | install_requires=['numpy', 'matplotlib', 'scipy', 'pandas'], 18 | zip_safe=True, 19 | classifiers=[ 20 | 'License :: OSI Approved :: MIT License', 21 | 'Intended Audience :: Science/Research', 22 | 'Operating System :: OS Independent', 23 | 'Programming Language :: Python :: 3' 24 | ], 25 | python_requires='>=3.6') 26 | 27 | --------------------------------------------------------------------------------