├── .dockerignore ├── .gitignore ├── .travis.yml ├── CHANGELOG.rst ├── Dockerfile ├── LICENSE ├── MANIFEST.in ├── README.rst ├── appveyor.yml ├── bidsify ├── __init__.py ├── data │ ├── dataset_description.json │ ├── example_config.yml │ ├── spinoza_cfg.yml │ ├── spinoza_cfg_dicom.yml │ ├── spinoza_metadata.yml │ └── test_data │ │ └── .gitkeep ├── docker.py ├── main.py ├── mri2nifti.py ├── phys2tsv.py ├── tests │ ├── __init__.py │ └── test_bidsify.py ├── utils.py └── version.py ├── build_docker_image ├── create_new_release ├── download_test_data.py ├── generate_dockerfile ├── requirements.txt └── setup.py /.dockerignore: -------------------------------------------------------------------------------- 1 | # python cache 2 | __pycache__/**/* 3 | __pycache__ 4 | *.pyc 5 | 6 | # python distribution 7 | build/**/* 8 | build 9 | dist/**/* 10 | dist 11 | bidsify.egg-info 12 | 13 | # git 14 | .gitignore 15 | .git/**/* 16 | .git 17 | 18 | bidsify/data/test_data 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | 103 | OLD/ 104 | .DS_Store 105 | 106 | bidsify/data/test_data 107 | bidsify/data/test_data 108 | bidsify/data/test_data 109 | bidsify/data/test_data 110 | .pytest_cache 111 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: false 3 | 4 | env: 5 | global: 6 | - PIP_DEPS="coveralls pytest-cov flake8" 7 | 8 | python: 9 | - '3.6' 10 | 11 | before_install: 12 | - bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh) 13 | - travis_retry sudo apt-get update -qq 14 | #- travis_retry sudo apt-get install dcm2niix 15 | - git clone https://github.com/neurolabusc/dcm2niix 16 | - mkdir dcm2niix/build 17 | - cd dcm2niix/build 18 | - cmake .. 19 | - make 20 | - sudo cp bin/dcm2niix /usr/bin 21 | - travis_retry sudo apt-get install nodejs 22 | - npm install -g bids-validator 23 | 24 | install: 25 | - travis_retry pip install --upgrade pytest # new pytest>3.3 for coveralls 26 | - travis_retry pip install $PIP_DEPS 27 | - cd $TRAVIS_BUILD_DIR 28 | - travis_retry pip install -r requirements.txt 29 | - travis_retry pip install -e . 30 | 31 | script: 32 | - python download_test_data.py 33 | - py.test --cov=bidsify/ 34 | 35 | after_success: 36 | - coveralls 37 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | CHANGELOG 2 | ========= 3 | 4 | Version 0.3.2 5 | ------------- 6 | Bugfixes + some added functionality. 7 | 8 | - ENH: simplify metadata structure 9 | - ENH: fix PAR headers of manually stopped scans (remove partial volumes) 10 | 11 | Version 0.3.1 12 | ------------- 13 | Hotfix pip install. 14 | 15 | - FIX: add MANIFEST.in to fix pip install issue 16 | 17 | Version 0.3 18 | ------------- 19 | Version 0.3 of bidsify will be the first release after the major refactor. 20 | It contains the following (major) changes: 21 | 22 | - ENH: accepts both json and yaml config files 23 | - ENH: major refactoring of package structure (now based on `shablona `_) 24 | - ENH: writes out a (default) dataset_description.json and participants.tsv file 25 | - ENH: option to run ``bidsify`` in a docker image! 26 | - ENH: (should) work with ``.dcm`` files 27 | - ENH: is now pip installable (``pip install bidsify``) 28 | 29 | Versions < 0.3.0 30 | ---------------- 31 | The changelog for versions < 0.3.0 has not been documented. 32 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Your version: 0.6.0 Latest version: 0.6.0 2 | # Generated by Neurodocker version 0.6.0 3 | # Timestamp: 2019-10-08 15:32:06 UTC 4 | # 5 | # Thank you for using Neurodocker. If you discover any issues 6 | # or ways to improve this software, please submit an issue or 7 | # pull request on our GitHub repository: 8 | # 9 | # https://github.com/kaczmarj/neurodocker 10 | 11 | FROM debian:stretch 12 | 13 | ARG DEBIAN_FRONTEND="noninteractive" 14 | 15 | ENV LANG="en_US.UTF-8" \ 16 | LC_ALL="en_US.UTF-8" \ 17 | ND_ENTRYPOINT="/neurodocker/startup.sh" 18 | RUN export ND_ENTRYPOINT="/neurodocker/startup.sh" \ 19 | && apt-get update -qq \ 20 | && apt-get install -y -q --no-install-recommends \ 21 | apt-utils \ 22 | bzip2 \ 23 | ca-certificates \ 24 | curl \ 25 | locales \ 26 | unzip \ 27 | && apt-get clean \ 28 | && rm -rf /var/lib/apt/lists/* \ 29 | && sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \ 30 | && dpkg-reconfigure --frontend=noninteractive locales \ 31 | && update-locale LANG="en_US.UTF-8" \ 32 | && chmod 777 /opt && chmod a+s /opt \ 33 | && mkdir -p /neurodocker \ 34 | && if [ ! -f "$ND_ENTRYPOINT" ]; then \ 35 | echo '#!/usr/bin/env bash' >> "$ND_ENTRYPOINT" \ 36 | && echo 'set -e' >> "$ND_ENTRYPOINT" \ 37 | && echo 'export USER="${USER:=`whoami`}"' >> "$ND_ENTRYPOINT" \ 38 | && echo 'if [ -n "$1" ]; then "$@"; else /usr/bin/env bash; fi' >> "$ND_ENTRYPOINT"; \ 39 | fi \ 40 | && chmod -R 777 /neurodocker && chmod a+s /neurodocker 41 | 42 | ENTRYPOINT ["/neurodocker/startup.sh"] 43 | 44 | RUN apt-get update -qq \ 45 | && apt-get install -y -q --no-install-recommends \ 46 | git \ 47 | && apt-get clean \ 48 | && rm -rf /var/lib/apt/lists/* 49 | 50 | ENV FSLDIR="/opt/fsl-6.0.1" \ 51 | PATH="/opt/fsl-6.0.1/bin:$PATH" 52 | RUN apt-get update -qq \ 53 | && apt-get install -y -q --no-install-recommends \ 54 | bc \ 55 | dc \ 56 | file \ 57 | libfontconfig1 \ 58 | libfreetype6 \ 59 | libgl1-mesa-dev \ 60 | libgl1-mesa-dri \ 61 | libglu1-mesa-dev \ 62 | libgomp1 \ 63 | libice6 \ 64 | libxcursor1 \ 65 | libxft2 \ 66 | libxinerama1 \ 67 | libxrandr2 \ 68 | libxrender1 \ 69 | libxt6 \ 70 | sudo \ 71 | wget \ 72 | && apt-get clean \ 73 | && rm -rf /var/lib/apt/lists/* \ 74 | && echo "Downloading FSL ..." \ 75 | && mkdir -p /opt/fsl-6.0.1 \ 76 | && curl -fsSL --retry 5 https://fsl.fmrib.ox.ac.uk/fsldownloads/fsl-6.0.1-centos6_64.tar.gz \ 77 | | tar -xz -C /opt/fsl-6.0.1 --strip-components 1 \ 78 | && sed -i '$iecho Some packages in this Docker container are non-free' $ND_ENTRYPOINT \ 79 | && sed -i '$iecho If you are considering commercial use of this container, please consult the relevant license:' $ND_ENTRYPOINT \ 80 | && sed -i '$iecho https://fsl.fmrib.ox.ac.uk/fsl/fslwiki/Licence' $ND_ENTRYPOINT \ 81 | && sed -i '$isource $FSLDIR/etc/fslconf/fsl.sh' $ND_ENTRYPOINT \ 82 | && echo "Installing FSL conda environment ..." \ 83 | && bash /opt/fsl-6.0.1/etc/fslconf/fslpython_install.sh -f /opt/fsl-6.0.1 \ 84 | && echo "Downgrading deprecation module per https://github.com/kaczmarj/neurodocker/issues/271#issuecomment-514523420" \ 85 | && /opt/fsl-6.0.1/fslpython/bin/conda install -n fslpython -c conda-forge -y deprecation==1.* \ 86 | && echo "Removing bundled with FSLeyes libz likely incompatible with the one from OS" \ 87 | && rm -f /opt/fsl-6.0.1/bin/FSLeyes/libz.so.1 88 | 89 | ENV PATH="/opt/dcm2niix-master/bin:$PATH" 90 | RUN apt-get update -qq \ 91 | && apt-get install -y -q --no-install-recommends \ 92 | cmake \ 93 | g++ \ 94 | gcc \ 95 | git \ 96 | make \ 97 | pigz \ 98 | zlib1g-dev \ 99 | && apt-get clean \ 100 | && rm -rf /var/lib/apt/lists/* \ 101 | && git clone https://github.com/rordenlab/dcm2niix /tmp/dcm2niix \ 102 | && mkdir /tmp/dcm2niix/build \ 103 | && cd /tmp/dcm2niix/build \ 104 | && cmake -DCMAKE_INSTALL_PREFIX:PATH=/opt/dcm2niix-master .. \ 105 | && make \ 106 | && make install \ 107 | && rm -rf /tmp/dcm2niix 108 | 109 | ENV CONDA_DIR="/opt/miniconda-latest" \ 110 | PATH="/opt/miniconda-latest/bin:$PATH" 111 | RUN export PATH="/opt/miniconda-latest/bin:$PATH" \ 112 | && echo "Downloading Miniconda installer ..." \ 113 | && conda_installer="/tmp/miniconda.sh" \ 114 | && curl -fsSL --retry 5 -o "$conda_installer" https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ 115 | && bash "$conda_installer" -b -p /opt/miniconda-latest \ 116 | && rm -f "$conda_installer" \ 117 | && conda update -yq -nbase conda \ 118 | && conda config --system --prepend channels conda-forge \ 119 | && conda config --system --set auto_update_conda false \ 120 | && conda config --system --set show_channel_urls true \ 121 | && sync && conda clean --all && sync \ 122 | && conda create -y -q --name neuro \ 123 | && conda install -y -q --name neuro \ 124 | "python=3.6" \ 125 | "numpy" \ 126 | "pandas" \ 127 | && sync && conda clean --all && sync \ 128 | && bash -c "source activate neuro \ 129 | && pip install --no-cache-dir \ 130 | "nipype" \ 131 | "git+https://github.com/poldracklab/pydeface.git@master" \ 132 | "pyyaml" \ 133 | "nibabel" \ 134 | "joblib"" \ 135 | && rm -rf ~/.cache/pip/* \ 136 | && sync \ 137 | && sed -i '$isource activate neuro' $ND_ENTRYPOINT 138 | 139 | RUN apt-get update -qq \ 140 | && apt-get install -y -q --no-install-recommends \ 141 | gnupg2 \ 142 | vim \ 143 | && apt-get clean \ 144 | && rm -rf /var/lib/apt/lists/* 145 | 146 | RUN curl --silent --location https://deb.nodesource.com/setup_10.x | bash - 147 | 148 | RUN apt-get update -qq \ 149 | && apt-get install -y -q --no-install-recommends \ 150 | nodejs \ 151 | && apt-get clean \ 152 | && rm -rf /var/lib/apt/lists/* 153 | 154 | RUN npm install -g bids-validator 155 | 156 | COPY [".", "/home/neuro/bidsify"] 157 | 158 | WORKDIR /home/neuro/bidsify 159 | 160 | RUN /opt/miniconda-latest/envs/neuro/bin/python setup.py install 161 | 162 | VOLUME ["/raw"] 163 | 164 | VOLUME ["/bids"] 165 | 166 | RUN echo '{ \ 167 | \n "pkg_manager": "apt", \ 168 | \n "instructions": [ \ 169 | \n [ \ 170 | \n "base", \ 171 | \n "debian:stretch" \ 172 | \n ], \ 173 | \n [ \ 174 | \n "install", \ 175 | \n [ \ 176 | \n "git" \ 177 | \n ] \ 178 | \n ], \ 179 | \n [ \ 180 | \n "fsl", \ 181 | \n { \ 182 | \n "version": "6.0.1" \ 183 | \n } \ 184 | \n ], \ 185 | \n [ \ 186 | \n "dcm2niix", \ 187 | \n { \ 188 | \n "version": "master", \ 189 | \n "method": "source" \ 190 | \n } \ 191 | \n ], \ 192 | \n [ \ 193 | \n "miniconda", \ 194 | \n { \ 195 | \n "create_env": "neuro", \ 196 | \n "conda_install": [ \ 197 | \n "python=3.6", \ 198 | \n "numpy", \ 199 | \n "pandas" \ 200 | \n ], \ 201 | \n "pip_install": [ \ 202 | \n "nipype", \ 203 | \n "git+https://github.com/poldracklab/pydeface.git@master", \ 204 | \n "pyyaml", \ 205 | \n "nibabel", \ 206 | \n "joblib" \ 207 | \n ], \ 208 | \n "activate": true \ 209 | \n } \ 210 | \n ], \ 211 | \n [ \ 212 | \n "install", \ 213 | \n [ \ 214 | \n "gnupg2", \ 215 | \n "vim" \ 216 | \n ] \ 217 | \n ], \ 218 | \n [ \ 219 | \n "run", \ 220 | \n "curl --silent --location https://deb.nodesource.com/setup_10.x | bash -" \ 221 | \n ], \ 222 | \n [ \ 223 | \n "install", \ 224 | \n [ \ 225 | \n "nodejs" \ 226 | \n ] \ 227 | \n ], \ 228 | \n [ \ 229 | \n "run", \ 230 | \n "npm install -g bids-validator" \ 231 | \n ], \ 232 | \n [ \ 233 | \n "copy", \ 234 | \n [ \ 235 | \n ".", \ 236 | \n "/home/neuro/bidsify" \ 237 | \n ] \ 238 | \n ], \ 239 | \n [ \ 240 | \n "workdir", \ 241 | \n "/home/neuro/bidsify" \ 242 | \n ], \ 243 | \n [ \ 244 | \n "run", \ 245 | \n "/opt/miniconda-latest/envs/neuro/bin/python setup.py install" \ 246 | \n ], \ 247 | \n [ \ 248 | \n "volume", \ 249 | \n [ \ 250 | \n "/raw" \ 251 | \n ] \ 252 | \n ], \ 253 | \n [ \ 254 | \n "volume", \ 255 | \n [ \ 256 | \n "/bids" \ 257 | \n ] \ 258 | \n ] \ 259 | \n ] \ 260 | \n}' > /neurodocker/neurodocker_specs.json 261 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | New BSD License (3 clause BSD) 2 | 3 | Copyright (c) 2016-2018, Lukas Snoek 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | * Neither the name of the nor the 14 | names of its contributors may be used to endorse or promote products 15 | derived from this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 18 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 | DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY 21 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 22 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 23 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 24 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 26 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | © 2018 GitHub, Inc. 28 | Terms 29 | Privacy 30 | Security 31 | Status 32 | Help 33 | Contact GitHub 34 | API 35 | Training 36 | Shop 37 | Blog 38 | About 39 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements.txt 2 | include README.rst 3 | include CHANGELOG.rst 4 | include download_test_data.py 5 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ``bidsify`` - converts your (raw) data to the BIDS-format 2 | ============================================================= 3 | 4 | .. _BIDS: http://bids.neuroimaging.io/ 5 | 6 | .. image:: https://travis-ci.org/spinoza-rec/bidsify.svg?branch=master 7 | :target: https://travis-ci.org/spinoza-rec/bidsify 8 | 9 | .. image:: https://ci.appveyor.com/api/projects/status/d9a7bjjqg204kofm?svg=true 10 | :target: https://ci.appveyor.com/project/lukassnoek/bidsify 11 | 12 | .. image:: https://coveralls.io/repos/github/spinoza-rec/bidsify/badge.svg?branch=master 13 | :target: https://coveralls.io/github/spinoza-rec/bidsify?branch=master 14 | 15 | .. image:: https://img.shields.io/badge/python-3.6-blue.svg 16 | :target: https://www.python.org/downloads/release/python-360 17 | 18 | ## This project has been archived as I finished my PhD! It won't be maintained anymore 19 | 20 | This package offers a tool to convert your raw (f)MRI data to the "Brain Imaging Data Structuce" (BIDS_) format. Using only a simple (json or yaml) config-file, it renames, reformats, and restructures your files such that it fits the BIDS naming scheme and conforms to file-formats specified by BIDS. After using ``bidsify``, you can run your data through BIDS-compatible analysis/preprocessing pipelines such as `fmriprep `_ 21 | and `mriqc `_ package. 22 | 23 | Currently, we use ``bidsify`` at the Spinoza Centre for Neuroimaging (location REC) to convert data to BIDS after each scan-session. We automated this process, including automatic preprocessing and quality control, using another package, `nitools `_ (which essentially "glues together" ``bidsify``, ``fmriprep``, and ``mriqc``). 24 | 25 | This package was originally developed to handle MRI-data from Philips scanners, which are traditionally exported 26 | in the "PAR/REC" format. Currently, ``bidsify`` also supports Philips (enhanced) DICOM (``DICOM``/``DICOMDIR`` format) and Siemens DICOM (``.dcm`` extension), but the latter has not been fully tested yet! 27 | 28 | ``bidsify`` is still very much in development, so there are probably still some bugs for data 29 | that differs from our standard format (at the Spinoza Centre in Amsterdam) and the API might change 30 | in the future. If you encounter any issues, please submit an issue or (better yet), submit a pull-request 31 | with your proposed solution! 32 | 33 | Installing ``bidsify`` & dependencies 34 | --------------------------------------- 35 | This package can be installed using ``pip``:: 36 | 37 | $ pip install bidsify 38 | 39 | To get the "bleeding edge" version, you can install the master branch from github:: 40 | 41 | $ pip install git+https://github.com/spinoza-rec/bidsify.git@master 42 | 43 | In terms of dependencies: ``bidsify`` uses `dcm2niix `_ 44 | under the hood to convert PAR/REC and DICOM files to nifti. Make sure you're using release `v1.0.20181125 `_ or newer. 45 | 46 | Apart from ``dcm2niix``, ``bidsify`` depends on the following Python packages: 47 | 48 | - nibabel 49 | - scipy 50 | - numpy 51 | - joblib (for parallelization) 52 | - pandas 53 | 54 | Moreover, if you want to use the defacing option (i.e., removing facial features from anatomical images), make sure you have `FSL `_ installed, as well as the `pydeface `_ Python package. Also, to enable validating the BIDS-conversion process,(i.e., running ``bidsify`` with the ``-v`` flag), make sure to install `bids-validator `_. 55 | 56 | Lastly, if you want to use the Docker interface (i.e., running ``bidsify`` with the `-D` flag), which obviates the need for installing dcm2niix/FSL/bids-validator, make sure to install Docker and make sure your user account has permission to run Docker (see below). 57 | 58 | Using Docker 59 | ------------ 60 | The current version (master branch) allows you to run ``bidsify`` from docker, so you don't 61 | have to install all the (large) dependencies (FSL, pydeface, dcm2niix, bids-validator, etc.). To do so, 62 | you need to do the following. 63 | 64 | 1. Install Docker (if you haven't already) and make sure you have permission to run Docker; 65 | 2. Pull the Docker image: ``docker pull lukassnoek/bidsify:0.x.x`` (fill in the latest version at the x.x); 66 | 3. Run bidsify with the `-D` flag (e.g., ``bidsify -c /home/user/config.yml -d /home/user/data -D``) 67 | 68 | Now you can use ``bidsify`` even without having FSL, dcm2niix, and other dependencies installed! 69 | (You do need to install ``bidsify`` itself though.) 70 | 71 | How does it work? 72 | ----------------- 73 | After installing, the ``bidsify`` command can be called as follows:: 74 | 75 | $ bidsify [-c config_file] [-d path_to_data_directory] [-o output_directory] [-v] [-D] 76 | 77 | The ``-c`` flag defaults to ``config.yml`` in the current working directory. 78 | 79 | The ``-d`` flag defaults to the current working directory. 80 | 81 | The ``-o`` flag defaults to the parent-directory of the data-directory. 82 | 83 | The ``-v`` flag calls `bids-validator `_ after BIDS-conversion (optional). 84 | 85 | The ``-D`` flag runs ``bidsify`` from Docker (recommended; see "Docker" section above). 86 | 87 | For example, if you would call the following command ... :: 88 | 89 | $ bidsify -c /home/user/data/config.yml -d /home/user/data 90 | 91 | ... your bidsified data will be in the following location:: 92 | 93 | /home/user 94 | ├── data 95 | | ├── config.yml 96 | | ├── s01 97 | | └── s02 98 | | 99 | └── bids 100 | ├── dataset_description.json 101 | ├── participants.tsv 102 | ├── sub-01 103 | └── sub-02 104 | 105 | Features 106 | -------- 107 | This package aims to take in any MRI-dataset and convert it to BIDS using information from the 108 | config-file provided by the user. Obviously, ``bidsify`` cannot handle *all* types of scans/data, 109 | but it can process most of the default scans/files we use at our MRI centre (Spinoza Centre), including 110 | 111 | - Standard (gradient-echo) EPI scans, both multiband and sequential 112 | - Standard (spin-echo) DWI scans 113 | - "Pepolar" (gradient-echo) EPI scans (also called "topup") 114 | - B0-based fieldmap scans (1 phase-difference + 1 magnitude image) 115 | - T1-weighted and T2-weighted scans 116 | 117 | ``bidsify`` can handle both PAR/REC and DICOM files. Moreover, in the future we want to enable processing of: 118 | 119 | - Philips physiology-files ("SCANPHYSLOG" files; WIP, not functional yet) 120 | 121 | In terms of "structure", this package allows the following "types" of datasets: 122 | 123 | - Multi-subject, multi-session datasets 124 | 125 | The config file 126 | --------------- 127 | ``bidsify`` only needs a config-file in either the json or YAML format. This file should contain 128 | information that can be used to rename and convert the raw files. 129 | 130 | The config file contains a couple of sections, which 131 | are explained below (we'll use the YAML format). 132 | 133 | "options" 134 | ~~~~~~~~~ 135 | The first (top-level) section (or "attribute" in JSON/YAML-lingo) in the file 136 | is the `"options"` section. An example of this section could be: 137 | 138 | .. code-block:: yaml 139 | 140 | options: 141 | mri_ext: PAR # alternatives: DICOM, dcm, nifti 142 | debug: False 143 | n_cores: -1 144 | subject_stem: sub 145 | deface: True 146 | spinoza_data: True 147 | out_dir: bids 148 | 149 | No options *need* to be set explicitly as they all have sensible defaults. 150 | The attribute-value pairs mean the following: 151 | 152 | - ``mri_type``: filetype of MRI-scans (PAR, dcm, DICOM, nifti; default: PAR) 153 | - ``n_cores``: how many CPUs to use during conversion (default: -1, all CPUs) 154 | - ``debug``: whether to print extra output for debugging (default: False) 155 | - ``subject_stem``: prefix for subject-directories, e.g. "subject" in "subject-001" (default: sub) 156 | - ``deface``: whether to deface the data (default: True, takes substantially longer though) 157 | - ``spinoza_data``: whether data is from the `Spinoza centre `_ (default: False) 158 | - ``out_dir``: name of directory to save results to (default: bids), relative to project-root. 159 | 160 | Note that with respect to DICOM files, the ``mri_type`` can be set to ``DICOM`` (referring to Philips [enhanced] DICOM files) or ``dcm`` (referring to Siemens DICOM files with the extension ``.dcm``). 161 | 162 | "mappings" 163 | ~~~~~~~~~~ 164 | The BIDS-format specifies the naming and format of several types of MRI(-related) filetypes. 165 | These filetypes have specific suffixes, which are appended to the filenames in the renaming 166 | process handled by ``bidsify``. The `"mappings"` section in the config is meant to 167 | tell ``bidsify`` what filetype can be identified by which "key". Thus, the mappings 168 | section consists of `"filetype": "identifier"` pairs. Basically, if BIDS requires a 169 | specific suffix for a filetype, you need to specify that here. For example, a standard 170 | dataset with several BOLD-fMRI files, a T1, and physiological recordings could have 171 | a mappings section like this: 172 | 173 | .. code-block:: yaml 174 | 175 | options: 176 | # ............. # 177 | 178 | mappings: 179 | bold: _func 180 | T1w: 3DT1 181 | dwi: DWI 182 | physio: ppuresp 183 | events: log 184 | phasediff: _ph 185 | magnitude: _mag 186 | epi: topup 187 | T2w: T2w 188 | 189 | Note that *every file should belong to one, and only one, file-type*! In other words, ``bidsify`` should be able to figure out what kind of file it's dealing with from the filename. For example, if you have a file named ``my_mri_file.PAR`` and you have configured the mappings as in the example above, ``bidsify`` won't be able to figure out what file-type it's dealing with (a ``bold`` file? A ``T1w`` file?), because the filename does not contain *any* of the mappings (e.g., ``_func``, ``3DT1``, or ``DWI``). 190 | 191 | Moreover, the filename should not contain *more than one file-type identifier*! Suppose you have a file named ``workingmemory_func_ppuresp.nii.gz``; with the above mappings, ``bidsify`` would conclude that it's either a ``bold`` file (because the name contains ``_func``) OR a ``physio`` file (because the name contains ``ppuresp``). As such, ``bidsify`` is going to skip converting/renaming this file and move it to the `unallocated` directory. In summary: files should contain one, and *only one*, identifier (such as ``_func``) mapping to a particular file-type (e.g., ``bold``). 192 | 193 | Also, check the BIDS-specification for all filetypes supported by the format. 194 | 195 | "metadata" 196 | ~~~~~~~~~~ 197 | At the same (hierarchical) level as the "mappings" and "options" sections, a section 198 | with the name "metadata" can be optionally specified. This attribute may contain an 199 | arbitrary amount of attribute-value pairs which will be appended to **each** 200 | JSON-metadata file during the conversion. These are thus "dataset-general" metadata 201 | parameters. For example, you could specify the data of conversion here, if you'd like: 202 | 203 | .. code-block:: yaml 204 | 205 | options: 206 | # some options 207 | 208 | mappings: 209 | # some mappings 210 | 211 | metadata: 212 | MagneticFieldStrength: 3 213 | ParallelAcquisitionTechnique: SENSE 214 | InstitutionName: Spinoza Centre for Neuroimaging, location REC 215 | 216 | The ``func``, ``anat``, ``dwi``, and ``fmap`` sections 217 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 218 | After the ``options``, ``mappings``, and (optionally) the ``metadata`` sections, 219 | the specifications for the four general "BIDS-datatypes" - ``func``, ``anat``, ``dwi``, and ``fmap`` - 220 | are listed in separate sections. 221 | 222 | Each section, like ``func``, can contain multiple sub-sections referring to different scans 223 | for that datatype. For example, you could have two different functional runs 224 | with each a different task ("workingmemory" and "nback"). In that case, the "func" 225 | section could look like: 226 | 227 | .. code-block:: yaml 228 | 229 | options: 230 | # some options 231 | 232 | mappings: 233 | # some mappings 234 | 235 | func: 236 | 237 | wm-task: 238 | id: wmtask 239 | task: workingmemory 240 | 241 | nback-task: 242 | id: nbacktask 243 | task: nback 244 | 245 | The exact naming of the "attributes" (here: ``wm-task`` and ``nback-task``) of the sub-sections 246 | do not matter, but the subsequent key-value pairs *do* matter. You *always* need to set the ``id`` 247 | key, which is used to identify the files that belong to this particular task. Any key-value pair 248 | besides the ``id`` key-value pair are append to the renamed filename along the BIDS-format. 249 | 250 | For example, suppose you have a raw file ``sub-001_wmtask.PAR``. With the above config-file, this file 251 | will be renamed into ``sub-001_task-workingmemory_bold.nii.gz``. 252 | 253 | As discussed, *any* key-value pair besides ``id`` will be appended (in the format "key-value") to the 254 | filename during the renaming-process. Imagine, for example, that you have only one task - "nback" - but 255 | you acquired four runs of it per subject, of which the first two were acquired with a sequential acquisition protocol, 256 | but the last two with a multiband protocol (e.g. if you'd want to do some methodological comparison). 257 | 258 | The config-file should, in that case, look like: 259 | 260 | .. code-block:: yaml 261 | 262 | options: 263 | # some options 264 | 265 | mappings: 266 | # some mappings 267 | 268 | func: 269 | 270 | nback-task1: 271 | id: nback1 272 | task: nback 273 | run: 1 274 | acq: sequential 275 | 276 | nback-task2: 277 | id: nback1 278 | task: nback 279 | run: 2 280 | acq: sequential 281 | 282 | nback-task3: 283 | id: nback3 284 | task: nback 285 | run: 3 286 | acq: multiband 287 | 288 | nback-task4: 289 | id: nback4 290 | task: nback 291 | run: 4 292 | acq: multiband 293 | 294 | ``bidsify`` will then create four files (assuming that they can be "found" using their corresponding ``id``s): 295 | 296 | - ``sub-001_task-nback_run-1_acq-sequential_bold.nii.gz`` 297 | - ``sub-001_task-nback_run-2_acq-sequential_bold.nii.gz`` 298 | - ``sub-001_task-nback_run-3_acq-multiband_bold.nii.gz`` 299 | - ``sub-001_task-nback_run-4_acq-multiband_bold.nii.gz`` 300 | 301 | The same logic can be applied to the "dwi", "anat", and "fmap" sections. For example, if you would have 302 | two T1-weighted structural scans, the "anat" section could look like: 303 | 304 | .. code-block:: yaml 305 | 306 | options: 307 | # some options 308 | 309 | mappings: 310 | # some mappings 311 | 312 | anat: 313 | 314 | firstT1: 315 | id: 3DT1_1 316 | run: 1 317 | 318 | secondT1: 319 | id: 3DT1_2 320 | run: 2 321 | 322 | Importantly, any UNIX-style wildcard (e.g. \*, ?, and [a,A,1-9]) can be used in the 323 | ``id`` values in these sections! 324 | 325 | Lastly, apart from the different elements (such as ``nback-task1`` in the previous example), 326 | each datatype-section (``func``, ``anat``, ``fmap``, and ``dwi``) also may include a 327 | ``metadata`` section, similar to the "toplevel" ``metadata`` section. This field may 328 | include key-value pairs that will be appended to *each* JSON-file within that 329 | datatype. This is especially nice if you'd want to add metadata that is needed for 330 | specific preprocessing/analysis pipelines that are based on the BIDS-format. 331 | For example, the `fmriprep `_ package provides 332 | preprocessing pipelines for BIDS-datasets, but sometimes need specific metadata. 333 | For example, for each BOLD-fMRI file, it needs a field ``EffectiveEchoSpacing`` in the 334 | corresponding JSON-file, and for B0-files (one phasediff, one magnitude image) it needs 335 | the fields ``EchoTime1`` and ``EchoTime2``. To include those metadata fields in the 336 | corresponding JSON-files, just include a ``metadata`` field under the appropriate 337 | datatype section. For example, to do so for the previous examples: 338 | 339 | .. code-block:: yaml 340 | 341 | func: 342 | 343 | metadata: 344 | EffectiveEchoSpacing: 0.00365 345 | PhaseEncodingDirection: "j" 346 | 347 | nback: 348 | id: nback 349 | task: nback 350 | 351 | fmap: 352 | 353 | metadata: 354 | EchoTime1: 0.003 355 | EchoTime2: 0.008 356 | 357 | B0: 358 | id: B0 359 | 360 | How to use ``bidsify`` 361 | ---------------------- 362 | After installing this package, the ``bidsify`` command should be available. 363 | This command assumes a specific organization of your directory with raw data. 364 | Below, I outlined the assumed structure for a simple dataset with one BOLD run and one T1-weighted scan across 365 | two sessions:: 366 | 367 | /home/user/data/ 368 | ├── config.yml 369 | ├── sub-01 370 | │   ├── ses-1 371 | │   │   ├── boldrun1.PAR 372 | │   │   ├── boldrun1.REC 373 | │   │   ├── T1.PAR 374 | │   │   └── T1.REC 375 | │   └── ses-2 376 | │   ├── boldrun1.PAR 377 | │   ├── boldrun1.REC 378 | │   ├── T1.PAR 379 | │   └── T1.REC 380 | └── sub-02 381 | ├── ses-1 382 | │   ├── boldrun1.PAR 383 | │   ├── boldrun1.REC 384 | │   ├── T1.PAR 385 | │   └── T1.REC 386 | └── ses-2 387 | ├── boldrun1.PAR 388 | ├── boldrun1.REC 389 | ├── T1.PAR 390 | └── T1.REC 391 | 392 | (If you have DICOM-files with the ``.dcm`` extension, just replace the PAR/REC files with a single `dcm` file.) 393 | 394 | So all raw files should be in a **single** directory, which can be the subject-directory or, optionally, 395 | a session-directory. **Note**: the session directory **must** be named "ses-". 396 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | build: false 2 | 3 | clone_folder: c:\projects\bidsify 4 | 5 | environment: 6 | matrix: 7 | - PYTHON: "C:\\Python36" 8 | PYTHON_VERSION: "3.6.5" 9 | PYTHON_ARCH: "64" 10 | MINICONDA: C:\Miniconda36 11 | 12 | init: 13 | - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH% %MINICONDA%" 14 | 15 | install: 16 | - git clone https://github.com/neurolabusc/dcm2niix c:\projects\dcm2niix 17 | - mkdir c:\projects\dcm2niix\build 18 | - cd c:\projects\dcm2niix\build 19 | - cmake -G "Visual Studio 14 2015 Win64" -DBATCH_VERSION=ON -DUSE_OPENJPEG=ON ..\ 20 | - MSBuild c:\projects\dcm2niix\build\dcm2niix.sln 21 | - set PATH=c:\projects\dcm2niix\build\bin;%PATH% 22 | - npm install -g bids-validator 23 | - cd c:\projects\bidsify 24 | - "set PATH=%MINICONDA%;%MINICONDA%\\Scripts;%PATH%" 25 | - conda config --set always_yes yes --set changeps1 no 26 | - conda update -q conda 27 | - conda info -a 28 | - "conda create -q -n test-environment python=%PYTHON_VERSION% numpy joblib pandas pytest pytest-cov" 29 | - activate test-environment 30 | - pip install coverage nibabel 31 | - python setup.py install 32 | 33 | test_script: 34 | - python download_test_data.py 35 | - cd c:\projects\bidsify 36 | - py.test --cov=bidsify 37 | -------------------------------------------------------------------------------- /bidsify/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, division, print_function 2 | from .main import bidsify # noqa 3 | -------------------------------------------------------------------------------- /bidsify/data/dataset_description.json: -------------------------------------------------------------------------------- 1 | { 2 | "Name": "Put the name of your experiment here", 3 | "BIDSVersion": "1.0.2", 4 | "License": "License under which your data is distributed", 5 | "Authors": ["Author1", "Author2", "Author3", "etc."], 6 | "Acknowledgements": "Put your acknowledgements here", 7 | "HowToAcknowledge": "Describe how you'd like to be acknowledged here", 8 | "Funding": "Put your funding sources here", 9 | "ReferencesAndLinks": ["e.g. data-paper", "(methods-)paper", "etc."], 10 | "DatasetDOI": "DOI of dataset (if there is one)" 11 | } 12 | -------------------------------------------------------------------------------- /bidsify/data/example_config.yml: -------------------------------------------------------------------------------- 1 | options: 2 | mri_ext: PAR # alternatives: nifti/dcm/DICOM 3 | debug: False # alternative: True, prints out a lot of stuff 4 | n_cores: -1 # number of CPU cores to use (for some operations) 5 | subject_stem: sub # subject identifier 6 | deface: True # whether to deface structural scans 7 | spinoza_data: False # only relevant for data acquired at the Spinoza Centre 8 | 9 | mappings: 10 | bold: _bold 11 | T1w: _T1w 12 | dwi: _dwi 13 | physio: _physio 14 | events: _events 15 | phasediff: _phasediff 16 | magnitude1: _magnitude1 17 | epi: _topup 18 | T2w: _T2w 19 | FLAIR: _FLAIR 20 | 21 | metadata: # will be appended to EACH file 22 | MagneticFieldStrength: 3 23 | ParallelAcquisitionTechnique: SENSE 24 | BIDSVersion: '1.1.0' 25 | InstitutionName: Spinoza Centre for Neuroimaging, location RE 26 | 27 | anat: 28 | single_T1: # this name doesn't matter 29 | id: t13d # identifier to this type of scan 30 | acq: 3min # optional 31 | 32 | func: 33 | restingstate: 34 | id: pioprs 35 | task: rest 36 | acq: MB3Sense2Mm3 37 | harriri: 38 | id: harriri 39 | task: harriri 40 | acq: SeqMm3Tr2000 41 | 42 | dwi: 43 | dwi: 44 | id: dti32 45 | acq: SeqSense2Dirs32 46 | -------------------------------------------------------------------------------- /bidsify/data/spinoza_cfg.yml: -------------------------------------------------------------------------------- 1 | options: 2 | mri_ext: PAR # may change to enh DICOM 3 | debug: False 4 | n_cores: -1 5 | subject_stem: sub 6 | deface: True 7 | spinoza_data: True 8 | out_dir: bids 9 | 10 | mappings: 11 | bold: _bold 12 | T1w: _T1w 13 | dwi: _dwi 14 | physio: _physio 15 | events: _events 16 | phasediff: _phasediff 17 | magnitude1: _magnitude1 18 | epi: _topup 19 | T2w: _T2w 20 | FLAIR: _FLAIR 21 | 22 | metadata: 23 | MagneticFieldStrength: 3 24 | ParallelAcquisitionTechnique: SENSE 25 | BIDSVersion: '1.1.0' 26 | InstitutionName: Spinoza Centre for Neuroimaging, location REC 27 | -------------------------------------------------------------------------------- /bidsify/data/spinoza_cfg_dicom.yml: -------------------------------------------------------------------------------- 1 | options: 2 | mri_ext: DICOM 3 | debug: False 4 | n_cores: -1 5 | subject_stem: sub 6 | deface: True 7 | spinoza_data: True 8 | out_dir: 'bids' 9 | 10 | mappings: 11 | bold: _bold 12 | T1w: _T1w 13 | T2w: _T2w 14 | FLAIR: _FLAIR 15 | dwi: _dwi 16 | physio: _physio 17 | events: _events 18 | phasediff: _phasediff 19 | magnitude1: _magnitude1 20 | epi: _topup 21 | 22 | metadata: 23 | MagneticFieldStrength: 3 24 | ParallelAcquisitionTechnique: SENSE 25 | BIDSVersion: '1.1.0' 26 | InstitutionName: Spinoza Centre for Neuroimaging, location REC 27 | -------------------------------------------------------------------------------- /bidsify/data/spinoza_metadata.yml: -------------------------------------------------------------------------------- 1 | # Structure: dtype --> mtype --> acqtype 2 | func: # dtype 3 | 4 | bold: # mtype 5 | 6 | SeqSense2Mm3: # acqtype 7 | # OLD: standard non-MB sequence 8 | PulseSequenceType: Gradient Echo EPI 9 | EchoTime: 0.028 10 | RepetitionTime: 2 11 | PhaseEncodingDirection: j 12 | SliceEncodingDirection: k 13 | ParallelAcquisitionTechnique: SENSE 14 | ParallelReductionFactorInPlane: 2 15 | WaterFatShift: 12.000 16 | FlipAngle: 76.10 17 | TotalReadoutTime: 0.0269452 18 | EffectiveEchoSpacing: 0.0006909 19 | 20 | SeqMm3Tr2000: # new name of above 21 | PulseSequenceType: Gradient Echo EPI 22 | EchoTime: 0.028 23 | RepetitionTime: 2 24 | PhaseEncodingDirection: j 25 | SliceEncodingDirection: k 26 | ParallelAcquisitionTechnique: SENSE 27 | ParallelReductionFactorInPlane: 2 28 | WaterFatShift: 12.000 29 | FlipAngle: 76.10 30 | TotalReadoutTime: 0.0269452 31 | EffectiveEchoSpacing: 0.0006909 32 | 33 | MB3Sense2Mm3: 34 | # old MB3 sequence 35 | PulseSequenceType: Multiband gradient echo EPI 36 | EchoTime: 0.028 37 | RepetitionTime: 0.764 38 | PhaseEncodingDirection: j 39 | SliceEncodingDirection: k 40 | ParallelAcquisitionTechnique: SENSE 41 | ParallelReductionFactorInPlane: 2 42 | MultibandAccelerationFactor: 3 43 | WaterFatShift: 12.000 44 | FlipAngle: 60 45 | TotalReadoutTime: 0.0269452 46 | EffectiveEchoSpacing: 0.0006909 47 | 48 | 49 | Mb4Mm3Tr550: 50 | # new MB4 sequence 51 | PulseSequenceType: Multiband gradient echo EPI 52 | EchoTime: 0.030 53 | RepetitionTime: 0.550 54 | PhaseEncodingDirection: j 55 | SliceEncodingDirection: k 56 | ParallelAcquisitionTechnique: SENSE 57 | ParallelReductionFactorInPlane: 1.51 58 | MultibandAccelerationFactor: 4 59 | WaterFatShift: 14.308 60 | FlipAngle: 55 61 | TotalReadoutTime: 0.0323413 62 | EffectiveEchoSpacing: 0.0006102 63 | 64 | MB4Mm3Tr550: # typfout 65 | # new MB4 sequence 66 | PulseSequenceType: Multiband gradient echo EPI 67 | EchoTime: 0.030 68 | RepetitionTime: 0.550 69 | PhaseEncodingDirection: j 70 | SliceEncodingDirection: k 71 | ParallelAcquisitionTechnique: SENSE 72 | ParallelReductionFactorInPlane: 1.51 73 | MultibandAccelerationFactor: 4 74 | WaterFatShift: 14.308 75 | FlipAngle: 55 76 | TotalReadoutTime: 0.0323413 77 | EffectiveEchoSpacing: 0.0006102 78 | 79 | Mb4Mm27Tr700: 80 | # new MB4 sequence 81 | PulseSequenceType: Multiband gradient echo EPI 82 | EchoTime: 0.030 83 | RepetitionTime: 0.700 84 | PhaseEncodingDirection: j 85 | SliceEncodingDirection: k 86 | ParallelAcquisitionTechnique: SENSE 87 | ParallelReductionFactorInPlane: 1.50 88 | MultibandAccelerationFactor: 4 89 | WaterFatShift: 14.388 90 | FlipAngle: 55 91 | TotalReadoutTime: 0.0325221 92 | EffectiveEchoSpacing: 0.0006136 93 | 94 | Mb4Mm2Tr1600: 95 | # new MB4 sequence 96 | PulseSequenceType: Multiband gradient echo EPI 97 | EchoTime: 0.030 98 | RepetitionTime: 1.6 99 | PhaseEncodingDirection: j 100 | SliceEncodingDirection: k 101 | ParallelAcquisitionTechnique: SENSE 102 | ParallelReductionFactorInPlane: 1.50 103 | MultibandAccelerationFactor: 4 104 | WaterFatShift: 23.709 105 | FlipAngle: 70 106 | TotalReadoutTime: 0.0538836 107 | EffectiveEchoSpacing: 0.0007184 108 | 109 | Mb4Mm2Tr1600EnhStab: 110 | # new MB4 sequence w/enhanced dynamic stabilization 111 | PulseSequenceType: Multiband gradient echo EPI 112 | EchoTime: 0.030 113 | RepetitionTime: 1.6 114 | PhaseEncodingDirection: j 115 | SliceEncodingDirection: k 116 | ParallelAcquisitionTechnique: SENSE 117 | ParallelReductionFactorInPlane: 1.50 118 | MultibandAccelerationFactor: 4 119 | WaterFatShift: 23.709 120 | FlipAngle: 70 121 | TotalReadoutTime: 0.0538836 122 | EffectiveEchoSpacing: 0.0007184 123 | 124 | Mb4Mm2Tr1800: 125 | # modified version of new MB4 sequence (project MindWandering) 126 | PulseSequenceType: Multiband gradient echo EPI 127 | EchoTime: 0.030 128 | RepetitionTime: 1.8 129 | PhaseEncodingDirection: j 130 | SliceEncodingDirection: k 131 | ParallelAcquisitionTechnique: SENSE 132 | ParallelReductionFactorInPlane: 1.50 133 | MultibandAccelerationFactor: 4 134 | WaterFatShift: 23.709 135 | FlipAngle: 70 136 | TotalReadoutTime: 0.0538836 137 | EffectiveEchoSpacing: 0.0007184 138 | 139 | fmap: # dtype 140 | 141 | phasediff: # mtype 142 | 143 | Mm2: # acqtype 144 | EchoTime1: 0.003 145 | EchoTime2: 0.008 146 | FlipAngle: 8 147 | ParallelAcquisitionTechnique: SENSE 148 | ParallelReductionFactorInPlane: 2.5 149 | WaterFatShift: 1.134 150 | 151 | epi: # a.k.a. topup (but 'epi' according to BIDS) 152 | 153 | SeqSense2Mm3: # acqtype 154 | # OLD: standard non-MB sequence 155 | PulseSequenceType: Gradient Echo EPI 156 | EchoTime: 0.028 157 | RepetitionTime: 2 158 | PhaseEncodingDirection: j- 159 | SliceEncodingDirection: k 160 | ParallelAcquisitionTechnique: SENSE 161 | ParallelReductionFactorInPlane: 2 162 | WaterFatShift: 12.000 163 | FlipAngle: 76.10 164 | TotalReadoutTime: 0.0269452 165 | EffectiveEchoSpacing: 0.0006909 166 | 167 | SeqMm3Tr2000: # new name of above 168 | PulseSequenceType: Gradient Echo EPI 169 | EchoTime: 0.028 170 | RepetitionTime: 2 171 | PhaseEncodingDirection: j- 172 | SliceEncodingDirection: k 173 | ParallelAcquisitionTechnique: SENSE 174 | ParallelReductionFactorInPlane: 2 175 | WaterFatShift: 12.000 176 | FlipAngle: 76.10 177 | TotalReadoutTime: 0.0269452 178 | EffectiveEchoSpacing: 0.0006909 179 | 180 | MB3Sense2Mm3: 181 | # old MB3 sequence 182 | PulseSequenceType: Multiband gradient echo EPI 183 | EchoTime: 0.028 184 | RepetitionTime: 0.764 185 | PhaseEncodingDirection: j- 186 | SliceEncodingDirection: k 187 | ParallelAcquisitionTechnique: SENSE 188 | ParallelReductionFactorInPlane: 2 189 | MultibandAccelerationFactor: 3 190 | WaterFatShift: 12.000 191 | FlipAngle: 60 192 | TotalReadoutTime: 0.0269452 193 | EffectiveEchoSpacing: 0.0006909 194 | 195 | Mb4Mm3Tr550: 196 | # new MB4 sequence 197 | PulseSequenceType: Multiband gradient echo EPI 198 | EchoTime: 0.030 199 | RepetitionTime: 0.550 200 | PhaseEncodingDirection: j- 201 | SliceEncodingDirection: k 202 | ParallelAcquisitionTechnique: SENSE 203 | ParallelReductionFactorInPlane: 1.51 204 | MultibandAccelerationFactor: 4 205 | WaterFatShift: 14.308 206 | FlipAngle: 55 207 | TotalReadoutTime: 0.0323413 208 | EffectiveEchoSpacing: 0.0006102 209 | 210 | MB4Mm3Tr550: # typfout 211 | # new MB4 sequence 212 | PulseSequenceType: Multiband gradient echo EPI 213 | EchoTime: 0.030 214 | RepetitionTime: 0.550 215 | PhaseEncodingDirection: j- 216 | SliceEncodingDirection: k 217 | ParallelAcquisitionTechnique: SENSE 218 | ParallelReductionFactorInPlane: 1.51 219 | MultibandAccelerationFactor: 4 220 | WaterFatShift: 14.308 221 | FlipAngle: 55 222 | TotalReadoutTime: 0.0323413 223 | EffectiveEchoSpacing: 0.0006102 224 | 225 | Mb4Mm27Tr700: 226 | # new MB4 sequence 227 | PulseSequenceType: Multiband gradient echo EPI 228 | EchoTime: 0.030 229 | RepetitionTime: 0.700 230 | PhaseEncodingDirection: j- 231 | SliceEncodingDirection: k 232 | ParallelAcquisitionTechnique: SENSE 233 | ParallelReductionFactorInPlane: 1.50 234 | MultibandAccelerationFactor: 4 235 | WaterFatShift: 14.388 236 | FlipAngle: 55 237 | TotalReadoutTime: 0.0325221 238 | EffectiveEchoSpacing: 0.0006136 239 | 240 | Mb4Mm2Tr1600: 241 | # new MB4 sequence 242 | PulseSequenceType: Multiband gradient echo EPI 243 | EchoTime: 0.030 244 | RepetitionTime: 1.6 245 | PhaseEncodingDirection: j- 246 | SliceEncodingDirection: k 247 | ParallelAcquisitionTechnique: SENSE 248 | ParallelReductionFactorInPlane: 1.50 249 | MultibandAccelerationFactor: 4 250 | WaterFatShift: 23.709 251 | FlipAngle: 70 252 | TotalReadoutTime: 0.0538836 253 | EffectiveEchoSpacing: 0.0007184 254 | 255 | Mb4Mm2Tr1600EnhStab: 256 | # new MB4 sequence w/enhanced dynamic stabilization 257 | PulseSequenceType: Multiband gradient echo EPI 258 | EchoTime: 0.030 259 | RepetitionTime: 1.6 260 | PhaseEncodingDirection: j- 261 | SliceEncodingDirection: k 262 | ParallelAcquisitionTechnique: SENSE 263 | ParallelReductionFactorInPlane: 1.50 264 | MultibandAccelerationFactor: 4 265 | WaterFatShift: 23.709 266 | FlipAngle: 70 267 | TotalReadoutTime: 0.0538836 268 | EffectiveEchoSpacing: 0.0007184 269 | 270 | Mb4Mm2Tr1800: 271 | # modified version of new MB4 sequence (project MindWandering) 272 | PulseSequenceType: Multiband gradient echo EPI 273 | EchoTime: 0.030 274 | RepetitionTime: 1.8 275 | PhaseEncodingDirection: j- 276 | SliceEncodingDirection: k 277 | ParallelAcquisitionTechnique: SENSE 278 | ParallelReductionFactorInPlane: 1.50 279 | MultibandAccelerationFactor: 4 280 | WaterFatShift: 23.709 281 | FlipAngle: 70 282 | TotalReadoutTime: 0.0538836 283 | EffectiveEchoSpacing: 0.0007184 284 | 285 | SeqSense2Dirs32: # old name 286 | EchoTime: 0.086 287 | RepetitionTime: 7.464 288 | PhaseEncodingDirection: j- 289 | ParallelAcquisitionTechnique: SENSE 290 | ParallelReductionFactorInPlane: 2 291 | WaterFatShift: 18.925 292 | FlipAngle: 90 293 | TotalReadoutTime: 0.0428062 294 | EffectiveEchoSpacing: 0.0007783 295 | 296 | SeqMm2Dirs32: # new name 297 | EchoTime: 0.086 298 | RepetitionTime: 7.464 299 | PhaseEncodingDirection: j- 300 | ParallelAcquisitionTechnique: SENSE 301 | ParallelReductionFactorInPlane: 2 302 | WaterFatShift: 18.925 303 | FlipAngle: 90 304 | TotalReadoutTime: 0.0428062 305 | EffectiveEchoSpacing: 0.0007783 306 | 307 | Mb3Dirs128: 308 | RepetitionTime: 2.550 309 | PhaseEncodingDirection: j- 310 | ParallelReductionFactorInPlane: 1.3 311 | MultibandAccelerationFactor: 3 312 | WaterFatShift: 18.925 313 | FlipAngle: 90 314 | TotalReadoutTime: 0.0428062 # to check 315 | EffectiveEchoSpacing: 0.0007783 # to check 316 | dwi: 317 | 318 | dwi: 319 | SeqSense2Dirs32: 320 | # Old name 321 | EchoTime: 0.086 322 | RepetitionTime: 7.464 323 | PhaseEncodingDirection: j 324 | ParallelAcquisitionTechnique: SENSE 325 | ParallelReductionFactorInPlane: 2 326 | WaterFatShift: 18.925 327 | FlipAngle: 90 328 | TotalReadoutTime: 0.0428062 329 | EffectiveEchoSpacing: 0.0007783 330 | 331 | SeqMm2Dirs32: 332 | # New name 333 | EchoTime: 0.086 334 | RepetitionTime: 7.464 335 | PhaseEncodingDirection: j- 336 | ParallelAcquisitionTechnique: SENSE 337 | ParallelReductionFactorInPlane: 2 338 | WaterFatShift: 18.925 339 | FlipAngle: 90 340 | TotalReadoutTime: 0.0428062 341 | EffectiveEchoSpacing: 0.0007783 342 | 343 | Mb3Dirs128: 344 | RepetitionTime: 2.550 345 | PhaseEncodingDirection: j 346 | ParallelReductionFactorInPlane: 1.3 347 | MultibandAccelerationFactor: 3 348 | WaterFatShift: 18.925 349 | FlipAngle: 90 350 | TotalReadoutTime: 0.0428062 # to check 351 | EffectiveEchoSpacing: 0.0007783 # to check 352 | -------------------------------------------------------------------------------- /bidsify/data/test_data/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NILAB-UvA/bidsify/95db2a19178375ffb09b46367b174d3fe13524e6/bidsify/data/test_data/.gitkeep -------------------------------------------------------------------------------- /bidsify/docker.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import os.path as op 4 | from datetime import datetime 5 | from .utils import _run_cmd 6 | from .version import __version__ 7 | 8 | 9 | def run_from_docker(cfg_path, directory, out_dir, validate, spinoza, uid=None, nolog=False, name=None): 10 | """ Runs bidsify from Docker. """ 11 | 12 | if name is None: 13 | today = datetime.now().strftime("%Y%m%d") 14 | basedir = op.basename(op.dirname(directory)) 15 | name = 'bidsify_%s_%s' % (basedir, today) 16 | 17 | proj_name = op.basename(op.dirname(directory)) 18 | date = str(datetime.now().strftime("%Y-%m-%d")) 19 | if not nolog: 20 | if spinoza: 21 | log_file = op.join(op.dirname(op.dirname(out_dir)), 'logs', 'project-%s_stage-bidsify_%s' % (proj_name, date)) 22 | else: 23 | log_file = op.join(out_dir, 'log') 24 | 25 | print("Writing logfile to %s ..." % log_file) 26 | 27 | if uid is None: 28 | uid = str(os.getuid()) # note: if run by CRON, this is root! 29 | else: 30 | str(uid) 31 | 32 | cmd = ['docker', 'run', '--rm', 33 | '-u', uid + ':' + uid, 34 | '-v', '%s:/data' % directory, 35 | '-v', '%s:/config.yml' % cfg_path, 36 | '-v', '%s:/bids' % out_dir, 37 | #'--name %s' % name, 38 | 'lukassnoek/bidsify:%s' % __version__, 'bidsify', '-c', '/config.yml', '-d', '/data', '-o', '/bids'] 39 | 40 | if validate: 41 | cmd.append('-v') 42 | 43 | if spinoza: 44 | cmd.append('-s') 45 | 46 | if not op.isdir(out_dir): 47 | # Need to create dir beforehand, otherwise it's owned by root 48 | os.makedirs(out_dir) 49 | 50 | 51 | print("RUNNING:") 52 | print(' '.join(cmd)) 53 | 54 | if not nolog: 55 | fout = open(log_file + '_stdout.txt', 'w') 56 | ferr = open(log_file + '_stderr.txt', 'w') 57 | subprocess.run(cmd, stdout=fout, stderr=ferr) 58 | fout.close() 59 | ferr.close() 60 | else: 61 | subprocess.run(cmd) 62 | -------------------------------------------------------------------------------- /bidsify/main.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, division, print_function 2 | import os 3 | import os.path as op 4 | import argparse 5 | import shutil 6 | import fnmatch 7 | import warnings 8 | import yaml 9 | import json 10 | import pandas as pd 11 | import nibabel as nib 12 | import numpy as np 13 | from copy import copy, deepcopy 14 | from glob import glob 15 | from joblib import Parallel, delayed 16 | from .mri2nifti import convert_mri 17 | from .phys2tsv import convert_phy 18 | from .docker import run_from_docker 19 | from .utils import (check_executable, _make_dir, _append_to_json, 20 | _run_cmd) 21 | from .version import __version__ 22 | 23 | 24 | __all__ = ['run_cmd', 'bidsify'] 25 | 26 | DTYPES = ['func', 'anat', 'fmap', 'dwi'] 27 | 28 | MTYPE_PER_DTYPE = dict( 29 | func=['bold'], 30 | anat=['T1w', 'T2w', 'FLAIR'], 31 | dwi=['dwi'], 32 | fmap=['phasediff', 'magnitude1', 'epi'] 33 | ) 34 | 35 | MTYPE_ORDERS = dict( 36 | T1w=dict(sub=0, ses=1, acq=2, ce=3, rec=4, run=5, T1w=6), 37 | T2w=dict(sub=0, ses=1, acq=2, ce=3, rec=4, run=5, T2w=6), 38 | FLAIR=dict(sub=0, ses=1, acq=2, ce=3, rec=4, run=5, FLAIR=6), 39 | bold=dict(sub=0, ses=1, task=2, acq=3, rec=4, run=5, echo=6, bold=7), 40 | events=dict(sub=0, ses=1, task=2, acq=3, rec=4, run=5, echo=6, events=7), 41 | physio=dict(sub=0, ses=1, task=2, acq=3, rec=4, run=5, echo=6, recording=7, 42 | physio=8), 43 | stim=dict(sub=0, ses=1, task=2, acq=3, rec=4, run=5, echo=6, recording=7, 44 | stim=8), 45 | dwi=dict(sub=0, ses=1, acq=2, run=3, dwi=4), 46 | phasediff=dict(sub=0, ses=1, acq=2, run=3, phasediff=4), 47 | magnitude1=dict(sub=0, ses=1, acq=2, run=3, magnitude=4), 48 | epi=dict(sub=0, ses=1, acq=2, dir=3, run=4, echo=5, epi=6) 49 | ) 50 | 51 | # For some reason, people seem to use periods in filenames, so 52 | # remove all unnecessary 'extensions' 53 | ALLOWED_EXTS = [ 54 | 'par', 'Par', 'rec', 'Rec', 'nii', 'Ni', 'gz', 'Gz', 'dcm', 55 | 'Dcm', 'dicom', 'Dicom', 'dicomdir', 'Dicomdir', 'pickle', 56 | 'json', 'edf', 'log', 'bz2', 'tar', 'phy', 'cPickle', 'pkl', 57 | 'jl', 'tsv', 'csv', 'txt', 'bval', 'bvec' 58 | ] 59 | ALLOWED_EXTS.extend([s.upper() for s in ALLOWED_EXTS]) 60 | 61 | 62 | def run_cmd(): 63 | """ Calls the bidsify function with cmd line arguments. """ 64 | 65 | DESC = ("This is a command line tool to convert " 66 | "unstructured data-directories to a BIDS-compatible format") 67 | 68 | parser = argparse.ArgumentParser(description=DESC) 69 | 70 | parser.add_argument('-d', '--directory', 71 | help='Directory to be converted.', 72 | required=False, 73 | default=os.getcwd()) 74 | 75 | parser.add_argument('-o', '--out', 76 | help='Directory for output.', 77 | required=False, 78 | default=None) 79 | 80 | parser.add_argument('-c', '--config_file', 81 | help='Config-file with img. acq. parameters', 82 | required=False, 83 | default=op.join(os.getcwd(), 'config.yml')) 84 | 85 | parser.add_argument('-v', '--validate', 86 | help='Run bids-validator', 87 | required=False, action='store_true', 88 | default=False) 89 | 90 | parser.add_argument('-D', '--docker', 91 | help='Whether to run in a Docker container', 92 | required=False, action='store_true', 93 | default=False) 94 | 95 | parser.add_argument('-s', '--spinoza', 96 | help='Whether is is Spinoza-REC data', 97 | required=False, action='store_true', 98 | default=False) 99 | 100 | parser.add_argument('-n', '--nolog', 101 | help='Do not write out log (stdout/err only)', 102 | required=False, action='store_true', 103 | default=False) 104 | args = parser.parse_args() 105 | 106 | if args.out is None: 107 | args.out = op.join(op.dirname(args.directory), 'bids') 108 | print("Setting output-dir to %s" % args.out) 109 | 110 | if args.spinoza: 111 | args.config_file = op.join(op.dirname(__file__), 'data', 'spinoza_cfg.yml') 112 | 113 | if not op.isfile(args.config_file): 114 | raise ValueError("Config-file %s does not exist!" % args.config_file) 115 | 116 | print("Running bidsify with the following arguments:\n" 117 | "\t directory=%s \n" 118 | "\t config=%s \n" 119 | "\t out_dir=%s \n" 120 | "\t validate=%s\n" % (args.directory, args.config_file, args.out, args.validate)) 121 | 122 | if args.docker: 123 | run_from_docker(cfg_path=args.config_file, directory=args.directory, 124 | out_dir=args.out, validate=args.validate, spinoza=args.spinoza, nolog=args.nolog) 125 | else: 126 | bidsify(cfg_path=args.config_file, directory=args.directory, 127 | out_dir=args.out, validate=args.validate) 128 | 129 | 130 | def bidsify(cfg_path, directory, out_dir, validate): 131 | """ Converts (raw) MRI datasets to the BIDS-format [1]. 132 | 133 | Parameters 134 | ---------- 135 | cfg_path : str 136 | Path to config-file (either json or YAML file) 137 | directory : str 138 | Path to directory with raw data 139 | out_dir : str 140 | Path to output-directory 141 | validate : bool 142 | Whether to run bids-validator on the bids-converted data 143 | 144 | Returns 145 | ------- 146 | layout : BIDSLayout object 147 | A BIDSLayout object from the pybids package. 148 | 149 | References 150 | ---------- 151 | .. [1] Gorgolewski, K. J., Auer, T., Calhoun, V. D., Craddock, R. C., 152 | Das, S., Duff, E. P., ... & Handwerker, D. A. (2016). The brain 153 | imaging data structure, a format for organizing and describing 154 | outputs of neuroimaging experiments. Scientific Data, 3, 160044. 155 | """ 156 | 157 | # First, parse the config file 158 | cfg = _parse_cfg(cfg_path, directory, out_dir) 159 | cfg['orig_cfg_path'] = cfg_path 160 | 161 | # Check whether everything is available 162 | if not check_executable('dcm2niix'): 163 | msg = """The program 'dcm2niix' was not found on this computer; 164 | install dcm2niix from neurodebian (Linux users) or download dcm2niix 165 | from Github (link) and compile locally (Mac/Windows); bidsify 166 | needs dcm2niix to convert MRI-files to nifti!. Alternatively, use 167 | the bidsify Docker image (not yet tested)!""" 168 | warnings.warn(msg) 169 | 170 | if not check_executable('bids-validator') and validate: 171 | msg = """The program 'bids-validator' was not found on your computer; 172 | setting the validate option to False""" 173 | warnings.warn(msg) 174 | validate = False 175 | 176 | # Extract some values from cfg for readability 177 | options = cfg['options'] 178 | out_dir = options['out_dir'] 179 | subject_stem = options['subject_stem'] 180 | 181 | # Find subject directories 182 | sub_dirs = [d for d in sorted(glob(op.join(directory, '%s*' % subject_stem))) 183 | if op.isdir(d)] 184 | 185 | if not sub_dirs: 186 | msg = ("Could not find subject dirs in directory %s with subject stem " 187 | "'%s'." % (directory, subject_stem)) 188 | raise ValueError(msg) 189 | 190 | # Process directories of each subject 191 | for sub_dir in sub_dirs: 192 | _process_directory(sub_dir, out_dir, cfg, is_sess=False) 193 | 194 | # Write example description_dataset.json to disk 195 | desc_json = op.join(op.dirname(__file__), 'data', 196 | 'dataset_description.json') 197 | dst = op.join(out_dir, 'dataset_description.json') 198 | shutil.copyfile(src=desc_json, dst=dst) 199 | 200 | # Copy .bidsignore (if any) 201 | bidsignore_file = op.join(directory, '.bidsignore') 202 | if op.isfile(bidsignore_file): 203 | shutil.copyfile(src=bidsignore_file, dst=op.join(out_dir, '.bidsignore')) 204 | 205 | # Write participants.tsv to disk 206 | found_sub_dirs = sorted(glob(op.join(cfg['options']['out_dir'], 'sub-*'))) 207 | sub_names = [op.basename(s) for s in found_sub_dirs] 208 | 209 | participants_tsv = pd.DataFrame(index=range(len(sub_names)), 210 | columns=['participant_id']) 211 | participants_tsv['participant_id'] = sub_names 212 | f_out = op.join(out_dir, 'participants.tsv') 213 | participants_tsv.to_csv(f_out, sep='\t', index=False) 214 | 215 | if validate: 216 | bids_validator_log = op.join(out_dir, 'bids_validator_log.txt') 217 | if op.isfile(bids_validator_log): 218 | print("Removing old BIDS-validator log prior to validation ...") 219 | os.remove(bids_validator_log) 220 | 221 | cmd = ['bids-validator', '--ignoreNiftiHeaders', out_dir] 222 | 223 | rs = _run_cmd(cmd, outfile=bids_validator_log, verbose=True) 224 | if rs == 0: 225 | msg = ("bidsify exited without errors and passed the " 226 | "bids-validator checks! For the complete bids-validator " 227 | "report, see %s." % bids_validator_log) 228 | print(msg) 229 | else: 230 | msg = ("bidsify exited without errors but the bids-validator " 231 | "raised one or more errors. Check the complete " 232 | "bids-validator report here: %s." % bids_validator_log) 233 | f = open(bids_validator_log, 'r') 234 | file_contents = f.read() 235 | print(file_contents) 236 | f.close() 237 | raise ValueError(msg) 238 | 239 | 240 | def _process_directory(cdir, out_dir, cfg, is_sess=False): 241 | """ Main workhorse of bidsify """ 242 | 243 | options = cfg['options'] 244 | n_cores = options['n_cores'] 245 | 246 | if is_sess: 247 | sub_name = _extract_sub_nr(options['subject_stem'], 248 | op.basename(op.dirname(cdir))) 249 | sess_name = op.basename(cdir) 250 | this_out_dir = op.join(out_dir, sub_name, sess_name) 251 | else: 252 | sub_name = _extract_sub_nr(options['subject_stem'], op.basename(cdir)) 253 | this_out_dir = op.join(out_dir, sub_name) 254 | 255 | # Important: to find session-dirs, they should be named 256 | # ses-*something* 257 | sess_dirs = sorted(glob(op.join(cdir, 'ses-*'))) 258 | 259 | if sess_dirs: 260 | # Recursive call to _process_directory 261 | for sess_dir in sess_dirs: 262 | _process_directory(sess_dir, out_dir, cfg, is_sess=True) 263 | 264 | return None # break out of recursive function 265 | 266 | already_exists = op.isdir(this_out_dir) 267 | if already_exists: 268 | print('Data from %s has been converted already - skipping ...' % sub_name) 269 | return None 270 | else: 271 | msg = 'Converting data from %s ...' % sub_name 272 | if is_sess: 273 | msg += ' (%s)' % sess_name 274 | print(msg) 275 | 276 | # Make dir and copy all files to this dir 277 | _make_dir(this_out_dir) 278 | all_files = sorted([f for f in glob(op.join(cdir, '*')) if op.isfile(f)]) 279 | 280 | if not all_files: 281 | all_files = sorted([f for f in glob(op.join(cdir, '*', '*')) if op.isfile(f)]) 282 | 283 | if not all_files: 284 | return None 285 | 286 | for f in all_files: 287 | dst = os.path.join(this_out_dir, op.basename(f)) 288 | if os.path.isdir(f): 289 | shutil.copytree(f, dst) 290 | else: 291 | shutil.copy2(f, dst) 292 | 293 | # First, convert all MRI-files 294 | convert_mri(this_out_dir, cfg) 295 | 296 | # Remove weird ADC file(s); no clue what they represent ... 297 | [os.remove(f) for f in glob(op.join(this_out_dir, '*ADC*.nii.gz'))] 298 | 299 | # If spinoza-data (there is no specific config file), try to infer elements 300 | # from converted data 301 | if 'spinoza_cfg' in op.basename(cfg['orig_cfg_path']): 302 | dtype_elements = _infer_dtype_elements(this_out_dir, cfg) 303 | cfg.update(dtype_elements) 304 | if cfg['options']['debug']: 305 | print("Creating the following config:") 306 | print(json.dumps(cfg, indent = 4)) 307 | 308 | # Check which datatypes (dtypes) are available (func, anat, fmap, dwi) 309 | cfg['data_types'] = [c for c in cfg.keys() if c in DTYPES] 310 | cfg = _extract_metadata_from_cfg(cfg) 311 | 312 | # Rename and move stuff 313 | data_dirs = [] 314 | for dtype in cfg['data_types']: 315 | ddir = _rename(this_out_dir, dtype, sub_name, cfg) 316 | if ddir is not None: 317 | data_dirs.append(ddir) 318 | 319 | # 2. Transform PHYS (if any) 320 | if cfg['mappings']['physio'] is not None: 321 | idf = cfg['mappings']['physio'] 322 | phys = sorted(glob(op.join(this_out_dir, '*', '*%s*' % idf))) 323 | Parallel(n_jobs=n_cores)(delayed(convert_phy)(f) for f in phys) 324 | 325 | # Also, while we're at it, remove bval/bvecs of dwi topups 326 | epi_bvals_bvecs = glob(op.join(this_out_dir, 'fmap', '*_epi.bv[e,a][c,l]')) 327 | [os.remove(f) for f in epi_bvals_bvecs] 328 | 329 | # Let's move stuff that's never allocated to a dtype to the unall dir 330 | unallocated = [f for f in glob(op.join(this_out_dir, '*')) if op.isfile(f)] 331 | if unallocated: 332 | print('Unallocated files for %s:' % sub_name) 333 | print('\n'.join(unallocated)) 334 | 335 | if is_sess: 336 | unall_dir = op.join(out_dir, 'unallocated', sub_name, 337 | sess_name) 338 | else: 339 | unall_dir = op.join(out_dir, 'unallocated', sub_name) 340 | _make_dir(unall_dir) 341 | 342 | for f in unallocated: 343 | # only move if doesn't exist already 344 | if not op.isfile(op.join(unall_dir, op.basename(f))): 345 | shutil.move(f, unall_dir) 346 | else: 347 | os.remove(f) 348 | 349 | # ... and extract some extra meta-data 350 | for data_dir in data_dirs: 351 | _add_missing_BIDS_metadata_and_save_to_disk(data_dir, cfg) 352 | 353 | # Reorient2std 354 | if not 'TRAVIS' in os.environ: 355 | # only run when not on Travis CI (on which FSL is not installed) 356 | all_niis = glob(op.join(this_out_dir, '*', '*.nii.gz')) 357 | Parallel(n_jobs=n_cores)(delayed(_reorient_file)(f) for f in all_niis) 358 | 359 | # Deface the anatomical data 360 | if options['deface']: 361 | anat_files = glob(op.join(this_out_dir, 'anat', '*.nii.gz')) 362 | magn_files = glob(op.join(this_out_dir, 'fmap', '*magnitude*.nii.gz')) 363 | to_deface = anat_files + magn_files 364 | Parallel(n_jobs=n_cores)(delayed(_deface)(f) for f in to_deface) 365 | 366 | if 'spinoza_cfg' in op.basename(cfg['orig_cfg_path']): 367 | for key in dtype_elements: 368 | cfg.pop(key) 369 | 370 | def _parse_cfg(cfg_file, raw_data_dir, out_dir): 371 | """ Parses config file and sets defaults. """ 372 | 373 | if not op.isfile(cfg_file): 374 | msg = "Couldn't find config-file: %s" % cfg_file 375 | raise IOError(msg) 376 | 377 | with open(cfg_file) as config: 378 | cfg = yaml.safe_load(config) 379 | 380 | # Set mappings to None if not present 381 | for mtype in MTYPE_ORDERS.keys(): 382 | 383 | if mtype not in cfg['mappings'].keys(): 384 | # Set non-existing mappings to None 385 | cfg['mappings'][mtype] = None 386 | 387 | options = cfg['options'].keys() 388 | 389 | if 'mri_ext' not in options: 390 | cfg['options']['mri_ext'] = 'PAR' 391 | 392 | if 'debug' not in options: 393 | cfg['options']['debug'] = False 394 | 395 | if 'n_cores' not in options: 396 | cfg['options']['n_cores'] = -1 397 | else: 398 | cfg['options']['n_cores'] = int(cfg['options']['n_cores']) 399 | 400 | if 'subject_stem' not in options: 401 | cfg['options']['subject_stem'] = 'sub' 402 | 403 | cfg['options']['out_dir'] = out_dir 404 | 405 | if 'spinoza_data' not in options: 406 | cfg['options']['spinoza_data'] = False 407 | 408 | if 'deface' not in options: 409 | cfg['options']['deface'] = True 410 | 411 | if cfg['options']['deface'] and 'FSLDIR' not in os.environ.keys(): 412 | warnings.warn("Cannot deface because FSL is not installed ...") 413 | cfg['options']['deface'] = False 414 | 415 | # Check if nipype/pydeface is installed; if not, deface = False 416 | if cfg['options']['deface']: 417 | try: 418 | import nipype 419 | except ImportError: 420 | msg = """To enable defacing, you need to install nipype (pip 421 | install nipype) manually! Setting deface to False for now""" 422 | warnings.warn(msg) 423 | cfg['options']['deface'] = False 424 | 425 | return cfg 426 | 427 | 428 | def _infer_dtype_elements(directory, cfg): 429 | """ Method to extract mtype/dtypes from data automatically. """ 430 | 431 | # Keep track of elements in a dictionary 432 | dtype_elements = dict() 433 | 434 | # Loop over all possible dtypes (data types: func, anat, fmap, dwi) 435 | for dtype in DTYPES: 436 | 437 | # Per dtype, loop over possible mtypes (modality types) 438 | for mtype in MTYPE_PER_DTYPE[dtype]: 439 | this_id = cfg['mappings'][mtype] 440 | files_found = glob(op.join(directory, '*%s*' % this_id)) 441 | counter = 1 442 | for f in files_found: 443 | 444 | # Very stupid hack to undo typo in test-dataset 445 | if '-acq' in f: 446 | os.rename(f, f.replace('-acq', '_acq')) 447 | f = f.replace('-acq', '_acq') 448 | 449 | # Another hack 450 | if mtype == 'epi' and 'task-' in f: 451 | os.rename(f, f.replace('task', 'dir')) 452 | f = f.replace('task', 'dir') 453 | 454 | info = op.basename(f).split('.')[0].split('_') 455 | info = [s for s in info if 'sub' not in s] 456 | info = [s for s in info if len(s.split('-')) > 1] 457 | 458 | # Remove everything that is not allowed for this mtype 459 | info = [s for s in info if s.split('-')[0] in MTYPE_ORDERS[mtype].keys()] 460 | info_dict = {s.split('-')[0]: s.split('-')[1] for s in info} 461 | info_dict['id'] = '_'.join(info) 462 | 463 | if mtype in ['phasediff', 'magnitude', 'epi']: 464 | info_dict['id'] += '*%s' % this_id 465 | 466 | if dtype_elements.get(dtype, None) is None: 467 | # If dtype is not yet a key, add it anyway 468 | dtype_elements.update({dtype: {'%s_%i' % (mtype, counter): info_dict}}) 469 | counter += 1 470 | else: 471 | if info_dict not in dtype_elements[dtype].values(): 472 | dtype_elements[dtype].update({'%s_%i' % (mtype, counter): info_dict}) 473 | counter += 1 474 | 475 | return dtype_elements 476 | 477 | 478 | def _extract_metadata_from_cfg(cfg): 479 | 480 | these_dtypes = cfg['data_types'] 481 | 482 | # Now, extract and set metadata 483 | metadata = dict() 484 | metadata['BidsifyVersion'] = __version__ 485 | if 'metadata' in cfg.keys(): 486 | metadata.update(cfg['metadata']) 487 | 488 | if cfg['options']['spinoza_data']: 489 | # If data is from Spinoza centre, set some sensible defaults! 490 | spi_cfg = op.join(op.dirname(__file__), 'data', 491 | 'spinoza_metadata.yml') 492 | with open(spi_cfg) as f: 493 | cfg['spinoza_metadata'] = yaml.safe_load(f) 494 | 495 | # Check config for metadata 496 | for dtype in these_dtypes: 497 | 498 | if 'metadata' in cfg[dtype].keys(): 499 | # Set specific dtype metadata 500 | metadata[dtype] = cfg[dtype]['metadata'] 501 | del cfg[dtype]['metadata'] 502 | 503 | cfg['metadata'] = metadata 504 | return cfg 505 | 506 | 507 | def _rename(cdir, dtype, sub_name, cfg): 508 | """ Does the actual work of processing/renaming/conversion. """ 509 | 510 | # Define out-Directory 511 | dtype_out_dir = op.join(cdir, dtype) # e.g. sub-01/ses-01/anat 512 | data_dir = None 513 | 514 | # The number of coherent elements for a given data-type (e.g. runs in 515 | # bold-fmri, or different T1 acquisitions for anat) ... 516 | mappings, options = cfg['mappings'], cfg['options'] 517 | n_elem = len(cfg[dtype]) 518 | 519 | if n_elem == 0: 520 | # If there are for some reason no elements, raise error 521 | raise ValueError("The category '%s' does not have any entries in your " 522 | "config-file!" % dtype) 523 | 524 | # Loop over contents of dtype (e.g. func) 525 | for elem in cfg[dtype].keys(): 526 | 527 | # Extract "key-value" pairs (info about element) 528 | kv_pairs = deepcopy(cfg[dtype][elem]) 529 | 530 | # Extract identifier (idf) from element ... 531 | idf = copy(kv_pairs['id']) 532 | # ... but delete the field, because we'll loop over the rest of the 533 | # fields! 534 | del kv_pairs['id'] 535 | 536 | common_kv_pairs = {sub_name.split('-')[0]: sub_name.split('-')[1]} 537 | # Add session-id pair to name if there are sessions! 538 | if 'ses-' in op.basename(cdir): 539 | sess_id = op.basename(cdir).split('ses-')[-1] 540 | common_kv_pairs.update(dict(ses=sess_id)) 541 | 542 | # Find files corresponding to func/anat/dwi/fieldmap 543 | files = [f for f in glob(op.join(cdir, '*%s*' % idf)) 544 | if op.isfile(f)] 545 | if not files: 546 | print("Could not find files for element %s (dtype %s) with " 547 | "identifier '%s'" % (elem, dtype, idf)) 548 | continue 549 | else: 550 | data_dir = _make_dir(dtype_out_dir) 551 | 552 | for f in files: 553 | # Rename files according to mapping 554 | these_kv_pairs = deepcopy(common_kv_pairs) 555 | types = [] 556 | for mtype, match in mappings.items(): 557 | if match is None: 558 | # if there's no mapping given, skip it 559 | continue 560 | 561 | # Try to find (unique) modality type (e.g. bold, dwi) 562 | match = '*%s*' % match 563 | if fnmatch.fnmatch(op.basename(f), match): 564 | types.append(mtype) 565 | 566 | if len(types) > 1: 567 | msg = ("Couldn't determine modality-type for file '%s' (i.e. " 568 | "there is no UNIQUE mapping); " 569 | "is one of the following:\n %r" % (f, types)) 570 | raise ValueError(msg) 571 | elif len(types) == 0: 572 | # No file found; ends up in unallocated (printed later). 573 | continue 574 | else: 575 | mtype = types[0] 576 | 577 | # Check if keys in config are allowed 578 | allowed_keys = list(MTYPE_ORDERS[mtype].keys()) 579 | for key, value in kv_pairs.items(): 580 | # Append key-value pair if in allowed keys 581 | if key in allowed_keys: 582 | these_kv_pairs.update({key: value}) 583 | else: 584 | print("Key '%s' in element '%s' (dtype %s) is not an " 585 | "allowed key! Choose from %r" % 586 | (key, elem, dtype, allowed_keys)) 587 | 588 | # Check if there are any keys in filename already 589 | these_keys = these_kv_pairs.keys() 590 | for key_value in op.basename(f).split('_'): 591 | key_value = key_value.split('.')[0] # remove extensions 592 | if len(key_value.split('-')) == 2: 593 | key, value = key_value.split('-') 594 | # If allowed (part of BIDS-spec) and not already added ... 595 | if key in allowed_keys and key not in these_keys: 596 | these_kv_pairs.update({key: value}) 597 | 598 | # Small hack to fix topups ('task' is not allowed; 'dir' is) 599 | if 'task' in these_kv_pairs.keys() and mtype == 'epi': 600 | these_kv_pairs['dir'] = these_kv_pairs.pop('task') 601 | 602 | if mtype == 'physio' and '.edf' in f: # eyedata 603 | these_kv_pairs['recording'] = 'eyetracker' 604 | elif mtype == 'physio' and not '.edf' in f: # ppu/resp 605 | these_kv_pairs['recording'] = 'respcardiac' 606 | 607 | # Sort kv-pairs using MTYPE_ORDERS 608 | this_order = MTYPE_ORDERS[mtype] 609 | ordered = sorted(zip(these_kv_pairs.keys(), 610 | these_kv_pairs.values()), 611 | key=lambda x: this_order[x[0]]) 612 | 613 | # Convert all values to strings 614 | ordered = [[str(s[0]), str(s[1])] for s in ordered] 615 | kv_string = '_'.join(['-'.join(s) for s in ordered]) 616 | 617 | # Create full name as common_name + unique filetype + original ext 618 | exts = op.basename(f).split('.')[1:] 619 | clean_exts = '.'.join([e for e in exts if e in ALLOWED_EXTS]) 620 | full_name = kv_string + '_%s.%s' % (mtype, clean_exts) 621 | full_name = op.join(data_dir, full_name) 622 | if mtype == 'bold': 623 | if 'task-' not in op.basename(full_name): 624 | msg = ("Could not assign task-name to file %s; please " 625 | "put this in the config-file under data-type 'func'" 626 | "and element '%s'" % (f, elem)) 627 | raise ValueError(msg) 628 | 629 | if options['debug']: 630 | print("Renaming '%s' to '%s'" % (f, full_name)) 631 | 632 | if not op.isfile(full_name): 633 | # only do it if it isn't already done 634 | shutil.move(f, full_name) 635 | 636 | return data_dir 637 | 638 | 639 | def _add_missing_BIDS_metadata_and_save_to_disk(data_dir, cfg): 640 | 641 | # Get metadata dict 642 | metadata, mappings = cfg['metadata'], cfg['mappings'] 643 | if 'spinoza_metadata' in cfg.keys(): 644 | spi_md = cfg['spinoza_metadata'] 645 | 646 | dtype = op.basename(data_dir) 647 | 648 | # Start with common metadata ("toplevel") 649 | common_metadata = {key: value for key, value in metadata.items() 650 | if not isinstance(value, dict)} 651 | 652 | # If there is dtype-specific metadata, append it 653 | if metadata.get(dtype, None) is not None: 654 | common_metadata.update(metadata.get(dtype)) 655 | 656 | # Used later for the IntendedFor field 657 | if 'ses-' in op.basename(op.dirname(data_dir)): 658 | ses2append = op.basename(op.dirname(data_dir)) 659 | else: 660 | ses2append = '' 661 | 662 | # Now loop over ftypes ('filetypes', e.g. bold, physio, etc.) 663 | for mtype in mappings.keys(): 664 | 665 | if dtype == 'fmap' and mtype == 'phasediff': 666 | # Find 'bold' files, needed for IntendedFor field of fmaps, 667 | # assuming a single phasediff file for all bold-files 668 | func_files = glob(op.join(op.dirname(data_dir), 669 | 'func', '*_bold.nii.gz')) 670 | 671 | common_metadata['IntendedFor'] = [op.join(ses2append, 'func', op.basename(f)) 672 | for f in func_files] 673 | 674 | # Find relevant jsons 675 | jsons = glob(op.join(data_dir, '*_%s.json' % mtype)) 676 | 677 | for this_json in jsons: 678 | # Loop over jsons 679 | fbase = op.basename(this_json) 680 | if 'acq' in fbase: 681 | acqtype = fbase.split('acq-')[-1].split('_')[0] 682 | else: 683 | acqtype = None 684 | 685 | # this_metadata refers to metadata meant for current json 686 | current_metadata = copy(common_metadata) 687 | if 'spinoza_metadata' in cfg.keys(): 688 | # Append spinoza metadata to current json according to dtype 689 | # (anat, func, etc.) and mtype (phasediff, bold, etc.) 690 | if spi_md.get(dtype, None) is not None: 691 | tmp_metadata = spi_md.get(dtype) 692 | if tmp_metadata.get(mtype, None) is not None: 693 | tmp_metadata = tmp_metadata.get(mtype) 694 | if tmp_metadata.get(acqtype, None) is not None: 695 | tmp_metadata = tmp_metadata.get(acqtype) 696 | else: 697 | msg = ("Trying to append metadata from dtype=%s, mtype=%s, " 698 | "acq=%s, but %s does not exist in spinoza_metadata.yml!" % 699 | (dtype, mtype, acqtype, acqtype)) 700 | raise ValueError(msg) 701 | else: 702 | # if there is no metadata, just append an empty dict 703 | tmp_metadata = dict() 704 | current_metadata.update(tmp_metadata) 705 | 706 | if mtype == 'epi': 707 | 708 | pardir = op.dirname(op.dirname(this_json)) 709 | acq_idf = fbase.split('acq-')[1].split('_')[0] 710 | 711 | # Stupid hack, but it works 712 | if 'Dirs' in acq_idf: 713 | cdwi = glob(op.join(pardir, 'dwi', '*%s*_dwi.nii.gz' % acq_idf)) 714 | 715 | if not cdwi: 716 | warnings.warn("Could not find DWI-file corresponding to topup (%s)!" % this_json) 717 | int_for = 'Could not find corresponding file; add this yourself!' 718 | else: 719 | cdwi = op.basename(cdwi[0]) 720 | int_for = op.join(ses2append, 'dwi', cdwi) 721 | else: # assume bold 722 | dir_idf = fbase.split('dir-')[1].split('_')[0] 723 | run_idf = fbase.split('run-') 724 | if len(run_idf) > 1: 725 | run_idf = run_idf[1].split('_')[0] 726 | cbold = glob(op.join(pardir, 'func', '*task-%s*acq-%s*_run-%s*_bold.nii.gz' % (dir_idf, acq_idf, run_idf))) 727 | else: 728 | cbold = glob(op.join(pardir, 'func', '*task-%s*acq-%s*_bold.nii.gz' % (dir_idf, acq_idf))) 729 | if not cbold: 730 | warnings.warn("Cound not find bold-file corresponding to topup (%s)!" % this_json) 731 | int_for = 'Could not find corresponding file; add this yourself!' 732 | elif len(cbold) > 1: 733 | warnings.warn("Found multiple bold-files (%s) corresponding to topup (%s)!" % (cbold, this_json)) 734 | 735 | cbold = op.basename(cbold[0]) 736 | int_for = op.join(ses2append, 'func', cbold) 737 | current_metadata['IntendedFor'] = int_for 738 | 739 | if mtype == 'bold': 740 | task_name = fbase.split('task-')[1].split('_')[0] 741 | current_metadata.update({'TaskName': task_name}) 742 | 743 | # Slicetiming info. Note: we assume ascending order! 744 | with open(this_json, 'r') as to_read: 745 | this_json_opened = json.load(to_read) 746 | 747 | if 'SliceEncodingDirection' in this_json_opened.keys(): 748 | sed = this_json_opened['SliceEncodingDirection'] 749 | else: 750 | sed = 'none' 751 | 752 | if 'SliceEncodingDirection' in current_metadata.keys(): 753 | sed = current_metadata['SliceEncodingDirection'] 754 | else: 755 | sed = 'none' 756 | 757 | if 'spinoza_metadata' in cfg.keys(): 758 | this_tr = this_json_opened['RepetitionTime'] 759 | corresp_func = this_json.replace('.json', '.nii.gz') 760 | nr_slices = nib.load(corresp_func).header.get_data_shape()[2] 761 | if 'MultibandAccelerationFactor' in this_json_opened.keys(): 762 | mb_factor = int(this_json_opened['MultibandAccelerationFactor']) 763 | else: 764 | mb_factor = 0 765 | 766 | if 'MultibandAccelerationFactor' in current_metadata.keys(): 767 | mb_factor = int(current_metadata['MultibandAccelerationFactor']) 768 | else: 769 | mb_factor = 0 770 | 771 | if mb_factor > 0: 772 | slice_timing = np.tile(np.linspace(0, this_tr, int(nr_slices/mb_factor)+1)[:-1], mb_factor) 773 | else: 774 | slice_timing = np.linspace(0, this_tr, nr_slices+1)[:-1] 775 | 776 | slice_timing = slice_timing.tolist() 777 | current_metadata.update({'SliceTiming': slice_timing}) 778 | 779 | _append_to_json(this_json, current_metadata) 780 | 781 | 782 | def _reorient_file(f): 783 | """ Reorient MRI file """ 784 | _run_cmd(['fslreorient2std', f, f]) 785 | 786 | 787 | def _deface(f): 788 | """ Deface anat data. """ 789 | 790 | _run_cmd(['pydeface', f]) # Run pydeface 791 | if op.isfile(f.replace('.nii.gz', '_defaced.nii.gz')): 792 | os.rename(f.replace('.nii.gz', '_defaced.nii.gz'), f) # Revert to old name 793 | 794 | 795 | def _extract_sub_nr(sub_stem, sub_name): 796 | nr = sub_name.split(sub_stem)[-1] 797 | nr = nr.replace('-', '').replace('_', '') 798 | return 'sub-' + nr 799 | -------------------------------------------------------------------------------- /bidsify/mri2nifti.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function, division 2 | import os 3 | import warnings 4 | import os.path as op 5 | from glob import glob 6 | from .utils import check_executable, _compress, _run_cmd 7 | from shutil import rmtree 8 | 9 | PIGZ = check_executable('pigz') 10 | 11 | 12 | def convert_mri(directory, cfg): 13 | 14 | compress = not cfg['options']['debug'] 15 | mri_ext = cfg['options']['mri_ext'] 16 | 17 | base_cmd = "dcm2niix -ba y" 18 | if compress: 19 | base_cmd += " -z y" if PIGZ else " -z i" 20 | else: 21 | base_cmd += " -z n" 22 | 23 | if mri_ext in ['PAR', 'dcm']: 24 | mri_files = glob(op.join(directory, '*.%s' % mri_ext)) 25 | for f in mri_files: 26 | 27 | if '.PAR' in f: 28 | info = _get_extra_info_from_par_header(f) 29 | 30 | basename, ext = op.splitext(op.basename(f)) 31 | if info['n_echoes'] > 1: 32 | basename += '_echo-%e' 33 | 34 | par_cmd = base_cmd + " -f %s %s" % (basename, f) 35 | # if debug, print dcm2niix output 36 | _run_cmd(par_cmd.split(' '), verbose=cfg['options']['debug']) 37 | os.remove(f) 38 | 39 | if mri_ext == 'PAR': 40 | [os.remove(f) for f in glob(op.join(directory, '*.REC'))] 41 | 42 | elif mri_ext == 'DICOM': 43 | # Experimental enh DICOM conversion 44 | dcm_cmd = base_cmd + " -f %n_%p " + directory 45 | _run_cmd(dcm_cmd.split(' ')) 46 | 47 | if op.isdir(op.join(directory, 'DICOM')): 48 | rmtree(op.join(directory, 'DICOM')) 49 | 50 | if op.isfile(op.join(directory, 'DICOMDIR')): 51 | os.remove(op.join(directory, 'DICOMDIR')) 52 | 53 | im_files = glob(op.join(directory, 'IM_????')) 54 | _ = [os.remove(f) for f in im_files] 55 | 56 | ps_files = glob(op.join(directory, 'PS_????')) 57 | _ = [os.remove(f) for f in ps_files] 58 | 59 | xx_files = glob(op.join(directory, 'XX_????')) 60 | _ = [os.remove(f) for f in xx_files] 61 | elif mri_ext == 'nifti': 62 | pass 63 | else: 64 | raise ValueError('Please select either PAR, dcm, DICOM or nifti for mri_ext!') 65 | 66 | niis = glob(op.join(directory, '*.nii')) 67 | if compress: 68 | for nii in niis: 69 | _compress(nii, PIGZ) 70 | os.remove(nii) 71 | 72 | if 'fmap' in cfg.keys(): 73 | idf = [elem['id'] for elem in cfg['fmap'].values()] 74 | else: 75 | idf = ['phasediff'] 76 | 77 | idf = list(set(idf)) 78 | _rename_phasediff_files(directory, cfg, idf=idf) 79 | 80 | 81 | def _rename_phasediff_files(directory, cfg, idf): 82 | """ Renames Philips "B0" files (1 phasediff / 1 magnitude) because dcm2niix 83 | appends (or sometimes prepends) '_ph' to the filename after conversion. 84 | """ 85 | 86 | if not isinstance(idf, list): 87 | idf = [idf] 88 | 89 | b0_files = [] 90 | for this_idf in idf: 91 | b0_files += sorted(glob(op.join(directory, '*%s*' % this_idf))) 92 | 93 | new_files = [] 94 | for f in b0_files: 95 | fnew = f.replace('phasediff', '') 96 | os.rename(f, fnew) 97 | new_files.append(fnew) 98 | 99 | for fnew in new_files: 100 | 101 | if '_real' in op.basename(fnew): 102 | os.rename(fnew, fnew.replace('_real', '_phasediff')) 103 | else: 104 | if '.nii.gz' in fnew: 105 | os.rename(fnew, fnew.replace('.nii.gz', '_magnitude1.nii.gz')) 106 | else: 107 | os.rename(fnew, fnew.replace('.', '_magnitude1.')) 108 | 109 | magnitude_jsons = glob(op.join(directory, '*_magnitude1.json')) 110 | [os.remove(tf) for tf in magnitude_jsons] 111 | 112 | 113 | def _get_extra_info_from_par_header(par): 114 | 115 | info = dict() 116 | 117 | with open(par, 'r') as f: 118 | lines = f.readlines() 119 | 120 | found = False 121 | for line in lines: 122 | found = 'Max. number of slices/locations' in line 123 | if found: 124 | info['n_slices'] = int(line.split(':')[-1].strip().replace('\n', '')) 125 | break 126 | 127 | if not found: 128 | raise ValueError("Could not determine number of slices from PAR header (%s)!" % par) 129 | 130 | found = False 131 | for line_nr_of_dyns, line in enumerate(lines): 132 | found = 'Max. number of dynamics' in line 133 | if found: 134 | info['n_dyns'] = int(line.split(':')[-1].strip().replace('\n', '')) 135 | break 136 | 137 | if not found: 138 | raise ValueError("Could not determine number of dynamics from PAR header (%s)!" % par) 139 | 140 | found = False 141 | for line in lines: 142 | found = 'Max. number of echoes' in line 143 | if found: 144 | info['n_echoes'] = int(line.split(':')[-1].strip().replace('\n', '')) 145 | break 146 | 147 | if info['n_echoes'] > 1: 148 | print("WARNING: file %s seems to be a multiecho file - this feature is experimental!" % op.basename(par)) 149 | 150 | if info['n_dyns'] == 1: 151 | # Not an fMRI file! skip the rest 152 | return info 153 | 154 | # Multiecho fMRI has n_dyns * n_echoes volumes in the 4th dim 155 | info['n_vols'] = int(info['n_dyns'] * info['n_echoes']) 156 | 157 | found = False 158 | for idx_start_slices, line in enumerate(lines): 159 | found = '# === IMAGE INFORMATION =' in line 160 | if found: 161 | idx_start_slices += 3 162 | break 163 | 164 | idx_stop_slices = len(lines) - 2 165 | slices = lines[idx_start_slices:idx_stop_slices] 166 | actual_n_vols = len(slices) / info['n_slices'] 167 | 168 | if actual_n_vols != info['n_vols']: 169 | print("Found %.3f vols (%i slices) for file %s, but expected %i dyns (%i slices);" 170 | " going to try to fix it by removing slices from the PAR header ..." % 171 | (actual_n_vols, len(slices), op.basename(par), info['n_vols'], info['n_vols']*info['n_slices'])) 172 | 173 | lines_to_remove = len(slices) % info['n_slices'] 174 | print("Number of excess slices: %i" % int(lines_to_remove)) 175 | if lines_to_remove != 0: 176 | for i in range(lines_to_remove): 177 | lines.pop(idx_stop_slices - (i+1)) 178 | 179 | slices = lines[idx_start_slices:(idx_stop_slices - lines_to_remove)] 180 | actual_n_dyns = len(slices) / info['n_slices'] / info['n_echoes'] 181 | if not actual_n_dyns.is_integer(): 182 | print("Couldn't fix PAR header (probably multiple randomly dropped frames)") 183 | return info 184 | else: 185 | actual_n_dyns = actual_n_vols 186 | 187 | # Replacing expected with actual number of dynamics 188 | lines[line_nr_of_dyns] = lines[line_nr_of_dyns].replace(str(info['n_dyns']), 189 | str(int(actual_n_dyns))) 190 | info['n_dyns'] = actual_n_dyns 191 | with open(par, 'w') as f_out: 192 | [f_out.write(line) for line in lines] 193 | 194 | return info 195 | 196 | return info 197 | -------------------------------------------------------------------------------- /bidsify/phys2tsv.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path as op 3 | import numpy as np 4 | import pandas as pd 5 | 6 | 7 | def convert_phy(f): 8 | 9 | ''' 10 | try: # Try to skip 5 rows (new version) 11 | df = pd.read_csv(f, delim_whitespace=True, skiprows=5, header=0, 12 | low_memory=False) 13 | _ = df['gx'] 14 | except KeyError: # Else skip 4 rows (old version) 15 | df = pd.read_csv(f, delim_whitespace=True, skiprows=4, header=0, 16 | low_memory=False) 17 | 18 | gradients = ['gx', 'gy', 'gz'] 19 | gradient_signal = np.array([df[g] for g in gradients]).sum(axis=0) 20 | gradient_signal[np.isnan(gradient_signal)] = 0 21 | centered = gradient_signal - gradient_signal.mean() 22 | gradient_signal = centered / gradient_signal.std() 23 | 24 | fn = op.join(op.dirname(f), op.splitext(op.basename(f))[0]) 25 | 26 | # Ideally, create sidecar-json with following fields: 27 | # "SamplingFrequency": 100.0, 28 | # "StartTime": -22.345, 29 | # "Columns": ["cardiac", "respiratory", "trigger"] 30 | 31 | df.to_csv(fn + '.tsv.gz', sep='\t', index=None, compression='gzip') 32 | os.remove(f) 33 | ''' 34 | pass 35 | -------------------------------------------------------------------------------- /bidsify/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NILAB-UvA/bidsify/95db2a19178375ffb09b46367b174d3fe13524e6/bidsify/tests/__init__.py -------------------------------------------------------------------------------- /bidsify/tests/test_bidsify.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, division, print_function 2 | import os 3 | import pytest 4 | import os.path as op 5 | from shutil import rmtree 6 | from bidsify import bidsify 7 | 8 | data_path = op.join(op.dirname(op.dirname(op.abspath(__file__))), 'data') 9 | testdata_path = op.join(data_path, 'test_data') 10 | datasets = [ 11 | op.join(testdata_path, 'PIOP_1'), 12 | op.join(testdata_path, 'Upgrade_2017'), 13 | op.join(testdata_path, 'SharedStates'), 14 | op.join(testdata_path, 'ME') 15 | ] 16 | 17 | 18 | @pytest.mark.parametrize('path_to_data', datasets) 19 | def test_bidsify(path_to_data): 20 | """ Tests bidsify """ 21 | 22 | if not op.isdir(path_to_data): 23 | # Not all datasets are on travis 24 | print("Couldn't find dataset %s." % path_to_data) 25 | return None 26 | else: 27 | print("Testing dataset %s ..." % path_to_data) 28 | 29 | bids_dir = op.join(path_to_data, 'bids') 30 | if op.isdir(bids_dir): 31 | rmtree(bids_dir) 32 | 33 | unall_dir = op.join(path_to_data, 'unallocated') 34 | if op.isdir(unall_dir): 35 | rmtree(unall_dir) 36 | 37 | if 'Upgrade' in path_to_data: 38 | cfg = op.join(data_path, 'spinoza_cfg.yml') 39 | else: 40 | cfg = op.join(path_to_data, 'raw', 'config.yml') 41 | 42 | bidsify(cfg_path=cfg, directory=op.join(path_to_data, 'raw'), 43 | validate=True, out_dir=bids_dir) 44 | rmtree(bids_dir) 45 | 46 | if op.isdir(unall_dir): 47 | rmtree(unall_dir) 48 | -------------------------------------------------------------------------------- /bidsify/utils.py: -------------------------------------------------------------------------------- 1 | import platform 2 | import subprocess 3 | import os 4 | import json 5 | import gzip 6 | import shutil 7 | import os.path as op 8 | from glob import glob 9 | 10 | 11 | def check_executable(executable): 12 | """ Checks if executable is available. 13 | 14 | Params 15 | ------ 16 | executable : str 17 | Command to check. 18 | 19 | Returns 20 | ------- 21 | bool 22 | """ 23 | cmd = "where" if platform.system() == "Windows" else "which" 24 | 25 | with open(os.devnull, 'w') as devnull: 26 | res = subprocess.call([cmd, executable], stdout=devnull) 27 | 28 | if res == 0: 29 | return True 30 | else: 31 | return False 32 | 33 | 34 | def _append_to_json(json_path, to_append): 35 | 36 | if op.isfile(json_path): 37 | 38 | with open(json_path, 'r') as metadata_file: 39 | metadata = json.load(metadata_file) 40 | metadata.update(to_append) # note: this overwrites if key exists! 41 | else: 42 | msg = "Constructing new meta-data json (%s)" % json_path 43 | print(msg) 44 | metadata = to_append 45 | 46 | with open(json_path, 'w') as new_metadata_file: 47 | json.dump(metadata, new_metadata_file, indent=4) 48 | 49 | 50 | def _compress(f, pigz): 51 | 52 | if pigz: 53 | cmd = ['pigz', f] 54 | with open(os.devnull, 'w') as devnull: 55 | subprocess.call(cmd, stdout=devnull) 56 | else: 57 | with open(f, 'rb') as f_in, gzip.open(f + '.gz', 'wb') as f_out: 58 | shutil.copyfileobj(f_in, f_out) 59 | os.remove(f) 60 | 61 | 62 | def _make_dir(path): 63 | """ Creates dir-if-not-exists-already. """ 64 | 65 | if not op.isdir(path): 66 | os.makedirs(path) 67 | 68 | return path 69 | 70 | 71 | def _glob(path, wildcards): 72 | """ Finds files with different wildcards. """ 73 | 74 | files = [] 75 | for w in wildcards: 76 | files.extend(glob(op.join(path, '*%s' % w))) 77 | 78 | return sorted(files) 79 | 80 | 81 | def _run_cmd(cmd, verbose=False, outfile=None): 82 | 83 | if verbose: 84 | if outfile is None: 85 | rs = subprocess.call(cmd) 86 | else: 87 | with open(outfile, 'w') as f: 88 | rs = subprocess.call(cmd, stdout=f) 89 | else: 90 | with open(os.devnull, 'w') as devnull: 91 | rs = subprocess.call(cmd, stdout=devnull) 92 | 93 | return rs 94 | -------------------------------------------------------------------------------- /bidsify/version.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, division, print_function 2 | import os.path as op 3 | 4 | # Format expected by setup.py and doc/source/conf.py: string of form "X.Y.Z" 5 | _version_major = 0 6 | _version_minor = 3 7 | _version_micro = 7 # use '' for first of series, number for 1 and above 8 | # _version_extra = 'dev' 9 | _version_extra = '' # Uncomment this for full releases 10 | 11 | # Construct full version string from these. 12 | _ver = [_version_major, _version_minor] 13 | if _version_micro: 14 | _ver.append(_version_micro) 15 | if _version_extra: 16 | _ver.append(_version_extra) 17 | 18 | __version__ = '.'.join(map(str, _ver)) 19 | 20 | CLASSIFIERS = ["Development Status :: 3 - Alpha", 21 | "Environment :: Console", 22 | "Intended Audience :: Science/Research", 23 | "License :: OSI Approved :: MIT License", 24 | "Operating System :: OS Independent", 25 | "Programming Language :: Python", 26 | "Topic :: Scientific/Engineering"] 27 | 28 | # Description should be a one-liner: 29 | description = "bidsify: Converts your (raw) data to the BIDS-format" 30 | 31 | NAME = "bidsify" 32 | MAINTAINER = "Lukas Snoek" 33 | MAINTAINER_EMAIL = "lukassnoek@gmail.com" 34 | DESCRIPTION = description 35 | URL = "https://github.com/spinoza-rec/bidsify" 36 | DOWNLOAD_URL = "" 37 | LICENSE = "3-clause BSD" 38 | AUTHOR = "Lukas Snoek" 39 | AUTHOR_EMAIL = "lukassnoek@gmail.com" 40 | PLATFORMS = "OS Independent" 41 | MAJOR = _version_major 42 | MINOR = _version_minor 43 | MICRO = _version_micro 44 | VERSION = __version__ 45 | PACKAGE_DATA = {'bidsify': [op.join('data', '*')]} 46 | -------------------------------------------------------------------------------- /build_docker_image: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker build -t lukassnoek/bidsify:$1 -t lukassnoek/bidsify:latest . 3 | -------------------------------------------------------------------------------- /create_new_release: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | python setup.py sdist 3 | python setup.py install 4 | twine upload dist/bidsify-$1.tar.gz 5 | ./generate_dockerfile 6 | ./build_docker_image $1 7 | docker push lukassnoek/bidsify:$1 8 | docker push lukassnoek/bidsify:latest 9 | -------------------------------------------------------------------------------- /download_test_data.py: -------------------------------------------------------------------------------- 1 | """ This script fetches test-data for bidsify. 2 | The data is stored at Surfdrive (a data storage repository/drive 3 | from the Dutch institute for IT in science/academia) and downloaded 4 | using cURL. """ 5 | 6 | from __future__ import print_function 7 | import subprocess 8 | import os 9 | import zipfile 10 | import os.path as op 11 | 12 | this_dir = op.dirname(op.realpath(__file__)) 13 | dst_dir = op.join(this_dir, 'bidsify', 'data', 'test_data') 14 | dst_file = op.join(dst_dir, 'test_data.zip') 15 | 16 | data_file = 'https://surfdrive.surf.nl/files/index.php/s/aQQTSghdmBPbHt7/download' 17 | 18 | if not op.isdir(op.join(dst_dir, 'PIOP_1_parrec')): 19 | 20 | print("Downloading the data ...\n") 21 | cmd = "curl -o %s %s" % (dst_file, data_file) 22 | return_code = subprocess.call(cmd, shell=True) 23 | print("\nDone!") 24 | print("Unzipping ...", end='') 25 | zip_ref = zipfile.ZipFile(dst_file, 'r') 26 | zip_ref.extractall(dst_dir) 27 | zip_ref.close() 28 | print(" done!") 29 | os.remove(dst_file) 30 | else: 31 | print("Data is already downloaded and located at %s/*" % dst_dir) 32 | -------------------------------------------------------------------------------- /generate_dockerfile: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker run --rm kaczmarj/neurodocker:0.6.0 generate docker \ 3 | --base debian:stretch --pkg-manager apt \ 4 | --install git \ 5 | --fsl version=6.0.1 \ 6 | --dcm2niix version=master method=source \ 7 | --miniconda create_env=neuro \ 8 | conda_install="python=3.6 numpy pandas" \ 9 | pip_install="nipype git+https://github.com/poldracklab/pydeface.git@master pyyaml nibabel joblib" \ 10 | activate=true \ 11 | --install gnupg2 vim \ 12 | --run "curl --silent --location https://deb.nodesource.com/setup_10.x | bash -" \ 13 | --install nodejs \ 14 | --run "npm install -g bids-validator" \ 15 | --copy . /home/neuro/bidsify \ 16 | --workdir /home/neuro/bidsify \ 17 | --run "/opt/miniconda-latest/envs/neuro/bin/python setup.py install" \ 18 | --volume /raw \ 19 | --volume /bids > Dockerfile 20 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | nibabel 3 | joblib 4 | pandas 5 | pyyaml 6 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from setuptools import setup, find_packages 3 | PACKAGES = find_packages() 4 | 5 | # Get version and release info, which is all stored in shablona/version.py 6 | ver_file = os.path.join('bidsify', 'version.py') 7 | 8 | with open(ver_file) as f: 9 | exec(f.read()) 10 | 11 | # Long description will go up on the pypi page 12 | with open('README.rst') as f: 13 | LONG_DESCRIPTION = f.read() 14 | 15 | with open('requirements.txt') as f: 16 | REQUIRES = f.readlines() 17 | 18 | opts = dict(name=NAME, 19 | maintainer=MAINTAINER, 20 | maintainer_email=MAINTAINER_EMAIL, 21 | description=DESCRIPTION, 22 | long_description=LONG_DESCRIPTION, 23 | url=URL, 24 | download_url=DOWNLOAD_URL, 25 | license=LICENSE, 26 | classifiers=CLASSIFIERS, 27 | author=AUTHOR, 28 | author_email=AUTHOR_EMAIL, 29 | platforms=PLATFORMS, 30 | version=VERSION, 31 | packages=PACKAGES, 32 | package_data=PACKAGE_DATA, 33 | install_requires=REQUIRES, 34 | requires=REQUIRES, 35 | entry_points={ 36 | 'console_scripts': [ 37 | 'bidsify = bidsify.main:run_cmd', 38 | ] 39 | } 40 | ) 41 | 42 | if __name__ == '__main__': 43 | setup(**opts) 44 | --------------------------------------------------------------------------------