├── .autorc ├── .codespellrc ├── .github ├── ISSUE_TEMPLATE.md ├── dependabot.yml └── workflows │ ├── docker.yml │ ├── lint.yml │ ├── release.yml │ ├── test.yml │ └── typing.yml ├── .gitignore ├── .mailmap ├── .pre-commit-config.yaml ├── .readthedocs.yaml ├── .zenodo.json ├── CHANGELOG.md ├── CONTRIBUTING.rst ├── Dockerfile ├── LICENSE ├── NOTES ├── README.rst ├── custom └── dbic │ ├── README │ └── singularity-env.def ├── dev-requirements.txt ├── docs ├── Makefile ├── api.rst ├── api │ ├── bids.rst │ ├── convert.rst │ ├── dicoms.rst │ ├── parser.rst │ ├── queue.rst │ └── utils.rst ├── changes.rst ├── commandline.rst ├── conf.py ├── container.rst ├── custom-heuristic.rst ├── figs ├── heuristics.rst ├── index.rst ├── installation.rst ├── quickstart.rst ├── reproin.rst ├── requirements.txt └── tutorials.rst ├── figs ├── environment.png └── workflow.png ├── heudiconv ├── __init__.py ├── bids.py ├── cli │ ├── __init__.py │ ├── monitor.py │ └── run.py ├── convert.py ├── dicoms.py ├── due.py ├── external │ ├── __init__.py │ ├── dlad.py │ └── tests │ │ ├── __init__.py │ │ └── test_dlad.py ├── heuristics │ ├── __init__.py │ ├── banda-bids.py │ ├── bids_ME.py │ ├── bids_PhoenixReport.py │ ├── bids_with_ses.py │ ├── cmrr_heuristic.py │ ├── convertall.py │ ├── convertall_custom.py │ ├── example.py │ ├── multires_7Tbold.py │ ├── reproin.py │ ├── reproin_validator.cfg │ ├── studyforrest_phase2.py │ ├── test_b0dwi_for_fmap.py │ ├── test_reproin.py │ └── uc_bids.py ├── info.py ├── main.py ├── parser.py ├── py.typed ├── queue.py ├── tests │ ├── __init__.py │ ├── anonymize_script.py │ ├── conftest.py │ ├── data │ │ ├── 01-anat-scout │ │ │ └── 0001.dcm │ │ ├── 01-fmap_acq-3mm │ │ │ └── 1.3.12.2.1107.5.2.43.66112.2016101409263663466202201.dcm │ │ ├── MRI_102TD_PHA_S.MR.Chen_Matthews_1.3.1.2022.11.16.15.50.20.357.31204541.dcm │ │ ├── Phoenix │ │ │ ├── 01+AA │ │ │ │ └── 01+AA+00001.dcm │ │ │ └── 99+PhoenixDocument │ │ │ │ └── 99+PhoenixDocument+00001.dcm │ │ ├── axasc35.dcm │ │ ├── b0dwiForFmap │ │ │ ├── b0dwi_for_fmap+00001.dcm │ │ │ ├── b0dwi_for_fmap+00002.dcm │ │ │ └── b0dwi_for_fmap+00003.dcm │ │ ├── non_zeros.bval │ │ ├── phantom.dcm │ │ ├── sample_nifti.nii.gz │ │ ├── sample_nifti_params.txt │ │ └── zeros.bval │ ├── test_archives.py │ ├── test_bids.py │ ├── test_convert.py │ ├── test_dicoms.py │ ├── test_heuristics.py │ ├── test_main.py │ ├── test_monitor.py │ ├── test_queue.py │ ├── test_regression.py │ ├── test_tarballs.py │ ├── test_utils.py │ └── utils.py └── utils.py ├── mypy.ini ├── pyproject.toml ├── requirements.txt ├── setup.py ├── tox.ini └── utils ├── anon-cmd ├── gen-docker-image.sh ├── link_issues_CHANGELOG ├── sensor-dicoms ├── test-compare-two-versions.sh └── update_changes.sh /.autorc: -------------------------------------------------------------------------------- 1 | { 2 | "onlyPublishWithReleaseLabel": true, 3 | "baseBranch": "master", 4 | "author": "auto ", 5 | "noVersionPrefix": false, 6 | "plugins": ["git-tag", "released"] 7 | } 8 | -------------------------------------------------------------------------------- /.codespellrc: -------------------------------------------------------------------------------- 1 | [codespell] 2 | skip = .git,.venv,venvs,*.svg,_build,build,venv,venvs 3 | # te -- TE as codespell is case insensitive 4 | ignore-words-list = bu,nd,te 5 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 5 | 6 | ### Summary 7 | 12 | 13 | 14 | 15 | ### Platform details: 16 | 17 | Choose one: 18 | - [ ] Local environment 19 | 20 | - [ ] Container 21 | 22 | 23 | - Heudiconv version: 24 | 25 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: github-actions 4 | directory: / 5 | schedule: 6 | interval: weekly 7 | commit-message: 8 | prefix: "[gh-actions]" 9 | include: scope 10 | labels: 11 | - internal 12 | -------------------------------------------------------------------------------- /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | name: Build Docker image 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | build-docker: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout source 13 | uses: actions/checkout@v4 14 | with: 15 | fetch-depth: 0 16 | 17 | 18 | - name: Generate Dockerfile 19 | run: bash gen-docker-image.sh 20 | working-directory: utils 21 | 22 | - name: Build Docker image 23 | run: | 24 | # build only if not release tag, i.e. has some "-" in describe 25 | # so we do not duplicate work with release workflow. 26 | git describe --match 'v[0-9]*' | grep -q -e - && \ 27 | docker build \ 28 | -t nipy/heudiconv:master \ 29 | -t nipy/heudiconv:unstable \ 30 | . 31 | 32 | - name: Push Docker image 33 | run: | 34 | git describe --match 'v[0-9]*' | grep -q -e - && ( 35 | docker login -u "$DOCKER_LOGIN" --password-stdin <<<"$DOCKER_TOKEN" 36 | docker push nipy/heudiconv:master 37 | docker push nipy/heudiconv:unstable 38 | ) 39 | env: 40 | DOCKER_LOGIN: ${{ secrets.DOCKER_LOGIN }} 41 | DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} 42 | 43 | # vim:set sts=2: 44 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Linters 2 | 3 | on: 4 | - push 5 | - pull_request 6 | 7 | jobs: 8 | lint: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Set up environment 12 | uses: actions/checkout@v4 13 | with: 14 | fetch-depth: 0 15 | 16 | - name: Set up Python 17 | uses: actions/setup-python@v5 18 | with: 19 | python-version: '3.9' 20 | 21 | - name: Install dependencies 22 | run: | 23 | python -m pip install --upgrade pip 24 | python -m pip install --upgrade tox 25 | 26 | - name: Run linters 27 | run: tox -e lint 28 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Auto-release on PR merge 2 | 3 | on: 4 | # ATM, this is the closest trigger to a PR merging 5 | push: 6 | branches: 7 | - master 8 | 9 | jobs: 10 | auto-release: 11 | runs-on: ubuntu-latest 12 | if: "!contains(github.event.head_commit.message, 'ci skip') && !contains(github.event.head_commit.message, 'skip ci')" 13 | steps: 14 | - name: Checkout source 15 | uses: actions/checkout@v4 16 | with: 17 | fetch-depth: 0 18 | 19 | - name: Download auto 20 | run: | 21 | #curl -vL -o - "$(curl -fsSL https://api.github.com/repos/intuit/auto/releases/latest | jq -r '.assets[] | select(.name == "auto-linux.gz") | .browser_download_url')" | gunzip > ~/auto 22 | # Pin so we don't break if & when 23 | # is fixed. 24 | # 11.0.5 is needed for 25 | wget -O- https://github.com/intuit/auto/releases/download/v11.0.5/auto-linux.gz | gunzip > ~/auto 26 | chmod a+x ~/auto 27 | 28 | - name: Query 'auto' on type of the release 29 | id: auto-version 30 | run: | 31 | # to be able to debug if something goes wrong 32 | set -o pipefail 33 | ~/auto version -vv | tee /tmp/auto-version 34 | version="$(sed -ne '/Calculated SEMVER bump:/s,.*: *,,p' /tmp/auto-version)" 35 | echo "version=$version" >> "$GITHUB_OUTPUT" 36 | env: 37 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 38 | 39 | - name: Set up Python 40 | if: steps.auto-version.outputs.version != '' 41 | uses: actions/setup-python@v5 42 | with: 43 | python-version: '^3.9' 44 | 45 | - name: Install Python dependencies 46 | if: steps.auto-version.outputs.version != '' 47 | run: python -m pip install build twine 48 | 49 | - name: Create release 50 | if: steps.auto-version.outputs.version != '' 51 | run: ~/auto shipit 52 | env: 53 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 54 | 55 | - name: Build & upload to PyPI 56 | if: steps.auto-version.outputs.version != '' 57 | run: | 58 | python -m build 59 | twine upload dist/* 60 | env: 61 | TWINE_USERNAME: __token__ 62 | TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} 63 | 64 | - name: Generate Dockerfile 65 | if: steps.auto-version.outputs.version != '' 66 | run: bash gen-docker-image.sh 67 | working-directory: utils 68 | 69 | - name: Build Docker images 70 | if: steps.auto-version.outputs.version != '' 71 | run: | 72 | docker build \ 73 | -t nipy/heudiconv:master \ 74 | -t nipy/heudiconv:unstable \ 75 | -t nipy/heudiconv:latest \ 76 | -t nipy/heudiconv:"$(git describe | sed -e 's,^v,,g')" \ 77 | . 78 | 79 | - name: Push Docker images 80 | if: steps.auto-version.outputs.version != '' 81 | run: | 82 | docker login -u "$DOCKER_LOGIN" --password-stdin <<<"$DOCKER_TOKEN" 83 | docker push --all-tags nipy/heudiconv 84 | env: 85 | DOCKER_LOGIN: ${{ secrets.DOCKER_LOGIN }} 86 | DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} 87 | 88 | # vim:set sts=2: 89 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | pull_request: 5 | push: 6 | schedule: 7 | # run weekly to ensure that we are still good 8 | - cron: '1 2 * * 3' 9 | 10 | jobs: 11 | test: 12 | runs-on: ubuntu-latest 13 | env: 14 | BOTO_CONFIG: /tmp/nowhere 15 | DATALAD_TESTS_SSH: '1' 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | python-version: 20 | - '3.9' 21 | - '3.10' 22 | - '3.11' 23 | - '3.12' 24 | # Seems needs work in traits: https://github.com/nipy/heudiconv/pull/799#issuecomment-2447298795 25 | # - '3.13' 26 | steps: 27 | - name: Check out repository 28 | uses: actions/checkout@v4 29 | with: 30 | fetch-depth: 0 31 | 32 | - name: Set up Python 33 | uses: actions/setup-python@v5 34 | with: 35 | python-version: ${{ matrix.python-version }} 36 | 37 | - name: Install git-annex 38 | run: | 39 | # The ultimate one-liner setup for NeuroDebian repository 40 | bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh) 41 | sudo apt-get update -qq 42 | sudo apt-get install git-annex-standalone dcm2niix 43 | 44 | - name: Install dependencies 45 | run: | 46 | python -m pip install --upgrade pip wheel 47 | pip install -r dev-requirements.txt 48 | pip install requests # below installs pyld but that assumes we have requests already 49 | pip install datalad 50 | pip install coverage pytest 51 | 52 | - name: Configure Git identity 53 | run: | 54 | git config --global user.email "test@github.land" 55 | git config --global user.name "GitHub Almighty" 56 | 57 | - name: Run tests with coverage 58 | run: coverage run `which pytest` -s -v heudiconv 59 | 60 | - name: Upload coverage to Codecov 61 | uses: codecov/codecov-action@v5 62 | with: 63 | fail_ci_if_error: false 64 | token: ${{ secrets.CODECOV_TOKEN }} 65 | 66 | # vim:set et sts=2: 67 | -------------------------------------------------------------------------------- /.github/workflows/typing.yml: -------------------------------------------------------------------------------- 1 | name: Type-check 2 | 3 | on: 4 | - push 5 | - pull_request 6 | 7 | jobs: 8 | typing: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Check out repository 12 | uses: actions/checkout@v4 13 | with: 14 | fetch-depth: 0 15 | 16 | - name: Set up Python 17 | uses: actions/setup-python@v5 18 | with: 19 | python-version: '3.9' 20 | 21 | - name: Install dependencies 22 | run: | 23 | python -m pip install --upgrade pip 24 | python -m pip install --upgrade tox 25 | 26 | - name: Run type checker 27 | run: tox -e typing 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info/ 2 | *.pyc 3 | .cache/ 4 | .coverage 5 | .idea/ 6 | .tox/ 7 | .vscode/ 8 | _build/ 9 | _version.py 10 | build/ 11 | dist/ 12 | sample_nifti.json 13 | venvs/ 14 | -------------------------------------------------------------------------------- /.mailmap: -------------------------------------------------------------------------------- 1 | Basile Pinsard 2 | Chris Filo Gorgolewski Chris Gorgolewski 3 | Christopher J. Markiewicz 4 | Dae Houlihan Dae 5 | Isaac To Isaac To 6 | John Lee 7 | John Lee 8 | John T. Wodder II 9 | Jörg Stadler Joerg Stadler 10 | Jörg Stadler Joerg Stadler 11 | Jörg Stadler Jörg Stadler 12 | Jörg Stadler Jörg Stadler 13 | Mathias Goncalves mathiasg 14 | Mathias Goncalves mathiasg 15 | Mathias Goncalves Mathias Goncalves 16 | Mathias Goncalves Mathias Goncalves 17 | Matteo Visconti di Oleggio Castello Matteo Visconti dOC 18 | Matteo Visconti di Oleggio Castello Matteo Visconti dOC 19 | Matteo Visconti di Oleggio Castello 20 | Michael Dayan <79224807+neurorepro@users.noreply.github.com> 21 | Michael Krause 22 | Pablo Velasco 23 | Pablo Velasco pvelasco 24 | Satrajit Ghosh Satrajit Ghosh 25 | Steven Tilley Steven Tilley 26 | Steven Tilley Steven Tilley 27 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.4.0 4 | hooks: 5 | - id: check-added-large-files 6 | - id: check-json 7 | - id: check-toml 8 | - id: check-yaml 9 | - id: end-of-file-fixer 10 | - id: trailing-whitespace 11 | 12 | - repo: https://github.com/codespell-project/codespell 13 | rev: v2.2.4 14 | hooks: 15 | - id: codespell 16 | 17 | - repo: https://github.com/psf/black 18 | rev: 23.3.0 19 | hooks: 20 | - id: black 21 | 22 | - repo: https://github.com/PyCQA/isort 23 | rev: 5.12.0 24 | hooks: 25 | - id: isort 26 | 27 | - repo: https://github.com/PyCQA/flake8 28 | rev: 6.0.0 29 | hooks: 30 | - id: flake8 31 | additional_dependencies: 32 | - flake8-bugbear 33 | - flake8-builtins 34 | - flake8-unused-arguments 35 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | formats: all 3 | python: 4 | install: 5 | - requirements: docs/requirements.txt 6 | - method: pip 7 | path: . 8 | build: 9 | os: ubuntu-20.04 10 | tools: 11 | python: "3.9" 12 | sphinx: 13 | configuration: docs/conf.py 14 | fail_on_warning: true 15 | -------------------------------------------------------------------------------- /.zenodo.json: -------------------------------------------------------------------------------- 1 | { 2 | "creators": [ 3 | { 4 | "name": "Yaroslav O. Halchenko", 5 | "orcid": "0000-0003-3456-2493", 6 | "affiliation": "Center for Open Neuroscience, Department of Psychological and Brain Sciences, Dartmouth College, Hanover, NH, USA" 7 | }, 8 | { 9 | "name": "Mathias Goncalves", 10 | "orcid": "0000-0002-7252-7771", 11 | "affiliation": "Department of Psychology, Stanford University, CA, USA" 12 | }, 13 | { 14 | "name": "Satrajit Ghosh", 15 | "orcid": "0000-0002-5312-6729", 16 | "affiliation": "McGovern Institute, Massachusetts Institute of Technology, Cambridge, MA, USA" 17 | }, 18 | { 19 | "name": "Pablo Velasco", 20 | "orcid": "0000-0002-5749-6049", 21 | "affiliation": "Flywheel Exchange LLC, Minneapolis, MN, USA" 22 | }, 23 | { 24 | "name": "Matteo Visconti di Oleggio Castello", 25 | "orcid": "0000-0001-7931-5272", 26 | "affiliation": "University of California, Berkeley, Berkeley, CA, USA" 27 | }, 28 | { 29 | "name": "Taylor Salo", 30 | "orcid": "0000-0001-9813-3167", 31 | "affiliation": "Perelman School of Medicine, University of Pennsylvania, Philadelphia, PA, USA" 32 | }, 33 | { 34 | "name": "John T. Wodder II", 35 | "affiliation": "Center for Open Neuroscience, Department of Psychological and Brain Sciences, Dartmouth College, Hanover, NH, USA" 36 | }, 37 | { 38 | "name": "Michael Hanke", 39 | "orcid": "0000-0001-6398-6370", 40 | "affiliation": "Institute of Neuroscience and Medicine, Brain & Behaviour (INM-7), Research Center J\u00fclich, J\u00fclich, Germany and Institute of Systems Neuroscience, Medical Faculty, Heinrich Heine University D\u00fcsseldorf, D\u00fcsseldorf, Germany" 41 | }, 42 | { 43 | "name": "Patrick Sadil", 44 | "orcid": "0000-0003-4141-1343", 45 | "affiliation": "Department of Biostatistics, Johns Hopkins Bloomberg School of Public Health, Baltimore, MD, USA" 46 | }, 47 | { 48 | "name": "Krzysztof Jacek Gorgolewski", 49 | "orcid": "0000-0003-3321-7583", 50 | "affiliation": "Emeritus of Department of Psychology, Stanford University, CA, USA" 51 | }, 52 | { 53 | "name": "Horea-Ioan Ioanas", 54 | "orcid": "0000-0001-7037-2449", 55 | "affiliation": "Center for Open Neuroscience, Department of Psychological and Brain Sciences, Dartmouth College, Hanover, NH, USA" 56 | }, 57 | { 58 | "name": "Chris Rorden", 59 | "orcid": "0000-0002-7554-6142", 60 | "affiliation": "Department of Psychology, University of South Carolina, Columbia, SC, USA" 61 | }, 62 | { 63 | "name": "Timothy J. Hendrickson", 64 | "orcid": "0000-0001-6862-6526", 65 | "affiliation": "Masonic Institute\u00a0for the Developing Brain, University of Minnesota, Minneapolis, MN, USA and Minnesota Supercomputing Institute, University of Minnesota, Minneapolis, MN, USA" 66 | }, 67 | { 68 | "name": "Michael Dayan", 69 | "orcid": "0000-0002-2666-0969", 70 | "affiliation": "Human Neuroscience Platform, Fondation Campus Biotech Geneva, Geneva, Switzerland" 71 | }, 72 | { 73 | "name": "Sean Dae Houlihan", 74 | "orcid": "0000-0001-5003-9278", 75 | "affiliation": "Center for Open Neuroscience, Department of Psychological and Brain Sciences, Dartmouth College, Hanover, NH, USA and Department of Brain and Cognitive Sciences, Massachusetts Institute of Technology, Cambridge, MA, USA" 76 | }, 77 | { 78 | "name": "James Kent", 79 | "orcid": "0000-0002-4892-2659", 80 | "affiliation": "Department of Psychology, University of Texas at Austin, Austin, TX, USA" 81 | }, 82 | { 83 | "name": "Ted Strauss", 84 | "orcid": "0000-0002-1927-666X", 85 | "affiliation": "McConnell Brain Imaging Centre, McGill University, Montreal, QC, Canada" 86 | }, 87 | { 88 | "name": "John Lee", 89 | "orcid": "0000-0001-5884-4247", 90 | "affiliation": "Data Science and Sharing Team, National Institute of Mental Health, Bethesda, MD, USA" 91 | }, 92 | { 93 | "name": "Isaac To", 94 | "orcid": "0000-0002-4740-0824", 95 | "affiliation": "Center for Open Neuroscience, Department of Psychological and Brain Sciences, Dartmouth College, Hanover, NH, USA" 96 | }, 97 | { 98 | "name": "Christopher J. Markiewicz", 99 | "orcid": "0000-0002-6533-164X", 100 | "affiliation": "Department of Psychology, Stanford University, CA, USA" 101 | }, 102 | { 103 | "name": "Darren Lukas", 104 | "orcid": "0009-0003-6941-0833", 105 | "affiliation": "Institute for Glycomics, Griffith University, QLD, Australia" 106 | }, 107 | { 108 | "name": "Ellyn R. Butler", 109 | "orcid": "0000-0001-6316-6444", 110 | "affiliation": "Department of Psychology, Northwestern University, Evanston, IL, USA" 111 | }, 112 | { 113 | "name": "Todd Thompson", 114 | "affiliation": "Department of Brain and Cognitive Sciences, Massachusetts Institute of Technology, Cambridge, MA, USA" 115 | }, 116 | { 117 | "name": "Maite Termenon", 118 | "orcid": "0000-0001-8102-5135", 119 | "affiliation": "Biomedical Engineering Department, Faculty of Engineering, Mondragon University, Mondragon, Spain and BCBL, Basque center on Cognition, Brain and Language, San Sebastian, Spain" 120 | }, 121 | { 122 | "name": "David V. Smith", 123 | "orcid": "0000-0001-5754-9633", 124 | "affiliation": "Department of Psychology and Neuroscience, Temple University, Philadelphia, PA, USA" 125 | }, 126 | { 127 | "name": "Austin Macdonald", 128 | "orcid": "0000-0002-8124-807X", 129 | "affiliation": "Center for Open Neuroscience, Department of Psychological and Brain Sciences, Dartmouth College, Hanover, NH, USA" 130 | }, 131 | { 132 | "name": "David N. Kennedy", 133 | "orcid": "0000-0002-9377-0797", 134 | "affiliation": "Departments of Psychiatry and Radiology, University of Massachusetts Chan Medical School, Worcester, MA, USA" 135 | } 136 | ], 137 | "keywords": [ 138 | "Python", 139 | "neuroscience", 140 | "standardization", 141 | "DICOM", 142 | "BIDS", 143 | "open science", 144 | "FOSS" 145 | ], 146 | "access_right": "open", 147 | "license": "Apache-2.0", 148 | "upload_type": "software", 149 | "title": "HeuDiConv — flexible DICOM conversion into structured directory layouts" 150 | } 151 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | ========================= 2 | Contributing to HeuDiConv 3 | ========================= 4 | 5 | Files organization 6 | ------------------ 7 | 8 | * `heudiconv/ <./heudiconv>`_ is the main Python module where major development is happening, with 9 | major submodules being: 10 | 11 | - ``cli/`` - wrappers and argument parsers bringing the HeuDiConv functionality to the command 12 | line. 13 | - ``external/`` - general compatibility layers for external functions HeuDiConv depends on. 14 | - ``heuristics/`` - heuristic evaluators for workflows, pull requests here are particularly 15 | welcome. 16 | 17 | * `docs/ <./docs>`_ - documentation directory. 18 | * `utils/ <./utils>`_ - helper utilities used during development, testing, and distribution of 19 | HeuDiConv. 20 | 21 | How to contribute 22 | ----------------- 23 | 24 | The preferred way to contribute to the HeuDiConv code base is 25 | to fork the `main repository `_ on GitHub. 26 | 27 | If you are unsure what that means, here is a set-up workflow you may wish to follow: 28 | 29 | 0. Fork the `project repository `_ on GitHub, by clicking 30 | on the “Fork” button near the top of the page — this will create a copy of the repository 31 | writeable by your GitHub user. 32 | 1. Set up a clone of the repository on your local machine and connect it to both the “official” 33 | and your copy of the repository on GitHub:: 34 | 35 | git clone git://github.com/nipy/heudiconv 36 | cd heudiconv 37 | git remote rename origin official 38 | git remote add origin git://github.com/YOUR_GITHUB_USERNAME/heudiconv 39 | 40 | 2. When you wish to start a new contribution, create a new branch:: 41 | 42 | git checkout -b topic_of_your_contribution 43 | 44 | 3. When you are done making the changes you wish to contribute, record them in Git:: 45 | 46 | git add the/paths/to/files/you/modified can/be/more/than/one 47 | git commit 48 | 49 | 3. Push the changes to your copy of the code on GitHub, following which Git will 50 | provide you with a link which you can click to initiate a pull request:: 51 | 52 | git push -u origin topic_of_your_contribution 53 | 54 | (If any of the above seems overwhelming, you can look up the `Git documentation 55 | `_ on the web.) 56 | 57 | 58 | Releases and Changelog 59 | ---------------------- 60 | 61 | HeuDiConv uses the `auto `_ tool to generate the changelog and automatically release the project. 62 | 63 | `auto` is used in the HeuDiConv GitHub actions, which monitors the labels on the pull request. 64 | HeuDiConv automation can add entries to the changelog, cut releases, and 65 | push new images to `dockerhub `_. 66 | 67 | The following pull request labels are respected: 68 | 69 | * major: Increment the major version when merged 70 | * minor: Increment the minor version when merged 71 | * patch: Increment the patch version when merged 72 | * skip-release: Preserve the current version when merged 73 | * release: Create a release when this pr is merged 74 | * internal: Changes only affect the internal API 75 | * documentation: Changes only affect the documentation 76 | * tests: Add or improve existing tests 77 | * dependencies: Update one or more dependencies version 78 | * performance: Improve performance of an existing feature 79 | 80 | 81 | Development environment 82 | ----------------------- 83 | 84 | We support Python 3 only (>= 3.7). 85 | 86 | Dependencies which you will need are `listed in the repository `_. 87 | Note that you will likely have these will already be available on your system if you used a 88 | package manager (e.g. Debian's ``apt-get``, Gentoo's ``emerge``, or simply PIP) to install the 89 | software. 90 | 91 | Development work might require live access to the copy of HeuDiConv which is being developed. 92 | If a system-wide release of HeuDiConv is already installed, or likely to be, it is best to keep 93 | development work sandboxed inside a dedicated virtual environment. 94 | This is best accomplished via:: 95 | 96 | cd /path/to/your/clone/of/heudiconv 97 | mkdir -p venvs/dev 98 | python -m venv venvs/dev 99 | source venvs/dev/bin/activate 100 | pip install -e .[all] 101 | 102 | 103 | Documentation 104 | ------------- 105 | 106 | To contribute to the documentation, we recommend building the docs 107 | locally prior to submitting a patch. 108 | 109 | To build the docs locally: 110 | 111 | 1. From the root of the heudiconv repository, `pip install -r docs/requirements.txt` 112 | 2. From the `docs/` directory, run `make html` 113 | 114 | 115 | Additional Hints 116 | ---------------- 117 | 118 | It is recommended to check that your contribution complies with the following 119 | rules before submitting a pull request: 120 | 121 | * All public functions (i.e. functions whose name does not start with an underscore) should have 122 | informative docstrings with sample usage presented as doctests when appropriate. 123 | * Docstrings are formatted in `NumPy style `_. 124 | * Lines are no longer than 120 characters. 125 | * All tests still pass:: 126 | 127 | cd /path/to/your/clone/of/heudiconv 128 | pytest -vvs . 129 | 130 | * New code should be accompanied by new tests. 131 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Generated by Neurodocker and Reproenv. 2 | 3 | FROM neurodebian:bookworm 4 | ENV PATH="/opt/dcm2niix-v1.0.20240202/bin:$PATH" 5 | RUN apt-get update -qq \ 6 | && apt-get install -y -q --no-install-recommends \ 7 | ca-certificates \ 8 | cmake \ 9 | g++ \ 10 | gcc \ 11 | git \ 12 | make \ 13 | pigz \ 14 | zlib1g-dev \ 15 | && rm -rf /var/lib/apt/lists/* \ 16 | && git clone https://github.com/rordenlab/dcm2niix /tmp/dcm2niix \ 17 | && cd /tmp/dcm2niix \ 18 | && git fetch --tags \ 19 | && git checkout v1.0.20240202 \ 20 | && mkdir /tmp/dcm2niix/build \ 21 | && cd /tmp/dcm2niix/build \ 22 | && cmake -DZLIB_IMPLEMENTATION=Cloudflare -DUSE_JPEGLS=ON -DUSE_OPENJPEG=ON -DCMAKE_INSTALL_PREFIX:PATH=/opt/dcm2niix-v1.0.20240202 .. \ 23 | && make -j1 \ 24 | && make install \ 25 | && rm -rf /tmp/dcm2niix 26 | RUN apt-get update -qq \ 27 | && apt-get install -y -q --no-install-recommends \ 28 | gcc \ 29 | git \ 30 | git-annex-standalone \ 31 | libc-dev \ 32 | liblzma-dev \ 33 | netbase \ 34 | pigz \ 35 | && rm -rf /var/lib/apt/lists/* 36 | COPY [".", \ 37 | "/src/heudiconv"] 38 | ENV CONDA_DIR="/opt/miniconda-py39_4.12.0" \ 39 | PATH="/opt/miniconda-py39_4.12.0/bin:$PATH" 40 | RUN apt-get update -qq \ 41 | && apt-get install -y -q --no-install-recommends \ 42 | bzip2 \ 43 | ca-certificates \ 44 | curl \ 45 | && rm -rf /var/lib/apt/lists/* \ 46 | # Install dependencies. 47 | && export PATH="/opt/miniconda-py39_4.12.0/bin:$PATH" \ 48 | && echo "Downloading Miniconda installer ..." \ 49 | && conda_installer="/tmp/miniconda.sh" \ 50 | && curl -fsSL -o "$conda_installer" https://repo.continuum.io/miniconda/Miniconda3-py39_4.12.0-Linux-x86_64.sh \ 51 | && bash "$conda_installer" -b -p /opt/miniconda-py39_4.12.0 \ 52 | && rm -f "$conda_installer" \ 53 | # Prefer packages in conda-forge 54 | && conda config --system --prepend channels conda-forge \ 55 | # Packages in lower-priority channels not considered if a package with the same 56 | # name exists in a higher priority channel. Can dramatically speed up installations. 57 | # Conda recommends this as a default 58 | # https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-channels.html 59 | && conda config --set channel_priority strict \ 60 | && conda config --system --set auto_update_conda false \ 61 | && conda config --system --set show_channel_urls true \ 62 | # Enable `conda activate` 63 | && conda init bash \ 64 | && conda install -y --name base \ 65 | "python=3.9" \ 66 | "traits>=4.6.0" \ 67 | "scipy" \ 68 | "numpy" \ 69 | "nomkl" \ 70 | "pandas" \ 71 | "gdcm" \ 72 | && bash -c "source activate base \ 73 | && python -m pip install --no-cache-dir --editable \ 74 | "/src/heudiconv[all]"" \ 75 | # Clean up 76 | && sync && conda clean --all --yes && sync \ 77 | && rm -rf ~/.cache/pip/* 78 | ENTRYPOINT ["heudiconv"] 79 | 80 | # Save specification to JSON. 81 | RUN printf '{ \ 82 | "pkg_manager": "apt", \ 83 | "existing_users": [ \ 84 | "root" \ 85 | ], \ 86 | "instructions": [ \ 87 | { \ 88 | "name": "from_", \ 89 | "kwds": { \ 90 | "base_image": "neurodebian:bookworm" \ 91 | } \ 92 | }, \ 93 | { \ 94 | "name": "env", \ 95 | "kwds": { \ 96 | "PATH": "/opt/dcm2niix-v1.0.20240202/bin:$PATH" \ 97 | } \ 98 | }, \ 99 | { \ 100 | "name": "run", \ 101 | "kwds": { \ 102 | "command": "apt-get update -qq\\napt-get install -y -q --no-install-recommends \\\\\\n ca-certificates \\\\\\n cmake \\\\\\n g++ \\\\\\n gcc \\\\\\n git \\\\\\n make \\\\\\n pigz \\\\\\n zlib1g-dev\\nrm -rf /var/lib/apt/lists/*\\ngit clone https://github.com/rordenlab/dcm2niix /tmp/dcm2niix\\ncd /tmp/dcm2niix\\ngit fetch --tags\\ngit checkout v1.0.20240202\\nmkdir /tmp/dcm2niix/build\\ncd /tmp/dcm2niix/build\\ncmake -DZLIB_IMPLEMENTATION=Cloudflare -DUSE_JPEGLS=ON -DUSE_OPENJPEG=ON -DCMAKE_INSTALL_PREFIX:PATH=/opt/dcm2niix-v1.0.20240202 ..\\nmake -j1\\nmake install\\nrm -rf /tmp/dcm2niix" \ 103 | } \ 104 | }, \ 105 | { \ 106 | "name": "install", \ 107 | "kwds": { \ 108 | "pkgs": [ \ 109 | "git", \ 110 | "gcc", \ 111 | "pigz", \ 112 | "liblzma-dev", \ 113 | "libc-dev", \ 114 | "git-annex-standalone", \ 115 | "netbase" \ 116 | ], \ 117 | "opts": null \ 118 | } \ 119 | }, \ 120 | { \ 121 | "name": "run", \ 122 | "kwds": { \ 123 | "command": "apt-get update -qq \\\\\\n && apt-get install -y -q --no-install-recommends \\\\\\n gcc \\\\\\n git \\\\\\n git-annex-standalone \\\\\\n libc-dev \\\\\\n liblzma-dev \\\\\\n netbase \\\\\\n pigz \\\\\\n && rm -rf /var/lib/apt/lists/*" \ 124 | } \ 125 | }, \ 126 | { \ 127 | "name": "copy", \ 128 | "kwds": { \ 129 | "source": [ \ 130 | ".", \ 131 | "/src/heudiconv" \ 132 | ], \ 133 | "destination": "/src/heudiconv" \ 134 | } \ 135 | }, \ 136 | { \ 137 | "name": "env", \ 138 | "kwds": { \ 139 | "CONDA_DIR": "/opt/miniconda-py39_4.12.0", \ 140 | "PATH": "/opt/miniconda-py39_4.12.0/bin:$PATH" \ 141 | } \ 142 | }, \ 143 | { \ 144 | "name": "run", \ 145 | "kwds": { \ 146 | "command": "apt-get update -qq\\napt-get install -y -q --no-install-recommends \\\\\\n bzip2 \\\\\\n ca-certificates \\\\\\n curl\\nrm -rf /var/lib/apt/lists/*\\n# Install dependencies.\\nexport PATH=\\"/opt/miniconda-py39_4.12.0/bin:$PATH\\"\\necho \\"Downloading Miniconda installer ...\\"\\nconda_installer=\\"/tmp/miniconda.sh\\"\\ncurl -fsSL -o \\"$conda_installer\\" https://repo.continuum.io/miniconda/Miniconda3-py39_4.12.0-Linux-x86_64.sh\\nbash \\"$conda_installer\\" -b -p /opt/miniconda-py39_4.12.0\\nrm -f \\"$conda_installer\\"\\n# Prefer packages in conda-forge\\nconda config --system --prepend channels conda-forge\\n# Packages in lower-priority channels not considered if a package with the same\\n# name exists in a higher priority channel. Can dramatically speed up installations.\\n# Conda recommends this as a default\\n# https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-channels.html\\nconda config --set channel_priority strict\\nconda config --system --set auto_update_conda false\\nconda config --system --set show_channel_urls true\\n# Enable `conda activate`\\nconda init bash\\nconda install -y --name base \\\\\\n \\"python=3.9\\" \\\\\\n \\"traits>=4.6.0\\" \\\\\\n \\"scipy\\" \\\\\\n \\"numpy\\" \\\\\\n \\"nomkl\\" \\\\\\n \\"pandas\\" \\\\\\n \\"gdcm\\"\\nbash -c \\"source activate base\\n python -m pip install --no-cache-dir --editable \\\\\\n \\"/src/heudiconv[all]\\"\\"\\n# Clean up\\nsync && conda clean --all --yes && sync\\nrm -rf ~/.cache/pip/*" \ 147 | } \ 148 | }, \ 149 | { \ 150 | "name": "entrypoint", \ 151 | "kwds": { \ 152 | "args": [ \ 153 | "heudiconv" \ 154 | ] \ 155 | } \ 156 | } \ 157 | ] \ 158 | }' > /.reproenv.json 159 | # End saving to specification to JSON. 160 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright [2014-2024] [HeuDiConv developers] 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | 15 | 16 | Some parts of the codebase/documentation are borrowed from other sources: 17 | 18 | - HeuDiConv tutorial from https://bitbucket.org/dpat/neuroimaging_core_docs/src 19 | 20 | Copyright 2023 Dianne Patterson 21 | -------------------------------------------------------------------------------- /NOTES: -------------------------------------------------------------------------------- 1 | Requires python-rdflib due to PROV generation. 2 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | **HeuDiConv** 3 | ============= 4 | 5 | `a heuristic-centric DICOM converter` 6 | 7 | .. image:: https://joss.theoj.org/papers/10.21105/joss.05839/status.svg 8 | :target: https://doi.org/10.21105/joss.05839 9 | :alt: JOSS Paper 10 | 11 | .. image:: https://img.shields.io/badge/docker-nipy/heudiconv:latest-brightgreen.svg?logo=docker&style=flat 12 | :target: https://hub.docker.com/r/nipy/heudiconv/tags/ 13 | :alt: Our Docker image 14 | 15 | .. image:: https://github.com/nipy/heudiconv/actions/workflows/test.yml/badge.svg?event=push 16 | :target: https://github.com/nipy/heudiconv/actions/workflows/test.yml 17 | :alt: GitHub Actions (test) 18 | 19 | .. image:: https://codecov.io/gh/nipy/heudiconv/branch/master/graph/badge.svg 20 | :target: https://codecov.io/gh/nipy/heudiconv 21 | :alt: CodeCoverage 22 | 23 | .. image:: https://readthedocs.org/projects/heudiconv/badge/?version=latest 24 | :target: http://heudiconv.readthedocs.io/en/latest/?badge=latest 25 | :alt: Readthedocs 26 | 27 | .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.1012598.svg 28 | :target: https://doi.org/10.5281/zenodo.1012598 29 | :alt: Zenodo (latest) 30 | 31 | .. image:: https://repology.org/badge/version-for-repo/debian_unstable/heudiconv.svg?header=Debian%20Unstable 32 | :target: https://repology.org/project/heudiconv/versions 33 | :alt: Debian Unstable 34 | 35 | .. image:: https://repology.org/badge/version-for-repo/gentoo_ovl_science/python:heudiconv.svg?header=Gentoo%20%28%3A%3Ascience%29 36 | :target: https://repology.org/project/python:heudiconv/versions 37 | :alt: Gentoo (::science) 38 | 39 | .. image:: https://repology.org/badge/version-for-repo/pypi/python:heudiconv.svg?header=PyPI 40 | :target: https://repology.org/project/python:heudiconv/versions 41 | :alt: PyPI 42 | 43 | .. image:: https://img.shields.io/badge/RRID-SCR__017427-blue 44 | :target: https://identifiers.org/RRID:SCR_017427 45 | :alt: RRID 46 | 47 | About 48 | ----- 49 | 50 | ``heudiconv`` is a flexible DICOM converter for organizing brain imaging data 51 | into structured directory layouts. 52 | 53 | - It allows flexible directory layouts and naming schemes through customizable heuristics implementations. 54 | - It only converts the necessary DICOMs and ignores everything else in a directory. 55 | - You can keep links to DICOM files in the participant layout. 56 | - Using `dcm2niix `_ under the hood, it's fast. 57 | - It can track the provenance of the conversion from DICOM to NIfTI in W3C PROV format. 58 | - It provides assistance in converting to `BIDS `_. 59 | - It integrates with `DataLad `_ to place converted and original data under git/git-annex 60 | version control while automatically annotating files with sensitive information (e.g., non-defaced anatomicals, etc). 61 | 62 | Heudiconv can be inserted into your workflow to provide automatic conversion as part of a data acquisition pipeline, as seen in the figure below: 63 | 64 | .. image:: figs/environment.png 65 | 66 | Installation 67 | ------------ 68 | 69 | See our `installation page `_ 70 | on heudiconv.readthedocs.io . 71 | 72 | HOWTO 101 73 | --------- 74 | 75 | In a nutshell -- ``heudiconv`` is given a file tree of DICOMs, and it produces a restructured file tree of NifTI files (conversion handled by `dcm2niix`_) with accompanying metadata files. 76 | The input and output structure is as flexible as your data, which is accomplished by using a Python file called a ``heuristic`` that knows how to read your input structure and decides how to name the resultant files. 77 | You can run your conversion automatically (which will produce a ``.heudiconv`` directory storing the used parameters), or generate the default parameters, edit them to customize file naming, and continue conversion via an additional invocation of `heudiconv`: 78 | 79 | .. image:: figs/workflow.png 80 | 81 | 82 | ``heudiconv`` comes with `existing heuristics `_ which can be used as is, or as examples. 83 | For instance, the Heuristic `convertall `_ extracts standard metadata from all matching DICOMs. 84 | ``heudiconv`` creates mapping files, ``.edit.text`` which lets researchers simply establish their own conversion mapping. 85 | 86 | In most use-cases of retrospective study data conversion, you would need to create your custom heuristic following the examples and the `"Heuristic" section `_ in the documentation. 87 | **Note** that `ReproIn heuristic `_ is 88 | generic and powerful enough to be adopted virtually for *any* study: For prospective studies, you would just need 89 | to name your sequences following the `ReproIn convention `_, and for 90 | retrospective conversions, you often would be able to create a new versatile heuristic by simply providing 91 | remappings into ReproIn as shown in `this issue (documentation is coming) `_. 92 | 93 | Having decided on a heuristic, you could use the command line:: 94 | 95 | heudiconv -f HEURISTIC-FILE-OR-NAME -o OUTPUT-PATH --files INPUT-PATHs 96 | 97 | with various additional options (see ``heudiconv --help`` or 98 | `"CLI Reference" in documentation `__) to tune its behavior to 99 | convert your data. 100 | 101 | For detailed examples and guides, please check out `ReproIn conversion invocation examples `_ 102 | and the `user tutorials `_ in the documentation. 103 | 104 | 105 | How to cite 106 | ----------- 107 | 108 | Please use `Zenodo record `_ for 109 | your specific version of HeuDiConv. We also support gathering 110 | all relevant citations via `DueCredit `_. 111 | 112 | 113 | How to contribute 114 | ----------------- 115 | 116 | For a detailed into, see our `contributing guide `_. 117 | 118 | Our releases are packaged using Intuit auto, with the corresponding workflow including 119 | Docker image preparation being found in ``.github/workflows/release.yml``. 120 | 121 | 122 | 3-rd party heuristics 123 | --------------------- 124 | 125 | - https://github.com/courtois-neuromod/ds_prep/blob/main/mri/convert/heuristics_unf.py 126 | 127 | 128 | Support 129 | ------- 130 | 131 | All bugs, concerns and enhancement requests for this software can be submitted here: 132 | https://github.com/nipy/heudiconv/issues. 133 | 134 | If you have a problem or would like to ask a question about how to use ``heudiconv``, 135 | please submit a question to `NeuroStars.org `_ with a ``heudiconv`` tag. 136 | NeuroStars.org is a platform similar to StackOverflow but dedicated to neuroinformatics. 137 | 138 | All previous ``heudiconv`` questions are available here: 139 | http://neurostars.org/tags/heudiconv/ . 140 | -------------------------------------------------------------------------------- /custom/dbic/README: -------------------------------------------------------------------------------- 1 | Scripts and configurations used alongside with heudiconv setup at DBIC 2 | (Dartmouth Brain Imaging Center). Might migrate to an independent repository 3 | eventually but for now placed here with hope they might come useful for some. 4 | -------------------------------------------------------------------------------- /custom/dbic/singularity-env.def: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2015-2016, Gregory M. Kurtzer. All rights reserved. 2 | # 3 | # Changes for NeuroDebian/DBIC setup are Copyright (c) 2017 Yaroslav Halchenko. 4 | # 5 | # The purpose of the environment is to provide a complete suite for running 6 | # heudiconv on the INBOX server to provide conversion into BIDS layout. 7 | # ATM it does not ship heudiconv itself which would be accessed directly 8 | # from the main drive for now. 9 | # 10 | # "Singularity" Copyright (c) 2016, The Regents of the University of California, 11 | # through Lawrence Berkeley National Laboratory (subject to receipt of any 12 | # required approvals from the U.S. Dept. of Energy). All rights reserved. 13 | 14 | # 15 | # Notes: 16 | # - Due to https://github.com/singularityware/singularity/issues/471 17 | # bootstrapping leads to non-usable/non-removable-without-reboot 18 | # image due to some rogue run away processes. 19 | # This line could help to kill them but should be used with caution 20 | # since could kill other unrelated processes 21 | # 22 | # grep -l loop /proc/*/mountinfo | sed -e 's,/proc/\(.*\)/.*,\1,g' | while read pid; do sudo kill $pid; done 23 | 24 | BootStrap: debootstrap 25 | #OSVersion: stable 26 | # needs nipype 0.12.1 but that one didn't build for stable since needs python-prov... 27 | # so trying stretch 28 | OSVersion: stretch 29 | MirrorURL: http://ftp.us.debian.org/debian/ 30 | 31 | # so if image is executed we just enter the environment 32 | %runscript 33 | echo "Welcome to the DBIC BIDS environment" 34 | /bin/bash 35 | 36 | 37 | %post 38 | echo "Configuring the environment" 39 | apt-get update 40 | apt-get -y install eatmydata 41 | eatmydata apt-get -y install vim wget strace time ncdu gnupg curl procps 42 | wget -q -O/tmp/nd-configurerepo https://raw.githubusercontent.com/neurodebian/neurodebian/4d26c8f30433145009aa3f74516da12f560a5a13/tools/nd-configurerepo 43 | bash /tmp/nd-configurerepo 44 | chmod a+r -R /etc/apt 45 | eatmydata apt-get -y install datalad python-nipype virtualenv dcm2niix python-dcmstack python-configparser python-funcsigs python-pytest dcmtk 46 | 47 | # for bids-validator 48 | curl -sL https://deb.nodesource.com/setup_6.x | bash - && \ 49 | eatmydata apt-get install -y nodejs 50 | npm install -g bids-validator@0.20.0 51 | chmod a+rX -R /usr/lib/node_modules/ 52 | 53 | chmod a+rX -R /etc/apt/sources.list.d 54 | rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 55 | apt-get clean 56 | 57 | # and wipe out apt lists since not to be used RW for further tuning 58 | # find /var/lib/apt/lists/ -type f -delete 59 | # /usr/bin/find /var/lib/apt/lists/ -type f -name \*Packages\* -o -name \*Contents\* 60 | # complicates later interrogation - thus disabled 61 | 62 | # Create some bind mount directories present on rolando 63 | mkdir -p /afs /inbox 64 | chmod a+rX /afs /inbox 65 | -------------------------------------------------------------------------------- /dev-requirements.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | tinydb 3 | inotify 4 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = . 8 | BUILDDIR = _build 9 | 10 | # Put it first so that "make" without argument is like "make help". 11 | help: 12 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 13 | 14 | .PHONY: help Makefile 15 | 16 | # Catch-all target: route all unknown targets to Sphinx using the new 17 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 18 | %: Makefile 19 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 20 | -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | API Reference 3 | ============= 4 | 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | api/bids 9 | api/convert 10 | api/dicoms 11 | api/parser 12 | api/queue 13 | api/utils 14 | -------------------------------------------------------------------------------- /docs/api/bids.rst: -------------------------------------------------------------------------------- 1 | ==== 2 | BIDS 3 | ==== 4 | 5 | .. automodule:: heudiconv.bids 6 | -------------------------------------------------------------------------------- /docs/api/convert.rst: -------------------------------------------------------------------------------- 1 | ========== 2 | Conversion 3 | ========== 4 | 5 | .. automodule:: heudiconv.convert 6 | -------------------------------------------------------------------------------- /docs/api/dicoms.rst: -------------------------------------------------------------------------------- 1 | ====== 2 | DICOMS 3 | ====== 4 | 5 | .. automodule:: heudiconv.dicoms 6 | -------------------------------------------------------------------------------- /docs/api/parser.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Parsing 3 | ======= 4 | 5 | .. automodule:: heudiconv.parser 6 | -------------------------------------------------------------------------------- /docs/api/queue.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | Batch Queuing 3 | ============= 4 | 5 | .. automodule:: heudiconv.queue 6 | -------------------------------------------------------------------------------- /docs/api/utils.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Utility 3 | ======= 4 | 5 | .. automodule:: heudiconv.utils 6 | -------------------------------------------------------------------------------- /docs/changes.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Changes 3 | ======= 4 | 5 | .. literalinclude:: ../CHANGELOG.md 6 | -------------------------------------------------------------------------------- /docs/commandline.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | CLI Reference 3 | ============= 4 | 5 | ``heudiconv`` processes DICOM files and converts the output into user defined 6 | paths. 7 | 8 | .. argparse:: 9 | :ref: heudiconv.cli.run.get_parser 10 | :prog: heudiconv 11 | :nodefault: 12 | :nodefaultconst: 13 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Configuration file for the Sphinx documentation builder. 4 | # 5 | # This file does only contain a selection of the most common options. For a 6 | # full list see the documentation: 7 | # http://www.sphinx-doc.org/en/master/config 8 | 9 | # -- Path setup -------------------------------------------------------------- 10 | 11 | # If extensions (or modules to document with autodoc) are in another directory, 12 | # add these directories to sys.path here. If the directory is relative to the 13 | # documentation root, use os.path.abspath to make it absolute, like shown here. 14 | # 15 | # import os 16 | # import sys 17 | # sys.path.insert(0, os.path.abspath('.')) 18 | 19 | 20 | # -- Project information ----------------------------------------------------- 21 | 22 | from heudiconv import __version__ 23 | 24 | project = "heudiconv" 25 | copyright = "2014-2022, Heudiconv team" # noqa: A001 26 | author = "Heudiconv team" 27 | 28 | # The short X.Y version 29 | version = ".".join(__version__.split(".")[:2]) 30 | # The full version, including alpha/beta/rc tags 31 | release = __version__ 32 | 33 | 34 | # -- General configuration --------------------------------------------------- 35 | 36 | # If your documentation needs a minimal Sphinx version, state it here. 37 | # 38 | # needs_sphinx = '1.0' 39 | 40 | # Add any Sphinx extension module names here, as strings. They can be 41 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 42 | # ones. 43 | extensions = [ 44 | "sphinx.ext.autodoc", 45 | "sphinxarg.ext", 46 | "sphinx.ext.napoleon", 47 | ] 48 | 49 | # Add any paths that contain templates here, relative to this directory. 50 | templates_path = ["_templates"] 51 | 52 | # The suffix(es) of source filenames. 53 | # You can specify multiple suffix as a list of string: 54 | # 55 | source_suffix = [".rst", ".md"] 56 | # source_suffix = '.rst' 57 | 58 | # The master toctree document. 59 | master_doc = "index" 60 | 61 | # The language for content autogenerated by Sphinx. Refer to documentation 62 | # for a list of supported languages. 63 | # 64 | # This is also used if you do content translation via gettext catalogs. 65 | # Usually you set "language" from the command line for these cases. 66 | language = "en" 67 | 68 | # List of patterns, relative to source directory, that match files and 69 | # directories to ignore when looking for source files. 70 | # This pattern also affects html_static_path and html_extra_path. 71 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 72 | 73 | # The name of the Pygments (syntax highlighting) style to use. 74 | pygments_style = None 75 | 76 | 77 | # -- Options for HTML output ------------------------------------------------- 78 | 79 | # The theme to use for HTML and HTML Help pages. See the documentation for 80 | # a list of builtin themes. 81 | # 82 | html_theme = "alabaster" 83 | 84 | # Theme options are theme-specific and customize the look and feel of a theme 85 | # further. For a list of options available for each theme, see the 86 | # documentation. 87 | # 88 | # html_theme_options = {} 89 | 90 | # Add any paths that contain custom static files (such as style sheets) here, 91 | # relative to this directory. They are copied after the builtin static files, 92 | # so a file named "default.css" will overwrite the builtin "default.css". 93 | html_static_path = [] 94 | 95 | # Custom sidebar templates, must be a dictionary that maps document names 96 | # to template names. 97 | # 98 | # The default sidebars (for documents that don't match any pattern) are 99 | # defined by theme itself. Builtin themes are using these templates by 100 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 101 | # 'searchbox.html']``. 102 | # 103 | # html_sidebars = {} 104 | 105 | 106 | # -- Options for HTMLHelp output --------------------------------------------- 107 | 108 | # Output file base name for HTML help builder. 109 | htmlhelp_basename = "heudiconvdoc" 110 | 111 | 112 | # -- Options for LaTeX output ------------------------------------------------ 113 | 114 | latex_elements = { 115 | # The paper size ('letterpaper' or 'a4paper'). 116 | # 117 | # 'papersize': 'letterpaper', 118 | # The font size ('10pt', '11pt' or '12pt'). 119 | # 120 | # 'pointsize': '10pt', 121 | # Additional stuff for the LaTeX preamble. 122 | # 123 | # 'preamble': '', 124 | # Latex figure (float) alignment 125 | # 126 | # 'figure_align': 'htbp', 127 | } 128 | 129 | # Grouping the document tree into LaTeX files. List of tuples 130 | # (source start file, target name, title, 131 | # author, documentclass [howto, manual, or own class]). 132 | latex_documents = [ 133 | ( 134 | master_doc, 135 | "heudiconv.tex", 136 | "heudiconv Documentation", 137 | "Heudiconv team", 138 | "manual", 139 | ), 140 | ] 141 | 142 | 143 | # -- Options for manual page output ------------------------------------------ 144 | 145 | # One entry per manual page. List of tuples 146 | # (source start file, name, description, authors, manual section). 147 | man_pages = [(master_doc, "heudiconv", "heudiconv Documentation", [author], 1)] 148 | 149 | 150 | # -- Options for Texinfo output ---------------------------------------------- 151 | 152 | # Grouping the document tree into Texinfo files. List of tuples 153 | # (source start file, target name, title, author, 154 | # dir menu entry, description, category) 155 | texinfo_documents = [ 156 | ( 157 | master_doc, 158 | "heudiconv", 159 | "heudiconv Documentation", 160 | author, 161 | "heudiconv", 162 | "One line description of project.", 163 | "Miscellaneous", 164 | ), 165 | ] 166 | 167 | 168 | # -- Options for Epub output ------------------------------------------------- 169 | 170 | # Bibliographic Dublin Core info. 171 | epub_title = project 172 | 173 | # The unique identifier of the text. This can be a ISBN number 174 | # or the project homepage. 175 | # 176 | # epub_identifier = '' 177 | 178 | # A unique identification for the text. 179 | # 180 | # epub_uid = '' 181 | 182 | # A list of files that should not be packed into the epub file. 183 | epub_exclude_files = ["search.html"] 184 | 185 | 186 | # -- Extension configuration ------------------------------------------------- 187 | autodoc_default_options = {"members": None} 188 | -------------------------------------------------------------------------------- /docs/container.rst: -------------------------------------------------------------------------------- 1 | ============================== 2 | Using heudiconv in a Container 3 | ============================== 4 | 5 | If heudiconv is :ref:`installed via a Docker container `, you 6 | can run the commands in the following format:: 7 | 8 | docker run nipy/heudiconv:latest [heudiconv options] 9 | 10 | So a user running via container would check the version with this command:: 11 | 12 | docker run nipy/heudiconv:latest --version 13 | 14 | Which is equivalent to the locally installed command:: 15 | 16 | heudiconv --version 17 | 18 | Bind mount 19 | ---------- 20 | 21 | Typically, users of heudiconv will be operating on data that is on their local machine. We can give heudiconv access to that data via a ``bind mount``, which is the ``-v`` syntax. 22 | 23 | Once common pattern is to share the working directory with ``-v $PWD:$PWD``, so heudiconv will behave as though it is installed on your system. However, you should be aware of how permissions work depending on your container toolset. 24 | 25 | 26 | Docker Permissions 27 | ****************** 28 | 29 | When you run a container with docker without specifying a user, it will be run as root. 30 | This isn't ideal if you are operating on data owned by your local user, so for ``Docker`` it is recommended to specify that the container will run as your user.:: 31 | 32 | docker run --user=$(id -u):$(id -g) -e "UID=$(id -u)" -e "GID=$(id -g)" --rm -t -v $PWD:$PWD nipy/heudiconv:latest --version 33 | 34 | Podman Permissions 35 | ****************** 36 | 37 | When running Podman without specifying a user, the container is run as root inside the container, but your user outside of the container. 38 | This default behavior usually works for heudiconv users:: 39 | 40 | docker run -v $PWD:PWD nipy/heudiconv:latest --version 41 | 42 | Other Common Options 43 | -------------------- 44 | 45 | We typically recommend users make use of the following flags to Docker and Podman 46 | 47 | * ``-it`` Interactive terminal 48 | * ``--rm`` Remove the changes to the container when it completes 49 | -------------------------------------------------------------------------------- /docs/figs: -------------------------------------------------------------------------------- 1 | ../figs/ -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. heudiconv documentation master file, created by 2 | sphinx-quickstart on Mon Mar 25 15:42:31 2019. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | .. include:: ../README.rst 7 | .. include:: ../CONTRIBUTING.rst 8 | 9 | Contents 10 | -------- 11 | 12 | .. toctree:: 13 | :maxdepth: 2 14 | 15 | installation 16 | changes 17 | tutorials 18 | heuristics 19 | commandline 20 | container 21 | api 22 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Installation 3 | ============ 4 | 5 | ``Heudiconv`` is packaged and available from many different sources. 6 | 7 | .. _install_local: 8 | 9 | Local 10 | ===== 11 | Released versions of HeuDiConv are available on `PyPI `_ 12 | and `conda `_. 13 | If installing through ``PyPI``, eg:: 14 | 15 | pip install heudiconv[all] 16 | 17 | Manual installation of `dcm2niix `_ 18 | is required. You can also benefit from an installer/downloader helper ``dcm2niix`` package 19 | on ``PyPI``, so you can simply ``pip install dcm2niix`` if you are installing in user space so 20 | subsequently it would be able to download and install dcm2niix binary. 21 | 22 | On Debian-based systems, we recommend using `NeuroDebian `_, 23 | which provides the `heudiconv package `_. 24 | 25 | .. _install_container: 26 | 27 | Containers 28 | ========== 29 | 30 | Our container image releases are available on `our Docker Hub `_ 31 | 32 | If `Docker `_ is available on your system, you can pull the latest release:: 33 | 34 | $ docker pull nipy/heudiconv:latest 35 | 36 | Additionally, HeuDiConv is available through the Docker image at `repronim/reproin `_ provided by 37 | `ReproIn heuristic project `_, which develops the ``reproin`` heuristic. 38 | 39 | To maintain provenance, it is recommended that you use the ``latest`` tag only when testing out heudiconv. 40 | Otherwise, it is recommended that you use an explicit version and record that information alongside the produced data. 41 | 42 | 43 | Singularity 44 | =========== 45 | If `Singularity `_ is available on your system, 46 | you can use it to pull and convert our Docker images! For example, to pull and 47 | build the latest release, you can run:: 48 | 49 | $ singularity pull docker://nipy/heudiconv:latest 50 | 51 | 52 | Singularity YODA style using ///repronim/containers 53 | =================================================== 54 | `ReproNim `_ provides a large collection of Singularity container images of popular 55 | neuroimaging tools, e.g. all the BIDS-Apps. This collection also includes the forementioned container 56 | images for `HeuDiConv `_ and 57 | `ReproIn `_ in the Singularity image format. This collection is available as a 58 | `DataLad `_ dataset at `///repronim/containers `_ 59 | on `datasets.datalad.org `_ and as `a GitHub repo `_. 60 | The HeuDiConv and ReproIn container images are named ``nipy-heudiconv`` and ``repronim-reproin``, respectively, in this collection. 61 | To use them, you can install the DataLad dataset and then use the ``datalad containers-run`` command to run. 62 | For a more detailed example of using images from this collection while fulfilling 63 | the `YODA Principles `_, please check out 64 | `A typical YODA workflow `_ in 65 | the documentation of this collection. 66 | 67 | **Note:** With the ``datalad containers-run`` command, the images in this collection work on macOS (OSX) 68 | as well for ``repronim/containers`` helpers automagically take care of running the Singularity containers via Docker. 69 | -------------------------------------------------------------------------------- /docs/quickstart.rst: -------------------------------------------------------------------------------- 1 | Quickstart 2 | ========== 3 | 4 | This tutorial is based on `Dianne Patterson's University of Arizona tutorials `_ 5 | 6 | This guide assumes you have already :ref:`installed heudiconv and dcm2niix ` and 7 | demonstrates how to use the heudiconv tool with a provided `heuristic.py` to convert DICOMS into the BIDS data structure. 8 | 9 | .. _prepare_dataset: 10 | 11 | Prepare Dataset 12 | *************** 13 | 14 | Download and unzip `sub-219_dicom.zip `_. 15 | 16 | We will be working from a directory called MRIS. Under the MRIS directory is the *dicom* subdirectory: Under the subject number *219* the session *itbs* is nested. Each dicom sequence folder is nested under the session:: 17 | 18 | dicom 19 | └── 219 20 | └── itbs 21 | ├── Bzero_verify_PA_17 22 | ├── DTI_30_DIRs_AP_15 23 | ├── Localizers_1 24 | ├── MoCoSeries_19 25 | ├── MoCoSeries_31 26 | ├── Post_TMS_restingstate_30 27 | ├── T1_mprage_1mm_13 28 | ├── field_mapping_20 29 | ├── field_mapping_21 30 | └── restingstate_18 31 | Nifti 32 | └── code 33 | └── heuristic1.py 34 | 35 | Basic Conversion 36 | **************** 37 | 38 | Next we will use heudiconv convert DICOMS into the BIDS data structure. 39 | The example dataset includes an example heuristic file, `heuristic1.py`. 40 | Typical use of heudiconv will require the creation and editing of your :doc:`heuristics file `, which we will cover 41 | in a :doc:`later tutorial `. 42 | 43 | .. note:: Heudiconv requires you to run the command from the parent 44 | directory of both the Dicom and Nifti directories, which is `MRIS` in 45 | our case. 46 | 47 | Run the following command:: 48 | 49 | heudiconv --files dicom/219/itbs/*/*.dcm -o Nifti -f Nifti/code/heuristic1.py -s 219 -ss itbs -c dcm2niix -b --minmeta --overwrite 50 | 51 | 52 | * We specify the dicom files to convert with `--files` 53 | * The heuristic file is provided with the `-f` option 54 | * We tell heudiconv to place our output in the Nifti dir with `-o` 55 | * `-b` indicates that we want to output in BIDS format 56 | * `--minmeta` guarantees that meta-information in the dcms does not get inserted into the JSON sidecar. This is good because the information is not needed but can overflow the JSON file causing some BIDS apps to crash. 57 | 58 | Output 59 | ****** 60 | 61 | The *Nifti* directory will contain a bids-compliant subject directory:: 62 | 63 | 64 | └── sub-219 65 | └── ses-itbs 66 | ├── anat 67 | ├── dwi 68 | ├── fmap 69 | └── func 70 | 71 | The following required BIDS text files are also created in the Nifti directory. Details for filling in these skeleton text files can be found under `tabular files `_ in the BIDS specification:: 72 | 73 | CHANGES 74 | README 75 | dataset_description.json 76 | participants.json 77 | participants.tsv 78 | task-rest_bold.json 79 | 80 | Validation 81 | ********** 82 | 83 | Ensure that everything is according to spec by using `bids validator `_ 84 | 85 | Click `Choose File` and then select the *Nifti* directory. There should be no errors (though there are a couple of warnings). 86 | 87 | .. Note:: Your files are not uploaded to the BIDS validator, so there are no privacy concerns! 88 | 89 | Next 90 | **** 91 | 92 | In the following sections, you will modify *heuristic.py* yourself so you can test different options and understand how to work with your own data. 93 | -------------------------------------------------------------------------------- /docs/reproin.rst: -------------------------------------------------------------------------------- 1 | ================ 2 | Reproin 3 | ================ 4 | 5 | This tutorial is based on `Dianne Patterson's University of Arizona tutorials `_ 6 | 7 | `Reproin `_ is a setup for 8 | automatic generation of sharable, version-controlled BIDS datasets from 9 | MR scanners. 10 | 11 | If you can control how your image sequences are named at the scanner, you can use the *reproin* naming convention. 12 | If you cannot control such naming, or already have collected data, you can provide your custom heuristic mapping into *reproin* and thus in effect use reproin heuristic. 13 | That will be a topic for another tutorial but meanwhile you can checkout `reproin/issues/18 `_ for a brief HOWTO. 14 | 15 | Get Example Dataset 16 | ------------------- 17 | 18 | This example uses a phantom dataset: `reproin_dicom.zip `_ generated by the University of Arizona on their Siemens Skyra 3T with Syngo MR VE11c software on 2018_02_08. 19 | 20 | The ``REPROIN`` directory is a simple reproin-compliant DICOM (.dcm) dataset without sessions. 21 | (Derived dwi images (ADC, FA etc.) that the scanner produced have been removed.:: 22 | 23 | [user@local ~/reproin_dicom/REPROIN]$ tree -I "*.dcm" 24 | 25 | REPROIN 26 | ├── data 27 | └── dicom 28 | └── 001 29 | └── Patterson_Coben\ -\ 1 30 | ├── Localizers_4 31 | ├── anatT1w_acqMPRAGE_6 32 | ├── dwi_dirAP_9 33 | ├── fmap_acq4mm_7 34 | ├── fmap_acq4mm_8 35 | ├── fmap_dirPA_15 36 | └── func_taskrest_16 37 | 38 | Convert and organize 39 | -------------------- 40 | 41 | From the ``REPROIN`` directory:: 42 | 43 | heudiconv -f reproin --bids -o data --files dicom/001 --minmeta 44 | 45 | * ``-f reproin`` specifies the converter file to use 46 | * ``-o data/`` specifies the output directory *data*. If the output directory does not exist, it will be created. 47 | * ``--files dicom/001`` identifies the path to the DICOM files. 48 | * ``--minmeta`` ensures that only the minimum necessary amount of data gets added to the JSON file when created. On the off chance that there is a LOT of meta-information in the DICOM header, the JSON file will not get swamped by it. Rumors are that fMRIPrep and MRIQC might be sensitive to excess of metadata and might crash crash, so minmeta provides a layer of protection against such corruption. 49 | 50 | 51 | Output Directory Structure 52 | -------------------------- 53 | 54 | Heudiconv's Reproin converter produces a hierarchy of directories with the BIDS dataset (here - `Cohen`) at the bottom:: 55 | 56 | data 57 | └── Patterson 58 | └── Coben 59 | ├── sourcedata 60 | │   └── sub-001 61 | │   ├── anat 62 | │   ├── dwi 63 | │   ├── fmap 64 | │   └── func 65 | └── sub-001 66 | ├── anat 67 | ├── dwi 68 | ├── fmap 69 | └── func 70 | 71 | The specific value for the hierarchy can be specified to HeuDiConv via `--locator PATH` option. 72 | If not, ReproIn heuristic bases it on the value of the DICOM "Study Description" field which is populated when user selects a specific *Exam* card located within some *Region* (see `ReproIn Walkthrough "Organization" `_). 73 | 74 | * The dataset is nested under two levels in the output directory: *Region* (Patterson) and *Exam* (Coben). *Tree* is reserved for other purposes at the UA research scanner. 75 | * Although the Program *Patient* is not visible in the output hierarchy, it is important. If you have separate sessions, then each session should have its own Program name. 76 | * **sourcedata** contains tarred gzipped (`.tgz`) sets of DICOM images corresponding to NIfTI images. 77 | * **sub-001/** contains a single subject data within this BIDS dataset. 78 | * The hidden directory is generated: *REPROIN/data/Patterson/Coben/.heudiconv* to contain derived mapping data, which could potentially be inspected or adjusted/used for re-conversion. 79 | 80 | 81 | 82 | Reproin Scanner File Names 83 | **************************** 84 | 85 | * For both BIDS and *reproin*, names are composed of an ordered series of key-value pairs, called `*entities* `_. 86 | Each key and its value are joined with a dash ``-`` (e.g., ``acq-MPRAGE``, ``dir-AP``). 87 | These key-value pairs are joined to other key-value pairs with underscores ``_``. 88 | The exception is the modality label, which is discussed more below. 89 | * *Reproin* scanner sequence names are simplified relative to the final BIDS output and generally conform to this scheme (but consult the `reproin heuristics file `_ for additional options): ``sequence type-modality label`` _ ``session-session name`` _ ``task-task name`` _ ``acquisition-acquisition detail`` _ ``run-run number`` _ ``direction-direction label``:: 90 | 91 | | func-bold_ses-pre_task-faces_acq-1mm_run-01_dir-AP 92 | 93 | * Each sequence name begins with the seqtype key. The seqtype key is the modality and corresponds to the name of the BIDS directory where the sequence belongs, e.g., ``anat``, ``dwi``, ``fmap`` or ``func``. 94 | * The seqtype key is optionally followed by a dash ``-`` and a modality label value (e.g., ``anat-scout`` or ``anat-T2W``). Often, the modality label is not needed because there is a predictable default for most seqtypes: 95 | * For **anat** the default modality is ``T1W``. Thus a sequence named ``anat`` will have the same output BIDS files as a sequence named ``anat-T1w``: *sub-001_T1w.nii.gz*. 96 | * For **fmap** the default modality is ``epi``. Thus ``fmap_dir-PA`` will have the same output as ``fmap-epi_dir-PA``: *sub-001_dir-PA_epi.nii.gz*. 97 | * For **func** the default modality is ``bold``. Thus, ``func-bold_task-rest`` will have the same output as ``func_task-rest``: *sub-001_task-rest_bold.nii.gz*. 98 | * *Reproin* gets the subject number from the DICOM metadata. 99 | * If you have multiple sessions, the session name does not need to be included in every sequence name in the program (i.e., Program= *Patient* level mentioned above). Instead, the session can be added to a single sequence name, usually the scout (localizer) sequence e.g. ``anat-scout_ses-pre``, and *reproin* will propagate the session information to the other sequence names in the *Program*. Interestingly, *reproin* does not add the localizer to your BIDS output. 100 | * When our scanner exports the DICOM sequences, all dashes are removed. But don't worry, *reproin* handles this just fine. 101 | * In the UA phantom reproin data, the subject was named ``01``. Horos reports the subject number as ``01`` but exports the DICOMS into a directory ``001``. If the data are copied to an external drive at the scanner, then the subject number is reported as ``001_001`` and the images are ``*.IMA`` instead of ``*.dcm``. *Reproin* does not care, it handles all of this gracefully. Your output tree (excluding *sourcedata* and *.heudiconv*) should look like this:: 102 | 103 | . 104 | |-- CHANGES 105 | |-- README 106 | |-- dataset_description.json 107 | |-- participants.tsv 108 | |-- sub-001 109 | | |-- anat 110 | | | |-- sub-001_acq-MPRAGE_T1w.json 111 | | | `-- sub-001_acq-MPRAGE_T1w.nii.gz 112 | | |-- dwi 113 | | | |-- sub-001_dir-AP_dwi.bval 114 | | | |-- sub-001_dir-AP_dwi.bvec 115 | | | |-- sub-001_dir-AP_dwi.json 116 | | | `-- sub-001_dir-AP_dwi.nii.gz 117 | | |-- fmap 118 | | | |-- sub-001_acq-4mm_magnitude1.json 119 | | | |-- sub-001_acq-4mm_magnitude1.nii.gz 120 | | | |-- sub-001_acq-4mm_magnitude2.json 121 | | | |-- sub-001_acq-4mm_magnitude2.nii.gz 122 | | | |-- sub-001_acq-4mm_phasediff.json 123 | | | |-- sub-001_acq-4mm_phasediff.nii.gz 124 | | | |-- sub-001_dir-PA_epi.json 125 | | | `-- sub-001_dir-PA_epi.nii.gz 126 | | |-- func 127 | | | |-- sub-001_task-rest_bold.json 128 | | | |-- sub-001_task-rest_bold.nii.gz 129 | | | `-- sub-001_task-rest_events.tsv 130 | | `-- sub-001_scans.tsv 131 | `-- task-rest_bold.json 132 | 133 | * Note that despite all the the different subject names (e.g., ``01``, ``001`` and ``001_001``), the subject is labeled ``sub-001``. 134 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | # should be "installed" from top directory since that one refers to .[all] 2 | sphinx-argparse 3 | sphinxcontrib-napoleon 4 | -r ../dev-requirements.txt 5 | -------------------------------------------------------------------------------- /docs/tutorials.rst: -------------------------------------------------------------------------------- 1 | .. _Tutorials: 2 | 3 | ========= 4 | Tutorials 5 | ========= 6 | 7 | .. toctree:: 8 | 9 | quickstart 10 | custom-heuristic 11 | reproin 12 | 13 | 14 | External Tutorials 15 | ****************** 16 | 17 | Luckily(?), we live in an era of plentiful information. Below are some links to 18 | other users' tutorials covering their experience with ``heudiconv``. 19 | 20 | - `YouTube tutorial `_ by `James Kent `_. 21 | 22 | - `Walkthrough `_ by the `Stanford Center for Reproducible Neuroscience `_. 23 | 24 | - `U of A Neuroimaging Core `_ by `Dianne Patterson `_. 25 | 26 | - `Sample Conversion: Coastal Coding 2019 `_. 27 | 28 | - `A joined DataLad and HeuDiConv tutorial for reproducible fMRI studies `_. 29 | 30 | - `The ReproIn conversion workflow overview `_. 31 | 32 | - `Slides `_ and 33 | `recording `_ 34 | of a ReproNim Webinar on ``heudiconv``. 35 | 36 | .. caution:: 37 | Some of these tutorials may not be up to date with 38 | the latest releases of ``heudiconv``. 39 | -------------------------------------------------------------------------------- /figs/environment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nipy/heudiconv/2a15cdfa913d2288f08198db2196047c90711e1b/figs/environment.png -------------------------------------------------------------------------------- /figs/workflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nipy/heudiconv/2a15cdfa913d2288f08198db2196047c90711e1b/figs/workflow.png -------------------------------------------------------------------------------- /heudiconv/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from ._version import __version__ 4 | from .info import __packagename__ 5 | 6 | __all__ = ["__packagename__", "__version__"] 7 | 8 | lgr = logging.getLogger(__name__) 9 | lgr.debug("Starting the abomination") # just to "run-test" logging 10 | -------------------------------------------------------------------------------- /heudiconv/cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nipy/heudiconv/2a15cdfa913d2288f08198db2196047c90711e1b/heudiconv/cli/__init__.py -------------------------------------------------------------------------------- /heudiconv/cli/monitor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import annotations 3 | 4 | import argparse 5 | from collections import OrderedDict 6 | import json 7 | import logging 8 | import os 9 | import os.path as op 10 | from pathlib import Path 11 | import re 12 | import shlex 13 | import subprocess 14 | import time 15 | from typing import Any, Optional 16 | 17 | import inotify.adapters 18 | from inotify.constants import IN_CREATE, IN_ISDIR, IN_MODIFY 19 | from tinydb import TinyDB 20 | 21 | _DEFAULT_LOG_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" 22 | _LOGGER = logging.getLogger(__name__) 23 | 24 | MASK = IN_MODIFY | IN_CREATE 25 | MASK_NEWDIR = IN_CREATE | IN_ISDIR 26 | WAIT_TIME = 86400 # in seconds 27 | 28 | 29 | # def _configure_logging(): 30 | _LOGGER.setLevel(logging.DEBUG) 31 | ch = logging.StreamHandler() 32 | formatter = logging.Formatter(_DEFAULT_LOG_FORMAT) 33 | ch.setFormatter(formatter) 34 | _LOGGER.addHandler(ch) 35 | 36 | 37 | def run_heudiconv(args: list[str]) -> tuple[str, dict[str, Any]]: 38 | info_dict: dict[str, Any] = dict() 39 | proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 40 | cmd = " ".join(map(shlex.quote, args)) 41 | return_code = proc.wait() 42 | if return_code == 0: 43 | _LOGGER.info("Done running %s", cmd) 44 | info_dict["success"] = 1 45 | else: 46 | _LOGGER.error("%s failed", cmd) 47 | info_dict["success"] = 0 48 | # get info on what we run 49 | stdout = proc.communicate()[0].decode("utf-8") 50 | match = re.match("INFO: PROCESSING STARTS: (.*)", stdout) 51 | info_dict_ = json.loads(match.group(1) if match else "{}") 52 | info_dict.update(info_dict_) 53 | return stdout, info_dict 54 | 55 | 56 | def process( 57 | paths2process: dict[str, float], 58 | db: Optional[TinyDB], 59 | wait: int | float = WAIT_TIME, 60 | logdir: str = "log", 61 | ) -> None: 62 | # if paths2process and 63 | # time.time() - os.path.getmtime(paths2process[0]) > WAIT_TIME: 64 | processed: list[str] = [] 65 | for path, mod_time in paths2process.items(): 66 | if time.time() - mod_time > wait: 67 | # process_me = paths2process.popleft().decode('utf-8') 68 | process_me = path 69 | process_dict: dict[str, Any] = { 70 | "input_path": process_me, 71 | "accession_number": op.basename(process_me), 72 | } 73 | print("Time to process {0}".format(process_me)) 74 | stdout, run_dict = run_heudiconv(["ls", "-l", process_me]) 75 | process_dict.update(run_dict) 76 | if db is not None: 77 | db.insert(process_dict) 78 | # save log 79 | log = Path(logdir, process_dict["accession_number"] + ".log") 80 | log.write_text(stdout) 81 | # if we processed it, or it failed, 82 | # we need to remove it to avoid running it again 83 | processed.append(path) 84 | for processed_path in processed: 85 | del paths2process[processed_path] 86 | 87 | 88 | def monitor( 89 | topdir: str = "/tmp/new_dir", 90 | check_ptrn: str = "/20../../..", 91 | db: Optional[TinyDB] = None, 92 | wait: int | float = WAIT_TIME, 93 | logdir: str = "log", 94 | ) -> None: 95 | # make logdir if not existent 96 | try: 97 | os.makedirs(logdir) 98 | except OSError: 99 | pass 100 | # paths2process = deque() 101 | paths2process: dict[str, float] = OrderedDict() 102 | # watch only today's folder 103 | path_re = re.compile("(%s%s)/?$" % (topdir, check_ptrn)) 104 | i = inotify.adapters.InotifyTree(topdir.encode()) # , mask=MASK) 105 | for event in i.event_gen(): 106 | if event is not None: 107 | (header, type_names, watch_path, filename) = event 108 | _LOGGER.info( 109 | "WD=(%d) MASK=(%d) COOKIE=(%d) LEN=(%d) MASK->NAMES=%s" 110 | " WATCH-PATH=[%s] FILENAME=[%s]", 111 | header.wd, 112 | header.mask, 113 | header.cookie, 114 | header.len, 115 | type_names, 116 | watch_path.decode("utf-8"), 117 | filename.decode("utf-8"), 118 | ) 119 | if header.mask == MASK_NEWDIR and path_re.match(watch_path.decode("utf-8")): 120 | # we got our directory, now let's do something on it 121 | newpath2process = op.join(watch_path, filename).decode("utf-8") 122 | # paths2process.append(newpath2process) 123 | # update time 124 | paths2process[newpath2process] = time.time() 125 | print(newpath2process, time.time()) 126 | # check if we need to update the time 127 | for path in paths2process.keys(): 128 | if path in watch_path.decode("utf-8"): 129 | paths2process[path] = time.time() 130 | print("Updating {0}: {1}".format(path, paths2process[path])) 131 | 132 | # check if there's anything to process 133 | process(paths2process, db, wait=wait, logdir=logdir) 134 | 135 | 136 | def parse_args() -> argparse.Namespace: 137 | parser = argparse.ArgumentParser( 138 | prog="monitor.py", 139 | description=( 140 | "Small monitoring script to detect new directories and " "process them" 141 | ), 142 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, 143 | ) 144 | parser.add_argument("path", help="Which directory to monitor") 145 | parser.add_argument( 146 | "--check_ptrn", 147 | "-p", 148 | help="regexp pattern for which subdirectories to check", 149 | default="/20../../..", 150 | ) 151 | parser.add_argument( 152 | "--database", "-d", help="database location", default="database.json" 153 | ) 154 | parser.add_argument( 155 | "--wait_time", 156 | "-w", 157 | help="After how long should we start processing datasets? (in seconds)", 158 | default=86400, 159 | type=float, 160 | ) 161 | parser.add_argument( 162 | "--logdir", "-l", help="Where should we save the logs?", default="log" 163 | ) 164 | return parser.parse_args() 165 | 166 | 167 | def main() -> None: 168 | parsed = parse_args() 169 | print("Got {0}".format(parsed)) 170 | # open database 171 | db = TinyDB(parsed.database, default_table="heudiconv") 172 | monitor( 173 | parsed.path, parsed.check_ptrn, db, wait=parsed.wait_time, logdir=parsed.logdir 174 | ) 175 | 176 | 177 | if __name__ == "__main__": 178 | main() 179 | -------------------------------------------------------------------------------- /heudiconv/cli/run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import annotations 3 | 4 | from argparse import ArgumentParser 5 | import logging 6 | import os 7 | import sys 8 | from typing import Optional 9 | 10 | from .. import __version__ 11 | from ..main import workflow 12 | 13 | lgr = logging.getLogger(__name__) 14 | 15 | 16 | def main(argv: Optional[list[str]] = None) -> None: 17 | logging.basicConfig( 18 | format="%(levelname)s: %(message)s", 19 | level=getattr(logging, os.environ.get("HEUDICONV_LOG_LEVEL", "INFO")), 20 | ) 21 | parser = get_parser() 22 | args = parser.parse_args(argv) 23 | # exit if nothing to be done 24 | if not args.files and not args.dicom_dir_template and not args.command: 25 | lgr.warning("Nothing to be done - displaying usage help") 26 | parser.print_help() 27 | sys.exit(1) 28 | 29 | kwargs = vars(args) 30 | workflow(**kwargs) 31 | 32 | 33 | def get_parser() -> ArgumentParser: 34 | docstr = """Example: 35 | heudiconv -d 'rawdata/{subject}' -o . -f heuristic.py -s s1 s2 s3""" 36 | parser = ArgumentParser(description=docstr) 37 | parser.add_argument("--version", action="version", version=__version__) 38 | group = parser.add_mutually_exclusive_group() 39 | group.add_argument( 40 | "-d", 41 | "--dicom_dir_template", 42 | dest="dicom_dir_template", 43 | help="Location of dicomdir that can be indexed with subject id " 44 | "{subject} and session {session}. Tarballs (can be compressed) " 45 | "are supported in addition to directory. All matching tarballs " 46 | "for a subject are extracted and their content processed in a " 47 | "single pass. If multiple tarballs are found, each is assumed to " 48 | "be a separate session and the --ses argument is ignored. Note " 49 | "that you might need to surround the value with quotes to avoid " 50 | "{...} being considered by shell", 51 | ) 52 | group.add_argument( 53 | "--files", 54 | nargs="*", 55 | help="Files (tarballs, dicoms) or directories containing files to " 56 | "process. Cannot be provided if using --dicom_dir_template.", 57 | ) 58 | parser.add_argument( 59 | "-s", 60 | "--subjects", 61 | dest="subjs", 62 | type=str, 63 | nargs="*", 64 | help="List of subjects - required for dicom template. If not " 65 | 'provided, DICOMS would first be "sorted" and subject IDs ' 66 | "deduced by the heuristic.", 67 | ) 68 | parser.add_argument( 69 | "-c", 70 | "--converter", 71 | choices=("dcm2niix", "none"), 72 | default="dcm2niix", 73 | help='Tool to use for DICOM conversion. Setting to "none" disables ' 74 | "the actual conversion step -- useful for testing heuristics.", 75 | ) 76 | parser.add_argument( 77 | "-o", 78 | "--outdir", 79 | default=os.getcwd(), 80 | help="Output directory for conversion setup (for further " 81 | "customization and future reference. This directory will refer " 82 | "to non-anonymized subject IDs.", 83 | ) 84 | parser.add_argument( 85 | "-l", 86 | "--locator", 87 | default=None, 88 | help="Study path under outdir. If provided, it overloads the value " 89 | "provided by the heuristic. If --datalad is enabled, every " 90 | "directory within locator becomes a super-dataset thus " 91 | 'establishing a hierarchy. Setting to "unknown" will skip that ' 92 | "dataset.", 93 | ) 94 | parser.add_argument( 95 | "-a", 96 | "--conv-outdir", 97 | default=None, 98 | help="Output directory for converted files. By default this is " 99 | "identical to --outdir. This option is most useful in " 100 | "combination with --anon-cmd.", 101 | ) 102 | parser.add_argument( 103 | "--anon-cmd", 104 | default=None, 105 | help="Command to run to convert subject IDs used for DICOMs to " 106 | "anonymized IDs. Such command must take a single argument and " 107 | "return a single anonymized ID. Also see --conv-outdir.", 108 | ) 109 | parser.add_argument( 110 | "-f", 111 | "--heuristic", 112 | dest="heuristic", 113 | help="Name of a known heuristic or path to the Python script " 114 | "containing heuristic.", 115 | ) 116 | parser.add_argument( 117 | "-p", 118 | "--with-prov", 119 | action="store_true", 120 | help="Store additional provenance information. Requires python-rdflib.", 121 | ) 122 | parser.add_argument( 123 | "-ss", 124 | "--ses", 125 | dest="session", 126 | default=None, 127 | help="Session for longitudinal study_sessions. Default is None.", 128 | ) 129 | parser.add_argument( 130 | "-b", 131 | "--bids", 132 | nargs="*", 133 | metavar=("BIDSOPTION1", "BIDSOPTION2"), 134 | choices=["notop"], 135 | dest="bids_options", 136 | help="Flag for output into BIDS structure. Can also take BIDS-" 137 | "specific options, e.g., --bids notop. The only currently " 138 | 'supported options is "notop", which skips creation of ' 139 | "top-level BIDS files. This is useful when running in batch " 140 | "mode to prevent possible race conditions.", 141 | ) 142 | parser.add_argument( 143 | "--overwrite", 144 | action="store_true", 145 | default=False, 146 | help="Overwrite existing converted files.", 147 | ) 148 | parser.add_argument( 149 | "--datalad", 150 | action="store_true", 151 | help="Store the entire collection as DataLad dataset(s). Small files " 152 | "will be committed directly to git, while large to annex. New " 153 | 'version (6) of annex repositories will be used in a "thin" ' 154 | "mode so it would look to mortals as just any other regular " 155 | "directory (i.e. no symlinks to under .git/annex). For now just " 156 | "for BIDS mode.", 157 | ) 158 | parser.add_argument( 159 | "--dbg", 160 | action="store_true", 161 | dest="debug", 162 | help="Do not catch exceptions and show exception traceback.", 163 | ) 164 | parser.add_argument( 165 | "--command", 166 | choices=( 167 | "heuristics", 168 | "heuristic-info", 169 | "ls", 170 | "populate-templates", 171 | "sanitize-jsons", 172 | "treat-jsons", 173 | "populate-intended-for", 174 | ), 175 | help="Custom action to be performed on provided files instead of " 176 | "regular operation.", 177 | ) 178 | parser.add_argument( 179 | "-g", 180 | "--grouping", 181 | default="studyUID", 182 | choices=("studyUID", "accession_number", "all", "custom"), 183 | help="How to group dicoms (default: by studyUID).", 184 | ) 185 | parser.add_argument( 186 | "--minmeta", 187 | action="store_true", 188 | help="Exclude dcmstack meta information in sidecar jsons.", 189 | ) 190 | parser.add_argument( 191 | "--random-seed", type=int, default=None, help="Random seed to initialize RNG." 192 | ) 193 | parser.add_argument( 194 | "--dcmconfig", 195 | default=None, 196 | help="JSON file for additional dcm2niix configuration.", 197 | ) 198 | submission = parser.add_argument_group("Conversion submission options") 199 | submission.add_argument( 200 | "-q", 201 | "--queue", 202 | choices=("SLURM", None), 203 | default=None, 204 | help="Batch system to submit jobs in parallel.", 205 | ) 206 | submission.add_argument( 207 | "--queue-args", 208 | dest="queue_args", 209 | default=None, 210 | help="Additional queue arguments passed as a single string of " 211 | "space-separated Argument=Value pairs.", 212 | ) 213 | return parser 214 | 215 | 216 | if __name__ == "__main__": 217 | main() 218 | -------------------------------------------------------------------------------- /heudiconv/due.py: -------------------------------------------------------------------------------- 1 | # emacs: at the end of the file 2 | # ex: set sts=4 ts=4 sw=4 et: 3 | # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### # 4 | """ 5 | 6 | Stub file for a guaranteed safe import of duecredit constructs: if duecredit 7 | is not available. 8 | 9 | To use it, place it into your project codebase to be imported, e.g. copy as 10 | 11 | cp stub.py /path/tomodule/module/due.py 12 | 13 | Note that it might be better to avoid naming it duecredit.py to avoid shadowing 14 | installed duecredit. 15 | 16 | Then use in your code as 17 | 18 | from .due import due, Doi, BibTeX, Text 19 | 20 | See https://github.com/duecredit/duecredit/blob/master/README.md for examples. 21 | 22 | Origin: Originally a part of the duecredit 23 | Copyright: 2015-2019 DueCredit developers 24 | License: BSD-2 25 | """ 26 | 27 | __version__ = "0.0.8" 28 | 29 | 30 | class InactiveDueCreditCollector(object): 31 | """Just a stub at the Collector which would not do anything""" 32 | 33 | def _donothing(self, *args, **kwargs): 34 | """Perform no good and no bad""" 35 | pass 36 | 37 | def dcite(self, *_args, **_kwargs): 38 | """If I could cite I would""" 39 | 40 | def nondecorating_decorator(func): 41 | return func 42 | 43 | return nondecorating_decorator 44 | 45 | active = False 46 | activate = add = cite = dump = load = _donothing 47 | 48 | def __repr__(self): 49 | return self.__class__.__name__ + "()" 50 | 51 | 52 | def _donothing_func(*args, **kwargs): 53 | """Perform no good and no bad""" 54 | pass 55 | 56 | 57 | try: 58 | from duecredit import BibTeX, Doi, Text, Url, due 59 | 60 | if "due" in locals() and not hasattr(due, "cite"): 61 | raise RuntimeError("Imported due lacks .cite. DueCredit is now disabled") 62 | except Exception as e: 63 | if not isinstance(e, ImportError): 64 | import logging 65 | 66 | logging.getLogger("duecredit").error( 67 | "Failed to import duecredit due to %s" % str(e) 68 | ) 69 | # Initiate due stub 70 | due = InactiveDueCreditCollector() 71 | BibTeX = Doi = Url = Text = _donothing_func 72 | 73 | # Emacs mode definitions 74 | # Local Variables: 75 | # mode: python 76 | # py-indent-offset: 4 77 | # tab-width: 4 78 | # indent-tabs-mode: nil 79 | # End: 80 | -------------------------------------------------------------------------------- /heudiconv/external/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nipy/heudiconv/2a15cdfa913d2288f08198db2196047c90711e1b/heudiconv/external/__init__.py -------------------------------------------------------------------------------- /heudiconv/external/dlad.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from glob import glob 4 | import inspect 5 | import logging 6 | import os 7 | import os.path as op 8 | from typing import TYPE_CHECKING, Optional 9 | 10 | from ..info import MIN_DATALAD_VERSION as MIN_VERSION 11 | from ..utils import SeqInfo, create_file_if_missing 12 | 13 | lgr = logging.getLogger(__name__) 14 | 15 | if TYPE_CHECKING: 16 | from datalad.api import Dataset 17 | 18 | 19 | def prepare_datalad( 20 | studydir: str, 21 | outdir: str, 22 | sid: Optional[str], 23 | session: str | int | None, 24 | seqinfo: Optional[dict[SeqInfo, list[str]]], 25 | dicoms: Optional[list[str]], 26 | bids: Optional[str], 27 | ) -> str: 28 | """Prepare data for datalad""" 29 | from datalad.api import Dataset 30 | 31 | datalad_msg_suf = " %s" % sid 32 | if session: 33 | datalad_msg_suf += ", session %s" % session 34 | if seqinfo: 35 | datalad_msg_suf += ", %d sequences" % len(seqinfo) 36 | datalad_msg_suf += ", %d dicoms" % len(sum(seqinfo.values(), [])) 37 | else: 38 | assert dicoms is not None 39 | datalad_msg_suf += ", %d dicoms" % len(dicoms) 40 | ds = Dataset(studydir) 41 | if not op.exists(outdir) or not ds.is_installed(): 42 | add_to_datalad( 43 | outdir, studydir, msg="Preparing for %s" % datalad_msg_suf, bids=bids 44 | ) 45 | return datalad_msg_suf 46 | 47 | 48 | def add_to_datalad( 49 | topdir: str, studydir: str, msg: Optional[str], bids: Optional[str] # noqa: U100 50 | ) -> None: 51 | """Do all necessary preparations (if were not done before) and save""" 52 | import datalad.api as dl 53 | from datalad.api import Dataset 54 | from datalad.support.annexrepo import AnnexRepo 55 | from datalad.support.external_versions import external_versions 56 | 57 | assert external_versions["datalad"] >= MIN_VERSION, "Need datalad >= {}".format( 58 | MIN_VERSION 59 | ) # add to reqs 60 | 61 | studyrelpath = op.relpath(studydir, topdir) 62 | assert not studyrelpath.startswith(op.pardir) # so we are under 63 | # now we need to test and initiate a DataLad dataset all along the path 64 | curdir_ = topdir 65 | superds = None 66 | subdirs = [""] + [d for d in studyrelpath.split(op.sep) if d != os.curdir] 67 | for isubdir, subdir in enumerate(subdirs): 68 | curdir_ = op.join(curdir_, subdir) 69 | ds = Dataset(curdir_) 70 | if not ds.is_installed(): 71 | lgr.info("Initiating %s", ds) 72 | # would require annex > 20161018 for correct operation on annex v6 73 | # need to add .gitattributes first anyways 74 | ds_ = dl.create( 75 | curdir_, 76 | dataset=superds, 77 | force=True, 78 | # initiate annex only at the bottom repository 79 | annex=isubdir == (len(subdirs) - 1), 80 | fake_dates=True, 81 | # shared_access='all', 82 | ) 83 | assert ds == ds_ 84 | assert ds.is_installed() 85 | superds = ds 86 | 87 | # TODO: we need a helper (in DataLad ideally) to ease adding such 88 | # specifications 89 | gitattributes_path = op.join(studydir, ".gitattributes") 90 | # We will just make sure that all our desired rules are present in it 91 | desired_attrs = """\ 92 | * annex.largefiles=(largerthan=100kb) 93 | *.json annex.largefiles=nothing 94 | *.txt annex.largefiles=nothing 95 | *.tsv annex.largefiles=nothing 96 | *.nii.gz annex.largefiles=anything 97 | *.tgz annex.largefiles=anything 98 | *_scans.tsv annex.largefiles=anything 99 | """ 100 | if op.exists(gitattributes_path): 101 | with open(gitattributes_path, "rb") as f: 102 | known_attrs = [line.decode("utf-8").rstrip() for line in f.readlines()] 103 | else: 104 | known_attrs = [] 105 | for attr in desired_attrs.split("\n"): 106 | if attr not in known_attrs: 107 | known_attrs.append(attr) 108 | with open(gitattributes_path, "wb") as f: 109 | f.write("\n".join(known_attrs).encode("utf-8")) 110 | 111 | # ds might have memories of having ds.repo GitRepo 112 | superds = Dataset(topdir) 113 | assert op.realpath(ds.path) == op.realpath(studydir) 114 | assert isinstance(ds.repo, AnnexRepo) 115 | # Add doesn't have all the options of save such as msg and supers 116 | ds.save(path=[".gitattributes"], message="Custom .gitattributes", to_git=True) 117 | dsh = dsh_path = None 118 | if op.lexists(op.join(ds.path, ".heudiconv")): 119 | dsh_path = op.join(ds.path, ".heudiconv") 120 | dsh = Dataset(dsh_path) 121 | if not dsh.is_installed(): 122 | # Previously we did not have it as a submodule, and since no 123 | # automagic migration is implemented, we just need to check first 124 | # if any path under .heudiconv is already under git control 125 | if any(x.startswith(".heudiconv/") for x in ds.repo.get_files()): 126 | lgr.warning( 127 | "%s has .heudiconv not as a submodule from previous" 128 | " versions of heudiconv. No automagic migration is " 129 | "yet provided", 130 | ds, 131 | ) 132 | else: 133 | dsh = ds.create( 134 | path=".heudiconv", 135 | force=True, 136 | # shared_access='all' 137 | ) 138 | # Since .heudiconv could contain sensitive information 139 | # we place all files under annex and then add 140 | if create_file_if_missing( 141 | op.join(dsh_path, ".gitattributes"), """* annex.largefiles=anything""" 142 | ): 143 | ds.save( 144 | ".heudiconv/.gitattributes", 145 | to_git=True, 146 | message="Added gitattributes to place all .heudiconv content" 147 | " under annex", 148 | ) 149 | save_res = ds.save( 150 | ".", 151 | recursive=True 152 | # not in effect! ? 153 | # annex_add_opts=['--include-dotfiles'] 154 | ) 155 | annexed_files = [sr["path"] for sr in save_res if sr.get("key", None)] 156 | 157 | # Provide metadata for sensitive information 158 | sensitive_patterns = [ 159 | "sourcedata/**", 160 | "*_scans.tsv", # top level 161 | "*/*_scans.tsv", # within subj 162 | "*/*/*_scans.tsv", # within sess/subj 163 | "*/anat/*", # within subj 164 | "*/*/anat/*", # within ses/subj 165 | ] 166 | for sp in sensitive_patterns: 167 | mark_sensitive(ds, sp, annexed_files) 168 | if dsh_path: 169 | mark_sensitive(ds, ".heudiconv") # entire .heudiconv! 170 | superds.save(path=ds.path, message=msg, recursive=True) 171 | 172 | assert not ds.repo.dirty 173 | # TODO: they are still appearing as native annex symlinked beasts 174 | """ 175 | TODOs: 176 | it needs 177 | - unlock (thin will be in effect) 178 | - save/commit (does modechange 120000 => 100644 179 | - could potentially somehow automate that all: 180 | http://git-annex.branchable.com/tips/automatically_adding_metadata/ 181 | - possibly even make separate sub-datasets for originaldata, derivatives ? 182 | """ 183 | 184 | 185 | def mark_sensitive(ds: Dataset, path_glob: str, files: list[str] | None = None) -> None: 186 | """ 187 | 188 | Parameters 189 | ---------- 190 | ds : Dataset to operate on 191 | path_glob : str 192 | glob of the paths within dataset to work on 193 | files : list[str] 194 | subset of files to mark 195 | 196 | Returns 197 | ------- 198 | None 199 | """ 200 | paths = glob(op.join(ds.path, path_glob)) 201 | if files: 202 | paths = [p for p in paths if p in files] 203 | if not paths: 204 | return 205 | lgr.debug("Marking %d files with distribution-restrictions field", len(paths)) 206 | # set_metadata can be a bloody generator 207 | res = ds.repo.set_metadata( 208 | paths, add=dict([("distribution-restrictions", "sensitive")]), recursive=True 209 | ) 210 | if inspect.isgenerator(res): 211 | res = list(res) 212 | -------------------------------------------------------------------------------- /heudiconv/external/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nipy/heudiconv/2a15cdfa913d2288f08198db2196047c90711e1b/heudiconv/external/tests/__init__.py -------------------------------------------------------------------------------- /heudiconv/external/tests/test_dlad.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from pathlib import Path 4 | 5 | import pytest 6 | 7 | from ..dlad import mark_sensitive 8 | from ...utils import create_tree 9 | 10 | dl = pytest.importorskip("datalad.api") 11 | 12 | 13 | def test_mark_sensitive(tmp_path: Path) -> None: 14 | ds = dl.Dataset(tmp_path).create(force=True) 15 | create_tree( 16 | str(tmp_path), 17 | { 18 | "f1": "d1", 19 | "f2": "d2", 20 | "g1": "d3", 21 | "g2": "d1", 22 | }, 23 | ) 24 | ds.save(".") 25 | mark_sensitive(ds, "f*") 26 | all_meta = dict(ds.repo.get_metadata(".")) 27 | target_rec = {"distribution-restrictions": ["sensitive"]} 28 | # g2 since the same content 29 | assert not all_meta.pop("g1", None) # nothing or empty record 30 | assert all_meta == {"f1": target_rec, "f2": target_rec, "g2": target_rec} 31 | 32 | 33 | def test_mark_sensitive_subset(tmp_path: Path) -> None: 34 | ds = dl.Dataset(tmp_path).create(force=True) 35 | create_tree( 36 | str(tmp_path), 37 | { 38 | "f1": "d1", 39 | "f2": "d2", 40 | "g1": "d3", 41 | "g2": "d1", 42 | }, 43 | ) 44 | ds.save(".") 45 | mark_sensitive(ds, "f*", [str(tmp_path / "f1")]) 46 | all_meta = dict(ds.repo.get_metadata(".")) 47 | target_rec = {"distribution-restrictions": ["sensitive"]} 48 | # g2 since the same content 49 | assert not all_meta.pop("g1", None) # nothing or empty record 50 | assert not all_meta.pop("f2", None) # nothing or empty record 51 | assert all_meta == {"f1": target_rec, "g2": target_rec} 52 | -------------------------------------------------------------------------------- /heudiconv/heuristics/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nipy/heudiconv/2a15cdfa913d2288f08198db2196047c90711e1b/heudiconv/heuristics/__init__.py -------------------------------------------------------------------------------- /heudiconv/heuristics/banda-bids.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import Optional 4 | 5 | from heudiconv.utils import SeqInfo 6 | 7 | 8 | def create_key( 9 | template: Optional[str], 10 | outtype: tuple[str, ...] = ("nii.gz", "dicom"), 11 | annotation_classes: None = None, 12 | ) -> tuple[str, tuple[str, ...], None]: 13 | if template is None or not template: 14 | raise ValueError("Template must be a valid format string") 15 | return (template, outtype, annotation_classes) 16 | 17 | 18 | def infotodict(seqinfo: list[SeqInfo]) -> dict[tuple[str, tuple[str, ...], None], list]: 19 | """Heuristic evaluator for determining which runs belong where 20 | 21 | allowed template fields - follow python string module: 22 | 23 | item: index within category 24 | subject: participant id 25 | seqitem: run number during scanning 26 | subindex: sub index within group 27 | """ 28 | t1 = create_key("sub-{subject}/anat/sub-{subject}_T1w") 29 | t2 = create_key("sub-{subject}/anat/sub-{subject}_T2w") 30 | rest = create_key("sub-{subject}/func/sub-{subject}_task-rest_run-{item:02d}_bold") 31 | rest_sbref = create_key( 32 | "sub-{subject}/func/sub-{subject}_task-rest_run-{item:02d}_sbref" 33 | ) 34 | face = create_key("sub-{subject}/func/sub-{subject}_task-face_run-{item:02d}_bold") 35 | face_sbref = create_key( 36 | "sub-{subject}/func/sub-{subject}_task-face_run-{item:02d}_sbref" 37 | ) 38 | gamble = create_key( 39 | "sub-{subject}/func/sub-{subject}_task-gambling_run-{item:02d}_bold" 40 | ) 41 | gamble_sbref = create_key( 42 | "sub-{subject}/func/sub-{subject}_task-gambling_run-{item:02d}_sbref" 43 | ) 44 | conflict = create_key( 45 | "sub-{subject}/func/sub-{subject}_task-conflict_run-{item:02d}_bold" 46 | ) 47 | conflict_sbref = create_key( 48 | "sub-{subject}/func/sub-{subject}_task-conflict_run-{item:02d}_sbref" 49 | ) 50 | dwi = create_key("sub-{subject}/dwi/sub-{subject}_run-{item:02d}_dwi") 51 | dwi_sbref = create_key("sub-{subject}/dwi/sub-{subject}_run-{item:02d}_sbref") 52 | fmap = create_key("sub-{subject}/fmap/sub-{subject}_dir-{dir}_run-{item:02d}_epi") 53 | 54 | info: dict[tuple[str, tuple[str, ...], None], list] = { 55 | t1: [], 56 | t2: [], 57 | rest: [], 58 | face: [], 59 | gamble: [], 60 | conflict: [], 61 | dwi: [], 62 | rest_sbref: [], 63 | face_sbref: [], 64 | gamble_sbref: [], 65 | conflict_sbref: [], 66 | dwi_sbref: [], 67 | fmap: [], 68 | } 69 | 70 | for s in seqinfo: 71 | # T1 and T2 scans 72 | if (s.dim3 == 208) and (s.dim4 == 1) and ("T1w" in s.protocol_name): 73 | info[t1] = [s.series_id] 74 | if (s.dim3 == 208) and ("T2w" in s.protocol_name): 75 | info[t2] = [s.series_id] 76 | # diffusion scans 77 | if "dMRI_dir9" in s.protocol_name: 78 | key = None 79 | if s.dim4 >= 99: 80 | key = dwi 81 | elif (s.dim4 == 1) and ("SBRef" in s.series_description): 82 | key = dwi_sbref 83 | if key: 84 | info[key].append({"item": s.series_id}) 85 | # functional scans 86 | if "fMRI" in s.protocol_name: 87 | tasktype = s.protocol_name.split("fMRI")[1].split("_")[1] 88 | key = None 89 | if s.dim4 in [420, 215, 338, 280]: 90 | if "rest" in tasktype: 91 | key = rest 92 | if "face" in tasktype: 93 | key = face 94 | if "conflict" in tasktype: 95 | key = conflict 96 | if "gambling" in tasktype: 97 | key = gamble 98 | if (s.dim4 == 1) and ("SBRef" in s.series_description): 99 | if "rest" in tasktype: 100 | key = rest_sbref 101 | if "face" in tasktype: 102 | key = face_sbref 103 | if "conflict" in tasktype: 104 | key = conflict_sbref 105 | if "gambling" in tasktype: 106 | key = gamble_sbref 107 | if key: 108 | info[key].append({"item": s.series_id}) 109 | if (s.dim4 == 3) and ("SpinEchoFieldMap" in s.protocol_name): 110 | dirtype = s.protocol_name.split("_")[-1] 111 | info[fmap].append({"item": s.series_id, "dir": dirtype}) 112 | 113 | # You can even put checks in place for your protocol 114 | msg = [] 115 | if len(info[t1]) != 1: 116 | msg.append("Missing correct number of t1 runs") 117 | if len(info[t2]) != 1: 118 | msg.append("Missing correct number of t2 runs") 119 | if len(info[dwi]) != 4: 120 | msg.append("Missing correct number of dwi runs") 121 | if len(info[rest]) != 4: 122 | msg.append("Missing correct number of resting runs") 123 | if len(info[face]) != 2: 124 | msg.append("Missing correct number of faceMatching runs") 125 | if len(info[conflict]) != 4: 126 | msg.append("Missing correct number of conflict runs") 127 | if len(info[gamble]) != 2: 128 | msg.append("Missing correct number of gamble runs") 129 | if msg: 130 | raise ValueError("\n".join(msg)) 131 | return info 132 | -------------------------------------------------------------------------------- /heudiconv/heuristics/bids_ME.py: -------------------------------------------------------------------------------- 1 | """Heuristic demonstrating conversion of the Multi-Echo sequences. 2 | 3 | It only cares about converting sequences which have _ME_ in their 4 | series_description and outputs to BIDS. 5 | """ 6 | 7 | from __future__ import annotations 8 | 9 | from typing import Optional 10 | 11 | from heudiconv.utils import SeqInfo 12 | 13 | 14 | def create_key( 15 | template: Optional[str], 16 | outtype: tuple[str, ...] = ("nii.gz",), 17 | annotation_classes: None = None, 18 | ) -> tuple[str, tuple[str, ...], None]: 19 | if template is None or not template: 20 | raise ValueError("Template must be a valid format string") 21 | return (template, outtype, annotation_classes) 22 | 23 | 24 | def infotodict( 25 | seqinfo: list[SeqInfo], 26 | ) -> dict[tuple[str, tuple[str, ...], None], list[str]]: 27 | """Heuristic evaluator for determining which runs belong where 28 | 29 | allowed template fields - follow python string module: 30 | 31 | item: index within category 32 | subject: participant id 33 | seqitem: run number during scanning 34 | subindex: sub index within group 35 | """ 36 | bold = create_key("sub-{subject}/func/sub-{subject}_task-test_run-{item}_bold") 37 | megre_mag = create_key("sub-{subject}/anat/sub-{subject}_part-mag_MEGRE") 38 | megre_phase = create_key("sub-{subject}/anat/sub-{subject}_part-phase_MEGRE") 39 | 40 | info: dict[tuple[str, tuple[str, ...], None], list[str]] = { 41 | bold: [], 42 | megre_mag: [], 43 | megre_phase: [], 44 | } 45 | for s in seqinfo: 46 | if "_ME_" in s.series_description: 47 | info[bold].append(s.series_id) 48 | if "GRE_QSM" in s.series_description: 49 | if s.image_type[2] == "M": 50 | info[megre_mag].append(s.series_id) 51 | elif s.image_type[2] == "P": 52 | info[megre_phase].append(s.series_id) 53 | return info 54 | -------------------------------------------------------------------------------- /heudiconv/heuristics/bids_PhoenixReport.py: -------------------------------------------------------------------------------- 1 | """Heuristic demonstrating conversion of the PhoenixZIPReport from Siemens. 2 | 3 | It only cares about converting a series with have PhoenixZIPReport in their 4 | series_description and outputs **only to sourcedata**. 5 | """ 6 | 7 | from __future__ import annotations 8 | 9 | from typing import Optional 10 | 11 | from heudiconv.utils import SeqInfo 12 | 13 | 14 | def create_key( 15 | template: Optional[str], 16 | outtype: tuple[str, ...] = ("nii.gz",), 17 | annotation_classes: None = None, 18 | ) -> tuple[str, tuple[str, ...], None]: 19 | if template is None or not template: 20 | raise ValueError("Template must be a valid format string") 21 | return (template, outtype, annotation_classes) 22 | 23 | 24 | def infotodict( 25 | seqinfo: list[SeqInfo], 26 | ) -> dict[tuple[str, tuple[str, ...], None], list[dict[str, str]]]: 27 | """Heuristic evaluator for determining which runs belong where 28 | 29 | allowed template fields - follow python string module: 30 | 31 | item: index within category 32 | subject: participant id 33 | seqitem: run number during scanning 34 | subindex: sub index within group 35 | """ 36 | sbref = create_key( 37 | "sub-{subject}/func/sub-{subject}_task-QA_sbref", 38 | outtype=( 39 | "nii.gz", 40 | "dicom", 41 | ), 42 | ) 43 | scout = create_key( 44 | "sub-{subject}/anat/sub-{subject}_T1w", 45 | outtype=( 46 | "nii.gz", 47 | "dicom", 48 | ), 49 | ) 50 | phoenix_doc = create_key( 51 | "sub-{subject}/misc/sub-{subject}_phoenix", outtype=("dicom",) 52 | ) 53 | 54 | info: dict[tuple[str, tuple[str, ...], None], list[dict[str, str]]] = { 55 | sbref: [], 56 | scout: [], 57 | phoenix_doc: [], 58 | } 59 | for s in seqinfo: 60 | if ( 61 | "PhoenixZIPReport" in s.series_description 62 | and s.image_type[3] == "CSA REPORT" 63 | ): 64 | info[phoenix_doc].append({"item": s.series_id}) 65 | if "scout" in s.series_description.lower(): 66 | info[scout].append({"item": s.series_id}) 67 | 68 | return info 69 | -------------------------------------------------------------------------------- /heudiconv/heuristics/bids_with_ses.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import Optional 4 | 5 | from heudiconv.utils import SeqInfo 6 | 7 | 8 | def create_key( 9 | template: Optional[str], 10 | outtype: tuple[str, ...] = ("nii.gz",), 11 | annotation_classes: None = None, 12 | ) -> tuple[str, tuple[str, ...], None]: 13 | if template is None or not template: 14 | raise ValueError("Template must be a valid format string") 15 | return (template, outtype, annotation_classes) 16 | 17 | 18 | def infotodict( 19 | seqinfo: list[SeqInfo], 20 | ) -> dict[tuple[str, tuple[str, ...], None], list]: 21 | """Heuristic evaluator for determining which runs belong where 22 | 23 | allowed template fields - follow python string module: 24 | 25 | item: index within category 26 | subject: participant id 27 | seqitem: run number during scanning 28 | subindex: sub index within group 29 | session: scan index for longitudinal acq 30 | """ 31 | # for this example, we want to include copies of the DICOMs just for our T1 32 | # and functional scans 33 | outdicom = ("dicom", "nii.gz") 34 | 35 | t1 = create_key( 36 | "{bids_subject_session_dir}/anat/{bids_subject_session_prefix}_T1w", 37 | outtype=outdicom, 38 | ) 39 | t2 = create_key("{bids_subject_session_dir}/anat/{bids_subject_session_prefix}_T2w") 40 | dwi_ap = create_key( 41 | "{bids_subject_session_dir}/dwi/{bids_subject_session_prefix}_dir-AP_dwi" 42 | ) 43 | dwi_pa = create_key( 44 | "{bids_subject_session_dir}/dwi/{bids_subject_session_prefix}_dir-PA_dwi" 45 | ) 46 | rs = create_key( 47 | "{bids_subject_session_dir}/func/{bids_subject_session_prefix}_task-rest_run-{item:02d}_bold", 48 | outtype=outdicom, 49 | ) 50 | boldt1 = create_key( 51 | "{bids_subject_session_dir}/func/{bids_subject_session_prefix}_task-bird1back_run-{item:02d}_bold", 52 | outtype=outdicom, 53 | ) 54 | boldt2 = create_key( 55 | "{bids_subject_session_dir}/func/{bids_subject_session_prefix}_task-letter1back_run-{item:02d}_bold", 56 | outtype=outdicom, 57 | ) 58 | boldt3 = create_key( 59 | "{bids_subject_session_dir}/func/{bids_subject_session_prefix}_task-letter2back_run-{item:02d}_bold", 60 | outtype=outdicom, 61 | ) 62 | 63 | info: dict[tuple[str, tuple[str, ...], None], list] = { 64 | t1: [], 65 | t2: [], 66 | dwi_ap: [], 67 | dwi_pa: [], 68 | rs: [], 69 | boldt1: [], 70 | boldt2: [], 71 | boldt3: [], 72 | } 73 | for s in seqinfo: 74 | if ( 75 | (s.dim3 == 176 or s.dim3 == 352) 76 | and (s.dim4 == 1) 77 | and ("MEMPRAGE" in s.protocol_name) 78 | ): 79 | info[t1] = [s.series_id] 80 | elif (s.dim4 == 1) and ("MEMPRAGE" in s.protocol_name): 81 | info[t1] = [s.series_id] 82 | elif ( 83 | (s.dim3 == 176 or s.dim3 == 352) 84 | and (s.dim4 == 1) 85 | and ("T2_SPACE" in s.protocol_name) 86 | ): 87 | info[t2] = [s.series_id] 88 | elif (s.dim4 >= 70) and ("DIFFUSION_HighRes_AP" in s.protocol_name): 89 | info[dwi_ap].append([s.series_id]) 90 | elif "DIFFUSION_HighRes_PA" in s.protocol_name: 91 | info[dwi_pa].append([s.series_id]) 92 | elif (s.dim4 == 144) and ("resting" in s.protocol_name): 93 | if not s.is_motion_corrected: 94 | info[rs].append([(s.series_id)]) 95 | elif (s.dim4 == 183 or s.dim4 == 366) and ("localizer" in s.protocol_name): 96 | if not s.is_motion_corrected: 97 | info[boldt1].append([s.series_id]) 98 | elif (s.dim4 == 227 or s.dim4 == 454) and ("transfer1" in s.protocol_name): 99 | if not s.is_motion_corrected: 100 | info[boldt2].append([s.series_id]) 101 | elif (s.dim4 == 227 or s.dim4 == 454) and ("transfer2" in s.protocol_name): 102 | if not s.is_motion_corrected: 103 | info[boldt3].append([s.series_id]) 104 | return info 105 | -------------------------------------------------------------------------------- /heudiconv/heuristics/cmrr_heuristic.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import Optional 4 | 5 | from heudiconv.utils import SeqInfo 6 | 7 | 8 | def create_key( 9 | template: Optional[str], 10 | outtype: tuple[str, ...] = ("nii.gz", "dicom"), 11 | annotation_classes: None = None, 12 | ) -> tuple[str, tuple[str, ...], None]: 13 | if template is None or not template: 14 | raise ValueError("Template must be a valid format string") 15 | return (template, outtype, annotation_classes) 16 | 17 | 18 | def infotodict( 19 | seqinfo: list[SeqInfo], 20 | ) -> dict[tuple[str, tuple[str, ...], None], list]: 21 | """Heuristic evaluator for determining which runs belong where 22 | 23 | allowed template fields - follow python string module: 24 | 25 | item: index within category 26 | subject: participant id 27 | seqitem: run number during scanning 28 | subindex: sub index within group 29 | """ 30 | t1 = create_key("anat/sub-{subject}_T1w") 31 | t2 = create_key("anat/sub-{subject}_T2w") 32 | rest = create_key("func/sub-{subject}_dir-{acq}_task-rest_run-{item:02d}_bold") 33 | face = create_key("func/sub-{subject}_task-face_run-{item:02d}_acq-{acq}_bold") 34 | gamble = create_key( 35 | "func/sub-{subject}_task-gambling_run-{item:02d}_acq-{acq}_bold" 36 | ) 37 | conflict = create_key( 38 | "func/sub-{subject}_task-conflict_run-{item:02d}_acq-{acq}_bold" 39 | ) 40 | dwi = create_key("dwi/sub-{subject}_dir-{acq}_run-{item:02d}_dwi") 41 | 42 | fmap_rest = create_key( 43 | "fmap/sub-{subject}_acq-func{acq}_dir-{dir}_run-{item:02d}_epi" 44 | ) 45 | fmap_dwi = create_key( 46 | "fmap/sub-{subject}_acq-dwi{acq}_dir-{dir}_run-{item:02d}_epi" 47 | ) 48 | 49 | info: dict[tuple[str, tuple[str, ...], None], list] = { 50 | t1: [], 51 | t2: [], 52 | rest: [], 53 | face: [], 54 | gamble: [], 55 | conflict: [], 56 | dwi: [], 57 | fmap_rest: [], 58 | fmap_dwi: [], 59 | } 60 | 61 | for idx, s in enumerate(seqinfo): 62 | if (s.dim3 == 208) and (s.dim4 == 1) and ("T1w" in s.protocol_name): 63 | info[t1] = [s.series_id] 64 | if (s.dim3 == 208) and ("T2w" in s.protocol_name): 65 | info[t2] = [s.series_id] 66 | if (s.dim4 >= 99) and ( 67 | ("dMRI_dir98_AP" in s.protocol_name) or ("dMRI_dir99_AP" in s.protocol_name) 68 | ): 69 | acq = s.protocol_name.split("dMRI_")[1].split("_")[0] + "AP" 70 | info[dwi].append({"item": s.series_id, "acq": acq}) 71 | if (s.dim4 >= 99) and ( 72 | ("dMRI_dir98_PA" in s.protocol_name) or ("dMRI_dir99_PA" in s.protocol_name) 73 | ): 74 | acq = s.protocol_name.split("dMRI_")[1].split("_")[0] + "PA" 75 | info[dwi].append({"item": s.series_id, "acq": acq}) 76 | if (s.dim4 == 1) and ( 77 | ("dMRI_dir98_AP" in s.protocol_name) or ("dMRI_dir99_AP" in s.protocol_name) 78 | ): 79 | acq = s.protocol_name.split("dMRI_")[1].split("_")[0] 80 | info[fmap_dwi].append({"item": s.series_id, "dir": "AP", "acq": acq}) 81 | if (s.dim4 == 1) and ( 82 | ("dMRI_dir98_PA" in s.protocol_name) or ("dMRI_dir99_PA" in s.protocol_name) 83 | ): 84 | acq = s.protocol_name.split("dMRI_")[1].split("_")[0] 85 | info[fmap_dwi].append({"item": s.series_id, "dir": "PA", "acq": acq}) 86 | if (s.dim4 == 420) and ("rfMRI_REST_AP" in s.protocol_name): 87 | info[rest].append({"item": s.series_id, "acq": "AP"}) 88 | if (s.dim4 == 420) and ("rfMRI_REST_PA" in s.protocol_name): 89 | info[rest].append({"item": s.series_id, "acq": "PA"}) 90 | if (s.dim4 == 1) and ("rfMRI_REST_AP" in s.protocol_name): 91 | if seqinfo[idx + 1][9] != 420: 92 | continue 93 | info[fmap_rest].append({"item": s.series_id, "dir": "AP", "acq": ""}) 94 | if (s.dim4 == 1) and ("rfMRI_REST_PA" in s.protocol_name): 95 | info[fmap_rest].append({"item": s.series_id, "dir": "PA", "acq": ""}) 96 | if (s.dim4 == 346) and ("tfMRI_faceMatching_AP" in s.protocol_name): 97 | info[face].append({"item": s.series_id, "acq": "AP"}) 98 | if (s.dim4 == 346) and ("tfMRI_faceMatching_PA" in s.protocol_name): 99 | info[face].append({"item": s.series_id, "acq": "PA"}) 100 | if (s.dim4 == 288) and ("tfMRI_conflict_AP" in s.protocol_name): 101 | info[conflict].append({"item": s.series_id, "acq": "AP"}) 102 | if (s.dim4 == 288) and ("tfMRI_conflict_PA" in s.protocol_name): 103 | info[conflict].append({"item": s.series_id, "acq": "PA"}) 104 | if (s.dim4 == 223) and ("tfMRI_gambling_AP" in (s.protocol_name)): 105 | info[gamble].append({"item": s.series_id, "acq": "AP"}) 106 | if (s.dim4 == 223) and ("tfMRI_gambling_PA" in s.protocol_name): 107 | info[gamble].append({"item": s.series_id, "acq": "PA"}) 108 | return info 109 | -------------------------------------------------------------------------------- /heudiconv/heuristics/convertall.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import logging 4 | from typing import Optional 5 | 6 | from heudiconv.utils import SeqInfo 7 | 8 | lgr = logging.getLogger("heudiconv") 9 | 10 | 11 | def create_key( 12 | template: Optional[str], 13 | outtype: tuple[str, ...] = ("nii.gz",), 14 | annotation_classes: None = None, 15 | ) -> tuple[str, tuple[str, ...], None]: 16 | if template is None or not template: 17 | raise ValueError("Template must be a valid format string") 18 | return (template, outtype, annotation_classes) 19 | 20 | 21 | def infotodict( 22 | seqinfo: list[SeqInfo], 23 | ) -> dict[tuple[str, tuple[str, ...], None], list[str]]: 24 | """Heuristic evaluator for determining which runs belong where 25 | 26 | allowed template fields - follow python string module: 27 | 28 | item: index within category 29 | subject: participant id 30 | seqitem: run number during scanning 31 | subindex: sub index within group 32 | """ 33 | 34 | data = create_key("run{item:03d}") 35 | info: dict[tuple[str, tuple[str, ...], None], list[str]] = {data: []} 36 | 37 | for s in seqinfo: 38 | """ 39 | The namedtuple `s` contains the following fields: 40 | 41 | * total_files_till_now 42 | * example_dcm_file 43 | * series_id 44 | * dcm_dir_name 45 | * unspecified2 46 | * unspecified3 47 | * dim1 48 | * dim2 49 | * dim3 50 | * dim4 51 | * TR 52 | * TE 53 | * protocol_name 54 | * is_motion_corrected 55 | * is_derived 56 | * patient_id 57 | * study_description 58 | * referring_physician_name 59 | * series_description 60 | * image_type 61 | """ 62 | 63 | info[data].append(s.series_id) 64 | return info 65 | -------------------------------------------------------------------------------- /heudiconv/heuristics/convertall_custom.py: -------------------------------------------------------------------------------- 1 | """A demo convertall heuristic with custom_seqinfo extracting affine and sample DICOM path 2 | 3 | This heuristic also demonstrates on how to create a "derived" heuristic which would augment 4 | behavior of an already existing heuristic without complete rewrite. Such approach could be 5 | useful for heuristic like reproin to overload mapping etc. 6 | """ 7 | from __future__ import annotations 8 | 9 | from typing import Any 10 | 11 | import nibabel.nicom.dicomwrappers as dw 12 | 13 | from .convertall import * # noqa: F403 14 | 15 | 16 | def custom_seqinfo( 17 | series_files: list[str], wrapper: dw.Wrapper, **kw: Any # noqa: U100 18 | ) -> tuple[str | None, str]: 19 | """Demo for extracting custom header fields into custom_seqinfo field 20 | 21 | Operates on already loaded DICOM data. 22 | Origin: https://github.com/nipy/heudiconv/pull/333 23 | """ 24 | 25 | from nibabel.nicom.dicomwrappers import WrapperError 26 | 27 | try: 28 | affine = str(wrapper.affine) 29 | except WrapperError: 30 | lgr.exception("Errored out while obtaining/converting affine") # noqa: F405 31 | affine = None 32 | return affine, series_files[0] 33 | -------------------------------------------------------------------------------- /heudiconv/heuristics/example.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import Optional 4 | 5 | from heudiconv.utils import SeqInfo 6 | 7 | # Dictionary to specify options for the `populate_intended_for`. 8 | # Valid options are defined in 'bids.py' (for 'matching_parameters': 9 | # ['Shims', 'ImagingVolume',]; for 'criterion': ['First', 'Closest'] 10 | POPULATE_INTENDED_FOR_OPTS = { 11 | "matching_parameters": "ImagingVolume", 12 | "criterion": "Closest", 13 | } 14 | 15 | 16 | def create_key( 17 | template: Optional[str], 18 | outtype: tuple[str, ...] = ("nii.gz",), 19 | annotation_classes: None = None, 20 | ) -> tuple[str, tuple[str, ...], None]: 21 | if template is None or not template: 22 | raise ValueError("Template must be a valid format string") 23 | return (template, outtype, annotation_classes) 24 | 25 | 26 | def infotodict( 27 | seqinfo: list[SeqInfo], 28 | ) -> dict[tuple[str, tuple[str, ...], None], list]: 29 | """Heuristic evaluator for determining which runs belong where 30 | 31 | allowed template fields - follow python string module: 32 | 33 | item: index within category 34 | subject: participant id 35 | seqitem: run number during scanning 36 | subindex: sub index within group 37 | """ 38 | 39 | rs = create_key("rsfmri/rest_run{item:03d}/rest", outtype=("dicom", "nii.gz")) 40 | boldt1 = create_key("BOLD/task001_run{item:03d}/bold") 41 | boldt2 = create_key("BOLD/task002_run{item:03d}/bold") 42 | boldt3 = create_key("BOLD/task003_run{item:03d}/bold") 43 | boldt4 = create_key("BOLD/task004_run{item:03d}/bold") 44 | boldt5 = create_key("BOLD/task005_run{item:03d}/bold") 45 | boldt6 = create_key("BOLD/task006_run{item:03d}/bold") 46 | boldt7 = create_key("BOLD/task007_run{item:03d}/bold") 47 | boldt8 = create_key("BOLD/task008_run{item:03d}/bold") 48 | fm1 = create_key("fieldmap/fm1_{item:03d}") 49 | fm2 = create_key("fieldmap/fm2_{item:03d}") 50 | fmrest = create_key("fieldmap/fmrest_{item:03d}") 51 | dwi = create_key("dmri/dwi_{item:03d}", outtype=("dicom", "nii.gz")) 52 | t1 = create_key("anatomy/T1_{item:03d}") 53 | asl = create_key("rsfmri/asl_run{item:03d}/asl") 54 | aslcal = create_key("rsfmri/asl_run{item:03d}/cal_{subindex:03d}") 55 | info: dict[tuple[str, tuple[str, ...], None], list] = { 56 | rs: [], 57 | boldt1: [], 58 | boldt2: [], 59 | boldt3: [], 60 | boldt4: [], 61 | boldt5: [], 62 | boldt6: [], 63 | boldt7: [], 64 | boldt8: [], 65 | fm1: [], 66 | fm2: [], 67 | fmrest: [], 68 | dwi: [], 69 | t1: [], 70 | asl: [], 71 | aslcal: [[]], 72 | } 73 | last_run = len(seqinfo) 74 | for s in seqinfo: 75 | series_num_str = s.series_id.split("-", 1)[0] 76 | if not series_num_str.isdecimal(): 77 | raise ValueError( 78 | f"This heuristic can operate only on data when series_id has form -, " 79 | f"and is a numeric number. Got series_id={s.series_id}" 80 | ) 81 | series_num: int = int(series_num_str) 82 | sl, nt = (s.dim3, s.dim4) 83 | if (sl == 176) and (nt == 1) and ("MPRAGE" in s.protocol_name): 84 | info[t1] = [s.series_id] 85 | elif (nt > 60) and ("ge_func_2x2x2_Resting" in s.protocol_name): 86 | if not s.is_motion_corrected: 87 | info[rs].append(s.series_id) 88 | elif ( 89 | (nt == 156) 90 | and ("ge_functionals_128_PACE_ACPC-30" in s.protocol_name) 91 | and series_num < last_run 92 | ): 93 | if not s.is_motion_corrected: 94 | info[boldt1].append(s.series_id) 95 | last_run = series_num 96 | elif (nt == 155) and ("ge_functionals_128_PACE_ACPC-30" in s.protocol_name): 97 | if not s.is_motion_corrected: 98 | info[boldt2].append(s.series_id) 99 | elif (nt == 222) and ("ge_functionals_128_PACE_ACPC-30" in s.protocol_name): 100 | if not s.is_motion_corrected: 101 | info[boldt3].append(s.series_id) 102 | elif (nt == 114) and ("ge_functionals_128_PACE_ACPC-30" in s.protocol_name): 103 | if not s.is_motion_corrected: 104 | info[boldt4].append(s.series_id) 105 | elif (nt == 156) and ("ge_functionals_128_PACE_ACPC-30" in s.protocol_name): 106 | if not s.is_motion_corrected and (series_num > last_run): 107 | info[boldt5].append(s.series_id) 108 | elif (nt == 324) and ("ge_func_3.1x3.1x4_PACE" in s.protocol_name): 109 | if not s.is_motion_corrected: 110 | info[boldt6].append(s.series_id) 111 | elif (nt == 250) and ("ge_func_3.1x3.1x4_PACE" in s.protocol_name): 112 | if not s.is_motion_corrected: 113 | info[boldt7].append(s.series_id) 114 | elif (nt == 136) and ("ge_func_3.1x3.1x4_PACE" in s.protocol_name): 115 | if not s.is_motion_corrected: 116 | info[boldt8].append(s.series_id) 117 | elif (nt == 101) and ("ep2d_pasl_FairQuipssII" in s.protocol_name): 118 | if not s.is_motion_corrected: 119 | info[asl].append(s.series_id) 120 | elif (nt == 1) and ("ep2d_pasl_FairQuipssII" in s.protocol_name): 121 | info[aslcal][0].append(s.series_id) 122 | elif (sl > 1) and (nt == 70) and ("DIFFUSION" in s.protocol_name): 123 | info[dwi].append(s.series_id) 124 | elif "field_mapping_128" in s.protocol_name: 125 | info[fm1].append(s.series_id) 126 | elif "field_mapping_3.1" in s.protocol_name: 127 | info[fm2].append(s.series_id) 128 | elif "field_mapping_Resting" in s.protocol_name: 129 | info[fmrest].append(s.series_id) 130 | else: 131 | pass 132 | return info 133 | -------------------------------------------------------------------------------- /heudiconv/heuristics/multires_7Tbold.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import Optional 4 | 5 | import pydicom as dcm 6 | 7 | from heudiconv.utils import SeqInfo 8 | 9 | scaninfo_suffix = ".json" 10 | 11 | 12 | def create_key( 13 | template: Optional[str], 14 | outtype: tuple[str, ...] = ("nii.gz",), 15 | annotation_classes: None = None, 16 | ) -> tuple[str, tuple[str, ...], None]: 17 | if template is None or not template: 18 | raise ValueError("Template must be a valid format string") 19 | return (template, outtype, annotation_classes) 20 | 21 | 22 | def filter_dicom(dcmdata: dcm.dataset.Dataset) -> bool: 23 | """Return True if a DICOM dataset should be filtered out, else False""" 24 | comments = getattr(dcmdata, "ImageComments", "") 25 | if len(comments): 26 | if "reference volume" in comments.lower(): 27 | print("Filter out image with comment '%s'" % comments) 28 | return True 29 | return False 30 | 31 | 32 | def extract_moco_params( 33 | basename: str, _outypes: tuple[str, ...], dicoms: list[str] 34 | ) -> None: 35 | if "_rec-dico" not in basename: 36 | return 37 | from pydicom import dcmread 38 | 39 | # get acquisition time for all dicoms 40 | dcm_times = [ 41 | (d, float(dcmread(d, stop_before_pixels=True).AcquisitionTime)) for d in dicoms 42 | ] 43 | # store MoCo info from image comments sorted by acquisition time 44 | moco = [ 45 | "\t".join( 46 | [ 47 | str(float(i)) 48 | for i in dcmread(fn, stop_before_pixels=True) 49 | .ImageComments.split()[1] 50 | .split(",") 51 | ] 52 | ) 53 | for fn, t in sorted(dcm_times, key=lambda x: x[1]) 54 | ] 55 | outname = basename[:-4] + "recording-motion_physio.tsv" 56 | with open(outname, "wt") as fp: 57 | for m in moco: 58 | fp.write("%s\n" % (m,)) 59 | 60 | 61 | custom_callable = extract_moco_params 62 | 63 | 64 | def infotodict( 65 | seqinfo: list[SeqInfo], 66 | ) -> dict[tuple[str, tuple[str, ...], None], list[str]]: 67 | """Heuristic evaluator for determining which runs belong where 68 | 69 | allowed template fields - follow python string module: 70 | 71 | item: index within category 72 | subject: participant id 73 | seqitem: run number during scanning 74 | subindex: sub index within group 75 | """ 76 | 77 | info: dict[tuple[str, tuple[str, ...], None], list[str]] = {} 78 | for s in seqinfo: 79 | if "_bold_" not in s.protocol_name: 80 | continue 81 | if "_coverage" not in s.protocol_name: 82 | label = "orientation%s_run-{item:02d}" 83 | else: 84 | label = "coverage%s" 85 | resolution = s.protocol_name.split("_")[5][:-3] 86 | assert float(resolution) 87 | if s.is_motion_corrected: 88 | label = label % ("_rec-dico",) 89 | else: 90 | label = label % ("",) 91 | 92 | templ = "ses-%smm/func/{subject}_ses-%smm_task-%s_bold" % ( 93 | resolution, 94 | resolution, 95 | label, 96 | ) 97 | 98 | key = create_key(templ) 99 | 100 | if key not in info: 101 | info[key] = [] 102 | info[key].append(s.series_id) 103 | 104 | return info 105 | -------------------------------------------------------------------------------- /heudiconv/heuristics/reproin_validator.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "ignore": [ 3 | "TOTAL_READOUT_TIME_NOT_DEFINED", 4 | "CUSTOM_COLUMN_WITHOUT_DESCRIPTION" 5 | ], 6 | "warn": [], 7 | "error": [], 8 | "ignoredFiles": [ 9 | "/.heudiconv/*", "/.heudiconv/*/*", "/.heudiconv/*/*/*", "/.heudiconv/*/*/*/*", 10 | "/.heudiconv/.git*", 11 | "/.heudiconv/.git/*", 12 | "/.heudiconv/.git/*/*", 13 | "/.heudiconv/.git/*/*/*", 14 | "/.heudiconv/.git/*/*/*/*", 15 | "/.heudiconv/.git/*/*/*/*/*", 16 | "/.heudiconv/.git/*/*/*/*/*/*", 17 | "/.git*", 18 | "/.datalad/*", "/.datalad/.*", 19 | "/.*/.datalad/*", "/.*/.datalad/.*", 20 | "/sub*/ses*/*/*__dup*", "/sub*/*/*__dup*" 21 | ] 22 | } 23 | -------------------------------------------------------------------------------- /heudiconv/heuristics/studyforrest_phase2.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import Optional 4 | 5 | from heudiconv.utils import SeqInfo 6 | 7 | scaninfo_suffix = ".json" 8 | 9 | 10 | def create_key( 11 | template: Optional[str], 12 | outtype: tuple[str, ...] = ("nii.gz",), 13 | annotation_classes: None = None, 14 | ) -> tuple[str, tuple[str, ...], None]: 15 | if template is None or not template: 16 | raise ValueError("Template must be a valid format string") 17 | return (template, outtype, annotation_classes) 18 | 19 | 20 | def infotodict( 21 | seqinfo: list[SeqInfo], 22 | ) -> dict[tuple[str, tuple[str, ...], None], list[str]]: 23 | """Heuristic evaluator for determining which runs belong where 24 | 25 | allowed template fields - follow python string module: 26 | 27 | item: index within category 28 | subject: participant id 29 | seqitem: run number during scanning 30 | subindex: sub index within group 31 | """ 32 | 33 | label_map = { 34 | "movie": "movielocalizer", 35 | "retmap": "retmap", 36 | "visloc": "objectcategories", 37 | } 38 | info: dict[tuple[str, tuple[str, ...], None], list[str]] = {} 39 | for s in seqinfo: 40 | if "EPI_3mm" not in s.protocol_name: 41 | continue 42 | label = s.protocol_name.split("_")[2].split()[0].strip("1234567890").lower() 43 | if label in ("movie", "retmap", "visloc"): 44 | key = create_key( 45 | "ses-localizer/func/{subject}_ses-localizer_task-%s_run-{item:01d}_bold" 46 | % label_map[label] 47 | ) 48 | elif label == "sense": 49 | # pilot retmap had different description 50 | key = create_key( 51 | "ses-localizer/func/{subject}_ses-localizer_task-retmap_run-{item:01d}_bold" 52 | ) 53 | elif label == "r": 54 | key = create_key( 55 | "ses-movie/func/{subject}_ses-movie_task-movie_run-%i_bold" 56 | % int(s.protocol_name.split("_")[2].split()[0][-1]) 57 | ) 58 | else: 59 | raise RuntimeError("YOU SHALL NOT PASS!") 60 | 61 | if key not in info: 62 | info[key] = [] 63 | 64 | info[key].append(s.series_id) 65 | 66 | return info 67 | -------------------------------------------------------------------------------- /heudiconv/heuristics/test_b0dwi_for_fmap.py: -------------------------------------------------------------------------------- 1 | """Heuristic to extract a b-value=0 DWI image (basically, a SE-EPI) 2 | both as a fmap and as dwi 3 | 4 | It is used just to test that a 'DIFFUSION' image that the user 5 | chooses to extract as fmap (pepolar case) doesn't produce _bvecs/ 6 | _bvals json files, while it does for dwi images 7 | """ 8 | 9 | from __future__ import annotations 10 | 11 | from typing import Optional 12 | 13 | from heudiconv.utils import SeqInfo 14 | 15 | 16 | def create_key( 17 | template: Optional[str], 18 | outtype: tuple[str, ...] = ("nii.gz",), 19 | annotation_classes: None = None, 20 | ) -> tuple[str, tuple[str, ...], None]: 21 | if template is None or not template: 22 | raise ValueError("Template must be a valid format string") 23 | return (template, outtype, annotation_classes) 24 | 25 | 26 | def infotodict( 27 | seqinfo: list[SeqInfo], 28 | ) -> dict[tuple[str, tuple[str, ...], None], list[str]]: 29 | """Heuristic evaluator for determining which runs belong where 30 | 31 | allowed template fields - follow python string module: 32 | 33 | item: index within category 34 | subject: participant id 35 | seqitem: run number during scanning 36 | subindex: sub index within group 37 | """ 38 | fmap = create_key("sub-{subject}/fmap/sub-{subject}_acq-b0dwi_epi") 39 | dwi = create_key("sub-{subject}/dwi/sub-{subject}_acq-b0dwi_dwi") 40 | 41 | info: dict[tuple[str, tuple[str, ...], None], list[str]] = {fmap: [], dwi: []} 42 | for s in seqinfo: 43 | if "DIFFUSION" in s.image_type: 44 | info[fmap].append(s.series_id) 45 | info[dwi].append(s.series_id) 46 | return info 47 | -------------------------------------------------------------------------------- /heudiconv/heuristics/test_reproin.py: -------------------------------------------------------------------------------- 1 | # 2 | # Tests for reproin.py 3 | # 4 | from __future__ import annotations 5 | 6 | import re 7 | from typing import NamedTuple 8 | from unittest.mock import patch 9 | 10 | import pytest 11 | 12 | from . import reproin 13 | from .reproin import ( 14 | filter_files, 15 | fix_canceled_runs, 16 | fix_dbic_protocol, 17 | fixup_subjectid, 18 | get_dups_marked, 19 | get_unique, 20 | md5sum, 21 | parse_series_spec, 22 | sanitize_str, 23 | ) 24 | 25 | 26 | class FakeSeqInfo(NamedTuple): 27 | accession_number: str 28 | study_description: str 29 | field1: str 30 | field2: str 31 | 32 | 33 | def test_get_dups_marked() -> None: 34 | no_dups: dict[tuple[str, tuple[str, ...], None], list[int]] = { 35 | ("some", ("foo",), None): [1] 36 | } 37 | assert get_dups_marked(no_dups) == no_dups 38 | 39 | info: dict[tuple[str, tuple[str, ...], None], list[int | str]] = { 40 | ("bu", ("du",), None): [1, 2], 41 | ("smth", (), None): [3], 42 | ("smth2", ("apple", "banana"), None): ["a", "b", "c"], 43 | } 44 | 45 | assert ( 46 | get_dups_marked(info) 47 | == get_dups_marked(info, True) 48 | == { 49 | ("bu__dup-01", ("du",), None): [1], 50 | ("bu", ("du",), None): [2], 51 | ("smth", (), None): [3], 52 | ("smth2__dup-01", ("apple", "banana"), None): ["a"], 53 | ("smth2__dup-02", ("apple", "banana"), None): ["b"], 54 | ("smth2", ("apple", "banana"), None): ["c"], 55 | } 56 | ) 57 | 58 | assert get_dups_marked(info, per_series=False) == { 59 | ("bu__dup-01", ("du",), None): [1], 60 | ("bu", ("du",), None): [2], 61 | ("smth", (), None): [3], 62 | ("smth2__dup-02", ("apple", "banana"), None): ["a"], 63 | ("smth2__dup-03", ("apple", "banana"), None): ["b"], 64 | ("smth2", ("apple", "banana"), None): ["c"], 65 | } 66 | 67 | 68 | def test_filter_files() -> None: 69 | # Filtering is currently disabled -- any sequence directory is Ok 70 | assert filter_files("/home/mvdoc/dbic/09-run_func_meh/0123432432.dcm") 71 | assert filter_files("/home/mvdoc/dbic/run_func_meh/012343143.dcm") 72 | 73 | 74 | def test_md5sum() -> None: 75 | assert md5sum("cryptonomicon") == "1cd52edfa41af887e14ae71d1db96ad1" 76 | assert md5sum("mysecretmessage") == "07989808231a0c6f522f9d8e34695794" 77 | 78 | 79 | def test_fix_canceled_runs() -> None: 80 | class FakeSeqInfo(NamedTuple): 81 | accession_number: str 82 | series_id: str 83 | protocol_name: str 84 | series_description: str 85 | 86 | seqinfo: list[FakeSeqInfo] = [] 87 | runname = "func_run+" 88 | for i in range(1, 6): 89 | seqinfo.append( 90 | FakeSeqInfo("accession1", "{0:02d}-".format(i) + runname, runname, runname) 91 | ) 92 | 93 | fake_accession2run = {"accession1": ["^01-", "^03-"]} 94 | 95 | with patch.object(reproin, "fix_accession2run", fake_accession2run): 96 | seqinfo_ = fix_canceled_runs(seqinfo) # type: ignore[arg-type] 97 | 98 | for i, s in enumerate(seqinfo_, 1): 99 | output = runname 100 | if i == 1 or i == 3: 101 | output = "cancelme_" + output 102 | for key in ["series_description", "protocol_name"]: 103 | value = getattr(s, key) 104 | assert value == output 105 | # check we didn't touch series_id 106 | assert s.series_id == "{0:02d}-".format(i) + runname 107 | 108 | 109 | def test_fix_dbic_protocol() -> None: 110 | accession_number = "A003" 111 | seq1 = FakeSeqInfo( 112 | accession_number, 113 | "mystudy", 114 | "02-anat-scout_run+_MPR_sag", 115 | "11-func_run-life2_acq-2mm692", 116 | ) 117 | seq2 = FakeSeqInfo(accession_number, "mystudy", "nochangeplease", "nochangeeither") 118 | 119 | seqinfos = [seq1, seq2] 120 | protocols2fix = { 121 | md5sum("mystudy"): [ 122 | (r"scout_run\+", "THESCOUT-runX"), 123 | ("run-life[0-9]", "run+_task-life"), 124 | ], 125 | re.compile("^my.*"): [("THESCOUT-runX", "THESCOUT")], 126 | # rely on 'catch-all' to fix up above scout 127 | "": [("THESCOUT", "scout")], 128 | } 129 | 130 | with patch.object(reproin, "protocols2fix", protocols2fix), patch.object( 131 | reproin, "series_spec_fields", ["field1"] 132 | ): 133 | seqinfos_ = fix_dbic_protocol(seqinfos) # type: ignore[arg-type] 134 | assert seqinfos[1] == seqinfos_[1] # type: ignore[comparison-overlap] 135 | # field2 shouldn't have changed since I didn't pass it 136 | assert seqinfos_[0] == FakeSeqInfo( # type: ignore[comparison-overlap] 137 | accession_number, "mystudy", "02-anat-scout_MPR_sag", seq1.field2 138 | ) 139 | 140 | # change also field2 please 141 | with patch.object(reproin, "protocols2fix", protocols2fix), patch.object( 142 | reproin, "series_spec_fields", ["field1", "field2"] 143 | ): 144 | seqinfos_ = fix_dbic_protocol(seqinfos) # type: ignore[arg-type] 145 | assert seqinfos[1] == seqinfos_[1] # type: ignore[comparison-overlap] 146 | # now everything should have changed 147 | assert seqinfos_[0] == FakeSeqInfo( # type: ignore[comparison-overlap] 148 | accession_number, 149 | "mystudy", 150 | "02-anat-scout_MPR_sag", 151 | "11-func_run+_task-life_acq-2mm692", 152 | ) 153 | 154 | 155 | def test_sanitize_str() -> None: 156 | assert sanitize_str("super@duper.faster") == "superduperfaster" 157 | assert sanitize_str("perfect") == "perfect" 158 | assert sanitize_str("never:use:colon:!") == "neverusecolon" 159 | 160 | 161 | def test_fixupsubjectid() -> None: 162 | assert fixup_subjectid("abra") == "abra" 163 | assert fixup_subjectid("sub") == "sub" 164 | assert fixup_subjectid("sid") == "sid" 165 | assert fixup_subjectid("sid000030") == "sid000030" 166 | assert fixup_subjectid("sid0000030") == "sid000030" 167 | assert fixup_subjectid("sid00030") == "sid000030" 168 | assert fixup_subjectid("sid30") == "sid000030" 169 | assert fixup_subjectid("SID30") == "sid000030" 170 | 171 | 172 | def test_parse_series_spec() -> None: 173 | pdpn = parse_series_spec 174 | 175 | assert pdpn("nondbic_func-bold") == {} 176 | assert pdpn("cancelme_func-bold") == {} 177 | 178 | assert ( 179 | pdpn("bids_func-bold") 180 | == pdpn("func-bold") 181 | == {"datatype": "func", "datatype_suffix": "bold"} 182 | ) 183 | 184 | # pdpn("bids_func_ses+_task-boo_run+") == \ 185 | # order and PREFIX: should not matter, as well as trailing spaces 186 | assert ( 187 | pdpn(" PREFIX:bids_func_ses+_task-boo_run+ ") 188 | == pdpn("PREFIX:bids_func_ses+_task-boo_run+") 189 | == pdpn("WIP func_ses+_task-boo_run+") 190 | == pdpn("bids_func_ses+_run+_task-boo") 191 | == { 192 | "datatype": "func", 193 | # 'datatype_suffix': 'bold', 194 | "session": "+", 195 | "run": "+", 196 | "task": "boo", 197 | } 198 | ) 199 | 200 | # TODO: fix for that 201 | assert ( 202 | pdpn("bids_func-pace_ses-1_task-boo_acq-bu_bids-please_run-2__therest") 203 | == pdpn("bids_func-pace_ses-1_run-2_task-boo_acq-bu_bids-please__therest") 204 | == pdpn("func-pace_ses-1_task-boo_acq-bu_bids-please_run-2") 205 | == { 206 | "datatype": "func", 207 | "datatype_suffix": "pace", 208 | "session": "1", 209 | "run": "2", 210 | "task": "boo", 211 | "acq": "bu", 212 | "bids": "bids-please", 213 | } 214 | ) 215 | 216 | assert pdpn("bids_anat-scout_ses+") == { 217 | "datatype": "anat", 218 | "datatype_suffix": "scout", 219 | "session": "+", 220 | } 221 | 222 | assert pdpn("anat_T1w_acq-MPRAGE_run+") == { 223 | "datatype": "anat", 224 | "run": "+", 225 | "acq": "MPRAGE", 226 | "datatype_suffix": "T1w", 227 | } 228 | 229 | # Check for currently used {date}, which should also should get adjusted 230 | # from (date) since Philips does not allow for {} 231 | assert ( 232 | pdpn("func_ses-{date}") 233 | == pdpn("func_ses-(date)") 234 | == {"datatype": "func", "session": "{date}"} 235 | ) 236 | 237 | assert pdpn("fmap_dir-AP_ses-01") == { 238 | "datatype": "fmap", 239 | "session": "01", 240 | "dir": "AP", 241 | } 242 | 243 | 244 | def test_get_unique() -> None: 245 | accession_number = "A003" 246 | acqs = [ 247 | FakeSeqInfo(accession_number, "mystudy", "nochangeplease", "nochangeeither"), 248 | FakeSeqInfo(accession_number, "mystudy2", "nochangeplease", "nochangeeither"), 249 | ] 250 | 251 | assert get_unique(acqs, "accession_number") == accession_number # type: ignore[arg-type] 252 | with pytest.raises(AssertionError) as ce: 253 | get_unique(acqs, "study_description") # type: ignore[arg-type] 254 | assert ( 255 | str(ce.value) 256 | == "Was expecting a single value for attribute 'study_description' but got: mystudy, mystudy2" 257 | ) 258 | -------------------------------------------------------------------------------- /heudiconv/heuristics/uc_bids.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import Optional 4 | 5 | from heudiconv.utils import SeqInfo 6 | 7 | 8 | def create_key( 9 | template: Optional[str], 10 | outtype: tuple[str, ...] = ("nii.gz",), 11 | annotation_classes: None = None, 12 | ) -> tuple[str, tuple[str, ...], None]: 13 | if template is None or not template: 14 | raise ValueError("Template must be a valid format string") 15 | return (template, outtype, annotation_classes) 16 | 17 | 18 | def infotodict( 19 | seqinfo: list[SeqInfo], 20 | ) -> dict[tuple[str, tuple[str, ...], None], list]: 21 | """Heuristic evaluator for determining which runs belong where 22 | 23 | allowed template fields - follow python string module: 24 | 25 | item: index within category 26 | subject: participant id 27 | seqitem: run number during scanning 28 | subindex: sub index within group 29 | """ 30 | t1w = create_key("anat/sub-{subject}_T1w") 31 | t2w = create_key("anat/sub-{subject}_acq-{acq}_T2w") 32 | flair = create_key("anat/sub-{subject}_acq-{acq}_FLAIR") 33 | rest = create_key("func/sub-{subject}_task-rest_acq-{acq}_run-{item:02d}_bold") 34 | 35 | info: dict[tuple[str, tuple[str, ...], None], list] = { 36 | t1w: [], 37 | t2w: [], 38 | flair: [], 39 | rest: [], 40 | } 41 | 42 | for seq in seqinfo: 43 | x, _, z, n_vol, protocol, dcm_dir = ( 44 | seq.dim1, 45 | seq.dim2, 46 | seq.dim3, 47 | seq.dim4, 48 | seq.protocol_name, 49 | seq.dcm_dir_name, 50 | ) 51 | # t1_mprage --> T1w 52 | if ( 53 | (z == 160) 54 | and (n_vol == 1) 55 | and ("t1_mprage" in protocol) 56 | and ("XX" not in dcm_dir) 57 | ): 58 | info[t1w] = [seq.series_id] 59 | # t2_tse --> T2w 60 | if ( 61 | (z == 35) 62 | and (n_vol == 1) 63 | and ("t2_tse" in protocol) 64 | and ("XX" not in dcm_dir) 65 | ): 66 | info[t2w].append({"item": seq.series_id, "acq": "TSE"}) 67 | # T2W --> T2w 68 | if ( 69 | (z == 192) 70 | and (n_vol == 1) 71 | and ("T2W" in protocol) 72 | and ("XX" not in dcm_dir) 73 | ): 74 | info[t2w].append({"item": seq.series_id, "acq": "highres"}) 75 | # t2_tirm --> FLAIR 76 | if ( 77 | (z == 35) 78 | and (n_vol == 1) 79 | and ("t2_tirm" in protocol) 80 | and ("XX" not in dcm_dir) 81 | ): 82 | info[flair].append({"item": seq.series_id, "acq": "TIRM"}) 83 | # t2_flair --> FLAIR 84 | if ( 85 | (z == 160) 86 | and (n_vol == 1) 87 | and ("t2_flair" in protocol) 88 | and ("XX" not in dcm_dir) 89 | ): 90 | info[flair].append({"item": seq.series_id, "acq": "highres"}) 91 | # T2FLAIR --> FLAIR 92 | if ( 93 | (z == 192) 94 | and (n_vol == 1) 95 | and ("T2-FLAIR" in protocol) 96 | and ("XX" not in dcm_dir) 97 | ): 98 | info[flair].append({"item": seq.series_id, "acq": "highres"}) 99 | # EPI (physio-matched) --> bold 100 | if ( 101 | (x == 128) 102 | and (z == 28) 103 | and (n_vol == 300) 104 | and ("EPI" in protocol) 105 | and ("XX" not in dcm_dir) 106 | ): 107 | info[rest].append({"item": seq.series_id, "acq": "128px"}) 108 | # EPI (physio-matched_NEW) --> bold 109 | if ( 110 | (x == 64) 111 | and (z == 34) 112 | and (n_vol == 300) 113 | and ("EPI" in protocol) 114 | and ("XX" not in dcm_dir) 115 | ): 116 | info[rest].append({"item": seq.series_id, "acq": "64px"}) 117 | return info 118 | -------------------------------------------------------------------------------- /heudiconv/info.py: -------------------------------------------------------------------------------- 1 | __author__ = "HeuDiConv team and contributors" 2 | __url__ = "https://github.com/nipy/heudiconv" 3 | __packagename__ = "heudiconv" 4 | __description__ = "Heuristic DICOM Converter" 5 | __license__ = "Apache 2.0" 6 | __longdesc__ = """Convert DICOM dirs based on heuristic info - HeuDiConv 7 | uses the dcmstack package and dcm2niix tool to convert DICOM directories or 8 | tarballs into collections of NIfTI files following pre-defined heuristic(s).""" 9 | 10 | CLASSIFIERS = [ 11 | "Environment :: Console", 12 | "Intended Audience :: Science/Research", 13 | "License :: OSI Approved :: Apache Software License", 14 | "Programming Language :: Python :: 3.9", 15 | "Programming Language :: Python :: 3.10", 16 | "Programming Language :: Python :: 3.11", 17 | "Programming Language :: Python :: 3.12", 18 | "Topic :: Scientific/Engineering", 19 | "Typing :: Typed", 20 | ] 21 | 22 | PYTHON_REQUIRES = ">=3.9" 23 | 24 | REQUIRES = [ 25 | # not usable in some use cases since might be just a downloader, not binary 26 | # 'dcm2niix', 27 | "dcmstack>=0.8", 28 | "etelemetry", 29 | "filelock>=3.0.12", 30 | "nibabel>=5.3.1", 31 | "nipype >=1.2.3", 32 | "pydicom >= 1.0.0", 33 | ] 34 | 35 | TESTS_REQUIRES = [ 36 | "pytest", 37 | "tinydb", 38 | "inotify", 39 | ] 40 | 41 | MIN_DATALAD_VERSION = "0.13.0" 42 | EXTRA_REQUIRES = { 43 | "tests": TESTS_REQUIRES, 44 | "extras": [ 45 | "duecredit", # optional dependency 46 | ], # Requires patched version ATM ['dcmstack'], 47 | "datalad": ["datalad >=%s" % MIN_DATALAD_VERSION], 48 | } 49 | 50 | # Flatten the lists 51 | EXTRA_REQUIRES["all"] = sum(EXTRA_REQUIRES.values(), []) 52 | -------------------------------------------------------------------------------- /heudiconv/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nipy/heudiconv/2a15cdfa913d2288f08198db2196047c90711e1b/heudiconv/py.typed -------------------------------------------------------------------------------- /heudiconv/queue.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import logging 4 | import os 5 | import subprocess 6 | import sys 7 | from typing import Optional 8 | 9 | from nipype.utils.filemanip import which 10 | 11 | lgr = logging.getLogger(__name__) 12 | 13 | 14 | def queue_conversion( 15 | queue: str, iterarg: str, iterables: int, queue_args: Optional[str] = None 16 | ) -> None: 17 | """ 18 | Write out conversion arguments to file and submit to a job scheduler. 19 | Parses `sys.argv` for heudiconv arguments. 20 | 21 | Parameters 22 | ---------- 23 | queue: string 24 | Batch scheduler to use 25 | iterarg: str 26 | Multi-argument to index (`subjects` OR `files`) 27 | iterables: int 28 | Number of `iterarg` arguments 29 | queue_args: string (optional) 30 | Additional queue arguments for job submission 31 | 32 | """ 33 | 34 | SUPPORTED_QUEUES = {"SLURM": "sbatch"} 35 | if queue not in SUPPORTED_QUEUES: 36 | raise NotImplementedError("Queuing with %s is not supported", queue) 37 | 38 | for i in range(iterables): 39 | args = clean_args(sys.argv[1:], iterarg, i) 40 | # make arguments executable 41 | heudiconv_exec = which("heudiconv") or "heudiconv" 42 | args.insert(0, heudiconv_exec) 43 | convertcmd = " ".join(args) 44 | 45 | # will overwrite across subjects 46 | queue_file = os.path.abspath("heudiconv-%s.sh" % queue) 47 | with open(queue_file, "wt") as fp: 48 | fp.write("#!/bin/bash\n") 49 | if queue_args: 50 | for qarg in queue_args.split(): 51 | fp.write("#SBATCH %s\n" % qarg) 52 | fp.write(convertcmd + "\n") 53 | 54 | cmd = [SUPPORTED_QUEUES[queue], queue_file] 55 | subprocess.call(cmd) 56 | lgr.info("Submitted %d jobs", iterables) 57 | 58 | 59 | def clean_args(hargs: list[str], iterarg: str, iteridx: int) -> list[str]: 60 | """ 61 | Filters arguments for batch submission. 62 | 63 | Parameters 64 | ---------- 65 | hargs: list 66 | Command-line arguments 67 | iterarg: str 68 | Multi-argument to index (`subjects` OR `files`) 69 | iteridx: int 70 | `iterarg` index to submit 71 | 72 | Returns 73 | ------- 74 | cmdargs : list 75 | Filtered arguments for batch submission 76 | 77 | Example 78 | -------- 79 | >>> from heudiconv.queue import clean_args 80 | >>> cmd = ['heudiconv', '-d', '/some/{subject}/path', 81 | ... '-q', 'SLURM', 82 | ... '-s', 'sub-1', 'sub-2', 'sub-3', 'sub-4'] 83 | >>> clean_args(cmd, 'subjects', 0) 84 | ['heudiconv', '-d', '/some/{subject}/path', '-s', 'sub-1'] 85 | """ 86 | 87 | if iterarg == "subjects": 88 | iterargs = ["-s", "--subjects"] 89 | elif iterarg == "files": 90 | iterargs = ["--files"] 91 | else: 92 | raise ValueError("Cannot index %s" % iterarg) 93 | 94 | # remove these or cause an infinite loop 95 | queue_args = ["-q", "--queue", "--queue-args"] 96 | 97 | # control variables for multi-argument parsing 98 | is_iterarg = False 99 | itercount = 0 100 | 101 | indices = [] 102 | cmdargs = hargs[:] 103 | 104 | for i, arg in enumerate(hargs): 105 | if arg.startswith("-") and is_iterarg: 106 | # moving on to another argument 107 | is_iterarg = False 108 | if is_iterarg: 109 | if iteridx != itercount: 110 | indices.append(i) 111 | itercount += 1 112 | if arg in iterargs: 113 | is_iterarg = True 114 | if arg in queue_args: 115 | indices.extend([i, i + 1]) 116 | 117 | for j in sorted(indices, reverse=True): 118 | del cmdargs[j] 119 | return cmdargs 120 | -------------------------------------------------------------------------------- /heudiconv/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nipy/heudiconv/2a15cdfa913d2288f08198db2196047c90711e1b/heudiconv/tests/__init__.py -------------------------------------------------------------------------------- /heudiconv/tests/anonymize_script.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import hashlib 4 | import re 5 | import sys 6 | 7 | 8 | def bids_id_(sid: str) -> str: 9 | m = re.compile(r"^(?:sub-|)(.+)$").search(sid) 10 | if m: 11 | parsed_id = m.group(1) 12 | return hashlib.md5(parsed_id.encode()).hexdigest()[:8] 13 | else: 14 | raise ValueError("invalid sid") 15 | 16 | 17 | def main() -> str: 18 | sid = sys.argv[1] 19 | return bids_id_(sid) 20 | 21 | 22 | if __name__ == "__main__": 23 | print(main()) 24 | -------------------------------------------------------------------------------- /heudiconv/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | 5 | 6 | @pytest.fixture(autouse=True, scope="session") 7 | def git_env() -> None: 8 | os.environ["GIT_AUTHOR_EMAIL"] = "maxm@example.com" 9 | os.environ["GIT_AUTHOR_NAME"] = "Max Mustermann" 10 | os.environ["GIT_COMMITTER_EMAIL"] = "maxm@example.com" 11 | os.environ["GIT_COMMITTER_NAME"] = "Max Mustermann" 12 | -------------------------------------------------------------------------------- /heudiconv/tests/data/01-anat-scout/0001.dcm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nipy/heudiconv/2a15cdfa913d2288f08198db2196047c90711e1b/heudiconv/tests/data/01-anat-scout/0001.dcm -------------------------------------------------------------------------------- /heudiconv/tests/data/01-fmap_acq-3mm/1.3.12.2.1107.5.2.43.66112.2016101409263663466202201.dcm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nipy/heudiconv/2a15cdfa913d2288f08198db2196047c90711e1b/heudiconv/tests/data/01-fmap_acq-3mm/1.3.12.2.1107.5.2.43.66112.2016101409263663466202201.dcm -------------------------------------------------------------------------------- /heudiconv/tests/data/MRI_102TD_PHA_S.MR.Chen_Matthews_1.3.1.2022.11.16.15.50.20.357.31204541.dcm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nipy/heudiconv/2a15cdfa913d2288f08198db2196047c90711e1b/heudiconv/tests/data/MRI_102TD_PHA_S.MR.Chen_Matthews_1.3.1.2022.11.16.15.50.20.357.31204541.dcm -------------------------------------------------------------------------------- /heudiconv/tests/data/Phoenix/01+AA/01+AA+00001.dcm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nipy/heudiconv/2a15cdfa913d2288f08198db2196047c90711e1b/heudiconv/tests/data/Phoenix/01+AA/01+AA+00001.dcm -------------------------------------------------------------------------------- /heudiconv/tests/data/Phoenix/99+PhoenixDocument/99+PhoenixDocument+00001.dcm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nipy/heudiconv/2a15cdfa913d2288f08198db2196047c90711e1b/heudiconv/tests/data/Phoenix/99+PhoenixDocument/99+PhoenixDocument+00001.dcm -------------------------------------------------------------------------------- /heudiconv/tests/data/axasc35.dcm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nipy/heudiconv/2a15cdfa913d2288f08198db2196047c90711e1b/heudiconv/tests/data/axasc35.dcm -------------------------------------------------------------------------------- /heudiconv/tests/data/b0dwiForFmap/b0dwi_for_fmap+00001.dcm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nipy/heudiconv/2a15cdfa913d2288f08198db2196047c90711e1b/heudiconv/tests/data/b0dwiForFmap/b0dwi_for_fmap+00001.dcm -------------------------------------------------------------------------------- /heudiconv/tests/data/b0dwiForFmap/b0dwi_for_fmap+00002.dcm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nipy/heudiconv/2a15cdfa913d2288f08198db2196047c90711e1b/heudiconv/tests/data/b0dwiForFmap/b0dwi_for_fmap+00002.dcm -------------------------------------------------------------------------------- /heudiconv/tests/data/b0dwiForFmap/b0dwi_for_fmap+00003.dcm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nipy/heudiconv/2a15cdfa913d2288f08198db2196047c90711e1b/heudiconv/tests/data/b0dwiForFmap/b0dwi_for_fmap+00003.dcm -------------------------------------------------------------------------------- /heudiconv/tests/data/non_zeros.bval: -------------------------------------------------------------------------------- 1 | 1000 0 1000 2000 2 | -------------------------------------------------------------------------------- /heudiconv/tests/data/phantom.dcm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nipy/heudiconv/2a15cdfa913d2288f08198db2196047c90711e1b/heudiconv/tests/data/phantom.dcm -------------------------------------------------------------------------------- /heudiconv/tests/data/sample_nifti.nii.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nipy/heudiconv/2a15cdfa913d2288f08198db2196047c90711e1b/heudiconv/tests/data/sample_nifti.nii.gz -------------------------------------------------------------------------------- /heudiconv/tests/data/sample_nifti_params.txt: -------------------------------------------------------------------------------- 1 | -1.99887669 0.0670080110 0.00144872535 109.236588 2 | 0.0650372952 1.94962955 -0.441264838 -71.7261658 3 | 0.0161963794 0.440969884 1.95071352 -64.1231308 4 | 0 0 0 1 5 | 5 5 5 1 6 | -------------------------------------------------------------------------------- /heudiconv/tests/data/zeros.bval: -------------------------------------------------------------------------------- 1 | 0 0 0 0 2 | -------------------------------------------------------------------------------- /heudiconv/tests/test_archives.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from glob import glob 4 | import os 5 | import os.path as op 6 | from pathlib import Path 7 | import shutil 8 | import stat 9 | 10 | import pytest 11 | 12 | from .utils import TESTS_DATA_PATH 13 | from ..parser import get_extracted_dicoms 14 | 15 | 16 | def _get_dicoms_archive(tmpdir: Path, fmt: str) -> list[str]: 17 | tmp_file = tmpdir / "dicom" 18 | archive = shutil.make_archive( 19 | str(tmp_file), format=fmt, root_dir=TESTS_DATA_PATH, base_dir="01-anat-scout" 20 | ) 21 | return [archive] 22 | 23 | 24 | @pytest.fixture 25 | def get_dicoms_archive(tmpdir: Path, request: pytest.FixtureRequest) -> list[str]: 26 | return _get_dicoms_archive(tmpdir, fmt=request.param) 27 | 28 | 29 | @pytest.fixture 30 | def get_dicoms_gztar(tmpdir: Path) -> list[str]: 31 | return _get_dicoms_archive(tmpdir, "gztar") 32 | 33 | 34 | @pytest.fixture 35 | def get_dicoms_list() -> list[str]: 36 | return glob(op.join(TESTS_DATA_PATH, "01-anat-scout", "*")) 37 | 38 | 39 | def test_get_extracted_dicoms_single_session_is_none( 40 | get_dicoms_gztar: list[str], 41 | ) -> None: 42 | for session_, _ in get_extracted_dicoms(get_dicoms_gztar): 43 | assert session_ is None 44 | 45 | 46 | def test_get_extracted_dicoms_multple_session_integers( 47 | get_dicoms_gztar: list[str], 48 | ) -> None: 49 | sessions = [ 50 | session 51 | for session, _ in get_extracted_dicoms(get_dicoms_gztar + get_dicoms_gztar) 52 | ] 53 | 54 | assert sessions == ["0", "1"] 55 | 56 | 57 | @pytest.mark.parametrize( 58 | "get_dicoms_archive", ("tar", "gztar", "zip", "bztar", "xztar"), indirect=True 59 | ) 60 | def test_get_extracted_dicoms_from_archives(get_dicoms_archive: list[str]) -> None: 61 | for _, files in get_extracted_dicoms(get_dicoms_archive): 62 | # check that the only file is the one called "0001.dcm" 63 | endswith = all(file.endswith("0001.dcm") for file in files) 64 | 65 | # check that permissions were set 66 | mode = all(stat.S_IMODE(os.stat(file).st_mode) == 448 for file in files) 67 | 68 | # check for absolute paths 69 | absolute = all(op.isabs(file) for file in files) 70 | 71 | assert all([endswith, mode, absolute]) 72 | 73 | 74 | def test_get_extracted_dicoms_from_file_list(get_dicoms_list: list[str]) -> None: 75 | for _, files in get_extracted_dicoms(get_dicoms_list): 76 | assert all(op.isfile(file) for file in files) 77 | 78 | 79 | def test_get_extracted_dicoms_from_mixed_list( 80 | get_dicoms_list: list[str], get_dicoms_gztar: list[str] 81 | ) -> None: 82 | for _, files in get_extracted_dicoms(get_dicoms_list + get_dicoms_gztar): 83 | assert all(op.isfile(file) for file in files) 84 | 85 | 86 | def test_get_extracted_from_empty_list() -> None: 87 | assert not len(get_extracted_dicoms([])) 88 | -------------------------------------------------------------------------------- /heudiconv/tests/test_dicoms.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import datetime 4 | from glob import glob 5 | import json 6 | import os.path as op 7 | from pathlib import Path 8 | 9 | import pydicom as dcm 10 | import pytest 11 | 12 | from heudiconv.cli.run import main as runner 13 | from heudiconv.convert import nipype_convert 14 | from heudiconv.dicoms import ( 15 | create_seqinfo, 16 | dw, 17 | embed_dicom_and_nifti_metadata, 18 | get_datetime_from_dcm, 19 | get_reproducible_int, 20 | group_dicoms_into_seqinfos, 21 | parse_private_csa_header, 22 | ) 23 | 24 | from .utils import TEST_DICOM_PATHS, TESTS_DATA_PATH 25 | 26 | # Public: Private DICOM tags 27 | DICOM_FIELDS_TO_TEST = {"ProtocolName": "tProtocolName"} 28 | 29 | 30 | def test_private_csa_header(tmp_path: Path) -> None: 31 | dcm_file = op.join(TESTS_DATA_PATH, "axasc35.dcm") 32 | dcm_data = dcm.dcmread(dcm_file, stop_before_pixels=True) 33 | for pub, priv in DICOM_FIELDS_TO_TEST.items(): 34 | # ensure missing public tag 35 | with pytest.raises(AttributeError): 36 | getattr(dcm, pub) 37 | # ensure private tag is found 38 | assert parse_private_csa_header(dcm_data, pub, priv) != "" 39 | # and quickly run heudiconv with no conversion 40 | runner( 41 | ["--files", dcm_file, "-c", "none", "-f", "reproin", "-o", str(tmp_path)] 42 | ) 43 | 44 | 45 | def test_embed_dicom_and_nifti_metadata( 46 | monkeypatch: pytest.MonkeyPatch, tmp_path: Path 47 | ) -> None: 48 | """Test dcmstack's additional fields""" 49 | monkeypatch.chdir(tmp_path) 50 | # set up testing files 51 | dcmfiles = [op.join(TESTS_DATA_PATH, "axasc35.dcm")] 52 | infofile = "infofile.json" 53 | 54 | out_prefix = str(tmp_path / "nifti") 55 | # 1) nifti does not exist -- no longer supported 56 | with pytest.raises(NotImplementedError): 57 | embed_dicom_and_nifti_metadata(dcmfiles, out_prefix + ".nii.gz", infofile, None) 58 | 59 | # we should produce nifti using our "standard" ways 60 | nipype_out, prov_file = nipype_convert( 61 | dcmfiles, 62 | prefix=out_prefix, 63 | with_prov=False, 64 | bids_options=None, 65 | tmpdir=str(tmp_path), 66 | ) 67 | niftifile = nipype_out.outputs.converted_files 68 | 69 | assert op.exists(niftifile) 70 | # 2) nifti exists 71 | embed_dicom_and_nifti_metadata(dcmfiles, niftifile, infofile, None) 72 | assert op.exists(infofile) 73 | with open(infofile) as fp: 74 | out2 = json.load(fp) 75 | 76 | # 3) with existing metadata 77 | bids = {"existing": "data"} 78 | embed_dicom_and_nifti_metadata(dcmfiles, niftifile, infofile, bids) 79 | with open(infofile) as fp: 80 | out3 = json.load(fp) 81 | 82 | assert out3.pop("existing") == "data" 83 | assert out3 == out2 84 | 85 | 86 | def test_group_dicoms_into_seqinfos() -> None: 87 | """Tests for group_dicoms_into_seqinfos""" 88 | 89 | # 1) Check that it works for PhoenixDocuments: 90 | # set up testing files 91 | dcmfolder = op.join(TESTS_DATA_PATH, "Phoenix") 92 | dcmfiles = glob(op.join(dcmfolder, "*", "*.dcm")) 93 | 94 | seqinfo = group_dicoms_into_seqinfos(dcmfiles, "studyUID", flatten=True) 95 | 96 | assert type(seqinfo) is dict 97 | assert len(seqinfo) == len(dcmfiles) 98 | assert [s.series_description for s in seqinfo] == [ 99 | "AAHead_Scout_32ch-head-coil", 100 | "PhoenixZIPReport", 101 | ] 102 | 103 | 104 | def test_custom_seqinfo() -> None: 105 | """Tests for custom seqinfo extraction""" 106 | 107 | from heudiconv.heuristics.convertall_custom import custom_seqinfo 108 | 109 | dcmfiles = glob(op.join(TESTS_DATA_PATH, "phantom.dcm")) 110 | 111 | seqinfos = group_dicoms_into_seqinfos( 112 | dcmfiles, "studyUID", flatten=True, custom_seqinfo=custom_seqinfo 113 | ) # type: ignore 114 | 115 | seqinfo = list(seqinfos.keys())[0] 116 | 117 | assert hasattr(seqinfo, "custom") 118 | assert isinstance(seqinfo.custom, tuple) 119 | assert len(seqinfo.custom) == 2 120 | assert seqinfo.custom[1] == dcmfiles[0] 121 | 122 | 123 | def test_get_datetime_from_dcm_from_acq_date_time() -> None: 124 | typical_dcm = dcm.dcmread( 125 | op.join(TESTS_DATA_PATH, "phantom.dcm"), stop_before_pixels=True 126 | ) 127 | 128 | # do we try to grab from AcquisitionDate/AcquisitionTime first when available? 129 | dt = get_datetime_from_dcm(typical_dcm) 130 | assert dt == datetime.datetime.strptime( 131 | typical_dcm.get("AcquisitionDate") + typical_dcm.get("AcquisitionTime"), 132 | "%Y%m%d%H%M%S.%f", 133 | ) 134 | 135 | 136 | def test_get_datetime_from_dcm_from_acq_datetime() -> None: 137 | # if AcquisitionDate and AcquisitionTime not there, can we rely on AcquisitionDateTime? 138 | XA30_enhanced_dcm = dcm.dcmread( 139 | op.join( 140 | TESTS_DATA_PATH, 141 | "MRI_102TD_PHA_S.MR.Chen_Matthews_1.3.1.2022.11.16.15.50.20.357.31204541.dcm", 142 | ), 143 | stop_before_pixels=True, 144 | ) 145 | dt = get_datetime_from_dcm(XA30_enhanced_dcm) 146 | assert dt == datetime.datetime.strptime( 147 | XA30_enhanced_dcm.get("AcquisitionDateTime"), "%Y%m%d%H%M%S.%f" 148 | ) 149 | 150 | 151 | def test_get_datetime_from_dcm_from_only_series_date_time() -> None: 152 | # if acquisition date/time/datetime not available, can we rely on SeriesDate & SeriesTime? 153 | XA30_enhanced_dcm = dcm.dcmread( 154 | op.join( 155 | TESTS_DATA_PATH, 156 | "MRI_102TD_PHA_S.MR.Chen_Matthews_1.3.1.2022.11.16.15.50.20.357.31204541.dcm", 157 | ), 158 | stop_before_pixels=True, 159 | ) 160 | del XA30_enhanced_dcm.AcquisitionDateTime 161 | dt = get_datetime_from_dcm(XA30_enhanced_dcm) 162 | assert dt == datetime.datetime.strptime( 163 | XA30_enhanced_dcm.get("SeriesDate") + XA30_enhanced_dcm.get("SeriesTime"), 164 | "%Y%m%d%H%M%S.%f", 165 | ) 166 | 167 | 168 | def test_get_datetime_from_dcm_wo_dt() -> None: 169 | # if there's no known source (e.g., after anonymization), are we still able to proceed? 170 | XA30_enhanced_dcm = dcm.dcmread( 171 | op.join( 172 | TESTS_DATA_PATH, 173 | "MRI_102TD_PHA_S.MR.Chen_Matthews_1.3.1.2022.11.16.15.50.20.357.31204541.dcm", 174 | ), 175 | stop_before_pixels=True, 176 | ) 177 | del XA30_enhanced_dcm.AcquisitionDateTime 178 | del XA30_enhanced_dcm.SeriesDate 179 | del XA30_enhanced_dcm.SeriesTime 180 | assert get_datetime_from_dcm(XA30_enhanced_dcm) is None 181 | 182 | 183 | @pytest.mark.parametrize("dcmfile", TEST_DICOM_PATHS) 184 | def test_create_seqinfo( 185 | dcmfile: str, 186 | ) -> None: 187 | mw = dw.wrapper_from_file(dcmfile) 188 | seqinfo = create_seqinfo(mw, [dcmfile], op.basename(dcmfile)) 189 | assert seqinfo.sequence_name 190 | 191 | 192 | @pytest.mark.parametrize("dcmfile", TEST_DICOM_PATHS) 193 | def test_get_reproducible_int(dcmfile: str) -> None: 194 | assert type(get_reproducible_int([dcmfile])) is int 195 | 196 | 197 | @pytest.mark.skip( 198 | reason="This test was mistakenly marked as a fixture, and removing the fixture decorator led to the test failing. Don't know how to fix." 199 | ) 200 | def test_get_reproducible_int_wo_dt(tmp_path: Path) -> None: 201 | # can this function return an int when we don't have any usable dates? 202 | typical_dcm = dcm.dcmread( 203 | op.join(TESTS_DATA_PATH, "phantom.dcm"), stop_before_pixels=True 204 | ) 205 | del typical_dcm.SeriesDate 206 | del typical_dcm.AcquisitionDate 207 | dcm.dcmwrite(tmp_path, typical_dcm) 208 | 209 | assert type(get_reproducible_int([str(tmp_path)])) is int 210 | 211 | 212 | @pytest.mark.skip( 213 | reason="This test was mistakenly marked as a fixture, and removing the fixture decorator led to the test failing. Don't know how to fix." 214 | ) 215 | def test_get_reproducible_int_raises_assertion_wo_dt(tmp_path: Path) -> None: 216 | # if there's no known source (e.g., after anonymization), is AssertionError Raised? 217 | XA30_enhanced_dcm = dcm.dcmread( 218 | op.join( 219 | TESTS_DATA_PATH, 220 | "MRI_102TD_PHA_S.MR.Chen_Matthews_1.3.1.2022.11.16.15.50.20.357.31204541.dcm", 221 | ), 222 | stop_before_pixels=True, 223 | ) 224 | del XA30_enhanced_dcm.AcquisitionDateTime 225 | del XA30_enhanced_dcm.SeriesDate 226 | del XA30_enhanced_dcm.SeriesTime 227 | dcm.dcmwrite(tmp_path, dataset=XA30_enhanced_dcm) 228 | with pytest.raises(AssertionError): 229 | get_reproducible_int([str(tmp_path)]) 230 | -------------------------------------------------------------------------------- /heudiconv/tests/test_heuristics.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import csv 4 | from glob import glob 5 | from io import StringIO 6 | import logging 7 | import os 8 | import os.path as op 9 | from os.path import dirname 10 | from os.path import join as pjoin 11 | from pathlib import Path 12 | import re 13 | from unittest.mock import patch 14 | 15 | import pytest 16 | 17 | from heudiconv.cli.run import main as runner 18 | 19 | from .utils import TESTS_DATA_PATH 20 | from .. import __version__ 21 | from ..bids import HEUDICONV_VERSION_JSON_KEY 22 | from ..utils import load_json 23 | 24 | lgr = logging.getLogger(__name__) 25 | 26 | try: 27 | from datalad.api import Dataset 28 | except ImportError: # pragma: no cover 29 | Dataset = None 30 | 31 | 32 | # this will fail if not in project's root directory 33 | def test_smoke_convertall(tmp_path: Path) -> None: 34 | args = [ 35 | "-c", 36 | "dcm2niix", 37 | "-o", 38 | str(tmp_path), 39 | "-b", 40 | "--datalad", 41 | "-s", 42 | "fmap_acq-3mm", 43 | "-d", 44 | f"{TESTS_DATA_PATH}/{{subject}}/*", 45 | ] 46 | 47 | # complain if no heurisitic 48 | with pytest.raises(RuntimeError): 49 | runner(args) 50 | 51 | args.extend(["-f", "convertall"]) 52 | runner(args) 53 | 54 | 55 | @pytest.mark.parametrize("heuristic", ["reproin", "convertall"]) 56 | @pytest.mark.parametrize( 57 | "invocation", 58 | [ 59 | ["--files", TESTS_DATA_PATH], # our new way with automated grouping 60 | ["-d", f"{TESTS_DATA_PATH}/{{subject}}/*", "-s", "01-fmap_acq-3mm"], 61 | # "old" way specifying subject 62 | # should produce the same results 63 | ], 64 | ) 65 | @pytest.mark.skipif(Dataset is None, reason="no datalad") 66 | def test_reproin_largely_smoke(tmp_path: Path, heuristic: str, invocation: str) -> None: 67 | is_bids = True if heuristic == "reproin" else False 68 | args = [ 69 | "--random-seed", 70 | "1", 71 | "-f", 72 | heuristic, 73 | "-c", 74 | "dcm2niix", 75 | "-o", 76 | str(tmp_path), 77 | ] 78 | if is_bids: 79 | args.append("-b") 80 | args.append("--datalad") 81 | args.extend(invocation) 82 | 83 | # Test some safeguards 84 | if invocation[0] == "--files": 85 | # Multiple subjects must not be specified -- only a single one could 86 | # be overridden from the command line 87 | with pytest.raises(ValueError): 88 | runner(args + ["--subjects", "sub1", "sub2"]) 89 | 90 | if heuristic != "reproin": 91 | # if subject is not overridden, raise error 92 | with pytest.raises(NotImplementedError): 93 | runner(args) 94 | return 95 | 96 | runner(args) 97 | ds = Dataset(tmp_path) 98 | assert ds.is_installed() 99 | assert not ds.repo.dirty 100 | head = ds.repo.get_hexsha() 101 | 102 | # and if we rerun -- should fail 103 | lgr.info( 104 | "RERUNNING, expecting to FAIL since the same everything " 105 | "and -c specified so we did conversion already" 106 | ) 107 | with pytest.raises(RuntimeError): 108 | runner(args) 109 | 110 | # but there should be nothing new 111 | assert not ds.repo.dirty 112 | # TODO: remove whenever https://github.com/datalad/datalad/issues/6843 113 | # is fixed/released 114 | buggy_datalad = (ds.pathobj / ".gitmodules").read_text().splitlines().count( 115 | '[submodule "Halchenko"]' 116 | ) > 1 117 | assert head == ds.repo.get_hexsha() or buggy_datalad 118 | 119 | # unless we pass 'overwrite' flag 120 | runner(args + ["--overwrite"]) 121 | # but result should be exactly the same, so it still should be clean 122 | # and at the same commit 123 | assert ds.is_installed() 124 | assert not ds.repo.dirty 125 | assert head == ds.repo.get_hexsha() or buggy_datalad 126 | 127 | 128 | @pytest.mark.parametrize( 129 | "invocation", 130 | [ 131 | ["--files", TESTS_DATA_PATH], # our new way with automated grouping 132 | ], 133 | ) 134 | def test_scans_keys_reproin(tmp_path: Path, invocation: list[str]) -> None: 135 | args = ["-f", "reproin", "-c", "dcm2niix", "-o", str(tmp_path), "-b"] 136 | args += invocation 137 | runner(args) 138 | # for now check it exists 139 | scans_keys = glob(pjoin(tmp_path, "*/*/*/*/*/*.tsv")) 140 | assert len(scans_keys) == 1 141 | with open(scans_keys[0]) as f: 142 | reader = csv.reader(f, delimiter="\t") 143 | for i, row in enumerate(reader): 144 | if i == 0: 145 | assert row == ["filename", "acq_time", "operator", "randstr"] 146 | assert len(row) == 4 147 | if i != 0: 148 | assert os.path.exists(pjoin(dirname(scans_keys[0]), row[0])) 149 | assert re.match( 150 | r"^[\d]{4}-[\d]{2}-[\d]{2}T[\d]{2}:[\d]{2}:[\d]{2}.[\d]{6}$", row[1] 151 | ) 152 | 153 | 154 | @patch("sys.stdout", new_callable=StringIO) 155 | def test_ls(stdout: StringIO) -> None: 156 | args = ["-f", "reproin", "--command", "ls", "--files", TESTS_DATA_PATH] 157 | runner(args) 158 | out = stdout.getvalue() 159 | assert "StudySessionInfo(locator=" in out 160 | assert "Halchenko/Yarik/950_bids_test4" in out 161 | 162 | 163 | def test_scout_conversion(tmp_path: Path) -> None: 164 | args = ["-b", "-f", "reproin", "--files", TESTS_DATA_PATH, "-o", str(tmp_path)] 165 | runner(args) 166 | 167 | dspath = tmp_path / "Halchenko/Yarik/950_bids_test4" 168 | sespath = dspath / "sub-phantom1sid1/ses-localizer" 169 | 170 | assert not (sespath / "anat").exists() 171 | assert ( 172 | dspath / "sourcedata/sub-phantom1sid1/ses-localizer/" 173 | "anat/sub-phantom1sid1_ses-localizer_scout.dicom.tgz" 174 | ).exists() 175 | 176 | # Let's do some basic checks on produced files 177 | j = load_json( 178 | sespath / "fmap/sub-phantom1sid1_ses-localizer_acq-3mm_phasediff.json" 179 | ) 180 | # We store HeuDiConv version in each produced .json file 181 | # TODO: test that we are not somehow overwriting that version in existing 182 | # files which we have not produced in a particular run. 183 | assert j[HEUDICONV_VERSION_JSON_KEY] == __version__ 184 | 185 | 186 | @pytest.mark.parametrize( 187 | "bidsoptions", 188 | [ 189 | ["notop"], 190 | [], 191 | ], 192 | ) 193 | def test_notop(tmp_path: Path, bidsoptions: list[str]) -> None: 194 | args = [ 195 | "-f", 196 | "reproin", 197 | "--files", 198 | TESTS_DATA_PATH, 199 | "-o", 200 | str(tmp_path), 201 | "-b", 202 | ] + bidsoptions 203 | runner(args) 204 | 205 | assert op.exists(pjoin(tmp_path, "Halchenko/Yarik/950_bids_test4")) 206 | for fname in [ 207 | "CHANGES", 208 | "dataset_description.json", 209 | "participants.tsv", 210 | "README", 211 | "participants.json", 212 | ]: 213 | if "notop" in bidsoptions: 214 | assert not op.exists( 215 | pjoin(tmp_path, "Halchenko/Yarik/950_bids_test4", fname) 216 | ) 217 | else: 218 | assert op.exists(pjoin(tmp_path, "Halchenko/Yarik/950_bids_test4", fname)) 219 | 220 | 221 | def test_phoenix_doc_conversion(tmp_path: Path) -> None: 222 | subID = "Phoenix" 223 | args = [ 224 | "-c", 225 | "dcm2niix", 226 | "-o", 227 | str(tmp_path), 228 | "-b", 229 | "-f", 230 | "bids_PhoenixReport", 231 | "--files", 232 | pjoin(TESTS_DATA_PATH, "Phoenix"), 233 | "-s", 234 | subID, 235 | ] 236 | runner(args) 237 | 238 | # check that the Phoenix document has been extracted (as gzipped dicom) in 239 | # the sourcedata/misc folder: 240 | assert op.exists( 241 | pjoin(tmp_path, "sourcedata", "sub-%s", "misc", "sub-%s_phoenix.dicom.tgz") 242 | % (subID, subID) 243 | ) 244 | # check that no "sub-/misc" folder has been created in the BIDS 245 | # structure: 246 | assert not op.exists(pjoin(tmp_path, "sub-%s", "misc") % subID) 247 | -------------------------------------------------------------------------------- /heudiconv/tests/test_monitor.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from collections.abc import Iterable, Iterator 4 | import os.path as op 5 | from pathlib import Path 6 | import sys 7 | from typing import TYPE_CHECKING, Any, Generic, NamedTuple, Optional, TypeVar 8 | from unittest.mock import patch 9 | 10 | import pytest 11 | 12 | try: 13 | from tinydb import Query, TinyDB 14 | except ImportError: 15 | pytest.importorskip("tinydb") 16 | from subprocess import CalledProcessError 17 | 18 | try: 19 | from heudiconv.cli.monitor import MASK_NEWDIR, monitor, process, run_heudiconv 20 | 21 | class Header(NamedTuple): 22 | wd: int 23 | mask: int 24 | cookie: int 25 | len: int 26 | 27 | header = Header(5, MASK_NEWDIR, 5, 5) 28 | watch_path = b"WATCHME" 29 | filename = b"FILE" 30 | type_names = b"TYPE" 31 | 32 | path2 = watch_path + b"/" + filename + b"/subpath" 33 | 34 | my_events = [ 35 | (header, type_names, watch_path, filename), 36 | (header, type_names, path2, b""), 37 | ] 38 | except AttributeError: 39 | # Import of inotify fails on mac os x with error 40 | # lsym(0x11fbeb780, inotify_init): symbol not found 41 | # because inotify doesn't seem to exist on Mac OS X 42 | my_events = [] 43 | pytestmark = pytest.mark.skip(reason="Unable to import inotify") 44 | 45 | 46 | if TYPE_CHECKING: 47 | if sys.version_info >= (3, 11): 48 | from typing import Self 49 | else: 50 | from typing_extensions import Self 51 | 52 | 53 | T = TypeVar("T") 54 | 55 | 56 | class MockInotifyTree(Generic[T]): 57 | def __init__(self, events: Iterable[T]) -> None: 58 | self.events: Iterator[T] = iter(events) 59 | 60 | def event_gen(self) -> Iterator[T]: 61 | for e in self.events: 62 | yield e 63 | 64 | def __call__(self, _topdir: Any) -> Self: 65 | return self 66 | 67 | 68 | class MockTime: 69 | def __init__(self, time: float) -> None: 70 | self.time = time 71 | 72 | def __call__(self) -> float: 73 | return self.time 74 | 75 | 76 | @pytest.mark.skip(reason="TODO") 77 | @patch("inotify.adapters.InotifyTree", MockInotifyTree(my_events)) 78 | @patch("time.time", MockTime(42)) 79 | def test_monitor(capsys: pytest.CaptureFixture[str]) -> None: 80 | monitor(watch_path.decode(), check_ptrn="") 81 | out, err = capsys.readouterr() 82 | desired_output = "{0}/{1} {2}\n".format(watch_path.decode(), filename.decode(), 42) 83 | desired_output += "Updating {0}/{1}: {2}\n".format( 84 | watch_path.decode(), filename.decode(), 42 85 | ) 86 | assert out == desired_output 87 | 88 | 89 | @pytest.mark.skip(reason="TODO") 90 | @patch("time.time", MockTime(42)) 91 | @pytest.mark.parametrize( 92 | "side_effect,success", [(None, 1), (CalledProcessError(1, "mycmd"), 0)] 93 | ) 94 | def test_process( 95 | tmp_path: Path, 96 | capsys: pytest.CaptureFixture[str], 97 | side_effect: Optional[BaseException], 98 | success: int, 99 | ) -> None: 100 | db_fn = tmp_path / "database.json" 101 | log_dir = tmp_path / "log" 102 | log_dir.mkdir() 103 | db = TinyDB(str(db_fn)) 104 | process_me = "/my/path/A12345" 105 | accession_number = op.basename(process_me) 106 | paths2process = {process_me: 42.0} 107 | with patch("subprocess.Popen") as mocked_popen: 108 | stdout = b"INFO: PROCESSING STARTS: {'just': 'a test'}" 109 | mocked_popen_instance = mocked_popen.return_value 110 | mocked_popen_instance.side_effect = side_effect 111 | mocked_popen_instance.communicate.return_value = (stdout,) 112 | # set return value for wait 113 | mocked_popen_instance.wait.return_value = 1 - success 114 | # mock also communicate to get the supposed stdout 115 | process(paths2process, db, wait=-30, logdir=str(log_dir)) 116 | out, err = capsys.readouterr() 117 | log_fn = log_dir / (accession_number + ".log") 118 | 119 | mocked_popen.assert_called_once() 120 | assert log_fn.exists() 121 | assert log_fn.read_text() == stdout.decode("utf-8") 122 | assert db_fn.exists() 123 | # dictionary should be empty 124 | assert not paths2process 125 | assert out == "Time to process {0}\n".format(process_me) 126 | 127 | # check what we have in the database 128 | path = Query() 129 | query = db.get(path.input_path == process_me) 130 | assert len(db) == 1 131 | assert query 132 | assert query["success"] == success 133 | assert query["accession_number"] == op.basename(process_me) 134 | assert query["just"] == "a test" 135 | 136 | 137 | @pytest.mark.skip(reason="TODO") 138 | def test_run_heudiconv() -> None: 139 | # echo should succeed always 140 | mydict = {"key1": "value1", "key2": "value2", "success": 1} 141 | args = ["echo", "INFO:", "PROCESSING", "STARTS:", str(mydict)] 142 | stdout, info_dict = run_heudiconv(args) 143 | assert info_dict == mydict 144 | assert stdout.strip() == " ".join(args[1:]) 145 | -------------------------------------------------------------------------------- /heudiconv/tests/test_queue.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from pathlib import Path 4 | import sys 5 | 6 | from nipype.utils.filemanip import which 7 | import pytest 8 | 9 | from heudiconv.cli.run import main as runner 10 | from heudiconv.queue import clean_args 11 | 12 | from .utils import TESTS_DATA_PATH 13 | 14 | 15 | @pytest.mark.skipif(bool(which("sbatch")), reason="skip a real slurm call") 16 | @pytest.mark.parametrize( 17 | "hargs", 18 | [ 19 | # our new way with automated grouping 20 | ["--files", f"{TESTS_DATA_PATH}/01-fmap_acq-3mm"], 21 | # "old" way specifying subject 22 | ["-d", f"{TESTS_DATA_PATH}/{{subject}}/*", "-s", "01-fmap_acq-3mm"], 23 | ], 24 | ) 25 | def test_queue_no_slurm( 26 | monkeypatch: pytest.MonkeyPatch, tmp_path: Path, hargs: list[str] 27 | ) -> None: 28 | monkeypatch.chdir(tmp_path) 29 | hargs.extend(["-f", "reproin", "-b", "--minmeta", "--queue", "SLURM"]) 30 | 31 | # simulate command-line call 32 | monkeypatch.setattr(sys, "argv", ["heudiconv"] + hargs) 33 | 34 | with pytest.raises(OSError): # SLURM should not be installed 35 | runner(hargs) 36 | # should have generated a slurm submission script 37 | slurm_cmd_file = str(tmp_path / "heudiconv-SLURM.sh") 38 | assert slurm_cmd_file 39 | # check contents and ensure args match 40 | with open(slurm_cmd_file) as fp: 41 | lines = fp.readlines() 42 | assert lines[0] == "#!/bin/bash\n" 43 | cmd = lines[1] 44 | 45 | # check that all flags we gave still being called 46 | for arg in hargs: 47 | # except --queue 48 | if arg in ["--queue", "SLURM"]: 49 | assert arg not in cmd 50 | else: 51 | assert arg in cmd 52 | 53 | 54 | def test_argument_filtering() -> None: 55 | cmd_files = [ 56 | "heudiconv", 57 | "--files", 58 | "/fake/path/to/files", 59 | "/another/fake/path", 60 | "-f", 61 | "convertall", 62 | "-q", 63 | "SLURM", 64 | "--queue-args", 65 | "--cpus-per-task=4 --contiguous --time=10", 66 | ] 67 | filtered = [ 68 | "heudiconv", 69 | "--files", 70 | "/another/fake/path", 71 | "-f", 72 | "convertall", 73 | ] 74 | assert clean_args(cmd_files, "files", 1) == filtered 75 | 76 | cmd_subjects = [ 77 | "heudiconv", 78 | "-d", 79 | "/some/{subject}/path", 80 | "--queue", 81 | "SLURM", 82 | "--subjects", 83 | "sub1", 84 | "sub2", 85 | "sub3", 86 | "sub4", 87 | "-f", 88 | "convertall", 89 | ] 90 | filtered = [ 91 | "heudiconv", 92 | "-d", 93 | "/some/{subject}/path", 94 | "--subjects", 95 | "sub3", 96 | "-f", 97 | "convertall", 98 | ] 99 | assert clean_args(cmd_subjects, "subjects", 2) == filtered 100 | -------------------------------------------------------------------------------- /heudiconv/tests/test_regression.py: -------------------------------------------------------------------------------- 1 | """Testing conversion with conversion saved on datalad""" 2 | from __future__ import annotations 3 | 4 | from glob import glob 5 | import os 6 | import os.path as op 7 | from pathlib import Path 8 | import re 9 | from typing import Optional 10 | 11 | import pydicom as dcm 12 | import pytest 13 | 14 | from heudiconv.cli.run import main as runner 15 | from heudiconv.parser import find_files 16 | from heudiconv.utils import load_json 17 | 18 | # testing utilities 19 | from .utils import TESTS_DATA_PATH, fetch_data, gen_heudiconv_args 20 | 21 | have_datalad = True 22 | try: 23 | from datalad.support.exceptions import IncompleteResultsError 24 | except ImportError: 25 | have_datalad = False 26 | 27 | 28 | @pytest.mark.skipif(not have_datalad, reason="no datalad") 29 | @pytest.mark.parametrize("subject", ["sid000143"]) 30 | @pytest.mark.parametrize("heuristic", ["reproin.py"]) 31 | @pytest.mark.parametrize("anon_cmd", [None, "anonymize_script.py"]) 32 | def test_conversion( 33 | monkeypatch: pytest.MonkeyPatch, 34 | tmp_path: Path, 35 | subject: str, 36 | heuristic: str, 37 | anon_cmd: Optional[str], 38 | ) -> None: 39 | monkeypatch.chdir(tmp_path) 40 | try: 41 | datadir = fetch_data( 42 | tmp_path, 43 | "dbic/QA", # path from datalad database root 44 | getpath=op.join("sourcedata", f"sub-{subject}"), 45 | ) 46 | except IncompleteResultsError as exc: 47 | pytest.skip("Failed to fetch test data: %s" % str(exc)) 48 | outdir = tmp_path / "out" 49 | outdir.mkdir() 50 | 51 | args = gen_heudiconv_args( 52 | datadir, 53 | str(outdir), 54 | subject, 55 | heuristic, 56 | anon_cmd, 57 | template="sourcedata/sub-{subject}/*/*/*.tgz", 58 | xargs=["--datalad"], 59 | ) 60 | runner(args) # run conversion 61 | 62 | # Get the possibly anonymized subject id and verify that it was 63 | # anonymized or not: 64 | subjects_maybe_anon = glob(f"{outdir}/sub-*") 65 | assert len(subjects_maybe_anon) == 1 # just one should be there 66 | subject_maybe_anon = op.basename(subjects_maybe_anon[0])[4:] 67 | 68 | if anon_cmd: 69 | assert subject_maybe_anon != subject 70 | else: 71 | assert subject_maybe_anon == subject 72 | 73 | # verify functionals were converted 74 | outfiles = sorted( 75 | [ 76 | f[len(str(outdir)) :] 77 | for f in glob(f"{outdir}/sub-{subject_maybe_anon}/func/*") 78 | ] 79 | ) 80 | assert outfiles 81 | datafiles = sorted( 82 | [f[len(datadir) :] for f in glob(f"{datadir}/sub-{subject}/ses-*/func/*")] 83 | ) 84 | # original data has ses- but because we are converting only func, and not 85 | # providing any session, we will not "match". Let's strip away the session 86 | datafiles = [re.sub(r"[/\\_]ses-[^/\\_]*", "", f) for f in datafiles] 87 | if not anon_cmd: 88 | assert outfiles == datafiles 89 | else: 90 | assert outfiles != datafiles # sid was anonymized 91 | assert len(outfiles) == len(datafiles) # but we have the same number of files 92 | 93 | # compare some json metadata 94 | json_ = "{}/task-rest_acq-24mm64sl1000tr32te600dyn_bold.json".format 95 | orig, conv = (load_json(json_(datadir)), load_json(json_(outdir))) 96 | keys = ["EchoTime", "MagneticFieldStrength", "Manufacturer", "SliceTiming"] 97 | for key in keys: 98 | assert orig[key] == conv[key] 99 | 100 | # validate sensitive marking 101 | from datalad.api import Dataset 102 | 103 | ds = Dataset(outdir) 104 | all_meta = dict(ds.repo.get_metadata(".")) 105 | target_rec = {"distribution-restrictions": ["sensitive"]} 106 | for pth, meta in all_meta.items(): 107 | if "anat" in pth or "scans.tsv" in pth: 108 | assert meta == target_rec 109 | else: 110 | assert meta == {} 111 | 112 | 113 | @pytest.mark.skipif(not have_datalad, reason="no datalad") 114 | def test_multiecho( 115 | monkeypatch: pytest.MonkeyPatch, 116 | tmp_path: Path, 117 | subject: str = "MEEPI", 118 | heuristic: str = "bids_ME.py", 119 | ) -> None: 120 | monkeypatch.chdir(tmp_path) 121 | try: 122 | datadir = fetch_data(tmp_path, "dicoms/velasco/MEEPI") 123 | except IncompleteResultsError as exc: 124 | pytest.skip("Failed to fetch test data: %s" % str(exc)) 125 | 126 | outdir = tmp_path / "out" 127 | outdir.mkdir() 128 | args = gen_heudiconv_args(datadir, str(outdir), subject, heuristic) 129 | runner(args) # run conversion 130 | 131 | # check if we have echo functionals 132 | echoes = glob(op.join("out", "sub-" + subject, "func", "*echo*nii.gz")) 133 | assert len(echoes) == 3 134 | 135 | # check EchoTime of each functional 136 | # ET1 < ET2 < ET3 137 | prev_echo = 0 138 | for echo in sorted(echoes): 139 | _json = echo.replace(".nii.gz", ".json") 140 | assert _json 141 | echotime = load_json(_json).get("EchoTime", None) 142 | assert echotime > prev_echo 143 | prev_echo = echotime 144 | 145 | events = glob(op.join("out", "sub-" + subject, "func", "*events.tsv")) 146 | for event in events: 147 | assert "echo-" not in event 148 | 149 | 150 | @pytest.mark.parametrize("subject", ["merged"]) 151 | def test_grouping(tmp_path: Path, subject: str) -> None: 152 | dicoms = [op.join(TESTS_DATA_PATH, fl) for fl in ["axasc35.dcm", "phantom.dcm"]] 153 | # ensure DICOMs are different studies 154 | studyuids = { 155 | dcm.dcmread(fl, stop_before_pixels=True).StudyInstanceUID for fl in dicoms 156 | } 157 | assert len(studyuids) == len(dicoms) 158 | # symlink to common location 159 | outdir = tmp_path / "out" 160 | outdir.mkdir() 161 | datadir = tmp_path / subject 162 | datadir.mkdir() 163 | for fl in dicoms: 164 | os.symlink(fl, datadir / op.basename(fl)) 165 | 166 | template = op.join("{subject}/*.dcm") 167 | hargs = gen_heudiconv_args( 168 | str(tmp_path), str(outdir), subject, "convertall.py", template=template 169 | ) 170 | 171 | with pytest.raises(AssertionError): 172 | runner(hargs) 173 | 174 | # group all found DICOMs under subject, despite conflicts 175 | hargs += ["-g", "all"] 176 | runner(hargs) 177 | assert ( 178 | len( 179 | list( 180 | find_files( 181 | rf"(^|{re.escape(os.sep)})run0", 182 | str(outdir), 183 | exclude_vcs=False, 184 | dirs=True, 185 | ) 186 | ) 187 | ) 188 | == 4 189 | ) 190 | tsv = outdir / "participants.tsv" 191 | assert tsv.exists() 192 | lines = tsv.open().readlines() 193 | assert len(lines) == 2 194 | assert lines[1].split("\t")[0] == "sub-{}".format(subject) 195 | -------------------------------------------------------------------------------- /heudiconv/tests/test_tarballs.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from glob import glob 4 | import os 5 | from os.path import join as opj 6 | from pathlib import Path 7 | import time 8 | 9 | from heudiconv.dicoms import compress_dicoms 10 | from heudiconv.utils import TempDirs, file_md5sum 11 | 12 | from .utils import TESTS_DATA_PATH 13 | 14 | 15 | def test_reproducibility(tmp_path: Path) -> None: 16 | dicom_list = glob(opj(TESTS_DATA_PATH, "01-fmap_acq-3mm", "*")) 17 | prefix = str(tmp_path / "precious") 18 | tempdirs = TempDirs() 19 | 20 | tarball = compress_dicoms(dicom_list, prefix, tempdirs, True) 21 | assert tarball is not None 22 | md5 = file_md5sum(tarball) 23 | # must not override, ensure overwrite is set to False 24 | assert compress_dicoms(dicom_list, prefix, tempdirs, False) is None 25 | 26 | os.unlink(tarball) 27 | 28 | time.sleep(1.1) # need to guarantee change of time 29 | tarball_ = compress_dicoms(dicom_list, prefix, tempdirs, True) 30 | assert tarball == tarball_ 31 | md5_ = file_md5sum(tarball_) 32 | assert md5 == md5_ 33 | -------------------------------------------------------------------------------- /heudiconv/tests/utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from glob import glob 4 | import logging 5 | import os.path as op 6 | from pathlib import Path 7 | from typing import Optional 8 | 9 | import heudiconv.heuristics 10 | 11 | HEURISTICS_PATH = op.join(heudiconv.heuristics.__path__[0]) 12 | TESTS_DATA_PATH = op.join(op.dirname(__file__), "data") 13 | # Do relative to curdir to shorten in a typical application, 14 | # and side-effect test that tests do not change curdir. 15 | TEST_DICOM_PATHS = [ 16 | op.relpath(x) 17 | for x in glob(op.join(TESTS_DATA_PATH, "**/*.dcm"), recursive=True) 18 | # exclude PhoenixDocuments 19 | if "PhoenixDocument" not in x 20 | ] 21 | 22 | lgr = logging.getLogger(__name__) 23 | 24 | 25 | def gen_heudiconv_args( 26 | datadir: str, 27 | outdir: str, 28 | subject: str, 29 | heuristic_file: str, 30 | anon_cmd: Optional[str] = None, 31 | template: Optional[str] = None, 32 | xargs: Optional[list[str]] = None, 33 | ) -> list[str]: 34 | heuristic = op.realpath(op.join(HEURISTICS_PATH, heuristic_file)) 35 | 36 | if template: 37 | # use --dicom_dir_template 38 | args = ["-d", op.join(datadir, template)] 39 | else: 40 | args = ["--files", datadir] 41 | 42 | args.extend( 43 | [ 44 | "-c", 45 | "dcm2niix", 46 | "-o", 47 | outdir, 48 | "-s", 49 | subject, 50 | "-f", 51 | heuristic, 52 | "--bids", 53 | "--minmeta", 54 | ] 55 | ) 56 | if anon_cmd: 57 | args += ["--anon-cmd", op.join(op.dirname(__file__), anon_cmd), "-a", outdir] 58 | if xargs: 59 | args += xargs 60 | 61 | return args 62 | 63 | 64 | def fetch_data(tmpdir: str | Path, dataset: str, getpath: Optional[str] = None) -> str: 65 | """ 66 | Utility function to interface with datalad database. 67 | Performs datalad `install` and datalad `get` operations. 68 | 69 | Parameters 70 | ---------- 71 | tmpdir : str or Path 72 | directory to temporarily store data 73 | dataset : str 74 | dataset path from `http://datasets-tests.datalad.org` 75 | getpath : str [optional] 76 | exclusive path to get 77 | 78 | Returns 79 | ------- 80 | targetdir : str 81 | directory with installed dataset 82 | """ 83 | from datalad import api 84 | 85 | targetdir = op.join(tmpdir, op.basename(dataset)) 86 | ds = api.install( 87 | path=targetdir, source="http://datasets-tests.datalad.org/{}".format(dataset) 88 | ) 89 | 90 | getdir = targetdir + (op.sep + getpath if getpath is not None else "") 91 | ds.get(getdir) 92 | return targetdir 93 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | allow_incomplete_defs = False 3 | allow_untyped_defs = False 4 | no_implicit_optional = True 5 | #implicit_reexport = False 6 | local_partial_types = True 7 | pretty = True 8 | show_error_codes = True 9 | show_traceback = True 10 | strict_equality = True 11 | warn_redundant_casts = True 12 | warn_return_any = True 13 | warn_unreachable = True 14 | exclude = due\.py 15 | 16 | [mypy-heudiconv.due] 17 | follow_imports = skip 18 | 19 | [mypy-datalad.*] 20 | ignore_missing_imports = True 21 | 22 | [mypy-dcmstack.*] 23 | ignore_missing_imports = True 24 | 25 | [mypy-duecredit.*] 26 | ignore_missing_imports = True 27 | 28 | [mypy-etelemetry.*] 29 | ignore_missing_imports = True 30 | 31 | [mypy-inotify.*] 32 | ignore_missing_imports = True 33 | 34 | [mypy-nibabel.*] 35 | # The CI runs type-checking using Python 3.7, yet nibabel only added type 36 | # annotations in v5.1.0, which requires Python 3.8+. 37 | ignore_missing_imports = True 38 | 39 | [mypy-nipype.*] 40 | ignore_missing_imports = True 41 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools >= 46.4.0", 4 | "versioningit ~= 2.3", 5 | "wheel ~= 0.32" 6 | ] 7 | build-backend = "setuptools.build_meta" 8 | 9 | [tool.versioningit.write] 10 | file = "heudiconv/_version.py" 11 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | .[all] 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 3 | # 4 | # See COPYING file distributed along with the Heudiconv package for the 5 | # copyright and license terms. 6 | # 7 | # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 8 | 9 | 10 | def main(): 11 | import os.path as op 12 | 13 | from setuptools import find_packages, findall, setup 14 | 15 | thispath = op.dirname(__file__) 16 | ldict = locals() 17 | 18 | # Get version and release info, which is all stored in heudiconv/info.py 19 | info_file = op.join(thispath, "heudiconv", "info.py") 20 | with open(info_file) as infofile: 21 | exec(infofile.read(), globals(), ldict) 22 | 23 | try: 24 | import versioningit # noqa: F401 25 | except ImportError: 26 | # versioningit isn't installed; assume we're building a Debian package 27 | # from an sdist on an older Debian that doesn't support pybuild 28 | vglobals = {} 29 | with open(op.join(op.dirname(__file__), "heudiconv", "_version.py")) as fp: 30 | exec(fp.read(), vglobals) 31 | kwargs = {"version": vglobals["__version__"]} 32 | else: 33 | kwargs = {} 34 | 35 | def findsome(subdir, extensions): 36 | """Find files under subdir having specified extensions 37 | 38 | Leading directory (datalad) gets stripped 39 | """ 40 | return [ 41 | f.split(op.sep, 1)[1] 42 | for f in findall(subdir) 43 | if op.splitext(f)[-1].lstrip(".") in extensions 44 | ] 45 | 46 | # Only recentish versions of find_packages support include 47 | # heudiconv_pkgs = find_packages('.', include=['heudiconv*']) 48 | # so we will filter manually for maximal compatibility 49 | heudiconv_pkgs = [pkg for pkg in find_packages(".") if pkg.startswith("heudiconv")] 50 | 51 | setup( 52 | name=ldict["__packagename__"], 53 | author=ldict["__author__"], 54 | # author_email="team@???", 55 | description=ldict["__description__"], 56 | long_description=ldict["__longdesc__"], 57 | license=ldict["__license__"], 58 | classifiers=ldict["CLASSIFIERS"], 59 | packages=heudiconv_pkgs, 60 | entry_points={ 61 | "console_scripts": [ 62 | "heudiconv=heudiconv.cli.run:main", 63 | "heudiconv_monitor=heudiconv.cli.monitor:main", 64 | ] 65 | }, 66 | python_requires=ldict["PYTHON_REQUIRES"], 67 | install_requires=ldict["REQUIRES"], 68 | extras_require=ldict["EXTRA_REQUIRES"], 69 | package_data={ 70 | "heudiconv": ["py.typed"], 71 | "heudiconv.tests": [ 72 | op.join("data", "*.dcm"), 73 | op.join("data", "*", "*.dcm"), 74 | op.join("data", "*", "*", "*.dcm"), 75 | op.join("data", "sample_nifti*"), 76 | ], 77 | }, 78 | **kwargs, 79 | ) 80 | 81 | 82 | if __name__ == "__main__": 83 | main() 84 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = lint,typing,py3 3 | 4 | [testenv] 5 | extras = all 6 | commands = pytest -s -v {posargs} heudiconv 7 | 8 | [testenv:lint] 9 | skip_install = True 10 | deps = 11 | codespell 12 | flake8 13 | flake8-bugbear 14 | flake8-builtins 15 | flake8-unused-arguments 16 | commands = 17 | flake8 heudiconv 18 | codespell 19 | 20 | [testenv:typing] 21 | deps = 22 | mypy 23 | extras = all 24 | commands = 25 | mypy heudiconv 26 | 27 | [pytest] 28 | # monitor.py requires optional linotify, but would blow tests discovery, does not contain tests within 29 | addopts = --doctest-modules --tb=short --ignore heudiconv/cli/monitor.py 30 | filterwarnings = 31 | error 32 | # 33 | ignore:module 'sre_.*' is deprecated:DeprecationWarning:traits 34 | # pytest generates a number of inscrutable warnings about open files never 35 | # being closed. I (jwodder) expect these are due to DataLad not shutting 36 | # down batch processes prior to garbage collection. 37 | ignore::pytest.PytestUnraisableExceptionWarning 38 | # I don't understand why this warning occurs, as we're using six 1.16, 39 | # which has the named method. 40 | ignore:_SixMetaPathImporter.find_spec\(\) not found:ImportWarning 41 | # 42 | ignore:.*pkg_resources:DeprecationWarning 43 | # 44 | ignore:.*Use setlocale.* instead:DeprecationWarning:nipype 45 | # 46 | ignore:.*datetime.datetime.utcnow\(\) is deprecated.*:DeprecationWarning:nipype 47 | 48 | [coverage:run] 49 | include = heudiconv/* 50 | setup.py 51 | 52 | [flake8] 53 | doctests = True 54 | exclude = .*/,build/,dist/,test/data,venv/ 55 | hang-closing = False 56 | unused-arguments-ignore-stub-functions = True 57 | select = A,B,B902,C,E,E242,F,U100,W 58 | ignore = A003,B005,E203,E262,E266,E501,W503 59 | 60 | [isort] 61 | atomic = True 62 | force_sort_within_sections = True 63 | honor_noqa = True 64 | lines_between_sections = 1 65 | profile = black 66 | reverse_relative = True 67 | sort_relative_in_force_sorted_sections = True 68 | known_first_party = heudiconv 69 | -------------------------------------------------------------------------------- /utils/anon-cmd: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Generic anonymization script which would anonymize sid based on what it had 3 | # seen in the past or simply what the translation dict already has. 4 | 5 | set -eu 6 | 7 | debug() { 8 | : echo "DEBUG: $*" >&2 9 | } 10 | 11 | # Translation file location 12 | # Store under .git by default to guarantee that it is not committed or locked by git-annex etc 13 | # But it might not fit some usecases where there is no .git 14 | anon_file_default=$(dirname "$0")/../.git/anon_sid_map.csv 15 | anon_file="${AC_ANON_FILE:-$anon_file_default}" 16 | anon_fmt="${AC_ANON_FMT:-%03d}" 17 | 18 | sid="$1" 19 | 20 | # harmonize since elderly awk on rolando seems to have no clue about IGNORECASE 21 | sid=$(echo "$sid" | tr '[:lower:]' '[:upper:]') 22 | 23 | debug "Using $anon_file to map $sid" 24 | 25 | if [ ! -e "$anon_file" ]; then 26 | touch "$anon_file" # initiate it 27 | fi 28 | 29 | # apparently heudiconv passes even those we provided in `-s` CLI option 30 | # to anonymization script. So, we will have to match those by our format 31 | # and then give back if matches. That would forbid plain remapping though if 32 | # original ids are in the same format, so some folks might want to disable that! 33 | sid_input_fmted=$(echo "$sid" | sed -e 's,^0*,,g' | xargs printf "$anon_fmt" 2>&1 || :) 34 | if [ "$sid" = "$sid_input_fmted" ]; then 35 | debug already in the anonymized format 36 | echo "$sid" 37 | exit 0 38 | fi 39 | 40 | res=$(grep "^$sid," "$anon_file" | head -n 1) 41 | if [ -n "$res" ]; then 42 | ann="${res##*,}" 43 | debug "Found $ann in '$res'" 44 | else 45 | echo "We have all sids mapped already! Will not create a new one for $sid" >&2; exit 1 46 | # need to take the latest one 47 | largest=$(sed -e 's/.*,//g' "$anon_file" | sort -n | tail -n1 | sed -e 's,^0*,,g') 48 | next=$((largest+1)) 49 | # shellcheck disable=SC2059 50 | ann=$(printf "$anon_fmt" $next) 51 | debug "Found $largest and $next to get $ann, storing" 52 | echo "$sid,$ann" >> "$anon_file" 53 | fi 54 | echo "$ann" 55 | -------------------------------------------------------------------------------- /utils/gen-docker-image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu 4 | 5 | thisd=$(dirname $0) 6 | VER=$(grep -Po '(?<=^__version__ = ).*' $thisd/../heudiconv/info.py | sed 's/"//g') 7 | 8 | image="kaczmarj/neurodocker:0.9.1" 9 | 10 | if hash podman; then 11 | OCI_BINARY=podman 12 | elif hash docker; then 13 | OCI_BINARY=docker 14 | else 15 | echo "ERROR: no podman or docker found" >&2 16 | exit 1 17 | fi 18 | 19 | ${OCI_BINARY:-docker} run --rm $image generate docker \ 20 | --base-image neurodebian:bookworm \ 21 | --pkg-manager apt \ 22 | --dcm2niix \ 23 | version=v1.0.20240202 \ 24 | method=source \ 25 | cmake_opts="-DZLIB_IMPLEMENTATION=Cloudflare -DUSE_JPEGLS=ON -DUSE_OPENJPEG=ON" \ 26 | --install \ 27 | git \ 28 | gcc \ 29 | pigz \ 30 | liblzma-dev \ 31 | libc-dev \ 32 | git-annex-standalone \ 33 | netbase \ 34 | --copy . /src/heudiconv \ 35 | --miniconda \ 36 | version="py39_4.12.0" \ 37 | conda_install="python=3.9 traits>=4.6.0 scipy numpy nomkl pandas gdcm" \ 38 | pip_install="/src/heudiconv[all]" \ 39 | pip_opts="--editable" \ 40 | --entrypoint "heudiconv" \ 41 | > $thisd/../Dockerfile 42 | -------------------------------------------------------------------------------- /utils/link_issues_CHANGELOG: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | in=CHANGELOG.md 4 | 5 | # Replace them with Markdown references 6 | sed -i -e 's/(\(#[0-9]\+\))/([\1][])/g' "$in" 7 | 8 | # Populate references 9 | tr ' ,' '\n\n' < "$in" | sponge | sed -n -e 's/.*(\[#\([0-9]\+\)\]\(\[\]*\)).*/\1/gp' | sort | uniq \ 10 | | while read issue; do 11 | #echo "issue $issue" 12 | # remove old one if exists 13 | sed -i -e "/^\[#$issue\]:.*/d" "$in" 14 | echo "[#$issue]: https://github.com/nipy/heudiconv/issues/$issue" >> "$in"; 15 | done 16 | -------------------------------------------------------------------------------- /utils/sensor-dicoms: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu 4 | 5 | # Function to show usage 6 | show_usage() { 7 | echo "Usage: $0 [--dry-run|-n] --move-to DIRNAME directory [directory2 ...]" 8 | } 9 | 10 | # On OSX we better use GNU one 11 | if [ -e /usr/local/opt/gnu-getopt/bin/getopt ]; then 12 | getopt=/usr/local/opt/gnu-getopt/bin/getopt 13 | else 14 | getopt=getopt 15 | fi 16 | 17 | # Parsing options 18 | TEMP=$("$getopt" -o 'n' --long move-to:,dry-run -n "$(basename "$0")" -- "$@") 19 | # shellcheck disable=SC2181 20 | if [ $? != 0 ]; then echo "Terminating..." >&2; exit 1; fi 21 | 22 | # Note the quotes around `$TEMP`: they are essential! 23 | eval set -- "$TEMP" 24 | 25 | # Initialize variables 26 | MOVE_TO_DIR="" 27 | DRY_RUN="" 28 | 29 | # Extract options and their arguments into variables 30 | while true; do 31 | case "$1" in 32 | --move-to) 33 | MOVE_TO_DIR="$2" 34 | shift 2 35 | ;; 36 | -n|--dry-run) 37 | DRY_RUN=1 38 | shift 39 | ;; 40 | --) 41 | shift 42 | break 43 | ;; 44 | *) 45 | echo "Internal error!" 46 | exit 1 47 | ;; 48 | esac 49 | done 50 | 51 | # Check for mandatory option 52 | if [ -z "$MOVE_TO_DIR" ]; then 53 | echo "Error: --move-to option is required." 54 | show_usage 55 | exit 1 56 | fi 57 | 58 | # Create MOVE_TO_DIR if it does not exist 59 | if [ ! -d "$MOVE_TO_DIR" ]; then 60 | mkdir -p "$MOVE_TO_DIR" 61 | fi 62 | 63 | TEMP=$(mktemp -d "${TMPDIR:-/tmp}/dl-XXXXXXX") 64 | 65 | # Process the remaining arguments (directories) 66 | for dir in "$@"; do 67 | echo "" 68 | echo "Processing directory: $dir" 69 | rm -rf "${TEMP:?}/*" 70 | failed= 71 | dcm2niix -z y -b y -o "$TEMP/" "$dir" 2>"$TEMP/stderr" >"$TEMP/stdout" || { 72 | echo " Exited with $?; We will proceed with the analysis. Standard error output was:" 73 | sed -e 's,^, ,g' "$TEMP/stderr" 74 | } 75 | 76 | if grep "Error: Check sorted order: 4D dataset has" "$TEMP/stderr"; then 77 | failed=1 78 | fi 79 | if grep "Error: Missing images." "$TEMP/stderr"; then 80 | failed=1 81 | fi 82 | if [ -n "$failed" ]; then 83 | if [ -n "$DRY_RUN" ]; then 84 | echo mv "$dir" "$MOVE_TO_DIR" 85 | else 86 | echo " Moving $dir to $MOVE_TO_DIR" 87 | mv "$dir" "$MOVE_TO_DIR" 88 | fi 89 | fi 90 | done 91 | -------------------------------------------------------------------------------- /utils/test-compare-two-versions.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # A script which is for now very ad-hoc and to be ran outside of this codebase and 3 | # be provided with two repos of heudiconv, 4 | # with virtualenvs setup inside under venvs/dev3. 5 | # Was used for https://github.com/nipy/heudiconv/pull/129 6 | # 7 | # Sample invocation 8 | # $> datalad install -g ///dicoms/dartmouth-phantoms/bids_test4-20161014/phantom-1 9 | # $> heudiconv/utils/test-compare-two-versions.sh heudiconv-{0.5.x,master} --bids -f reproin --files dartmouth-phantoms/bids_test4-20161014/phantom-1 10 | # where heudiconv-0.5.x and heudiconv-master have two worktrees with different 11 | # branches checked out and envs/dev3 environments in each 12 | 13 | PS1=+ 14 | set -eu 15 | 16 | outdir=${OUTDIR:=compare-versions} 17 | 18 | RUN=echo 19 | RUN=time 20 | 21 | 22 | function run() { 23 | heudiconvdir="$1" 24 | out=$outdir/$2 25 | shift 26 | shift 27 | source $heudiconvdir/venvs/dev3/bin/activate 28 | whichheudiconv=$(which heudiconv) 29 | # to get "reproducible" dataset UUIDs (might be detrimental if we had multiple datalad calls 30 | # but since we use python API for datalad, should be Ok) 31 | export DATALAD_SEED=1 32 | 33 | 34 | if [ ! -e "$out" ]; then 35 | # just do full conversion 36 | echo "Running $whichheudiconv with log in $out.log" 37 | $RUN heudiconv --random-seed 1 -o $out "$@" >| $out.log 2>&1 \ 38 | || { 39 | echo "Exited with $? Check $out.log" >&2 40 | exit $? 41 | } 42 | else 43 | echo "Not running heudiconv since $out already exists" 44 | fi 45 | } 46 | 47 | d1=$1; v1=$(git -C "$d1" describe); shift 48 | d2=$1; v2=$(git -C "$d2" describe); shift 49 | diff="$v1-$v2.diff" 50 | 51 | function show_diff() { 52 | cd $outdir 53 | diff_full="$PWD/$diff" 54 | #git remote add rolando "$outdir/rolando" 55 | #git fetch rolando 56 | # git diff --stat rolando/master.. 57 | if diff -Naur --exclude=.git --ignore-matching-lines='^\s*\(id\s*=.*\|"HeudiconvVersion": \)' "$v1" "$v2" >| "$diff_full"; then 58 | echo "Results are identical" 59 | else 60 | echo "Results differ: $diff_full" 61 | cat "$diff_full" | diffstat 62 | fi 63 | if hash xsel; then 64 | echo "$diff_full" | xsel -i 65 | fi 66 | } 67 | 68 | mkdir -p $outdir 69 | 70 | if [ ! -e "$outdir/$diff" ]; then 71 | run "$d1" "$v1" "$@" 72 | run "$d2" "$v2" "$@" 73 | fi 74 | 75 | show_diff 76 | -------------------------------------------------------------------------------- /utils/update_changes.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Adapted from https://github.com/nipy/nipype/blob/master/tools/update_changes.sh 4 | # 5 | # This is a script to be run before releasing a new version. 6 | # 7 | # Usage /bin/bash update_changes.sh 0.5.1 8 | # 9 | 10 | # Setting # $ help set 11 | set -u # Treat unset variables as an error when substituting. 12 | set -x # Print command traces before executing command. 13 | 14 | CHANGES=../CHANGELOG.md 15 | 16 | 17 | # Add changelog documentation 18 | cat > newchanges <<'_EOF' 19 | # Changelog 20 | All notable changes to this project will be documented (for humans) in this file. 21 | 22 | The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) 23 | and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). 24 | 25 | _EOF 26 | 27 | # List all merged PRs 28 | curl -s https://api.github.com/repos/nipy/heudiconv/pulls?state=closed+milestone=$1 | jq -r \ 29 | '.[] | "\(.title) #\(.number) milestone:\(.milestone.title) \(.merged_at)"' | sed '/null/d' | sed '/milestone:0.5 /d' >> newchanges 30 | echo "" >> newchanges 31 | echo "" >> newchanges 32 | 33 | 34 | # Elaborate today's release header 35 | HEADER="## [$1] - $(date '+%Y-%m-%d')" 36 | echo $HEADER >> newchanges 37 | echo "TODO Summary" >> newchanges 38 | echo "### Added" >> newchanges 39 | echo "" >> newchanges 40 | echo "### Changed" >> newchanges 41 | echo "" >> newchanges 42 | echo "### Deprecated" >> newchanges 43 | echo "" >> newchanges 44 | echo "### Fixed" >> newchanges 45 | echo "" >> newchanges 46 | echo "### Removed" >> newchanges 47 | echo "" >> newchanges 48 | echo "### Security" >> newchanges 49 | echo "" >> newchanges 50 | 51 | # Append old CHANGES 52 | tail -n+7 $CHANGES >> newchanges 53 | 54 | # Replace old CHANGES with new file 55 | mv newchanges $CHANGES 56 | --------------------------------------------------------------------------------