├── .dockerignore ├── .flake8 ├── .github └── workflows │ ├── build.yml │ └── docs.yml ├── .gitignore ├── .isort.cfg ├── .pre-commit-config.yaml ├── .vscode ├── launch.json ├── settings.json └── tasks.json ├── AUTHORS.md ├── CHANGELOG.md ├── CONTRIBUTING.md ├── CONTRIBUTORS.md ├── Dockerfile ├── LICENSE ├── MANIFEST.in ├── README.md ├── conftest.py ├── docker-compose.yml ├── docs ├── authors.md ├── changelog.md ├── cli │ ├── apps.md │ ├── classification.md │ ├── export.md │ ├── intro.md │ ├── measurement.md │ ├── preprocessing.md │ ├── segmentation.md │ ├── utils.md │ └── visualization.md ├── contributing.md ├── contributors.md ├── directories.md ├── file-types.md ├── img │ ├── steinbock-favicon.png │ ├── steinbock-logo-white.png │ └── steinbock-logo.png ├── index.md ├── install-docker.md ├── install-python.md ├── license.md ├── mkdocstrings.css └── python │ ├── api │ ├── steinbock.classification.md │ ├── steinbock.export.md │ ├── steinbock.io.md │ ├── steinbock.measurement.md │ ├── steinbock.preprocessing.md │ ├── steinbock.segmentation.md │ ├── steinbock.utils.md │ └── steinbock.visualization.md │ └── intro.md ├── entrypoint.sh ├── fixuid.yml ├── mkdocs.yml ├── pyproject.toml ├── requirements.txt ├── requirements_devel.txt ├── requirements_docs.txt ├── requirements_test.txt ├── setup.cfg ├── steinbock ├── __init__.py ├── __main__.py ├── _cli │ ├── __init__.py │ ├── _cli.py │ ├── apps.py │ ├── utils.py │ └── visualization.py ├── _env.py ├── _steinbock.py ├── classification │ ├── __init__.py │ ├── _classification.py │ ├── _cli │ │ ├── __init__.py │ │ ├── _cli.py │ │ └── ilastik.py │ └── ilastik │ │ ├── __init__.py │ │ ├── _ilastik.py │ │ └── data │ │ ├── __init__.py │ │ └── pixel_classifier.ilp ├── export │ ├── __init__.py │ ├── _cli │ │ ├── __init__.py │ │ ├── _cli.py │ │ ├── data.py │ │ └── graphs.py │ ├── data.py │ └── graphs.py ├── io.py ├── measurement │ ├── __init__.py │ ├── _cli │ │ ├── __init__.py │ │ ├── _cli.py │ │ ├── cellprofiler.py │ │ ├── intensities.py │ │ ├── neighbors.py │ │ └── regionprops.py │ ├── _measurement.py │ ├── cellprofiler │ │ ├── __init__.py │ │ ├── _cellprofiler.py │ │ └── data │ │ │ ├── __init__.py │ │ │ └── cell_measurement.cppipe │ ├── intensities.py │ ├── neighbors.py │ └── regionprops.py ├── preprocessing │ ├── __init__.py │ ├── _cli │ │ ├── __init__.py │ │ ├── _cli.py │ │ ├── external.py │ │ └── imc.py │ ├── _preprocessing.py │ ├── external.py │ └── imc.py ├── segmentation │ ├── __init__.py │ ├── _cli │ │ ├── __init__.py │ │ ├── _cli.py │ │ ├── cellpose.py │ │ ├── cellprofiler.py │ │ └── deepcell.py │ ├── _segmentation.py │ ├── cellpose.py │ ├── cellprofiler │ │ ├── __init__.py │ │ ├── _cellprofiler.py │ │ └── data │ │ │ ├── __init__.py │ │ │ └── cell_segmentation.cppipe │ └── deepcell.py ├── utils │ ├── __init__.py │ ├── _cli │ │ ├── __init__.py │ │ ├── _cli.py │ │ ├── expansion.py │ │ ├── matching.py │ │ └── mosaics.py │ ├── _utils.py │ ├── expansion.py │ ├── matching.py │ └── mosaics.py └── visualization.py └── tests ├── classification └── test_ilastik_classification.py ├── export ├── test_data_export.py └── test_graphs_export.py ├── measurement ├── test_cellprofiler_measurement.py ├── test_intensities_measurement.py ├── test_neighbors_measurement.py └── test_regionprops_measurement.py ├── preprocessing ├── test_external_preprocessing.py └── test_imc_preprocessing.py ├── segmentation ├── test_cellpose_segmentation.py ├── test_cellprofiler_segmentation.py └── test_deepcell_segmentation.py ├── test_io.py ├── test_viewer.py └── utils ├── test_expansion_utils.py ├── test_matching_utils.py └── test_mosaics_utils.py /.dockerignore: -------------------------------------------------------------------------------- 1 | /.github/ 2 | /.vscode/ 3 | /data/ 4 | /docs/ 5 | /steinbock/_version.py 6 | /.dockerignore 7 | /.flake8 8 | /.gitignore 9 | /.isort.cfg 10 | /.pre-commit-config.yaml 11 | /AUTHORS.md 12 | /CHANGELOG.md 13 | /CONTRIBUTING.md 14 | /CONTRIBUTORS.md 15 | /docker-compose.yml 16 | /Dockerfile 17 | /LICENSE.md 18 | /mkdocs.yml 19 | /README.md 20 | 21 | # Byte-compiled / optimized / DLL files 22 | __pycache__/ 23 | *.py[cod] 24 | *$py.class 25 | 26 | # C extensions 27 | *.so 28 | 29 | # Distribution / packaging 30 | .Python 31 | build/ 32 | develop-eggs/ 33 | dist/ 34 | downloads/ 35 | eggs/ 36 | .eggs/ 37 | lib/ 38 | lib64/ 39 | parts/ 40 | sdist/ 41 | var/ 42 | wheels/ 43 | share/python-wheels/ 44 | *.egg-info/ 45 | .installed.cfg 46 | *.egg 47 | MANIFEST 48 | 49 | # PyInstaller 50 | # Usually these files are written by a python script from a template 51 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 52 | *.manifest 53 | *.spec 54 | 55 | # Installer logs 56 | pip-log.txt 57 | pip-delete-this-directory.txt 58 | 59 | # Unit test / coverage reports 60 | htmlcov/ 61 | .tox/ 62 | .nox/ 63 | .coverage 64 | .coverage.* 65 | .cache 66 | nosetests.xml 67 | coverage.xml 68 | *.cover 69 | *.py,cover 70 | .hypothesis/ 71 | .pytest_cache/ 72 | cover/ 73 | 74 | # Translations 75 | *.mo 76 | *.pot 77 | 78 | # Django stuff: 79 | *.log 80 | local_settings.py 81 | db.sqlite3 82 | db.sqlite3-journal 83 | 84 | # Flask stuff: 85 | instance/ 86 | .webassets-cache 87 | 88 | # Scrapy stuff: 89 | .scrapy 90 | 91 | # Sphinx documentation 92 | docs/_build/ 93 | 94 | # PyBuilder 95 | .pybuilder/ 96 | target/ 97 | 98 | # Jupyter Notebook 99 | .ipynb_checkpoints 100 | 101 | # IPython 102 | profile_default/ 103 | ipython_config.py 104 | 105 | # pyenv 106 | # For a library or package, you might want to ignore these files since the code is 107 | # intended to run in multiple environments; otherwise, check them in: 108 | # .python-version 109 | 110 | # pipenv 111 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 112 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 113 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 114 | # install all needed dependencies. 115 | #Pipfile.lock 116 | 117 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 118 | __pypackages__/ 119 | 120 | # Celery stuff 121 | celerybeat-schedule 122 | celerybeat.pid 123 | 124 | # SageMath parsed files 125 | *.sage.py 126 | 127 | # Environments 128 | .env 129 | .venv 130 | env/ 131 | venv/ 132 | ENV/ 133 | env.bak/ 134 | venv.bak/ 135 | 136 | # Spyder project settings 137 | .spyderproject 138 | .spyproject 139 | 140 | # Rope project settings 141 | .ropeproject 142 | 143 | # mkdocs documentation 144 | /site 145 | 146 | # mypy 147 | .mypy_cache/ 148 | .dmypy.json 149 | dmypy.json 150 | 151 | # Pyre type checker 152 | .pyre/ 153 | 154 | # pytype static type analyzer 155 | .pytype/ 156 | 157 | # Cython debug symbols 158 | cython_debug/ 159 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 88 3 | extend-ignore = E203 4 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: docs 2 | 3 | on: 4 | 5 | push: 6 | branches: 7 | - main 8 | tags: 9 | - "v*" 10 | 11 | workflow_dispatch: 12 | 13 | jobs: 14 | 15 | docs: 16 | runs-on: ubuntu-latest 17 | steps: 18 | 19 | - name: Checkout repository 20 | uses: actions/checkout@v3 21 | with: 22 | fetch-depth: 0 23 | 24 | - name: Setup Python 25 | uses: actions/setup-python@v4 26 | with: 27 | python-version: "3.8" 28 | 29 | - name: Install dependencies 30 | run: | 31 | python -m pip install --upgrade pip setuptools wheel 32 | python -m pip install -r requirements.txt 33 | python -m pip install -r requirements_docs.txt 34 | python -m pip install -e ".[imc,cellpose,deepcell,napari]" 35 | 36 | - name: Build documentation (versioned) 37 | if: startsWith(github.ref, 'refs/tags') 38 | run: | 39 | git config user.name github-actions 40 | git config user.email github-actions@github.com 41 | git fetch origin gh-pages --verbose 42 | mike deploy -p -u ${GITHUB_REF#refs/tags/} 43 | 44 | - name: Build documentation (latest) 45 | if: github.ref == 'refs/heads/main' 46 | run: | 47 | git config user.name github-actions 48 | git config user.email github-actions@github.com 49 | git fetch origin gh-pages --verbose 50 | mike deploy -p -u latest 51 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /data/ 2 | /steinbock/_version.py 3 | images.csv 4 | 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | *.DS_Store 10 | 11 | # C extensions 12 | *.so 13 | 14 | # Distribution / packaging 15 | .Python 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | downloads/ 20 | eggs/ 21 | .eggs/ 22 | lib/ 23 | lib64/ 24 | parts/ 25 | sdist/ 26 | var/ 27 | wheels/ 28 | share/python-wheels/ 29 | *.egg-info/ 30 | .installed.cfg 31 | *.egg 32 | MANIFEST 33 | 34 | # PyInstaller 35 | # Usually these files are written by a python script from a template 36 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 37 | *.manifest 38 | *.spec 39 | 40 | # Installer logs 41 | pip-log.txt 42 | pip-delete-this-directory.txt 43 | 44 | # Unit test / coverage reports 45 | htmlcov/ 46 | .tox/ 47 | .nox/ 48 | .coverage 49 | .coverage.* 50 | .cache 51 | nosetests.xml 52 | coverage.xml 53 | *.cover 54 | *.py,cover 55 | .hypothesis/ 56 | .pytest_cache/ 57 | cover/ 58 | 59 | # Translations 60 | *.mo 61 | *.pot 62 | 63 | # Django stuff: 64 | *.log 65 | local_settings.py 66 | db.sqlite3 67 | db.sqlite3-journal 68 | 69 | # Flask stuff: 70 | instance/ 71 | .webassets-cache 72 | 73 | # Scrapy stuff: 74 | .scrapy 75 | 76 | # Sphinx documentation 77 | docs/_build/ 78 | 79 | # PyBuilder 80 | .pybuilder/ 81 | target/ 82 | 83 | # Jupyter Notebook 84 | .ipynb_checkpoints 85 | 86 | # IPython 87 | profile_default/ 88 | ipython_config.py 89 | 90 | # pyenv 91 | # For a library or package, you might want to ignore these files since the code is 92 | # intended to run in multiple environments; otherwise, check them in: 93 | # .python-version 94 | 95 | # pipenv 96 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 97 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 98 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 99 | # install all needed dependencies. 100 | #Pipfile.lock 101 | 102 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 103 | __pypackages__/ 104 | 105 | # Celery stuff 106 | celerybeat-schedule 107 | celerybeat.pid 108 | 109 | # SageMath parsed files 110 | *.sage.py 111 | 112 | # Environments 113 | .env 114 | .venv 115 | env/ 116 | venv/ 117 | ENV/ 118 | env.bak/ 119 | venv.bak/ 120 | 121 | # Spyder project settings 122 | .spyderproject 123 | .spyproject 124 | 125 | # Rope project settings 126 | .ropeproject 127 | 128 | # mkdocs documentation 129 | /site 130 | 131 | # mypy 132 | .mypy_cache/ 133 | .dmypy.json 134 | dmypy.json 135 | 136 | # Pyre type checker 137 | .pyre/ 138 | 139 | # pytype static type analyzer 140 | .pytype/ 141 | 142 | # Cython debug symbols 143 | cython_debug/ 144 | -------------------------------------------------------------------------------- /.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | profile=black 3 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | exclude: ^\.vscode/.* 2 | repos: 3 | - repo: https://github.com/pre-commit/pre-commit-hooks 4 | rev: v4.4.0 5 | hooks: 6 | - id: check-added-large-files 7 | - id: check-case-conflict 8 | - id: check-docstring-first 9 | - id: check-executables-have-shebangs 10 | - id: check-merge-conflict 11 | - id: check-shebang-scripts-are-executable 12 | - id: check-toml 13 | - id: check-yaml 14 | - id: debug-statements 15 | - id: end-of-file-fixer 16 | - id: requirements-txt-fixer 17 | - id: trailing-whitespace 18 | - repo: https://github.com/PyCQA/isort 19 | rev: "5.12.0" 20 | hooks: 21 | - id: isort 22 | - repo: https://github.com/PyCQA/autoflake 23 | rev: v2.0.1 24 | hooks: 25 | - id: autoflake 26 | args: [--in-place, --remove-all-unused-imports] 27 | - repo: https://github.com/psf/black 28 | rev: '23.1.0' 29 | hooks: 30 | - id: black 31 | - repo: https://github.com/PyCQA/flake8 32 | rev: "6.0.0" 33 | hooks: 34 | - id: flake8 35 | additional_dependencies: [flake8-typing-imports] 36 | - repo: https://github.com/pre-commit/mirrors-mypy 37 | rev: v1.0.0 38 | hooks: 39 | - id: mypy 40 | additional_dependencies: [types-requests, types-PyYAML] 41 | ci: 42 | autoupdate_branch: develop 43 | skip: [flake8, mypy] 44 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "name": "Docker: Python - General", 6 | "type": "docker", 7 | "request": "launch", 8 | "preLaunchTask": "docker-run", 9 | "python": { 10 | "pathMappings": [ 11 | { 12 | "localRoot": "${workspaceFolder}", 13 | "remoteRoot": "/app/steinbock" 14 | } 15 | ], 16 | "projectType": "general" 17 | } 18 | }, 19 | { 20 | "name": "Python: Remote Attach (steinbock-debug)", 21 | "type": "python", 22 | "request": "attach", 23 | "preLaunchTask": "compose-up: steinbock-debug", 24 | "connect": { 25 | "host": "localhost", 26 | "port": 5678 27 | }, 28 | "pathMappings": [ 29 | { 30 | "localRoot": "${workspaceFolder}", 31 | "remoteRoot": "/app/steinbock" 32 | } 33 | ], 34 | "justMyCode": false 35 | }, 36 | { 37 | "name": "Python: Remote Attach (pytest-debug)", 38 | "type": "python", 39 | "request": "attach", 40 | "preLaunchTask": "compose-up: pytest-debug", 41 | "connect": { 42 | "host": "localhost", 43 | "port": 5678 44 | }, 45 | "pathMappings": [ 46 | { 47 | "localRoot": "${workspaceFolder}", 48 | "remoteRoot": "/app/steinbock" 49 | } 50 | ], 51 | "justMyCode": false 52 | } 53 | ] 54 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.testing.pytestArgs": [ 3 | "tests" 4 | ], 5 | "python.testing.pytestEnabled": true, 6 | "python.linting.flake8Enabled": true, 7 | "python.linting.enabled": true 8 | } -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "label": "docker-build", 6 | "type": "docker-build", 7 | "platform": "python", 8 | "dockerBuild": { 9 | "tag": "ghcr.io/bodenmillergroup/steinbock:latest", 10 | "dockerfile": "${workspaceFolder}/Dockerfile", 11 | "context": "${workspaceFolder}", 12 | "pull": true, 13 | "buildArgs": { 14 | "TENSORFLOW_TARGET": "tensorflow", 15 | "STEINBOCK_TARGET": "steinbock" 16 | }, 17 | "target": "steinbock" 18 | } 19 | }, 20 | { 21 | "label": "docker-run", 22 | "type": "docker-run", 23 | "dependsOn": [ 24 | "docker-build" 25 | ], 26 | "python": { 27 | "module": "steinbock" 28 | } 29 | }, 30 | { 31 | "label": "compose-up: steinbock-debug", 32 | "type": "shell", 33 | "dependsOn": [ 34 | "docker-build" 35 | ], 36 | "command": "docker-compose up -d steinbock-debug" 37 | }, 38 | { 39 | "label": "compose-up: pytest-debug", 40 | "type": "shell", 41 | "dependsOn": [ 42 | "docker-build" 43 | ], 44 | "command": "docker-compose up -d pytest-debug" 45 | } 46 | ] 47 | } -------------------------------------------------------------------------------- /AUTHORS.md: -------------------------------------------------------------------------------- 1 | # Authors 2 | 3 | Created and maintained by [Jonas Windhager](mailto:jonas@windhager.io) until February 2023. 4 | 5 | Maintained by [Milad Adibi](mailto:milad.adibi@uzh.ch) from February 2023. 6 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Pull requests are welcome. Please make sure to update tests and documentation as appropriate. 4 | 5 | For major changes, please open an issue first to discuss what you would like to change. 6 | 7 | ## Toolchain 8 | 9 | *steinbock* is developed using [Visual Studio Code](https://code.visualstudio.com). The Python development toolchain includes black, flake8, isort, mypy and pre-commit (see setuptools `devel` target). For building the *steinbock* Docker image, BuildKit needs to be enabled for Docker. Workflows for continuous integration/continuous delivery (CI/CD) are configured using GitHub Actions. 10 | 11 | ## Development 12 | 13 | For convenience, the following [Docker Compose](https://docs.docker.com/compose) services are available: 14 | 15 | - `steinbock` for running *steinbock* 16 | - `steinbock-debug` for debugging *steinbock* using [debugpy](https://github.com/microsoft/debugpy) 17 | - `pytest` for running unit tests with [pytest](https://pytest.org) 18 | - `pytest-debug` for debugging unit tests with [pytest](https://pytest.org) and [debugpy](https://github.com/microsoft/debugpy) 19 | 20 | Matching Visual Studio Code launch configurations are provided for debugging: 21 | 22 | - `Docker: Python General` for debugging *steinbock* using [Docker](https://www.docker.com) directly 23 | - `Python: Remote Attach (steinbock-debug)` for debugging *steinbock* using [Docker Compose](https://docs.docker.com/compose) 24 | - `Python: Remote Attach (pytest-debug)` for debugging unit tests with [pytest](https://pytest.org) using [Docker Compose](https://docs.docker.com/compose) 25 | 26 | To debug specific *steinbock* commands using e.g. the `Python: Remote Attach (steinbock-debug)` launch configuration, adapt the respective `command` in the [docker-compose.yml](https://github.com/BodenmillerGroup/steinbock/blob/main/docker-compose.yml) file (e.g. add `--version` after `-m steinbock`). Launch configurations may have to be invoked multiple times in order for them to work. 27 | 28 | To debug unit tests on the host system (i.e., not within the Docker container), run `pytest tests` in the project root folder or use the "Testing" tab in Visual Studio Code. 29 | -------------------------------------------------------------------------------- /CONTRIBUTORS.md: -------------------------------------------------------------------------------- 1 | # Contributors 2 | 3 | ## Nils Eling 4 | 5 | [nils.eling@uzh.ch](mailto:nils.eling@uzh.ch) 6 | 7 | - Logo 8 | - Documentation 9 | - Testing, feedback & discussion 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 University of Zurich 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS.md 2 | include CONTRIBUTING.md 3 | include CONTRIBUTORS.md 4 | include LICENSE 5 | include README.md 6 | include requirements*.txt 7 | include steinbock/**/data/* 8 | 9 | recursive-exclude .github * 10 | recursive-exclude .vscode * 11 | exclude .dockerignore 12 | exclude .gitignore 13 | exclude docker-compose.yml 14 | exclude Dockerfile 15 | exclude entrypoint.sh 16 | exclude fixuid.yml 17 | 18 | recursive-exclude * __pycache__ 19 | recursive-exclude * *.py[co] 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Logo 2 | 3 | # steinbock 4 | 5 | Python 6 | Build 7 | PyPI 8 | Coverage 9 | Documentation 10 | Issues 11 | Pull requests 12 | License 13 | 14 | A toolkit for processing multiplexed tissue images 15 | 16 | Documentation is available at https://bodenmillergroup.github.io/steinbock 17 | 18 | ## Citation 19 | 20 | Please cite the following paper when using `steinbock` in your work: 21 | 22 | > Windhager, J., Zanotelli, V.R.T., Schulz, D. et al. An end-to-end workflow for multiplexed image processing and analysis. Nat Protoc (2023). https://doi.org/10.1038/s41596-023-00881-0 23 | 24 | @article{Windhager2023, 25 | author = {Windhager, Jonas and Zanotelli, Vito R.T. and Schulz, Daniel and Meyer, Lasse and Daniel, Michelle and Bodenmiller, Bernd and Eling, Nils}, 26 | title = {An end-to-end workflow for multiplexed image processing and analysis}, 27 | year = {2023}, 28 | doi = {10.1038/s41596-023-00881-0}, 29 | URL = {https://www.nature.com/articles/s41596-023-00881-0}, 30 | journal = {Nature Protocols} 31 | } 32 | -------------------------------------------------------------------------------- /conftest.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | from pathlib import Path 3 | from typing import Generator 4 | 5 | import pytest 6 | import requests 7 | 8 | _imc_test_data_steinbock_url = ( 9 | "https://github.com/BodenmillerGroup/TestData" 10 | "/releases/download/v1.0.7/210308_ImcTestData_steinbock.tar.gz" 11 | ) 12 | _imc_test_data_steinbock_dir = "datasets/210308_ImcTestData/steinbock" 13 | 14 | 15 | def _download_and_extract_asset(tmp_dir_path: Path, asset_url: str): 16 | asset_file_path = tmp_dir_path / "asset.tar.gz" 17 | response = requests.get(asset_url, stream=True) 18 | if response.status_code == 200: 19 | with asset_file_path.open(mode="wb") as f: 20 | f.write(response.raw.read()) 21 | shutil.unpack_archive(asset_file_path, tmp_dir_path) 22 | 23 | 24 | @pytest.fixture(scope="session") 25 | def imc_test_data_steinbock_path( 26 | tmp_path_factory, 27 | ) -> Generator[Path, None, None]: 28 | tmp_dir_path = tmp_path_factory.mktemp("raw") 29 | _download_and_extract_asset(tmp_dir_path, _imc_test_data_steinbock_url) 30 | yield tmp_dir_path / Path(_imc_test_data_steinbock_dir) 31 | shutil.rmtree(tmp_dir_path) 32 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.4' 2 | services: 3 | steinbock: 4 | image: ghcr.io/bodenmillergroup/steinbock:latest 5 | build: 6 | context: . 7 | dockerfile: Dockerfile 8 | target: steinbock 9 | args: 10 | - TENSORFLOW_TARGET=tensorflow 11 | - STEINBOCK_TARGET=steinbock 12 | volumes: 13 | - ./data:/data:rw 14 | - /tmp/.X11-unix:/tmp/.X11-unix:rw 15 | - ~/.Xauthority:/home/steinbock/.Xauthority:ro 16 | environment: 17 | - DISPLAY 18 | steinbock-debug: 19 | image: ghcr.io/bodenmillergroup/steinbock:latest 20 | volumes: 21 | - ./data:/data:rw 22 | - /tmp/.X11-unix:/tmp/.X11-unix:rw 23 | - ~/.Xauthority:/home/steinbock/.Xauthority:ro 24 | environment: 25 | - DISPLAY 26 | entrypoint: /bin/bash 27 | command: ['-c', 'python -m pip install debugpy -t /tmp && python /tmp/debugpy --wait-for-client --listen 0.0.0.0:5678 -m steinbock'] 28 | ports: 29 | - '5678:5678' 30 | pytest: 31 | image: ghcr.io/bodenmillergroup/steinbock:latest 32 | volumes: 33 | - .:/app/steinbock 34 | entrypoint: /bin/bash 35 | command: ['-c', 'python -m pytest tests --cov=steinbock --cov-report xml:coverage.xml'] 36 | working_dir: /app/steinbock 37 | user: root 38 | pytest-debug: 39 | image: ghcr.io/bodenmillergroup/steinbock:latest 40 | volumes: 41 | - .:/app/steinbock 42 | entrypoint: /bin/bash 43 | command: ['-c', 'python -m pip install debugpy -t /tmp && python /tmp/debugpy --wait-for-client --listen 0.0.0.0:5678 -m pytest tests --cov=steinbock --cov-report xml:coverage.xml'] 44 | ports: 45 | - '5678:5678' 46 | working_dir: /app/steinbock 47 | user: root 48 | -------------------------------------------------------------------------------- /docs/authors.md: -------------------------------------------------------------------------------- 1 | ../AUTHORS.md -------------------------------------------------------------------------------- /docs/changelog.md: -------------------------------------------------------------------------------- 1 | ../CHANGELOG.md -------------------------------------------------------------------------------- /docs/cli/apps.md: -------------------------------------------------------------------------------- 1 | # Apps 2 | 3 | The *steinbock* Docker container exposes various third-party apps via the `apps` command. 4 | 5 | ## Ilastik 6 | 7 | !!! note "Docker and graphical user interfaces" 8 | Running Ilastik using steinbock Docker containers requires support for graphical user interfaces (e.g. X forwarding). 9 | 10 | To run [Ilastik](https://www.ilastik.org): 11 | 12 | steinbock apps ilastik 13 | 14 | Without additional arguments, this will start the graphical user interface of Ilastik. 15 | 16 | ## CellProfiler 17 | 18 | !!! note "Docker and graphical user interfaces" 19 | Running CellProfiler using steinbock Docker containers requires support for a graphical user interfaces (e.g. X forwarding). 20 | 21 | To run [CellProfiler](https://cellprofiler.org): 22 | 23 | steinbock apps cellprofiler 24 | 25 | Without additional arguments, this will start the graphical user interface of CellProfiler. 26 | 27 | ## Jupyter Notebook 28 | 29 | !!! note "Port" 30 | Jupyter Notebook requires the specified port exposed and published via Docker. 31 | 32 | To run [Jupyter Notebook](https://jupyter.org): 33 | 34 | steinbock apps jupyter 35 | 36 | Without additional arguments, this will start Jupyter Notebook on http://localhost:8888. 37 | 38 | ## Jupyter Lab 39 | 40 | !!! note "Port" 41 | Jupyter Lab requires the specified port exposed and published via Docker. 42 | 43 | To run [Jupyter Lab](https://jupyter.org): 44 | 45 | steinbock apps jupyterlab 46 | 47 | Without additional arguments, this will start Jupyter Lab on http://localhost:8888. 48 | -------------------------------------------------------------------------------- /docs/cli/classification.md: -------------------------------------------------------------------------------- 1 | # Pixel classification 2 | 3 | In this step, for each image, the probabilities of pixels belonging to a given class (e.g. nucleus, cytoplasm, background) will be determined. This will result in *probability images*, with one color per class encoding the probability of pixels belonging to that class (see [File types](../file-types.md#probabilities)). 4 | 5 | !!! note "Pixel classification-based image segmentation" 6 | Probability images generated by pixel classification can be used to segment images, see [Object segmentation](segmentation.md). 7 | 8 | Various classification approaches are supported, each of which is described in the following. 9 | 10 | ## Ilastik 11 | 12 | [Ilastik](https://www.ilastik.org) is an application for interactive learning and segmentation. Here, Ilastik's semantic [pixel classification workflow](https://www.ilastik.org/documentation/pixelclassification/pixelclassification) is used to perform pixel classification using random forests. 13 | 14 | ### Data preparation 15 | 16 | In a first step, input data are prepared for processing with Ilastik: 17 | 18 | steinbock classify ilastik prepare --cropsize 50 --seed 123 19 | 20 | With default desination file/directory paths shown in brackets, this will: 21 | 22 | - aggregate, scale and convert images to *steinbock* Ilastik format (`ilastik_img`) 23 | - extract and save one random crop of 50x50 pixels per image for training (`ilastik_crops`) 24 | - create a default *steinbock* Ilastik pixel classification project file (`pixel_classifier.ilp`) 25 | 26 | By specifying the `--seed` parameter, this command reproducibly extracts crops from the same pseudo-random locations when executed repeatedly. 27 | 28 | !!! note "Ilastik image data" 29 | All generated image data are saved in *steinbock* Ilastik HDF5 format (undocumented). 30 | 31 | If an `ilastik` column is present in the *steinbock* panel file, channels are sorted and grouped according to values in that column: For each image, each group of channels is aggregated by computing the mean along the channel axis (use the `--aggr` option to specify a different aggregation strategy). The generated Ilastik images consist of one channel per group; channels without a group label are ignored. In addition, the mean of all included channels is prepended to the generated Ilastik images as an additional channel, unless `--no-mean` is specified. 32 | 33 | Furthermore, all generated Ilastik images are scaled two-fold in x and y, unless specified otherwise using the `--scale` command-line option. This helps with more accurately identifying object borders in segmentation workflows for images of relatively low resolution (e.g. Imaging Mass Cytometry). In applications with higher resolution (e.g. sequential immunofluorescence), it is recommended to not scale the image data, i.e., to specify `--scale 1`. 34 | 35 | ### Training the classifier 36 | 37 | To interactively train a new classifier, open the pixel classification project in Ilastik (see [Apps](apps.md#ilastik)): 38 | 39 | steinbock apps ilastik 40 | 41 | !!! note "Data/working directory" 42 | Within the container, your data/working directory containing the Ilastik project file is accessible under `/data`. 43 | 44 | More detailed instructions on how to use Ilastik for training a pixel classifier can be found [here](https://www.ilastik.org/documentation/pixelclassification/pixelclassification). 45 | 46 | !!! note "Class labels for segmentation" 47 | By default, the Ilastik pixel classification project is configured for training three classes (Nucleus, Cytoplasm, Background) for cell segmentation. Other segmentation workflows may require different numbers of classes and class labels (e.g. two classes for Tumor/Stroma segmentation). While the number and order of classes is arbitrary and can be changed by the user, it needs to be compatible with downstream [segmentation steps](segmentation.md). 48 | 49 | !!! note "Feature selection" 50 | The choice of features in Ilastik's feature selection step depends on the input data. For relatively small IMC datasets, the selection of all default features greater than or equal to 1 pixel [is recommended](https://github.com/BodenmillerGroup/ImcSegmentationPipeline/blob/main/scripts/imc_preprocessing.ipynb). 51 | 52 | ### Existing training data 53 | 54 | !!! danger "Experimental feature" 55 | Reusing existing training data is an experimental feature. Use at own risk. Always make backups of your data. 56 | 57 | Instead of training a new classifier, one can use an existing classifier by 58 | 59 | - replacing the generated Ilastik pixel classification project file with a pre-trained project, and 60 | 61 | - replacing the image crops (see [Data preparation](#data-preparation)) with the crops originally used for training. 62 | 63 | Subsequently, to ensure compatibility of the external Ilastik project file/crops: 64 | 65 | steinbock classify ilastik fix 66 | 67 | This will attempt to in-place patch the Ilastik pixel classification project and the image crops after creating a backup (`.bak` file/directory extension), unless `--no-backup` is specified. 68 | 69 | !!! note "Patching existing training data" 70 | This command will convert image crops to 32-bit floating point images with CYX dimension order and save them in *steinbock* Ilastik HDF5 format (undocumented). It will then adjust the metadata in the Ilastik project file accordingly. 71 | 72 | ### Batch processing 73 | 74 | After training the pixel classifier on the image crops (or providing and patching a pre-trained one), it can be applied to a batch of full-size images created in the [Data preparation](#data-preparation) step as follows: 75 | 76 | steinbock classify ilastik run 77 | 78 | By default, this will create probability images in `ilastik_probabilities`, with one color per class encoding the probability of pixels belonging to that class (see [File types](../file-types.md#probabilities)). 79 | 80 | !!! note "Probability images" 81 | The size of the generated probability images are equal to the size of the Ilastik input images, i.e., scaled by a user-specified factor that defaults to 2 (see above). If applicable, make sure to adapt downstream segmentation workflows accordingly to create object masks matching the original (i.e., unscaled) images. 82 | 83 | If the default three-class structure is used, the probability images are RGB images with the following color code: 84 | 85 | - Red: Nuclei 86 | - Green: Cytoplasm 87 | - Blue: Background 88 | -------------------------------------------------------------------------------- /docs/cli/export.md: -------------------------------------------------------------------------------- 1 | # Data export 2 | 3 | Data generated by *steinbock* can be exported to various formats for downstream data analysis. 4 | 5 | ## OME-TIFF 6 | 7 | To export images to OME-TIFF, with channel names determined by the panel file: 8 | 9 | steinbock export ome 10 | 11 | The exported OME-TIFF files are generated by [xtiff](https://github.com/BodenmillerGroup/xtiff); the default destination directory is `ome`. 12 | 13 | ## histoCAT 14 | 15 | To export images and masks to a folder structure compatible with [histoCAT for MATLAB](https://bodenmillergroup.github.io/histoCAT/): 16 | 17 | steinbock export histocat 18 | 19 | This will create a histoCAT-compatible folder structure (defaults to `histocat`), with one subfolder per image, where each subfolder contains one image file per channel. Additionally, if masks are available, each image subfolder contains a single mask file. 20 | 21 | ## CSV 22 | 23 | To export specified object data from all images as a single .csv file: 24 | 25 | steinbock export csv intensities regionprops -o objects.csv 26 | 27 | This will collect object data from the `intensities` and `regionprops` directories and create a single object data table in [object data format](../file-types.md#object-data), with an additional first column indicating the source image. 28 | 29 | ## FCS 30 | 31 | To export specified object data from all images as a single .fcs file: 32 | 33 | steinbock export fcs intensities regionprops -o objects.fcs 34 | 35 | This will collect object data from the `intensities` and `regionprops` directories and create a single FCS file using the [fcswrite](https://github.com/ZELLMECHANIK-DRESDEN/fcswrite) package. 36 | 37 | ## AnnData 38 | 39 | To export specified object data to [AnnData](https://github.com/theislab/anndata): 40 | 41 | steinbock export anndata --intensities intensities --data regionprops --neighbors neighbors -o objects.h5ad 42 | 43 | This will generate a single .h5ad file, with object intensities as main data, object regionprops as observation annotations, and neighbors as pairwise observation annotations (adjacency matrix in `adj`, distances in `dists`). 44 | 45 | !!! note "AnnData file format" 46 | To export the data as .loom or .zarr, specify `--format loom` or `--format zarr`, respectively. 47 | 48 | Currently, the .h5ad format does not allow for storing panel/image metadata, see [issue #66](https://github.com/BodenmillerGroup/steinbock/issues/66). 49 | 50 | !!! note "Multiple data sources" 51 | The `--data` option can be specified multiple times to include different object data as observation annotationss. 52 | 53 | ## Graphs 54 | 55 | To export neighbors as spatial object graphs, with object data as node attributes: 56 | 57 | steinbock export graphs --data intensities 58 | 59 | By default, this will generate one .graphml file per graph using the [networkx](https://networkx.org) Python package, with object intensities as node attributes. The default destination directory is `graphs`. 60 | 61 | !!! note "NetworkX file format" 62 | To export the graphs as .gexf or .gml, specify `--format gexf` or `--format gml`, respectively. 63 | 64 | !!! note "Multiple graph attributes sources" 65 | The `--data` option can be specified multiple times to include different object data as graph attributes. 66 | -------------------------------------------------------------------------------- /docs/cli/intro.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | The following sections document the usage of the *steinbock* command-line interface (CLI). 4 | 5 | !!! note "Prerequisites" 6 | From this point onwards, it is assumed that the `steinbock` command alias was [configured](../install-docker.md) correctly. To make efficient use of the *steinbock* Docker container, basic command line skills are absolutely required. Furthermore, understanding key concepts of containerization using Docker may be helpful in resolving issues. 7 | 8 | ## Trying it out 9 | 10 | To try out *steinbock*, the [IMC mock dataset](https://github.com/BodenmillerGroup/TestData/tree/main/datasets/210308_ImcTestData) can be used as follows: 11 | 12 | 1. Copy the [raw](https://github.com/BodenmillerGroup/TestData/tree/main/datasets/210308_ImcTestData/raw) directory to your *steinbock* data/working directory 13 | 2. Place the [panel.csv](https://github.com/BodenmillerGroup/TestData/blob/main/datasets/210308_ImcTestData/panel.csv) into the `raw` directory in your *steinbock* data/working directory 14 | 3. Continue with [Imaging Mass Cytometry (IMC) preprocessing](preprocessing.md#imaging-mass-cytometry-imc) and subsequent steps 15 | 16 | !!! note 17 | Existing Ilastik training data ([Ilastik pixel classification project](https://github.com/BodenmillerGroup/TestData/blob/main/datasets/210308_ImcTestData/ilastik.ilp), [Ilastik crops](https://github.com/BodenmillerGroup/TestData/tree/main/datasets/210308_ImcTestData/analysis/ilastik)) can be used for testing the classification step. 18 | 19 | ## Getting help 20 | 21 | At any time, use the `--help` option to show help about a *steinbock* command, e.g.: 22 | 23 | > steinbock --help 24 | 25 | Usage: steinbock [OPTIONS] COMMAND [ARGS]... 26 | 27 | Options: 28 | --version Show the version and exit. 29 | --help Show this message and exit. 30 | 31 | Commands: 32 | preprocess Extract and preprocess images from raw data 33 | classify Perform pixel classification to extract probabilities 34 | segment Perform image segmentation to create object masks 35 | measure Extract object data from segmented images 36 | export Export data to third-party formats 37 | utils Various utilities and tools 38 | view View image using napari GUI 39 | apps Third-party applications 40 | 41 | !!! note "Directory structure" 42 | Unless specified otherwise, all *steinbock* commands adhere to the default [directory structure](../directories.md). 43 | 44 | For bug reports or further help, please do not hesitate to reach out via [GitHub Issues/Discussions](https://github.com/BodenmillerGroup/steinbock). 45 | -------------------------------------------------------------------------------- /docs/cli/measurement.md: -------------------------------------------------------------------------------- 1 | # Object measurement 2 | 3 | In this step, object-level (e.g. single-cell) data will be extracted from segmented images. 4 | 5 | Various types of data can be extracted, each of which is described in the following. 6 | 7 | !!! note "Collecting multiple object data from multiple images" 8 | To collect all object data (e.g. intensities, region properties) from all images into a single file, see [Data export](export.md#object-data). 9 | 10 | ## Object intensities 11 | 12 | To extract mean object intensities per channel: 13 | 14 | steinbock measure intensities 15 | 16 | This will create object data tables in CSV format (see [File types](../file-types.md#object-data), one file per image). The default destination directory is `intensities`. 17 | 18 | !!! note "Pixel aggregation" 19 | By default, pixels belonging to an object are aggregated by taking the mean. To specify a different [numpy](https://numpy.org) function for aggregation, use the `--aggr` option (e.g. specify `--aggr median` to measure "median object intensities"). 20 | 21 | ## Region properties 22 | 23 | To extract spatial object properties ("region properties"): 24 | 25 | steinbock measure regionprops 26 | 27 | This will create object data tables in CSV format (see [File types](../file-types.md#object-data), one file per image). The default destination directory is `regionprops`. 28 | 29 | !!! note "Region property selection" 30 | By default, the following [scikit-image region properties](https://scikit-image.org/docs/dev/api/skimage.measure.html#skimage.measure.regionprops) will be computed: 31 | 32 | - `area` 33 | - `centroid` 34 | - `major_axis_length` 35 | - `minor_axis_length` 36 | - `eccentricity` 37 | 38 | An alternative selection of scikit-image region properties can be specified in the `regionprops` command, e.g.: 39 | 40 | steinbock measure regionprops area convex_area perimeter 41 | 42 | ## Object neighbors 43 | 44 | Neighbors can be measured (i.e., identified) based on distances between object centroids or object borders, or by pixel expansion. For distance-based neighbor identification, the maximum distance and/or number of neighbors can be specified. 45 | 46 | !!! note "Spatial object graphs" 47 | Pairs of neighbors can be represented as edges on a spatial object graph, where each cell is a vertex (node), and neighboring cells are connected by an edge associated with a spatial distance. 48 | 49 | The following commands will create directed edge lists in CSV format (see [File types](../file-types.md#object-neighbors), one file per image). For undirected graphs, i.e., graphs constructed by distance thresholding/pixel expansion, each edge will appear twice. The default destination directory is `neighbors`. 50 | 51 | ### Centroid distances 52 | 53 | To find neighbors by thresholding on distances between object centroids: 54 | 55 | steinbock measure neighbors --type centroids --dmax 15 56 | 57 | To construct k-nearest neighbor (kNN) graphs based on object centroid distances: 58 | 59 | steinbock measure neighbors --type centroids --kmax 5 60 | 61 | !!! note "Distance metric" 62 | By default, the Euclidean distance distance is used. Other metrics can be specified using the `--metric` option, e.g.: 63 | 64 | steinbock measure neighbors --type centroids --dmax 15 --metric cityblock 65 | 66 | Available distance metrics are listed in the [documentation for scipy.spatial.distance.pdist](https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.pdist.html). 67 | 68 | !!! note "Distance-thresholded kNN graphs" 69 | The options `--dmax` and `--kmax` options can be combined to construct distance-thresholded kNN graphs, e.g.: 70 | 71 | steinbock measure neighbors --type centroids --dmax 15 --kmax 5 72 | 73 | ### Border distances 74 | 75 | To find neighbors by thresholding on Euclidean distances between object borders: 76 | 77 | steinbock measure neighbors --type borders --dmax 4 78 | 79 | To construct k-nearest neighbor (kNN) graphs based on Euclidean object border distances: 80 | 81 | steinbock measure neighbors --type borders --kmax 5 --dmax 20 82 | 83 | !!! note "Computational complexity" 84 | The construction of spatial kNN graphs based on Euclidean distances between object borders is computationally expensive. To speed up the computation, always specify a suitable `--dmax` value like in the example above. 85 | 86 | ### Pixel expansion 87 | 88 | To find neighbors by Euclidean pixel expansion (morphological dilation): 89 | 90 | steinbock measure neighbors --type expansion --dmax 4 91 | 92 | !!! note "Pixel expansion versus border distances" 93 | Neighbor identification by pixel expansion is a special case of finding neighbors based on Euclidean distances between object borders, in which, after pixel expansion, only *touching* objects (i.e., objects within a 4-neighborhood) are considered neighbors. 94 | 95 | ## CellProfiler (legacy) 96 | 97 | !!! danger "Legacy operation" 98 | The output of this operation is not actively supported by downstream processing steps. 99 | 100 | ### Pipeline preparation 101 | 102 | To prepare a CellProfiler measurement pipeline: 103 | 104 | steinbock measure cellprofiler prepare 105 | 106 | !!! note "Data/working directory" 107 | Within the container, your data/working directory containing the CellProfiler pipeline is accessible under `/data`. 108 | 109 | By default, this will create a CellProfiler pipeline file `cell_measurement.cppipe` and collect all images and masks (both in 16-bit unsigned integer format) into the `cellprofiler_input` directory. 110 | 111 | !!! note "CellProfiler plugins" 112 | The generated CellProfiler pipeline makes use of [custom plugins for multi-channel images](https://github.com/BodenmillerGroup/ImcPluginsCP), which are pre-installed in the *steinbock* Docker container. It can be inspected using CellProfiler as described in the following section. 113 | 114 | ### Modifying the pipeline 115 | 116 | To interactively inspect, modify and run the pipeline, import it in CellProfiler (see [Apps](apps.md#cellprofiler)): 117 | 118 | steinbock apps cellprofiler 119 | 120 | More detailed instructions on how to create CellProfiler pipelines can be found [here](https://cellprofiler-manual.s3.amazonaws.com/CellProfiler-4.1.3/help/pipelines_building.html). 121 | 122 | ### Batch processing 123 | 124 | After the pipeline has been configured, it can be applied to a batch of images and masks: 125 | 126 | steinbock measure cellprofiler run 127 | 128 | By default, this will generate (undocumented and unstandardized) CellProfiler output as configured in the pipeline and store it in the `cellprofiler_output` directory. 129 | -------------------------------------------------------------------------------- /docs/cli/preprocessing.md: -------------------------------------------------------------------------------- 1 | # Preprocessing 2 | 3 | In this step, image data will be prepared for processing with *steinbock*. 4 | 5 | Various sources for raw data are supported by *steinbock*, each of which is described in the following. If you miss support for an imaging modality, please consider [filing an issue on GitHub](https://github.com/BodenmillerGroup/steinbock/issues). 6 | 7 | !!! note "Optional preprocessing" 8 | The *steinbock* toolkit natively supports input images saved in Tag Image File Format (TIFF), see [File types](../file-types.md#images). If you already have preprocessed TIFF files, you can directly use those for further processing. If you have preprocessed images in another file format supported by [imageio](https://imageio.readthedocs.io), you need to convert them to *steinbock*-compatible TIFF files first, see [External images](#external-images). 9 | 10 | !!! note "Computational resources" 11 | Unless specified otherwise, *steinbock* converts all input images to 32-bit floating point images upon loading, see [File types](../file-types.md#images). For large images, this may exhaust a system's available random access memory (RAM). In these situations, it is recommended to run all operations on image tiles instead, see [mosaics](utils.md#mosaics). 12 | 13 | ## Imaging mass cytometry (IMC) 14 | 15 | Preprocessing of IMC data consists of two steps: 16 | 17 | 1. Create a *steinbock* panel file and, optionally, edit it to select channels 18 | 2. Extract images from .mcd/.txt files according to the created *steinbock* panel file 19 | 20 | !!! note "Panel-based image extraction" 21 | The *steinbock* panel determines the presence and order of channels in the extracted images. 22 | 23 | ### Panel creation 24 | 25 | A *steinbock* panel file contains information about the channels in an image, such as channel ID (e.g. metal tag), channel name (e.g. antibody target), or whether a channel will be used in certain tasks (e.g. classification, segmentation). Multiple options exist for creating a *steinbock* panel file for IMC applications: 26 | 27 | - Manual *steinbock* panel file creation, following the [*steinbock* panel format specification](../file-types.md#panel) 28 | - Automatic *steinbock* panel file creation from metadata embedded in raw MCD/TXT files 29 | - Conversion from an "IMC panel file" in *IMC Segmentation Pipeline*[^1] format (undocumented) 30 | 31 | !!! note "Panel file types" 32 | The *steinbock* panel file is different from the "IMC panel file" used in the original *IMC Segmentation Pipeline*[^1] in that it is ordered (i.e., the channel order in the panel matches the channel order in the images) and only requires `channel` and `name` columns (see [File types](../file-types.md#panel)). By default, channels in a *steinbock* panel file generated from IMC raw data are sorted by mass. As the *steinbock* panel format allows for further arbitrary columns, unmapped columns from an original "IMC panel" will be "passed through" to the generated *steinbock* panel. 33 | 34 | When manually creating the *steinbock* panel file, no further actions are required; proceed with image conversion. Otherwise, to create a *steinbock* panel file for IMC data processing: 35 | 36 | steinbock preprocess imc panel 37 | 38 | This will create a *steinbock* panel at the specified location (defaults to `panel.csv`) as follows: 39 | 40 | - If an IMC panel file (in *IMC Segmentation Pipeline*[^1] format, undocumented) exists at the specified location (defaults to `raw/panel.csv`), it is converted to the [*steinbock* panel format](../file-types.md#panel). 41 | - If no IMC panel file was found, the *steinbock* panel is created based on all acquisitions in all .mcd files found at the specified location (defaults to `raw`). 42 | - If no IMC panel file and no .mcd file were found, the *steinbock* panel is created based on all .txt files found at the specified location (defaults to `raw`). 43 | 44 | !!! note "Different panels" 45 | In principle, IMC supports acquiring a different panel for each .mcd/.txt file and acquisition. When creating a *steinbock* panel from .mcd/.txt files, the created panel will contain all targets found in any of the input files. During image conversion (see below), only targets marked as `keep=1` in the panel file will be retained; imaging data with missing channels (identified by the `channel` column in the panel file) are skipped. 46 | 47 | ### Image conversion 48 | 49 | To convert .mcd/.txt files in the raw data directory to TIFF and filter hot pixels: 50 | 51 | steinbock preprocess imc images --hpf 50 52 | 53 | This will extract images from raw files (source directory defaults to `raw`) and save them at the specified location (defaults to `img`). Each image corresponds to one acquisition in one file, with the image channels filtered (`keep` column) and sorted according to the *steinbock* panel file at the specified location (defaults to `panel.csv`). For corrupted .mcd files, *steinbock* will try to recover the missing acquisitions from matching .txt files. In a second step, images from *unmatched* .txt files are extracted as well. 54 | 55 | Furthermore, this commands also creates an image information table as described in [File types](../file-types.md#image-information). In addition to the default columns, the following IMC-specific columns will be added: 56 | 57 | - `source_file`: the raw .mcd/.txt file name 58 | - `recovery_file`: the corresponding .txt file name, if available 59 | - `recovered`: *True* if the .mcd acquisition was recovered from the corresponding .txt file 60 | - Acquisition-specific information (only for images extracted from .mcd files): 61 | - `acquisition_id`: numeric acquisition ID 62 | - `acquisition_description`: user-specified acquisition description 63 | - `acquisition_posx_um`, `acquisition_posy_um`: start position, in micrometers 64 | - `acquisition_width_um`, `acquisition_height_um`: dimensions, in micrometers 65 | 66 | !!! note "IMC file matching" 67 | Matching of .txt files to .mcd files is performed by file name: If a .txt file name starts with the file name of an .mcd file (without extension) AND ends with `_{acquisition}.txt`, where `{acquisition}` is the numeric acquisition ID, it is considered matching that particular acquisition from the .mcd file. 68 | 69 | !!! note "ZIP archives" 70 | If .zip archives are found in the raw data directory, contained .txt/.mcd files will be automatically extracted to a temporary directory, unless disabled using the `--no-unzip` command-line option. After image extraction, this temporary directory and its contents will be removed. 71 | 72 | After image extraction, if the `--hpf` option is specified, the images are filtered for hot pixels. The value of the `--hpf` option (`50` in the example above) determines the *hot pixel filtering threshold*. 73 | 74 | !!! note "Hot pixel filtering" 75 | Hot pixel filtering works by comparing each pixel to its 8-neighborhood (i.e., neighboring pixels at a [Chebyshev distance](https://en.wikipedia.org/wiki/Chebyshev_distance) of 1). If the difference (not: absolute difference) between the pixel and any of its 8 neighbor pixels exceeds a *hot pixel filtering threshold*, the pixel is set to the maximum neighbor pixel value ("hot pixel-filtered"). In the original implementation of the *IMC Segmentation Pipeline*[^1], a *hot pixel filtering threshold* of 50 is recommended. 76 | 77 | [^1]: Zanotelli et al. ImcSegmentationPipeline: A pixel classification-based multiplexed image segmentation pipeline. Zenodo, 2017. DOI: [10.5281/zenodo.3841961](https://doi.org/10.5281/zenodo.3841961). 78 | 79 | ## External images 80 | 81 | *External images* are images preprocessed externally (i.e., without *steinbock*) that are saved in an image format supported by [imageio](https://imageio.readthedocs.io). 82 | 83 | For convenience, to create a template panel file based on external image data stored at the specified location (defaults to `external`): 84 | 85 | steinbock preprocess external panel 86 | 87 | To convert external image data to *steinbock*-supported TIFF files (see [File types](../file-types.md#images)) and save them to the specified location (defaults to `external`): 88 | 89 | steinbock preprocess external images 90 | -------------------------------------------------------------------------------- /docs/cli/utils.md: -------------------------------------------------------------------------------- 1 | # Utilities 2 | 3 | Various built-in utilities are exposed via *steinbock*'s `utils` command. 4 | 5 | ## Matching 6 | 7 | The following command will, for each pair of masks, identify overlapping (intersecting) objects: 8 | 9 | steinbock utils match cell_masks tumor_masks -o matched_objects 10 | 11 | Here, `cell_masks` and `tumor_masks` are path to directories containing masks. Masks from both directories are matched by name. This will generate tables in CSV format (undocumented, one file per mask pair), with each row indicating IDs from overlapping objects in both masks. 12 | 13 | !!! note "Usage example" 14 | Identifying overlapping objects can be useful in multi-segmentation contexts. For example, one may be interested in cells from tumor regions only, in which case two segmentation workflows would be followed sequentially: 15 | 16 | - "Global" tumor/stroma segmentation 17 | - "Local" cell segmentation 18 | 19 | Afterwards, one could match the generated masks to restrict downstream analyses to cells in tumor regions. 20 | 21 | ## Mosaics 22 | 23 | This *steinbock* utility for tiling and stitching images allows the processing of large image files. 24 | 25 | !!! note "Data type" 26 | Unlike other *steinbock* operations, all `mosaic` commands load and save images in their original data type. 27 | 28 | ### Tiling images 29 | 30 | The following command will split all images in `img_full` into tiles of 4096x4096 pixels (the recommended maximum image size for *steinbock* on local installations) and save them to `img`: 31 | 32 | steinbock utils mosaics tile img_full --size 4096 -o img 33 | 34 | The created image tiles will have the following file name, where `{IMG}` is the original file name (without extension), `{X}` and `{Y}` indicate the tile position (in pixels) and `{W}` and `{H}` indicate the tile width and height, respectively: 35 | 36 | {IMG}_tx{X}_ty{Y}_tw{W}_th{H}.tiff 37 | 38 | ### Stitching mosaics 39 | 40 | The following command will stitch all mask tiles in `masks` (following the file conventions above) to assemble masks of original size and save them to `masks_full`: 41 | 42 | steinbock utils mosaics stitch masks -o masks_full 43 | -------------------------------------------------------------------------------- /docs/cli/visualization.md: -------------------------------------------------------------------------------- 1 | # Visualization 2 | 3 | ## Images 4 | 5 | In this step, images and masks are visualized using the [napari](https://napari.org) image viewer. 6 | 7 | !!! danger "Experimental feature" 8 | This is a highly experimental feature. It requires OpenGL-enabled X forwarding and will not work on most systems. Alternatively, one can resort to Xpra-enabled Docker containers to run a steinbock-enabled desktop environment within a web browser (undocumented). 9 | 10 | !!! note "Docker and graphical user interfaces" 11 | Running napari using steinbock Docker containers requires support for graphical user interfaces (e.g. X forwarding). 12 | 13 | To open the image `myimage.tiff` (and, optionally, the corresponding mask) in napari: 14 | 15 | steinbock view myimage.tiff 16 | -------------------------------------------------------------------------------- /docs/contributing.md: -------------------------------------------------------------------------------- 1 | ../CONTRIBUTING.md -------------------------------------------------------------------------------- /docs/contributors.md: -------------------------------------------------------------------------------- 1 | ../CONTRIBUTORS.md -------------------------------------------------------------------------------- /docs/directories.md: -------------------------------------------------------------------------------- 1 | # Directories 2 | 3 | The input and output files and directories of all *steinbock* CLI commands can be controlled via command-line options. However, without explicitly specifying input/output arguments, the following typical directory structure is assumed when working with the *steinbock* CLI: 4 | 5 | steinbock data/working directory 6 | | 7 | ├── raw (user-provided, when starting from raw data) 8 | | 9 | ├── img (user-provided, when not starting from raw data) 10 | ├── panel.csv (user-provided, when not starting from raw data) 11 | ├── images.csv 12 | | 13 | ├── ilastik_img 14 | ├── ilastik_crops 15 | ├── pixel_classifier.ilp 16 | ├── ilastik_probabilities 17 | | 18 | ├── cell_segmentation.cppipe 19 | ├── masks 20 | | 21 | ├── intensities 22 | ├── regionprops 23 | └── neighbors 24 | 25 | Depending on the choice of [preprocessing approaches](cli/preprocessing.md), either the `raw` directory containing the raw data, or the `img` directory containing the images and a `panel.csv` file must be provided by the user. All other files and directories are generated by the *steinbock* Docker container when following a supported workflow. 26 | -------------------------------------------------------------------------------- /docs/file-types.md: -------------------------------------------------------------------------------- 1 | # File types 2 | 3 | ## Panel 4 | 5 | File extension: .csv 6 | 7 | User-provided list of channels present in the images (in order) 8 | 9 | Comma-separated values (CSV) file with column headers and no index 10 | 11 | | Column | Description | Type | Required? | 12 | | --- | --- | --- | --- | 13 | | `channel` | Unique channel ID, e.g. metal isotope | Text | yes | 14 | | `name` | Unique channel name, e.g. antibody target
(can be empty only for rows with `keep=0`) | Text or empty | yes | 15 | | `keep` | Whether the channel is present in preprocessed images
(if column is absent, all channels are assumed present) | Boolean (`0` or `1`) | no | 16 | | `ilastik` | Group label for creating [*steinbock* Ilastik images](cli/classification.md#ilastik)
(if column is absent, all channels are used separately) | Numeric or empty | no | 17 | | `deepcell` | Group label for [DeepCell segmentation](cli/segmentation.md#deepcell)
(if column is absent, all channels are used separately) | Numeric or empty | no | 18 | | `cellpose` | Group label for [Cellpose segmentation](cli/segmentation.md#cellpose)
(if column is absent, all channels are used separately) | Numeric or empty | no | 19 | 20 | The *steinbock* panel allows for further arbitrary columns. 21 | 22 | ## Images 23 | 24 | File extension: .tiff 25 | 26 | Multi-channel images, where each channel corresponds to a panel entry 27 | 28 | Tag Image File Format (TIFF) images of any data type in CYX dimension order 29 | 30 | !!! note "Image data type" 31 | Unless explicitly mentioned, images are converted to 32-bit floating point upon loading (without rescaling). 32 | 33 | ## Image information 34 | 35 | File extension: .csv 36 | 37 | Image information (e.g. image dimensions) extracted during preprocessing 38 | 39 | CSV file with image file name as index (`Image` column) and the following columns: 40 | 41 | | Column | Description | Type | 42 | | --- | --- | --- | 43 | | `image` | Unique image file name | Text | 44 | | `width_px` | Image width, in pixels | Numeric | 45 | | `height_px` | Image height, in pixels | Numeric | 46 | | `num_channels` | Number of image channels | Numeric | 47 | 48 | Further columns may be added by [modality-specific preprocessing commands](cli/preprocessing.md). 49 | 50 | 51 | ## Probabilities 52 | 53 | File extension: .tiff 54 | 55 | Color images, with one color per class encoding the probability of pixels belonging to that class 56 | 57 | 16-bit unsigned integer TIFF images in YXS dimension order, same YX ratio as source image 58 | 59 | !!! danger "Probability image size" 60 | The size of probability images may be different from the original images (see [Ilastik pixel classification](cli/classification.md#ilastik)). 61 | 62 | ## Object masks 63 | 64 | File extension: .tiff 65 | 66 | Grayscale images, with one unique value per object ("object ID", 0 for background) 67 | 68 | 16-bit unsigned integer TIFF images in YX dimension order, same YX shape as source image 69 | 70 | ## Object data 71 | 72 | File extension: .csv 73 | 74 | Object measurements (e.g. mean intensities, morphological features) 75 | 76 | CSV file with object IDs as index (`Object` column) and feature/channel names as columns 77 | 78 | !!! note "Combined object data" 79 | For data containing measurements from multiple images, a combined index of image name and object ID is used. 80 | 81 | ## Object neighbors 82 | 83 | File extension: .csv 84 | 85 | List of directed edges defining a spatial object neighborhood graph 86 | 87 | CSV file (one per image) with no index and three columns (`Object`, `Neighbor`, `Distance`) 88 | 89 | !!! note "Undirected graphs" 90 | For undirected graphs, each edge appears twice (one edge per direction) 91 | -------------------------------------------------------------------------------- /docs/img/steinbock-favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BodenmillerGroup/steinbock/7b73f2925b68a351925e89aa145a7a739741b2b7/docs/img/steinbock-favicon.png -------------------------------------------------------------------------------- /docs/img/steinbock-logo-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BodenmillerGroup/steinbock/7b73f2925b68a351925e89aa145a7a739741b2b7/docs/img/steinbock-logo-white.png -------------------------------------------------------------------------------- /docs/img/steinbock-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BodenmillerGroup/steinbock/7b73f2925b68a351925e89aa145a7a739741b2b7/docs/img/steinbock-logo.png -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | Logo 2 | 3 | # Welcome 4 | 5 | *steinbock* is a toolkit for processing multiplexed tissue images 6 | 7 | The *steinbock* toolkit comprises the following components: 8 | 9 | - The [*steinbock* Python package](https://pypi.org/project/steinbock) with the integrated *steinbock command-line interface* (CLI) 10 | - The [*steinbock* Docker container](https://github.com/BodenmillerGroup/steinbock/pkgs/container/steinbock) interactively exposing the *steinbock* command-line interface, with supported third-party software (e.g. Ilastik, CellProfiler) pre-installed 11 | 12 | !!! note "Modes of usage" 13 | *steinbock* can be used both [interactively](cli/intro.md) using the command-line interface (CLI) and [programmatically](python/intro.md) from within Python scripts. 14 | 15 | ## Overview 16 | 17 | At its core, *steinbock* provides the following functionality: 18 | 19 | - Image preprocessing, including utilities for tiling/stitching images 20 | - Pixel classification, to enable pixel classification-based image segmentation 21 | - Image segmentation, to identify objects (e.g. cells or other regions of interest) 22 | - Object measurement, to extract single-cell data, cell neighbors, etc. 23 | - Data export, to facilitate downstream data analysis 24 | - Visualization of multiplexed tissue images 25 | 26 | !!! note "Downstream single-cell data analysis" 27 | *steinbock* is a toolkit for extracting single-cell data from multiplexed tissue images and NOT for downstream single-cell data analysis. 28 | 29 | While all *steinbock* functionality can be used in a modular fashion, the toolkit was designed for - and explicitly supports - the following image segmentation workflows: 30 | 31 | - **[Ilastik/CellProfiler]** Zanotelli et al. ImcSegmentationPipeline: A pixel classification-based multiplexed image segmentation pipeline. Zenodo, 2017. DOI: [10.5281/zenodo.3841961](https://doi.org/10.5281/zenodo.3841961). 32 | - **[DeepCell/Mesmer]** Greenwald et al. Whole-cell segmentation of tissue images with human-level performance using large-scale data annotation and deep learning. Nature Biotechnology, 2021. DOI: [10.1038/s41587-021-01094-0](https://doi.org/10.1038/s41587-021-01094-0). 33 | - **[Cellpose]** Stringer et al. Cellpose: a generalist algorithm for cellular segmentation. Nature methods, 2021. DOI: [10.1038/s41592-020-01018-x](https://doi.org/10.1038/s41592-020-01018-x) 34 | 35 | The *steinbock* toolkit is extensible and support for further workflows may be added in the future. If you are missing support for a workflow, please consider [filing an issue on GitHub](https://github.com/BodenmillerGroup/steinbock/issues). 36 | 37 | ## Resources 38 | 39 | Code: [https://github.com/BodenmillerGroup/steinbock](https://github.com/BodenmillerGroup/steinbock) 40 | 41 | Documentation: [https://bodenmillergroup.github.io/steinbock](https://bodenmillergroup.github.io/steinbock) 42 | 43 | Issue tracker: [https://github.com/BodenmillerGroup/steinbock/issues](https://github.com/BodenmillerGroup/steinbock/issues) 44 | 45 | Discussions: [https://github.com/BodenmillerGroup/steinbock/discussions](https://github.com/BodenmillerGroup/steinbock/discussions) 46 | 47 | Workshop 2023: [https://github.com/BodenmillerGroup/ImagingWorkshop2023](https://github.com/BodenmillerGroup/ImagingWorkshop2023) 48 | 49 | ## Citing steinbock 50 | 51 | Please cite the following paper when using *steinbock* in your work: 52 | 53 | !!! quote 54 | Windhager, J., Zanotelli, V.R.T., Schulz, D. et al. An end-to-end workflow for multiplexed image processing and analysis. Nat Protoc (2023). https://doi.org/10.1038/s41596-023-00881-0 55 | 56 | ``` 57 | @article{Windhager2023, 58 | author = {Windhager, Jonas and Zanotelli, Vito R.T. and Schulz, Daniel and Meyer, Lasse and Daniel, Michelle and Bodenmiller, Bernd and Eling, Nils}, 59 | title = {An end-to-end workflow for multiplexed image processing and analysis}, 60 | year = {2023}, 61 | doi = {10.1038/s41596-023-00881-0}, 62 | URL = {https://www.nature.com/articles/s41596-023-00881-0}, 63 | journal = {Nature Protocols} 64 | } 65 | ``` 66 | 67 | If you have issues accessing the manuscript, please reach out to us and we can share the PDF version. 68 | -------------------------------------------------------------------------------- /docs/install-python.md: -------------------------------------------------------------------------------- 1 | # Python package 2 | 3 | The *steinbock* toolkit can be used programmatically using the *steinbock* Python package. 4 | 5 | In this section, the installation of the *steinbock* Python package is described. 6 | 7 | !!! danger "For scripting use only" 8 | Installing/using the *steinbock* Python package directly is not recommended for regular users. Please use the *steinbock* Docker containers instead. 9 | 10 | ## Requirements 11 | 12 | [Python](https://www.python.org) 3.8 or newer 13 | 14 | Tested versions of Python package dependencies can be found in [requirements.txt](https://github.com/BodenmillerGroup/steinbock/blob/main/requirements.txt). 15 | 16 | ## Installation 17 | 18 | The *steinbock* Python package can be installed [from PyPI](https://pypi.org/project/steinbock) as follows: 19 | 20 | pip install steinbock 21 | 22 | The following extras are available: 23 | 24 | - `imc` to enable IMC preprocessing functionality 25 | - `deepcell` to enable DeepCell segmentation functionality 26 | - `cellpose` to enable Cellpose segmentation functionality 27 | - `napari` to enable image visualization functionality 28 | 29 | ## Usage 30 | 31 | Please refer to [Python usage](python/intro.md) for usage instructions. 32 | -------------------------------------------------------------------------------- /docs/license.md: -------------------------------------------------------------------------------- 1 | # License 2 | 3 | Copyright (c) 2021 University of Zurich 4 | 5 | *steinbock* is licensed under the MIT License 6 | 7 | [https://github.com/BodenmillerGroup/steinbock/blob/main/LICENSE](https://github.com/BodenmillerGroup/steinbock/blob/main/LICENSE) 8 | -------------------------------------------------------------------------------- /docs/mkdocstrings.css: -------------------------------------------------------------------------------- 1 | /* Indentation. */ 2 | div.doc-contents:not(.first) { 3 | padding-left: 25px; 4 | border-left: 4px solid rgba(230, 230, 230); 5 | margin-bottom: 80px; 6 | } 7 | -------------------------------------------------------------------------------- /docs/python/api/steinbock.classification.md: -------------------------------------------------------------------------------- 1 | # steinbock.classification 2 | 3 | ::: steinbock.classification 4 | -------------------------------------------------------------------------------- /docs/python/api/steinbock.export.md: -------------------------------------------------------------------------------- 1 | # steinbock.export 2 | 3 | ::: steinbock.export 4 | -------------------------------------------------------------------------------- /docs/python/api/steinbock.io.md: -------------------------------------------------------------------------------- 1 | # steinbock.io 2 | 3 | ::: steinbock.io 4 | -------------------------------------------------------------------------------- /docs/python/api/steinbock.measurement.md: -------------------------------------------------------------------------------- 1 | # steinbock.measurement 2 | 3 | ::: steinbock.measurement 4 | -------------------------------------------------------------------------------- /docs/python/api/steinbock.preprocessing.md: -------------------------------------------------------------------------------- 1 | # steinbock.preprocessing 2 | 3 | ::: steinbock.preprocessing 4 | -------------------------------------------------------------------------------- /docs/python/api/steinbock.segmentation.md: -------------------------------------------------------------------------------- 1 | # steinbock.segmentation 2 | 3 | ::: steinbock.segmentation 4 | -------------------------------------------------------------------------------- /docs/python/api/steinbock.utils.md: -------------------------------------------------------------------------------- 1 | # steinbock.utils 2 | 3 | ::: steinbock.utils 4 | -------------------------------------------------------------------------------- /docs/python/api/steinbock.visualization.md: -------------------------------------------------------------------------------- 1 | # steinbock.visualization 2 | 3 | ::: steinbock.visualization 4 | -------------------------------------------------------------------------------- /docs/python/intro.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | The *steinbock* toolkit can be used as a regular Python package: 4 | 5 | import steinbock 6 | 7 | Please refer to the *steinbock* API documentation for further details. 8 | 9 | Usage examples can be found in the [examples](https://github.com/BodenmillerGroup/steinbock/tree/main/examples) folder on GitHub. 10 | 11 | !!! danger "Work in progress" 12 | The *steinbock* API documentation is work in progress. 13 | -------------------------------------------------------------------------------- /entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | test ${RUN_FIXUID} && eval $( fixuid -q ) 3 | python -m steinbock "$@" 4 | -------------------------------------------------------------------------------- /fixuid.yml: -------------------------------------------------------------------------------- 1 | user: steinbock 2 | group: steinbock 3 | paths: 4 | - /run # required for Xpra 5 | - /tmp # required for Xpra 6 | - /home/steinbock 7 | - /data 8 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: steinbock 2 | repo_url: https://github.com/BodenmillerGroup/steinbock 3 | site_description: The steinbock toolkit documentation 4 | copyright: University of Zurich 5 | nav: 6 | - Home: 7 | - Welcome: index.md 8 | - Installation: 9 | - install-docker.md 10 | - install-python.md 11 | - Specifications: 12 | - file-types.md 13 | - directories.md 14 | - Development: 15 | - authors.md 16 | - contributors.md 17 | - contributing.md 18 | - changelog.md 19 | - license.md 20 | - Command-line usage: 21 | - cli/intro.md 22 | - cli/preprocessing.md 23 | - cli/classification.md 24 | - cli/segmentation.md 25 | - cli/measurement.md 26 | - cli/visualization.md 27 | - cli/export.md 28 | - cli/utils.md 29 | - cli/apps.md 30 | - Python usage: 31 | - python/intro.md 32 | - API documentation: 33 | - python/api/steinbock.preprocessing.md 34 | - python/api/steinbock.classification.md 35 | - python/api/steinbock.segmentation.md 36 | - python/api/steinbock.measurement.md 37 | - python/api/steinbock.visualization.md 38 | - python/api/steinbock.export.md 39 | - python/api/steinbock.utils.md 40 | - python/api/steinbock.io.md 41 | - Issues: https://github.com/BodenmillerGroup/steinbock/issues 42 | - Discussions: https://github.com/BodenmillerGroup/steinbock/discussions 43 | theme: 44 | name: material 45 | locale: en 46 | # theme-specific keywords 47 | logo: img/steinbock-logo-white.png 48 | favicon: img/steinbock-favicon.png 49 | features: 50 | - navigation.tabs 51 | - navigation.tabs.sticky 52 | - navigation.expand 53 | - navigation.top 54 | - search.suggest 55 | - search.highlight 56 | extra_css: 57 | - mkdocstrings.css 58 | extra: 59 | version: 60 | provider: mike 61 | watch: 62 | - steinbock 63 | markdown_extensions: 64 | - admonition 65 | - footnotes 66 | plugins: 67 | - search: {} 68 | - mike: 69 | version_selector: true 70 | - mkdocstrings: 71 | default_handler: python 72 | handlers: 73 | python: 74 | paths: 75 | - steinbock 76 | options: 77 | show_root_toc_entry: no 78 | filters: 79 | - '!^_' 80 | show_if_no_docstring: yes 81 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=64", "wheel", "setuptools_scm[toml]>=6.2"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.setuptools_scm] 6 | write_to = "steinbock/_version.py" 7 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # This file lists tested package versions and is used to build the Docker containers 2 | anndata==0.8.0 3 | # cellpose # not included in default Docker containers (see Dockerfile for version) 4 | click==8.1.3 5 | click-log==0.4.0 6 | deepcell==0.12.6 7 | fcswrite==0.6.2 8 | h5py==3.8.0 9 | imageio==2.25.0 10 | lxml_html_clean==0.1.1 11 | napari[all]==0.4.19 12 | networkx==3.0 13 | numpy==1.23.5 # deepcell 0.12.4 requires <1.24 14 | opencv-python-headless==4.7.0.68 15 | pandas==1.5.3 16 | pyyaml==6.0 17 | readimc==0.7.0 18 | scikit-image==0.19.3 19 | scipy==1.10.0 20 | tifffile==2023.1.23.1 21 | xtiff==0.7.9 22 | -------------------------------------------------------------------------------- /requirements_devel.txt: -------------------------------------------------------------------------------- 1 | black 2 | flake8 3 | isort 4 | mypy 5 | pre-commit 6 | -------------------------------------------------------------------------------- /requirements_docs.txt: -------------------------------------------------------------------------------- 1 | mike 2 | mkdocs-material 3 | mkdocstrings[python-legacy] 4 | -------------------------------------------------------------------------------- /requirements_test.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | pytest-cov 3 | requests 4 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = steinbock 3 | url = https://github.com/BodenmillerGroup/steinbock 4 | author = Jonas Windhager 5 | author_email = jonas@windhager.io 6 | maintainer = Milad Adibi 7 | maintainer_email = milad.adibi@uzh.ch 8 | classifiers = 9 | Operating System :: OS Independent 10 | Programming Language :: Python :: 3 11 | Programming Language :: Python :: 3.8 12 | Programming Language :: Python :: 3.9 13 | License :: OSI Approved :: MIT License 14 | license = MIT 15 | license_files = LICENSE 16 | description = A toolkit for processing multiplexed tissue images 17 | long_description = file: README.md 18 | long_description_content_type = text/markdown 19 | 20 | [options] 21 | zip_safe = True 22 | install_requires = 23 | anndata 24 | click 25 | click-log 26 | fcswrite 27 | h5py 28 | imageio 29 | networkx 30 | numpy 31 | opencv-python-headless 32 | pandas 33 | scikit-image 34 | scipy 35 | tifffile 36 | xtiff 37 | python_requires = >=3.8 38 | packages = find_namespace: 39 | 40 | [options.packages.find] 41 | include = steinbock* 42 | 43 | [options.package_data] 44 | steinbock = **/data/* 45 | 46 | [options.extras_require] 47 | imc = 48 | readimc 49 | cellpose = 50 | cellpose 51 | deepcell = 52 | deepcell 53 | pyyaml 54 | napari = 55 | napari[all] 56 | 57 | [options.entry_points] 58 | console_scripts = 59 | steinbock = steinbock._cli:steinbock_cmd_group 60 | -------------------------------------------------------------------------------- /steinbock/__init__.py: -------------------------------------------------------------------------------- 1 | from ._steinbock import SteinbockException, logger 2 | 3 | __all__ = ["SteinbockException", "logger"] 4 | -------------------------------------------------------------------------------- /steinbock/__main__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import click_log 4 | 5 | from ._cli import steinbock_cmd_group 6 | from ._steinbock import logger 7 | 8 | # click_log.basic_config(logger=logger) 9 | logger_handler = click_log.ClickHandler() 10 | logger_handler.formatter = logging.Formatter( 11 | fmt="%(asctime)s %(levelname)s %(name)s - %(message)s" 12 | ) 13 | logger.handlers = [logger_handler] 14 | logger.propagate = False 15 | 16 | if __name__ == "__main__": 17 | steinbock_cmd_group(prog_name="steinbock") 18 | -------------------------------------------------------------------------------- /steinbock/_cli/__init__.py: -------------------------------------------------------------------------------- 1 | from ._cli import steinbock_cmd_group 2 | 3 | __all__ = ["steinbock_cmd_group"] 4 | -------------------------------------------------------------------------------- /steinbock/_cli/_cli.py: -------------------------------------------------------------------------------- 1 | import click 2 | 3 | from .._version import version as steinbock_version 4 | from ..classification._cli import classify_cmd_group 5 | from ..export._cli import export_cmd_group 6 | from ..measurement._cli import measure_cmd_group 7 | from ..preprocessing._cli import preprocess_cmd_group 8 | from ..segmentation._cli import segment_cmd_group 9 | from ..utils._cli import utils_cmd_group 10 | from .apps import apps_cmd_group 11 | from .utils import OrderedClickGroup 12 | from .visualization import view_cmd 13 | 14 | 15 | @click.group(name="steinbock", cls=OrderedClickGroup) 16 | @click.version_option(steinbock_version) 17 | def steinbock_cmd_group(): 18 | pass 19 | 20 | 21 | steinbock_cmd_group.add_command(preprocess_cmd_group) 22 | steinbock_cmd_group.add_command(classify_cmd_group) 23 | steinbock_cmd_group.add_command(segment_cmd_group) 24 | steinbock_cmd_group.add_command(measure_cmd_group) 25 | steinbock_cmd_group.add_command(export_cmd_group) 26 | steinbock_cmd_group.add_command(utils_cmd_group) 27 | steinbock_cmd_group.add_command(view_cmd) 28 | steinbock_cmd_group.add_command(apps_cmd_group) 29 | -------------------------------------------------------------------------------- /steinbock/_cli/apps.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from pathlib import Path 3 | 4 | import click 5 | import click_log 6 | 7 | from .._env import run_captured, use_ilastik_env 8 | from .._steinbock import SteinbockException 9 | from .._steinbock import logger as steinbock_logger 10 | from .utils import OrderedClickGroup, catch_exception 11 | 12 | 13 | @click.group(name="apps", cls=OrderedClickGroup, help="Third-party applications") 14 | def apps_cmd_group(): 15 | pass 16 | 17 | 18 | @apps_cmd_group.command( 19 | name="ilastik", 20 | context_settings={"ignore_unknown_options": True}, 21 | help="Run Ilastik GUI", 22 | add_help_option=False, 23 | ) 24 | @click.option( 25 | "--ilastik", 26 | "ilastik_binary", 27 | type=click.STRING, 28 | default="/opt/ilastik/run_ilastik.sh", 29 | show_default=True, 30 | help="Ilastik binary", 31 | ) 32 | @click.argument("ilastik_args", nargs=-1, type=click.UNPROCESSED) 33 | @click_log.simple_verbosity_option(logger=steinbock_logger) 34 | @catch_exception(handle=SteinbockException) 35 | @use_ilastik_env 36 | def ilastik_cmd(ilastik_binary, ilastik_args, ilastik_env): 37 | args = [ilastik_binary] + list(ilastik_args) 38 | result = run_captured(args, env=ilastik_env) 39 | sys.exit(result.returncode) 40 | 41 | 42 | @apps_cmd_group.command( 43 | name="cellprofiler", 44 | context_settings={"ignore_unknown_options": True}, 45 | help="Run CellProfiler GUI", 46 | add_help_option=False, 47 | ) 48 | @click.option( 49 | "--python", 50 | "python_path", 51 | type=click.Path(dir_okay=False), 52 | default="/opt/cellprofiler-venv/bin/python", 53 | show_default=True, 54 | help="Python path", 55 | ) 56 | @click.option( 57 | "--cellprofiler", 58 | "cellprofiler_module", 59 | type=click.STRING, 60 | default="cellprofiler", 61 | show_default=True, 62 | help="CellProfiler module", 63 | ) 64 | @click.option( 65 | "--plugins-directory", 66 | "cellprofiler_plugin_dir", 67 | type=click.Path(file_okay=False), 68 | default="/opt/cellprofiler_plugins", 69 | show_default=True, 70 | help="Path to the CellProfiler plugin directory", 71 | ) 72 | @click.argument("cellprofiler_args", nargs=-1, type=click.UNPROCESSED) 73 | @click_log.simple_verbosity_option(logger=steinbock_logger) 74 | @catch_exception(handle=SteinbockException) 75 | def cellprofiler_cmd( 76 | python_path, cellprofiler_module, cellprofiler_plugin_dir, cellprofiler_args 77 | ): 78 | args = [python_path, "-m", cellprofiler_module] + list(cellprofiler_args) 79 | if Path(cellprofiler_plugin_dir).is_dir(): 80 | args.append(f"--plugins-directory={cellprofiler_plugin_dir}") 81 | result = run_captured(args) 82 | sys.exit(result.returncode) 83 | 84 | 85 | @apps_cmd_group.command( 86 | name="jupyter", 87 | context_settings={"ignore_unknown_options": True}, 88 | help="Run Jupyter Notebook", 89 | add_help_option=False, 90 | ) 91 | @click.option( 92 | "--python", 93 | "python_path", 94 | type=click.Path(dir_okay=False), 95 | default="python", 96 | show_default=True, 97 | help="Python path", 98 | ) 99 | @click.option( 100 | "--jupyter", 101 | "jupyter_module", 102 | type=click.STRING, 103 | default="jupyter", 104 | show_default=True, 105 | help="Jupyter module", 106 | ) 107 | @click.argument("jupyter_args", nargs=-1, type=click.UNPROCESSED) 108 | @click_log.simple_verbosity_option(logger=steinbock_logger) 109 | @catch_exception(handle=SteinbockException) 110 | def jupyter_cmd(python_path, jupyter_module, jupyter_args): 111 | jupyter_args = list(jupyter_args) 112 | if not any(arg.startswith("--ip=") for arg in jupyter_args): 113 | jupyter_args.append("--ip='0.0.0.0'") 114 | args = [python_path, "-m", jupyter_module, "notebook"] + jupyter_args 115 | result = run_captured(args) 116 | sys.exit(result.returncode) 117 | 118 | 119 | @apps_cmd_group.command( 120 | name="jupyterlab", 121 | context_settings={"ignore_unknown_options": True}, 122 | help="Run Jupyter Lab", 123 | add_help_option=False, 124 | ) 125 | @click.option( 126 | "--python", 127 | "python_path", 128 | type=click.Path(dir_okay=False), 129 | default="python", 130 | show_default=True, 131 | help="Python path", 132 | ) 133 | @click.option( 134 | "--jupyter", 135 | "jupyter_module", 136 | type=click.STRING, 137 | default="jupyter", 138 | show_default=True, 139 | help="Jupyter module", 140 | ) 141 | @click.argument("jupyterlab_args", nargs=-1, type=click.UNPROCESSED) 142 | @click_log.simple_verbosity_option(logger=steinbock_logger) 143 | @catch_exception(handle=SteinbockException) 144 | def jupyterlab_cmd(python_path, jupyter_module, jupyterlab_args): 145 | jupyterlab_args = list(jupyterlab_args) 146 | if not any(arg.startswith("--ip=") for arg in jupyterlab_args): 147 | jupyterlab_args.append("--ip='0.0.0.0'") 148 | args = [python_path, "-m", jupyter_module, "lab"] + jupyterlab_args 149 | result = run_captured(args) 150 | sys.exit(result.returncode) 151 | -------------------------------------------------------------------------------- /steinbock/_cli/utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from collections import OrderedDict 3 | from functools import partial, wraps 4 | from typing import Dict 5 | 6 | import click 7 | from click.core import Command 8 | 9 | from .._steinbock import SteinbockException 10 | 11 | logger = logging.getLogger(__name__.rpartition(".")[0].rpartition(".")[0]) 12 | 13 | 14 | class SteinbockCLIException(SteinbockException): 15 | pass 16 | 17 | 18 | class OrderedClickGroup(click.Group): 19 | def __init__(self, *args, commands=None, **kwargs) -> None: 20 | super(OrderedClickGroup, self).__init__(*args, **kwargs) 21 | self.commands = commands or OrderedDict() 22 | 23 | def list_commands(self, ctx) -> Dict[str, Command]: 24 | return self.commands 25 | 26 | 27 | def catch_exception(func=None, *, handle=SteinbockException): 28 | if not func: 29 | return partial(catch_exception, handle=handle) 30 | 31 | @wraps(func) 32 | def wrapper(*args, **kwargs): 33 | try: 34 | return func(*args, **kwargs) 35 | except handle as e: 36 | raise click.ClickException(e) 37 | 38 | return wrapper 39 | -------------------------------------------------------------------------------- /steinbock/_cli/visualization.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import click 4 | import click_log 5 | 6 | from .. import io 7 | from .._steinbock import SteinbockException 8 | from .._steinbock import logger as steinbock_logger 9 | from ..visualization import view 10 | from .utils import catch_exception 11 | 12 | 13 | @click.command(name="view", help="View image using napari GUI") 14 | @click.option( 15 | "--img", 16 | "img_dir", 17 | type=click.Path(exists=True, file_okay=False), 18 | default="img", 19 | show_default=True, 20 | help="Path to the image directory", 21 | ) 22 | @click.option( 23 | "--masks", 24 | "mask_dirs", 25 | multiple=True, 26 | type=click.Path(exists=True, file_okay=False), 27 | default=["masks"], 28 | show_default=True, 29 | help="Path(s) to the mask directory", 30 | ) 31 | @click.option( 32 | "--panel", 33 | "panel_file", 34 | type=click.Path(exists=True, dir_okay=False), 35 | default="panel.csv", 36 | show_default=True, 37 | help="Path to the panel file", 38 | ) 39 | @click.option( 40 | "--pixelsize", 41 | "pixel_size_um", 42 | type=click.FLOAT, 43 | default=1.0, 44 | show_default=True, 45 | help="Pixel size in micrometers", 46 | ) 47 | @click.argument("img_file_name", type=click.STRING) 48 | @click_log.simple_verbosity_option(logger=steinbock_logger) 49 | @catch_exception(handle=SteinbockException) 50 | def view_cmd(img_dir, mask_dirs, panel_file, pixel_size_um, img_file_name): 51 | img = io.read_image(Path(img_dir) / img_file_name, native_dtype=True) 52 | masks = None 53 | if len(mask_dirs) > 0: 54 | masks = { 55 | f"Mask ({Path(mask_dir).name})": io.read_mask( 56 | Path(mask_dir) / img_file_name, native_dtype=True 57 | ) 58 | for mask_dir in mask_dirs 59 | } 60 | channel_names = None 61 | if Path(panel_file).is_file(): 62 | panel = io.read_panel(panel_file) 63 | if "channel" in panel: 64 | channel_names = panel["name"].tolist() 65 | view(img, masks=masks, channel_names=channel_names, pixel_size_um=pixel_size_um) 66 | -------------------------------------------------------------------------------- /steinbock/_env.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import selectors 4 | import subprocess 5 | import sys 6 | from functools import wraps 7 | 8 | logger = logging.getLogger(__name__.rpartition(".")[0]) 9 | 10 | 11 | def run_captured(args, **popen_kwargs) -> subprocess.CompletedProcess: 12 | with subprocess.Popen( 13 | args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **popen_kwargs 14 | ) as process: 15 | selector = selectors.DefaultSelector() 16 | selector.register(process.stdout, selectors.EVENT_READ) # type: ignore 17 | selector.register(process.stderr, selectors.EVENT_READ) # type: ignore 18 | running = True 19 | while running: 20 | for key, _ in selector.select(): 21 | data = key.fileobj.read1() # type: ignore 22 | if not data: 23 | running = False 24 | elif key.fileobj is process.stdout: 25 | sys.stdout.buffer.write(data) 26 | elif key.fileobj is process.stderr: 27 | sys.stderr.buffer.write(data) 28 | process.wait() 29 | return subprocess.CompletedProcess( 30 | process.args, process.returncode, process.stdout, process.stderr 31 | ) 32 | 33 | 34 | def use_ilastik_env(func): 35 | @wraps(func) 36 | def use_ilastik_env_wrapper(*args, **kwargs): 37 | if "ilastik_env" not in kwargs: 38 | kwargs["ilastik_env"] = os.environ.copy() 39 | kwargs["ilastik_env"].pop("PYTHONPATH", None) 40 | kwargs["ilastik_env"].pop("PYTHONHOME", None) 41 | kwargs["ilastik_env"].pop("LD_LIBRARY_PATH", None) 42 | return func(*args, **kwargs) 43 | 44 | return use_ilastik_env_wrapper 45 | -------------------------------------------------------------------------------- /steinbock/_steinbock.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | logger = logging.getLogger(__name__.rpartition(".")[0]) 4 | 5 | 6 | class SteinbockException(Exception): 7 | pass 8 | -------------------------------------------------------------------------------- /steinbock/classification/__init__.py: -------------------------------------------------------------------------------- 1 | from ._classification import SteinbockClassificationException 2 | 3 | __all__ = ["SteinbockClassificationException"] 4 | -------------------------------------------------------------------------------- /steinbock/classification/_classification.py: -------------------------------------------------------------------------------- 1 | from .._steinbock import SteinbockException 2 | 3 | 4 | class SteinbockClassificationException(SteinbockException): 5 | pass 6 | -------------------------------------------------------------------------------- /steinbock/classification/_cli/__init__.py: -------------------------------------------------------------------------------- 1 | from ._cli import classify_cmd_group 2 | 3 | __all__ = ["classify_cmd_group"] 4 | -------------------------------------------------------------------------------- /steinbock/classification/_cli/_cli.py: -------------------------------------------------------------------------------- 1 | import click 2 | 3 | from ..._cli.utils import OrderedClickGroup 4 | from .ilastik import ilastik_cmd_group 5 | 6 | 7 | @click.group( 8 | name="classify", 9 | cls=OrderedClickGroup, 10 | help="Perform pixel classification to extract probabilities", 11 | ) 12 | def classify_cmd_group(): 13 | pass 14 | 15 | 16 | classify_cmd_group.add_command(ilastik_cmd_group) 17 | -------------------------------------------------------------------------------- /steinbock/classification/ilastik/__init__.py: -------------------------------------------------------------------------------- 1 | from ._ilastik import ( 2 | SteinbockIlastikClassificationException, 3 | create_and_save_ilastik_project, 4 | create_ilastik_crop, 5 | create_ilastik_image, 6 | fix_ilastik_project_file_inplace, 7 | list_ilastik_crop_files, 8 | list_ilastik_image_files, 9 | logger, 10 | read_ilastik_crop, 11 | read_ilastik_image, 12 | run_pixel_classification, 13 | try_create_ilastik_crops_from_disk, 14 | try_create_ilastik_images_from_disk, 15 | try_fix_ilastik_crops_from_disk, 16 | write_ilastik_crop, 17 | write_ilastik_image, 18 | ) 19 | 20 | __all__ = [ 21 | "SteinbockIlastikClassificationException", 22 | "create_and_save_ilastik_project", 23 | "create_ilastik_crop", 24 | "create_ilastik_image", 25 | "fix_ilastik_project_file_inplace", 26 | "list_ilastik_crop_files", 27 | "list_ilastik_image_files", 28 | "logger", 29 | "read_ilastik_crop", 30 | "read_ilastik_image", 31 | "run_pixel_classification", 32 | "try_create_ilastik_crops_from_disk", 33 | "try_create_ilastik_images_from_disk", 34 | "try_fix_ilastik_crops_from_disk", 35 | "write_ilastik_crop", 36 | "write_ilastik_image", 37 | ] 38 | -------------------------------------------------------------------------------- /steinbock/classification/ilastik/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BodenmillerGroup/steinbock/7b73f2925b68a351925e89aa145a7a739741b2b7/steinbock/classification/ilastik/data/__init__.py -------------------------------------------------------------------------------- /steinbock/classification/ilastik/data/pixel_classifier.ilp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BodenmillerGroup/steinbock/7b73f2925b68a351925e89aa145a7a739741b2b7/steinbock/classification/ilastik/data/pixel_classifier.ilp -------------------------------------------------------------------------------- /steinbock/export/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BodenmillerGroup/steinbock/7b73f2925b68a351925e89aa145a7a739741b2b7/steinbock/export/__init__.py -------------------------------------------------------------------------------- /steinbock/export/_cli/__init__.py: -------------------------------------------------------------------------------- 1 | from ._cli import export_cmd_group, histocat_cmd, ome_cmd 2 | 3 | __all__ = ["export_cmd_group", "histocat_cmd", "ome_cmd"] 4 | -------------------------------------------------------------------------------- /steinbock/export/_cli/_cli.py: -------------------------------------------------------------------------------- 1 | import re 2 | from pathlib import Path 3 | 4 | import click 5 | import click_log 6 | import numpy as np 7 | import tifffile 8 | import xtiff 9 | 10 | from ... import io 11 | from ..._cli.utils import OrderedClickGroup, catch_exception, logger 12 | from ..._steinbock import SteinbockException 13 | from ..._steinbock import logger as steinbock_logger 14 | from .data import anndata_cmd, csv_cmd, fcs_cmd 15 | from .graphs import graphs_cmd 16 | 17 | 18 | @click.group( 19 | name="export", 20 | cls=OrderedClickGroup, 21 | help="Export data to third-party formats", 22 | ) 23 | def export_cmd_group(): 24 | pass 25 | 26 | 27 | @export_cmd_group.command(name="ome", help="Export images as OME-TIFF") 28 | @click.option( 29 | "--img", 30 | "img_dir", 31 | type=click.Path(exists=True, file_okay=False), 32 | default="img", 33 | show_default=True, 34 | help="Path to the image directory", 35 | ) 36 | @click.option( 37 | "--panel", 38 | "panel_file", 39 | type=click.Path(exists=True, dir_okay=False), 40 | default="panel.csv", 41 | show_default=True, 42 | help="Path to the panel file", 43 | ) 44 | @click.option( 45 | "-o", 46 | "ome_dir", 47 | type=click.Path(file_okay=False), 48 | default="ome", 49 | show_default=True, 50 | help="Path to the OME-TIFF export directory", 51 | ) 52 | @click_log.simple_verbosity_option(logger=steinbock_logger) 53 | @catch_exception(handle=SteinbockException) 54 | def ome_cmd(img_dir, panel_file, ome_dir): 55 | panel = io.read_panel(panel_file) 56 | channel_names = [ 57 | f"{channel_id}_{channel_name}" 58 | for channel_id, channel_name in zip( 59 | panel["channel"].values, panel["name"].values 60 | ) 61 | ] 62 | Path(ome_dir).mkdir(exist_ok=True) 63 | for img_file in io.list_image_files(img_dir): 64 | img = io.read_image(img_file, native_dtype=True) 65 | ome_file = io._as_path_with_suffix(Path(ome_dir) / img_file.name, ".ome.tiff") 66 | xtiff.to_tiff( 67 | io._to_dtype(img, np.float32), 68 | ome_file, 69 | channel_names=channel_names, 70 | ) 71 | logger.info(ome_file) 72 | del img 73 | 74 | 75 | @export_cmd_group.command(name="histocat", help="Export images to histoCAT for MATLAB") 76 | @click.option( 77 | "--img", 78 | "img_dir", 79 | type=click.Path(exists=True, file_okay=False), 80 | default="img", 81 | show_default=True, 82 | help="Path to the image directory", 83 | ) 84 | @click.option( 85 | "--masks", 86 | "mask_dir", 87 | type=click.Path(file_okay=False), 88 | default="masks", 89 | show_default=True, 90 | help="Path to the mask directory", 91 | ) 92 | @click.option( 93 | "--panel", 94 | "panel_file", 95 | type=click.Path(exists=True, dir_okay=False), 96 | default="panel.csv", 97 | show_default=True, 98 | help="Path to the panel file", 99 | ) 100 | @click.option( 101 | "-o", 102 | "histocat_dir", 103 | type=click.Path(file_okay=False), 104 | default="histocat", 105 | show_default=True, 106 | help="Path to the histoCAT export directory", 107 | ) 108 | @click_log.simple_verbosity_option(logger=steinbock_logger) 109 | @catch_exception(handle=SteinbockException) 110 | def histocat_cmd(img_dir, mask_dir, panel_file, histocat_dir): 111 | panel = io.read_panel(panel_file) 112 | channel_names = [ 113 | f"{channel_id}_{channel_name}" 114 | for channel_id, channel_name in zip( 115 | panel["channel"].values, panel["name"].values 116 | ) 117 | ] 118 | img_files = io.list_image_files(img_dir) 119 | mask_files = None 120 | if Path(mask_dir).is_dir(): 121 | mask_files = io.list_mask_files(mask_dir, base_files=img_files) 122 | Path(histocat_dir).mkdir(exist_ok=True) 123 | for i, img_file in enumerate(img_files): 124 | img = io.read_image(img_file, native_dtype=True) 125 | histocat_img_dir = Path(histocat_dir) / img_file.stem 126 | histocat_img_dir.mkdir(exist_ok=True) 127 | for channel_name, channel_img in zip(channel_names, img): 128 | histocat_img = io._to_dtype(channel_img, np.float32)[ 129 | np.newaxis, np.newaxis, np.newaxis, :, :, np.newaxis 130 | ] 131 | channel_name = re.sub(r"\s*/\s*", "_", channel_name) 132 | channel_name = re.sub(r"\s", "_", channel_name) 133 | histocat_img_file = histocat_img_dir / f"{channel_name}.tiff" 134 | tifffile.imwrite( 135 | histocat_img_file, 136 | data=histocat_img, 137 | imagej=histocat_img.dtype in (np.uint8, np.uint16, np.float32), 138 | ) 139 | logger.info(histocat_img_file) 140 | mask = None 141 | if mask_files is not None: 142 | mask = io.read_mask(mask_files[i], native_dtype=True) 143 | histocat_mask = io._to_dtype(mask, np.uint16)[ 144 | np.newaxis, np.newaxis, np.newaxis, :, :, np.newaxis 145 | ] 146 | histocat_mask_file = histocat_img_dir / f"{img_file.stem}_mask.tiff" 147 | tifffile.imwrite( 148 | histocat_mask_file, 149 | data=histocat_mask, 150 | imagej=histocat_mask.dtype in (np.uint8, np.uint16, np.float32), 151 | ) 152 | logger.info(histocat_mask_file) 153 | del img, mask 154 | 155 | 156 | export_cmd_group.add_command(csv_cmd) 157 | export_cmd_group.add_command(fcs_cmd) 158 | export_cmd_group.add_command(anndata_cmd) 159 | export_cmd_group.add_command(graphs_cmd) 160 | -------------------------------------------------------------------------------- /steinbock/export/_cli/graphs.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import click 4 | import click_log 5 | import networkx as nx 6 | 7 | from ... import io 8 | from ..._cli.utils import catch_exception, logger 9 | from ..._steinbock import SteinbockException 10 | from ..._steinbock import logger as steinbock_logger 11 | from .. import graphs 12 | 13 | 14 | @click.command(name="graphs", help="Export neighbors as spatial object graphs") 15 | @click.option( 16 | "--neighbors", 17 | "neighbors_dir", 18 | type=click.Path(exists=True, file_okay=False), 19 | default="neighbors", 20 | show_default=True, 21 | help="Path to the neighbors directory", 22 | ) 23 | @click.option( 24 | "--data", 25 | "data_dirs", 26 | multiple=True, 27 | type=click.STRING, 28 | help="Object data (e.g. intensities, regionprops) to use as attributes", 29 | ) 30 | @click.option( 31 | "--format", 32 | "graph_format", 33 | type=click.Choice(["graphml", "gexf", "gml"], case_sensitive=False), 34 | default="graphml", 35 | show_default=True, 36 | help="AnnData file format to use", 37 | ) 38 | @click.option( 39 | "-o", 40 | "graph_dir", 41 | type=click.Path(file_okay=False), 42 | default="graphs", 43 | show_default=True, 44 | help="Path to the networkx output directory", 45 | ) 46 | @click_log.simple_verbosity_option(logger=steinbock_logger) 47 | @catch_exception(handle=SteinbockException) 48 | def graphs_cmd(neighbors_dir, data_dirs, graph_format, graph_dir): 49 | neighbors_files = io.list_neighbors_files(neighbors_dir) 50 | data_file_lists = [ 51 | io.list_data_files(data_dir, base_files=neighbors_files) 52 | for data_dir in data_dirs 53 | ] 54 | Path(graph_dir).mkdir(exist_ok=True) 55 | for ( 56 | neighbors_file, 57 | data_files, 58 | graph, 59 | ) in graphs.try_convert_to_networkx_from_disk(neighbors_files, *data_file_lists): 60 | graph_file = Path(graph_dir) / neighbors_file.name 61 | if graph_format == "graphml": 62 | graph_file = io._as_path_with_suffix(graph_file, ".graphml") 63 | nx.write_graphml(graph, str(graph_file)) 64 | elif graph_format == "gexf": 65 | graph_file = io._as_path_with_suffix(graph_file, ".gexf") 66 | nx.write_gexf(graph, str(graph_file)) 67 | elif graph_format == "gml": 68 | graph_file = io._as_path_with_suffix(graph_file, ".gml") 69 | nx.write_gml(graph, str(graph_file)) 70 | else: 71 | raise NotImplementedError() 72 | logger.info(graph_file) 73 | -------------------------------------------------------------------------------- /steinbock/export/data.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from os import PathLike 3 | from pathlib import Path 4 | from typing import Generator, Optional, Sequence, Tuple, Union 5 | 6 | import numpy as np 7 | import pandas as pd 8 | from anndata import AnnData 9 | from scipy.sparse import csr_matrix 10 | 11 | from .. import io 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | def try_convert_to_dataframe_from_disk( 17 | *data_file_lists, 18 | ) -> Generator[Tuple[str, Tuple[Path, ...], pd.DataFrame], None, None]: 19 | for data_files in zip(*data_file_lists): 20 | data_files = tuple(Path(data_file) for data_file in data_files) 21 | img_file_name = io._as_path_with_suffix(data_files[0], ".tiff").name 22 | try: 23 | df = io.read_data(data_files[0]) 24 | for data_file in data_files[1:]: 25 | df = pd.merge( 26 | df, 27 | io.read_data(data_file), 28 | left_index=True, 29 | right_index=True, 30 | ) 31 | yield img_file_name, data_files, df 32 | del df 33 | except Exception as e: 34 | logger.exception( 35 | f"Error creating DataFrame for image {img_file_name}: {e}; " 36 | "skipping image" 37 | ) 38 | 39 | 40 | def try_convert_to_anndata_from_disk( 41 | intensity_files: Sequence[Union[str, PathLike]], 42 | *data_file_lists, 43 | neighbors_files: Optional[Sequence[Union[str, PathLike]]] = None, 44 | panel: Optional[pd.DataFrame] = None, 45 | image_info: Optional[pd.DataFrame] = None, 46 | ) -> Generator[Tuple[str, Path, Tuple[Path, ...], Optional[Path], AnnData], None, None]: 47 | if panel is not None: 48 | panel = panel.set_index("name", drop=False, verify_integrity=True) 49 | if image_info is not None: 50 | image_info = image_info.set_index("image", drop=False, verify_integrity=True) 51 | for i, intensity_file in enumerate(intensity_files): 52 | intensity_file = Path(intensity_file) 53 | data_files = tuple(Path(dfl[i]) for dfl in data_file_lists) 54 | neighbors_file = None 55 | if neighbors_files is not None: 56 | neighbors_file = Path(neighbors_files[i]) 57 | img_file_name = io._as_path_with_suffix(intensity_file, ".tiff").name 58 | try: 59 | x = io.read_data(intensity_file) 60 | obs = None 61 | if len(data_files) > 0: 62 | obs = io.read_data(data_files[0]) 63 | for data_file in data_files[1:]: 64 | obs = pd.merge( 65 | obs, 66 | io.read_data(data_file), 67 | left_index=True, 68 | right_index=True, 69 | ) 70 | obs = obs.loc[x.index, :] 71 | if image_info is not None: 72 | image_obs = ( 73 | pd.concat([image_info.loc[img_file_name, :]] * len(x.index), axis=1) 74 | .transpose() 75 | .astype(image_info.dtypes.to_dict()) 76 | ) 77 | image_obs.index = x.index 78 | image_obs.columns = "image_" + image_obs.columns 79 | image_obs.rename(columns={"image_image": "image"}, inplace=True) 80 | if obs is not None: 81 | obs = pd.merge( 82 | obs, 83 | image_obs, 84 | how="inner", # preserves order of left keys 85 | left_index=True, 86 | right_index=True, 87 | ) 88 | else: 89 | obs = image_obs 90 | var = None 91 | if panel is not None: 92 | var = panel.loc[x.columns, :].copy() 93 | if obs is not None: 94 | obs.index = [f"Object {object_id}" for object_id in x.index] 95 | if var is not None: 96 | var.index = x.columns.astype(str).tolist() 97 | # convert nullable string dtype to generic object dtype 98 | # https://github.com/BodenmillerGroup/steinbock/issues/66 99 | if obs is not None: 100 | for col, dtype in zip(obs.columns, obs.dtypes): 101 | if dtype == "string": 102 | obs[col] = obs[col].astype(str) 103 | if var is not None: 104 | for col, dtype in zip(var.columns, var.dtypes): 105 | if dtype == "string": 106 | var[col] = var[col].astype(str) 107 | adata = AnnData(X=x.values, obs=obs, var=var, dtype=np.float32) 108 | if neighbors_file is not None: 109 | neighbors = io.read_neighbors(neighbors_file) 110 | row_ind = [x.index.get_loc(a) for a in neighbors["Object"]] 111 | col_ind = [x.index.get_loc(b) for b in neighbors["Neighbor"]] 112 | adata.obsp["adj"] = csr_matrix( 113 | ([True] * len(neighbors.index), (row_ind, col_ind)), 114 | shape=(adata.n_obs, adata.n_obs), 115 | dtype=np.uint8, 116 | ) 117 | if neighbors["Distance"].notna().any(): 118 | adata.obsp["dist"] = csr_matrix( 119 | (neighbors["Distance"].values, (row_ind, col_ind)), 120 | shape=(adata.n_obs, adata.n_obs), 121 | dtype=np.float32, 122 | ) 123 | del neighbors 124 | yield ( 125 | img_file_name, 126 | intensity_file, 127 | data_files, 128 | neighbors_file, 129 | adata, 130 | ) 131 | del x, obs, var, adata 132 | except Exception as e: 133 | logger.exception( 134 | f"Error creating AnnData object for image {img_file_name}: {e}; " 135 | "skipping image" 136 | ) 137 | -------------------------------------------------------------------------------- /steinbock/export/graphs.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from collections import Counter 3 | from os import PathLike 4 | from pathlib import Path 5 | from typing import Generator, Sequence, Tuple, Union 6 | 7 | import networkx as nx 8 | import pandas as pd 9 | 10 | from .. import io 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | def convert_to_networkx(neighbors: pd.DataFrame, *data_list) -> nx.Graph: 16 | edges = neighbors[["Object", "Neighbor"]].astype(int).values.tolist() 17 | undirected_edges = [tuple(sorted(edge)) for edge in edges] 18 | is_directed = any([x != 2 for x in Counter(undirected_edges).values()]) 19 | graph: nx.Graph = nx.from_pandas_edgelist( 20 | neighbors, 21 | source="Object", 22 | target="Neighbor", 23 | edge_attr=True, 24 | create_using=nx.DiGraph if is_directed else nx.Graph, 25 | ) 26 | if len(data_list) > 0: 27 | merged_data = data_list[0] 28 | for data in data_list[1:]: 29 | merged_data = pd.merge(merged_data, data, left_index=True, right_index=True) 30 | node_attributes = { 31 | int(object_id): object_data.to_dict() 32 | for object_id, object_data in merged_data.iterrows() 33 | } 34 | nx.set_node_attributes(graph, node_attributes) 35 | return graph 36 | 37 | 38 | def try_convert_to_networkx_from_disk( 39 | neighbors_files: Sequence[Union[str, PathLike]], *data_file_lists 40 | ) -> Generator[Tuple[Path, Tuple[Path, ...], nx.Graph], None, None]: 41 | for neighbors_file, *data_files in zip(neighbors_files, *data_file_lists): 42 | data_files = tuple(Path(data_file) for data_file in data_files) 43 | try: 44 | neighbors = io.read_neighbors(neighbors_file) 45 | data_list = [io.read_data(data_file) for data_file in data_files] 46 | graph = convert_to_networkx(neighbors, *data_list) 47 | yield Path(neighbors_file), data_files, graph 48 | del neighbors, data_list, graph 49 | except Exception as e: 50 | logger.exception(f"Error converting {neighbors_file} to networkx: {e}") 51 | -------------------------------------------------------------------------------- /steinbock/measurement/__init__.py: -------------------------------------------------------------------------------- 1 | from ._measurement import SteinbockMeasurementException 2 | 3 | __all__ = ["SteinbockMeasurementException"] 4 | -------------------------------------------------------------------------------- /steinbock/measurement/_cli/__init__.py: -------------------------------------------------------------------------------- 1 | from ._cli import measure_cmd_group 2 | 3 | __all__ = ["measure_cmd_group"] 4 | -------------------------------------------------------------------------------- /steinbock/measurement/_cli/_cli.py: -------------------------------------------------------------------------------- 1 | import click 2 | 3 | from ..._cli.utils import OrderedClickGroup 4 | from .cellprofiler import cellprofiler_cmd_group 5 | from .intensities import intensities_cmd 6 | from .neighbors import neighbors_cmd 7 | from .regionprops import regionprops_cmd 8 | 9 | 10 | @click.group( 11 | name="measure", 12 | cls=OrderedClickGroup, 13 | help="Extract object data from segmented images", 14 | ) 15 | def measure_cmd_group(): 16 | pass 17 | 18 | 19 | measure_cmd_group.add_command(intensities_cmd) 20 | measure_cmd_group.add_command(regionprops_cmd) 21 | measure_cmd_group.add_command(neighbors_cmd) 22 | measure_cmd_group.add_command(cellprofiler_cmd_group) 23 | -------------------------------------------------------------------------------- /steinbock/measurement/_cli/cellprofiler.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from pathlib import Path 3 | 4 | import click 5 | import click_log 6 | import numpy as np 7 | import tifffile 8 | 9 | from ... import io 10 | from ..._cli.utils import OrderedClickGroup, catch_exception, logger 11 | from ..._steinbock import SteinbockException 12 | from ..._steinbock import logger as steinbock_logger 13 | from .. import cellprofiler 14 | 15 | 16 | @click.group( 17 | name="cellprofiler", 18 | cls=OrderedClickGroup, 19 | help="Run a CellProfiler measurement pipeline (legacy)", 20 | ) 21 | def cellprofiler_cmd_group(): 22 | pass 23 | 24 | 25 | @cellprofiler_cmd_group.command( 26 | name="prepare", help="Prepare a CellProfiler measurement pipeline" 27 | ) 28 | @click.option( 29 | "--img", 30 | "img_dir", 31 | type=click.Path(exists=True, file_okay=False), 32 | default="img", 33 | show_default=True, 34 | help="Path to the image directory", 35 | ) 36 | @click.option( 37 | "--masks", 38 | "mask_dir", 39 | type=click.Path(exists=True, file_okay=False), 40 | default="masks", 41 | show_default=True, 42 | help="Path to the mask directory", 43 | ) 44 | @click.option( 45 | "--panel", 46 | "panel_file", 47 | type=click.Path(exists=True, dir_okay=False), 48 | default="panel.csv", 49 | show_default=True, 50 | help="Path to the panel file", 51 | ) 52 | @click.option( 53 | "--pipeout", 54 | "measurement_pipeline_file", 55 | type=click.Path(dir_okay=False), 56 | default="cell_measurement.cppipe", 57 | show_default=True, 58 | help="Path to the CellProfiler measurement pipeline output file", 59 | ) 60 | @click.option( 61 | "--dataout", 62 | "cpdata_dir", 63 | type=click.Path(file_okay=False), 64 | default="cellprofiler_input", 65 | show_default=True, 66 | help="Path to the CellProfiler input directory", 67 | ) 68 | @click_log.simple_verbosity_option(logger=steinbock_logger) 69 | @catch_exception(handle=SteinbockException) 70 | def prepare_cmd( 71 | img_dir, 72 | mask_dir, 73 | panel_file, 74 | measurement_pipeline_file, 75 | cpdata_dir, 76 | ): 77 | panel = io.read_panel(panel_file) 78 | img_files = io.list_image_files(img_dir) 79 | mask_files = io.list_mask_files(mask_dir, base_files=img_files) 80 | Path(cpdata_dir).mkdir(exist_ok=True) 81 | for img_file, mask_file in zip(img_files, mask_files): 82 | img = io.read_image(img_file, native_dtype=True) 83 | cp_img = io._to_dtype(img, np.uint16)[ 84 | np.newaxis, np.newaxis, :, :, :, np.newaxis 85 | ] 86 | cp_img_file = Path(cpdata_dir) / img_file.name 87 | tifffile.imwrite( 88 | cp_img_file, 89 | data=cp_img, 90 | imagej=cp_img.dtype in (np.uint8, np.uint16, np.float32), 91 | ) 92 | logger.info(cp_img_file) 93 | del img 94 | mask = io.read_mask(mask_file, native_dtype=True) 95 | cp_mask = io._to_dtype(mask, np.uint16)[ 96 | np.newaxis, np.newaxis, np.newaxis, :, :, np.newaxis 97 | ] 98 | cp_mask_file = Path(cpdata_dir) / f"{mask_file.stem}_mask{mask_file.suffix}" 99 | tifffile.imwrite( 100 | cp_mask_file, 101 | data=cp_mask, 102 | imagej=cp_mask.dtype in (np.uint8, np.uint16, np.float32), 103 | ) 104 | logger.info(cp_mask_file) 105 | del mask 106 | cellprofiler.create_and_save_measurement_pipeline( 107 | measurement_pipeline_file, len(panel.index) 108 | ) 109 | logger.info(measurement_pipeline_file) 110 | 111 | 112 | @cellprofiler_cmd_group.command( 113 | name="run", help="Run an object measurement batch using CellProfiler" 114 | ) 115 | @click.option( 116 | "--python", 117 | "python_path", 118 | type=click.Path(exists=True, dir_okay=False), 119 | default="/opt/cellprofiler-venv/bin/python", 120 | show_default=True, 121 | help="Python path", 122 | ) 123 | @click.option( 124 | "--cellprofiler", 125 | "cellprofiler_module", 126 | type=click.STRING, 127 | default="cellprofiler", 128 | show_default=True, 129 | help="CellProfiler module", 130 | ) 131 | @click.option( 132 | "--plugins-directory", 133 | "cellprofiler_plugin_dir", 134 | type=click.Path(file_okay=False), 135 | default="/opt/cellprofiler_plugins", 136 | show_default=True, 137 | help="Path to the CellProfiler plugin directory", 138 | ) 139 | @click.option( 140 | "--pipe", 141 | "measurement_pipeline_file", 142 | type=click.Path(exists=True, dir_okay=False), 143 | default="cell_measurement.cppipe", 144 | show_default=True, 145 | help="Path to the CellProfiler measurement pipeline file", 146 | ) 147 | @click.option( 148 | "--data", 149 | "cpdata_dir", 150 | type=click.Path(exists=True, file_okay=False), 151 | default="cellprofiler_input", 152 | show_default=True, 153 | help="Path to the CellProfiler input directory", 154 | ) 155 | @click.option( 156 | "-o", 157 | "cpout_dir", 158 | type=click.Path(file_okay=False), 159 | default="cellprofiler_output", 160 | show_default=True, 161 | help="Path to the CellProfiler output directory", 162 | ) 163 | @click_log.simple_verbosity_option(logger=steinbock_logger) 164 | @catch_exception(handle=SteinbockException) 165 | def run_cmd( 166 | python_path, 167 | cellprofiler_module, 168 | cellprofiler_plugin_dir, 169 | measurement_pipeline_file, 170 | cpdata_dir, 171 | cpout_dir, 172 | ): 173 | Path(cpout_dir).mkdir(exist_ok=True) 174 | result = cellprofiler.try_measure_objects( 175 | python_path, 176 | cellprofiler_module, 177 | measurement_pipeline_file, 178 | cpdata_dir, 179 | cpout_dir, 180 | cellprofiler_plugin_dir=cellprofiler_plugin_dir, 181 | ) 182 | sys.exit(result.returncode) 183 | -------------------------------------------------------------------------------- /steinbock/measurement/_cli/intensities.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import click 4 | import click_log 5 | 6 | from ... import io 7 | from ..._cli.utils import catch_exception, logger 8 | from ..._steinbock import SteinbockException 9 | from ..._steinbock import logger as steinbock_logger 10 | from ..intensities import IntensityAggregation, try_measure_intensities_from_disk 11 | 12 | _intensity_aggregations = { 13 | "sum": IntensityAggregation.SUM, 14 | "min": IntensityAggregation.MIN, 15 | "max": IntensityAggregation.MAX, 16 | "mean": IntensityAggregation.MEAN, 17 | "median": IntensityAggregation.MEDIAN, 18 | "std": IntensityAggregation.STD, 19 | "var": IntensityAggregation.VAR, 20 | } 21 | 22 | 23 | @click.command(name="intensities", help="Measure object intensities") 24 | @click.option( 25 | "--img", 26 | "img_dir", 27 | type=click.Path(exists=True, file_okay=False), 28 | default="img", 29 | show_default=True, 30 | help="Path to the image directory", 31 | ) 32 | @click.option( 33 | "--masks", 34 | "mask_dir", 35 | type=click.Path(exists=True, file_okay=False), 36 | default="masks", 37 | show_default=True, 38 | help="Path to the mask directory", 39 | ) 40 | @click.option( 41 | "--panel", 42 | "panel_file", 43 | type=click.Path(exists=True, dir_okay=False), 44 | default="panel.csv", 45 | show_default=True, 46 | help="Path to the panel file", 47 | ) 48 | @click.option( 49 | "--aggr", 50 | "intensity_aggregation_name", 51 | type=click.Choice(list(_intensity_aggregations.keys()), case_sensitive=True), 52 | default="mean", 53 | show_default=True, 54 | help="Function for aggregating cell pixels", 55 | ) 56 | @click.option( 57 | "--mmap/--no-mmap", 58 | "mmap", 59 | default=False, 60 | show_default=True, 61 | help="Use memory mapping for reading images/masks", 62 | ) 63 | @click.option( 64 | "-o", 65 | "intensities_dir", 66 | type=click.Path(file_okay=False), 67 | default="intensities", 68 | show_default=True, 69 | help="Path to the object intensities output directory", 70 | ) 71 | @click_log.simple_verbosity_option(logger=steinbock_logger) 72 | @catch_exception(handle=SteinbockException) 73 | def intensities_cmd( 74 | img_dir, 75 | mask_dir, 76 | panel_file, 77 | intensity_aggregation_name, 78 | mmap, 79 | intensities_dir, 80 | ): 81 | panel = io.read_panel(panel_file) 82 | channel_names = panel["name"].tolist() 83 | img_files = io.list_image_files(img_dir) 84 | mask_files = io.list_mask_files(mask_dir, base_files=img_files) 85 | Path(intensities_dir).mkdir(exist_ok=True) 86 | for img_file, mask_file, intensities in try_measure_intensities_from_disk( 87 | img_files, 88 | mask_files, 89 | channel_names, 90 | _intensity_aggregations[intensity_aggregation_name], 91 | mmap=mmap, 92 | ): 93 | intensities_file = io._as_path_with_suffix( 94 | Path(intensities_dir) / img_file.name, ".csv" 95 | ) 96 | io.write_data(intensities, intensities_file) 97 | logger.info(intensities_file) 98 | del intensities 99 | -------------------------------------------------------------------------------- /steinbock/measurement/_cli/neighbors.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import click 4 | import click_log 5 | 6 | from ... import io 7 | from ..._cli.utils import catch_exception, logger 8 | from ..._steinbock import SteinbockException 9 | from ..._steinbock import logger as steinbock_logger 10 | from ..neighbors import NeighborhoodType, try_measure_neighbors_from_disk 11 | 12 | _neighborhood_types = { 13 | "centroids": NeighborhoodType.CENTROID_DISTANCE, 14 | "borders": NeighborhoodType.EUCLIDEAN_BORDER_DISTANCE, 15 | "expansion": NeighborhoodType.EUCLIDEAN_PIXEL_EXPANSION, 16 | } 17 | 18 | 19 | @click.command(name="neighbors", help="Measure object neighbors") 20 | @click.option( 21 | "--masks", 22 | "mask_dir", 23 | type=click.Path(exists=True, file_okay=False), 24 | default="masks", 25 | show_default=True, 26 | help="Path to the mask directory", 27 | ) 28 | @click.option( 29 | "--type", 30 | "neighborhood_type_name", 31 | type=click.Choice(list(_neighborhood_types.keys()), case_sensitive=True), 32 | required=True, 33 | help="Neighborhood type", 34 | ) 35 | @click.option( 36 | "--metric", 37 | "metric", 38 | type=click.STRING, 39 | default="euclidean", 40 | help="Spatial distance metric (see scipy's pdist)", 41 | ) 42 | @click.option( 43 | "--dmax", 44 | "dmax", 45 | type=click.FLOAT, 46 | help="Maximum spatial distance between objects", 47 | ) 48 | @click.option( 49 | "--kmax", 50 | "kmax", 51 | type=click.INT, 52 | help="Maximum number of neighbors per object", 53 | ) 54 | @click.option( 55 | "--mmap/--no-mmap", 56 | "mmap", 57 | default=False, 58 | show_default=True, 59 | help="Use memory mapping for reading images/masks", 60 | ) 61 | @click.option( 62 | "-o", 63 | "neighbors_dir", 64 | type=click.Path(file_okay=False), 65 | default="neighbors", 66 | show_default=True, 67 | help="Path to the object neighbors output directory", 68 | ) 69 | @click_log.simple_verbosity_option(logger=steinbock_logger) 70 | @catch_exception(handle=SteinbockException) 71 | def neighbors_cmd( 72 | mask_dir, neighborhood_type_name, metric, dmax, kmax, mmap, neighbors_dir 73 | ): 74 | mask_files = io.list_mask_files(mask_dir) 75 | Path(neighbors_dir).mkdir(exist_ok=True) 76 | for mask_file, neighbors in try_measure_neighbors_from_disk( 77 | mask_files, 78 | _neighborhood_types[neighborhood_type_name], 79 | metric=metric, 80 | dmax=dmax, 81 | kmax=kmax, 82 | mmap=mmap, 83 | ): 84 | neighbors_file = io._as_path_with_suffix( 85 | Path(neighbors_dir) / Path(mask_file).name, ".csv" 86 | ) 87 | io.write_neighbors(neighbors, neighbors_file) 88 | logger.info(neighbors_file) 89 | del neighbors 90 | -------------------------------------------------------------------------------- /steinbock/measurement/_cli/regionprops.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import click 4 | import click_log 5 | 6 | from ... import io 7 | from ..._cli.utils import catch_exception, logger 8 | from ..._steinbock import SteinbockException 9 | from ..._steinbock import logger as steinbock_logger 10 | from ..regionprops import try_measure_regionprops_from_disk 11 | 12 | 13 | @click.command(name="regionprops", help="Measure object region properties") 14 | @click.option( 15 | "--img", 16 | "img_dir", 17 | type=click.Path(exists=True, file_okay=False), 18 | default="img", 19 | show_default=True, 20 | help="Path to the image directory", 21 | ) 22 | @click.option( 23 | "--masks", 24 | "mask_dir", 25 | type=click.Path(exists=True, file_okay=False), 26 | default="masks", 27 | show_default=True, 28 | help="Path to the mask directory", 29 | ) 30 | @click.option( 31 | "--mmap/--no-mmap", 32 | "mmap", 33 | default=False, 34 | show_default=True, 35 | help="Use memory mapping for reading images/masks", 36 | ) 37 | @click.argument("skimage_regionprops", nargs=-1, type=click.STRING) 38 | @click.option( 39 | "-o", 40 | "regionprops_dir", 41 | type=click.Path(file_okay=False), 42 | default="regionprops", 43 | show_default=True, 44 | help="Path to the object region properties output directory", 45 | ) 46 | @click_log.simple_verbosity_option(logger=steinbock_logger) 47 | @catch_exception(handle=SteinbockException) 48 | def regionprops_cmd(img_dir, mask_dir, mmap, skimage_regionprops, regionprops_dir): 49 | img_files = io.list_image_files(img_dir) 50 | mask_files = io.list_mask_files(mask_dir, base_files=img_files) 51 | Path(regionprops_dir).mkdir(exist_ok=True) 52 | if not skimage_regionprops: 53 | skimage_regionprops = [ 54 | "area", 55 | "centroid", 56 | "axis_major_length", 57 | "axis_minor_length", 58 | "eccentricity", 59 | ] 60 | for img_file, mask_file, regionprops in try_measure_regionprops_from_disk( 61 | img_files, mask_files, skimage_regionprops, mmap=mmap 62 | ): 63 | regionprops_file = io._as_path_with_suffix( 64 | Path(regionprops_dir) / img_file.name, ".csv" 65 | ) 66 | io.write_data(regionprops, regionprops_file) 67 | logger.info(regionprops_file) 68 | del regionprops 69 | -------------------------------------------------------------------------------- /steinbock/measurement/_measurement.py: -------------------------------------------------------------------------------- 1 | from .._steinbock import SteinbockException 2 | 3 | 4 | class SteinbockMeasurementException(SteinbockException): 5 | pass 6 | -------------------------------------------------------------------------------- /steinbock/measurement/cellprofiler/__init__.py: -------------------------------------------------------------------------------- 1 | from ._cellprofiler import create_and_save_measurement_pipeline, try_measure_objects 2 | 3 | __all__ = ["create_and_save_measurement_pipeline", "try_measure_objects"] 4 | -------------------------------------------------------------------------------- /steinbock/measurement/cellprofiler/_cellprofiler.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | from importlib import resources 3 | from os import PathLike 4 | from pathlib import Path 5 | from typing import Union 6 | 7 | from ..._env import run_captured 8 | from . import data as cellprofiler_data 9 | 10 | 11 | def create_and_save_measurement_pipeline( 12 | measurement_pipeline_file: Union[str, PathLike], num_channels: int 13 | ) -> None: 14 | s = resources.read_text(cellprofiler_data, "cell_measurement.cppipe") 15 | s = s.replace("{{NUM_CHANNELS}}", str(num_channels)) 16 | with Path(measurement_pipeline_file).open(mode="w") as f: 17 | f.write(s) 18 | 19 | 20 | def try_measure_objects( 21 | python_path: str, 22 | cellprofiler_module: str, 23 | measurement_pipeline_file: Union[str, PathLike], 24 | cpdata_dir: Union[str, PathLike], 25 | cpout_dir: Union[str, PathLike], 26 | cellprofiler_plugin_dir: Union[str, PathLike, None] = None, 27 | ) -> subprocess.CompletedProcess: 28 | args = [ 29 | python_path, 30 | "-m", 31 | cellprofiler_module, 32 | "-c", 33 | "-r", 34 | "-p", 35 | str(measurement_pipeline_file), 36 | "-i", 37 | str(cpdata_dir), 38 | "-o", 39 | str(cpout_dir), 40 | ] 41 | if cellprofiler_plugin_dir is not None and Path(cellprofiler_plugin_dir).is_dir(): 42 | args.append(f"--plugins-directory={cellprofiler_plugin_dir}") 43 | return run_captured(args) 44 | -------------------------------------------------------------------------------- /steinbock/measurement/cellprofiler/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BodenmillerGroup/steinbock/7b73f2925b68a351925e89aa145a7a739741b2b7/steinbock/measurement/cellprofiler/data/__init__.py -------------------------------------------------------------------------------- /steinbock/measurement/cellprofiler/data/cell_measurement.cppipe: -------------------------------------------------------------------------------- 1 | CellProfiler Pipeline: http://www.cellprofiler.org 2 | Version:5 3 | DateRevision:413 4 | GitHash: 5 | ModuleCount:10 6 | HasImagePlaneDetails:False 7 | 8 | Images:[module_num:1|svn_version:'Unknown'|variable_revision_number:2|show_window:False|notes:[]|batch_state:array([], dtype=uint8)|enabled:True|wants_pause:False] 9 | : 10 | Filter images?:Images only 11 | Select the rule criteria:and (extension does isimage) (directory doesnot containregexp "[\\\\/]\\.") 12 | 13 | Metadata:[module_num:2|svn_version:'Unknown'|variable_revision_number:6|show_window:False|notes:[]|batch_state:array([], dtype=uint8)|enabled:True|wants_pause:False] 14 | Extract metadata?:No 15 | Metadata data type:Text 16 | Metadata types:{} 17 | Extraction method count:1 18 | Metadata extraction method:Extract from file/folder names 19 | Metadata source:File name 20 | Regular expression to extract from file name: 21 | Regular expression to extract from folder name: 22 | Extract metadata from:All images 23 | Select the filtering criteria:and (file does contain "") 24 | Metadata file location:Elsewhere...| 25 | Match file and image metadata:[] 26 | Use case insensitive matching?:No 27 | Metadata file name: 28 | Does cached metadata exist?:No 29 | 30 | NamesAndTypes:[module_num:3|svn_version:'Unknown'|variable_revision_number:8|show_window:False|notes:[]|batch_state:array([], dtype=uint8)|enabled:True|wants_pause:False] 31 | Assign a name to:Images matching rules 32 | Select the image type:Grayscale image 33 | Name to assign these images: 34 | Match metadata:[{'Image': None, 'Mask': None}] 35 | Image set matching method:Order 36 | Set intensity range from:Image bit-depth 37 | Assignments count:2 38 | Single images count:0 39 | Maximum intensity:1 40 | Process as 3D?:No 41 | Relative pixel spacing in X:1.0 42 | Relative pixel spacing in Y:1.0 43 | Relative pixel spacing in Z:1.0 44 | Select the rule criteria:and (file doesnot endwith "_mask.tiff") 45 | Name to assign these images:Image 46 | Name to assign these objects:Cell 47 | Select the image type:Color image 48 | Set intensity range from:Image bit-depth 49 | Maximum intensity:255.0 50 | Select the rule criteria:and (file does endwith "_mask.tiff") 51 | Name to assign these images:Mask 52 | Name to assign these objects:cell 53 | Select the image type:Grayscale image 54 | Set intensity range from:Image bit-depth 55 | Maximum intensity:255.0 56 | 57 | Groups:[module_num:4|svn_version:'Unknown'|variable_revision_number:2|show_window:False|notes:[]|batch_state:array([], dtype=uint8)|enabled:True|wants_pause:False] 58 | Do you want to group your images?:No 59 | grouping metadata count:1 60 | Metadata category:Site 61 | 62 | ConvertImageToObjects:[module_num:5|svn_version:'Unknown'|variable_revision_number:1|show_window:False|notes:[]|batch_state:array([], dtype=uint8)|enabled:True|wants_pause:False] 63 | Select the input image:Mask 64 | Name the output object:Cell 65 | Convert to boolean image:No 66 | Preserve original labels:Yes 67 | Background label:0 68 | Connectivity:0 69 | 70 | MeasureObjectNeighbors:[module_num:6|svn_version:'Unknown'|variable_revision_number:3|show_window:False|notes:[]|batch_state:array([], dtype=uint8)|enabled:True|wants_pause:False] 71 | Select objects to measure:Cell 72 | Select neighboring objects to measure:Cell 73 | Method to determine neighbors:Within a specified distance 74 | Neighbor distance:8 75 | Consider objects discarded for touching image border?:Yes 76 | Retain the image of objects colored by numbers of neighbors?:No 77 | Name the output image: 78 | Select colormap:Default 79 | Retain the image of objects colored by percent of touching pixels?:No 80 | Name the output image: 81 | Select colormap:Default 82 | 83 | MeasureObjectIntensityMultichannel:[module_num:7|svn_version:'Unknown'|variable_revision_number:4|show_window:False|notes:[]|batch_state:array([], dtype=uint8)|enabled:True|wants_pause:False] 84 | Select images to measure:Image 85 | Select objects to measure:Cell 86 | How many channels does the image have?:{{NUM_CHANNELS}} 87 | 88 | MeasureImageIntensityMultichannel:[module_num:8|svn_version:'Unknown'|variable_revision_number:3|show_window:False|notes:[]|batch_state:array([], dtype=uint8)|enabled:True|wants_pause:False] 89 | Select images to measure:Image 90 | Measure the intensity only from areas enclosed by objects?:No 91 | Select input object sets: 92 | How many channels does the image have?:{{NUM_CHANNELS}} 93 | 94 | ExportToSpreadsheet:[module_num:9|svn_version:'Unknown'|variable_revision_number:13|show_window:False|notes:[]|batch_state:array([], dtype=uint8)|enabled:True|wants_pause:False] 95 | Select the column delimiter:Comma (",") 96 | Add image metadata columns to your object data file?:No 97 | Add image file and folder names to your object data file?:No 98 | Select the measurements to export:No 99 | Calculate the per-image mean values for object measurements?:No 100 | Calculate the per-image median values for object measurements?:No 101 | Calculate the per-image standard deviation values for object measurements?:No 102 | Output file location:Default Output Folder| 103 | Create a GenePattern GCT file?:No 104 | Select source of sample row name:Metadata 105 | Select the image to use as the identifier:None 106 | Select the metadata to use as the identifier:None 107 | Export all measurement types?:Yes 108 | Press button to select measurements: 109 | Representation of Nan/Inf:NaN 110 | Add a prefix to file names?:No 111 | Filename prefix:MyExpt_ 112 | Overwrite existing files without warning?:Yes 113 | Data to export:Do not use 114 | Combine these object measurements with those of the previous object?:No 115 | File name:DATA.csv 116 | Use the object name for the file name?:Yes 117 | 118 | ExportToSpreadsheet:[module_num:10|svn_version:'Unknown'|variable_revision_number:13|show_window:False|notes:[]|batch_state:array([], dtype=uint8)|enabled:True|wants_pause:False] 119 | Select the column delimiter:Comma (",") 120 | Add image metadata columns to your object data file?:No 121 | Add image file and folder names to your object data file?:No 122 | Select the measurements to export:No 123 | Calculate the per-image mean values for object measurements?:No 124 | Calculate the per-image median values for object measurements?:No 125 | Calculate the per-image standard deviation values for object measurements?:No 126 | Output file location:Default Output Folder| 127 | Create a GenePattern GCT file?:No 128 | Select source of sample row name:Metadata 129 | Select the image to use as the identifier:None 130 | Select the metadata to use as the identifier:None 131 | Export all measurement types?:No 132 | Press button to select measurements: 133 | Representation of Nan/Inf:NaN 134 | Add a prefix to file names?:No 135 | Filename prefix:MyExpt_ 136 | Overwrite existing files without warning?:Yes 137 | Data to export:Object relationships 138 | Combine these object measurements with those of the previous object?:No 139 | File name:DATA.csv 140 | Use the object name for the file name?:Yes 141 | -------------------------------------------------------------------------------- /steinbock/measurement/intensities.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from enum import Enum 3 | from functools import partial 4 | from os import PathLike 5 | from pathlib import Path 6 | from typing import Generator, Sequence, Tuple, Union 7 | 8 | import numpy as np 9 | import pandas as pd 10 | import scipy.ndimage 11 | 12 | from .. import io 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | 17 | class IntensityAggregation(Enum): 18 | SUM = partial(scipy.ndimage.sum_labels) 19 | MIN = partial(scipy.ndimage.minimum) 20 | MAX = partial(scipy.ndimage.maximum) 21 | MEAN = partial(scipy.ndimage.mean) 22 | MEDIAN = partial(scipy.ndimage.median) 23 | STD = partial(scipy.ndimage.standard_deviation) 24 | VAR = partial(scipy.ndimage.variance) 25 | 26 | 27 | def measure_intensites( 28 | img: np.ndarray, 29 | mask: np.ndarray, 30 | channel_names: Sequence[str], 31 | intensity_aggregation: IntensityAggregation, 32 | ) -> pd.DataFrame: 33 | object_ids = np.unique(mask[mask != 0]) 34 | data = { 35 | channel_name: intensity_aggregation.value(img[i], labels=mask, index=object_ids) 36 | for i, channel_name in enumerate(channel_names) 37 | } 38 | return pd.DataFrame( 39 | data=data, 40 | index=pd.Index(object_ids, dtype=io.mask_dtype, name="Object"), 41 | ) 42 | 43 | 44 | def try_measure_intensities_from_disk( 45 | img_files: Sequence[Union[str, PathLike]], 46 | mask_files: Sequence[Union[str, PathLike]], 47 | channel_names: Sequence[str], 48 | intensity_aggregation: IntensityAggregation, 49 | mmap: bool = False, 50 | ) -> Generator[Tuple[Path, Path, pd.DataFrame], None, None]: 51 | for img_file, mask_file in zip(img_files, mask_files): 52 | try: 53 | if mmap: 54 | img = io.mmap_image(img_file) 55 | mask = io.mmap_mask(mask_file) 56 | else: 57 | img = io.read_image(img_file) 58 | mask = io.read_mask(mask_file) 59 | intensities = measure_intensites( 60 | img, mask, channel_names, intensity_aggregation 61 | ) 62 | del img, mask 63 | yield Path(img_file), Path(mask_file), intensities 64 | del intensities 65 | except Exception as e: 66 | logger.exception(f"Error measuring intensities in {img_file}: {e}") 67 | -------------------------------------------------------------------------------- /steinbock/measurement/regionprops.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from os import PathLike 3 | from pathlib import Path 4 | from typing import Generator, Sequence, Tuple, Union 5 | 6 | import numpy as np 7 | import pandas as pd 8 | from skimage.measure import regionprops_table 9 | 10 | from .. import io 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | def measure_regionprops( 16 | img: np.ndarray, mask: np.ndarray, skimage_regionprops: Sequence[str] 17 | ) -> pd.DataFrame: 18 | skimage_regionprops = list(skimage_regionprops) 19 | if "label" not in skimage_regionprops: 20 | skimage_regionprops.insert(0, "label") 21 | data = regionprops_table( 22 | mask, 23 | intensity_image=np.moveaxis(img, 0, -1), 24 | properties=skimage_regionprops, 25 | ) 26 | object_ids = data.pop("label") 27 | return pd.DataFrame( 28 | data=data, 29 | index=pd.Index(object_ids, dtype=io.mask_dtype, name="Object"), 30 | ) 31 | 32 | 33 | def try_measure_regionprops_from_disk( 34 | img_files: Sequence[Union[str, PathLike]], 35 | mask_files: Sequence[Union[str, PathLike]], 36 | skimage_regionprops: Sequence[str], 37 | mmap: bool = False, 38 | ) -> Generator[Tuple[Path, Path, pd.DataFrame], None, None]: 39 | for img_file, mask_file in zip(img_files, mask_files): 40 | try: 41 | if mmap: 42 | img = io.mmap_image(img_file) 43 | mask = io.mmap_mask(mask_file) 44 | else: 45 | img = io.read_image(img_file) 46 | mask = io.read_mask(mask_file) 47 | regionprops = measure_regionprops(img, mask, skimage_regionprops) 48 | del img, mask 49 | yield Path(img_file), Path(mask_file), regionprops 50 | del regionprops 51 | except Exception as e: 52 | logger.exception(f"Error measuring regionprops in {img_file}: {e}") 53 | -------------------------------------------------------------------------------- /steinbock/preprocessing/__init__.py: -------------------------------------------------------------------------------- 1 | from ._preprocessing import SteinbockPreprocessingException 2 | 3 | __all__ = ["SteinbockPreprocessingException"] 4 | -------------------------------------------------------------------------------- /steinbock/preprocessing/_cli/__init__.py: -------------------------------------------------------------------------------- 1 | from ._cli import preprocess_cmd_group 2 | 3 | __all__ = ["preprocess_cmd_group"] 4 | -------------------------------------------------------------------------------- /steinbock/preprocessing/_cli/_cli.py: -------------------------------------------------------------------------------- 1 | import click 2 | 3 | from ..._cli.utils import OrderedClickGroup 4 | from .external import external_cmd_group 5 | from .imc import imc_cli_available, imc_cmd_group 6 | 7 | 8 | @click.group( 9 | name="preprocess", 10 | cls=OrderedClickGroup, 11 | help="Extract and preprocess images from raw data", 12 | ) 13 | def preprocess_cmd_group(): 14 | pass 15 | 16 | 17 | if imc_cli_available: 18 | preprocess_cmd_group.add_command(imc_cmd_group) 19 | 20 | preprocess_cmd_group.add_command(external_cmd_group) 21 | -------------------------------------------------------------------------------- /steinbock/preprocessing/_cli/external.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import click 4 | import click_log 5 | import numpy as np 6 | import pandas as pd 7 | 8 | from ... import io 9 | from ..._cli.utils import OrderedClickGroup, catch_exception, logger 10 | from ..._steinbock import SteinbockException 11 | from ..._steinbock import logger as steinbock_logger 12 | from .. import external 13 | 14 | 15 | @click.group( 16 | name="external", 17 | cls=OrderedClickGroup, 18 | help="Preprocess external image data", 19 | ) 20 | def external_cmd_group(): 21 | pass 22 | 23 | 24 | @external_cmd_group.command( 25 | name="panel", help="Create a panel from external image data" 26 | ) 27 | @click.option( 28 | "--img", 29 | "ext_img_dir", 30 | type=click.Path(exists=True, file_okay=False), 31 | default="external", 32 | show_default=True, 33 | help="Path to the external image file directory", 34 | ) 35 | @click.option( 36 | "-o", 37 | "panel_file", 38 | type=click.Path(dir_okay=False), 39 | default="panel.csv", 40 | show_default=True, 41 | help="Path to the panel output file", 42 | ) 43 | @click_log.simple_verbosity_option(logger=steinbock_logger) 44 | @catch_exception(handle=SteinbockException) 45 | def panel_cmd(ext_img_dir, panel_file): 46 | ext_img_files = external.list_image_files(ext_img_dir) 47 | panel = external.create_panel_from_image_files(ext_img_files) 48 | io.write_panel(panel, panel_file) 49 | logger.info(panel_file) 50 | 51 | 52 | @external_cmd_group.command(name="images", help="Extract external images") 53 | @click.option( 54 | "--img", 55 | "ext_img_dir", 56 | type=click.Path(exists=True, file_okay=False), 57 | default="external", 58 | show_default=True, 59 | help="Path to the external image file directory", 60 | ) 61 | @click.option( 62 | "--panel", 63 | "panel_file", 64 | type=click.Path(dir_okay=False), 65 | default="panel.csv", 66 | show_default=True, 67 | help="Path to the steinbock panel file", 68 | ) 69 | @click.option( 70 | "--mmap/--no-mmap", 71 | "mmap", 72 | default=False, 73 | show_default=True, 74 | help="Use memory mapping for writing images", 75 | ) 76 | @click.option( 77 | "--imgout", 78 | "img_dir", 79 | type=click.Path(file_okay=False), 80 | default="img", 81 | show_default=True, 82 | help="Path to the image output directory", 83 | ) 84 | @click.option( 85 | "--infoout", 86 | "image_info_file", 87 | type=click.Path(dir_okay=False), 88 | default="images.csv", 89 | show_default=True, 90 | help="Path to the image information output file", 91 | ) 92 | @click_log.simple_verbosity_option(logger=steinbock_logger) 93 | @catch_exception(handle=SteinbockException) 94 | def images_cmd(ext_img_dir, panel_file, mmap, img_dir, image_info_file): 95 | channel_indices = None 96 | if Path(panel_file).is_file(): 97 | panel = io.read_panel(panel_file) 98 | if "channel" in panel: 99 | channel_indices = panel["channel"].astype(int).sub(1).tolist() 100 | ext_img_files = external.list_image_files(ext_img_dir) 101 | image_info_data = [] 102 | Path(img_dir).mkdir(exist_ok=True) 103 | for ext_img_file, img in external.try_preprocess_images_from_disk(ext_img_files): 104 | # filter channels here rather than in try_preprocess_images_from_disk, 105 | # to avoid advanced indexing creating a copy of img (relevant for mmap) 106 | if channel_indices is not None: 107 | if max(channel_indices) > img.shape[0]: 108 | logger.warning( 109 | f"Channel indices out of bounds for file {ext_img_file} " 110 | f"with {img.shape[0]} channels" 111 | ) 112 | continue 113 | cur_channel_indices = channel_indices 114 | else: 115 | cur_channel_indices = list(range(img.shape[0])) 116 | img_file = io._as_path_with_suffix( 117 | Path(img_dir) / Path(ext_img_file).name, ".tiff" 118 | ) 119 | out_shape = list(img.shape) 120 | out_shape[0] = len(cur_channel_indices) 121 | out_shape = tuple(out_shape) 122 | if mmap: 123 | out = io.mmap_image(img_file, mode="r+", shape=out_shape, dtype=img.dtype) 124 | else: 125 | out = np.empty(out_shape, dtype=img.dtype) 126 | for i, channel_index in enumerate(cur_channel_indices): 127 | out[i, :, :] = img[channel_index, :, :] 128 | if mmap: 129 | out.flush() 130 | if not mmap: 131 | io.write_image(out, img_file) 132 | image_info_row = { 133 | "image": img_file.name, 134 | "width_px": img.shape[2], 135 | "height_px": img.shape[1], 136 | "num_channels": img.shape[0], 137 | } 138 | image_info_data.append(image_info_row) 139 | logger.info(img_file) 140 | del img 141 | image_info = pd.DataFrame(data=image_info_data) 142 | io.write_image_info(image_info, image_info_file) 143 | -------------------------------------------------------------------------------- /steinbock/preprocessing/_preprocessing.py: -------------------------------------------------------------------------------- 1 | from .._steinbock import SteinbockException 2 | 3 | 4 | class SteinbockPreprocessingException(SteinbockException): 5 | pass 6 | -------------------------------------------------------------------------------- /steinbock/preprocessing/external.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from os import PathLike 3 | from pathlib import Path 4 | from typing import Generator, List, Sequence, Tuple, Union 5 | 6 | import imageio 7 | import numpy as np 8 | import pandas as pd 9 | 10 | from .. import io 11 | from ._preprocessing import SteinbockPreprocessingException 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | class SteinbockExternalPreprocessingException(SteinbockPreprocessingException): 17 | pass 18 | 19 | 20 | def _read_external_image(ext_img_file: Union[str, PathLike]) -> np.ndarray: 21 | # try tifffile-based reading first, for memory reasons 22 | try: 23 | return io.read_image(ext_img_file, native_dtype=True) 24 | except Exception: 25 | pass # skipped intentionally 26 | ext_img = imageio.volread(ext_img_file) 27 | orig_img_shape = ext_img.shape 28 | while ext_img.ndim > 3 and ext_img.shape[0] == 1: 29 | ext_img = np.squeeze(ext_img, axis=0) 30 | while ext_img.ndim > 3 and ext_img.shape[-1] == 1: 31 | ext_img = np.squeeze(ext_img, axis=-1) 32 | if ext_img.ndim == 2: 33 | ext_img = ext_img[np.newaxis, :, :] 34 | elif ext_img.ndim != 3: 35 | raise SteinbockExternalPreprocessingException( 36 | f"Unsupported shape {orig_img_shape} for image {ext_img_file}" 37 | ) 38 | return ext_img 39 | 40 | 41 | def list_image_files(ext_img_dir: Union[str, PathLike]) -> List[Path]: 42 | return sorted(Path(ext_img_dir).rglob("[!.]*.*")) 43 | 44 | 45 | def create_panel_from_image_files( 46 | ext_img_files: Sequence[Union[str, PathLike]] 47 | ) -> pd.DataFrame: 48 | num_channels = None 49 | for ext_img_file in ext_img_files: 50 | try: 51 | ext_img = _read_external_image(ext_img_file) 52 | num_channels = ext_img.shape[0] 53 | break 54 | except Exception: 55 | pass # skipped intentionally 56 | if num_channels is None: 57 | raise SteinbockExternalPreprocessingException("No valid images found") 58 | panel = pd.DataFrame( 59 | data={ 60 | "channel": range(1, num_channels + 1), 61 | "name": np.nan, 62 | "keep": True, 63 | "ilastik": range(1, num_channels + 1), 64 | "deepcell": np.nan, 65 | "cellpose": np.nan, 66 | }, 67 | ) 68 | panel["channel"] = panel["channel"].astype(pd.StringDtype()) 69 | panel["name"] = panel["name"].astype(pd.StringDtype()) 70 | panel["keep"] = panel["keep"].astype(pd.BooleanDtype()) 71 | panel["ilastik"] = panel["ilastik"].astype(pd.UInt8Dtype()) 72 | panel["deepcell"] = panel["deepcell"].astype(pd.UInt8Dtype()) 73 | panel["cellpose"] = panel["cellpose"].astype(pd.UInt8Dtype()) 74 | return panel 75 | 76 | 77 | def try_preprocess_images_from_disk( 78 | ext_img_files: Sequence[Union[str, PathLike]] 79 | ) -> Generator[Tuple[Path, np.ndarray], None, None]: 80 | for ext_img_file in ext_img_files: 81 | try: 82 | img = _read_external_image(ext_img_file) 83 | except Exception: 84 | logger.warning(f"Unsupported file format: {ext_img_file}") 85 | continue 86 | yield Path(ext_img_file), img 87 | del img 88 | -------------------------------------------------------------------------------- /steinbock/segmentation/__init__.py: -------------------------------------------------------------------------------- 1 | from ._segmentation import SteinbockSegmentationException 2 | 3 | __all__ = ["SteinbockSegmentationException"] 4 | -------------------------------------------------------------------------------- /steinbock/segmentation/_cli/__init__.py: -------------------------------------------------------------------------------- 1 | from ._cli import segment_cmd_group 2 | 3 | __all__ = ["segment_cmd_group"] 4 | -------------------------------------------------------------------------------- /steinbock/segmentation/_cli/_cli.py: -------------------------------------------------------------------------------- 1 | import click 2 | 3 | from ..._cli.utils import OrderedClickGroup 4 | from .cellpose import cellpose_cli_available, cellpose_cmd 5 | from .cellprofiler import cellprofiler_cmd_group 6 | from .deepcell import deepcell_cli_available, deepcell_cmd 7 | 8 | 9 | @click.group( 10 | name="segment", 11 | cls=OrderedClickGroup, 12 | help="Perform image segmentation to create object masks", 13 | ) 14 | def segment_cmd_group(): 15 | pass 16 | 17 | 18 | segment_cmd_group.add_command(cellprofiler_cmd_group) 19 | if deepcell_cli_available: 20 | segment_cmd_group.add_command(deepcell_cmd) 21 | if cellpose_cli_available: 22 | segment_cmd_group.add_command(cellpose_cmd) 23 | -------------------------------------------------------------------------------- /steinbock/segmentation/_cli/cellpose.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import click 4 | import click_log 5 | import numpy as np 6 | 7 | from ... import io 8 | from ..._cli.utils import catch_exception, logger 9 | from ..._steinbock import SteinbockException 10 | from ..._steinbock import logger as steinbock_logger 11 | from .. import cellpose 12 | 13 | cellpose_cli_available = cellpose.cellpose_available 14 | 15 | 16 | @click.command(name="cellpose", help="Run an object segmentation batch using Cellpose") 17 | @click.option( 18 | "--model", 19 | "model_name", 20 | type=click.Choice(["nuclei", "cyto", "cyto2"]), 21 | default="cyto2", 22 | show_default=True, 23 | help="Name of the Cellpose model", 24 | ) 25 | @click.option( 26 | "--img", 27 | "img_dir", 28 | type=click.Path(exists=True, file_okay=False), 29 | default="img", 30 | show_default=True, 31 | help="Path to the image directory", 32 | ) 33 | @click.option( 34 | "--minmax/--no-minmax", 35 | "channelwise_minmax", 36 | default=False, 37 | show_default=True, 38 | help="Channel-wise min-max normalization", 39 | ) 40 | @click.option( 41 | "--zscore/--no-zscore", 42 | "channelwise_zscore", 43 | default=False, 44 | show_default=True, 45 | help="Channel-wise z-score normalization", 46 | ) 47 | @click.option( 48 | "--panel", 49 | "panel_file", 50 | type=click.Path(exists=True, dir_okay=False), 51 | default="panel.csv", 52 | show_default=True, 53 | help="Path to the panel file", 54 | ) 55 | @click.option( 56 | "--aggr", 57 | "aggr_func_name", 58 | type=click.STRING, 59 | default="mean", 60 | show_default=True, 61 | help="Numpy function for aggregating channel pixels", 62 | ) 63 | @click.option( 64 | "--net-avg/--no-net-avg", 65 | "net_avg", 66 | default=True, 67 | show_default=True, 68 | help="See Cellpose documentation", 69 | ) 70 | @click.option( 71 | "--batch-size", 72 | "batch_size", 73 | type=click.INT, 74 | default=8, 75 | show_default=True, 76 | help="See Cellpose documentation", 77 | ) 78 | @click.option( 79 | "--normalize/--no-normalize", 80 | "normalize", 81 | default=True, 82 | show_default=True, 83 | help="See Cellpose documentation", 84 | ) 85 | @click.option( 86 | "--diameter", 87 | "diameter", 88 | type=click.FLOAT, 89 | help="See Cellpose documentation", 90 | ) 91 | @click.option( 92 | "--tile/--no-tile", 93 | "tile", 94 | default=False, 95 | show_default=True, 96 | help="See Cellpose documentation", 97 | ) 98 | @click.option( 99 | "--tile-overlap", 100 | "tile_overlap", 101 | type=click.FLOAT, 102 | default=0.1, 103 | show_default=True, 104 | help="See Cellpose documentation", 105 | ) 106 | @click.option( 107 | "--resample/--no-resample", 108 | "resample", 109 | default=True, 110 | show_default=True, 111 | help="See Cellpose documentation", 112 | ) 113 | @click.option( 114 | "--interp/--no-interp", 115 | "interp", 116 | default=True, 117 | show_default=True, 118 | help="See Cellpose documentation", 119 | ) 120 | @click.option( 121 | "--flow-threshold", 122 | "flow_threshold", 123 | type=click.FLOAT, 124 | default=0.4, 125 | show_default=True, 126 | help="See Cellpose documentation", 127 | ) 128 | @click.option( 129 | "--cellprobab-threshold", 130 | "cellprobab_threshold", 131 | type=click.FLOAT, 132 | default=0.0, 133 | show_default=True, 134 | help="See Cellpose documentation", 135 | ) 136 | @click.option( 137 | "--min-size", 138 | "min_size", 139 | type=click.INT, 140 | default=15, 141 | show_default=True, 142 | help="See Cellpose documentation", 143 | ) 144 | @click.option( 145 | "-o", 146 | "mask_dir", 147 | type=click.Path(file_okay=False), 148 | default="masks", 149 | show_default=True, 150 | help="Path to the mask output directory", 151 | ) 152 | @click_log.simple_verbosity_option(logger=steinbock_logger) 153 | @catch_exception(handle=SteinbockException) 154 | def cellpose_cmd( 155 | model_name: str, 156 | img_dir, 157 | channelwise_minmax, 158 | channelwise_zscore, 159 | panel_file, 160 | aggr_func_name, 161 | net_avg, 162 | batch_size, 163 | normalize, 164 | diameter, 165 | tile, 166 | tile_overlap, 167 | resample, 168 | interp, 169 | flow_threshold, 170 | cellprobab_threshold, 171 | min_size, 172 | mask_dir, 173 | ): 174 | channel_groups = None 175 | if Path(panel_file).is_file(): 176 | panel = io.read_panel(panel_file) 177 | if "cellpose" in panel and panel["cellpose"].notna().any(): 178 | channel_groups = panel["cellpose"].values 179 | aggr_func = getattr(np, aggr_func_name) 180 | img_files = io.list_image_files(img_dir) 181 | Path(mask_dir).mkdir(exist_ok=True) 182 | for img_file, mask, flow, style, diam in cellpose.try_segment_objects( 183 | model_name, 184 | img_files, 185 | channelwise_minmax=channelwise_minmax, 186 | channelwise_zscore=channelwise_zscore, 187 | channel_groups=channel_groups, 188 | aggr_func=aggr_func, 189 | net_avg=net_avg, 190 | batch_size=batch_size, 191 | normalize=normalize, 192 | diameter=diameter, 193 | tile=tile, 194 | tile_overlap=tile_overlap, 195 | resample=resample, 196 | interp=interp, 197 | flow_threshold=flow_threshold, 198 | cellprob_threshold=cellprobab_threshold, 199 | min_size=min_size, 200 | ): 201 | mask_file = io._as_path_with_suffix(Path(mask_dir) / img_file.name, ".tiff") 202 | io.write_mask(mask, mask_file) 203 | logger.info(mask_file) 204 | -------------------------------------------------------------------------------- /steinbock/segmentation/_cli/cellprofiler.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from pathlib import Path 3 | 4 | import click 5 | import click_log 6 | 7 | from ..._cli.utils import OrderedClickGroup, catch_exception, logger 8 | from ..._steinbock import SteinbockException 9 | from ..._steinbock import logger as steinbock_logger 10 | from .. import cellprofiler 11 | 12 | 13 | @click.group( 14 | name="cellprofiler", 15 | cls=OrderedClickGroup, 16 | help="Segment objects in probability images using CellProfiler", 17 | ) 18 | def cellprofiler_cmd_group(): 19 | pass 20 | 21 | 22 | @cellprofiler_cmd_group.command( 23 | name="prepare", help="Prepare a CellProfiler segmentation pipeline" 24 | ) 25 | @click.option( 26 | "-o", 27 | "segmentation_pipeline_file", 28 | type=click.Path(dir_okay=False), 29 | default="cell_segmentation.cppipe", 30 | show_default=True, 31 | help="Path to the CellProfiler segmentation pipeline output file", 32 | ) 33 | @click_log.simple_verbosity_option(logger=steinbock_logger) 34 | @catch_exception(handle=SteinbockException) 35 | def prepare_cmd(segmentation_pipeline_file): 36 | cellprofiler.create_and_save_segmentation_pipeline(segmentation_pipeline_file) 37 | logger.info(segmentation_pipeline_file) 38 | 39 | 40 | @cellprofiler_cmd_group.command( 41 | name="run", help="Run a object segmentation batch using CellProfiler" 42 | ) 43 | @click.option( 44 | "--python", 45 | "python_path", 46 | type=click.Path(exists=True, dir_okay=False), 47 | default="/opt/cellprofiler-venv/bin/python", 48 | show_default=True, 49 | help="Python path", 50 | ) 51 | @click.option( 52 | "--cellprofiler", 53 | "cellprofiler_module", 54 | type=click.STRING, 55 | default="cellprofiler", 56 | show_default=True, 57 | help="CellProfiler module", 58 | ) 59 | @click.option( 60 | "--plugins-directory", 61 | "cellprofiler_plugin_dir", 62 | type=click.Path(file_okay=False), 63 | default="/opt/cellprofiler_plugins", 64 | show_default=True, 65 | help="Path to the CellProfiler plugin directory", 66 | ) 67 | @click.option( 68 | "--pipe", 69 | "segmentation_pipeline_file", 70 | type=click.Path(exists=True, dir_okay=False), 71 | default="cell_segmentation.cppipe", 72 | show_default=True, 73 | help="Path to the CellProfiler segmentation pipeline file", 74 | ) 75 | @click.option( 76 | "--probabs", 77 | "probabilities_dir", 78 | type=click.Path(exists=True, file_okay=False), 79 | default="ilastik_probabilities", 80 | show_default=True, 81 | help="Path to the probabilities directory", 82 | ) 83 | @click.option( 84 | "-o", 85 | "mask_dir", 86 | type=click.Path(file_okay=False), 87 | default="masks", 88 | show_default=True, 89 | help="Path to the mask output directory", 90 | ) 91 | @click_log.simple_verbosity_option(logger=steinbock_logger) 92 | @catch_exception(handle=SteinbockException) 93 | def run_cmd( 94 | python_path, 95 | cellprofiler_module, 96 | cellprofiler_plugin_dir, 97 | segmentation_pipeline_file, 98 | probabilities_dir, 99 | mask_dir, 100 | ): 101 | if probabilities_dir not in ("ilastik_probabilities"): 102 | logger.warning( 103 | "When using custom probabilities from unknown origins, " 104 | "make sure to adapt the CellProfiler pipeline accordingly" 105 | ) 106 | Path(mask_dir).mkdir(exist_ok=True) 107 | result = cellprofiler.try_segment_objects( 108 | python_path, 109 | cellprofiler_module, 110 | segmentation_pipeline_file, 111 | probabilities_dir, 112 | mask_dir, 113 | cellprofiler_plugin_dir=cellprofiler_plugin_dir, 114 | ) 115 | sys.exit(result.returncode) 116 | -------------------------------------------------------------------------------- /steinbock/segmentation/_cli/deepcell.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import click 4 | import click_log 5 | import numpy as np 6 | 7 | from ... import io 8 | from ..._cli.utils import catch_exception, logger 9 | from ..._steinbock import SteinbockException 10 | from ..._steinbock import logger as steinbock_logger 11 | from .. import deepcell 12 | 13 | if deepcell.deepcell_available: 14 | import yaml 15 | 16 | deepcell_cli_available = deepcell.deepcell_available 17 | 18 | _applications = { 19 | "mesmer": deepcell.Application.MESMER, 20 | } 21 | 22 | 23 | @click.command(name="deepcell", help="Run an object segmentation batch using DeepCell") 24 | @click.option( 25 | "--app", 26 | "application_name", 27 | type=click.Choice(list(_applications.keys()), case_sensitive=True), 28 | show_choices=True, 29 | default="mesmer", 30 | show_default=True, 31 | help="DeepCell application name", 32 | ) 33 | @click.option( 34 | "--model", 35 | "model_path_or_name", 36 | type=click.STRING, 37 | default="MultiplexSegmentation", 38 | show_default=True, 39 | help="Path/name of custom Keras model", 40 | ) 41 | @click.option( 42 | "--modeldir", 43 | "keras_model_dir", 44 | type=click.Path(file_okay=False), 45 | default="/opt/keras/models", 46 | show_default=True, 47 | help="Path to Keras model directory", 48 | ) 49 | @click.option( 50 | "--img", 51 | "img_dir", 52 | type=click.Path(exists=True, file_okay=False), 53 | default="img", 54 | show_default=True, 55 | help="Path to the image directory", 56 | ) 57 | @click.option( 58 | "--minmax/--no-minmax", 59 | "channelwise_minmax", 60 | default=False, 61 | show_default=True, 62 | help="Channel-wise min-max normalization", 63 | ) 64 | @click.option( 65 | "--zscore/--no-zscore", 66 | "channelwise_zscore", 67 | default=False, 68 | show_default=True, 69 | help="Channel-wise z-score normalization", 70 | ) 71 | @click.option( 72 | "--panel", 73 | "panel_file", 74 | type=click.Path(exists=True, dir_okay=False), 75 | default="panel.csv", 76 | show_default=True, 77 | help="Path to the panel file", 78 | ) 79 | @click.option( 80 | "--aggr", 81 | "aggr_func_name", 82 | type=click.STRING, 83 | default="mean", 84 | show_default=True, 85 | help="Numpy function for aggregating channel pixels", 86 | ) 87 | @click.option( 88 | "--pixelsize", 89 | "pixel_size_um", 90 | type=click.FLOAT, 91 | default=1.0, 92 | show_default=True, 93 | help="[Mesmer] Pixel size in micrometers", 94 | ) 95 | @click.option( 96 | "--type", 97 | "segmentation_type", 98 | type=click.Choice(["whole-cell", "nuclear"]), 99 | default="whole-cell", 100 | show_default=True, 101 | show_choices=True, 102 | help="[Mesmer] Segmentation type", 103 | ) 104 | @click.option( 105 | "--preprocess", 106 | "preprocess_file", 107 | type=click.Path(exists=True, dir_okay=False), 108 | help="[Mesmer] Preprocessing parameters (YAML file)", 109 | ) 110 | @click.option( 111 | "--postprocess", 112 | "postprocess_file", 113 | type=click.Path(exists=True, dir_okay=False), 114 | help="[Mesmer] Postprocessing parameters (YAML file)", 115 | ) 116 | @click.option( 117 | "-o", 118 | "mask_dir", 119 | type=click.Path(file_okay=False), 120 | default="masks", 121 | show_default=True, 122 | help="Path to the mask output directory", 123 | ) 124 | @click_log.simple_verbosity_option(logger=steinbock_logger) 125 | @catch_exception(handle=SteinbockException) 126 | def deepcell_cmd( 127 | application_name, 128 | model_path_or_name, 129 | keras_model_dir, 130 | img_dir, 131 | channelwise_minmax, 132 | channelwise_zscore, 133 | panel_file, 134 | aggr_func_name, 135 | pixel_size_um, 136 | segmentation_type, 137 | preprocess_file, 138 | postprocess_file, 139 | mask_dir, 140 | ): 141 | channel_groups = None 142 | if Path(panel_file).is_file(): 143 | panel = io.read_panel(panel_file) 144 | if "deepcell" in panel and panel["deepcell"].notna().any(): 145 | channel_groups = panel["deepcell"].values 146 | aggr_func = getattr(np, aggr_func_name) 147 | img_files = io.list_image_files(img_dir) 148 | model = None 149 | if model_path_or_name is not None: 150 | from tensorflow.keras.models import load_model # type: ignore 151 | 152 | if Path(model_path_or_name).exists(): 153 | model = load_model(model_path_or_name, compile=False) 154 | elif Path(keras_model_dir).joinpath(model_path_or_name).exists(): 155 | model = load_model( 156 | Path(keras_model_dir).joinpath(model_path_or_name), 157 | compile=False, 158 | ) 159 | preprocess_kwargs = None 160 | if preprocess_file is not None: 161 | with Path(preprocess_file).open() as f: 162 | preprocess_kwargs = yaml.load(f, yaml.Loader) 163 | postprocess_kwargs = None 164 | if postprocess_file is not None: 165 | with Path(postprocess_file).open() as f: 166 | postprocess_kwargs = yaml.load(f, yaml.Loader) 167 | Path(mask_dir).mkdir(exist_ok=True) 168 | for img_file, mask in deepcell.try_segment_objects( 169 | img_files, 170 | _applications[application_name], 171 | model=model, 172 | channelwise_minmax=channelwise_minmax, 173 | channelwise_zscore=channelwise_zscore, 174 | channel_groups=channel_groups, 175 | aggr_func=aggr_func, 176 | pixel_size_um=pixel_size_um, 177 | segmentation_type=segmentation_type, 178 | preprocess_kwargs=preprocess_kwargs, 179 | postprocess_kwargs=postprocess_kwargs, 180 | ): 181 | mask_file = io._as_path_with_suffix(Path(mask_dir) / img_file.name, ".tiff") 182 | io.write_mask(mask, mask_file) 183 | logger.info(mask_file) 184 | -------------------------------------------------------------------------------- /steinbock/segmentation/_segmentation.py: -------------------------------------------------------------------------------- 1 | from .._steinbock import SteinbockException 2 | 3 | 4 | class SteinbockSegmentationException(SteinbockException): 5 | pass 6 | -------------------------------------------------------------------------------- /steinbock/segmentation/cellpose.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from importlib.util import find_spec 3 | from os import PathLike 4 | from pathlib import Path 5 | from typing import Generator, Optional, Protocol, Sequence, Tuple, Union 6 | 7 | import numpy as np 8 | 9 | from .. import io 10 | from ._segmentation import SteinbockSegmentationException 11 | 12 | try: 13 | import cellpose.models 14 | 15 | cellpose_available = True 16 | except Exception: 17 | cellpose_available = False 18 | 19 | logger = logging.getLogger(__name__) 20 | cellpose_available = find_spec("cellpose") is not None 21 | 22 | 23 | class SteinbockCellposeSegmentationException(SteinbockSegmentationException): 24 | pass 25 | 26 | 27 | class AggregationFunction(Protocol): 28 | def __call__(self, img: np.ndarray, axis: Optional[int] = None) -> np.ndarray: 29 | ... 30 | 31 | 32 | def create_segmentation_stack( 33 | img: np.ndarray, 34 | channelwise_minmax: bool = False, 35 | channelwise_zscore: bool = False, 36 | channel_groups: Optional[np.ndarray] = None, 37 | aggr_func: AggregationFunction = np.mean, 38 | ) -> np.ndarray: 39 | if channelwise_minmax: 40 | channel_mins = np.nanmin(img, axis=(1, 2)) 41 | channel_maxs = np.nanmax(img, axis=(1, 2)) 42 | channel_ranges = channel_maxs - channel_mins 43 | img -= channel_mins[:, np.newaxis, np.newaxis] 44 | img[channel_ranges > 0] /= channel_ranges[ 45 | channel_ranges > 0, np.newaxis, np.newaxis 46 | ] 47 | if channelwise_zscore: 48 | channel_means = np.nanmean(img, axis=(1, 2)) 49 | channel_stds = np.nanstd(img, axis=(1, 2)) 50 | img -= channel_means[:, np.newaxis, np.newaxis] 51 | img[channel_stds > 0] /= channel_stds[channel_stds > 0, np.newaxis, np.newaxis] 52 | if channel_groups is not None: 53 | img = np.stack( 54 | [ 55 | aggr_func(img[channel_groups == channel_group], axis=0) 56 | for channel_group in np.unique(channel_groups) 57 | if not np.isnan(channel_group) 58 | ] 59 | ) 60 | return img 61 | 62 | 63 | def try_segment_objects( 64 | model_name: str, 65 | img_files: Sequence[Union[str, PathLike]], 66 | channelwise_minmax: bool = False, 67 | channelwise_zscore: bool = False, 68 | channel_groups: Optional[np.ndarray] = None, 69 | aggr_func: AggregationFunction = np.mean, 70 | net_avg: bool = True, 71 | batch_size: int = 8, 72 | normalize: bool = True, 73 | diameter: Optional[int] = None, 74 | tile: bool = False, 75 | tile_overlap: float = 0.1, 76 | resample: bool = True, 77 | interp: bool = True, 78 | flow_threshold: float = 0.4, 79 | cellprob_threshold: float = 0.0, 80 | min_size: int = 15, 81 | ) -> Generator[Tuple[Path, np.ndarray, np.ndarray, np.ndarray, float], None, None]: 82 | model = cellpose.models.Cellpose(model_type=model_name, net_avg=net_avg) 83 | for img_file in img_files: 84 | try: 85 | img = create_segmentation_stack( 86 | io.read_image(img_file), 87 | channelwise_minmax=channelwise_minmax, 88 | channelwise_zscore=channelwise_zscore, 89 | channel_groups=channel_groups, 90 | aggr_func=aggr_func, 91 | ) 92 | # channels: [cytoplasmic, nuclear] 93 | if img.shape[0] == 1: 94 | channels = [0, 0] # grayscale image (cytoplasmic channel only) 95 | elif img.shape[0] == 2: 96 | channels = [2, 1] # R=1 G=2 B=3 image (nuclear & cytoplasmic channels) 97 | else: 98 | raise SteinbockCellposeSegmentationException( 99 | f"Invalid number of aggregated channels: " 100 | f"expected 1 or 2, got {img.shape[0]}" 101 | ) 102 | masks, flows, styles, diams = model.eval( 103 | [img], 104 | batch_size=batch_size, 105 | channels=channels, 106 | channel_axis=0, 107 | normalize=normalize, 108 | diameter=diameter, 109 | net_avg=net_avg, 110 | tile=tile, 111 | tile_overlap=tile_overlap, 112 | resample=resample, 113 | interp=interp, 114 | flow_threshold=flow_threshold, 115 | cellprob_threshold=cellprob_threshold, 116 | min_size=min_size, 117 | progress=False, 118 | ) 119 | diam = diams if isinstance(diams, float) else diams[0] 120 | yield Path(img_file), masks[0], flows[0], styles[0], diam 121 | del img, masks, flows, styles, diams 122 | except Exception as e: 123 | logger.exception(f"Error segmenting objects in {img_file}: {e}") 124 | -------------------------------------------------------------------------------- /steinbock/segmentation/cellprofiler/__init__.py: -------------------------------------------------------------------------------- 1 | from ._cellprofiler import create_and_save_segmentation_pipeline, try_segment_objects 2 | 3 | __all__ = ["create_and_save_segmentation_pipeline", "try_segment_objects"] 4 | -------------------------------------------------------------------------------- /steinbock/segmentation/cellprofiler/_cellprofiler.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import subprocess 3 | from importlib import resources 4 | from os import PathLike 5 | from pathlib import Path 6 | from typing import Union 7 | 8 | from ..._env import run_captured 9 | from . import data as cellprofiler_data 10 | 11 | 12 | def create_and_save_segmentation_pipeline( 13 | segmentation_pipeline_file: Union[str, PathLike] 14 | ) -> None: 15 | with resources.open_binary(cellprofiler_data, "cell_segmentation.cppipe") as fsrc: 16 | with open(segmentation_pipeline_file, mode="wb") as fdst: 17 | shutil.copyfileobj(fsrc, fdst) 18 | 19 | 20 | def try_segment_objects( 21 | python_path: str, 22 | cellprofiler_module: str, 23 | segmentation_pipeline_file: Union[str, PathLike], 24 | probabilities_dir: Union[str, PathLike], 25 | mask_dir: Union[str, PathLike], 26 | cellprofiler_plugin_dir: Union[str, PathLike, None] = None, 27 | ) -> subprocess.CompletedProcess: 28 | args = [ 29 | python_path, 30 | "-m", 31 | cellprofiler_module, 32 | "-c", 33 | "-r", 34 | "-p", 35 | str(segmentation_pipeline_file), 36 | "-i", 37 | str(probabilities_dir), 38 | "-o", 39 | str(mask_dir), 40 | ] 41 | if cellprofiler_plugin_dir is not None and Path(cellprofiler_plugin_dir).is_dir(): 42 | args.append(f"--plugins-directory={cellprofiler_plugin_dir}") 43 | return run_captured(args) 44 | -------------------------------------------------------------------------------- /steinbock/segmentation/cellprofiler/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BodenmillerGroup/steinbock/7b73f2925b68a351925e89aa145a7a739741b2b7/steinbock/segmentation/cellprofiler/data/__init__.py -------------------------------------------------------------------------------- /steinbock/segmentation/deepcell.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from enum import Enum 3 | from functools import partial 4 | from importlib.util import find_spec 5 | from os import PathLike 6 | from pathlib import Path 7 | from typing import ( 8 | TYPE_CHECKING, 9 | Any, 10 | Generator, 11 | Mapping, 12 | Optional, 13 | Protocol, 14 | Sequence, 15 | Tuple, 16 | Union, 17 | ) 18 | 19 | import numpy as np 20 | 21 | from .. import io 22 | from ._segmentation import SteinbockSegmentationException 23 | 24 | if TYPE_CHECKING: 25 | from tensorflow.keras.models import Model # type: ignore 26 | 27 | 28 | logger = logging.getLogger(__name__) 29 | deepcell_available = find_spec("deepcell") is not None 30 | 31 | 32 | class SteinbockDeepcellSegmentationException(SteinbockSegmentationException): 33 | pass 34 | 35 | 36 | def _mesmer_application(model=None): 37 | from deepcell.applications import Mesmer 38 | 39 | app = Mesmer(model=model) 40 | 41 | def predict( 42 | img: np.ndarray, 43 | *, 44 | pixel_size_um: Optional[float] = None, 45 | segmentation_type: Optional[str] = None, 46 | preprocess_kwargs: Optional[Mapping[str, Any]] = None, 47 | postprocess_kwargs: Optional[Mapping[str, Any]] = None, 48 | ) -> np.ndarray: 49 | assert img.ndim == 3 50 | if pixel_size_um is None: 51 | raise SteinbockDeepcellSegmentationException("Unknown pixel size") 52 | if segmentation_type is None: 53 | raise SteinbockDeepcellSegmentationException("Unknown segmentation type") 54 | mask = app.predict( 55 | np.expand_dims(np.moveaxis(img, 0, -1), 0), 56 | batch_size=1, 57 | image_mpp=pixel_size_um, 58 | compartment=segmentation_type, 59 | preprocess_kwargs=preprocess_kwargs or {}, 60 | postprocess_kwargs_whole_cell=postprocess_kwargs or {}, 61 | postprocess_kwargs_nuclear=postprocess_kwargs or {}, 62 | )[0, :, :, 0] 63 | assert mask.shape == img.shape[1:] 64 | return mask 65 | 66 | return app, predict 67 | 68 | 69 | class Application(Enum): 70 | MESMER = partial(_mesmer_application) 71 | 72 | 73 | class AggregationFunction(Protocol): 74 | def __call__(self, img: np.ndarray, axis: Optional[int] = None) -> np.ndarray: 75 | ... 76 | 77 | 78 | def create_segmentation_stack( 79 | img: np.ndarray, 80 | channelwise_minmax: bool = False, 81 | channelwise_zscore: bool = False, 82 | channel_groups: Optional[np.ndarray] = None, 83 | aggr_func: AggregationFunction = np.mean, 84 | ) -> np.ndarray: 85 | if channelwise_minmax: 86 | channel_mins = np.nanmin(img, axis=(1, 2)) 87 | channel_maxs = np.nanmax(img, axis=(1, 2)) 88 | channel_ranges = channel_maxs - channel_mins 89 | img -= channel_mins[:, np.newaxis, np.newaxis] 90 | img[channel_ranges > 0] /= channel_ranges[ 91 | channel_ranges > 0, np.newaxis, np.newaxis 92 | ] 93 | if channelwise_zscore: 94 | channel_means = np.nanmean(img, axis=(1, 2)) 95 | channel_stds = np.nanstd(img, axis=(1, 2)) 96 | img -= channel_means[:, np.newaxis, np.newaxis] 97 | img[channel_stds > 0] /= channel_stds[channel_stds > 0, np.newaxis, np.newaxis] 98 | if channel_groups is not None: 99 | img = np.stack( 100 | [ 101 | aggr_func(img[channel_groups == channel_group], axis=0) 102 | for channel_group in np.unique(channel_groups) 103 | if not np.isnan(channel_group) 104 | ] 105 | ) 106 | return img 107 | 108 | 109 | def try_segment_objects( 110 | img_files: Sequence[Union[str, PathLike]], 111 | application: Application, 112 | model: Optional["Model"] = None, 113 | channelwise_minmax: bool = False, 114 | channelwise_zscore: bool = False, 115 | channel_groups: Optional[np.ndarray] = None, 116 | aggr_func: AggregationFunction = np.mean, 117 | **predict_kwargs, 118 | ) -> Generator[Tuple[Path, np.ndarray], None, None]: 119 | app, predict = application.value(model=model) 120 | for img_file in img_files: 121 | try: 122 | img = create_segmentation_stack( 123 | io.read_image(img_file), 124 | channelwise_minmax=channelwise_minmax, 125 | channelwise_zscore=channelwise_zscore, 126 | channel_groups=channel_groups, 127 | aggr_func=aggr_func, 128 | ) 129 | if img.shape[0] != 2: 130 | raise SteinbockDeepcellSegmentationException( 131 | f"Invalid number of aggregated channels: " 132 | f"expected 2, got {img.shape[0]}" 133 | ) 134 | mask = predict(img, **predict_kwargs) 135 | yield Path(img_file), mask 136 | del img, mask 137 | except Exception as e: 138 | logger.exception(f"Error segmenting objects in {img_file}: {e}") 139 | -------------------------------------------------------------------------------- /steinbock/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from ._utils import SteinbockUtilsException 2 | 3 | __all__ = ["SteinbockUtilsException"] 4 | -------------------------------------------------------------------------------- /steinbock/utils/_cli/__init__.py: -------------------------------------------------------------------------------- 1 | from ._cli import utils_cmd_group 2 | 3 | __all__ = ["utils_cmd_group"] 4 | -------------------------------------------------------------------------------- /steinbock/utils/_cli/_cli.py: -------------------------------------------------------------------------------- 1 | import click 2 | 3 | from ..._cli.utils import OrderedClickGroup 4 | from .expansion import expand_cmd 5 | from .matching import match_cmd 6 | from .mosaics import mosaics_cmd_group 7 | 8 | 9 | @click.group(name="utils", cls=OrderedClickGroup, help="Various utilities and tools") 10 | def utils_cmd_group(): 11 | pass 12 | 13 | 14 | utils_cmd_group.add_command(expand_cmd) 15 | utils_cmd_group.add_command(match_cmd) 16 | utils_cmd_group.add_command(mosaics_cmd_group) 17 | -------------------------------------------------------------------------------- /steinbock/utils/_cli/expansion.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import click 4 | import click_log 5 | 6 | from ... import io 7 | from ..._cli.utils import catch_exception, logger 8 | from ..._steinbock import SteinbockException 9 | from ..._steinbock import logger as steinbock_logger 10 | from .. import expansion 11 | 12 | 13 | @click.command(name="expand", help="Expand mask objects by an Euclidean distance") 14 | @click.argument("masks", type=click.Path(exists=True, file_okay=False)) 15 | @click.argument("distance", type=click.INT) 16 | @click.option( 17 | "--mmap/--no-mmap", 18 | "mmap", 19 | default=False, 20 | show_default=True, 21 | help="Use memory mapping for reading images/masks", 22 | ) 23 | @click.option( 24 | "-o", 25 | "expanded_masks_dir", 26 | type=click.Path(file_okay=False), 27 | help="Path to the expanded masks output directory", 28 | ) 29 | @click_log.simple_verbosity_option(logger=steinbock_logger) 30 | @catch_exception(handle=SteinbockException) 31 | def expand_cmd(masks, distance, mmap, expanded_masks_dir): 32 | if Path(masks).is_file(): 33 | mask_files = [Path(masks)] 34 | elif Path(masks).is_dir(): 35 | mask_files = io.list_mask_files(masks) 36 | if expanded_masks_dir is not None: 37 | Path(expanded_masks_dir).mkdir(exist_ok=True) 38 | else: 39 | expanded_masks_dir = Path(masks) 40 | for mask_file, expanded_mask in expansion.try_expand_masks_from_disk( 41 | mask_files, distance, mmap=mmap 42 | ): 43 | expanded_mask_file = Path(expanded_masks_dir) / mask_file.name 44 | io.write_mask(expanded_mask, expanded_mask_file, ignore_dtype=True) 45 | logger.info(expanded_mask_file) 46 | del expanded_mask 47 | -------------------------------------------------------------------------------- /steinbock/utils/_cli/matching.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import click 4 | import click_log 5 | 6 | from ... import io 7 | from ..._cli.utils import catch_exception, logger 8 | from ..._steinbock import SteinbockException 9 | from ..._steinbock import logger as steinbock_logger 10 | from .. import matching 11 | 12 | 13 | @click.command(name="match", help="Match mask objects") 14 | @click.argument("masks1", nargs=1, type=click.Path(exists=True, file_okay=False)) 15 | @click.argument("masks2", nargs=1, type=click.Path(exists=True, file_okay=False)) 16 | @click.option( 17 | "--mmap/--no-mmap", 18 | "mmap", 19 | default=False, 20 | show_default=True, 21 | help="Use memory mapping for reading images/masks", 22 | ) 23 | @click.option( 24 | "-o", 25 | "csv_dir", 26 | type=click.Path(file_okay=False), 27 | required=True, 28 | help="Path to the object table CSV output directory", 29 | ) 30 | @click_log.simple_verbosity_option(logger=steinbock_logger) 31 | @catch_exception(handle=SteinbockException) 32 | def match_cmd(masks1, masks2, mmap, csv_dir): 33 | if Path(masks1).is_file() and Path(masks2).is_file(): 34 | mask_files1 = [Path(masks1)] 35 | mask_files2 = [Path(masks2)] 36 | elif Path(masks1).is_dir() and Path(masks2).is_dir(): 37 | mask_files1 = io.list_mask_files(masks1) 38 | mask_files2 = io.list_mask_files(masks2, base_files=mask_files1) 39 | Path(csv_dir).mkdir(exist_ok=True) 40 | for mask_file1, mask_file2, df in matching.try_match_masks_from_disk( 41 | mask_files1, mask_files2, mmap=mmap 42 | ): 43 | csv_file = io._as_path_with_suffix(Path(csv_dir) / mask_file1.name, ".csv") 44 | df.columns = [Path(masks1).name, Path(masks2).name] 45 | df.to_csv(csv_file, index=False) 46 | logger.info(csv_file) 47 | del df 48 | -------------------------------------------------------------------------------- /steinbock/utils/_cli/mosaics.py: -------------------------------------------------------------------------------- 1 | from os import PathLike 2 | from pathlib import Path 3 | from typing import List, Sequence, Union 4 | 5 | import click 6 | import click_log 7 | 8 | from ..._cli.utils import OrderedClickGroup, catch_exception, logger 9 | from ..._steinbock import SteinbockException 10 | from ..._steinbock import logger as steinbock_logger 11 | from .. import mosaics 12 | 13 | 14 | def _collect_tiff_files( 15 | img_files_or_dirs: Sequence[Union[str, PathLike]] 16 | ) -> List[Path]: 17 | img_files = [] 18 | for img_file_or_dir in img_files_or_dirs: 19 | if Path(img_file_or_dir).is_file(): 20 | img_files.append(Path(img_file_or_dir)) 21 | else: 22 | img_files += sorted(Path(img_file_or_dir).rglob("[!.]*.tiff")) 23 | return img_files 24 | 25 | 26 | @click.group(name="mosaics", cls=OrderedClickGroup, help="Mosaic tiling/stitching") 27 | def mosaics_cmd_group(): 28 | pass 29 | 30 | 31 | @mosaics_cmd_group.command(name="tile", help="Extract tiles from images") 32 | @click.argument("images", nargs=-1, type=click.Path(exists=True)) 33 | @click.option( 34 | "--size", 35 | "tile_size", 36 | type=click.INT, 37 | required=True, 38 | help="Tile size (in pixels)", 39 | ) 40 | @click.option( 41 | "--mmap/--no-mmap", 42 | "mmap", 43 | default=False, 44 | show_default=True, 45 | help="Use memory mapping for reading images", 46 | ) 47 | @click.option( 48 | "-o", 49 | "tile_dir", 50 | type=click.Path(file_okay=False), 51 | required=True, 52 | help="Path to the tile output directory", 53 | ) 54 | @click_log.simple_verbosity_option(logger=steinbock_logger) 55 | @catch_exception(handle=SteinbockException) 56 | def tile_cmd(images, tile_size, mmap, tile_dir): 57 | img_files = _collect_tiff_files(images) 58 | Path(tile_dir).mkdir(exist_ok=True) 59 | for tile_file, tile in mosaics.try_extract_tiles_from_disk_to_disk( 60 | img_files, tile_dir, tile_size, mmap=mmap 61 | ): 62 | logger.info(tile_file) 63 | del tile 64 | 65 | 66 | @mosaics_cmd_group.command(name="stitch", help="Combine tiles into images") 67 | @click.argument("tiles", nargs=-1, type=click.Path(exists=True)) 68 | @click.option( 69 | "--relabel/--no-relabel", 70 | "relabel", 71 | default=False, 72 | show_default=True, 73 | help="Relabel objects", 74 | ) 75 | @click.option( 76 | "--mmap/--no-mmap", 77 | "mmap", 78 | default=False, 79 | show_default=True, 80 | help="Use memory mapping for writing images", 81 | ) 82 | @click.option( 83 | "-o", 84 | "img_dir", 85 | type=click.Path(file_okay=False), 86 | required=True, 87 | help="Path to the tile output directory", 88 | ) 89 | @click_log.simple_verbosity_option(logger=steinbock_logger) 90 | @catch_exception(handle=SteinbockException) 91 | def stitch_cmd(tiles, relabel, mmap, img_dir): 92 | tile_files = _collect_tiff_files(tiles) 93 | Path(img_dir).mkdir(exist_ok=True) 94 | for img_file, img in mosaics.try_stitch_tiles_from_disk_to_disk( 95 | tile_files, img_dir, relabel=relabel, mmap=mmap 96 | ): 97 | logger.info(img_file) 98 | del img 99 | -------------------------------------------------------------------------------- /steinbock/utils/_utils.py: -------------------------------------------------------------------------------- 1 | from .._steinbock import SteinbockException 2 | 3 | 4 | class SteinbockUtilsException(SteinbockException): 5 | pass 6 | -------------------------------------------------------------------------------- /steinbock/utils/expansion.py: -------------------------------------------------------------------------------- 1 | from os import PathLike 2 | from pathlib import Path 3 | from typing import Generator, Sequence, Tuple, Union 4 | 5 | import numpy as np 6 | from skimage.segmentation import expand_labels 7 | 8 | from .. import io 9 | 10 | 11 | def expand_mask(mask: np.ndarray, distance: int) -> np.ndarray: 12 | expanded_mask = expand_labels(mask, distance=distance) 13 | return expanded_mask 14 | 15 | 16 | def try_expand_masks_from_disk( 17 | mask_files: Sequence[Union[str, PathLike]], distance: int, mmap: bool = False 18 | ) -> Generator[Tuple[Path, np.ndarray], None, None]: 19 | for mask_file in mask_files: 20 | if mmap: 21 | mask = io.mmap_mask(mask_file) 22 | else: 23 | mask = io.read_mask(mask_file, native_dtype=True) 24 | expanded_mask = expand_mask(mask, distance=distance) 25 | yield Path(mask_file), expanded_mask 26 | -------------------------------------------------------------------------------- /steinbock/utils/matching.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from os import PathLike 3 | from pathlib import Path 4 | from typing import Generator, Sequence, Tuple, Union 5 | 6 | import numpy as np 7 | import pandas as pd 8 | 9 | from .. import io 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | def match_masks(mask1: np.ndarray, mask2: np.ndarray) -> pd.DataFrame: 15 | nz1 = mask1 != 0 16 | nz2 = mask2 != 0 17 | object_ids1 = [] 18 | object_ids2 = [] 19 | for object_id1 in np.unique(mask1[nz1]): 20 | for object_id2 in np.unique(mask2[nz2 & (mask1 == object_id1)]): 21 | object_ids1.append(object_id1) 22 | object_ids2.append(object_id2) 23 | for object_id2 in np.unique(mask2[nz2]): 24 | for object_id1 in np.unique(mask1[nz1 & (mask2 == object_id2)]): 25 | object_ids1.append(object_id1) 26 | object_ids2.append(object_id2) 27 | df = pd.DataFrame(data={"Object1": object_ids1, "Object2": object_ids2}) 28 | df.drop_duplicates(inplace=True, ignore_index=True) 29 | return df 30 | 31 | 32 | def try_match_masks_from_disk( 33 | mask_files1: Sequence[Union[str, PathLike]], 34 | mask_files2: Sequence[Union[str, PathLike]], 35 | mmap: bool = False, 36 | ) -> Generator[Tuple[Path, Path, pd.DataFrame], None, None]: 37 | for mask_file1, mask_file2 in zip(mask_files1, mask_files2): 38 | try: 39 | if mmap: 40 | mask1 = io.mmap_mask(mask_file1) 41 | mask2 = io.mmap_mask(mask_file2) 42 | else: 43 | mask1 = io.read_mask(mask_file1) 44 | mask2 = io.read_mask(mask_file2) 45 | df = match_masks(mask1, mask2) 46 | del mask1, mask2 47 | yield Path(mask_file1), Path(mask_file2), df 48 | del df 49 | except Exception as e: 50 | logger.exception(f"Error matching masks {mask_file1, mask_file2}: {e}") 51 | -------------------------------------------------------------------------------- /steinbock/utils/mosaics.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import re 3 | from os import PathLike 4 | from pathlib import Path 5 | from typing import Dict, Generator, List, NamedTuple, Sequence, Tuple, Union 6 | 7 | import numpy as np 8 | from skimage import measure 9 | 10 | from .. import io 11 | from ._utils import SteinbockUtilsException 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | class SteinbockMosaicsUtilsException(SteinbockUtilsException): 17 | pass 18 | 19 | 20 | def try_extract_tiles_from_disk_to_disk( 21 | img_files: Sequence[Union[str, PathLike]], 22 | tile_dir: Union[str, PathLike], 23 | tile_size: int, 24 | mmap: bool = False, 25 | ) -> Generator[Tuple[Path, np.ndarray], None, None]: 26 | for img_file in img_files: 27 | try: 28 | if mmap: 29 | img = io.mmap_image(img_file) 30 | else: 31 | img = io.read_image(img_file, native_dtype=True) 32 | if img.shape[-1] % tile_size == 1 or img.shape[-2] % tile_size == 1: 33 | logger.warning( 34 | "Chosen tile size yields UNSTITCHABLE tiles of 1 pixel " 35 | f"width or height for image {img_file}" 36 | ) 37 | for tile_x in range(0, img.shape[-1], tile_size): 38 | for tile_y in range(0, img.shape[-2], tile_size): 39 | tile = img[ 40 | :, 41 | tile_y : (tile_y + tile_size), 42 | tile_x : (tile_x + tile_size), 43 | ] 44 | tile_file = Path(tile_dir) / ( 45 | f"{Path(img_file).stem}_tx{tile_x}_ty{tile_y}" 46 | f"_tw{tile.shape[-1]}_th{tile.shape[-2]}.tiff" 47 | ) 48 | io.write_image(tile, tile_file, ignore_dtype=True) 49 | yield tile_file, tile 50 | del tile 51 | del img 52 | except Exception as e: 53 | logger.exception(f"Error extracting tiles: {img_file}: {e}") 54 | 55 | 56 | def try_stitch_tiles_from_disk_to_disk( 57 | tile_files: Sequence[Union[str, PathLike]], 58 | img_dir: Union[str, PathLike], 59 | relabel: bool = False, 60 | mmap: bool = False, 61 | ) -> Generator[Tuple[Path, np.ndarray], None, None]: 62 | class TileInfo(NamedTuple): 63 | tile_file: Path 64 | x: int 65 | y: int 66 | width: int 67 | height: int 68 | 69 | tile_file_stem_pattern = re.compile( 70 | r"(?P.+)_tx(?P\d+)_ty(?P\d+)" 71 | r"_tw(?P\d+)_th(?P\d+)" 72 | ) 73 | img_tile_infos: Dict[str, List[TileInfo]] = {} 74 | for tile_file in tile_files: 75 | m = tile_file_stem_pattern.fullmatch(Path(tile_file).stem) 76 | if m is None: 77 | raise SteinbockMosaicsUtilsException( 78 | f"Malformed tile file name: {tile_file}" 79 | ) 80 | img_file_stem = m.group("img_file_stem") 81 | tile_info = TileInfo( 82 | Path(tile_file), 83 | int(m.group("x")), 84 | int(m.group("y")), 85 | int(m.group("width")), 86 | int(m.group("height")), 87 | ) 88 | if img_file_stem not in img_tile_infos: 89 | img_tile_infos[img_file_stem] = [] 90 | img_tile_infos[img_file_stem].append(tile_info) 91 | for img_file_stem, tile_infos in img_tile_infos.items(): 92 | img_file = Path(img_dir) / f"{img_file_stem}.tiff" 93 | try: 94 | tile = io.read_image(tile_infos[0].tile_file, native_dtype=True) 95 | img_shape = ( 96 | tile.shape[0], 97 | max(ti.y + ti.height for ti in tile_infos), 98 | max(ti.x + ti.width for ti in tile_infos), 99 | ) 100 | if mmap: 101 | img = io.mmap_image( 102 | img_file, mode="r+", shape=img_shape, dtype=tile.dtype 103 | ) 104 | else: 105 | img = np.zeros(img_shape, dtype=tile.dtype) 106 | for i, tile_info in enumerate(tile_infos): 107 | if i > 0: 108 | tile = io.read_image(tile_info.tile_file, native_dtype=True) 109 | img[ 110 | :, 111 | tile_info.y : tile_info.y + tile_info.height, 112 | tile_info.x : tile_info.x + tile_info.width, 113 | ] = tile 114 | if mmap: 115 | img.flush() 116 | if relabel: 117 | img[0, :, :] = measure.label(img[0, :, :]) 118 | if mmap: 119 | img.flush() 120 | else: 121 | io.write_image(img, img_file, ignore_dtype=True) 122 | yield img_file, img 123 | del img 124 | except Exception as e: 125 | logger.exception(f"Error stitching tiles: {img_file}: {e}") 126 | -------------------------------------------------------------------------------- /steinbock/visualization.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Optional, Sequence 2 | 3 | import napari 4 | import numpy as np 5 | 6 | 7 | def view( 8 | img: np.ndarray, 9 | masks: Optional[Dict[str, np.ndarray]] = None, 10 | channel_names: Optional[Sequence[str]] = None, 11 | pixel_size_um: float = 1.0, 12 | run: bool = True, 13 | **viewer_kwargs, 14 | ) -> Optional[napari.Viewer]: 15 | viewer = napari.Viewer(**viewer_kwargs) 16 | viewer.axes.visible = True 17 | viewer.dims.axis_labels = ("y", "x") 18 | viewer.scale_bar.visible = True 19 | viewer.scale_bar.unit = "um" 20 | viewer.add_image( 21 | data=img, 22 | channel_axis=0, 23 | colormap="gray", 24 | name=channel_names, 25 | scale=(pixel_size_um, pixel_size_um), 26 | blending="additive", 27 | visible=False, 28 | ) 29 | if masks is not None: 30 | for mask_name, mask in masks.items(): 31 | viewer.add_labels( 32 | data=mask, 33 | name=mask_name, 34 | scale=(pixel_size_um, pixel_size_um), 35 | blending="translucent", 36 | visible=False, 37 | ) 38 | if run: 39 | napari.run() 40 | return None 41 | return viewer 42 | -------------------------------------------------------------------------------- /tests/classification/test_ilastik_classification.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | from pathlib import Path 3 | 4 | import numpy as np 5 | import pytest 6 | 7 | from steinbock import io 8 | from steinbock.classification import ilastik 9 | 10 | ilastik_binary = "/opt/ilastik/run_ilastik.sh" 11 | 12 | 13 | class TestIlastikClassification: 14 | def test_list_ilastik_image_files(self, imc_test_data_steinbock_path: Path): 15 | ilastik.list_ilastik_image_files( 16 | imc_test_data_steinbock_path / "ilastik_img" 17 | ) # TODO 18 | 19 | def test_list_ilastik_crop_files(self, imc_test_data_steinbock_path: Path): 20 | ilastik.list_ilastik_crop_files( 21 | imc_test_data_steinbock_path / "ilastik_crops" 22 | ) # TODO 23 | 24 | def test_read_ilastik_image(self, imc_test_data_steinbock_path: Path): 25 | ilastik.read_ilastik_image( 26 | imc_test_data_steinbock_path / "ilastik_img" / "20210305_NE_mockData1_1.h5" 27 | ) # TODO 28 | 29 | def test_read_ilastik_crop(self, imc_test_data_steinbock_path: Path): 30 | ilastik.read_ilastik_crop( 31 | imc_test_data_steinbock_path 32 | / "ilastik_crops" 33 | / "20210305_NE_mockData1_s0_a1_ac_ilastik_x0_y0_w120_h120.h5" 34 | ) # TODO 35 | 36 | def test_write_ilastik_image(self, tmp_path: Path): 37 | ilastik_img = np.array( 38 | [ 39 | [ 40 | [1, 2, 3], 41 | [4, 5, 6], 42 | [7, 8, 9], 43 | ] 44 | ], 45 | dtype=io.img_dtype, 46 | ) 47 | ilastik.write_ilastik_image(ilastik_img, tmp_path / "ilastik_img.h5") # TODO 48 | 49 | def test_write_ilastik_crop(self, tmp_path: Path): 50 | ilastik_crop = np.array( 51 | [ 52 | [ 53 | [1, 2, 3], 54 | [4, 5, 6], 55 | [7, 8, 9], 56 | ] 57 | ], 58 | dtype=io.img_dtype, 59 | ) 60 | ilastik.write_ilastik_crop(ilastik_crop, tmp_path / "ilastik_crop.h5") # TODO 61 | 62 | def test_create_ilastik_image(self): 63 | img = np.array( 64 | [ 65 | [ 66 | [1, 2, 3], 67 | [4, 5, 6], 68 | [7, 8, 9], 69 | ] 70 | ], 71 | dtype=io.img_dtype, 72 | ) 73 | ilastik_img = ilastik.create_ilastik_image(img) 74 | expected_ilastik_img = np.array( 75 | [ 76 | [ 77 | [100, 200, 300], 78 | [400, 500, 600], 79 | [700, 800, 900], 80 | ], 81 | [ 82 | [1, 2, 3], 83 | [4, 5, 6], 84 | [7, 8, 9], 85 | ], 86 | ], 87 | dtype=io.img_dtype, 88 | ) 89 | assert np.all(ilastik_img == expected_ilastik_img) 90 | 91 | def test_try_create_ilastik_images_from_disk( 92 | self, imc_test_data_steinbock_path: Path 93 | ): 94 | img_files = io.list_image_files(imc_test_data_steinbock_path / "img") 95 | gen = ilastik.try_create_ilastik_images_from_disk(img_files) 96 | for img_file, ilastik_img in gen: 97 | pass # TODO 98 | 99 | def test_create_ilastik_crop(self): 100 | ilastik_img = np.array( 101 | [ 102 | [ 103 | [100, 200, 300], 104 | [400, 500, 600], 105 | [700, 800, 900], 106 | ], 107 | [ 108 | [1, 2, 3], 109 | [4, 5, 6], 110 | [7, 8, 9], 111 | ], 112 | ], 113 | dtype=io.img_dtype, 114 | ) 115 | rng = np.random.default_rng() 116 | x, y, ilastik_crop = ilastik.create_ilastik_crop(ilastik_img, 3, rng) 117 | expected_ilastik_crop = np.array( 118 | [ 119 | [ 120 | [100, 200, 300], 121 | [400, 500, 600], 122 | [700, 800, 900], 123 | ], 124 | [ 125 | [1, 2, 3], 126 | [4, 5, 6], 127 | [7, 8, 9], 128 | ], 129 | ], 130 | dtype=io.img_dtype, 131 | ) 132 | assert np.all(ilastik_crop == expected_ilastik_crop) 133 | 134 | def test_try_create_ilastik_crops_from_disk( 135 | self, imc_test_data_steinbock_path: Path 136 | ): 137 | ilastik_img_files = ilastik.list_ilastik_image_files( 138 | imc_test_data_steinbock_path / "ilastik_img" 139 | ) 140 | rng = np.random.default_rng(seed=123) 141 | gen = ilastik.try_create_ilastik_crops_from_disk(ilastik_img_files, 50, rng) 142 | for ilastik_img_file, x, y, ilastik_crop in gen: 143 | pass # TODO 144 | 145 | def test_create_and_save_ilastik_project( 146 | self, imc_test_data_steinbock_path: Path, tmp_path: Path 147 | ): 148 | shutil.copytree( 149 | imc_test_data_steinbock_path / "ilastik_crops", 150 | tmp_path / "ilastik_crops", 151 | ) 152 | ilastik_crop_files = ilastik.list_ilastik_crop_files(tmp_path / "ilastik_crops") 153 | ilastik.create_and_save_ilastik_project( 154 | ilastik_crop_files, tmp_path / "pixel_classifier.ilp" 155 | ) # TODO 156 | 157 | @pytest.mark.skip(reason="Test would take too long") 158 | @pytest.mark.skipif( 159 | shutil.which(ilastik_binary) is None, reason="Ilastik is not available" 160 | ) 161 | def test_run_pixel_classification( 162 | self, imc_test_data_steinbock_path: Path, tmp_path: Path 163 | ): 164 | shutil.copytree( 165 | imc_test_data_steinbock_path / "ilastik_img", 166 | tmp_path / "ilastik_img", 167 | ) 168 | ilastik_img_files = ilastik.list_ilastik_image_files(tmp_path / "ilastik_img") 169 | ilastik.run_pixel_classification( 170 | ilastik_binary, 171 | imc_test_data_steinbock_path / "pixel_classifier.ilp", 172 | ilastik_img_files, 173 | tmp_path / "ilastik_probabilities", 174 | ) # TODO 175 | 176 | def test_try_fix_ilastik_crops_from_disk(self, imc_test_data_steinbock_path: Path): 177 | pass # TODO 178 | 179 | def test_fix_ilastik_project_file_inplace(self, imc_test_data_steinbock_path: Path): 180 | pass # TODO 181 | -------------------------------------------------------------------------------- /tests/export/test_data_export.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from steinbock import io 4 | from steinbock.export import data 5 | 6 | 7 | class TestDataExport: 8 | def test_try_convert_to_dataframe_from_disk( 9 | self, imc_test_data_steinbock_path: Path 10 | ): 11 | intensities_files = io.list_data_files( 12 | imc_test_data_steinbock_path / "intensities" 13 | ) 14 | regionprops_files = io.list_data_files( 15 | imc_test_data_steinbock_path / "regionprops", 16 | base_files=intensities_files, 17 | ) 18 | gen = data.try_convert_to_dataframe_from_disk( 19 | intensities_files, regionprops_files 20 | ) 21 | for img_file_name, data_files, df in gen: 22 | pass # TODO 23 | 24 | def test_try_convert_to_anndata_from_disk(self, imc_test_data_steinbock_path: Path): 25 | intensities_files = io.list_data_files( 26 | imc_test_data_steinbock_path / "intensities" 27 | ) 28 | regionprops_files = io.list_data_files( 29 | imc_test_data_steinbock_path / "regionprops", 30 | base_files=intensities_files, 31 | ) 32 | neighbors_files = io.list_neighbors_files( 33 | imc_test_data_steinbock_path / "neighbors", 34 | base_files=intensities_files, 35 | ) 36 | gen = data.try_convert_to_anndata_from_disk( 37 | intensities_files, 38 | regionprops_files, 39 | neighbors_files=neighbors_files, 40 | ) 41 | for ( 42 | img_file_name, 43 | intensities_file, 44 | (regionprops_file,), 45 | neighbors_file, 46 | adata, 47 | ) in gen: 48 | pass # TODO 49 | -------------------------------------------------------------------------------- /tests/export/test_graphs_export.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import pandas as pd 4 | 5 | from steinbock import io 6 | from steinbock.export import graphs 7 | 8 | 9 | class TestGraphsExport: 10 | def test_convert_to_networkx(self): 11 | neighbors = pd.DataFrame( 12 | data={ 13 | "Object": [1, 2], 14 | "Neighbor": [2, 1], 15 | "Distance": [1.0, 1.0], 16 | } 17 | ) 18 | neighbors["Object"] = neighbors["Object"].astype(io.mask_dtype) 19 | neighbors["Neighbor"] = neighbors["Neighbor"].astype(io.mask_dtype) 20 | intensities = pd.DataFrame( 21 | data={ 22 | "Channel 1": [1.0, 2.0], 23 | "Channel 2": [100.0, 200.0], 24 | }, 25 | index=pd.Index([1, 2], name="Object", dtype=io.mask_dtype), 26 | ) 27 | graphs.convert_to_networkx(neighbors, intensities) # TODO 28 | 29 | def test_try_convert_to_networkx_from_disk( 30 | self, imc_test_data_steinbock_path: Path 31 | ): 32 | neighbors_files = io.list_neighbors_files( 33 | imc_test_data_steinbock_path / "neighbors" 34 | ) 35 | intensities_files = io.list_data_files( 36 | imc_test_data_steinbock_path / "intensities", 37 | base_files=neighbors_files, 38 | ) 39 | gen = graphs.try_convert_to_networkx_from_disk( 40 | neighbors_files, intensities_files 41 | ) 42 | for neighbors_file, (intensities_file,), graph in gen: 43 | pass # TODO 44 | -------------------------------------------------------------------------------- /tests/measurement/test_cellprofiler_measurement.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | from pathlib import Path 3 | 4 | import pytest 5 | 6 | from steinbock.measurement import cellprofiler 7 | 8 | cellprofiler_binary = "cellprofiler" 9 | cellprofiler_plugin_dir = "/opt/cellprofiler_plugins" 10 | 11 | 12 | class TestCellprofilerMeasurement: 13 | def test_create_and_save_measurement_pipeline(self, tmp_path: Path): 14 | cellprofiler.create_and_save_measurement_pipeline( 15 | tmp_path / "cell_measurement.cppipe", 5 16 | ) # TODO 17 | 18 | @pytest.mark.skip(reason="Test would take too long") 19 | @pytest.mark.skipif( 20 | shutil.which(cellprofiler_binary) is None, 21 | reason="CellProfiler is not available", 22 | ) 23 | def test_try_measure_objects( 24 | self, imc_test_data_steinbock_path: Path, tmp_path: Path 25 | ): 26 | cellprofiler.try_measure_objects( 27 | cellprofiler_binary, 28 | imc_test_data_steinbock_path / "cell_measurement.cppipe", 29 | imc_test_data_steinbock_path / "cellprofiler_input", 30 | tmp_path / "cellprofiler_output", 31 | cellprofiler_plugin_dir=cellprofiler_plugin_dir, 32 | ) # TODO 33 | -------------------------------------------------------------------------------- /tests/measurement/test_intensities_measurement.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import numpy as np 4 | 5 | from steinbock import io 6 | from steinbock.measurement import intensities 7 | from steinbock.measurement.intensities import IntensityAggregation 8 | 9 | 10 | class TestIntensitiesMeasurement: 11 | def test_measure_intensites(self): 12 | img = np.array( 13 | [ 14 | [ 15 | [0.5, 1.5, 0.1], 16 | [0.1, 0.2, 0.3], 17 | [0.1, 6.5, 3.5], 18 | ], 19 | [ 20 | [20, 30, 1], 21 | [1, 2, 3], 22 | [1, 100, 200], 23 | ], 24 | ], 25 | dtype=io.img_dtype, 26 | ) 27 | mask = np.array( 28 | [ 29 | [1, 1, 0], 30 | [0, 0, 0], 31 | [0, 2, 2], 32 | ], 33 | dtype=io.mask_dtype, 34 | ) 35 | channel_names = ["Channel 1", "Channel 2"] 36 | df = intensities.measure_intensites( 37 | img, mask, channel_names, IntensityAggregation.MEAN 38 | ) 39 | assert np.all(df.index.values == np.array([1, 2])) 40 | assert np.all(df.columns.values == np.array(channel_names)) 41 | assert np.all(df.values == np.array([[1.0, 25.0], [5.0, 150.0]])) 42 | 43 | def test_try_measure_intensities_from_disk( 44 | self, imc_test_data_steinbock_path: Path 45 | ): 46 | panel = io.read_panel(imc_test_data_steinbock_path / "panel.csv") 47 | img_files = io.list_image_files(imc_test_data_steinbock_path / "img") 48 | mask_files = io.list_mask_files( 49 | imc_test_data_steinbock_path / "masks", base_files=img_files 50 | ) 51 | gen = intensities.try_measure_intensities_from_disk( 52 | img_files, 53 | mask_files, 54 | panel["name"].tolist(), 55 | IntensityAggregation.MEAN, 56 | ) 57 | for img_file, mask_file, df in gen: 58 | pass # TODO 59 | -------------------------------------------------------------------------------- /tests/measurement/test_neighbors_measurement.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import numpy as np 4 | 5 | from steinbock import io 6 | from steinbock.measurement import neighbors 7 | from steinbock.measurement.neighbors import NeighborhoodType 8 | 9 | 10 | class TestNeighborsMeasurement: 11 | def test_measure_neighbors_centroid(self): 12 | mask = np.array( 13 | [ 14 | [1, 1, 0], 15 | [3, 0, 4], 16 | [0, 2, 2], 17 | ], 18 | dtype=io.mask_dtype, 19 | ) 20 | neighbors.measure_neighbors( 21 | mask, 22 | NeighborhoodType.CENTROID_DISTANCE, 23 | metric="euclidean", 24 | dmax=1.0, 25 | ) # TODO 26 | 27 | def test_measure_neighbors_euclidean_border(self): 28 | mask = np.array( 29 | [ 30 | [1, 1, 0], 31 | [3, 0, 4], 32 | [0, 2, 2], 33 | ], 34 | dtype=io.mask_dtype, 35 | ) 36 | neighbors.measure_neighbors( 37 | mask, 38 | NeighborhoodType.EUCLIDEAN_BORDER_DISTANCE, 39 | metric="euclidean", 40 | dmax=1.0, 41 | ) # TODO 42 | 43 | def test_measure_neighbors_euclidean_pixel_expansion(self): 44 | mask = np.array( 45 | [ 46 | [1, 1, 0], 47 | [3, 0, 4], 48 | [0, 2, 2], 49 | ], 50 | dtype=io.mask_dtype, 51 | ) 52 | neighbors.measure_neighbors( 53 | mask, 54 | NeighborhoodType.EUCLIDEAN_PIXEL_EXPANSION, 55 | metric="euclidean", 56 | dmax=1.0, 57 | ) # TODO 58 | 59 | def test_try_measure_neighbors_from_disk(self, imc_test_data_steinbock_path: Path): 60 | mask_files = io.list_mask_files(imc_test_data_steinbock_path / "masks") 61 | gen = neighbors.try_measure_neighbors_from_disk( 62 | mask_files, 63 | NeighborhoodType.CENTROID_DISTANCE, 64 | metric="euclidean", 65 | dmax=4.0, 66 | ) 67 | for mask_file, df in gen: 68 | pass # TODO 69 | -------------------------------------------------------------------------------- /tests/measurement/test_regionprops_measurement.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import numpy as np 4 | 5 | from steinbock import io 6 | from steinbock.measurement import regionprops 7 | 8 | 9 | class TestRegionpropsMeasurement: 10 | def test_measure_regionprops(self): 11 | img = np.array( 12 | [ 13 | [ 14 | [0.5, 1.5, 0.1], 15 | [0.1, 0.2, 0.3], 16 | [0.1, 6.5, 3.5], 17 | ], 18 | [ 19 | [20, 30, 1], 20 | [1, 2, 3], 21 | [1, 100, 200], 22 | ], 23 | ], 24 | dtype=io.img_dtype, 25 | ) 26 | mask = np.array( 27 | [ 28 | [1, 1, 0], 29 | [0, 0, 0], 30 | [0, 2, 2], 31 | ], 32 | dtype=io.mask_dtype, 33 | ) 34 | regionprops.measure_regionprops(img, mask, ["area"]) # TODO 35 | 36 | def test_try_measure_regionprops_from_disk( 37 | self, imc_test_data_steinbock_path: Path 38 | ): 39 | img_files = io.list_image_files(imc_test_data_steinbock_path / "img") 40 | mask_files = io.list_mask_files( 41 | imc_test_data_steinbock_path / "masks", base_files=img_files 42 | ) 43 | gen = regionprops.try_measure_regionprops_from_disk( 44 | img_files, mask_files, ["area"] 45 | ) 46 | for img_file, mask_file, df in gen: 47 | pass # TODO 48 | -------------------------------------------------------------------------------- /tests/preprocessing/test_external_preprocessing.py: -------------------------------------------------------------------------------- 1 | class TestExternalPreprocessing: 2 | def test_list_image_files(self): 3 | pass # TODO 4 | 5 | def test_create_panel_from_image_files(self): 6 | pass # TODO 7 | 8 | def test_try_preprocess_images_from_disk(self): 9 | pass # TODO 10 | -------------------------------------------------------------------------------- /tests/preprocessing/test_imc_preprocessing.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import numpy as np 4 | import pytest 5 | 6 | from steinbock import io 7 | from steinbock.preprocessing import imc 8 | 9 | 10 | @pytest.mark.skipif(not imc.imc_available, reason="IMC is not available") 11 | class TestIMCPreprocessing: 12 | def test_list_mcd_files(self, imc_test_data_steinbock_path: Path): 13 | imc.list_mcd_files(imc_test_data_steinbock_path / "raw") # TODO 14 | 15 | def test_list_txt_files(self, imc_test_data_steinbock_path: Path): 16 | imc.list_txt_files(imc_test_data_steinbock_path / "raw") # TODO 17 | 18 | def test_create_panel_from_imc_panel(self, imc_test_data_steinbock_path: Path): 19 | imc.create_panel_from_imc_panel( 20 | imc_test_data_steinbock_path / "raw" / "panel.csv" 21 | ) # TODO 22 | 23 | def test_create_panel_from_mcd_file(self, imc_test_data_steinbock_path: Path): 24 | pass # TODO 25 | 26 | def test_create_panel_from_mcd_files(self, imc_test_data_steinbock_path: Path): 27 | mcd_files = imc.list_mcd_files( 28 | imc_test_data_steinbock_path / "raw" / "20210305_NE_mockData1" 29 | ) 30 | imc.create_panel_from_mcd_files(mcd_files) # TODO 31 | 32 | def test_create_panel_from_txt_file(self, imc_test_data_steinbock_path: Path): 33 | pass # TODO 34 | 35 | def test_create_panel_from_txt_files(self, imc_test_data_steinbock_path: Path): 36 | txt_files = imc.list_txt_files( 37 | imc_test_data_steinbock_path / "raw" / "20210305_NE_mockData1" 38 | ) 39 | imc.create_panel_from_txt_files(txt_files) # TODO 40 | 41 | def test_create_image_info(self, imc_test_data_steinbock_path: Path): 42 | pass # TODO 43 | 44 | def test_filter_hot_pixels(self): 45 | img = np.array( 46 | [ 47 | [ 48 | [1, 2, 3], 49 | [4, 13, 6], 50 | [7, 8, 9], 51 | ] 52 | ], 53 | dtype=io.img_dtype, 54 | ) 55 | filtered_img = imc.filter_hot_pixels(img, 3.0) 56 | expected_filtered_img = np.array( 57 | [ 58 | [ 59 | [1, 2, 3], 60 | [4, 9, 6], 61 | [7, 8, 9], 62 | ] 63 | ], 64 | dtype=io.img_dtype, 65 | ) 66 | assert np.all(filtered_img == expected_filtered_img) 67 | 68 | def test_preprocess_image(self): 69 | img = np.array( 70 | [ 71 | [ 72 | [1, 2, 3], 73 | [4, 13, 6], 74 | [7, 8, 9], 75 | ], 76 | ], 77 | dtype=io.img_dtype, 78 | ) 79 | preprocessed_img = imc.preprocess_image(img, hpf=3.0) 80 | expected_preprocessed_img = np.array( 81 | [ 82 | [ 83 | [1, 2, 3], 84 | [4, 9, 6], 85 | [7, 8, 9], 86 | ] 87 | ], 88 | dtype=io.img_dtype, 89 | ) 90 | assert np.all(preprocessed_img == expected_preprocessed_img) 91 | 92 | def test_try_preprocess_images_from_disk(self, imc_test_data_steinbock_path: Path): 93 | mcd_files = imc.list_mcd_files(imc_test_data_steinbock_path / "raw") 94 | txt_files = imc.list_txt_files(imc_test_data_steinbock_path / "raw") 95 | gen = imc.try_preprocess_images_from_disk(mcd_files, txt_files) 96 | for mcd_txt_file, acquisition, img, recovery_file, recovered in gen: 97 | pass # TODO 98 | -------------------------------------------------------------------------------- /tests/segmentation/test_cellpose_segmentation.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import pytest 4 | 5 | from steinbock.segmentation import cellpose 6 | 7 | 8 | @pytest.mark.skipif(not cellpose.cellpose_available, reason="Cellpose is not available") 9 | class TestCellposeSegmentation: 10 | def test_create_segmentation_stack(self, imc_test_data_steinbock_path: Path): 11 | pass # TODO 12 | 13 | @pytest.mark.skip(reason="Test would take too long") 14 | def test_try_segment_objects_nuclei(self, imc_test_data_steinbock_path: Path): 15 | pass # TODO 16 | -------------------------------------------------------------------------------- /tests/segmentation/test_cellprofiler_segmentation.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | from pathlib import Path 3 | 4 | import pytest 5 | 6 | from steinbock.segmentation import cellprofiler 7 | 8 | cellprofiler_binary = "cellprofiler" 9 | cellprofiler_plugin_dir = "/opt/cellprofiler_plugins" 10 | 11 | 12 | class TestCellprofilerSegmentation: 13 | def test_create_and_save_segmentation_pipeline(self, tmp_path: Path): 14 | cellprofiler.create_and_save_segmentation_pipeline( 15 | tmp_path / "cell_segmentation.cppipe" 16 | ) # TODO 17 | 18 | @pytest.mark.skip(reason="Test would take too long") 19 | @pytest.mark.skipif( 20 | shutil.which(cellprofiler_binary) is None, 21 | reason="CellProfiler is not available", 22 | ) 23 | def test_try_segment_objects( 24 | self, imc_test_data_steinbock_path: Path, tmp_path: Path 25 | ): 26 | cellprofiler.try_segment_objects( 27 | cellprofiler_binary, 28 | imc_test_data_steinbock_path / "cell_segmentation.cppipe", 29 | imc_test_data_steinbock_path / "ilastik_probabilities", 30 | tmp_path / "masks", 31 | cellprofiler_plugin_dir=cellprofiler_plugin_dir, 32 | ) # TODO 33 | -------------------------------------------------------------------------------- /tests/segmentation/test_deepcell_segmentation.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import numpy as np 4 | import pytest 5 | 6 | from steinbock import io 7 | from steinbock.segmentation import deepcell 8 | from steinbock.segmentation.deepcell import Application 9 | 10 | keras_models_dir = "/opt/keras/models" 11 | 12 | 13 | @pytest.mark.skipif(not deepcell.deepcell_available, reason="DeepCell is not available") 14 | class TestDeepcellSegmentation: 15 | def test_create_segmentation_stack(self, imc_test_data_steinbock_path: Path): 16 | pass # TODO 17 | 18 | @pytest.mark.skip(reason="Test would take too long") 19 | @pytest.mark.filterwarnings("ignore::DeprecationWarning") 20 | def test_try_segment_objects_mesmer(self, imc_test_data_steinbock_path: Path): 21 | from tensorflow.keras.models import load_model # type: ignore 22 | 23 | img_files = io.list_image_files(imc_test_data_steinbock_path / "img") 24 | model = None 25 | model_path = Path(keras_models_dir) / "MultiplexSegmentation" 26 | if model_path.exists(): 27 | model = load_model(model_path, compile=False) 28 | channel_groups = np.array([np.nan, 2, np.nan, np.nan, 1]) 29 | deepcell.try_segment_objects( 30 | img_files, 31 | Application.MESMER, 32 | model=model, 33 | channelwise_minmax=True, 34 | channel_groups=channel_groups, 35 | ) # TODO 36 | -------------------------------------------------------------------------------- /tests/test_io.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from steinbock import io 4 | 5 | 6 | class TestIO: 7 | def test_read_panel(self, imc_test_data_steinbock_path: Path): 8 | io.read_panel(imc_test_data_steinbock_path / "panel.csv") # TODO 9 | 10 | def test_write_panel(self, imc_test_data_steinbock_path: Path): 11 | pass # TODO 12 | 13 | def test_list_image_files(self, imc_test_data_steinbock_path: Path): 14 | io.list_image_files(imc_test_data_steinbock_path / "img") # TODO 15 | 16 | def test_read_image(self, imc_test_data_steinbock_path: Path): 17 | io.read_image( 18 | imc_test_data_steinbock_path / "img" / "20210305_NE_mockData1_1.tiff" 19 | ) # TODO 20 | 21 | def test_write_image(self, imc_test_data_steinbock_path: Path): 22 | pass # TODO 23 | 24 | def test_read_image_info(self, imc_test_data_steinbock_path: Path): 25 | io.read_image_info(imc_test_data_steinbock_path / "images.csv") # TODO 26 | 27 | def test_write_image_info(self, imc_test_data_steinbock_path: Path): 28 | pass # TODO 29 | 30 | def test_list_mask_files(self, imc_test_data_steinbock_path: Path): 31 | io.list_mask_files(imc_test_data_steinbock_path / "masks") # TODO 32 | 33 | def test_read_mask(self, imc_test_data_steinbock_path: Path): 34 | io.read_mask( 35 | imc_test_data_steinbock_path / "masks" / "20210305_NE_mockData1_1.tiff" 36 | ) # TODO 37 | 38 | def test_write_mask(self, imc_test_data_steinbock_path: Path): 39 | pass # TODO 40 | 41 | def test_list_data_files(self, imc_test_data_steinbock_path: Path): 42 | io.list_data_files(imc_test_data_steinbock_path / "intensities") # TODO 43 | 44 | def test_read_data(self, imc_test_data_steinbock_path: Path): 45 | io.read_data( 46 | imc_test_data_steinbock_path / "intensities" / "20210305_NE_mockData1_1.csv" 47 | ) # TODO 48 | 49 | def test_write_data(self, imc_test_data_steinbock_path: Path): 50 | pass # TODO 51 | 52 | def test_list_neighbors_files(self, imc_test_data_steinbock_path: Path): 53 | io.list_neighbors_files(imc_test_data_steinbock_path / "neighbors") # TODO 54 | 55 | def test_read_neighbors(self, imc_test_data_steinbock_path: Path): 56 | io.read_neighbors( 57 | imc_test_data_steinbock_path / "neighbors" / "20210305_NE_mockData1_1.csv" 58 | ) # TODO 59 | 60 | def test_write_neighbors(self, imc_test_data_steinbock_path: Path): 61 | pass # TODO 62 | -------------------------------------------------------------------------------- /tests/test_viewer.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | 4 | class TestViewer: 5 | def test_view(self, imc_test_data_steinbock_path: Path): 6 | pass # TODO 7 | -------------------------------------------------------------------------------- /tests/utils/test_expansion_utils.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | 4 | class TestExpansionUtils: 5 | def test_expand_mask(self, imc_test_data_steinbock_path: Path): 6 | pass # TODO 7 | 8 | def test_try_expand_masks_from_disk(self, imc_test_data_steinbock_path: Path): 9 | pass # TODO 10 | -------------------------------------------------------------------------------- /tests/utils/test_matching_utils.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import numpy as np 4 | 5 | from steinbock import io 6 | from steinbock.utils import matching 7 | 8 | 9 | class TestMatchingUtils: 10 | def test_match_masks(self): 11 | mask1 = np.array( 12 | [ 13 | [1, 1, 0], 14 | [0, 0, 0], 15 | [0, 2, 2], 16 | ], 17 | dtype=io.mask_dtype, 18 | ) 19 | mask2 = np.array( 20 | [ 21 | [3, 3, 0], 22 | [0, 0, 0], 23 | [0, 4, 4], 24 | ], 25 | dtype=io.mask_dtype, 26 | ) 27 | matching.match_masks(mask1, mask2) # TODO 28 | 29 | def test_try_match_masks_from_disk(self, imc_test_data_steinbock_path: Path): 30 | mask_files1 = io.list_mask_files(imc_test_data_steinbock_path / "masks") 31 | mask_files2 = io.list_mask_files( 32 | imc_test_data_steinbock_path / "masks", base_files=mask_files1 33 | ) 34 | gen = matching.try_match_masks_from_disk(mask_files1, mask_files2) 35 | for mask_file1, mask_file2, df in gen: 36 | pass # TODO 37 | -------------------------------------------------------------------------------- /tests/utils/test_mosaics_utils.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | 4 | class TestMosaicsUtils: 5 | def test_try_extract_tiles_from_disk_to_disk( 6 | self, imc_test_data_steinbock_path: Path 7 | ): 8 | # img_files = io.list_image_files(imc_test_data_steinbock_path / "img") 9 | # gen = mosaics.try_extract_tiles_from_disk(img_files, 50) 10 | # for img_file, tile_info, tile in gen: 11 | # pass # TODO 12 | pass # TODO 13 | 14 | def test_try_stitch_tiles_from_disk_to_disk( 15 | self, imc_test_data_steinbock_path: Path, tmp_path: Path 16 | ): 17 | # img_files = io.list_image_files(imc_test_data_steinbock_path / "img") 18 | # img_file_stems = [] 19 | # img_tile_files = {} 20 | # img_tile_infos = {} 21 | # gen = mosaics.try_extract_tiles_from_disk(img_files, 50) 22 | # for img_file, tile_info, tile in gen: 23 | # tile_file = tmp_path / ( 24 | # f"{tile_info.img_file_stem}_tx{tile_info.x}_ty{tile_info.y}" 25 | # f"_tw{tile_info.width}_th{tile_info.height}.tiff" 26 | # ) 27 | # if tile_info.img_file_stem not in img_file_stems: 28 | # img_file_stems.append(tile_info.img_file_stem) 29 | # img_tile_files[tile_info.img_file_stem] = [] 30 | # img_tile_infos[tile_info.img_file_stem] = [] 31 | # img_tile_files[tile_info.img_file_stem].append(tile_file) 32 | # img_tile_infos[tile_info.img_file_stem].append(tile_info) 33 | # io.write_image(tile, tile_file, ignore_dtype=True) 34 | # del tile 35 | # gen = mosaics.try_stitch_tiles_from_disk( 36 | # img_file_stems, img_tile_files, img_tile_infos 37 | # ) 38 | # for img_file_stem, img in gen: 39 | # pass # TODO 40 | pass # TODO 41 | --------------------------------------------------------------------------------