├── .flake8 ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── workflows │ ├── build-and-test.yml │ ├── lint.yml │ ├── pypi.yml │ └── version_check.yml ├── .gitignore ├── .python-version ├── .readthedocs.yaml ├── CITATION.cff ├── LICENSE ├── README.md ├── docs ├── _config.yml ├── _toc.yml ├── api.rst ├── conf.py ├── examples.md ├── intro.md ├── logo.PNG ├── notebooks │ ├── STATSnc.ipynb │ ├── background_correction.ipynb │ ├── big_datasets.ipynb │ ├── cli.ipynb │ ├── config.toml │ ├── exploring_pipeline_data.ipynb │ ├── montaging.ipynb │ ├── pipeline_step_by_step.ipynb │ ├── processing_raw_data.ipynb │ ├── stats.ipynb │ └── toml_config.ipynb └── requirements.txt ├── environment.yml ├── notebooks ├── config-holo.toml ├── config.toml ├── pipeline-holo.ipynb ├── pyopia-classifier │ └── pyopia-default-classifier.ipynb ├── single-image-stats-holo.ipynb └── single-image-stats.ipynb ├── pyopia ├── __init__.py ├── background.py ├── cf_metadata.json ├── classify.py ├── cli.py ├── exampledata.py ├── instrument │ ├── __init__.py │ ├── common.py │ ├── holo.py │ ├── silcam.py │ └── uvp.py ├── io.py ├── pipeline.py ├── plotting.py ├── process.py ├── simulator │ ├── __init__.py │ └── silcam.py ├── statistics.py └── tests │ ├── __init__.py │ ├── test_classify.py │ ├── test_io.py │ ├── test_notebooks.py │ └── test_pipeline.py └── pyproject.toml /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 130 3 | max-complexity = 10 -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Detailed steps to reproduce the behavior. 15 | 16 | **Expected behavior** 17 | A clear and concise description of what you expected to happen. 18 | 19 | **Screenshots** 20 | If applicable add screenshots to help explain your problem. 21 | 22 | **Additional context** 23 | Add any other context about the problem here. 24 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/workflows/build-and-test.yml: -------------------------------------------------------------------------------- 1 | name: Build and test 2 | 3 | on: [pull_request] 4 | 5 | jobs: 6 | Windows_uv: 7 | runs-on: windows-latest 8 | timeout-minutes: 60 9 | 10 | steps: 11 | - name: Check out code 12 | uses: actions/checkout@v2 13 | 14 | - name: Install uv 15 | uses: astral-sh/setup-uv@v5 16 | with: 17 | version: "0.6.10" 18 | 19 | - name: Install PyOPIA 20 | run: uv sync --all-extras --dev 21 | 22 | - name: Run the automated tests 23 | run: uv run pytest -v 24 | 25 | Ubuntu_uv: 26 | runs-on: ubuntu-latest 27 | timeout-minutes: 60 28 | 29 | steps: 30 | - name: Check out code 31 | uses: actions/checkout@v3 32 | 33 | - name: Install uv 34 | uses: astral-sh/setup-uv@v5 35 | with: 36 | version: "0.6.10" 37 | 38 | - name: Install PyOPIA 39 | run: uv sync --all-extras --dev 40 | 41 | - name: Run the automated tests 42 | run: uv run pytest -v 43 | 44 | MacOS_uv: 45 | runs-on: macos-latest 46 | timeout-minutes: 60 47 | 48 | steps: 49 | - name: Check out code 50 | uses: actions/checkout@v3 51 | 52 | - name: Install uv 53 | uses: astral-sh/setup-uv@v5 54 | with: 55 | version: "0.6.10" 56 | 57 | - name: Install PyOPIA 58 | run: uv sync --all-extras --dev 59 | 60 | - name: Run the automated tests 61 | run: uv run pytest -v 62 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: flake8 Lint 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | flake8-lint: 7 | runs-on: ubuntu-latest 8 | name: Lint 9 | steps: 10 | - name: Check out source repository 11 | uses: actions/checkout@v2 12 | - name: Set up Python environment 13 | uses: actions/setup-python@v2 14 | with: 15 | python-version: "3.10" 16 | - name: flake8 Lint 17 | uses: py-actions/flake8@v2 18 | with: 19 | exclude: "docs/conf.py" 20 | -------------------------------------------------------------------------------- /.github/workflows/pypi.yml: -------------------------------------------------------------------------------- 1 | name: Publish 2 | 3 | on: 4 | release: 5 | types: 6 | - released 7 | push: 8 | branches: 9 | - main 10 | workflow_dispatch: 11 | 12 | jobs: 13 | pypi-publish: 14 | name: upload release to PyPI 15 | runs-on: ubuntu-latest 16 | environment: 17 | name: release 18 | url: https://pypi.org/p/pyopia 19 | permissions: 20 | id-token: write 21 | steps: 22 | - uses: actions/checkout@v4 23 | 24 | - name: Install uv 25 | uses: astral-sh/setup-uv@v3 26 | 27 | - name: Install Python 28 | run: uv python install 3.12 29 | 30 | - name: Build 31 | run: uv build 32 | 33 | - name: Publish package distributions to PyPI 34 | uses: pypa/gh-action-pypi-publish@release/v1 35 | -------------------------------------------------------------------------------- /.github/workflows/version_check.yml: -------------------------------------------------------------------------------- 1 | name: Version check 2 | 3 | on: [pull_request] 4 | 5 | env: 6 | IMAGE_NAME: sintef/pyopia 7 | IMAGE_TAG: github-ci 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | timeout-minutes: 60 13 | 14 | steps: 15 | - name: Check out code 16 | uses: actions/checkout@v2 17 | with: 18 | fetch-depth: 0 19 | 20 | - name: Diff 21 | id: main 22 | run: | 23 | check=$(git diff origin/main -- pyopia/__init__.py | grep __version | wc -l) && 24 | echo "::set-output name=check::$check" 25 | 26 | - name: Check 27 | if: steps.main.outputs.check != 2 28 | run: | 29 | echo "__init__.py is unchanged" 30 | exit 1 31 | 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/vim,python,pycharm,visualstudiocode 3 | # Edit at https://www.gitignore.io/?templates=vim,python,pycharm,visualstudiocode 4 | 5 | ### PyCharm ### 6 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 7 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 8 | 9 | # User-specific stuff 10 | .idea/ 11 | .idea/**/workspace.xml 12 | .idea/**/tasks.xml 13 | .idea/**/usage.statistics.xml 14 | .idea/**/dictionaries 15 | .idea/**/shelf 16 | 17 | # Generated files 18 | .idea/**/contentModel.xml 19 | 20 | # Sensitive or high-churn files 21 | .idea/**/dataSources/ 22 | .idea/**/dataSources.ids 23 | .idea/**/dataSources.local.xml 24 | .idea/**/sqlDataSources.xml 25 | .idea/**/dynamic.xml 26 | .idea/**/uiDesigner.xml 27 | .idea/**/dbnavigator.xml 28 | 29 | # Gradle 30 | .idea/**/gradle.xml 31 | .idea/**/libraries 32 | 33 | # Gradle and Maven with auto-import 34 | # When using Gradle or Maven with auto-import, you should exclude module files, 35 | # since they will be recreated, and may cause churn. Uncomment if using 36 | # auto-import. 37 | # .idea/modules.xml 38 | # .idea/*.iml 39 | # .idea/modules 40 | # *.iml 41 | # *.ipr 42 | 43 | # CMake 44 | cmake-build-*/ 45 | 46 | # Mongo Explorer plugin 47 | .idea/**/mongoSettings.xml 48 | 49 | # File-based project format 50 | *.iws 51 | 52 | # IntelliJ 53 | out/ 54 | 55 | # mpeltonen/sbt-idea plugin 56 | .idea_modules/ 57 | 58 | # JIRA plugin 59 | atlassian-ide-plugin.xml 60 | 61 | # Cursive Clojure plugin 62 | .idea/replstate.xml 63 | 64 | # Crashlytics plugin (for Android Studio and IntelliJ) 65 | com_crashlytics_export_strings.xml 66 | crashlytics.properties 67 | crashlytics-build.properties 68 | fabric.properties 69 | 70 | # Editor-based Rest Client 71 | .idea/httpRequests 72 | 73 | # Android studio 3.1+ serialized cache file 74 | .idea/caches/build_file_checksums.ser 75 | 76 | ### PyCharm Patch ### 77 | # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 78 | 79 | # *.iml 80 | # modules.xml 81 | # .idea/misc.xml 82 | # *.ipr 83 | 84 | # Sonarlint plugin 85 | .idea/**/sonarlint/ 86 | 87 | # SonarQube Plugin 88 | .idea/**/sonarIssues.xml 89 | 90 | # Markdown Navigator plugin 91 | .idea/**/markdown-navigator.xml 92 | .idea/**/markdown-navigator/ 93 | 94 | ### Python ### 95 | # Byte-compiled / optimized / DLL files 96 | __pycache__/ 97 | *.py[cod] 98 | *$py.class 99 | 100 | # C extensions 101 | *.so 102 | 103 | # Distribution / packaging 104 | .Python 105 | build/ 106 | develop-eggs/ 107 | dist/ 108 | downloads/ 109 | eggs/ 110 | .eggs/ 111 | lib/ 112 | lib64/ 113 | parts/ 114 | sdist/ 115 | var/ 116 | wheels/ 117 | pip-wheel-metadata/ 118 | share/python-wheels/ 119 | *.egg-info/ 120 | .installed.cfg 121 | *.egg 122 | MANIFEST 123 | 124 | # PyInstaller 125 | # Usually these files are written by a python script from a template 126 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 127 | *.manifest 128 | *.spec 129 | 130 | # Installer logs 131 | pip-log.txt 132 | pip-delete-this-directory.txt 133 | 134 | # Unit test / coverage reports 135 | htmlcov/ 136 | .tox/ 137 | .nox/ 138 | .coverage 139 | .coverage.* 140 | .cache 141 | nosetests.xml 142 | coverage.xml 143 | *.cover 144 | .hypothesis/ 145 | .pytest_cache/ 146 | 147 | # Translations 148 | *.mo 149 | *.pot 150 | 151 | # Scrapy stuff: 152 | .scrapy 153 | 154 | # Sphinx documentation 155 | docs/_build/ 156 | docs/_autosummary/ 157 | docs/source/ 158 | !docs/source/conf.py 159 | !docs/source/index.rst 160 | 161 | # PyBuilder 162 | target/ 163 | 164 | # pyenv 165 | .python-version 166 | 167 | # pipenv 168 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 169 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 170 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 171 | # install all needed dependencies. 172 | #Pipfile.lock 173 | 174 | # celery beat schedule file 175 | celerybeat-schedule 176 | 177 | # SageMath parsed files 178 | *.sage.py 179 | 180 | # Spyder project settings 181 | .spyderproject 182 | .spyproject 183 | 184 | # Rope project settings 185 | .ropeproject 186 | 187 | # Mr Developer 188 | .mr.developer.cfg 189 | .project 190 | .pydevproject 191 | 192 | # mkdocs documentation 193 | /site 194 | 195 | # mypy 196 | .mypy_cache/ 197 | .dmypy.json 198 | dmypy.json 199 | 200 | # Pyre type checker 201 | .pyre/ 202 | 203 | ### Vim ### 204 | # Swap 205 | [._]*.s[a-v][a-z] 206 | [._]*.sw[a-p] 207 | [._]s[a-rt-v][a-z] 208 | [._]ss[a-gi-z] 209 | [._]sw[a-p] 210 | 211 | # Session 212 | Session.vim 213 | Sessionx.vim 214 | 215 | # Temporary 216 | .netrwhist 217 | *~ 218 | 219 | # Auto-generated tag files 220 | tags 221 | 222 | # Persistent undo 223 | [._]*.un~ 224 | 225 | # Coc configuration directory 226 | .vim 227 | 228 | ### VisualStudioCode ### 229 | .vscode 230 | .vscode/* 231 | !.vscode/settings.json 232 | !.vscode/tasks.json 233 | !.vscode/launch.json 234 | !.vscode/extensions.json 235 | 236 | ### VisualStudioCode Patch ### 237 | # Ignore all local history of files 238 | .history 239 | 240 | ### ipynb checkpoints 241 | **/*.ipynb_checkpoints/ 242 | 243 | ### data files 244 | *.silc 245 | *.bmp 246 | *.png 247 | *.h5 248 | *.pgm 249 | *.h5 250 | !docs/logo.PNG 251 | 252 | # End of https://www.gitignore.io/api/vim,python,pycharm,visualstudiocode 253 | *.zip 254 | notebooks/header.tfl.txt 255 | notebooks/keras_model.h5 256 | notebooks/github-test-holo.ipynb 257 | 258 | test-report/* 259 | docs/.DS_Store 260 | .DS_Store 261 | docs/notebooks/header.tfl.txt 262 | docs/jupyter_execute/ 263 | *.nc 264 | 265 | poetry.lock 266 | /notebooks/gas_silcam_images 267 | /notebooks/oil_silcam_images 268 | header.tfl.txt 269 | *.tiff 270 | notebooks/__MACOSX/* 271 | notebooks/silcam240822.keras 272 | model/* 273 | notebooks/model/* 274 | *.keras 275 | *.png 276 | *-summary.txt 277 | /dev/* 278 | -------------------------------------------------------------------------------- /.python-version: -------------------------------------------------------------------------------- 1 | 3.12 2 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | build: 9 | os: "ubuntu-22.04" 10 | tools: 11 | python: "3.12" 12 | 13 | # Build documentation in the docs/ directory with Sphinx 14 | sphinx: 15 | builder: html 16 | configuration: docs/conf.py 17 | fail_on_warning: false 18 | 19 | # Optionally build your docs in additional formats such as PDF 20 | #formats: 21 | # - pdf 22 | 23 | # Optionally set the version of Python and requirements required to build your docs 24 | python: 25 | install: 26 | - requirements: docs/requirements.txt 27 | - path: . 28 | 29 | # By default readthedocs does not checkout git submodules 30 | submodules: 31 | include: all 32 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | message: "If you use this software, please cite it as below." 3 | authors: 4 | - family-names: Davies 5 | given-names: Emlyn 6 | - family-names: Nimmo-Smith 7 | given-names: Alex 8 | - family-names: Nepstad 9 | given-names: Raymond 10 | - family-names: Nordam 11 | given-names: Tor 12 | - family-names: Brönner 13 | given-names: Ute 14 | - family-names: Steinvika 15 | given-names: Andreas 16 | - family-names: Sari 17 | given-names: Giering 18 | - family-names: Masoudi 19 | given-names: Mojtaba 20 | - family-names: Liu 21 | given-names: Zonghua 22 | - family-names: Hélaouët 23 | given-names: Pierre 24 | - family-names: Cursons 25 | given-names: Kairan 26 | - family-names: Rau 27 | given-names: Matthew 28 | - family-names: Song 29 | given-names: Yixuan 30 | - family-names: Mostaani 31 | given-names: Arsalan 32 | - family-names: Barstein 33 | given-names: Karoline 34 | - family-names: Buscombe 35 | given-names: Daniel 36 | title: "PyOPIA: A Python Ocean Particle Image Analysis toolbox" 37 | url: "https://pyopia.readthedocs.io" 38 | repository-code: "https://github.com/sintef/pyopia" 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2018 SINTEF OCEAN 2 | 3 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 4 | 5 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 6 | 7 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | 9 | 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 10 | 11 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | PyOPIA 2 | =============================== 3 | 4 | A Python Ocean Particle Image Analysis toolbox 5 | 6 | # Quick tryout of PyOPIA 7 | 8 | 1) Install [uv](https://docs.astral.sh/uv/getting-started/installation) 9 | 2) Run PyOPIA classification tests on database particles 10 | ```bash 11 | uv run --python 3.12 --with git+https://github.com/SINTEF/pyopia --with tensorflow==2.16.2 --with keras==3.5.0 python -m pyopia.tests.test_classify 12 | ``` 13 | 14 | # Documentation: 15 | 16 | [![Jupyter Book Badge](https://jupyterbook.org/badge.svg)](https://pyopia.readthedocs.io) [![Documentation](https://readthedocs.org/projects/pyopia/badge/?version=latest)](https://pyopia.readthedocs.io/en/latest/?badge=latest) 17 | 18 | [pyopia.readthedocs.io](https://pyopia.readthedocs.io) 19 | # Current status: 20 | 21 | - Under development. See/regester issues, [here](https://github.com/SINTEF/pyopia/issues) 22 | 23 | ---- 24 | # Development targets for PyOpia: 25 | 26 | 1) Allow nonfamiliar users to install and use PyOpia, and to contribute & commit code changes 27 | 2) Not hardware specific 28 | 3) Smaller dependency list than PySilCam -Eventual optional dependencies (e.g. for classification) 29 | 4) Can be imported by pysilcam or other hardware-specific tools 30 | 5) Work on a single-image basis (...primarily, with options for multiprocess to be considered later) 31 | 6) No use of settings/config files within the core code - pass arguments directly. Eventual use of settings/config files should be handled by high-level wrappers that provide arguments to functions. 32 | 7) Github workflows 33 | 8) Tests 34 | 35 | Normal functions within PyOpia should: 36 | 37 | 1) take inputs 38 | 2) return new outputs 39 | 3) don't modify state of input 40 | 4) minimum possible disc IO during processing 41 | 42 | ## Contributions 43 | 44 | We welcome additions and improvements to the code! We request that you follow a few guidelines. These are in place to make sure the code improves over time. 45 | 46 | 1. All code changes must be submitted as pull requests, either from a branch or a fork. 47 | 2. Good documentation of the code is needed for PyOpia to succeed and so please include up-to-date docstrings as you make changes, so that the auto-build on readthedocs is complete and useful for users. (A version of the new docs will complie when you make a pull request and a link to this can be found in the pull request checks) 48 | 3. All pull requests are required to pass all tests before merging. Please do not disable or remove tests just to make your branch pass the pull request. 49 | 4. All pull requests must be reviewed by a person. The benefits from code review are plenty, but we like to emphasise that code reviews help spreading the awarenes of code changes. Please note that code reviews should be a pleasant experience, so be plesant, polite and remember that there is a human being with good intentions on the other side of the screen. 50 | 5. All contributions are linted with flake8. We recommend that you run flake8 on your code while developing to fix any issues as you go. We recommend using autopep8 to autoformat your Python code (but please check the code behaviour is not affected by autoformatting before pushing). This makes flake8 happy, and makes it easier for us all to maintain a consistent and readable code base. 51 | 52 | ## Docstrings 53 | 54 | Use the NumPy style in docstrings. See style guide [here](https://numpydoc.readthedocs.io/en/latest/format.html#documenting-classes) 55 | 56 | # Installing 57 | 58 | ## For users 59 | 60 | Users are expected to be familiar with Python. Please refer to the recommended installation instructions provided on the documentation pages, [here](https://pyopia.readthedocs.io/en/latest/intro.html#installing) 61 | 62 | ## For developers from source 63 | 64 | 65 | Install (uv)[https://docs.astral.sh/uv/getting-started/installation/] 66 | 67 | 1. Navigate to the folder where you want to install pyopia using the 'cd' command. 68 | 69 | If you use git: 70 | Download repository from github, and move into the new directory: 71 | 72 | ```bash 73 | git clone https://github.com/SINTEF/pyopia.git 74 | cd pyopia 75 | ``` 76 | 77 | For the next steps, you need to be located in the PyOPIA root directory that contains the file 'pyproject.toml'. 78 | 79 | 2. Install all requirements with 80 | 81 | ```bash 82 | uv sync --all-extras 83 | ``` 84 | 85 | 3. (optional) Run local tests: 86 | 87 | ```bash 88 | uv run pytest 89 | ``` 90 | 91 | #### Version numbering 92 | 93 | The version number of PyOPIA is split into three sections: MAJOR.MINOR.PATCH 94 | 95 | * MAJOR: Changes in high-level pipeline use and/or data output that are not backwards-compatible. 96 | * MINOR: New features that are backwards-compatible. 97 | * PATCH: Backwards-compatible bug fixes or enhancements to existing functionality 98 | 99 | ## Build docs locally 100 | 101 | ``` 102 | sphinx-apidoc -f -o docs/source docs/build --separate 103 | 104 | sphinx-build -b html ./docs/ ./docs/build 105 | ``` 106 | 107 | ---- 108 | # License 109 | 110 | PyOpia is licensed under the BSD3 license. See LICENSE. All contributors should be recognised & aknowledged. 111 | -------------------------------------------------------------------------------- /docs/_config.yml: -------------------------------------------------------------------------------- 1 | # Book settings 2 | # Learn more at https://jupyterbook.org/customize/config.html 3 | 4 | title: PyOPIA 5 | author: The PyOPIA Community 6 | logo: logo.PNG 7 | 8 | # Force re-execution of notebooks on each build. 9 | # See https://jupyterbook.org/content/execute.html 10 | execute: 11 | execute_notebooks: false 12 | 13 | # Define the name of the latex output file for PDF builds 14 | #latex: 15 | # latex_documents: 16 | # targetname: book.tex 17 | 18 | # Add a bibtex file so that we can create citations 19 | # bibtex_bibfiles: 20 | # - references.bib 21 | 22 | # Information about where the book exists on the web 23 | repository: 24 | url: https://github.com/SINTEF/pyopia # Online location of your book 25 | path_to_book: docs # Optional path to your book, relative to the repository root 26 | branch: master # Which branch of the repository should be used when creating links (optional) 27 | 28 | # Add GitHub buttons to your book 29 | # See https://jupyterbook.org/customize/config.html#add-a-link-to-your-repository 30 | html: 31 | use_issues_button: true 32 | use_repository_button: true 33 | 34 | 35 | sphinx: 36 | extra_extensions: 37 | - 'sphinx.ext.autodoc' 38 | - 'sphinx.ext.napoleon' 39 | - 'sphinx.ext.autosummary' 40 | - 'sphinx.ext.sphinx_togglebutton' 41 | config: 42 | autosummary_generate: True -------------------------------------------------------------------------------- /docs/_toc.yml: -------------------------------------------------------------------------------- 1 | # Table of contents 2 | # Learn more at https://jupyterbook.org/customize/toc.html 3 | 4 | format: jb-book 5 | root: intro 6 | 7 | parts: 8 | - caption: 9 | chapters: 10 | - file: examples 11 | - caption: Processing examples 12 | chapters: 13 | - file: notebooks/processing_raw_data.ipynb 14 | - file: notebooks/toml_config.ipynb 15 | - file: notebooks/cli 16 | - file: notebooks/big_datasets.ipynb 17 | - caption: STATS particle statistics data 18 | chapters: 19 | - file: notebooks/stats 20 | - file: notebooks/STATSnc 21 | - caption: Advanced analysis 22 | chapters: 23 | - file: notebooks/pipeline_step_by_step 24 | - file: notebooks/background_correction 25 | - file: notebooks/exploring_pipeline_data 26 | - file: notebooks/montaging 27 | - caption: Code docs 28 | chapters: 29 | - file: api 30 | -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- 1 | API documentation 2 | ================= 3 | 4 | .. autosummary:: 5 | :toctree: _autosummary 6 | :recursive: 7 | 8 | pyopia 9 | 10 | ---- 11 | 12 | **Content** 13 | 14 | * `CLI`_ 15 | * `Pipeline`_ 16 | * `Background`_ 17 | * `Process`_ 18 | * `Statistics`_ 19 | * `Plotting`_ 20 | * `IO`_ 21 | * `Classify`_ 22 | * `ExampleData`_ 23 | 24 | **Instrument-Specific** 25 | 26 | * `SilCam`_ 27 | * `Holo`_ 28 | * `UVP`_ 29 | 30 | **Simulators** 31 | 32 | * `SilCam-Simulator`_ 33 | 34 | **Indices and tables** 35 | 36 | * :ref:`genindex` 37 | * :ref:`modindex` 38 | 39 | 40 | ---- 41 | 42 | CLI 43 | -------------- 44 | .. automodule:: pyopia.cli 45 | :members: 46 | 47 | Pipeline 48 | -------------- 49 | .. automodule:: pyopia.pipeline 50 | :members: 51 | 52 | Background 53 | -------------- 54 | .. automodule:: pyopia.background 55 | :members: 56 | 57 | Process 58 | -------------- 59 | .. automodule:: pyopia.process 60 | :members: 61 | 62 | Statistics 63 | -------------- 64 | .. automodule:: pyopia.statistics 65 | :members: 66 | 67 | Plotting 68 | -------------- 69 | .. automodule:: pyopia.plotting 70 | :members: 71 | 72 | IO 73 | -------------- 74 | .. automodule:: pyopia.io 75 | :members: 76 | 77 | Classify 78 | -------------- 79 | .. automodule:: pyopia.classify 80 | :members: 81 | 82 | ExampleData 83 | -------------- 84 | .. automodule:: pyopia.exampledata 85 | :members: 86 | 87 | Instruments 88 | ================== 89 | SilCam 90 | -------------- 91 | .. automodule:: pyopia.instrument.silcam 92 | :members: 93 | 94 | Holo 95 | -------------- 96 | .. automodule:: pyopia.instrument.holo 97 | :members: 98 | 99 | UVP 100 | -------------- 101 | .. automodule:: pyopia.instrument.uvp 102 | :members: 103 | 104 | Common 105 | ------------- 106 | .. automodule:: pyopia.instrument.common 107 | :members: 108 | 109 | Simulators 110 | ================== 111 | Silcam-Simulator 112 | -------------- 113 | .. automodule:: pyopia.simulator.silcam 114 | :members: 115 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Auto-generated by `jupyter-book config` 3 | # If you wish to continue using _config.yml, make edits to that file and 4 | # re-generate this one. 5 | ############################################################################### 6 | author = 'The PyOPIA Community' 7 | autosummary_generate = True 8 | comments_config = {'hypothesis': False, 'utterances': False} 9 | copyright = '2022' 10 | exclude_patterns = ['**.ipynb_checkpoints', '.DS_Store', 'Thumbs.db', '_build'] 11 | extensions = ['sphinx_togglebutton', 'sphinx_copybutton', 'myst_nb', 'jupyter_book', 'sphinx_thebe', 'sphinx_comments', 'sphinx_external_toc', 'sphinx.ext.intersphinx', 'sphinx_design', 'sphinx_book_theme', 'sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.autosummary', 'sphinx_jupyterbook_latex'] 12 | external_toc_exclude_missing = False 13 | external_toc_path = '_toc.yml' 14 | html_baseurl = '' 15 | html_favicon = '' 16 | html_logo = 'logo.PNG' 17 | html_sourcelink_suffix = '' 18 | html_theme = 'sphinx_book_theme' 19 | html_theme_options = {'search_bar_text': 'Search this book...', 'launch_buttons': {'notebook_interface': 'classic', 'binderhub_url': '', 'jupyterhub_url': '', 'thebe': False, 'colab_url': ''}, 'path_to_docs': 'docs', 'repository_url': 'https://github.com/SINTEF/pyopia', 'repository_branch': 'master', 'extra_footer': '', 'home_page_in_toc': True, 'announcement': '', 'analytics': {'google_analytics_id': ''}, 'use_repository_button': True, 'use_edit_page_button': False, 'use_issues_button': True} 20 | html_context = {"default_mode": "light"} 21 | html_title = 'PyOPIA' 22 | latex_engine = 'pdflatex' 23 | myst_enable_extensions = ['colon_fence', 'dollarmath', 'linkify', 'substitution', 'tasklist'] 24 | myst_url_schemes = ['mailto', 'http', 'https'] 25 | nb_execution_allow_errors = False 26 | nb_execution_cache_path = '' 27 | nb_execution_excludepatterns = [] 28 | nb_execution_in_temp = False 29 | nb_execution_mode = 'off' 30 | nb_execution_timeout = 30 31 | nb_output_stderr = 'show' 32 | numfig = True 33 | pygments_style = 'sphinx' 34 | suppress_warnings = ['myst.domains'] 35 | use_jupyterbook_latex = True 36 | use_multitoc_numbering = True 37 | -------------------------------------------------------------------------------- /docs/examples.md: -------------------------------------------------------------------------------- 1 | PyOPIA by example 2 | ================================== 3 | 4 | Here are some simple guides to get you started with PyOPIA. If you can't find what you are looking for, have a look at the [notebooks](https://github.com/SINTEF/pyopia/tree/main/notebooks) or log an issue with a request, [here](https://github.com/SINTEF/pyopia/issues/new/choose) 5 | 6 | Some examples of how to use {class}`pyopia.pipeline.Pipeline` for silcam can be found for SilCam [here](https://github.com/SINTEF/pyopia/blob/main/notebooks/single-image-stats.ipynb) and holographic analysis [here](https://github.com/SINTEF/pyopia/blob/main/notebooks/pipeline-holo.ipynb). 7 | -------------------------------------------------------------------------------- /docs/intro.md: -------------------------------------------------------------------------------- 1 | Welcome! 2 | ================================== 3 | 4 | This is documentation for PyOPIA: Python Ocean Particle Image Analysis, hosted by [SINTEF Ocean](https://www.sintef.no/en/ocean/), in collaboration with University of Plymouth, NTNU, The MBA, George Washington University, and National Oceanography Center (Southampton). We hope to encourage wider collaboration on analysis tools for ocean particles and welcome new contributors! 5 | 6 | PyOPIA started in Feb. 2022 as a 'spin-off' from elements of [PySilCam](https://github.com/SINTEF/PySilCam/wiki) that could be relevant for other hardware and image types, including holography. 7 | 8 | The code repository for PyOPIA can be found [here](https://github.com/SINTEF/pyopia/). 9 | 10 | Pipelines 11 | ================================== 12 | PyOPIA aims to provide a pipeline-based workflow as a standard for analysis of particle images in the ocean. 13 | 14 | This pipeline should be consistent across different instruments (hardware), and therefore has flexibility to adapt analysis steps to meet instrument-specific processing needs (i.e. holographic reconstruction), while maintaining a traceable workflow that is attached as metadata to a standard output file (HDF5) that helps users follow FAIR data principles. 15 | 16 | See {class}`pyopia.pipeline.Pipeline` for more details and examples of how to process images with PyOPIA. 17 | 18 | A function-based toolbox 19 | ================================== 20 | 21 | PyOPIA tools are organised into the following modules: 22 | 23 | * background correction {mod}`pyopia.background`. 24 | * processing {mod}`pyopia.process`. 25 | * statistical analysis {mod}`pyopia.statistics`. 26 | * classification {mod}`pyopia.classify`. 27 | * metadata & datafile formatting {mod}`pyopia.io`. 28 | 29 | You can combine these tools for exploring different analysis approaches (i.e. in notebooks). 30 | We hope this can help more exploratory development and contributions to the PyOPIA code. 31 | 32 | If you are analysing data for publication, we recommend using the {class}`pyopia.pipeline.Pipeline` standard so that your analysis steps are documented and the output format is more easily shareable. 33 | 34 | Full documentation for the code is [here](api) 35 | 36 | Installing 37 | ================================== 38 | 39 | Users are expected to be familiar with Python and [uv](https://docs.astral.sh/uv/getting-started/installation/). You can create a new uv project and install PyOPIA like this: 40 | 41 | ``` 42 | uv init --python 3.12 mypyopiaproject 43 | cd mypyopiaproject 44 | uv add pyopia[classification] 45 | ``` 46 | 47 | To run PyOPIA, either use uv (uv run pyopia --help), or activate the venv first (source .venv/bin/activate), before running pyopia (pyopia --help). 48 | 49 | The [classification] part installs tensorflow which is required by PyOPIA's Classification module, and is optional. 50 | 51 | To confirm that everything was installed correctly, you can run a PyOPIA test for the classifier: 52 | 53 | ``` 54 | uv run python -m pyopia.tests.test_classify 55 | ``` 56 | 57 | If you would like to install a development environment, please refer to the instructions in the README on GitHub, [here](https://github.com/SINTEF/pyopia/blob/main/README.md) 58 | 59 | Links to libraries PyOPIA uses 60 | ================================== 61 | 62 | PyOPIA is a high-level tool that makes use of several open source libraries. Please see the list of libraries listed in the pyproject.toml file if you are interested in what is used. 63 | 64 | Processing and plotting modeuls makes routine use of several functions provided by libraries including: [scikit-image](https://scikit-image.org/), 65 | [numpy](https://numpy.org/), [pandas](https://pandas.pydata.org/), [xarray](https://docs.xarray.dev), [matplotlib](https://matplotlib.org/), 66 | [cmocean](https://matplotlib.org/cmocean/), [tensorflow](https://www.tensorflow.org/), and [keras](https://keras.io/). 67 | -------------------------------------------------------------------------------- /docs/logo.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SINTEF/pyopia/bac90c61d39b668e5c06cbb966b73b8db605f5cb/docs/logo.PNG -------------------------------------------------------------------------------- /docs/notebooks/background_correction.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Different ways to do background correction\n", 8 | "Background correction is an optional step in the analysis pipeline, and is used to remove static elements in an image for improved analysis results.\n", 9 | "\n", 10 | "In PyOpia there are several ways to use the background correction functionality, illustrated in this notebook.\n", 11 | "\n", 12 | "The default behavior is to set up the background for the first N images of a pipeline run, and not perform any analysis. \n", 13 | "After the background is completely set up, analysis starts (from image N), with either a dynamic running average or static background, depending on the configuration choices. You can customize the behavior you want by changing the configuration and by adding custom code, illustrated at the end of this notebook." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 7, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "from glob import glob\n", 23 | "import numpy as np\n", 24 | "import matplotlib.pyplot as plt\n", 25 | "from skimage.exposure import rescale_intensity\n", 26 | "import zipfile\n", 27 | "\n", 28 | "import pyopia.exampledata\n", 29 | "from pyopia.pipeline import Pipeline" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": 6, 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "# Download example image files\n", 39 | "pyopia.exampledata.get_file_from_pysilcam_blob('oil.zip')\n", 40 | "with zipfile.ZipFile('oil.zip', 'r') as zipit:\n", 41 | " zipit.extractall('.')" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": null, 47 | "metadata": {}, 48 | "outputs": [], 49 | "source": [ 50 | "# These imports are indirectly needed for the Pipeline\n", 51 | "import pyopia.background\n", 52 | "import pyopia.instrument.silcam\n", 53 | "\n", 54 | "# Manually define PyOpia pipeline configuration\n", 55 | "NUM_IMAGES_FOR_BACKGROUND = 5\n", 56 | "\n", 57 | "pipeline_config = {\n", 58 | " 'general': {\n", 59 | " 'raw_files': f'oil/*.silc',\n", 60 | " 'pixel_size': 24 # pixel size in um \n", 61 | " },\n", 62 | " 'steps': {\n", 63 | " ### start of steps applied to every image\n", 64 | " # load the image using instrument-specific loading function \n", 65 | " 'load': {\n", 66 | " 'pipeline_class': 'pyopia.instrument.silcam.SilCamLoad'\n", 67 | " },\n", 68 | " # apply background correction - argument is which method to use:\n", 69 | " # 'accurate' - recommended method for moving background\n", 70 | " # 'fast' - faster method for realtime applications\n", 71 | " # 'pass' - omit background correction\n", 72 | " 'correctbackground': {\n", 73 | " 'pipeline_class': 'pyopia.background.CorrectBackgroundAccurate',\n", 74 | " 'average_window': NUM_IMAGES_FOR_BACKGROUND,\n", 75 | " 'bgshift_function': 'accurate'\n", 76 | " }\n", 77 | " }\n", 78 | "}\n", 79 | "\n", 80 | "# now initialise the pipeline\n", 81 | "processing_pipeline = Pipeline(pipeline_config)" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "metadata": {}, 87 | "source": [ 88 | "# Create a background from multiple images" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": null, 94 | "metadata": {}, 95 | "outputs": [], 96 | "source": [ 97 | "# The background stack (of raw images) and background image (mean of bgstack) is built during the first\n", 98 | "# N run steps of the pipeline. During this process, further analysis steps are skipped.\n", 99 | "\n", 100 | "# Get a sorted list of image files\n", 101 | "image_files = sorted(glob(pipeline_config['general']['raw_files']))\n", 102 | "print(f'Found {len(image_files)} image files')\n", 103 | "\n", 104 | "# Process first N images to create the background.\n", 105 | "for filename in image_files[:NUM_IMAGES_FOR_BACKGROUND]:\n", 106 | " processing_pipeline.run(filename)" 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": null, 112 | "metadata": {}, 113 | "outputs": [], 114 | "source": [ 115 | "# Inspect the background image \n", 116 | "fig, ax = plt.subplots()\n", 117 | "ax.imshow(processing_pipeline.data['imbg'])" 118 | ] 119 | }, 120 | { 121 | "cell_type": "markdown", 122 | "metadata": {}, 123 | "source": [ 124 | "# Run background correction on a single image" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": null, 130 | "metadata": {}, 131 | "outputs": [], 132 | "source": [ 133 | "# Process one image using the already prepared background of the first N images\n", 134 | "# NB: Each time you call run(), the background stack and background image will be updated! (Unless 'pass' was set as bgshift_function - see below)\n", 135 | "processing_pipeline.run(image_files[NUM_IMAGES_FOR_BACKGROUND])" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": null, 141 | "metadata": {}, 142 | "outputs": [], 143 | "source": [ 144 | "# Plot raw and corrected image\n", 145 | "fig, axes = plt.subplots(1, 2, figsize=(2*6, 4))\n", 146 | "axes[0].imshow(processing_pipeline.data['imraw'])\n", 147 | "axes[0].set_title(f'Raw image #{NUM_IMAGES_FOR_BACKGROUND}')\n", 148 | "axes[1].imshow(processing_pipeline.data['im_corrected'])\n", 149 | "axes[1].set_title('Background corrected image')" 150 | ] 151 | }, 152 | { 153 | "cell_type": "markdown", 154 | "metadata": {}, 155 | "source": [ 156 | "# Static vs running average background\n", 157 | "The CorrectBackgroundAccurate class have two different modes for a dynamic (running) background correction (bgshift_function either 'fast' and 'accurate'), \n", 158 | "and one mode for a static background that is created once and then not updated (bgshift_function='pass').\n", 159 | "The static background is set up in the same way as the dynamic one, by the N initial calls to the pipeline run. \n", 160 | "You can choose how many and which images to use for the background, illustrated below." 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": null, 166 | "metadata": {}, 167 | "outputs": [], 168 | "source": [ 169 | "# Recreate pipeline and update background step config for static background correction\n", 170 | "processing_pipeline = Pipeline(processing_pipeline.settings)\n", 171 | "processing_pipeline.settings['steps'].update(\n", 172 | " {\n", 173 | " 'correctbackground':\n", 174 | " {\n", 175 | " 'pipeline_class': 'pyopia.background.CorrectBackgroundAccurate',\n", 176 | " 'average_window': NUM_IMAGES_FOR_BACKGROUND,\n", 177 | " 'bgshift_function': 'pass'\n", 178 | " }\n", 179 | " }\n", 180 | ")\n", 181 | "\n", 182 | "# Process first N images to create the static background.\n", 183 | "for filename in image_files[:NUM_IMAGES_FOR_BACKGROUND]:\n", 184 | " processing_pipeline.run(filename)\n", 185 | "\n", 186 | "# With a static background, the processing order does not matter, so we can for instance process the last image in the list.\n", 187 | "# Now processing an image will not cause the background to be updated\n", 188 | "imbg_before = processing_pipeline.data['imbg'].copy()\n", 189 | "processing_pipeline.run(image_files[-1])\n", 190 | "\n", 191 | "# Check difference in imbg before and after analysis step, should be zero\n", 192 | "np.abs(imbg_before - processing_pipeline.data['imbg']).sum()\n" 193 | ] 194 | }, 195 | { 196 | "cell_type": "markdown", 197 | "metadata": {}, 198 | "source": [ 199 | "# Correct images by subtracting vs dividing average background\n", 200 | "The CorrectBackgroundAccurate class have two modes for a subtracting background correction (divide_bg=False), \n", 201 | "or dividing background correction (divide_bg=True) that provides the corrected image ('im_corrected') for further analysis.\n", 202 | "For dividing background mode, the zero-value pixels of the average background image are initially rescaled to 1/255 to prevent division by zero.\n", 203 | "For more information, refer to: https://doi.org/10.1016/j.marpolbul.2016.11.063).\n", 204 | "You can select the subtracting/dividing correction modes used in the pipeline to process raw images, as illustrated below." 205 | ] 206 | }, 207 | { 208 | "cell_type": "code", 209 | "execution_count": null, 210 | "metadata": {}, 211 | "outputs": [], 212 | "source": [ 213 | "# Corrected image uisng subtracting method\n", 214 | "im_corrected_subtract = processing_pipeline.data['im_corrected'].copy()" 215 | ] 216 | }, 217 | { 218 | "cell_type": "code", 219 | "execution_count": null, 220 | "metadata": {}, 221 | "outputs": [], 222 | "source": [ 223 | "# Update pipeline config and background step with dividing background correction (divide_bg=True)\n", 224 | "\n", 225 | "pipeline_config['steps']['correctbackground']['divide_bg'] = True\n", 226 | "\n", 227 | "# Run the first N images to creat the background\n", 228 | "for filename in image_files[:NUM_IMAGES_FOR_BACKGROUND]:\n", 229 | " processing_pipeline.run(filename)\n", 230 | "\n", 231 | "# Now process one of the raw images\n", 232 | "processing_pipeline.run(image_files[NUM_IMAGES_FOR_BACKGROUND])" 233 | ] 234 | }, 235 | { 236 | "cell_type": "code", 237 | "execution_count": null, 238 | "metadata": {}, 239 | "outputs": [], 240 | "source": [ 241 | "# Corrected image uisng dividing mode\n", 242 | "im_corrected_division = processing_pipeline.data['im_corrected'].copy()" 243 | ] 244 | }, 245 | { 246 | "cell_type": "code", 247 | "execution_count": null, 248 | "metadata": {}, 249 | "outputs": [], 250 | "source": [ 251 | "# Plot raw, im_corrected by subtracing and dividing averaged background image\n", 252 | "fig, axes = plt.subplots(1, 3, figsize=(3*6, 5))\n", 253 | "axes[0].imshow(processing_pipeline.data['imraw'])\n", 254 | "axes[0].set_title(f'Raw image #{NUM_IMAGES_FOR_BACKGROUND}')\n", 255 | "axes[1].imshow(im_corrected_subtract)\n", 256 | "axes[1].set_title('Corrected image by background subtraction')\n", 257 | "axes[2].imshow(im_corrected_division)\n", 258 | "axes[2].set_title('Corrected image by background division')" 259 | ] 260 | }, 261 | { 262 | "cell_type": "markdown", 263 | "metadata": {}, 264 | "source": [ 265 | "# Custom background correction\n", 266 | "You can write your own custom background correction class, here is a simple example of how to do that.\n", 267 | "\n" 268 | ] 269 | }, 270 | { 271 | "cell_type": "code", 272 | "execution_count": null, 273 | "metadata": {}, 274 | "outputs": [], 275 | "source": [ 276 | "class MyCustomBackgroundClass():\n", 277 | " '''\n", 278 | " Example custom background class: use a randomly generated image to \"correct\" the background\n", 279 | " '''\n", 280 | "\n", 281 | " def __call__(self, data):\n", 282 | " # Create a random background image\n", 283 | " data['imbg'] = np.random.random(data['imraw'].shape)\n", 284 | " data['bgstack'] = [data['imbg']]\n", 285 | "\n", 286 | " # Correct\n", 287 | " data['im_corrected'] = np.maximum(data['imraw'] - data['imbg'], 0)\n", 288 | "\n", 289 | " # Stretch contrast\n", 290 | " data['im_corrected'] = rescale_intensity(data['im_corrected'], out_range=(0, 1))\n", 291 | "\n", 292 | " return data\n", 293 | "\n", 294 | "\n", 295 | "# Monkey patch the custom class into PyOpia\n", 296 | "pyopia.background.MyCustomBackgroundClass = MyCustomBackgroundClass" 297 | ] 298 | }, 299 | { 300 | "cell_type": "code", 301 | "execution_count": null, 302 | "metadata": {}, 303 | "outputs": [], 304 | "source": [ 305 | "# Recreate pipeline and update background step config with cutsom background correction\n", 306 | "processing_pipeline = Pipeline(processing_pipeline.settings)\n", 307 | "processing_pipeline.settings['steps'].update(\n", 308 | " {\n", 309 | " 'correctbackground':\n", 310 | " {\n", 311 | " 'pipeline_class': 'pyopia.background.MyCustomBackgroundClass',\n", 312 | " }\n", 313 | " }\n", 314 | ")\n", 315 | "\n", 316 | "processing_pipeline.run(image_files[0])" 317 | ] 318 | }, 319 | { 320 | "cell_type": "code", 321 | "execution_count": null, 322 | "metadata": {}, 323 | "outputs": [], 324 | "source": [ 325 | "# Plot raw and corrected image\n", 326 | "fig, axes = plt.subplots(1, 2, figsize=(2*6, 4))\n", 327 | "axes[0].imshow(processing_pipeline.data['imraw'])\n", 328 | "axes[0].set_title('Raw image')\n", 329 | "axes[1].imshow(processing_pipeline.data['im_corrected'])\n", 330 | "axes[1].set_title('Background corrected image')" 331 | ] 332 | } 333 | ], 334 | "metadata": { 335 | "kernelspec": { 336 | "display_name": "pyopia", 337 | "language": "python", 338 | "name": "python3" 339 | }, 340 | "language_info": { 341 | "codemirror_mode": { 342 | "name": "ipython", 343 | "version": 3 344 | }, 345 | "file_extension": ".py", 346 | "mimetype": "text/x-python", 347 | "name": "python", 348 | "nbconvert_exporter": "python", 349 | "pygments_lexer": "ipython3", 350 | "version": "3.12.5" 351 | } 352 | }, 353 | "nbformat": 4, 354 | "nbformat_minor": 2 355 | } 356 | -------------------------------------------------------------------------------- /docs/notebooks/big_datasets.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "65dc6c46", 6 | "metadata": {}, 7 | "source": [ 8 | "(big-data)=\n", 9 | "# Big datasets\n", 10 | "\n", 11 | "If you have data containing a lot of particles, then there are some config settings that will significantly speed up processing. Here are some pointers.\n", 12 | "\n", 13 | "When processing, use the non-appending functionality in {class}`pyopia.io.StatsToDisc`\n", 14 | "\n", 15 | "```\n", 16 | " [steps.output]\n", 17 | " pipeline_class = 'pyopia.io.StatsToDisc'\n", 18 | " output_datafile = 'proc/test' # prefix path for output nc file\n", 19 | " append = false\n", 20 | "```\n", 21 | "\n", 22 | "Using the above output step in you pipeline will create a directory 'proc' filled with nc files conforming to the pattern: 'test-Image-D*-STATS.nc'\n", 23 | "\n", 24 | "These can be combined using {func}`pyopia.io.merge_and_save_mfdataset` of command line tool `pyopia merge-mfdata`, which will produce a new single -STATS.nc file of the whole dataset (for faster loading). Or you can do this manually like this:\n", 25 | "\n", 26 | "```python\n", 27 | "xstats, image_stats = pyopia.io.combine_stats_netcdf_files('proc/')\n", 28 | "```\n", 29 | "\n", 30 | "And the make a new nc file of the whole dataset for faster loading later:\n", 31 | "\n", 32 | "```python\n", 33 | "settings = pyopia.pipeline.steps_from_xstats(xstats)\n", 34 | "\n", 35 | "pyopia.io.write_stats(xstats.to_dataframe(),\n", 36 | " 'proc/test2-test',\n", 37 | " settings,\n", 38 | " image_stats=image_stats.to_dataframe())\n", 39 | "\n", 40 | "xstats = pyopia.io.load_stats('proc/test2-test-STATS.nc')\n", 41 | "```" 42 | ] 43 | }, 44 | { 45 | "cell_type": "markdown", 46 | "id": "25bae4d6", 47 | "metadata": {}, 48 | "source": [ 49 | "# Parallell processing\n", 50 | "\n", 51 | "If you have data containing a lot of particles and/or a lot of raw images, you can use the num-chunks functionality in the {ref}(pyopia-process) command line tool e.g.:\n", 52 | "\n", 53 | "```bash\n", 54 | "pyopia process config.toml --num-chunks 4\n", 55 | "```\n", 56 | "\n", 57 | "This will split the list of raw files into 4 chunks to be processed in parallell using multiprocessing. This tool will organise the chunks of file names so that the appropriate background files into the correct places (i.e. for moving background, the last `average_window` number of files in the previous chunk are added to the start of the next chunk; and for fixed background the same initial `average_window` number of files are added to the top of each chunk)." 58 | ] 59 | } 60 | ], 61 | "metadata": { 62 | "kernelspec": { 63 | "display_name": "Python 3.8.13 64-bit ('pyopia')", 64 | "language": "python", 65 | "name": "python3" 66 | }, 67 | "language_info": { 68 | "codemirror_mode": { 69 | "name": "ipython", 70 | "version": 3 71 | }, 72 | "file_extension": ".py", 73 | "mimetype": "text/x-python", 74 | "name": "python", 75 | "nbconvert_exporter": "python", 76 | "pygments_lexer": "ipython3", 77 | "version": "3.12.5" 78 | }, 79 | "vscode": { 80 | "interpreter": { 81 | "hash": "35ab0c005d63a5587fede8db5b2b16b081d9aece903d58f7211748c218ea86a0" 82 | } 83 | } 84 | }, 85 | "nbformat": 4, 86 | "nbformat_minor": 5 87 | } 88 | -------------------------------------------------------------------------------- /docs/notebooks/config.toml: -------------------------------------------------------------------------------- 1 | [general] # general setting independent of processing steps 2 | raw_files = 'raw_data/*.silc' # string used to obtain list of raw data files for processing 3 | pixel_size = 24 # pixel size of imaging system in microns 4 | 5 | [steps] # setup of analysis pipeline order, functions, and parameters 6 | 7 | [steps.classifier] 8 | pipeline_class = 'pyopia.classify.Classify' 9 | model_path = 'keras_model.h5' # path to trained nn model 10 | 11 | [steps.load] 12 | pipeline_class = 'pyopia.instrument.silcam.SilCamLoad' 13 | 14 | [steps.imageprep] 15 | pipeline_class = 'pyopia.instrument.silcam.ImagePrep' 16 | image_level = 'imraw' # the level of processing for further analysis. Either 'im_corrected' for ignoring background or 'imc' for using the backgroun-corrected image. Defaults to 'imc' if not defined. 17 | 18 | [steps.segmentation] 19 | pipeline_class = 'pyopia.process.Segment' 20 | threshold = 0.85 # threshold used for segmentation 21 | segment_source = 'im_minimum' 22 | 23 | [steps.statextract] 24 | pipeline_class = 'pyopia.process.CalculateStats' 25 | export_outputpath = "silcam_rois" 26 | roi_source = 'imref' 27 | 28 | [steps.output] 29 | pipeline_class = 'pyopia.io.StatsToDisc' 30 | output_datafile = './test' # prefix path for output nc file 31 | append = true 32 | -------------------------------------------------------------------------------- /docs/notebooks/processing_raw_data.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "(processing-raw-data)=\n", 8 | "# Processing raw data" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "metadata": {}, 14 | "source": [ 15 | "## 0) Activate the pyopia environment\n", 16 | "\n", 17 | "If you installed PyOPIA within as per the guide [here](https://pyopia.readthedocs.io/en/latest/intro.html#installing), then you should activate this environment first, e.g.:\n", 18 | "\n", 19 | "```\n", 20 | "uv sync\n", 21 | "```\n", 22 | "and\n", 23 | "```\n", 24 | "source .venv/bin/activate\n", 25 | "```" 26 | ] 27 | }, 28 | { 29 | "cell_type": "markdown", 30 | "metadata": {}, 31 | "source": [ 32 | "## 1) Get yourself a config file.\n", 33 | "You can do this either by copy-paste from the page on {ref}`toml-config` into a new toml file (you might call it 'silcam-config.toml', for example), or from generating a very basic config from the command line tool: `pyopia generate-config`, e.g. for silcam:\n", 34 | "\n", 35 | "```\n", 36 | "pyopia generate-config silcam 'rawdatapath/*.silc' 'modelpath/keras_model.keras' 'proc_folder_path' 'testdata'\n", 37 | "```\n", 38 | "\n", 39 | "If you want help on what these options are, do: `pyopia generate-config --help`\n", 40 | "\n", 41 | "You should now have a toml file (e.g. called 'silcam-config.toml')" 42 | ] 43 | }, 44 | { 45 | "cell_type": "markdown", 46 | "metadata": {}, 47 | "source": [ 48 | "## 2) Make sure you are happy with your config file\n", 49 | "\n", 50 | "Refer to the comments in the examples given here {ref}`toml-config`\n", 51 | "\n", 52 | "If you need detailed help on arguments specific to a pipeline class, then you may wish to refer to the API documentation for that specific class.\n", 53 | "\n", 54 | "If you want to do classification, you need to give the `model_path` argument within `[steps.classifier]` a path to a trained keras model. You can download a silcam example [here](https://pysilcam.blob.core.windows.net/test-data/silcam-classification_database_20240822-200-20240829T091048.zip)" 55 | ] 56 | }, 57 | { 58 | "cell_type": "markdown", 59 | "metadata": {}, 60 | "source": [ 61 | "## 3) Process!\n", 62 | "\n", 63 | "Run the command line processing which simply needs to know which config file you want it to work on, e.g.:\n", 64 | "\n", 65 | "```\n", 66 | "pyopia process silcam-config.toml\n", 67 | "```" 68 | ] 69 | }, 70 | { 71 | "cell_type": "markdown", 72 | "metadata": {}, 73 | "source": [ 74 | "## 4) Output\n", 75 | "\n", 76 | "* You should expect an output folder defined by the `output_datafile` argument within the `[steps.output]` step. \n", 77 | " * This will either contain a new .nc file or several .nc files, depending on if you used the `append = false` option (intended for {ref}`big-data`) or not.\n", 78 | "* If you defined the `export_outputpath` argument in `[steps.statextract]`, then you will also have a folder containing a series of .h5 files, that contains all the particle ROIs" 79 | ] 80 | } 81 | ], 82 | "metadata": { 83 | "kernelspec": { 84 | "display_name": "pyopia", 85 | "language": "python", 86 | "name": "python3" 87 | }, 88 | "language_info": { 89 | "codemirror_mode": { 90 | "name": "ipython", 91 | "version": 3 92 | }, 93 | "file_extension": ".py", 94 | "mimetype": "text/x-python", 95 | "name": "python", 96 | "nbconvert_exporter": "python", 97 | "pygments_lexer": "ipython3", 98 | "version": "3.12.7" 99 | } 100 | }, 101 | "nbformat": 4, 102 | "nbformat_minor": 2 103 | } 104 | -------------------------------------------------------------------------------- /docs/notebooks/toml_config.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "(toml-config)=\n", 8 | "# Pipeline config files" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "metadata": {}, 14 | "source": [ 15 | "PyOPIA makes use of TOML for configuration files, which can used loaded and given to {class}`pyopia.pipeline.Pipeline`, or passed directly from command line, like this: `pyopia process config.toml` ('Commmand line tools' page)." 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "Config files can be loaded for use in scripts or notebooks using:\n", 23 | "\n", 24 | "```\n", 25 | "toml_settings = pyopia.io.load_toml('config.toml')\n", 26 | "```" 27 | ] 28 | }, 29 | { 30 | "cell_type": "markdown", 31 | "metadata": {}, 32 | "source": [ 33 | "### Main components of the config\n", 34 | "\n", 35 | "The `[general]` section contains information that applies generally to the dataset, or to several steps within a pipeline.\n", 36 | "\n", 37 | "The `[steps]` section contains sub-steps describing the {mod}`pyopia.pipeline` class and input arguments that perform each step of the processing." 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "## Examples" 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "metadata": {}, 50 | "source": [ 51 | "### SilCam\n", 52 | "\n", 53 | "Below is a typical example for a SilCam processing pipeline. This setup could be used for other standard images by adapting the load function.\n", 54 | "\n", 55 | "```toml\n", 56 | "[general] # general settings independent of processing steps\n", 57 | "raw_files = 'raw_data/*.silc' # string used to obtain list of raw data files for processing\n", 58 | "pixel_size = 24 # pixel size of imaging system in microns\n", 59 | "log_level = 'INFO' # (defaults to INFO) sets the level of printed output or details in the log (see python logging library for details)\n", 60 | "log_file = 'pyopia.log' # (optional) path to logfile - logfile not written if not defined here\n", 61 | "\n", 62 | "[steps] # setup of analysis pipeline order, functions, and parameters\n", 63 | "\n", 64 | " [steps.classifier]\n", 65 | " pipeline_class = 'pyopia.classify.Classify'\n", 66 | " model_path = 'keras_model.keras' # path to trained nn model\n", 67 | "\n", 68 | " [steps.load]\n", 69 | " pipeline_class = 'pyopia.instrument.silcam.SilCamLoad'\n", 70 | "\n", 71 | " [steps.correctbackground]\n", 72 | " pipeline_class = 'pyopia.background.CorrectBackgroundAccurate'\n", 73 | " average_window = 10 # number of images used to create background\n", 74 | " bgshift_function = 'accurate' # optional 'fast' or 'accurate' method for moving backgrounds. For static background use 'pass' or comment this line.\n", 75 | "\n", 76 | " [steps.imageprep]\n", 77 | " pipeline_class = 'pyopia.instrument.silcam.ImagePrep'\n", 78 | " image_level = 'imraw' # the level of processing for further analysis. Either 'imraw' for ignoring background or 'im_corrected' for using the backgroun-corrected image. Defaults to 'imc' if not defined.\n", 79 | "\n", 80 | " [steps.segmentation]\n", 81 | " pipeline_class = 'pyopia.process.Segment'\n", 82 | " threshold = 0.85 # threshold used for segmentation\n", 83 | " segment_source = \"im_minimum\" # the image used for segmentation\n", 84 | "\n", 85 | " [steps.statextract]\n", 86 | " pipeline_class = 'pyopia.process.CalculateStats'\n", 87 | " propnames = ['major_axis_length', 'minor_axis_length', 'equivalent_diameter', 'solidity'] # optional parameters to request from skimage.regionprops when calculating particle geometries. Defaults to ['major_axis_length', 'minor_axis_length', 'equivalent_diameter']. 'solidity' is needed for oil and gas analysis to help remove occluded particles.\n", 88 | " export_outputpath = 'exported_rois' # Path to folder to put extracted particle ROIs (in h5 files). Required for making montages later. Leave this option out if you don't want to export ROIs\n", 89 | " roi_source = \"imref\" # the image used to extract ROIs, and give to the classifier\n", 90 | "\n", 91 | " [steps.output]\n", 92 | " pipeline_class = 'pyopia.io.StatsToDisc'\n", 93 | " output_datafile = 'proc/test' # prefix path for output nc file\n", 94 | "\n", 95 | "```" 96 | ] 97 | }, 98 | { 99 | "cell_type": "markdown", 100 | "metadata": {}, 101 | "source": [ 102 | "### Holo\n", 103 | "\n", 104 | "Here is a typical configuration of a holographic reconstruction pipeline." 105 | ] 106 | }, 107 | { 108 | "cell_type": "markdown", 109 | "metadata": {}, 110 | "source": [ 111 | "```toml\n", 112 | "[general] # general setting independent of processing steps\n", 113 | "raw_files = 'holo_test_data_01/*.pgm' # string used to obtain list of raw data files for processing\n", 114 | "pixel_size = 4.4 # pixel size of imaging system in microns'wavelength = 658, # laser wavelength in nm\n", 115 | "\n", 116 | "[steps] # setup of analysis pipeline order, functions, and parameters\n", 117 | " [steps.initial]\n", 118 | " pipeline_class = 'pyopia.instrument.holo.Initial'\n", 119 | " # hologram reconstruction settings\n", 120 | " wavelength = 658 # laser wavelength in nm\n", 121 | " n = 1.33 # index of refraction of sample volume medium (1.33 for water)\n", 122 | " offset = 27 # offset to start of sample volume in mm\n", 123 | " minZ = 0 # minimum reconstruction distance within sample volume in mm\n", 124 | " maxZ = 50 # maximum reconstruction distance within sample volume in mm\n", 125 | " stepZ = 0.5 #step size in mm\n", 126 | "\n", 127 | " [steps.load]\n", 128 | " pipeline_class = 'pyopia.instrument.holo.Load'\n", 129 | "\n", 130 | " [steps.correctbackground]\n", 131 | " pipeline_class = 'pyopia.background.CorrectBackgroundAccurate'\n", 132 | " average_window = 1 # number of images used to create background\n", 133 | " bgshift_function = 'accurate' # optional 'fast' or 'accurate' method for moving backgrounds. For static background use 'pass' or comment this line.\n", 134 | "\n", 135 | " [steps.reconstruct]\n", 136 | " pipeline_class = 'pyopia.instrument.holo.Reconstruct'\n", 137 | " stack_clean = 0.02\n", 138 | "\n", 139 | " [steps.focus]\n", 140 | " pipeline_class = 'pyopia.instrument.holo.Focus'\n", 141 | " stacksummary_function = 'max_map'\n", 142 | " threshold = 1\n", 143 | " increase_depth_of_field = true\n", 144 | " focus_function = 'find_focus_sobel'\n", 145 | " merge_adjacent_particles = 2\n", 146 | "\n", 147 | " [steps.segmentation]\n", 148 | " pipeline_class = 'pyopia.process.Segment'\n", 149 | " threshold = 0.99 # threshold used for segmentation\n", 150 | " segment_source = 'im_focussed' # the image used for segmentation\n", 151 | "\n", 152 | " [steps.statextract]\n", 153 | " pipeline_class = 'pyopia.process.CalculateStats'\n", 154 | " propnames = ['major_axis_length', 'minor_axis_length', 'equivalent_diameter', \n", 155 | " 'feret_diameter_max', 'equivalent_diameter_area']\n", 156 | " roi_source = 'im_focussed'\n", 157 | "\n", 158 | " [steps.output]\n", 159 | " pipeline_class = 'pyopia.io.StatsToDisc'\n", 160 | " output_datafile = 'proc-holo-singleimage/test' # prefix path for output nc file\n", 161 | "```" 162 | ] 163 | }, 164 | { 165 | "cell_type": "markdown", 166 | "metadata": {}, 167 | "source": [ 168 | "### UVP\n", 169 | "\n", 170 | "Here is a typical configuration for basic analysis of raw UVP data.\n", 171 | "\n", 172 | "```toml\n", 173 | "[general]\n", 174 | "raw_files = \"uvp_data/*.png\"\n", 175 | "pixel_size = 80\n", 176 | "\n", 177 | "[steps]\n", 178 | "\n", 179 | " [steps.classifier]\n", 180 | " pipeline_class = \"pyopia.classify.Classify\"\n", 181 | " model_path = \"model/silcam-classification_database_20240822-200-20240829T091048-best-epoch.keras\"\n", 182 | "\n", 183 | " [steps.load]\n", 184 | " pipeline_class = \"pyopia.instrument.uvp.UVPLoad\"\n", 185 | "\n", 186 | " [steps.segmentation]\n", 187 | " pipeline_class = \"pyopia.process.Segment\"\n", 188 | " threshold = 0.95\n", 189 | " segment_source = \"imraw\"\n", 190 | "\n", 191 | " [steps.statextract]\n", 192 | " pipeline_class = \"pyopia.process.CalculateStats\"\n", 193 | " roi_source = \"imraw\"\n", 194 | " export_outputpath = \"uvp_rois\"\n", 195 | "\n", 196 | " [steps.output]\n", 197 | " pipeline_class = \"pyopia.io.StatsToDisc\"\n", 198 | " output_datafile = \"proc/uvp-test\"\n", 199 | "\n", 200 | "```" 201 | ] 202 | } 203 | ], 204 | "metadata": { 205 | "language_info": { 206 | "name": "python" 207 | }, 208 | "orig_nbformat": 4 209 | }, 210 | "nbformat": 4, 211 | "nbformat_minor": 2 212 | } 213 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | flake8 2 | docopt 3 | setuptools 4 | pytest-error-for-skips 5 | sphinx_rtd_theme>=0.5.0 6 | sphinxcontrib-napoleon>=0.7 7 | sphinx-togglebutton 8 | sphinx-copybutton 9 | readthedocs-sphinx-search 10 | myst-nb 11 | jupyter_book 12 | ipykernel>=6.19.4 13 | tensorflow==2.16.2 14 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: pyopia 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - python=3.12 6 | - poetry 7 | -------------------------------------------------------------------------------- /notebooks/config-holo.toml: -------------------------------------------------------------------------------- 1 | [general] # general setting independent of processing steps 2 | raw_files = 'holo_test_data_01/*.pgm' # string used to obtain list of raw data files for processing 3 | pixel_size = 4.4 # pixel size of imaging system in microns'wavelength = 658, # laser wavelength in nm 4 | 5 | [steps] # setup of analysis pipeline order, functions, and parameters 6 | [steps.initial] 7 | pipeline_class = 'pyopia.instrument.holo.Initial' 8 | # hologram reconstruction settings 9 | wavelength = 658 # laser wavelength in nm 10 | n = 1.33 # index of refraction of sample volume medium (1.33 for water) 11 | offset = 27 # offset to start of sample volume in mm 12 | minZ = 0 # minimum reconstruction distance within sample volume in mm 13 | maxZ = 50 # maximum reconstruction distance within sample volume in mm 14 | stepZ = 0.5 #step size in mm 15 | 16 | [steps.load] 17 | pipeline_class = 'pyopia.instrument.holo.Load' 18 | 19 | [steps.correctbackground] 20 | pipeline_class = 'pyopia.background.CorrectBackgroundAccurate' 21 | average_window = 1 # number of images used to create background 22 | bgshift_function = 'accurate' # optional 'fast' or 'accurate' method for moving backgrounds. For static background use 'pass' or comment this line. 23 | 24 | [steps.reconstruct] 25 | pipeline_class = 'pyopia.instrument.holo.Reconstruct' 26 | stack_clean = 0.02 27 | 28 | [steps.focus] 29 | pipeline_class = 'pyopia.instrument.holo.Focus' 30 | stacksummary_function = 'max_map' 31 | threshold = 1 32 | increase_depth_of_field = true 33 | focus_function = 'find_focus_sobel' 34 | merge_adjacent_particles = 2 35 | 36 | [steps.segmentation] 37 | pipeline_class = 'pyopia.process.Segment' 38 | threshold = 0.99 # threshold used for segmentation 39 | segment_source = 'im_focussed' 40 | 41 | [steps.statextract] 42 | pipeline_class = 'pyopia.process.CalculateStats' 43 | propnames = ['major_axis_length', 'minor_axis_length', 'equivalent_diameter', 44 | 'feret_diameter_max', 'equivalent_diameter_area'] 45 | roi_source = 'im_focussed' 46 | 47 | [steps.output] 48 | pipeline_class = 'pyopia.io.StatsToDisc' 49 | output_datafile = 'proc-holo-singleimage/test' # prefix path for output nc file -------------------------------------------------------------------------------- /notebooks/config.toml: -------------------------------------------------------------------------------- 1 | [general] # general setting independent of processing steps 2 | raw_files = 'raw_data/*.silc' # string used to obtain list of raw data files for processing 3 | pixel_size = 24 # pixel size of imaging system in microns 4 | 5 | [steps] # setup of analysis pipeline order, functions, and parameters 6 | 7 | [steps.classifier] 8 | pipeline_class = 'pyopia.classify.Classify' 9 | model_path = 'pyopia-default-classifier-20250409.keras' # path to trained nn model 10 | 11 | [steps.load] 12 | pipeline_class = 'pyopia.instrument.silcam.SilCamLoad' 13 | 14 | [steps.imageprep] 15 | pipeline_class = 'pyopia.instrument.silcam.ImagePrep' 16 | image_level = 'imraw' # the level of processing for further analysis. Either 'imraw' for ignoring background or 'im_corrected' for using the backgroun-corrected image. Defaults to 'imc' if not defined. 17 | 18 | [steps.segmentation] 19 | pipeline_class = 'pyopia.process.Segment' 20 | threshold = 0.85 # threshold used for segmentation 21 | segment_source = "im_minimum" 22 | 23 | [steps.statextract] 24 | pipeline_class = 'pyopia.process.CalculateStats' 25 | export_outputpath = "silcam_rois" 26 | roi_source = "imref" 27 | 28 | [steps.output] 29 | pipeline_class = 'pyopia.io.StatsToDisc' 30 | output_datafile = 'proc-single-image-stats/test' # prefix path for output nc file -------------------------------------------------------------------------------- /pyopia/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "2.10.0" 2 | -------------------------------------------------------------------------------- /pyopia/background.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Background correction module (inherited from PySilCam) 3 | ''' 4 | import numpy as np 5 | 6 | 7 | def ini_background(bgfiles, load_function): 8 | ''' 9 | Create and initial background stack and average image 10 | 11 | Parameters 12 | ----------- 13 | bgfiles : list 14 | List of strings of filenames to be used in background creation 15 | load_function : object 16 | This function should take a filename and return an image, for example: :func:`pyopia.instrument.silcam.load_image` 17 | 18 | Returns 19 | ------- 20 | bgstack : list 21 | list of all images in the background stack 22 | imbg : array 23 | background image 24 | ''' 25 | bgstack = [] 26 | for f in bgfiles: 27 | im = load_function(f) 28 | bgstack.append(im) 29 | 30 | imbg = np.mean(bgstack, axis=0) # average the images in the stack 31 | 32 | return bgstack, imbg 33 | 34 | 35 | def shift_bgstack_accurate(bgstack, imbg, imnew): 36 | ''' 37 | Shifts the background by popping the oldest and added a new image 38 | 39 | The new background is calculated slowly by computing the mean of all images 40 | in the background stack. 41 | 42 | Parameters 43 | ----------- 44 | bgstack : list 45 | list of all images in the background stack 46 | imbg : array 47 | background image 48 | imnew : array 49 | new image to be added to stack 50 | 51 | Returns 52 | ------- 53 | bgstack : list 54 | updated list of all background images 55 | imbg : array 56 | updated actual background image 57 | ''' 58 | bgstack.pop(0) # pop the oldest image from the stack, 59 | bgstack.append(imnew) # append the new image to the stack 60 | imbg = np.mean(bgstack, axis=0) 61 | return bgstack, imbg 62 | 63 | 64 | def shift_bgstack_fast(bgstack, imbg, imnew): 65 | ''' 66 | Shifts the background by popping the oldest and added a new image 67 | 68 | The new background is appoximated quickly by subtracting the old image and 69 | adding the new image (both scaled by the stacklength). 70 | This is close to a running mean, but not quite. 71 | 72 | Parameters 73 | ----------- 74 | bgstack : list 75 | list of all images in the background stack 76 | imbg : uint8 77 | background image 78 | imnew : unit8 79 | new image to be added to stack 80 | 81 | Returns 82 | ------- 83 | bgstack : list 84 | updated list of all background images 85 | imbg : array 86 | updated actual background image 87 | ''' 88 | stacklength = len(bgstack) 89 | imold = bgstack.pop(0) # pop the oldest image from the stack, 90 | # subtract the old image from the average (scaled by the average window) 91 | imbg -= (imold / stacklength) 92 | # add the new image to the average (scaled by the average window) 93 | imbg += (imnew / stacklength) 94 | bgstack.append(imnew) # append the new image to the stack 95 | return bgstack, imbg 96 | 97 | 98 | def correct_im_accurate(imbg, imraw, divide_bg=False): 99 | ''' 100 | Corrects raw image by subtracting or dividing the background and scaling the output 101 | 102 | For dividing method see: https://doi.org/10.1016/j.marpolbul.2016.11.063) 103 | 104 | There is a small chance of clipping of imc in both crushed blacks and blown 105 | highlights if the background or raw images are very poorly obtained 106 | 107 | Parameters 108 | ----------- 109 | imbg : float64 110 | background averaged image 111 | imraw : float64 112 | raw image 113 | divide_bg : (bool, optional) 114 | If True, the correction will be performed by dividing the raw image by the background 115 | Default to False 116 | 117 | Returns 118 | ------- 119 | im_corrected : float64 120 | corrected image, same type as input 121 | ''' 122 | 123 | if divide_bg: 124 | imbg = np.clip(imbg, a_min=1/255, a_max=None) # Clipping the zero_value pixels 125 | im_corrected = imraw / imbg 126 | im_corrected += (1 / 2 - np.percentile(im_corrected, 50)) 127 | im_corrected -= im_corrected.min() # Shift the negative values to zero 128 | im_corrected = np.clip(im_corrected, a_min=0, a_max=1) 129 | else: 130 | im_corrected = imraw - imbg 131 | im_corrected += (1 / 2 - np.percentile(im_corrected, 50)) 132 | im_corrected += 1 - im_corrected.max() # Shift the positive values exceeding unity to one 133 | 134 | return im_corrected 135 | 136 | 137 | def correct_im_fast(imbg, imraw): 138 | ''' 139 | Corrects raw image by subtracting the background and clipping the ouput 140 | without scaling 141 | 142 | There is high potential for clipping of imc in both crushed blacks an blown 143 | highlights, especially if the background or raw images are not properly obtained 144 | 145 | Parameters 146 | ----------- 147 | imraw : array 148 | raw image 149 | imbg : array 150 | background averaged image 151 | 152 | Returns 153 | ------- 154 | im_corrected : array 155 | corrected image 156 | ''' 157 | im_corrected = imraw - imbg 158 | 159 | im_corrected += 215/255 160 | im_corrected = np.clip(im_corrected, 0, 1) 161 | 162 | return im_corrected 163 | 164 | 165 | def shift_and_correct(bgstack, imbg, imraw, stacklength, real_time_stats=False): 166 | ''' 167 | Shifts the background stack and averaged image and corrects the new 168 | raw image. 169 | 170 | This is a wrapper for shift_bgstack and correct_im 171 | 172 | Parameters 173 | ----------- 174 | bgstack : list 175 | list of all images in the background stack 176 | imbg : float64 177 | background image 178 | imraw : float64 179 | raw image 180 | stacklength : int 181 | unused int here - just there to maintain the same behaviour as shift_bgstack_fast() 182 | real_time_stats : Bool, optional 183 | True use fast functions, if False use accurate functions., by default False 184 | 185 | Returns 186 | ------- 187 | bgstack : list 188 | list of all images in the background stack 189 | imbg : float64 190 | background averaged image 191 | im_corrected : float64 192 | corrected image 193 | ''' 194 | 195 | if real_time_stats: 196 | im_corrected = correct_im_fast(imbg, imraw) 197 | bgstack, imbg = shift_bgstack_fast(bgstack, imbg, imraw, stacklength) 198 | else: 199 | im_corrected = correct_im_accurate(imbg, imraw) 200 | bgstack, imbg = shift_bgstack_accurate(bgstack, imbg, imraw, stacklength) 201 | 202 | return bgstack, imbg, im_corrected 203 | 204 | 205 | class CorrectBackgroundAccurate(): 206 | ''' 207 | :class:`pyopia.pipeline` compatible class that calls: :func:`pyopia.background.correct_im_accurate` 208 | and will shift the background using a moving average function if given. 209 | 210 | The background stack and background image are created during the first 'average_window' (int) calls 211 | to this class, and the skip_next_steps flag is set in the pipeline Data. No background correction 212 | is performed during these steps. 213 | 214 | Required keys in :class:`pyopia.pipeline.Data`: 215 | - :attr:`pyopia.pipeline.Data.imraw` 216 | 217 | Parameters 218 | ---------- 219 | bgshift_function : (string, optional) 220 | Function used to shift the background. Defaults to passing (i.e. static background) 221 | Available options are 'accurate', 'fast', or 'pass' to apply a static background correction: 222 | 223 | :func:`pyopia.background.shift_bgstack_accurate` 224 | 225 | :func:`pyopia.background.shift_bgstack_fast` 226 | 227 | average_window : int 228 | number of images to use in the background image stack 229 | 230 | image_source: (str, optional) 231 | The key in Pipeline.data of the image to be background corrected. 232 | Defaults to 'imraw' 233 | 234 | divide_bg : (bool) 235 | If True, it performs background correction by dividing the raw image by the background. 236 | Default to False. 237 | 238 | Returns 239 | ------- 240 | data : :class:`pyopia.pipeline.Data` 241 | containing the following new keys: 242 | 243 | :attr:`pyopia.pipeline.Data.im_corrected` 244 | :attr:`pyopia.pipeline.Data.im_corrected` 245 | 246 | :attr:`pyopia.pipeline.Data.bgstack` 247 | 248 | :attr:`pyopia.pipeline.Data.imbg` 249 | 250 | Examples 251 | -------- 252 | Apply moving average using :func:`pyopia.background.shift_bgstack_accurate` : 253 | 254 | .. code-block:: toml 255 | 256 | [steps.correctbackground] 257 | pipeline_class = 'pyopia.background.CorrectBackgroundAccurate' 258 | bgshift_function = 'accurate' 259 | average_window = 5 260 | 261 | Apply static background correction: 262 | 263 | .. code-block:: toml 264 | 265 | [steps.correctbackground] 266 | pipeline_class = 'pyopia.background.CorrectBackgroundAccurate' 267 | bgshift_function = 'pass' 268 | average_window = 5 269 | 270 | If you do not want to do background correction, leave this step out of the pipeline. 271 | Then you could use :class:`pyopia.pipeline.CorrectBackgroundNone` if you need to instead. 272 | ''' 273 | 274 | def __init__(self, bgshift_function='pass', average_window=1, image_source='imraw', divide_bg=False): 275 | self.bgshift_function = bgshift_function 276 | self.average_window = average_window 277 | self.image_source = image_source 278 | self.divide_bg = divide_bg 279 | 280 | def _build_background_step(self, data): 281 | '''Add one layer to the background stack from the raw image in data pipeline, and update the background image.''' 282 | if 'bgstack' not in data: 283 | data['bgstack'] = [] 284 | 285 | init_complete = True 286 | if len(data['bgstack']) < self.average_window: 287 | data['bgstack'].append(data[self.image_source]) 288 | data['imbg'] = np.mean(data['bgstack'], axis=0) 289 | init_complete = False 290 | 291 | return init_complete 292 | 293 | def __call__(self, data): 294 | # Initialize the background while required bgstack size not reached 295 | init_complete = self._build_background_step(data) 296 | 297 | # If we are still building the bgstack, return without doing image correction and bgstack update 298 | if not init_complete: 299 | # Flag to the pipeline that remaining steps should be skipped since we are still building the background 300 | data['skip_next_steps'] = True 301 | return data 302 | 303 | data['im_corrected'] = correct_im_accurate(data['imbg'], data[self.image_source], divide_bg=self.divide_bg) 304 | 305 | match self.bgshift_function: 306 | case 'pass': 307 | return data 308 | case 'accurate': 309 | data['bgstack'], data['imbg'] = shift_bgstack_accurate(data['bgstack'], 310 | data['imbg'], 311 | data[self.image_source]) 312 | case 'fast': 313 | data['bgstack'], data['imbg'] = shift_bgstack_fast(data['bgstack'], 314 | data['imbg'], 315 | data[self.image_source]) 316 | return data 317 | 318 | 319 | class CorrectBackgroundNone(): 320 | ''' 321 | :class:`pyopia.pipeline` compatible class for use when no background correction is required. 322 | This simply makes `data['im_corrected'] = data['imraw'] in the pipeline. 323 | This simply makes `data['im_corrected'] = data['imraw'] in the pipeline. 324 | 325 | Required keys in :class:`pyopia.pipeline.Data`: 326 | - :attr:`pyopia.pipeline.Data.imraw` 327 | 328 | Parameters 329 | ----------- 330 | None 331 | 332 | Returns 333 | -------- 334 | data : :class:`pyopia.pipeline.Data` 335 | containing the following new keys: 336 | 337 | :attr:`pyopia.pipeline.Data.im_corrected` 338 | 339 | 340 | Example pipeline uses: 341 | ---------------------- 342 | Don't apply any background correction after image load step : 343 | 344 | .. code-block:: toml 345 | 346 | [steps.nobackground] 347 | pipeline_class = 'pyopia.background.CorrectBackgroundNone' 348 | 349 | ''' 350 | 351 | def __init__(self): 352 | pass 353 | 354 | def __call__(self, data): 355 | data['im_corrected'] = data['imraw'] 356 | 357 | return data 358 | -------------------------------------------------------------------------------- /pyopia/cf_metadata.json: -------------------------------------------------------------------------------- 1 | {"major_axis_length": { 2 | "standard_name": "major_axis_length", 3 | "long_name": "The length of the major axis of the ellipse that has the same normalized second central moments as the region", 4 | "units": "micrometer", 5 | "calculation_method": "Computed using skimage.measure.regionprops (axis_major_length)", 6 | "pyopia_process_level": 1}, 7 | "minor_axis_length": { 8 | "standard_name": "minor_axis_length", 9 | "long_name": "The length of the minor axis of the ellipse that has the same normalized second central moments as the region", 10 | "units": "micrometer", 11 | "calculation_method": "Computed using skimage.measure.regionprops (axis_minor_length)", 12 | "pyopia_process_level": 1}, 13 | "equivalent_diameter": { 14 | "standard_name": "equivalent_circular_diameter", 15 | "long_name": "Diameter of a circle with the same area as the particle", 16 | "units": "micrometer", 17 | "calculation_method": "Computed using skimage.measure.regionprops (equivalent_diameter)", 18 | "pyopia_process_level": 1}, 19 | "minr": { 20 | "standard_name": "minimum_row_index", 21 | "long_name": "Minimum row index of the particle bounding box", 22 | "units": "pixels", 23 | "calculation_method": "Extracted from skimage.measure.regionprops (bbox[0])", 24 | "pyopia_process_level": 1}, 25 | "maxr": { 26 | "standard_name": "maximum_row_index", 27 | "long_name": "Maximum row index of the particle bounding box", 28 | "units": "pixels", 29 | "calculation_method": "Extracted from skimage.measure.regionprops (bbox[2])", 30 | "pyopia_process_level": 1}, 31 | "minc": { 32 | "standard_name": "minimum_column_index", 33 | "long_name": "Minimum column index of the particle bounding box", 34 | "units": "pixels", 35 | "calculation_method": "Extracted from skimage.measure.regionprops (bbox[1])", 36 | "pyopia_process_level": 1}, 37 | "maxc": { 38 | "standard_name": "maximum_column_index", 39 | "long_name": "Maximum column index of the particle bounding box", 40 | "units": "pixels", 41 | "calculation_method": "Extracted from skimage.measure.regionprops (bbox[3])", 42 | "pyopia_process_level": 1}, 43 | "saturation": { 44 | "standard_name": "image_saturation", 45 | "long_name": "Percentage saturation of the image", 46 | "units": "percent", 47 | "calculation_method": "Computed as the percentage of the image covered by particles relative to the maximum acceptable coverage", 48 | "pyopia_process_level": 1}, 49 | "index": { 50 | "standard_name": "index", 51 | "long_name": "Index of the particle in the dataset", 52 | "units": "", 53 | "calculation_method": "Sequential numbering of particles in the dataset", 54 | "pyopia_process_level": 1}, 55 | "export_name": { 56 | "standard_name": "export_name", 57 | "long_name": "Name of the exported particle ROI file", 58 | "units": "", 59 | "calculation_method": "Generated during particle export", 60 | "pyopia_process_level": 1}, 61 | "time": { 62 | "standard_name": "time", 63 | "long_name": "Time of particle observation", 64 | "calculation_method": "Extracted from the timestamp of the observation", 65 | "pyopia_process_level": 0}, 66 | "timestamp": { 67 | "standard_name": "timestamp", 68 | "long_name": "Timestamp of particle observation", 69 | "calculation_method": "Recorded during particle observation", 70 | "pyopia_process_level": 0}} -------------------------------------------------------------------------------- /pyopia/classify.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module containing tools for classifying particle ROIs 3 | """ 4 | 5 | import os 6 | import numpy as np 7 | import pandas as pd 8 | import logging 9 | from skimage.exposure import rescale_intensity 10 | 11 | logger = logging.getLogger() 12 | 13 | # import tensorflow here. It must be imported on the processor where it will be used! 14 | # import is therefore here instead of at the top of file. 15 | # consider # noqa: E(?) for flake8 / linting 16 | try: 17 | from tensorflow import keras 18 | import tensorflow as tf 19 | except ImportError: 20 | info_str = "ERROR: Could not import Keras. Classify will not work" 21 | info_str += " until you install tensorflow.\n" 22 | info_str += "Use: pip install pyopia[classification]\n" 23 | info_str += " or: pip install pyopia[classification-arm64]" 24 | info_str += " for tensorflow-macos (silicon chips)" 25 | raise ImportError(info_str) 26 | 27 | 28 | class Classify: 29 | """ 30 | A classifier class for PyOPIA workflow. 31 | This is intended as a parent class that can be used as a template for flexible classification methods 32 | 33 | Parameters 34 | ---------- 35 | model_path : str 36 | path to particle-classifier e.g. '/testdata/model_name/particle_classifier.keras' 37 | normalize_intensity : bool 38 | Scale input image intensity to [0-1] range before classification 39 | correct_whitebalance : bool 40 | Perform whitebalance correction before classification 41 | 42 | Example 43 | ------- 44 | 45 | .. code-block:: python 46 | 47 | cl = Classify(model_path='/testdata/model_name/particle_classifier.h5') 48 | 49 | prediction = cl.proc_predict(roi) # roi is an image roi to be classified 50 | 51 | Note 52 | ---- 53 | :meth:`Classify.load_model()` 54 | is run when the :class:`Classify` class is initialised. 55 | If this is used in combination with multiprocessing then the model must be loaded 56 | on the process where it will be used and not passed between processers 57 | (i.e. cl must be initialised on that process). 58 | 59 | The config setup looks like this: 60 | 61 | .. code-block:: toml 62 | 63 | [steps.classifier] 64 | pipeline_class = 'pyopia.classify.Classify' 65 | model_path = 'keras_model.h5' # path to trained nn model 66 | 67 | If '[steps.classifier]' is not defined, the classification will be skipped and no probabilities reported. 68 | 69 | See Also 70 | -------- 71 | 72 | If you want to use an example trained model for SilCam data 73 | (no guarantee of accuracy for other applications), you can get it using :mod:`pyopia.exampledata`: 74 | 75 | .. code-block:: python 76 | 77 | import pyopia.exampledata 78 | model_path = exampledata.get_example_model() 79 | 80 | """ 81 | 82 | def __init__( 83 | self, 84 | model_path=None, 85 | normalize_intensity=True, 86 | correct_whitebalance=False, 87 | ): 88 | self.model_path = model_path 89 | self.load_model() 90 | 91 | # Get config for image resizing from the model 92 | _, self.img_height, self.img_width, _ = self.model.get_config()["layers"][0][ 93 | "config" 94 | ]["batch_shape"] 95 | self.pad_to_aspect_ratio = getattr( 96 | self.model.layers[0], "pad_to_aspect_ratio", False 97 | ) 98 | 99 | # Enable this to perform whitebalance correction in the preprocessing step 100 | self.correct_whitebalance = correct_whitebalance 101 | 102 | # Enable this to rescale intensity in the preprocessing step 103 | self.normalize_intensity = normalize_intensity 104 | 105 | def __call__(self): 106 | return self 107 | 108 | def load_model(self): 109 | """ 110 | Load a trained Keras model into the Classify class. 111 | 112 | Parameters 113 | ---------- 114 | model : tf model object 115 | loaded Keras model 116 | class_names: list 117 | names for the model output classes 118 | """ 119 | model_path = self.model_path 120 | 121 | os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" 122 | keras.backend.clear_session() 123 | 124 | # Instantiate Keras model from file 125 | path, filename = os.path.split(model_path) 126 | self.model = keras.models.load_model(model_path) 127 | 128 | # Try to create model output class name list from last model layer name 129 | class_labels = None 130 | try: 131 | class_labels = self.model.layers[-1].name.split(".") 132 | except: # noqa E722 133 | logger.info( 134 | "Could not get class names from model layer name, reverting to old method with header file." 135 | ) 136 | 137 | # If we could not create correct class names above, revert to old header file method 138 | expected_class_number = self.model.layers[-1].output.shape[1] 139 | if class_labels is None or len(class_labels) != expected_class_number: 140 | header = pd.read_csv(os.path.join(path, "header.tfl.txt")) 141 | class_labels = header.columns 142 | 143 | self.class_labels = class_labels 144 | logger.info(self.class_labels) 145 | 146 | def preprocessing(self, img_input): 147 | """ 148 | Preprocess ROI ready for prediction. example here based on the pysilcam network setup 149 | 150 | Parameters 151 | ---------- 152 | img_input : float 153 | A particle ROI before preprocessing with range 0-1 154 | 155 | Returns 156 | ------- 157 | img_preprocessed : float 158 | A particle ROI with range 0.-255., corrected and preprocessed, ready for prediction 159 | """ 160 | 161 | whitebalanced = img_input.astype(np.float64) 162 | 163 | # Do white-balance correction as a per-channel histogram shift 164 | if self.correct_whitebalance: 165 | p = 99 166 | for c in range(3): 167 | whitebalanced[:, :, c] += (p / 100) - np.percentile( 168 | whitebalanced[:, :, c], p 169 | ) 170 | whitebalanced[whitebalanced > 1] = 1 171 | whitebalanced[whitebalanced < 0] = 0 172 | 173 | # Rescale intensity to 0-1 range 174 | if self.normalize_intensity: 175 | whitebalanced = rescale_intensity(whitebalanced) 176 | 177 | # convert back to 0-255 scaling (because of this layer in the network: 178 | # layers.Rescaling(1./255, input_shape=(img_height, img_width, 3))) 179 | # This is useful because it allows training to use tf.keras.utils.image_dataset_from_directory, 180 | # which loads images in 0-255 range 181 | img = keras.utils.img_to_array(whitebalanced * 255) 182 | 183 | # resize to match the dimentions expected by the network 184 | img = tf.image.resize( 185 | img, 186 | [self.img_height, self.img_width], 187 | method=tf.image.ResizeMethod.BILINEAR, 188 | preserve_aspect_ratio=self.pad_to_aspect_ratio, 189 | ) 190 | 191 | img_array = tf.keras.utils.img_to_array(img) 192 | img_preprocessed = tf.expand_dims(img_array, 0) # Create a batch 193 | return img_preprocessed 194 | 195 | @tf.function 196 | def predict(self, img_preprocessed): 197 | """ 198 | Use tensorflow model to classify particles. example here based on the pysilcam network setup. 199 | 200 | Parameters 201 | ---------- 202 | img_preprocessed: float 203 | A particle ROI arry, corrected and preprocessed using :meth:`Classify.preprocessing`, 204 | ready for prediction using :meth:`Classify.predict` 205 | 206 | Returns 207 | ------- 208 | prediction : array 209 | The probability of the roi belonging to each class 210 | """ 211 | 212 | prediction = self.model(img_preprocessed, training=False) 213 | prediction = tf.nn.softmax(prediction[0]) 214 | return prediction 215 | 216 | def proc_predict(self, img_input): 217 | """ 218 | Run pre-processing (:meth:`Classify.preprocessing`) and prediction (:meth:`Classify.predict`) 219 | using tensorflow model to classify particles. example here based on the pysilcam network setup. 220 | 221 | Parameters 222 | ---------- 223 | img_input : float 224 | Aparticle ROI with range 0-1 before preprocessing 225 | 226 | Returns 227 | ------- 228 | prediction : array 229 | The probability of the roi belonging to each class 230 | """ 231 | img_preprocessed = self.preprocessing(img_input) 232 | prediction = self.predict(img_preprocessed) 233 | 234 | return prediction 235 | -------------------------------------------------------------------------------- /pyopia/cli.py: -------------------------------------------------------------------------------- 1 | ''' 2 | PyOPIA top-level code primarily for managing cmd line entry points 3 | ''' 4 | 5 | import typer 6 | import toml 7 | import os 8 | import time 9 | import datetime 10 | import traceback 11 | import logging 12 | from rich.progress import Progress 13 | from rich.logging import RichHandler 14 | import rich.progress 15 | import pandas as pd 16 | import multiprocessing 17 | 18 | import pyopia 19 | import pyopia.background 20 | import pyopia.instrument.silcam 21 | import pyopia.instrument.holo 22 | import pyopia.instrument.uvp 23 | import pyopia.instrument.common 24 | import pyopia.io 25 | import pyopia.pipeline 26 | import pyopia.plotting 27 | import pyopia.process 28 | import pyopia.statistics 29 | 30 | app = typer.Typer() 31 | 32 | 33 | @app.command() 34 | def docs(): 35 | '''Open browser at PyOPIA's readthedocs page 36 | ''' 37 | print("Opening PyOPIA's docs") 38 | typer.launch("https://pyopia.readthedocs.io") 39 | 40 | 41 | @app.command() 42 | def modify_config(existing_filename: str, modified_filename: str, 43 | raw_files=None, pixel_size=None, 44 | step_name=None, modify_arg=None, modify_value=None): 45 | '''Modify a existing config.toml file and write a new one to disc 46 | 47 | Parameters 48 | ---------- 49 | existing_filename : str 50 | e.g. config.toml 51 | modified_filename : str 52 | e.g. config_new.toml 53 | raw_files : str, optional 54 | modify the raw file input in the `[general]` settings, by default None 55 | pixel_size : str, optional 56 | modify the pixel size in the `[general]` settings, by default None 57 | step_name : str, optional 58 | the name of the step to modify e.g. `segmentation`, by default None 59 | modify_arg : str, optional 60 | the name of the step to modify e.g. `threshold`. 61 | existing arguments will be overwritten, non-existent arguments will be created, by default None 62 | modify_value : str or floar, optional 63 | new value to attach to the 'modify_arg' setting e.g. 0.85. 64 | Accepts either string or float input, by default None 65 | ''' 66 | toml_settings = pyopia.io.load_toml(existing_filename) 67 | 68 | if raw_files is not None: 69 | toml_settings['general']['raw_files'] = f'{raw_files}' 70 | if pixel_size is not None: 71 | toml_settings['general']['pixel_size'] = float(pixel_size) 72 | 73 | if step_name is not None: 74 | try: 75 | if modify_arg == 'average_window': 76 | modify_value = int(modify_value) 77 | elif modify_arg == 'threshold': 78 | modify_value = float(modify_value) 79 | else: 80 | modify_value = str(modify_value) 81 | except ValueError: 82 | pass 83 | 84 | toml_settings['steps'][step_name][modify_arg] = modify_value 85 | 86 | with open(modified_filename, "w") as toml_file: 87 | toml.dump(toml_settings, toml_file) 88 | 89 | 90 | @app.command() 91 | def generate_config(instrument: str, raw_files: str, model_path: str, outfolder: str, output_prefix: str): 92 | '''Put an example config.toml file in the current directory 93 | 94 | Parameters 95 | ---------- 96 | instrument : str 97 | either `silcam`, `holo` or `uvp` 98 | raw_files : str 99 | raw_files 100 | model_path : str 101 | model_path 102 | outfolder : str 103 | outfolder 104 | output_prefix : str 105 | output_prefix 106 | ''' 107 | match instrument: 108 | case 'silcam': 109 | pipeline_config = pyopia.instrument.silcam.generate_config(raw_files, model_path, outfolder, output_prefix) 110 | case 'holo': 111 | pipeline_config = pyopia.instrument.holo.generate_config(raw_files, model_path, outfolder, output_prefix) 112 | case 'uvp': 113 | pipeline_config = pyopia.instrument.uvp.generate_config(raw_files, model_path, outfolder, output_prefix) 114 | 115 | config_filename = instrument + "-config.toml" 116 | with open(config_filename, "w") as toml_file: 117 | toml.dump(pipeline_config, toml_file) 118 | 119 | 120 | @app.command() 121 | def process(config_filename: str, num_chunks: int = 1, strategy: str = 'block'): 122 | '''Run a PyOPIA processing pipeline based on given a config.toml 123 | 124 | Parameters 125 | ---------- 126 | config_filename : str 127 | Config filename 128 | 129 | numchunks : int, optional 130 | Split the dataset into chucks, and process in parallell, by default 1 131 | 132 | strategy : str, optional 133 | Strategy to use for chunking dataset, either `block` or `interleave`. Defult: `block` 134 | ''' 135 | t1 = time.time() 136 | 137 | with Progress(transient=True) as progress: 138 | progress.console.print(f"[blue]PYOPIA VERSION {pyopia.__version__}") 139 | 140 | progress.console.print("[blue]LOAD CONFIG") 141 | pipeline_config = pyopia.io.load_toml(config_filename) 142 | 143 | setup_logging(pipeline_config) 144 | logger = logging.getLogger('rich') 145 | logger.info(f'PyOPIA process started {pd.Timestamp.now()}') 146 | 147 | check_chunks(num_chunks, pipeline_config) 148 | 149 | progress.console.print("[blue]OBTAIN IMAGE LIST") 150 | conf_corrbg = pipeline_config['steps'].get('correctbackground', dict()) 151 | average_window = conf_corrbg.get('average_window', 0) 152 | bgshift_function = conf_corrbg.get('bgshift_function', 'pass') 153 | raw_files = pyopia.pipeline.FilesToProcess(pipeline_config['general']['raw_files']) 154 | raw_files.prepare_chunking(num_chunks, average_window, bgshift_function, strategy=strategy) 155 | 156 | # Write the dataset list of images to a text file 157 | raw_files.to_filelist_file('filelist.txt') 158 | 159 | progress.console.print('[blue]PREPARE FOLDERS') 160 | if 'output' not in pipeline_config['steps']: 161 | raise Exception('The given config file is missing an "output" step.\n' + 162 | 'This is needed to setup how to save data to disc.') 163 | output_datafile = pipeline_config['steps']['output']['output_datafile'] 164 | os.makedirs(os.path.split(output_datafile)[:-1][0], 165 | exist_ok=True) 166 | 167 | if os.path.isfile(output_datafile + '-STATS.nc'): 168 | dt_now = datetime.datetime.now().strftime('D%Y%m%dT%H%M%S') 169 | newname = output_datafile + '-conflict-' + str(dt_now) + '-STATS.nc' 170 | logger.warning(f'Renaming conflicting file to: {newname}') 171 | os.rename(output_datafile + '-STATS.nc', newname) 172 | 173 | progress.console.print("[blue]INITIALISE PIPELINE") 174 | 175 | # With one chunk we keep the non-multiprocess functionality to ensure backwards compatibility 176 | job_list = [] 177 | if num_chunks == 1: 178 | process_file_list(raw_files, pipeline_config, 0) 179 | else: 180 | for c, chunk in enumerate(raw_files.chunked_files): 181 | job = multiprocessing.Process(target=process_file_list, args=(chunk, pipeline_config, c)) 182 | job_list.append(job) 183 | 184 | # Start all the jobs 185 | [job.start() for job in job_list] 186 | 187 | # If we are using multiprocessing, make sure all jobs have finished 188 | [job.join() for job in job_list] 189 | 190 | # Calculate and print total processing time 191 | time_total = pd.to_timedelta(time.time() - t1, 'seconds') 192 | with Progress(transient=True) as progress: 193 | progress.console.print(f"[blue]PROCESSING COMPLETED IN {time_total}") 194 | 195 | 196 | @app.command() 197 | def merge_mfdata(path_to_data: str, prefix='*', overwrite_existing_partials: bool = True, 198 | chunk_size: int = None): 199 | '''Combine a multi-file directory of STATS.nc files into a single '-STATS.nc' file 200 | that can then be loaded with {func}`pyopia.io.load_stats` 201 | 202 | Parameters 203 | ---------- 204 | path_to_data : str 205 | Folder name containing nc files with pattern '*Image-D*-STATS.nc' 206 | 207 | prefix : str 208 | Prefix to multi-file dataset (for replacing the wildcard in '*Image-D*-STATS.nc'). 209 | Defaults to '*' 210 | 211 | overwrite_existing_partials : bool 212 | Do not reprocess existing merged netcdf files for each chunk if False. 213 | Otherwise reprocess (load) and overwrite. This can be used to restart 214 | or continue a previous merge operation as new files become available. 215 | 216 | chunk_size : int 217 | Process this many files together and store as partially merged netcdf files, which 218 | are then merged at the end. Default: None, process all files together. 219 | ''' 220 | setup_logging({'general': {}}) 221 | 222 | pyopia.io.merge_and_save_mfdataset(path_to_data, prefix=prefix, 223 | overwrite_existing_partials=overwrite_existing_partials, 224 | chunk_size=chunk_size) 225 | 226 | 227 | def process_file_list(file_list, pipeline_config, c): 228 | '''Run a PyOPIA processing pipeline for a chuncked list of files based on a given config.toml 229 | 230 | Parameters 231 | ---------- 232 | file_list : str 233 | List of file paths to process, where each file will be passed individually through the processing pipeline 234 | 235 | pipeline_config : str 236 | Loaded config.toml file to initialize the processing pipeline and setup logging 237 | 238 | c : int 239 | Chunk index for tracking progress and logging. If set to 0, enables the 240 | progress bar; for other values, the progress bar is disabled. 241 | ''' 242 | processing_pipeline = pyopia.pipeline.Pipeline(pipeline_config) 243 | setup_logging(pipeline_config) 244 | logger = logging.getLogger('rich') 245 | 246 | with get_custom_progress_bar(f'[blue]Processing progress (chunk {c})', disable=c != 0) as pbar: 247 | for filename in pbar.track(file_list, description=f'[blue]Processing progress (chunk {c})'): 248 | try: 249 | logger.debug(f'Chunk {c} starting to process {filename}') 250 | processing_pipeline.run(filename) 251 | except Exception as e: 252 | logger.warning('[red]An error occured in processing, ' + 253 | 'skipping rest of pipeline and moving to next image.' + 254 | f'(chunk {c})') 255 | logger.error(e) 256 | logger.debug(''.join(traceback.format_tb(e.__traceback__))) 257 | 258 | 259 | def setup_logging(pipeline_config): 260 | '''Configure logging 261 | 262 | Parameters 263 | ---------- 264 | pipeline_config : dict 265 | TOML settings 266 | ''' 267 | # Get user parameters or default values for logging 268 | log_file = pipeline_config['general'].get('log_file', None) 269 | log_level_name = pipeline_config['general'].get('log_level', 'INFO') 270 | log_level = getattr(logging, log_level_name) 271 | 272 | # Either log to file (silent console) or to console with Rich 273 | if log_file is None: 274 | handlers = [RichHandler(show_time=True, show_level=False)] 275 | else: 276 | handlers = [logging.FileHandler(log_file, mode='a')] 277 | 278 | # Configure logger 279 | log_format = '%(asctime)s %(levelname)s %(processName)s [%(module)s.%(funcName)s] %(message)s' 280 | logging.basicConfig(level=log_level, datefmt='%Y-%m-%d %H:%M:%S', format=log_format, handlers=handlers) 281 | 282 | 283 | def check_chunks(chunks, pipeline_config): 284 | if chunks < 1: 285 | raise RuntimeError('You must have at least 1 chunk') 286 | 287 | append_enabled = pipeline_config['steps']['output'].get('append', True) 288 | if chunks > 1 and append_enabled: 289 | raise RuntimeError('Output mode must be set to "append = false" in "output" step when using more than one chunk') 290 | 291 | 292 | def get_custom_progress_bar(description, disable): 293 | ''' Create a custom rich.progress.Progress object for displaying progress bars''' 294 | progress = Progress( 295 | rich.progress.TextColumn(description), 296 | rich.progress.BarColumn(), 297 | rich.progress.TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), 298 | rich.progress.MofNCompleteColumn(), 299 | rich.progress.TextColumn("•"), 300 | rich.progress.TimeElapsedColumn(), 301 | rich.progress.TextColumn("•"), 302 | rich.progress.TimeRemainingColumn(), 303 | disable=disable 304 | ) 305 | return progress 306 | 307 | 308 | if __name__ == "__main__": 309 | app() 310 | -------------------------------------------------------------------------------- /pyopia/exampledata.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import zipfile 3 | import os 4 | import gdown 5 | from pathlib import Path 6 | 7 | import logging 8 | 9 | logger = logging.getLogger() 10 | 11 | 12 | def get_classifier_database_from_pysilcam_blob(download_directory="./"): 13 | """Downloads a specified filename from the pysilcam.blob into the working dir. if it doesn't already exist 14 | 15 | only works for known filenames that are on this blob 16 | 17 | Parameters 18 | ---------- 19 | filename : string 20 | known filename on the blob 21 | 22 | """ 23 | if os.path.exists(os.path.join(download_directory)): 24 | logger.info(download_directory, "already exists. Returning nothing") 25 | return download_directory 26 | os.makedirs(download_directory, exist_ok=False) 27 | url = "https://pysilcam.blob.core.windows.net/test-data/silcam_database.zip" 28 | logger.info("Downloading....") 29 | urllib.request.urlretrieve(url, download_directory + "/silcam_database.zip") 30 | logger.info("Unzipping....") 31 | with zipfile.ZipFile( 32 | os.path.join(download_directory, "silcam_database.zip"), "r" 33 | ) as zipit: 34 | zipit.extractall(os.path.join(download_directory, "../")) 35 | logger.info("Removing zip file") 36 | os.remove(os.path.join(download_directory, "silcam_database.zip")) 37 | logger.info("Done.") 38 | return download_directory 39 | 40 | 41 | def get_file_from_pysilcam_blob(filename, download_directory="./"): 42 | """Downloads a specified filename from the pysilcam.blob into the working dir. if it doesn't already exist 43 | 44 | only works for known filenames that are on this blob 45 | 46 | Parameters 47 | ---------- 48 | filename : string 49 | known filename on the blob 50 | 51 | """ 52 | if os.path.exists(os.path.join(download_directory, filename)): 53 | return filename 54 | url = "https://pysilcam.blob.core.windows.net/test-data/" + filename 55 | urllib.request.urlretrieve(url, os.path.join(download_directory, filename)) 56 | return download_directory 57 | 58 | 59 | def get_example_silc_image(download_directory="./"): 60 | """calls `get_file_from_pysilcam_blob` for a silcam iamge 61 | 62 | Returns 63 | ------- 64 | string 65 | filename 66 | """ 67 | filename = "D20181101T142731.838206.silc" 68 | if os.path.isfile(filename): 69 | logger.info("Example image already exists. Skipping download.") 70 | return filename 71 | get_file_from_pysilcam_blob(filename, download_directory) 72 | return filename 73 | 74 | 75 | def get_example_model(download_directory="./"): 76 | """Download PyOPIA default CNN model classifier 77 | 78 | Download from the pysilcam blob storage into the working dir. 79 | If the file exists, skip the download. 80 | 81 | Returns 82 | ------- 83 | string 84 | model_filename 85 | """ 86 | model_filename = "pyopia-default-classifier-20250409.keras" 87 | model_path = Path(download_directory, model_filename) 88 | model_url = ( 89 | f"https://pysilcam.blob.core.windows.net/test-data/{str(model_filename)}" 90 | ) 91 | if not model_path.exists(): 92 | logger.info("Downloading example model...") 93 | urllib.request.urlretrieve(model_url, model_path) 94 | return str(model_path) 95 | 96 | 97 | def get_example_hologram_and_background(download_directory="./"): 98 | """calls `get_file_from_pysilcam_blob` for a raw hologram, and its associated background image. 99 | 100 | Returns 101 | ------- 102 | string 103 | holo_filename 104 | 105 | string 106 | holo_background_filename 107 | """ 108 | holo_filename = "001-2082.pgm" 109 | holo_background_filename = "imbg-" + holo_filename 110 | get_file_from_pysilcam_blob(holo_filename, download_directory) 111 | get_file_from_pysilcam_blob(holo_background_filename, download_directory) 112 | 113 | holo_filename = os.path.join(download_directory, holo_filename) 114 | holo_background_filename = os.path.join( 115 | download_directory, holo_background_filename 116 | ) 117 | 118 | return holo_filename, holo_background_filename 119 | 120 | 121 | def get_folder_from_holo_repository(foldername="holo_test_data_01", existsok=False): 122 | """Downloads a specified folder from the holo testing repository into the working dir. if it doesn't already exist 123 | 124 | only works for known folders that are on the GoogleDrive repository 125 | by default will download a known-good folder. Additional elif statements can be added to implement additional folders. 126 | 127 | Parameters 128 | ---------- 129 | foldername : string 130 | known filename on the blob 131 | existsok : (bool, optional) 132 | if True, then don't download if the specified folder already exists, defaults to False 133 | 134 | """ 135 | if foldername == "holo_test_data_01": 136 | url = "https://drive.google.com/drive/folders/1yNatOaKdWwYQp-5WVEDItoibr-k0lGsP?usp=share_link" 137 | 138 | elif foldername == "holo_test_data_02": 139 | url = "https://drive.google.com/drive/folders/1E5iNSyfeKcVMLVe4PNEwF2Q2mo3WVjF5?usp=share_link" 140 | 141 | else: 142 | foldername == "holo_test_data_01" 143 | url = "https://drive.google.com/drive/folders/1yNatOaKdWwYQp-5WVEDItoibr-k0lGsP?usp=share_link" 144 | 145 | if os.path.exists(foldername) and existsok: 146 | logger.info(foldername + " already exists. Skipping download.") 147 | return foldername 148 | 149 | gdown.download_folder(url, quiet=True, use_cookies=False) 150 | return foldername 151 | -------------------------------------------------------------------------------- /pyopia/instrument/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /pyopia/instrument/common.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Non-instrument-specific functions that operates on the image loading or initial processing level. 3 | ''' 4 | import numpy as np 5 | 6 | 7 | class RectangularImageMask(): 8 | '''PyOpia pipline-compatible class for masking out part of the raw image. 9 | 10 | Required keys in :class:`pyopia.pipeline.Data`: 11 | - :attr:`pyopia.pipeline.Data.imraw` 12 | 13 | Parameters 14 | ---------- 15 | mask_bbox : (list, optional) 16 | Pixel corner coordinates of rectangle to mask (image outside the rectangle is set to 0) 17 | 18 | Returns 19 | ------- 20 | data : :class:`pyopia.pipeline.Data` 21 | containing the new key: 22 | 23 | :attr:`pyopia.pipeline.Data.im_masked` 24 | 25 | 26 | Example pipeline use: 27 | ---------------------- 28 | Put this in your pipeline right after load step to mask out border outside specified pixel coordinates: 29 | 30 | .. code-block:: toml 31 | 32 | [steps.mask] 33 | pipeline_class = 'pyopia.instrument.common.RectangularImageMask' 34 | mask_bbox = [[200, 1850], [400, 2048], [0, 3]] 35 | 36 | The mask_bbox is [[start_row, end_row], [start_col, end_col], [start_colorchan, end_colorchan]] 37 | ''' 38 | 39 | def __init__(self, mask_bbox=None): 40 | if mask_bbox is None: 41 | self.mask_bbox = (slice(None), slice(None), slice(None)) 42 | else: 43 | self.mask_bbox = (slice(mask_bbox[0][0], mask_bbox[0][1]), 44 | slice(mask_bbox[1][0], mask_bbox[1][1]), 45 | slice(mask_bbox[2][0], mask_bbox[2][1])) 46 | 47 | def __call__(self, data): 48 | # Create a masked version of imraw, where space between defined mask rectangle and border is set to 0, 49 | # while inside is kept. 50 | imraw_masked = np.zeros_like(data['imraw']) 51 | imraw_masked[self.mask_bbox] = data['imraw'][self.mask_bbox] 52 | data['im_masked'] = imraw_masked 53 | 54 | return data 55 | -------------------------------------------------------------------------------- /pyopia/instrument/holo.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This is a module containing basic processing for reconstruction of in-line holographic images with :mod:`pyopia.pipeline`. 3 | 4 | See (and references therein): 5 | Davies EJ, Buscombe D, Graham GW & Nimmo-Smith WAM (2015) 6 | 'Evaluating Unsupervised Methods to Size and Classify Suspended Particles 7 | Using Digital In-Line Holography' 8 | Journal of Atmospheric and Oceanic Technology 32, (6) 1241-1256, 9 | https://doi.org/10.1175/JTECH-D-14-00157.1 10 | https://journals.ametsoc.org/view/journals/atot/32/6/jtech-d-14-00157_1.xml 11 | 12 | 2022-11-01 Alex Nimmo-Smith alex.nimmo.smith@plymouth.ac.uk 13 | ''' 14 | 15 | import os 16 | import numpy as np 17 | import pandas as pd 18 | from scipy import fft 19 | from skimage.io import imread 20 | from skimage.filters import sobel 21 | from skimage.morphology import disk, erosion, dilation 22 | import pyopia.process 23 | import struct 24 | from datetime import timedelta, datetime 25 | from glob import glob 26 | 27 | import logging 28 | logger = logging.getLogger() 29 | 30 | 31 | class Initial(): 32 | '''PyOpia pipline-compatible class for one-time setup of holograhic reconstruction 33 | 34 | Parameters 35 | ---------- 36 | wavelength : float 37 | laser wavelength in nm 38 | n : float 39 | refractive index of medium 40 | offset : float 41 | offset of focal plane from hologram plane in mm 42 | minZ : float 43 | minimum reconstruction distance in mm 44 | maxZ : float 45 | maximum reconstruction distance in mm 46 | stepZ : float 47 | step size in mm (i.e. resolution of reconstruction between minZ and maxZ) 48 | 49 | Returns 50 | ------- 51 | kern : np.arry 52 | reconstruction kernel 53 | im_stack : np.array 54 | pre-allocated array to receive reconstruction 55 | 56 | ''' 57 | 58 | def __init__(self, wavelength, n, offset, minZ, maxZ, stepZ): 59 | self.wavelength = wavelength 60 | self.n = n 61 | self.offset = offset 62 | self.minZ = minZ 63 | self.maxZ = maxZ 64 | self.stepZ = stepZ 65 | 66 | def __call__(self, data): 67 | logger.info('Using first raw file from list in general settings to determine image dimensions') 68 | raw_files = glob(data['settings']['general']['raw_files']) 69 | self.filename = raw_files[0] 70 | imtmp = load_image(self.filename) 71 | self.pixel_size = data['settings']['general']['pixel_size'] 72 | logger.info('Build kernel with pixel_size = ', self.pixel_size, 'um') 73 | kern = create_kernel(imtmp, self.pixel_size, self.wavelength, self.n, self.offset, self.minZ, self.maxZ, self.stepZ) 74 | im_stack = np.zeros(np.shape(kern)).astype(np.float64) 75 | logger.info('HoloInitial done', datetime.now()) 76 | data['kern'] = kern 77 | data['im_stack'] = im_stack 78 | return data 79 | 80 | 81 | def load_image(filename): 82 | '''load a hologram image file from disc 83 | 84 | Parameters 85 | ---------- 86 | filename : string 87 | filename to load 88 | 89 | Returns 90 | ------- 91 | array 92 | raw image 93 | ''' 94 | img = imread(filename).astype(np.float64) / 255 95 | return img 96 | 97 | 98 | class Load(): 99 | '''PyOpia pipline-compatible class for loading a single holo image 100 | 101 | Parameters 102 | ---------- 103 | filename : string 104 | hologram filename (.pgm) 105 | 106 | Returns 107 | ------- 108 | timestamp : timestamp 109 | timestamp @todo 110 | imraw : np.arraym 111 | hologram 112 | ''' 113 | 114 | def __init__(self): 115 | pass 116 | 117 | def __call__(self, data): 118 | logger.info(data['filename']) 119 | try: 120 | timestamp = read_lisst_holo_info(data['filename']) 121 | except ValueError: 122 | timestamp = pd.to_datetime(os.path.splitext(os.path.basename(data['filename']))[0][1:]) 123 | logger.info(timestamp) 124 | im = load_image(data['filename']) 125 | data['timestamp'] = timestamp 126 | data['imraw'] = im 127 | return data 128 | 129 | 130 | class Reconstruct(): 131 | '''PyOpia pipline-compatible class for reconstructing a single holo image 132 | 133 | Required keys in :class:`pyopia.pipeline.Data`: 134 | - :attr:`pyopia.pipeline.Data.im_corrected` 135 | 136 | Parameters 137 | ---------- 138 | stack_clean : float 139 | defines amount of cleaning of stack (fraction of max value below which to zero) 140 | forward_filter_option : int 141 | switch to control filtering in frequency domain (0=none,1=DC only,2=zero ferquency/default) 142 | inverse_output_option : int 143 | switch to control optional scaling of output intensity (0=square/default,1=linear) 144 | 145 | Returns 146 | ------- 147 | data : :class:`pyopia.pipeline.Data` 148 | containing the following new keys: 149 | 150 | :attr:`pyopia.pipeline.Data.im_stack` 151 | ''' 152 | 153 | def __init__(self, stack_clean=0, forward_filter_option=0, inverse_output_option=0): 154 | self.stack_clean = stack_clean 155 | self.forward_filter_option = forward_filter_option 156 | self.inverse_output_option = inverse_output_option 157 | 158 | def __call__(self, data): 159 | imc = data['im_corrected'] 160 | kern = data['kern'] 161 | im_stack = data['im_stack'] 162 | 163 | im_fft = forward_transform(imc, self.forward_filter_option) 164 | im_stack = inverse_transform(im_fft, kern, im_stack, self.inverse_output_option) 165 | data['im_stack'] = clean_stack(im_stack, self.stack_clean) 166 | 167 | return data 168 | 169 | 170 | def forward_transform(im, forward_filter_option=2): 171 | '''Perform forward transform with optional filtering 172 | 173 | Parameters 174 | ---------- 175 | im : np.array 176 | hologram (usually background-corrected) 177 | forward_filter_option : int 178 | filtering in frequency domain (0=none/default,1=DC only,2=zero ferquency) 179 | 180 | Returns 181 | ------- 182 | im_fft : np.array 183 | im_fft 184 | ''' 185 | 186 | # Perform forward transform 187 | im_fft = fft.fft2(im, workers=os.cpu_count()) 188 | 189 | # apply filtering if required 190 | match forward_filter_option: 191 | case 1: 192 | im_fft[0, 0] = 0 193 | case 2: 194 | im_fft[:, 0] = 0 195 | im_fft[0, :] = 0 196 | case _: 197 | pass 198 | 199 | # fftshift 200 | im_fft = fft.fftshift(im_fft) 201 | 202 | return im_fft 203 | 204 | 205 | def create_kernel(im, pixel_size, wavelength, n, offset, minZ, maxZ, stepZ): 206 | '''create reconstruction kernel 207 | 208 | Parameters 209 | ---------- 210 | im : np.arry 211 | hologram 212 | pixel_size : float 213 | pixel_size in microns per pixel (i.e. usually 4.4 for lisst-holo type of resolution) 214 | wavelength : float 215 | laser wavelength in nm 216 | minZ : float 217 | minimum reconstruction distance in mm 218 | maxZ : float 219 | maximum reconstruction distance in mm 220 | stepZ : float 221 | step size in mm (i.e. resolution of reconstruction between minZ and maxZ) 222 | 223 | Returns 224 | ------- 225 | np.array 226 | holographic reconstruction kernel (3D array of complex numbers) 227 | ''' 228 | cx = im.shape[1] / 2 229 | cy = im.shape[0] / 2 230 | 231 | x = (np.arange(0, im.shape[1]) - cx) / cx 232 | y = (np.arange(0, im.shape[0]) - cy) / cy 233 | y.shape = (im.shape[0], 1) 234 | 235 | f1 = np.tile(x, (im.shape[0], 1)) 236 | f2 = np.tile(y, (1, im.shape[1])) 237 | 238 | f = (np.pi / (pixel_size / 1e6)) * (f1**2 + f2**2)**0.5 239 | 240 | z = (np.arange(minZ * 1e-3, (maxZ + stepZ) * 1e-3, stepZ * 1e-3) / n) + (offset * 1e-3) 241 | 242 | wavelength_m = wavelength * 1e-9 243 | k = 2 * np.pi / wavelength_m 244 | 245 | kern = -1j * np.zeros((im.shape[0], im.shape[1], len(z))) 246 | for i, z_ in enumerate(z): 247 | 248 | kern[:, :, i] = np.exp(-1j * f**2 * z_ / (2 * k)) 249 | return kern 250 | 251 | 252 | def inverse_transform(im_fft, kern, im_stack, inverse_output_option=0): 253 | '''create the reconstructed hologram stack of real images 254 | 255 | Parameters 256 | ---------- 257 | im_fft : np.array 258 | calculated from forward_transform 259 | kern : np.array 260 | calculated from create_kernel 261 | im_stack : np.array 262 | pre-allocated array to receive output 263 | inverse_output_option: int 264 | optional scaling of output intensity (0=square/default,1=linear) 265 | 266 | Returns 267 | ------- 268 | np.arry 269 | im_stack 270 | ''' 271 | 272 | for i in range(np.shape(kern)[2]): 273 | im_tmp = np.multiply(im_fft, kern[:, :, i]) 274 | match inverse_output_option: 275 | case 1: 276 | im_stack[:, :, i] = fft.ifft2(im_tmp, workers=os.cpu_count()).real 277 | case _: 278 | im_stack[:, :, i] = (fft.ifft2(im_tmp, workers=os.cpu_count()).real)**2 279 | 280 | return im_stack 281 | 282 | 283 | def clean_stack(im_stack, stack_clean): 284 | '''clean the im_stack by removing low value pixels - set to 0 to disable 285 | 286 | Parameters 287 | ---------- 288 | im_stack : np.array 289 | 290 | stack_clean : flaot 291 | pixels below this value will be zeroed 292 | 293 | Returns 294 | ------- 295 | np.array 296 | cleaned version of im_stack 297 | ''' 298 | if stack_clean > 0.0: 299 | im_max = np.amax(im_stack, axis=None) 300 | im_stack[im_stack < im_max * stack_clean] = 0 301 | return im_stack 302 | 303 | 304 | def std_map(im_stack): 305 | '''_summary_ 306 | 307 | Parameters 308 | ---------- 309 | im_stack : _type_ 310 | _description_ 311 | 312 | Returns 313 | ------- 314 | _type_ 315 | _description_ 316 | ''' 317 | std_map = np.std(im_stack, axis=2) 318 | return std_map 319 | 320 | 321 | def max_map(im_stack): 322 | '''_summary_ 323 | 324 | Parameters 325 | ---------- 326 | im_stack : _type_ 327 | _description_ 328 | 329 | Returns 330 | ------- 331 | _type_ 332 | _description_ 333 | ''' 334 | max_map = np.max(im_stack, axis=2) 335 | return max_map 336 | 337 | 338 | def rescale_image(im): 339 | '''rescale im (e.g. may be stack summary) to be dark particles on light background 340 | 341 | Parameters 342 | ---------- 343 | im : image 344 | input image to be scaled 345 | 346 | Returns 347 | ------- 348 | im : image 349 | scaled and inverted image 350 | ''' 351 | im_max = np.max(im) 352 | im_min = np.min(im) 353 | im = (im - im_min) / (im_max - im_min) 354 | im = 1 - im 355 | return im 356 | 357 | 358 | def find_focus_imax(im_stack, bbox, increase_depth_of_field): 359 | '''finds and returns the focussed image for the bbox region within im_stack 360 | using intensity of bbox area 361 | 362 | Parameters 363 | ---------- 364 | im_stack : nparray 365 | image stack 366 | 367 | bbox : tuple 368 | Bounding box (min_row, min_col, max_row, max_col) 369 | 370 | increase_depth_of_field : bool 371 | set to True to use max values from planes either side of main focus plane to create focussed image (default False) 372 | 373 | Returns 374 | ------- 375 | im : image 376 | focussed image for bbox 377 | 378 | ifocus: int 379 | index through stack of focussed image 380 | ''' 381 | roi = im_stack[bbox[0]:bbox[2], bbox[1]:bbox[3], :] 382 | focus = np.sum(roi, axis=(0, 1)) 383 | ifocus = np.argmax(focus) 384 | 385 | if increase_depth_of_field: 386 | im_focus = np.max(roi[:, :, np.max([ifocus-1, 0]):np.min([ifocus+1, roi.shape[2]])], axis=2) 387 | else: 388 | im_focus = roi[:, :, ifocus] 389 | 390 | return im_focus, ifocus 391 | 392 | 393 | def find_focus_sobel(im_stack, bbox, increase_depth_of_field): 394 | '''finds and returns the focussed image for the bbox region within im_stack 395 | using edge magnitude of bbox area 396 | 397 | Parameters 398 | ---------- 399 | im_stack : nparray 400 | image stack 401 | 402 | bbox : tuple 403 | Bounding box (min_row, min_col, max_row, max_col) 404 | 405 | increase_depth_of_field : bool 406 | set to True to use max values from planes either side of main focus plane to create focussed image (default False) 407 | 408 | Returns 409 | ------- 410 | im : image 411 | focussed image for bbox 412 | 413 | ifocus: int 414 | index through stack of focussed image 415 | ''' 416 | im_bbox = im_stack[bbox[0]:bbox[2], bbox[1]:bbox[3], :] 417 | roi = np.zeros_like(im_bbox) 418 | for zi in range(roi.shape[2]): 419 | roi[:, :, zi] = sobel(im_bbox[:, :, zi]) 420 | 421 | focus = np.sum(roi, axis=(0, 1)) 422 | ifocus = np.argmax(focus) 423 | 424 | if increase_depth_of_field: 425 | im_focus = np.max(roi[:, :, np.max([ifocus-1, 0]):np.min([ifocus+1, roi.shape[2]])], axis=2) 426 | else: 427 | im_focus = roi[:, :, ifocus] 428 | 429 | return im_focus, ifocus 430 | 431 | 432 | class Focus(): 433 | '''PyOpia pipline-compatible class for creating a focussed image from an image stack 434 | 435 | Required keys in :class:`pyopia.pipeline.Data`: 436 | - :attr:`pyopia.pipeline.Data.im_stack` 437 | 438 | Parameters 439 | ---------- 440 | stacksummary_function : (string, optional) 441 | Function used to summarise the stack 442 | Available functions are: 443 | 444 | :func:`pyopia.instrument.holo.max_map` 445 | 446 | :func:`pyopia.instrument.holo.std_map` (default) 447 | 448 | threshold : float 449 | threshold to apply during initial segmentation 450 | 451 | focus_function : (string, optional) 452 | Function used to focus particles within the stack 453 | Available functions are: 454 | 455 | :func:`pyopia.instrument.holo.find_focus_imax` (default) 456 | 457 | :func:`pyopia.instrument.holo.find_focus_sobel` 458 | 459 | discard_end_slices : (bool, optional) 460 | set to True to discard particles that focus at either first or last slice 461 | 462 | increase_depth_of_field : (bool, optional) 463 | set to True to use max values from planes either side of main focus plane to create focussed image (default False) 464 | 465 | merge_adjacent_particles : (bool, optional) 466 | set to 0 (default) to deactivate, set to positive integer to give radius in pixels of smoothing of stack 467 | summary image to merge adjacent particles 468 | 469 | Returns 470 | ------- 471 | data : :class:`pyopia.pipeline.Data` 472 | 473 | containing the following keys: 474 | 475 | :attr:`pyopia.pipeline.Data.im_focussed` 476 | 477 | :attr:`pyopia.pipeline.Data.imss` 478 | 479 | :attr:`pyopia.pipeline.Data.stack_rp` 480 | 481 | :attr:`pyopia.pipeline.Data.stack_ifocus` 482 | ''' 483 | 484 | def __init__(self, stacksummary_function='std_map', threshold=0.9, focus_function='find_focus_imax', 485 | discard_end_slices=True, increase_depth_of_field=False, merge_adjacent_particles=0): 486 | self.stacksummary_function = stacksummary_function 487 | self.threshold = threshold 488 | self.focus_function = focus_function 489 | self.discard_end_slices = discard_end_slices 490 | self.increase_depth_of_field = increase_depth_of_field 491 | self.merge_adjacent_particles = merge_adjacent_particles 492 | 493 | def __call__(self, data): 494 | im_stack = data['im_stack'] 495 | 496 | match self.stacksummary_function: 497 | case 'std_map': 498 | imss = std_map(im_stack) 499 | case 'max_map': 500 | imss = max_map(im_stack) 501 | case _: 502 | raise ValueError('stacksummary_function in pyopia.instrument.holo.Focus not recognised') 503 | 504 | imss = rescale_image(imss) 505 | if self.merge_adjacent_particles: 506 | se = disk(self.merge_adjacent_particles) 507 | imss = dilation(imss, se) 508 | imss = erosion(imss, se) 509 | data['imss'] = imss 510 | 511 | # segment imss to find particle x-y locations 512 | imssbw = pyopia.process.segment(imss, self.threshold) 513 | # identify particles 514 | region_properties = pyopia.process.measure_particles(imssbw) 515 | # loop through bounding boxes to focus each particle and add to output imc 516 | im_focussed = np.zeros_like(im_stack[:, :, 0]) 517 | ifocus = [] 518 | rp_out = [] 519 | for rp in region_properties: 520 | 521 | match self.focus_function: 522 | case 'find_focus_imax': 523 | im_focus_, ifocus_ = find_focus_imax(im_stack, rp.bbox, self.increase_depth_of_field) 524 | case 'find_focus_sobel': 525 | im_focus_, ifocus_ = find_focus_sobel(im_stack, rp.bbox, self.increase_depth_of_field) 526 | case _: 527 | raise ValueError('focus_function in pyopia.instrument.holo.Focus not recognised') 528 | 529 | if self.discard_end_slices and (ifocus_ == 0 or ifocus_ == im_stack.shape[2]): 530 | continue 531 | ifocus.append(ifocus_) 532 | rp_out.append(rp) 533 | im_focussed[rp.bbox[0]:rp.bbox[2], rp.bbox[1]:rp.bbox[3]] = im_focus_ 534 | 535 | data['im_focussed'] = 1 - im_focussed 536 | data['stack_rp'] = rp_out 537 | data['stack_ifocus'] = ifocus 538 | return data 539 | 540 | 541 | class MergeStats(): 542 | '''PyOpia pipline-compatible class for merging holo-specific statistics into output stats 543 | 544 | Parameters 545 | ---------- 546 | None 547 | 548 | Returns 549 | ------- 550 | data : :class:`pyopia.pipeline.Data` 551 | Updated pipeline data, where data['stats'] includes the new columns: 'holo_filename', 'z', and 'ifocus' 552 | ''' 553 | 554 | def __init__(self): 555 | pass 556 | 557 | def __call__(self, data): 558 | stats = data['stats'] 559 | stack_rp = data['stack_rp'] 560 | stack_ifocus = data['stack_ifocus'] 561 | 562 | bbox = np.empty((0, 4), int) 563 | for rp in stack_rp: 564 | bbox = np.append(bbox, [rp.bbox], axis=0) 565 | 566 | ifocus = [] 567 | for idx, minr in enumerate(stats.minr): 568 | total_diff = (abs(bbox[:, 0] - stats.minr[idx]) + abs(bbox[:, 1] - stats.minc[idx]) 569 | + abs(bbox[:, 2] - stats.maxr[idx]) + abs(bbox[:, 3] - stats.maxc[idx])) 570 | ifocus.append(stack_ifocus[np.argmin(total_diff)]) 571 | 572 | stats['ifocus'] = np.array(ifocus, dtype=np.int64) 573 | z = (np.arange(data['settings']['steps']['initial']['minZ'], 574 | (data['settings']['steps']['initial']['maxZ'] + data['settings']['steps']['initial']['stepZ']), 575 | data['settings']['steps']['initial']['stepZ'])) 576 | stats['z'] = z[stats['ifocus']-1] 577 | stats['holo_filename'] = data['filename'] 578 | data['stats'] = stats 579 | return data 580 | 581 | 582 | def read_lisst_holo_info(filename): 583 | '''reads the non-image information (timestamp, etc) from LISST-HOLO holograms 584 | 585 | Parameters 586 | ---------- 587 | filename : string 588 | filename to load 589 | 590 | Returns 591 | ------- 592 | timestamp : timestamp 593 | timestamp 594 | ''' 595 | f = open(filename, 'rb') 596 | assert f.readline().decode('ascii').strip() == 'P5' 597 | (width, height, bitdepth) = [int(i) for i in f.readline().split()] 598 | assert bitdepth <= 255 599 | f.seek(width * height, 1) 600 | 601 | timestamp = (pd.to_datetime(struct.unpack('i', f.read(4)), unit='s')) 602 | filenum = filename.rsplit('-', 1)[-1] 603 | filenum = int(filenum.rsplit('.', 1)[0]) 604 | timestamp = timestamp + timedelta(microseconds=filenum) 605 | timestamp = timestamp[0] 606 | logger.info(timestamp.strftime('D%Y%m%dT%H%M%S.%f')) 607 | f.close() 608 | 609 | return timestamp 610 | 611 | 612 | def generate_config(raw_files: str, model_path: str, outfolder: str, output_prefix: str): 613 | '''Generaste example holo config.toml as a dict 614 | 615 | Parameters 616 | ---------- 617 | raw_files : str 618 | raw_files 619 | model_path : str 620 | model_path 621 | outfolder : str 622 | outfolder 623 | output_prefix : str 624 | output_prefix 625 | 626 | Returns 627 | ------- 628 | dict 629 | pipeline_config toml dict 630 | ''' 631 | pipeline_config = { 632 | 'general': { 633 | 'raw_files': raw_files, 634 | 'pixel_size': 4.4 # pixel size in um 635 | }, 636 | 'steps': { 637 | 'initial': { 638 | 'pipeline_class': 'pyopia.instrument.holo.Initial', 639 | 'wavelength': 658, # laser wavelength in nm 640 | 'n': 1.33, # index of refraction of sample volume medium (1.33 for water) 641 | 'offset': 27, # offset to start of sample volume in mm 642 | 'minZ': 0, # minimum reconstruction distance within sample volume in mm 643 | 'maxZ': 50, # maximum reconstruction distance within sample volume in mm 644 | 'stepZ': 0.5 # step size in mm 645 | }, 646 | 'load': { 647 | 'pipeline_class': 'pyopia.instrument.holo.Load' 648 | }, 649 | 'correctbackground': { 650 | 'pipeline_class': 'pyopia.background.CorrectBackgroundAccurate', 651 | 'bgshift_function': 'accurate', 652 | 'average_window': 10 653 | }, 654 | 'reconstruct': { 655 | 'pipeline_class': 'pyopia.instrument.holo.Reconstruct', 656 | 'stack_clean': 0.02, 657 | 'forward_filter_option': 2, 658 | 'inverse_output_option': 0 659 | }, 660 | 'focus': { 661 | 'pipeline_class': 'pyopia.instrument.holo.Focus', 662 | 'stacksummary_function': 'max_map', 663 | 'threshold': 0.9, 664 | 'focus_function': 'find_focus_sobel', 665 | 'increase_depth_of_field': False, 666 | 'merge_adjacent_particles': 2 667 | }, 668 | 'segmentation': { 669 | 'pipeline_class': 'pyopia.process.Segment', 670 | 'threshold': 0.9, 671 | 'segment_source': 'im_focussed' 672 | }, 673 | 'statextract': { 674 | 'pipeline_class': 'pyopia.process.CalculateStats', 675 | 'export_outputpath': outfolder, 676 | 'propnames': ['major_axis_length', 'minor_axis_length', 'equivalent_diameter', 677 | 'feret_diameter_max', 'equivalent_diameter_area'], 678 | 'roi_source': 'im_focussed' 679 | }, 680 | 'mergeholostats': { 681 | 'pipeline_class': 'pyopia.instrument.holo.MergeStats', 682 | }, 683 | 'output': { 684 | 'pipeline_class': 'pyopia.io.StatsToDisc', 685 | 'output_datafile': os.path.join(outfolder, output_prefix) 686 | } 687 | } 688 | } 689 | return pipeline_config 690 | -------------------------------------------------------------------------------- /pyopia/instrument/silcam.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Module containing SilCam specific tools to enable compatability with the :mod:`pyopia.pipeline` 3 | 4 | See: 5 | Davies, E. J., Brandvik, P. J., Leirvik, F., & Nepstad, R. (2017). The use of wide-band transmittance imaging to size and 6 | classify suspended particulate matter in seawater. Marine Pollution Bulletin, 7 | 115(1–2). https://doi.org/10.1016/j.marpolbul.2016.11.063 8 | ''' 9 | 10 | import os 11 | import numpy as np 12 | import pandas as pd 13 | from skimage.exposure import rescale_intensity 14 | import skimage.io 15 | 16 | 17 | def timestamp_from_filename(filename): 18 | '''get a pandas timestamp from a silcam filename 19 | 20 | Parameters 21 | ---------- 22 | filename (string): silcam filename (.silc) 23 | 24 | Returns 25 | ------- 26 | timestamp: timestamp 27 | timestamp from pandas.to_datetime() 28 | ''' 29 | 30 | # get the timestamp of the image (in this case from the filename) 31 | timestamp = pd.to_datetime(os.path.splitext(os.path.basename(filename))[0][1:]) 32 | return timestamp 33 | 34 | 35 | def load_mono8(filename): 36 | '''load a mono8 .msilc file from disc 37 | 38 | Assumes 8-bit mono image in range 0-255 39 | 40 | Parameters 41 | ---------- 42 | filename : string 43 | filename to load 44 | 45 | Returns 46 | ------- 47 | array 48 | raw image float between 0-1 49 | ''' 50 | im_mono = np.load(filename, allow_pickle=False).astype(np.float64) / 255 51 | image_shape = np.shape(im_mono) 52 | if len(image_shape) > 2: 53 | if image_shape[2] == 1: 54 | img = im_mono[:, :, 0] 55 | else: 56 | raise RuntimeError('Invalid image dimension') 57 | return img 58 | 59 | 60 | def load_bayer_rgb8(filename): 61 | '''load an RG8 .bsilc file from disc and convert it to RGB image 62 | 63 | Assumes 8-bit Bayer-RG (Red-Green) image in range 0-255 64 | 65 | Parameters 66 | ---------- 67 | filename : string 68 | filename to load 69 | 70 | Returns 71 | ------- 72 | array 73 | raw image float between 0-1 74 | ''' 75 | img_bayer = np.load(filename, allow_pickle=False).astype(np.int16) 76 | 77 | # Check the image dimension 78 | image_shape = np.shape(img_bayer) 79 | if len(image_shape) > 2: 80 | if image_shape[2] == 1: 81 | img_bayer = img_bayer[:, :, 0] 82 | else: 83 | raise RuntimeError('Invalid image dimension') 84 | 85 | M, N = img_bayer.shape[:2] # Number of pixels in image height and width 86 | img_bayer_min, img_bayer_max = np.min(img_bayer), np.max(img_bayer) 87 | 88 | # img is a reconstructed RGB image 89 | img = np.zeros((M, N, 3), dtype=np.uint8) 90 | img[0:M:2, 0:N:2, 0] = img_bayer[0:M:2, 0:N:2] # Red pixels 91 | img[0:M:2, 1:N:2, 1] = img_bayer[0:M:2, 1:N:2] # Green pixels on the first row 92 | img[1:M:2, 0:N:2, 1] = img_bayer[1:M:2, 0:N:2] # Green pixels on the second row 93 | img[1:M:2, 1:N:2, 2] = img_bayer[1:M:2, 1:N:2] # Blue pixels 94 | 95 | # Boundary pixels interpolation in the first and last rows 96 | # ***Red pixels 97 | img[0, 2:N-1:2, 1] = (img_bayer[1, 2:N-1:2] + img_bayer[0, 1:N-2:2] + img_bayer[0, 3:N:2] + 1) // 3 # Interpolated G 98 | img[0, 2:N-1:2, 2] = (img_bayer[1, 1:N-2:2] + img_bayer[1, 3:N:2] + 1) // 2 # Interpolated B 99 | # ***Green pixels (odd columns) 100 | img[0, 1:N-2:2, 0] = (img_bayer[0, 0:N-3:2] + img_bayer[0, 2:N-1:2] + 1) // 2 # Interpolated R 101 | img[0, 1:N-2:2, 2] = img_bayer[1, 1:N-2:2] # Interpolated B 102 | # ***Blue pixels 103 | img[M-1, 1:N-2:2, 1] = (img_bayer[M-2, 1:N-2:2] 104 | + img_bayer[M-1, 0:N-3:2] + img_bayer[M-1, 2:N-1:2] + 1) // 3 # Interpolated G 105 | img[M-1, 1:N-2:2, 0] = (img_bayer[M-2, 0:N-3:2] + img_bayer[M-2, 2:N-1:2] + 1) // 2 # Interpolated R 106 | # ***Green pixels (even columns) 107 | img[M-1, 2:N-1:2, 2] = (img_bayer[M-1, 1:N-2:2] + img_bayer[M-1, 3:N:2] + 1) // 2 # Interpolated B 108 | img[M-1, 2:N-1:2, 0] = img_bayer[M-2, 2:N-1:2] # Interpolated R 109 | 110 | # Boundary pixels interpolation in the first and last cols 111 | # ***Red pixels 112 | img[2:M-1:2, 0, 1] = (img_bayer[3:M:2, 0] + img_bayer[1:M-2:2, 0] + img_bayer[2:M-1:2, 1] + 1) // 3 # Interpolated G 113 | img[2:M-1:2, 0, 2] = (img_bayer[3:M:2, 1] + img_bayer[1:M-2:2, 1] + 1) // 2 # Interpolated B 114 | # ***Green pixels (odd columns) 115 | img[1:M-2:2, 0, 0] = (img_bayer[0:M-3:2, 0] + img_bayer[2:M-1:2, 0] + 1) // 2 # Interpolated R 116 | img[1:M-2:2, 0, 2] = img_bayer[1:M-2:2, 1] # Interpolated B 117 | # ***Blue pixels 118 | img[1:M-2:2, N-1, 1] = (img_bayer[0:M-3:2, N-1] 119 | + img_bayer[2:M-1:2, N-1] + img_bayer[1:M-2:2, N-2] + 1) // 3 # Interpolated G 120 | img[1:M-2:2, N-1, 0] = (img_bayer[0:M-3:2, N-2] + img_bayer[2:M-1:2, N-2] + 1) // 2 # Interpolated R 121 | # ***Green pixels (even columns) 122 | img[2:M-1:2, N-1, 0] = img_bayer[2:M-1:2, N-2] # Interpolated R 123 | img[2:M-1:2, N-1, 2] = (img_bayer[1:M-2:2, N-1]+img_bayer[3:M:2, N-1] + 1)//2 # Interpolated B 124 | 125 | # Corner pixels interpolation 126 | # *** top-left 127 | img[0, 0, 1] = (img_bayer[1, 0] + img_bayer[0, 1] + 1) // 2 # Interpolated G 128 | img[0, 0, 2] = img_bayer[1, 1] # Interpolated B 129 | # *** top-right 130 | img[0, N-1, 0] = img_bayer[0, N-2] # Interpolated R 131 | img[0, N-1, 2] = img_bayer[1, N-1] # Interpolated B 132 | # *** bottom-left 133 | img[M-1, 0, 0] = img_bayer[M-2, 0] # Interpolated R 134 | img[M-1, 0, 2] = img_bayer[M-1, 1] # Interpolated B 135 | # *** bottom-right 136 | img[M-1, N-1, 1] = (img_bayer[M-2, N-1] + img_bayer[M-1, N-2] + 1) // 2 # Interpolated G 137 | img[M-1, N-1, 0] = img_bayer[M-2, N-2] # Interpolated R 138 | 139 | # Internal pixels interpolation 140 | # ***G pixel on odd row, even column 141 | img[1:M-2:2, 2:N-1:2, 0] = (img_bayer[0:M-3:2, 2:N-1:2] + img_bayer[2:M-1:2, 2:N-1:2] + 1) // 2 # Interpolated R 142 | img[1:M-2:2, 2:N-1:2, 2] = (img_bayer[1:M-2:2, 1:N-2:2] + img_bayer[1:M-2:2, 3:N:2] + 1) // 2 # Interpolated B 143 | # ***G pixel on even row, odd column 144 | img[2:M-1:2, 1:N-2:2, 0] = (img_bayer[2:M-1:2, 0:N-3:2] + img_bayer[2:M-1:2, 2:N-1:2] + 1) // 2 # Interpolated R 145 | img[2:M-1:2, 1:N-2:2, 2] = (img_bayer[1:M-2:2, 1:N-2:2] + img_bayer[3:M:2, 1:N-2:2] + 1) // 2 # Interpolated B 146 | # ***R pixel 147 | img[2:M-1:2, 2:N-1:2, 1] = (img_bayer[1:M-2:2, 2:N-1:2] 148 | + img_bayer[3:M:2, 2:N-1:2] + img_bayer[2:M-1:2, 1:N-2:2] 149 | + img_bayer[2:M-1:2, 3:N:2] + 2) // 4 # Interpolated G 150 | img[2:M-1:2, 2:N-1:2, 2] = (img_bayer[1:M-2:2, 1:N-2:2] 151 | + img_bayer[3:M:2, 1:N-2:2] + img_bayer[3:M:2, 3:N:2] 152 | + img_bayer[1:M-2:2, 3:N:2] + 2) // 4 # Interpolated B 153 | # ***B pixel 154 | img[1:M-2:2, 1:N-2:2, 0] = (img_bayer[0:M-3:2, 0:N-3:2] 155 | + img_bayer[2:M-1:2, 0:N-3:2] + img_bayer[2:M-1:2, 2:N-1:2] 156 | + img_bayer[0:M-3:2, 2:N-1:2] + 2) // 4 # Interpolated R 157 | img[1:M-2:2, 1:N-2:2, 1] = (img_bayer[0:M-3:2, 1:N-2:2] 158 | + img_bayer[2:M-1:2, 1:N-2:2] + img_bayer[1:M-2:2, 0:N-3:2] 159 | + img_bayer[1:M-2:2, 2:N-1:2] + 2) // 4 # Interpolated G 160 | img_min, img_max = np.min(img), np.max(img) 161 | 162 | if img_min < 0 or img_max > 255 or (img_max - img_bayer_max) != 0 or (img_min - img_bayer_min) != 0: 163 | raise ValueError( 164 | "The converted RGB image is not suitable for further analysis. Check the pixel intensity ranges of the input image." 165 | ) 166 | 167 | img = img.astype(np.float64) / 255 168 | return img 169 | 170 | 171 | def load_rgb8(filename): 172 | '''load an RGB .silc file from disc 173 | 174 | Assumes 8-bit RGB image in range 0-255 175 | 176 | Parameters 177 | ---------- 178 | filename : string 179 | filename to load 180 | 181 | Returns 182 | ------- 183 | array 184 | raw image float between 0-1 185 | ''' 186 | img = np.load(filename, allow_pickle=False).astype(np.float64) / 255 187 | return img 188 | 189 | 190 | def load_image(filename): 191 | '''.. deprecated:: 2.4.6 192 | :func:`pyopia.instrument.silcam.load_image` will be removed in version 3.0.0, it is replaced by 193 | :func:`pyopia.instrument.silcam.load_rgb8` because this is more explicit to that image type. 194 | 195 | Load an RGB .silc file from disc 196 | 197 | Parameters 198 | ---------- 199 | filename : string 200 | filename to load 201 | 202 | Returns 203 | ------- 204 | array 205 | raw image float between 0-1 206 | ''' 207 | 208 | return load_rgb8(filename) 209 | 210 | 211 | class SilCamLoad(): 212 | '''PyOpia pipline-compatible class for loading a single silcam image 213 | and extracting the timestamp using 214 | :func:`pyopia.instrument.silcam.timestamp_from_filename` 215 | 216 | Required keys in :class:`pyopia.pipeline.Data`: 217 | - :attr:`pyopia.pipeline.Data.filename` conforming to the format 'DYYYYmmddTHHMMSS.ffffff.silc' 218 | e.g. 'D20240919T074500.183294.silc' 219 | 220 | Parameters 221 | ---------- 222 | image_format : str, optional 223 | Image file format. Can be either 'infer', 'rgb8', 'bayer_rg8' or 'mono8', by default 'infer'. 224 | 225 | Note 226 | ---- 227 | 'infer' uses the file extension to determine the image format using the following convention: 228 | - '.silc' for RGB8 229 | - '.msilc' for MONO8 230 | - '.bsilc' for BAYER_RG8 231 | - '.bmp' for using skimage.io.imread 232 | 233 | Returns 234 | ------- 235 | data : :class:`pyopia.pipeline.Data` 236 | containing the following new keys: 237 | 238 | :attr:`pyopia.pipeline.Data.timestamp` 239 | 240 | :attr:`pyopia.pipeline.Data.img` 241 | ''' 242 | 243 | def __init__(self, image_format='infer'): 244 | self.image_format = image_format 245 | self.extension_load = {'.silc': load_rgb8, 246 | '.msilc': load_mono8, 247 | '.bsilc': load_bayer_rgb8, 248 | '.bmp': lambda filename: skimage.io.imread(filename).astype(np.float64) / 255} 249 | self.format_load = {'RGB8': load_rgb8, 250 | 'MONO8': load_mono8, 'BAYER_RG8': load_bayer_rgb8} 251 | 252 | def __call__(self, data): 253 | data['timestamp'] = timestamp_from_filename(data['filename']) 254 | data['imraw'] = self.load_image(data['filename']) 255 | return data 256 | 257 | def load_image(self, filename): 258 | if self.image_format == 'infer': 259 | file_extension = os.path.splitext(os.path.basename(filename))[-1] 260 | load_function = self.extension_load[file_extension] 261 | else: 262 | load_function = self.format_load[self.image_format] 263 | img = load_function(filename) 264 | return img 265 | 266 | 267 | class ImagePrep(): 268 | '''PyOpia pipline-compatible class for preparing silcam images for further analysis 269 | 270 | Required keys in :class:`pyopia.pipeline.Data`: 271 | - :attr:`pyopia.pipeline.Data.img` 272 | 273 | Returns 274 | ------- 275 | data : :class:`pyopia.pipeline.Data` 276 | containing the following new keys: 277 | 278 | :attr:`pyopia.pipeline.Data.im_minimum` 279 | ''' 280 | def __init__(self, image_level='im_corrected'): 281 | self.image_level = image_level 282 | pass 283 | 284 | def __call__(self, data): 285 | image = data[self.image_level] 286 | 287 | # simplify processing by squeezing the image dimensions into a 2D array 288 | # min is used for squeezing to represent the highest attenuation of all wavelengths 289 | data['im_minimum'] = np.min(image, axis=2) 290 | 291 | data['imref'] = rescale_intensity(image, out_range=(0, 1)) 292 | return data 293 | 294 | 295 | def generate_config(raw_files: str, model_path: str, outfolder: str, output_prefix: str): 296 | '''Generate example silcam config.toml as a dict 297 | 298 | Parameters 299 | ---------- 300 | raw_files : str 301 | raw_files 302 | model_path : str 303 | model_path 304 | outfolder : str 305 | outfolder 306 | output_prefix : str 307 | output_prefix 308 | 309 | Returns 310 | ------- 311 | dict 312 | pipeline_config toml dict 313 | ''' 314 | # define the configuration to use in the processing pipeline - given as a dictionary - with some values defined above 315 | pipeline_config = { 316 | 'general': { 317 | 'raw_files': raw_files, 318 | 'pixel_size': 28 # pixel size in um 319 | }, 320 | 'steps': { 321 | 'classifier': { 322 | 'pipeline_class': 'pyopia.classify.Classify', 323 | 'model_path': model_path 324 | }, 325 | 'load': { 326 | 'pipeline_class': 'pyopia.instrument.silcam.SilCamLoad' 327 | }, 328 | 'imageprep': { 329 | 'pipeline_class': 'pyopia.instrument.silcam.ImagePrep', 330 | 'image_level': 'imraw' 331 | }, 332 | 'segmentation': { 333 | 'pipeline_class': 'pyopia.process.Segment', 334 | 'threshold': 0.85, 335 | 'segment_source': 'im_minimum' 336 | }, 337 | 'statextract': { 338 | 'pipeline_class': 'pyopia.process.CalculateStats', 339 | 'roi_source': 'imref' 340 | }, 341 | 'output': { 342 | 'pipeline_class': 'pyopia.io.StatsToDisc', 343 | 'output_datafile': os.path.join(outfolder, output_prefix) 344 | } 345 | } 346 | } 347 | return pipeline_config 348 | -------------------------------------------------------------------------------- /pyopia/instrument/uvp.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Module containing UVP specific tools to enable compatability with the :mod:`pyopia.pipeline` 3 | ''' 4 | 5 | import os 6 | import numpy as np 7 | import pandas as pd 8 | import skimage.io 9 | 10 | 11 | def timestamp_from_filename(filename): 12 | '''get a pandas timestamp from a UVP vignette image filename 13 | 14 | Parameters 15 | ---------- 16 | filename (string): UVP filename (.png) 17 | 18 | Returns 19 | ------- 20 | timestamp: timestamp from pandas.to_datetime() 21 | ''' 22 | 23 | # get the timestamp of the image (in this case from the filename) 24 | timestr = os.path.split(filename)[-1].strip('.png') 25 | timestamp = pd.to_datetime(timestr) 26 | return timestamp 27 | 28 | 29 | def load_image(filename): 30 | '''load a UVP .png file from disc 31 | 32 | Parameters 33 | ---------- 34 | filename : string 35 | filename to load 36 | 37 | Returns 38 | ------- 39 | array 40 | raw image float between 0-1, inverted so that particles are dark on a light background 41 | ''' 42 | img_darkfield = skimage.io.imread(filename).astype(np.float64) 43 | img_inverted = (255 - img_darkfield) / 255 44 | return img_inverted 45 | 46 | 47 | class UVPLoad(): 48 | '''PyOpia pipline-compatible class for loading a single UVP image 49 | using :func:`pyopia.instrument.uvp.load_image` 50 | and extracting the timestamp using 51 | :func:`pyopia.instrument.uvp.timestamp_from_filename` 52 | 53 | Required keys in :class:`pyopia.pipeline.Data`: 54 | - :attr:`pyopia.pipeline.Data.filename` 55 | 56 | Returns 57 | ------- 58 | data : :class:`pyopia.pipeline.Data` 59 | containing the following new keys: 60 | 61 | :attr:`pyopia.pipeline.Data.timestamp` 62 | 63 | :attr:`pyopia.pipeline.Data.img` 64 | ''' 65 | 66 | def __init__(self): 67 | pass 68 | 69 | def __call__(self, data): 70 | timestamp = timestamp_from_filename(data['filename']) 71 | img = load_image(data['filename']) 72 | data['timestamp'] = timestamp 73 | data['imraw'] = img 74 | return data 75 | 76 | 77 | def generate_config(raw_files: str, model_path: str, outfolder: str, output_prefix: str): 78 | '''Generate example uvp config.toml as a dict 79 | 80 | Parameters 81 | ---------- 82 | raw_files : str 83 | raw_files 84 | model_path : str 85 | model_path 86 | outfolder : str 87 | outfolder 88 | output_prefix : str 89 | output_prefix 90 | 91 | Returns 92 | ------- 93 | dict 94 | pipeline_config toml dict 95 | ''' 96 | # define the configuration to use in the processing pipeline - given as a dictionary - with some values defined above 97 | pipeline_config = { 98 | 'general': { 99 | 'raw_files': raw_files, 100 | 'pixel_size': 80 # pixel size in um 101 | }, 102 | 'steps': { 103 | 'classifier': { 104 | 'pipeline_class': 'pyopia.classify.Classify', 105 | 'model_path': model_path 106 | }, 107 | 'load': { 108 | 'pipeline_class': 'pyopia.instrument.uvp.UVPLoad' 109 | }, 110 | 'segmentation': { 111 | 'pipeline_class': 'pyopia.process.Segment', 112 | 'threshold': 0.95, 113 | 'segment_source': 'imraw' 114 | }, 115 | 'statextract': { 116 | 'pipeline_class': 'pyopia.process.CalculateStats', 117 | 'roi_source': 'imraw' 118 | }, 119 | 'output': { 120 | 'pipeline_class': 'pyopia.io.StatsToDisc', 121 | 'output_datafile': os.path.join(outfolder, output_prefix) 122 | } 123 | } 124 | } 125 | return pipeline_config 126 | -------------------------------------------------------------------------------- /pyopia/pipeline.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Module for managing the PyOpia processing pipeline 3 | 4 | Refer to the :class:`Pipeline` class documentation for examples of how to process datasets and images 5 | ''' 6 | from typing import TypedDict 7 | import time 8 | import datetime 9 | import pandas as pd 10 | from operator import methodcaller 11 | import sys 12 | from pyopia.io import steps_from_xstats as steps_from_xstats # noqa: E(F401) 13 | import logging 14 | from glob import glob 15 | import numpy as np 16 | 17 | logger = logging.getLogger() 18 | 19 | 20 | class Pipeline(): 21 | '''The processing pipeline class 22 | 23 | Note 24 | ---- 25 | The classes called in the Pipeline steps can be modified, and the names of the steps changed. 26 | New steps can be added or deleted as required. 27 | 28 | The classes called in the Pipeline steps need to take a TOML-formatted dictionary as input 29 | and return a dictionary of data as output. This common data dictionary: :class:`pyopia.pipeline.Data` 30 | is therefore passed between steps so that data or variables generated by each step can be passed along the pipeline. 31 | 32 | By default, the step names: `initial`, `classifier`, and `createbackground` 33 | are run when initialising `Pipeline`. 34 | The remaining steps will be run on Pipeline.run(). 35 | You can add initial steps with the optional input `initial_steps`, 36 | which takes a list of strings of the step key names that should only be run on initialisation of the pipeline. 37 | i.e.: `processing_pipeline = pyopia.pipeline.Pipeline(toml_settings, initial_steps=['classifier', 'novel_initial_process'])` 38 | 39 | The step called 'classifier' must return a dict containing: 40 | :attr:`pyopia.pipeline.Data.cl` in order to run successfully. 41 | 42 | :func:`Pipeline.run()` takes a string as input. 43 | This string is put into :class:`pyopia.pipeline.Data`, available to the steps in the pipeline as `data['filename']`. 44 | This is intended for use in looping through several files during processing, so run can be 45 | called multiple times with different filenames. 46 | 47 | Examples 48 | -------- 49 | 50 | Examples of setting up and running a pipeline 51 | can be found for SilCam `here `_, 52 | and holographic analysis `here `_. 53 | 54 | Example config files can be found for SilCam `here `_, 55 | and for holographic analysis `here `_. 56 | 57 | You can check the workflow used by reading the steps from the metadata in the 58 | output file using :func:`pyopia.io.steps_from_xstats` 59 | 60 | More examples and guides can be found on the `PyOIA By Example `_ page. 61 | ''' 62 | 63 | def __init__(self, settings, 64 | initial_steps=['initial', 'classifier', 'createbackground']): 65 | 66 | self.settings = settings 67 | self.stepnames = list(settings['steps'].keys()) 68 | 69 | self.initial_steps = initial_steps 70 | logger.info('Initialising pipeline') 71 | self.data = Data() 72 | self.data['cl'] = None 73 | self.data['settings'] = settings 74 | 75 | # Flag used to control whether remaining pipeline steps should be skipped once it has been set to True 76 | self.data['skip_next_steps'] = False 77 | 78 | self.pass_general_settings() 79 | 80 | for stepname in self.stepnames: 81 | if not self.initial_steps.__contains__(stepname): 82 | continue 83 | self.run_step(stepname) 84 | 85 | def run(self, filename): 86 | '''Method for executing the processing pipeline. 87 | 88 | Parameters 89 | ---------- 90 | filename : str 91 | file to be processed 92 | 93 | Returns 94 | ------- 95 | stats : DataFrame 96 | particle statistics associated with 'filename' 97 | 98 | Note 99 | ---- 100 | The returned stats from this function are single-image only and not appended 101 | if you loop through several filenames! It is recommended to use this step in the pipeline 102 | for properly appending data into NetCDF format when processing several files. 103 | 104 | .. code-block:: toml 105 | 106 | [steps.output] 107 | pipeline_class = 'pyopia.io.StatsDisc' 108 | output_datafile = 'proc/test' # prefix path for output nc file 109 | ''' 110 | 111 | self.data['filename'] = filename 112 | 113 | for stepname in self.stepnames: 114 | if self.initial_steps.__contains__(stepname): 115 | continue 116 | 117 | logger.info(f'Running pipeline step: {stepname}') 118 | t1 = time.time() 119 | self.run_step(stepname) 120 | t2 = time.time() 121 | logger.debug(f'Running pipeline step {stepname} took {t2-t1:.3f} seconds') 122 | 123 | # Check for signal from this step that we should skip remaining pipeline for this image 124 | if self.data['skip_next_steps']: 125 | logger.info('Skipping remaining steps of the pipeline and returning') 126 | 127 | # Reset skip flag 128 | self.data['skip_next_steps'] = False 129 | return 130 | 131 | return 132 | 133 | def run_step(self, stepname): 134 | '''Execute a pipeline step and update the pipeline data 135 | 136 | Parameters 137 | ---------- 138 | stepname : str 139 | Name of the step defined in the settings 140 | ''' 141 | if stepname == 'classifier': 142 | import pyopia.classify # noqa: E(F410) 143 | callobj = self.step_callobj(stepname) 144 | self.data['cl'] = callobj() 145 | else: 146 | callobj = self.step_callobj(stepname) 147 | self.data = callobj(self.data) 148 | 149 | def step_callobj(self, stepname): 150 | '''Generate a callable object for use in run_step() 151 | 152 | Parameters 153 | ---------- 154 | stepname : str 155 | Name of the step defined in the settings 156 | 157 | Returns 158 | ------- 159 | obj 160 | callable object for use in run_step() 161 | ''' 162 | 163 | pipeline_class = self.settings['steps'][stepname]['pipeline_class'] 164 | classname = pipeline_class.split('.')[-1] 165 | modulename = pipeline_class.replace(classname, '')[:-1] 166 | 167 | keys = [k for k in self.settings['steps'][stepname] if k != 'pipeline_class'] 168 | 169 | arguments = dict() 170 | for k in keys: 171 | arguments[k] = self.settings['steps'][stepname][k] 172 | 173 | m = methodcaller(classname, **arguments) 174 | callobj = m(sys.modules[modulename]) 175 | logger.debug(f'{classname} ready with: {arguments} and data: {self.data.keys()}') 176 | return callobj 177 | 178 | def pass_general_settings(self): 179 | self.data['raw_files'] = self.settings['general']['raw_files'] 180 | 181 | def print_steps(self): 182 | '''Print the version number and steps dict (for log_level = DEBUG) 183 | ''' 184 | 185 | # an eventual metadata parser could replace this below printing 186 | # and format into an appropriate standard 187 | logger.info('\n-- Pipeline configuration --\n') 188 | from pyopia import __version__ as pyopia_version 189 | logger.info(f'PyOpia version: {pyopia_version} + \n') 190 | logger.debug(steps_to_string(self.steps)) 191 | logger.info('\n---------------------------------\n') 192 | 193 | 194 | class Data(TypedDict): 195 | '''Data dictionary which is passed between :class:`pyopia.pipeline` steps. 196 | ''' 197 | 198 | raw_files: str 199 | '''String used by glob to obtain file list of data to be processed 200 | This is exracted automatically from 'general.raw_files' in the toml config 201 | during pipeline initialisation. 202 | ''' 203 | imraw: float 204 | '''Raw uncorrected image''' 205 | img: float 206 | '''Deprecatied. Replaced by imraw''' 207 | imc: float 208 | '''Deprecatied. Replaced by im_corrected''' 209 | im_corrected: float 210 | '''Single composite image of focussed particles ready for segmentation 211 | Obtained from e.g. :class:`pyopia.background.CorrectBackgroundAccurate` 212 | ''' 213 | im_minimum: float 214 | '''A 2-d flattened RGB image representing the minmum intensity of all channels 215 | Obtained from e.g. :class:`pyopia.instrument.silcam.ImagePrep`''' 216 | bgstack: float 217 | '''List of images making up the background (either static or moving) 218 | Obtained from :class:`pyopia.background.CorrectBackgroundAccurate` 219 | ''' 220 | imbg: float 221 | '''Background image that can be used to correct :attr:`pyopia.pipeline.Data.imraw` 222 | and calcaulte :attr:`pyopia.pipeline.Data.im_corrected` 223 | Obtained from :class:`pyopia.background.CorrectBackgroundAccurate` 224 | ''' 225 | filename: str 226 | '''Filename string''' 227 | steps_string: str 228 | '''String documenting the steps given to :class:`pyopia.pipeline` 229 | This is put here for documentation purposes, and saving as metadata. 230 | ''' 231 | cl: object 232 | '''classifier object from :class:`pyopia.classify.Classify`''' 233 | timestamp: datetime.datetime 234 | '''timestamp from e.g. :func:`pyopia.instrument.silcam.timestamp_from_filename()`''' 235 | imbw: float 236 | '''Segmented binary image identifying particles from water 237 | Obtained from e.g. :class:`pyopia.process.Segment` 238 | ''' 239 | stats: pd.DataFrame 240 | '''stats DataFrame containing particle statistics of every particle 241 | Obtained from e.g. :class:`pyopia.process.CalculateStats` 242 | ''' 243 | im_stack: float 244 | '''3-d array of reconstructed real hologram images 245 | Obtained from :class:`pyopia.instrument.holo.Reconstruct` 246 | ''' 247 | imss: float 248 | '''Stack summary image used to locate possible particles 249 | Obtained from :class:`pyopia.instrument.holo.Focus` 250 | ''' 251 | im_focussed: float 252 | '''Focussed holographic image''' 253 | imref: float 254 | '''Refereence background corrected image passed to silcam classifier''' 255 | im_masked: float 256 | '''Masked raw image with removed potentially noisy border region before further processsing 257 | Obtained from e.g. :class:`pyopia.instrument.common.RectangularImageMask`''' 258 | 259 | 260 | def steps_to_string(steps): 261 | '''Deprecated. Convert pipeline steps dictionary to a human-readable string 262 | 263 | Parameters 264 | ---------- 265 | steps : dict 266 | pipeline steps dictionary 267 | 268 | Returns 269 | ------- 270 | steps_str : str 271 | human-readable string of the types and variables 272 | ''' 273 | 274 | steps_str = '\n' 275 | for i, key in enumerate(steps.keys()): 276 | steps_str += (str(i + 1) + ') Step: ' + key 277 | + '\n Type: ' + str(type(steps[key])) 278 | + '\n Vars: ' + str(vars(steps[key])) 279 | + '\n') 280 | return steps_str 281 | 282 | 283 | def build_repr(toml_steps, step_name): 284 | '''Build a callable object from settings, which can be used to construct the pipeline steps dict 285 | 286 | Parameters 287 | ---------- 288 | toml_steps : dict 289 | TOML-formatted steps 290 | step_name : str 291 | the key of the TOML-formatted steps which should be use to create a callable object 292 | 293 | Returns 294 | ------- 295 | obj 296 | callable object, useable in a pipeline steps dict 297 | ''' 298 | pipeline_class = toml_steps[step_name]['pipeline_class'] 299 | classname = pipeline_class.split('.')[-1] 300 | modulename = pipeline_class.replace(classname, '')[:-1] 301 | 302 | keys = [k for k in toml_steps[step_name] if k != 'pipeline_class'] 303 | 304 | arguments = dict() 305 | for k in keys: 306 | arguments[k] = toml_steps[step_name][k] 307 | 308 | m = methodcaller(classname, **arguments) 309 | callobj = m(sys.modules[modulename]) 310 | return callobj 311 | 312 | 313 | def build_steps(toml_steps): 314 | '''Build a steps dictionary, ready for pipeline use, from a TOML-formatted steps dict 315 | 316 | Parameters 317 | ---------- 318 | toml_steps : dict 319 | TOML-formatted steps (usually loaded from a config.toml file) 320 | 321 | Returns 322 | ------- 323 | dict 324 | steps dict that is useable by `pyopia.pipeline.Pipeline` 325 | ''' 326 | step_names = list(toml_steps.keys()) 327 | steps = dict() 328 | for step_name in step_names: 329 | steps[step_name] = build_repr(toml_steps, step_name) 330 | 331 | return steps 332 | 333 | 334 | class FilesToProcess: 335 | '''Build file list from glob pattern if specified. 336 | Create FilesToProcess.chunked_files is chunks specified 337 | File list from glob will be sorted. 338 | If a filelist file is specified, load the list from there without sorting. 339 | 340 | Parameters 341 | ---------- 342 | glob_pattern : str, optional 343 | Glob pattern, by default None. If it ends with .txt, interpret as a filelist file. 344 | ''' 345 | def __init__(self, glob_pattern=None): 346 | self.files = None 347 | self.background_files = [] 348 | self.chunked_files = [] 349 | if glob_pattern is not None: 350 | # If a .txt file is specified, this indicates we should get the filelist from there 351 | if glob_pattern.endswith('.txt'): 352 | self.from_filelist_file(glob_pattern) 353 | else: 354 | self.files = sorted(glob(glob_pattern)) 355 | 356 | def from_filelist_file(self, path_to_filelist): 357 | ''' 358 | Initialize explicit list of files to process from a text file. 359 | The text file should contain one path to an image per line, which should be processed in order. 360 | ''' 361 | logger.info(f'Loading explicit image file list from file: {path_to_filelist}') 362 | with open(path_to_filelist, 'r') as fh: 363 | self.files = [line.rstrip() for line in fh.readlines()] 364 | 365 | def to_filelist_file(self, path_to_filelist): 366 | '''Write file list to a txt file 367 | 368 | Parameters 369 | ---------- 370 | path_to_filelist : str 371 | Path to txt file to write 372 | ''' 373 | with open(path_to_filelist, 'w') as fh: 374 | [fh.writelines(L + '\n') for L in self.files] 375 | 376 | with open(path_to_filelist+'.chunks', 'w') as fh: 377 | for i, chunk in enumerate(self.chunked_files): 378 | fh.write(f'--- Chunk {i}, N = {len(chunk)} ---\n') 379 | [fh.writelines(L + '\n') for L in chunk] 380 | 381 | def prepare_chunking(self, num_chunks, average_window, bgshift_function, strategy='block'): 382 | '''Chunk the file list and add initial background files to each chunk 383 | 384 | Parameters 385 | ---------- 386 | num_chunks : int 387 | Number of chunks to produce (must be at least 1) 388 | average_window : int 389 | Number of images to use for background correction 390 | bgshift_function : str 391 | Background update strategy, either `pass` (static background) or `accurate` 392 | strategy : str, optional 393 | Strategy to use for chunking dataset, either `block` or `interleave`. Defult: `block` 394 | ''' 395 | if num_chunks > len(self.files) // 2: 396 | raise RuntimeError('Number of chunks exceeds more than half the number of files to process. Use less chunks.') 397 | self.chunk_files(num_chunks, strategy) 398 | self.build_initial_background_files(average_window=average_window) 399 | self.insert_bg_files_into_chunks(bgshift_function=bgshift_function) 400 | 401 | def chunk_files(self, num_chunks: int, strategy: str = 'block'): 402 | '''Chunk the file list and create FilesToProcess.chunked_files 403 | 404 | Parameters 405 | ---------- 406 | num_chunks : int 407 | number of chunks to produce (must be at least 1) 408 | strategy : str, optional 409 | Strategy to use for chunking dataset, either `block` or `interleave`. Defult: `block` 410 | ''' 411 | if num_chunks < 1: 412 | raise RuntimeError('You must have at least one chunk') 413 | chunk_length = int(np.ceil(len(self.files) / num_chunks)) 414 | if strategy == 'block': 415 | logging.debug('Chunking file list with strategy: block') 416 | self.chunked_files = [self.files[i:i + chunk_length] for i in range(0, len(self.files), chunk_length)] 417 | elif strategy == 'interleave': 418 | logging.debug('Chunking file list with strategy: interleave') 419 | self.chunked_files = [self.files[i::num_chunks] for i in range(0, num_chunks)] 420 | else: 421 | logging.debug(f'Invalid chunking strategy: {strategy}') 422 | raise RuntimeError(f'Unknown strategy: {strategy}. Should be either block or interleave') 423 | 424 | def insert_bg_files_into_chunks(self, bgshift_function='pass'): 425 | average_window = len(self.background_files) 426 | for i, chunk in enumerate(self.chunked_files): 427 | if i > 0 and bgshift_function != 'pass': 428 | # If the bgshift_function is not pass then we need to find a new set of 429 | # background images for the start of next chunk. These will be the last 430 | # average_window number of files from the previous chunk. 431 | # If bgshift_function is 'pass', then we should use the same background files for all chunks 432 | # so there is no need to extend the list of background files here 433 | self.background_files.extend(self.chunked_files[i-1][-average_window:]) 434 | # we have to loop backwards over bg_files because we are inserting into the top of the chunk 435 | chunk = [chunk.insert(0, bg_file) for bg_file in reversed(self.background_files[-average_window:])] 436 | 437 | def build_initial_background_files(self, average_window=0): 438 | '''Create a list of files to use for initializing the background in the first chunk 439 | 440 | Parameters 441 | ---------- 442 | average_window : int, optional 443 | number of images to use in creating a background, by default 0 444 | ''' 445 | self.background_files = [] 446 | for f in self.files[0:average_window]: 447 | self.background_files.append(f) 448 | 449 | def __len__(self): 450 | return len(self.files) 451 | 452 | def __iter__(self): 453 | for filename in self.files: 454 | yield filename 455 | -------------------------------------------------------------------------------- /pyopia/plotting.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Particle plotting functionality for standardised figures 4 | e.g. image presentation, size distributions, montages etc. 5 | """ 6 | 7 | import matplotlib.pyplot as plt 8 | import numpy as np 9 | import seaborn as sns 10 | import pandas as pd 11 | import pyopia.statistics 12 | 13 | 14 | def show_image(image, pixel_size): 15 | """Plots a scaled figure (in mm) of an image 16 | 17 | Parameters 18 | ---------- 19 | image : float 20 | Image (usually a corrected image, such as im_corrected) 21 | pixel_size : float 22 | the pixel size (um) of the imaging system used 23 | """ 24 | r, c = np.shape(image[:, :, 0]) 25 | 26 | plt.imshow( 27 | image, 28 | extent=[0, c * pixel_size / 1000, 0, r * pixel_size / 1000], 29 | interpolation="nearest", 30 | ) 31 | plt.xlabel("mm") 32 | plt.ylabel("mm") 33 | 34 | return 35 | 36 | 37 | def montage_plot(montage, pixel_size): 38 | """ 39 | Plots a SilCam particle montage with a 1mm scale reference 40 | 41 | Parameters 42 | ---------- 43 | montage : uint8 44 | a montage created with scpp.make_montage 45 | pixel_size : float 46 | the pixel size (um) of the imaging system used 47 | """ 48 | msize = np.shape(montage)[0] 49 | ex = pixel_size * np.float64(msize) / 1000.0 50 | 51 | ax = plt.gca() 52 | ax.imshow(montage, extent=[0, ex, 0, ex], cmap="grey") 53 | ax.set_xticks([1, 2], []) 54 | ax.set_xticklabels([" 1mm", ""]) 55 | ax.set_yticks([], []) 56 | ax.xaxis.set_ticks_position("bottom") 57 | 58 | 59 | def classify_rois(roilist, classifier): 60 | """Classify list of single-object images 61 | 62 | If true_class is specified, mark ROIs not matching this class in the figure. 63 | 64 | Parameters 65 | ---------- 66 | roilist: list 67 | List of ROI images to classify 68 | 69 | classifier: pyopia.classify.Classify 70 | Used to classify ROIs 71 | 72 | Returns 73 | ------- 74 | df: pd.DataFrame 75 | Class probabilities for each item in roifiles 76 | """ 77 | 78 | # Get class labels from classifier 79 | class_labels = [f"probability_{cl}" for cl in classifier.class_labels] 80 | 81 | # Classify all ROIs 82 | classify_data = [] 83 | for img in roilist: 84 | prediction = classifier.proc_predict(img).numpy() 85 | classify_data.append(prediction) 86 | df = pd.DataFrame(columns=class_labels, data=classify_data) 87 | 88 | return df 89 | 90 | 91 | def plot_classified_rois(roilist, df_class_labels, true_class=None): 92 | """Plot classified single-object images and show them in a figure grid with classification info 93 | 94 | If true_class is specified, mark ROIs not matching this class in the figure. 95 | 96 | Parameters 97 | ---------- 98 | roilist: list 99 | List of ROI image to classify 100 | 101 | df_class_labels: pd.DataFrame 102 | Class label for each image in roilist in a column named "best guess" 103 | 104 | true_class: str 105 | True class of listed ROIs 106 | 107 | 108 | Returns 109 | ------- 110 | fig: matplotlib figure 111 | ax: matplotlib axes 112 | """ 113 | if "best guess" not in df_class_labels: 114 | raise RuntimeError( 115 | "df_class_clabels must contain column 'best guess'. " 116 | "See e.g. pyopia.statistics.add_best_guesses_to_stats" 117 | ) 118 | 119 | # Get class labels and class index for each ROI 120 | label_maxprob_list = df_class_labels["best guess"].values 121 | class_maxprob_list = pd.Categorical( 122 | df_class_labels["best guess"].values, 123 | categories=[ 124 | cl 125 | for cl in df_class_labels.columns 126 | if cl not in ["best guess", "best guess value"] 127 | ], 128 | ordered=True, 129 | ).codes 130 | 131 | # Set up figure with 15 axes columns 132 | N = len(roilist) 133 | ncols = img_per_col = min(N, 15) 134 | nrows = 1 135 | if N > img_per_col: 136 | ncols = img_per_col 137 | nrows = N // ncols + int((N % img_per_col) > 0) 138 | fig, axes = plt.subplots(nrows, ncols, figsize=(1 * ncols, 1 * nrows)) 139 | 140 | # Plot ROIs in an ncols x nrows grid 141 | colors = sns.color_palette() 142 | for i, (img, ax) in enumerate(zip(roilist, axes.flatten())): 143 | ax.imshow(img) 144 | ax.set_xticks([]) 145 | ax.set_yticks([]) 146 | plt.setp(ax.spines.values(), color=colors[class_maxprob_list[i]], lw=4) 147 | ax.text( 148 | 0, 149 | 1, 150 | f"{class_maxprob_list[i]} {label_maxprob_list[i][:4].upper()}", 151 | ha="left", 152 | va="top", 153 | fontsize=8, 154 | transform=ax.transAxes, 155 | ) 156 | if true_class and (label_maxprob_list[i] != true_class): 157 | ax.text( 158 | 0.5, 159 | 0.5, 160 | "X", 161 | ha="center", 162 | va="center", 163 | fontsize=20, 164 | color="red", 165 | alpha=0.5, 166 | transform=ax.transAxes, 167 | ) 168 | 169 | # Hide non-used axes in the grid 170 | for ax in axes.flatten()[len(roilist):]: 171 | ax.set_visible(False) 172 | 173 | fig.patch.set_linewidth(10) 174 | fig.patch.set_edgecolor("k") 175 | 176 | return fig, ax 177 | 178 | 179 | def classify_plot_class_rois(class_name, classifier, filelist): 180 | """Classify single-object (ROI) images and plot images in a grid with best guess class. 181 | 182 | Parameters 183 | ---------- 184 | class_name: str 185 | Name of class ROI files belong to (e.g. 'copepod') 186 | classifier: pyopia.classify.Classify 187 | PyOPIA classifier instance 188 | filelist: list 189 | List of single-object (ROI) files 190 | 191 | Returns 192 | ------- 193 | df_: pandas.DataFrame 194 | Classification results for each image 195 | """ 196 | # Load single-object images (ROIs) 197 | roilist = [np.float64(plt.imread(f)) / 255.0 for f in filelist] 198 | 199 | # Classify ROIs 200 | df_ = classify_rois(roilist, classifier) 201 | df_ = pyopia.statistics.add_best_guesses_to_stats(df_) 202 | 203 | # Remove "_probability" from labels 204 | df_ = df_.replace("probability_", "", regex=True) 205 | df_.columns = df_.columns.str.replace("probability_", "", regex=True) 206 | 207 | # Plot 208 | fig, ax = plot_classified_rois(roilist, df_, true_class=class_name) 209 | 210 | # Print classification info 211 | num_correct = (df_["best guess"] == class_name).sum() 212 | num_images = len(roilist) 213 | frac_class = num_correct / num_images 214 | print( 215 | f"Correctly identified {class_name} was {100 * frac_class:.1f}% ({num_correct}/{num_images})" 216 | ) 217 | fig.suptitle( 218 | f"Class: {class_name} ({num_correct}/{num_images}, {100 * frac_class:.1f}%)" 219 | ) 220 | 221 | return df_.style.format(precision=1, decimal=".") 222 | -------------------------------------------------------------------------------- /pyopia/simulator/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /pyopia/simulator/silcam.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Module containing tools for assessing statistical reliability of silcam size distributions 3 | ''' 4 | import numpy as np 5 | import skimage.draw 6 | import matplotlib.pyplot as plt 7 | import skimage.util 8 | import pandas as pd 9 | import os 10 | 11 | import pyopia.statistics 12 | import pyopia.plotting 13 | import pyopia.process 14 | import pyopia.instrument.silcam 15 | from pyopia.pipeline import Pipeline 16 | 17 | 18 | class SilcamSimulator(): 19 | '''SilCam simulator 20 | 21 | Parameters 22 | ---------- 23 | total_volume_concentration : int, optional 24 | total volume concentration, by default 1000 25 | d50 : int, optional 26 | median particle size, by default 1000 27 | MinD : int, optional 28 | minimum diameter to simulate, by default 10 29 | PIX_SIZE : int, optional 30 | pixel size (um), by default 28 31 | PATH_LENGTH : int, optional 32 | path length (mm), by default 40 33 | imx : int, optional 34 | image x dimension, by default 2048 35 | imy : int, optional 36 | image y dimension, by default 2448 37 | nims : int, optional 38 | number of images to simulate, by default 50 39 | 40 | Example 41 | ------- 42 | 43 | .. code-block:: python 44 | 45 | from pyopia.simulator.silcam import SilcamSimulator 46 | 47 | sim = SilcamSimulator() 48 | sim.check_convergence() 49 | sim.synthesize() 50 | sim.process_synthetic_image() 51 | sim.plot() 52 | 53 | ''' 54 | def __init__(self, total_volume_concentration=1000, 55 | d50=1000, 56 | MinD=10, 57 | PIX_SIZE=28, 58 | PATH_LENGTH=40, 59 | imx=2048, 60 | imy=2448, 61 | nims=50): 62 | self.total_volume_concentration = total_volume_concentration 63 | self.d50 = d50 64 | self.MinD = MinD 65 | self.PIX_SIZE = PIX_SIZE 66 | self.PATH_LENGTH = PATH_LENGTH 67 | self.imx = imx 68 | self.imy = imy 69 | self.nims = nims 70 | 71 | self.dias, self.bin_limits = pyopia.statistics.get_size_bins() 72 | 73 | # calculate the sample volume of the SilCam specified 74 | self.sample_volume = pyopia.statistics.get_sample_volume(self.PIX_SIZE, 75 | path_length=self.PATH_LENGTH, 76 | imx=self.imx, imy=self.imy) 77 | 78 | self.data = dict() 79 | 80 | def weibull_distribution(self, x): 81 | '''calculate weibull distribution 82 | 83 | Parameters 84 | ---------- 85 | x : array 86 | size bins of input 87 | 88 | Returns 89 | ------- 90 | array 91 | weibull distribution 92 | ''' 93 | a = 2.8 94 | n = self.d50 * 1.5723270440251573 # scaling required for the log-spaced size bins to match the input d50 95 | return (a / n) * (x / n) ** (a - 1) * np.exp(-(x / n) ** a) 96 | 97 | def check_convergence(self): 98 | '''Check statistical convergence of randomly selected size distributions 99 | over the `nims`number of images 100 | 101 | Parameters 102 | ---------- 103 | data['volume_distribution'] : array 104 | volume distribution of shape (nims, dias) 105 | data['cumulative_volume_concentration'] : float 106 | cumulative mean volume concentration of length `nims` 107 | data['cumulative_d50'] : float 108 | cumulative average d50 of length `nims` 109 | ''' 110 | self.data['weibull_x'] = np.linspace(np.min(self.dias), np.max(self.dias), 10000) 111 | self.data['weibull_y'] = self.weibull_distribution(self.data['weibull_x']) 112 | 113 | self.data['volume_distribution_input'] = self.weibull_distribution(self.dias) 114 | self.data['volume_distribution_input'] = self.data['volume_distribution_input'] / \ 115 | np.sum(self.data['volume_distribution_input']) * \ 116 | self.total_volume_concentration # scale the distribution according to concentration 117 | 118 | DropletVolume = ((4 / 3) * np.pi * ((self.dias * 1e-6) / 2) ** 3) # the volume of each droplet in m3 119 | # the number distribution in each bin 120 | self.data['number_distribution'] = self.data['volume_distribution_input'] / (DropletVolume * 1e9) 121 | self.data['number_distribution'][self.dias < self.MinD] = 0 # remove small particles for speed purposes 122 | 123 | # scale the number distribution by the sample volume so resulting units are #/L/bin 124 | self.data['number_distribution'] = self.data['number_distribution'] * self.sample_volume 125 | nc = int(sum(self.data['number_distribution'])) # calculate the total number concentration. must be integer number 126 | 127 | # convert the number distribution to volume distribution in uL/L/bin 128 | vd2 = pyopia.statistics.vd_from_nd(self.data['number_distribution'], self.dias, self.sample_volume) 129 | 130 | # obtain the resulting concentration, now having remove small particles 131 | self.data['initial_volume_concentration'] = sum(vd2) 132 | 133 | # calculate the d50 in um 134 | self.data['d50_theoretical_best'] = pyopia.statistics.d50_from_vd(vd2, self.dias) 135 | 136 | # preallocate variables 137 | self.data['volume_distribution'] = np.zeros((self.nims, len(self.dias))) 138 | self.data['cumulative_volume_concentration'] = np.zeros(self.nims) 139 | self.data['cumulative_d50'] = np.zeros(self.nims) 140 | 141 | for i in range(self.nims): 142 | # randomly select a droplet radius from the input distribution 143 | # radius is in pixels 144 | rad = np.random.choice(self.dias / 2, 145 | size=nc, 146 | p=self.data['number_distribution'] / sum(self.data['number_distribution'])) / self.PIX_SIZE 147 | log_ecd = rad * 2 * self.PIX_SIZE # log this size as a diameter in um 148 | 149 | necd, edges = np.histogram(log_ecd, self.bin_limits) # count particles into number distribution 150 | 151 | # convert to volume distribution 152 | self.data['volume_distribution'][i, :] = pyopia.statistics.vd_from_nd(necd, 153 | self.dias, 154 | sample_volume=self.sample_volume) 155 | 156 | # calculated the cumulate volume distribution over image number 157 | self.data['cumulative_volume_concentration'][i] = np.sum(np.mean(self.data['volume_distribution'][0:i, :], 158 | axis=0)) 159 | 160 | # calcualte the cumulate d50 over image number 161 | self.data['cumulative_d50'][i] = pyopia.statistics.d50_from_vd(np.mean(self.data['volume_distribution'], 162 | axis=0), 163 | self.dias) 164 | 165 | def synthesize(self, add_noise=False, noise_var=0.001, database_path='', database_image_ext='tiff'): 166 | '''Synthesize an image and document the distributions of particles used as input 167 | 168 | Parameters 169 | ---------- 170 | add_noise : bool, optional 171 | Uses skimage.util.random_noise() to add gaussian noise with variance defined by `noise_var`, by default False 172 | noise_var : float, optional 173 | Passed to the var argument of skimage.util.random_noise(), by default 0.001 174 | database_path : str, optional 175 | Path to a folder of particle ROI images to be randomly selected from to build the synthetic image. 176 | If this is an empty string (default), then black discs will be used instead of real images., by default '' 177 | database_image_ext : str, optional 178 | Image file extension to look for within the folder specified by `database_path` 179 | (must be a type that is loadable by skimage.io.imread() e.g. png of tiff), by default 'tiff' 180 | 181 | Parameters 182 | ---------- 183 | data['synthetic_image_data']['image'] : array 184 | synthetic image 185 | data['synthetic_image_data']['input_volume_distribution'] : array 186 | Volume distribution used to create the synthetic image 187 | ''' 188 | 189 | number_concentration = int(sum(self.data['number_distribution'])) # number concentration 190 | 191 | # preallocate the image and logged volume distribution variables 192 | img = np.zeros((self.imx, self.imy, 3), dtype=np.uint8()) + 230 # scale the initial brightness down a bit 193 | log_ecd = np.zeros(number_concentration) 194 | # randomly select a droplet radii from the input distribution 195 | # radius is in pixels 196 | radii = np.random.choice(self.dias / 2, 197 | size=number_concentration, 198 | p=self.data['number_distribution'] / sum(self.data['number_distribution'])) / self.PIX_SIZE 199 | log_ecd = radii * 2 * self.PIX_SIZE # log these sizes as a diameter in um 200 | 201 | if database_path != '': 202 | from pyopia.pipeline import FilesToProcess 203 | file_list = FilesToProcess(os.path.join(database_path, '*.' + database_image_ext)).files 204 | 205 | self.example_images = [] 206 | for radius in radii: 207 | # randomly decide where to put particles within the image 208 | row = np.random.randint(low=radius * 2, high=self.imx - (radius * 2)) 209 | col = np.random.randint(low=radius * 2, high=self.imy - (radius * 2)) 210 | 211 | if database_path != '': 212 | example_image = extract_and_scale_example_image(radius, file_list) 213 | img[row:row + example_image.shape[0], 214 | col:col + example_image.shape[1], 215 | :] = example_image 216 | if np.min(np.shape(example_image[:, :, 0])) > 5: 217 | self.example_images.append(example_image) 218 | else: 219 | rr, cc = skimage.draw.disk((row, col), radius) # make a cirle of the radius selected from the distribution 220 | img[rr, cc, :] = 0 221 | 222 | necd, edges = np.histogram(log_ecd, self.bin_limits) # count the input diameters into a number distribution 223 | # convert to a volume distribution 224 | temporal_volume_distribution = pyopia.statistics.vd_from_nd(necd, self.dias, sample_volume=self.sample_volume) 225 | 226 | if add_noise: 227 | # add some noise to the synthesized image 228 | img = np.uint8(255 * skimage.util.random_noise(np.float64(img) / 255, mode='gaussian', var=noise_var)) 229 | 230 | img = np.uint8(img) # convert to uint8 231 | self.data['synthetic_image_data'] = dict() 232 | self.data['synthetic_image_data']['image'] = img 233 | self.data['synthetic_image_data']['input_volume_distribution'] = temporal_volume_distribution 234 | 235 | def process_synthetic_image(self): 236 | '''Put the synthetic image `data['synthetic_image_data']['image']` through a basic pyopia processing pipeline 237 | 238 | Parameters 239 | ---------- 240 | data['synthetic_image_data']['pyopia_processed_volume_distribution'] : array 241 | pyopia processed volume distribution associated with `dias`size classes 242 | ''' 243 | pipeline_config = { 244 | 'general': { 245 | 'raw_files': '', 246 | 'pixel_size': 28 # pixel size in um 247 | }, 248 | 'steps': { 249 | 'imageprep': { 250 | 'pipeline_class': 'pyopia.instrument.silcam.ImagePrep', 251 | 'image_level': 'im_synthetic' 252 | }, 253 | 'segmentation': { 254 | 'pipeline_class': 'pyopia.process.Segment', 255 | 'threshold': 0.85, 256 | 'segment_source': 'im_minimum' 257 | }, 258 | 'statextract': { 259 | 'pipeline_class': 'pyopia.process.CalculateStats', 260 | 'roi_source': 'im_synthetic' 261 | } 262 | } 263 | } 264 | pipeline = Pipeline(pipeline_config) 265 | pipeline.data['im_synthetic'] = self.data['synthetic_image_data']['image'] 266 | pipeline.data['timestamp'] = pd.Timestamp.now() 267 | pipeline.run('') 268 | dias, vd = pyopia.statistics.vd_from_stats(pipeline.data['stats'], pipeline_config['general']['pixel_size']) 269 | vd /= self.sample_volume 270 | self.data['synthetic_image_data']['pyopia_processed_volume_distribution'] = vd 271 | 272 | def plot(self): 273 | f, a = plt.subplots(2, 2, figsize=(15, 10)) 274 | 275 | plt.sca(a[0, 0]) 276 | pyopia.plotting.show_image(self.data['synthetic_image_data']['image'], self.PIX_SIZE) 277 | plt.title(f'Synthetic image. Path lengh: {self.PATH_LENGTH}') 278 | 279 | plt.sca(a[0, 1]) 280 | plt.plot(self.dias, self.data['volume_distribution'].T, '0.8', alpha=0.2) 281 | plt.plot(-10, 0, '0.8', alpha=0.2, label='Simulated') 282 | plt.plot(self.dias, np.mean(self.data['volume_distribution'].T, axis=1), 'k', label=f'{self.nims} statistical average') 283 | plt.plot(self.dias, self.data['synthetic_image_data']['input_volume_distribution'], 'b', 284 | label='Best possible from synthetic image\n(without occlusion)') 285 | plt.plot(self.dias, self.data['synthetic_image_data']['pyopia_processed_volume_distribution'], 'g', 286 | label='PyOPIA processed from synthetic image\n') 287 | plt.plot(self.dias, self.data['volume_distribution_input'], 'r', label='target') 288 | plt.xscale('log') 289 | plt.xlabel('Diameter [um]') 290 | plt.ylabel('Volume concentration [uL/L]') 291 | plt.legend() 292 | plt.xlim(np.min(self.dias), np.max(self.dias)) 293 | 294 | plt.sca(a[1, 0]) 295 | plt.plot(self.data['cumulative_volume_concentration'], '0.8', label='simulated') 296 | plt.hlines(self.total_volume_concentration, xmin=0, xmax=self.nims, colors='r', label='target') 297 | plt.xlabel('n-images') 298 | plt.ylabel('Volume concentration of n-images [uL/L]') 299 | plt.xlim(0, self.nims) 300 | plt.legend() 301 | 302 | plt.sca(a[1, 1]) 303 | plt.plot(self.data['cumulative_d50'], '0.8', label='simulated') 304 | plt.hlines(self.d50, xmin=0, xmax=self.nims, colors='r', label='target') 305 | plt.xlabel('n-images') 306 | plt.ylabel('D50 over n-images [um]') 307 | plt.xlim(0, self.nims) 308 | plt.legend() 309 | 310 | plt.tight_layout() 311 | 312 | 313 | def extract_and_scale_example_image(output_length, file_list): 314 | '''Randomly select a file from the input file_list list, 315 | load this and then scale it (maintaining aspect ratio) to match the longest x-y dimention 316 | to rad_ number of pixel 317 | 318 | Parameters 319 | ---------- 320 | output_length : float 321 | wanted longest dimention 322 | file_list : list 323 | list of filenames to chose from (to be read with skimage.io.imread) 324 | 325 | Returns 326 | ------- 327 | example_image : array 328 | resized image 329 | ''' 330 | filename = np.random.choice(file_list, size=1)[0] 331 | raw_image = skimage.io.imread(filename).astype(float) 332 | 333 | longest_axis = np.max(raw_image.shape[0:2]) 334 | 335 | scale_factor = output_length * 2 / longest_axis 336 | 337 | size_row = scale_factor * np.float64(raw_image.shape[0]) 338 | size_col = scale_factor * np.float64(raw_image.shape[1]) 339 | 340 | example_image = skimage.transform.resize(raw_image, 341 | (size_row, size_col), 342 | anti_aliasing=True) 343 | 344 | if np.min(np.shape(example_image)) > 0: 345 | example_image += 255 - np.max(example_image) 346 | return example_image 347 | -------------------------------------------------------------------------------- /pyopia/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /pyopia/tests/test_classify.py: -------------------------------------------------------------------------------- 1 | """ 2 | A high level test for the basic processing pipeline. 3 | 4 | """ 5 | 6 | from glob import glob 7 | import tempfile 8 | import os 9 | from tqdm import tqdm 10 | 11 | import pyopia.exampledata as exampledata 12 | import pyopia.io 13 | import pyopia.classify 14 | import pyopia.pipeline 15 | import pyopia.process 16 | import pyopia.statistics 17 | import pyopia.background # noqa: F401 18 | import pandas as pd 19 | import skimage.io 20 | import numpy as np 21 | import pyopia.instrument.silcam 22 | 23 | 24 | ACCURACY = 60 25 | 26 | 27 | def test_match_to_database(): 28 | """ 29 | Basic check of classification prediction against the training database. 30 | Therefore, if correct positive matches are not high percentages, then something is wrong with the prediction. 31 | 32 | @todo include more advanced testing of the classification feks. assert values in a confusion matrix. 33 | """ 34 | 35 | # location of the training data 36 | with tempfile.TemporaryDirectory() as tempdir: 37 | # location of the training data 38 | database_path = os.path.join(tempdir, "silcam_classification_database") 39 | 40 | exampledata.get_classifier_database_from_pysilcam_blob(database_path) 41 | os.makedirs(os.path.join(tempdir, "model"), exist_ok=True) 42 | model_path = exampledata.get_example_model(os.path.join(tempdir, "model")) 43 | 44 | # Load the trained tensorflow model and class names 45 | cl = pyopia.classify.Classify(model_path=model_path) 46 | class_labels = cl.class_labels 47 | 48 | # class_labels should match the training data 49 | classes = sorted(glob(os.path.join(database_path, "*"))) 50 | 51 | # @todo write a quick check that classes and class_labels agree before doing the proper test. 52 | 53 | def correct_positives(category): 54 | """ 55 | calculate the percentage positive matches for a given category 56 | """ 57 | print("Checking", category) 58 | # list the files in this category of the training data 59 | files = glob(os.path.join(database_path, category, "*.tiff")) 60 | 61 | assert len(files) > 50, "less then 50 files in test data." 62 | 63 | # start a counter of incorrectly classified images 64 | failed = 0 65 | time_limit = len(files) * 0.02 66 | t1 = pd.Timestamp.now() 67 | 68 | # loop through the database images 69 | for file in tqdm(files): 70 | img = skimage.io.imread(file) # load ROI 71 | img = np.float64(img) / 255 72 | prediction = cl.proc_predict(img) # run prediction from silcam_classify 73 | 74 | ind = np.argmax(prediction) # find the highest score 75 | 76 | # check if the highest score matches the correct category 77 | if not class_labels[ind] == category: 78 | # if not, the add to the failure count 79 | failed += 1 80 | 81 | # turn failed count into a success percent 82 | success = 100 - (failed / len(files)) * 100 83 | 84 | t2 = pd.Timestamp.now() 85 | td = t2 - t1 86 | assert td < pd.to_timedelta(time_limit, "s"), "Processing time too long." 87 | 88 | return success 89 | 90 | # loop through each category and calculate the success percentage 91 | for cat in classes: 92 | name = os.path.split(cat)[-1] 93 | success = correct_positives(name) 94 | print(name, success) 95 | assert success > ACCURACY, ( 96 | name + " was poorly classified at only " + str(success) + "percent." 97 | ) 98 | 99 | 100 | def test_pipeline_classification(): 101 | """Check that the pipeline doesn't change the outcome of the classification. 102 | Do this by putting rois of know classificion (which we know get correctly classified independintly), 103 | and then use the same model in a pipeline analysing the synthetic image. 104 | """ 105 | 106 | with tempfile.TemporaryDirectory() as tempdir: 107 | # location of the training data 108 | database_path = os.path.join(tempdir, "silcam_classification_database") 109 | 110 | exampledata.get_classifier_database_from_pysilcam_blob(database_path) 111 | os.makedirs("model", exist_ok=True) 112 | model_path = exampledata.get_example_model("model") 113 | 114 | # Load the trained tensorflow model and class names 115 | cl = pyopia.classify.Classify(model_path=model_path) 116 | 117 | def get_good_roi(category): 118 | """ 119 | calculate the percentage positive matches for a given category 120 | """ 121 | 122 | print("category", category) 123 | # list the files in this category of the training data 124 | files = sorted(glob(os.path.join(database_path, category, "*.tiff"))) 125 | print(len(files), "files") 126 | 127 | found_match = 0 128 | # loop through the database images 129 | for file in tqdm(files): 130 | img = np.uint8(skimage.io.imread(file)) # load ROI 131 | img = np.float64(img) / 255 132 | prediction = cl.proc_predict(img) # run prediction from silcam_classify 133 | 134 | if np.max(prediction) < (ACCURACY / 100): 135 | continue 136 | 137 | ind = np.argmax(prediction) # find the highest score 138 | 139 | # check if the highest score matches the correct category 140 | if cl.class_labels[ind] == category: 141 | print("roi file", file) 142 | return img, category 143 | assert found_match == 1, ( 144 | f"classifier not finding matching particle for {category}" 145 | ) 146 | 147 | canvas = np.ones((2048, 2448, 3), np.float64) 148 | 149 | rc_shift = int(2048 / len(cl.class_labels) / 1.5) 150 | rc = rc_shift 151 | 152 | classes = sorted(glob(os.path.join(database_path, "*"))) 153 | 154 | categories = [] 155 | 156 | for cat in classes: 157 | name = os.path.split(cat)[-1] 158 | img, category = get_good_roi(name) 159 | categories.append(category) 160 | img_shape = np.shape(img) 161 | rc += rc_shift 162 | canvas[rc: rc + img_shape[0], rc: rc + img_shape[1], :] = np.float64(img) 163 | 164 | settings = { 165 | "general": {"raw_files": None, "pixel_size": 24}, 166 | "steps": {"note": "non-standard pipeline."}, 167 | } 168 | 169 | # Initialise the pipeline class without running anything 170 | MyPipeline = pyopia.pipeline.Pipeline(settings=settings, initial_steps="") 171 | 172 | # Get the example trained model 173 | model_path = pyopia.exampledata.get_example_model(os.getcwd()) 174 | 175 | # Add the classifier step description to settings (i.e. metadata) 176 | MyPipeline.settings["steps"].update( 177 | { 178 | "classifier": { 179 | "pipeline_class": "pyopia.classify.Classify", 180 | "model_path": model_path, 181 | } 182 | } 183 | ) 184 | 185 | # Execute the classifier step we defined above 186 | MyPipeline.run_step("classifier") 187 | # This is the same as running: 188 | # MyPipeline.data['cl'] = pyopia.classify.Classify(model_path=model_path) 189 | # Note: the classifier step is special in that it's output is specifically data['cl'], rather than other new keys in data 190 | 191 | MyPipeline.data["imraw"] = canvas 192 | MyPipeline.data["timestamp"] = pd.Timestamp.now() 193 | MyPipeline.data["filename"] = "" 194 | 195 | # Add the imageprep step description 196 | MyPipeline.settings["steps"].update( 197 | { 198 | "imageprep": { 199 | "pipeline_class": "pyopia.instrument.silcam.ImagePrep", 200 | "image_level": "imraw", 201 | } 202 | } 203 | ) 204 | # Run the step 205 | MyPipeline.run_step("imageprep") 206 | # This is the same as running: 207 | # ImagePrep = pyopia.instrument.silcam.ImagePrep(image_level='imraw') 208 | # MyPipeline.data = ImagePrep(MyPipeline.data) 209 | 210 | # Add the segmentation step description 211 | MyPipeline.settings["steps"].update( 212 | { 213 | "segmentation": { 214 | "pipeline_class": "pyopia.process.Segment", 215 | "threshold": 1, 216 | "segment_source": "im_minimum", 217 | } 218 | } 219 | ) 220 | # Run the step 221 | MyPipeline.run_step("segmentation") 222 | # This is the same as running: 223 | # Segment = pyopia.process.Segment(threshold=settings['steps']['segmentation']['threshold']) 224 | # data = Segment(data) 225 | 226 | # Add the segmentation step description 227 | MyPipeline.settings["steps"].update( 228 | { 229 | "statextract": { 230 | "pipeline_class": "pyopia.process.CalculateStats", 231 | "roi_source": "imref", 232 | } 233 | } 234 | ) 235 | 236 | # Run the step 237 | MyPipeline.run_step("statextract") 238 | # This is the same as running: 239 | # CalculateStats = pyopia.process.CalculateStats() 240 | # data = CalculateStats(data) 241 | 242 | stats = pyopia.statistics.add_best_guesses_to_stats(MyPipeline.data["stats"]) 243 | 244 | out = [x[12:] for x in stats["best guess"].values] 245 | 246 | print("classes input", categories) 247 | print("classes measured", out) 248 | 249 | assert categories == out, ( 250 | "Classes returned from classifier do not match what was given to the pipeline" 251 | ) 252 | 253 | 254 | if __name__ == "__main__": 255 | test_match_to_database() 256 | test_pipeline_classification() 257 | -------------------------------------------------------------------------------- /pyopia/tests/test_io.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | import pytest 4 | import pandas as pd 5 | from pyopia.io import write_stats, load_stats, get_cf_metadata_spec 6 | from pyopia.instrument.silcam import generate_config 7 | 8 | 9 | pipeline_config = generate_config( 10 | raw_files="images/*.silc", 11 | model_path="None", 12 | outfolder="processed", 13 | output_prefix="test", 14 | ) 15 | 16 | 17 | def test_write_and_load_stats(tmp_path: Path): 18 | # Create a temporary directory for testing 19 | temp_dir = tmp_path / "test_data" 20 | temp_dir.mkdir() 21 | 22 | # Create a sample DataFrame to write 23 | data = { 24 | "major_axis_length": [10.5, 20.3], 25 | "minor_axis_length": [5.2, 10.1], 26 | "equivalent_diameter": [7.8, 15.2], 27 | "saturation": [50.0, 75.0], 28 | } 29 | # Convert timestamp to datetime format 30 | data["timestamp"] = pd.to_datetime(["2025-04-25T10:00:00", "2025-04-25T10:05:00"]) 31 | stats_df = pd.DataFrame(data) 32 | 33 | # Define the output file path 34 | output_file = os.path.join(temp_dir, "test") 35 | 36 | # Provide a minimal valid settings object 37 | config = pipeline_config 38 | 39 | # Write the stats to a NetCDF file 40 | write_stats(stats_df, output_file, settings=config, dataformat="nc", append=True) 41 | 42 | # Check if the file was created 43 | assert os.path.exists(output_file + "-STATS.nc") 44 | 45 | # Load the stats back 46 | loaded_stats = load_stats(output_file + "-STATS.nc") 47 | 48 | # Verify the data matches 49 | for var in data.keys(): 50 | assert var in loaded_stats.data_vars 51 | assert all(loaded_stats[var].values == stats_df[var].values) 52 | 53 | # Verify CF_METADATA attributes 54 | for var, metadata in get_cf_metadata_spec().items(): 55 | if var in loaded_stats.data_vars: 56 | for attr, value in metadata.items(): 57 | assert loaded_stats[var].attrs.get(attr) == value 58 | 59 | # Check that version tag is in the attributes 60 | assert "PyOPIA_version" in loaded_stats.attrs 61 | 62 | 63 | if __name__ == "__main__": 64 | pytest.main() 65 | -------------------------------------------------------------------------------- /pyopia/tests/test_notebooks.py: -------------------------------------------------------------------------------- 1 | """ 2 | A high level test for executing the ipynb notebooks in the notebooks folder. 3 | 4 | Can only be run from top-level directory (i.e. with 'poetry run pytest -v') 5 | 6 | """ 7 | 8 | from nbconvert.preprocessors import ExecutePreprocessor 9 | import nbformat 10 | from pathlib import Path 11 | 12 | 13 | def test_notebooks(): 14 | notebooks = sorted(Path("notebooks/").rglob("*.ipynb")) 15 | notebooks.append("docs/notebooks/background_correction.ipynb") 16 | for notebook_filename in notebooks: 17 | with open(notebook_filename, encoding="utf8") as f: 18 | nb = nbformat.read(f, as_version=4) 19 | ep = ExecutePreprocessor() 20 | print("running", notebook_filename) 21 | ep.preprocess(nb, {"metadata": {"path": "notebooks/"}}) 22 | print(notebook_filename, "complete") 23 | 24 | 25 | if __name__ == "__main__": 26 | test_notebooks() 27 | -------------------------------------------------------------------------------- /pyopia/tests/test_pipeline.py: -------------------------------------------------------------------------------- 1 | ''' 2 | A high level test for the basic processing pipeline. 3 | 4 | ''' 5 | 6 | from glob import glob 7 | import tempfile 8 | import os 9 | import numpy as np 10 | import skimage.io 11 | 12 | import pyopia.exampledata as testdata 13 | import pyopia.io 14 | import pyopia.classify 15 | from pyopia.pipeline import Pipeline 16 | import pyopia.process 17 | import pyopia.statistics 18 | import pyopia.background # noqa: F401 19 | import xarray 20 | 21 | 22 | def test_holo_pipeline(): 23 | ''' 24 | Runs a holo pipeline on a single image with a pre-created background file. 25 | This test is primarily to detect errors when running the pipeline. 26 | 27 | Asserts that the number of particles counted after analysis is as-expected for the settings used in the test 28 | (although based on a course step-size, for speed purposes) 29 | 30 | Note: This does not properly test the background creation, and loads a pre-created background 31 | ''' 32 | import pyopia.instrument.holo # noqa: F401 33 | with tempfile.TemporaryDirectory() as tempdir: 34 | print('tmpdir created:', tempdir) 35 | os.makedirs(tempdir, exist_ok=True) 36 | tempdir_proc = os.path.join(tempdir, 'proc') 37 | os.makedirs(tempdir_proc, exist_ok=True) 38 | 39 | holo_filename, holo_background_filename = testdata.get_example_hologram_and_background(tempdir) 40 | datafile_prefix = os.path.join(tempdir_proc, 'test') 41 | 42 | # define the configuration to use in the processing pipeline - given as a dictionary - with some values defined above 43 | pipeline_config = { 44 | 'general': { 45 | 'raw_files': os.path.join(tempdir, '*.pgm'), 46 | 'pixel_size': 4.4 # pixel size in um 47 | }, 48 | 'steps': { 49 | 'initial': { 50 | 'pipeline_class': 'pyopia.instrument.holo.Initial', 51 | 'wavelength': 658, # laser wavelength in nm 52 | 'n': 1.33, # index of refraction of sample volume medium (1.33 for water) 53 | 'offset': 27, # offset to start of sample volume in mm 54 | 'minZ': 0, # minimum reconstruction distance within sample volume in mm 55 | 'maxZ': 50, # maximum reconstruction distance within sample volume in mm 56 | 'stepZ': 0.5 # step size in mm 57 | }, 58 | 'load': { 59 | 'pipeline_class': 'pyopia.instrument.holo.Load' 60 | }, 61 | 'correctbackground': { 62 | 'pipeline_class': 'pyopia.background.CorrectBackgroundAccurate', 63 | 'bgshift_function': 'accurate', 64 | 'average_window': 1 65 | }, 66 | 'reconstruct': { 67 | 'pipeline_class': 'pyopia.instrument.holo.Reconstruct', 68 | 'stack_clean': 0.02, 69 | 'forward_filter_option': 2, 70 | 'inverse_output_option': 0 71 | }, 72 | 'focus': { 73 | 'pipeline_class': 'pyopia.instrument.holo.Focus', 74 | 'stacksummary_function': 'max_map', 75 | 'threshold': 0.97, 76 | 'focus_function': 'find_focus_sobel', 77 | 'increase_depth_of_field': False, 78 | 'merge_adjacent_particles': 2 79 | }, 80 | 'segmentation': { 81 | 'pipeline_class': 'pyopia.process.Segment', 82 | 'threshold': 0.97, 83 | 'segment_source': 'im_focussed' 84 | }, 85 | 'statextract': { 86 | 'pipeline_class': 'pyopia.process.CalculateStats', 87 | 'export_outputpath': tempdir_proc, 88 | 'propnames': ['major_axis_length', 'minor_axis_length', 'equivalent_diameter', 89 | 'feret_diameter_max', 'equivalent_diameter_area'], 90 | 'roi_source': 'im_focussed' 91 | }, 92 | 'mergeholostats': { 93 | 'pipeline_class': 'pyopia.instrument.holo.MergeStats', 94 | }, 95 | 'output': { 96 | 'pipeline_class': 'pyopia.io.StatsToDisc', 97 | 'output_datafile': datafile_prefix 98 | } 99 | } 100 | } 101 | 102 | processing_pipeline = Pipeline(pipeline_config) 103 | 104 | # Manually initialize the background from a pre-computed and stored image 105 | background_img = skimage.io.imread(holo_background_filename) 106 | processing_pipeline.data['bgstack'] = [background_img] 107 | processing_pipeline.data['imbg'] = np.mean(processing_pipeline.data['bgstack'], axis=0) 108 | 109 | print('Run processing on: ', holo_filename) 110 | processing_pipeline.run(holo_filename) 111 | with xarray.open_dataset(datafile_prefix + '-STATS.nc') as stats: 112 | stats.load() 113 | 114 | print('stats header: ', stats.data_vars) 115 | print('Total number of particles: ', len(stats.major_axis_length)) 116 | assert len(stats.major_axis_length) == 40, ('Number of particles expected in this test is 56 for main' + 117 | ' (or 40 for dev-1.2.)' + 118 | ' This test counted ' + str(len(stats.major_axis_length)) + 119 | ' Something has altered the number of particles detected') 120 | 121 | 122 | def test_silcam_pipeline(): 123 | ''' 124 | Asserts that the number of images counted in the processed hdf5 stats is the same as the 125 | number of images that should have been downloaded for the test. 126 | 127 | This test is primarily to detect errors when running the pipeline. 128 | ''' 129 | import pyopia.instrument.silcam 130 | with tempfile.TemporaryDirectory() as tempdir: 131 | os.makedirs(tempdir, exist_ok=True) 132 | tempdir_proc = os.path.join(tempdir, 'proc') 133 | os.makedirs(tempdir_proc, exist_ok=True) 134 | 135 | filename = testdata.get_example_silc_image(tempdir) 136 | print('filename got:', filename) 137 | 138 | files = glob(os.path.join(tempdir, '*.silc')) 139 | print('file list available for test:') 140 | print(files) 141 | 142 | datafile_prefix = os.path.join(tempdir_proc, 'test') 143 | 144 | pipeline_config = { 145 | 'general': { 146 | 'raw_files': files, 147 | 'pixel_size': 28 # pixel size in um 148 | }, 149 | 'steps': { 150 | 'load': { 151 | 'pipeline_class': 'pyopia.instrument.silcam.SilCamLoad' 152 | }, 153 | 'imageprep': { 154 | 'pipeline_class': 'pyopia.instrument.silcam.ImagePrep', 155 | 'image_level': 'imraw' 156 | }, 157 | 'segmentation': { 158 | 'pipeline_class': 'pyopia.process.Segment', 159 | 'threshold': 0.85, 160 | 'segment_source': 'im_minimum' 161 | }, 162 | 'statextract': { 163 | 'pipeline_class': 'pyopia.process.CalculateStats', 164 | 'roi_source': 'im_minimum' 165 | }, 166 | 'output': { 167 | 'pipeline_class': 'pyopia.io.StatsToDisc', 168 | 'output_datafile': datafile_prefix 169 | } 170 | } 171 | } 172 | 173 | processing_pipeline = Pipeline(pipeline_config) 174 | 175 | for filename in files[:2]: 176 | stats = processing_pipeline.run(filename) 177 | 178 | with xarray.open_dataset(datafile_prefix + '-STATS.nc') as stats: 179 | stats.load() 180 | 181 | print('stats header: ', stats.data_vars) 182 | print('Total number of particles: ', len(stats.major_axis_length)) 183 | num_images = pyopia.statistics.count_images_in_stats(stats) 184 | print('Number of raw images: ', num_images) 185 | assert num_images == 1, ('Number of images expected is 1.' + 186 | 'This test counted' + str(num_images)) 187 | assert len(stats.major_axis_length) == 870, ('Number of particles expected in this test is 870.' + 188 | 'This test counted ' + str(len(stats.major_axis_length)) + 189 | ' Something has altered the number of particles detected') 190 | 191 | 192 | if __name__ == "__main__": 193 | test_holo_pipeline() 194 | test_silcam_pipeline() 195 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "PyOPIA" 3 | dynamic = ["version"] 4 | description = "A Python Ocean Particle Image Analysis toolbox." 5 | authors = [ 6 | { name = "Emlyn Davies", email = "emlyn.davies@sintef.no" }, 7 | { name = "Alex Nimmo Smith@plymouth.ac.uk", email = "alex.nimmo.smith@plymouth.ac.uk" }, 8 | ] 9 | requires-python = "~=3.12" 10 | readme = "README.md" 11 | keywords = [ 12 | "Ocean", 13 | "Particles", 14 | "Imaging", 15 | "Measurement", 16 | "Size distribution", 17 | ] 18 | dependencies = [ 19 | "flake8>=6.1.0,<7", 20 | "numpy>=1.24.0", 21 | "scipy>=1.11.2,<2", 22 | "pytest>=7.2.0", 23 | "imageio>=2.31.3,<3", 24 | "matplotlib>=3.7", 25 | "tqdm>=4.66.1,<5", 26 | "pytest-error-for-skips>=2.0.2,<3", 27 | "nbclient==0.7", 28 | "sphinx==5.0", 29 | "sphinx-rtd-theme>=0.5.0", 30 | "sphinxcontrib-napoleon>=0.7", 31 | "sphinx-togglebutton>=0.3.2,<0.4", 32 | "sphinx-copybutton>=0.5.2,<0.6", 33 | "readthedocs-sphinx-search>=0.3.2,<0.4", 34 | "myst-nb>=0.17.2,<0.18", 35 | "jupyter-book>=0.15.1,<0.16", 36 | "ipykernel>=6.19.4", 37 | "urllib3<2.0", 38 | "gdown>=4.7.1,<5", 39 | "cmocean>=3.0.3,<4", 40 | "toml>=0.10.2,<0.11", 41 | "xarray>=2023.12.0,<2024", 42 | "typer[all]>=0.9.0,<0.10", 43 | "pandas[computation]>=2.1.1,<3", 44 | "h5py>=3.9.0,<4", 45 | "poetry-version-plugin>=0.2.0,<0.3", 46 | "dask>=2024.8.1", 47 | "nbconvert>=7.16.4,<8", 48 | "h5netcdf>= 1.3.0", 49 | "scikit-image>=0.24.0,<0.25", 50 | "click<8.2.0", 51 | "seaborn>=0.13.2", 52 | ] 53 | 54 | [project.optional-dependencies] 55 | 56 | classification = [ 57 | "tensorflow>=2.19.0", 58 | "keras==3.9.1", 59 | ] 60 | 61 | [project.urls] 62 | Repository = "https://github.com/sintef/pyopia" 63 | Documentation = "https://pyopia.readthedocs.io" 64 | 65 | [project.scripts] 66 | pyopia = "pyopia.cli:app" 67 | 68 | [tool.hatch.build.targets.sdist] 69 | include = ["pyopia"] 70 | 71 | [tool.hatch.build.targets.wheel] 72 | include = ["pyopia"] 73 | 74 | [tool.hatch.version] 75 | path = "pyopia/__init__.py" 76 | 77 | [build-system] 78 | requires = ["hatchling"] 79 | build-backend = "hatchling.build" 80 | --------------------------------------------------------------------------------