├── .github └── workflows │ ├── ci.yml │ ├── env.yml │ └── o2.yml ├── .gitignore ├── .gitpod.yml ├── CHANGES.md ├── LICENSE ├── README.md ├── config ├── defaults.yml ├── nf │ ├── O2base.config │ ├── aws.config │ ├── docker.config │ ├── sage.config │ ├── singularity.config │ ├── tma.config │ ├── tower.config │ └── wsi.config └── schema.yml ├── docs ├── .gitignore ├── 404.md ├── CNAME ├── Gemfile ├── _config.yml ├── _includes │ ├── dev.md │ ├── footer_custom.html │ ├── install.md │ └── params.md ├── _sass │ └── color_schemes │ │ ├── lsp.scss │ │ └── mcmicro.scss ├── assets │ ├── css │ │ └── style.scss │ └── images │ │ ├── bg-2.png │ │ ├── hero_background.jpg │ │ └── mcmicro-logo.svg ├── cite │ ├── index.md │ └── publications.md ├── community │ ├── code_of_conduct.md │ └── index.md ├── datasets │ └── index.md ├── how-to-use.md ├── images │ ├── EMIT_TMA22.png │ ├── FAQ-binning.png │ ├── Fig1.png │ ├── Segmentation_crop2.png │ ├── addmod │ │ ├── Step1.png │ │ ├── Step2.png │ │ └── Step3.png │ ├── coreograph-crop.png │ ├── coreograph-probmap.jpg │ ├── coreograph-raw.jpg │ ├── coreograph1.png │ ├── coreograph1av2.png │ ├── coreograph3a.png │ ├── coreograph3b.png │ ├── coreographIntro.png │ ├── coreographbannerv10.png │ ├── cycifoverview.png │ ├── cycifoverview1.png │ ├── cylinter_banner.png │ ├── galaxy-build-collection.png │ ├── galaxy-collection-history.png │ ├── galaxy-inputs-select.png │ ├── galaxy-inputs.png │ ├── galaxy-order-collection.png │ ├── galaxy-run.png │ ├── galaxy-success-invocation.png │ ├── galaxy-wf-select.png │ ├── galaxy-wf-upload.png │ ├── galaxy-wf.png │ ├── hande.png │ ├── if.png │ ├── imgmap_modules.svg │ ├── imgmap_overview.svg │ ├── main-menu-1.png │ ├── main-menu-2.png │ ├── main-menu-3.png │ ├── mcmicro-exemplar-001.jpg │ ├── mcmicro-exemplar-002.jpg │ ├── minerva-examp.png │ ├── modules │ │ ├── SCIMAP.png │ │ ├── ashlar.png │ │ ├── basic.png │ │ ├── coreo.png │ │ ├── cylinter.png │ │ ├── mcquant.png │ │ ├── minerva.png │ │ ├── others.png │ │ ├── s3seg.png │ │ └── unmicst.png │ ├── pipeline-no-microscope-white.png │ ├── pipeline-no-microscope.png │ ├── pipeline-two-rows-v3.png │ ├── scimap.png │ ├── screenshot-after-run.jpg │ ├── segbannerv7.png │ ├── segbannerv8.png │ ├── segmentation1.jpg │ ├── segmentation1.png │ ├── segmentation2.png │ ├── segmentation3.png │ ├── segmentation3b.png │ ├── segmentation3c.png │ ├── segmentation4aa.png │ ├── segmentation4ab.png │ ├── segmentation4bi.png │ ├── segmentation4bii.png │ ├── segmentation4c.png │ ├── segmentation5ci.png │ ├── segmentation5cii.png │ ├── segmentation5i.png │ ├── segmentation5ii.png │ ├── squareoutlines.png │ ├── stitching-art.png │ ├── tutorials │ │ ├── exemplar-001-cycle-06-dfp.png │ │ ├── exemplar-001-cycle-06-ffp.png │ │ ├── exemplar-001-cycle-06.png │ │ ├── exemplar-001-message.PNG │ │ ├── exemplar-001-outlinemerge.PNG │ │ ├── exemplar-002-message.PNG │ │ ├── tma-map.png │ │ └── vizguide.png │ ├── unmicst2.png │ ├── unmicst3.png │ ├── unmicst4.png │ ├── unmicst5.png │ ├── unmicst6.png │ └── unmicstbannerv2.png ├── index.md ├── io.md ├── overview │ ├── exhibit.json │ ├── exp.md │ ├── index.md │ └── pipeline-visual-guide.html ├── parameters │ ├── core.md │ ├── index.md │ ├── other.md │ ├── specs.md │ └── workflow.md ├── platforms │ ├── galaxy │ │ ├── galaxy-running.md │ │ ├── index.md │ │ └── installation.md │ ├── index.md │ ├── run-AWS.md │ ├── run-O2.md │ └── run_tower.md ├── troubleshooting │ ├── O2.md │ ├── faq.md │ ├── index.md │ └── tuning │ │ ├── coreograph.md │ │ ├── index.md │ │ ├── s3seg.md │ │ └── unmicst.md ├── tutorial │ ├── installation.md │ └── tutorial.md └── updates │ ├── future.md │ └── index.md ├── env ├── auto-minerva.Dockerfile ├── dev.Dockerfile └── roadie.Dockerfile ├── exemplar.nf ├── lib ├── mcmicro │ ├── Flow.groovy │ ├── Opts.groovy │ └── Util.groovy └── worker.nf ├── main.nf ├── modules ├── background.nf ├── dearray.nf ├── downstream.nf ├── ext │ └── story.py ├── illumination.nf ├── quantification.nf ├── registration.nf ├── segmentation.nf ├── staging.nf ├── template.nf └── viz.nf ├── nextflow.config ├── nextflow_schema.json ├── roadie.nf ├── roadie ├── scripts │ ├── pyramidize.py │ └── recyze.py └── tasks.yml └── setup ├── O2ext.nf └── cloudformation ├── create-batch.sh ├── create-s3.sh ├── mcmicro-nextflow-batch.yml ├── mcmicro-nextflow-s3.yml ├── parameters-batch.json ├── parameters-s3.json ├── update-batch.sh └── update-s3.sh /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | # Controls when the action will run. 4 | on: 5 | # Triggers the workflow on push or pull request events but only for the master branch 6 | push: 7 | branches: [ master ] 8 | paths-ignore: 9 | - README.md 10 | - CHANGES.md 11 | - .gitpod.yml 12 | - 'docs/**' 13 | - 'setup/**' 14 | - 'env/*' 15 | pull_request: 16 | branches: [ master ] 17 | paths-ignore: 18 | - README.md 19 | - CHANGES.md 20 | - .gitpod.yml 21 | - 'docs/**' 22 | - 'setup/**' 23 | - 'env/*' 24 | 25 | # Allows you to run this workflow manually from the Actions tab 26 | workflow_dispatch: 27 | 28 | jobs: 29 | 30 | # Downloads exemplar data from S3 as needed 31 | setup: 32 | runs-on: ubuntu-latest 33 | steps: 34 | - uses: actions/checkout@v4 35 | - name: Install Nextflow 36 | run: curl -fsSL get.nextflow.io | bash 37 | 38 | # Exemplars will be cached 39 | - name: Exemplar-001 cache 40 | uses: actions/cache@v4 41 | id: cache-ex001 42 | with: 43 | path: ~/data/exemplar-001 44 | key: ex001-2022-02-24 45 | - name: Exemplar-001 (w/ autofluorescence) cache 46 | uses: actions/cache@v4 47 | id: cache-ex001-af 48 | with: 49 | path: ~/data/af/exemplar-001 50 | key: ex001-af-2023-01-14 51 | - name: Exemplar-002 cache 52 | uses: actions/cache@v4 53 | id: cache-ex002 54 | with: 55 | path: ~/data/exemplar-002 56 | key: ex002-2022-02-24 57 | 58 | # Download data only if no cache is available 59 | - name: Exemplar-001 download 60 | if: steps.cache-ex001.outputs.cache-hit != 'true' 61 | run: ./nextflow exemplar.nf --name exemplar-001 --path ~/data 62 | - name: Exemplar-001 (w/ autofluorescence) download 63 | if: steps.cache-ex001-af.outputs.cache-hit != 'true' 64 | run: ./nextflow exemplar.nf --name exemplar-001 --path ~/data/af --from-cycle 1 --to-cycle 3 65 | - name: Exemplar-002 download 66 | if: steps.cache-ex002.outputs.cache-hit != 'true' 67 | run: ./nextflow exemplar.nf --name exemplar-002 --path ~/data --from-cycle 6 --to-cycle 7 68 | 69 | # Test exemplar-001 70 | ex001: 71 | strategy: 72 | matrix: 73 | include: 74 | - params: --viz 75 | upload_artifact: true 76 | - params: --start-at illumination --stop-at registration 77 | - params: --background 78 | - params: --background --background-method imagej-rolling-ball 79 | - params: --segmentation cypository --cypository-opts '--channel 10 --scalingFactor 2' --s3seg-opts '--logSigma 45 300' 80 | - params: --segmentation ilastik,mesmer 81 | - params: --segmentation cellpose --mcquant-opts '--masks *masks.tif' 82 | needs: setup 83 | runs-on: ubuntu-latest 84 | steps: 85 | - uses: actions/checkout@v4 86 | - name: Install Nextflow 87 | run: curl -fsSL get.nextflow.io | bash 88 | - name: Restore exemplar-001 cache 89 | uses: actions/cache@v4 90 | if: ${{ !contains(matrix.params, 'background') }} 91 | with: 92 | path: ~/data/exemplar-001 93 | key: ex001-2022-02-24 94 | - name: Restore exemplar-001 (w/ autofluorescence) cache 95 | uses: actions/cache@v4 96 | if: ${{ contains(matrix.params, 'background') }} 97 | with: 98 | path: ~/data/af/exemplar-001 99 | key: ex001-af-2023-01-14 100 | - name: Adjust data location for the autofluorescence exemplar 101 | if: ${{ contains(matrix.params, 'background') }} 102 | run: mv ~/data/af/exemplar-001 ~/data 103 | - name: Test exemplar-001 with additional parameters 104 | run: | 105 | ./nextflow main.nf --in ~/data/exemplar-001 ${{ matrix.params }} 106 | ls -R ~/data/exemplar-001 107 | - name: Upload processed result 108 | uses: actions/upload-artifact@v4 109 | if: ${{ github.event_name == 'push' && matrix.upload_artifact }} 110 | with: 111 | name: exemplar-001 112 | path: | 113 | ~/data/exemplar-001 114 | !~/data/exemplar-001/raw 115 | !~/data/exemplar-001/illumination 116 | 117 | # Step-by-step test of exemplar-001 118 | ex001-by-step: 119 | needs: setup 120 | runs-on: ubuntu-latest 121 | steps: 122 | - uses: actions/checkout@v4 123 | - name: Install Nextflow 124 | run: curl -fsSL get.nextflow.io | bash 125 | - name: Restore exemplar-001 cache 126 | uses: actions/cache@v4 127 | with: 128 | path: ~/data/exemplar-001 129 | key: ex001-2022-02-24 130 | - name: Test exemplar-001 step-by-step 131 | run: | 132 | ./nextflow main.nf --in ~/data/exemplar-001 --start-at registration --stop-at registration 133 | ./nextflow clean -f last 134 | ./nextflow main.nf --in ~/data/exemplar-001 --start-at segmentation --stop-at segmentation 135 | ./nextflow main.nf --in ~/data/exemplar-001 --start-at watershed --stop-at watershed 136 | ./nextflow main.nf --in ~/data/exemplar-001 --start-at quantification --stop-at quantification 137 | docker rmi -f $(docker images -a -q) 138 | ./nextflow main.nf --in ~/data/exemplar-001 --start-at downstream --stop-at downstream --downstream scimap 139 | ./nextflow main.nf --in ~/data/exemplar-001 --start-at downstream --stop-at downstream --downstream naivestates 140 | ./nextflow main.nf --in ~/data/exemplar-001 --start-at downstream --stop-at downstream --downstream fastpg 141 | ./nextflow main.nf --in ~/data/exemplar-001 --start-at downstream --stop-at downstream --downstream scanpy 142 | ./nextflow main.nf --in ~/data/exemplar-001 --start-at downstream --stop-at downstream --downstream flowsom 143 | ls -R ~/data/exemplar-001 144 | 145 | # One-shot test of exemplar-002 146 | ex002: 147 | needs: setup 148 | runs-on: ubuntu-latest 149 | steps: 150 | - uses: actions/checkout@v4 151 | - name: Install Nextflow 152 | run: curl -fsSL get.nextflow.io | bash 153 | - name: Restore exemplar-002 cache 154 | uses: actions/cache@v4 155 | with: 156 | path: ~/data/exemplar-002 157 | key: ex002-2022-02-24 158 | - name: Test exemplar-002 159 | run: | 160 | ./nextflow main.nf --in ~/data/exemplar-002 --tma --start-at registration --stop-at segmentation --segmentation mesmer 161 | docker rmi -f $(docker images -a -q) 162 | ./nextflow main.nf --in ~/data/exemplar-002 --tma --start-at quantification 163 | ls -R ~/data/exemplar-002 164 | 165 | # Test exemplar-001 with singularity containers 166 | ex001-singularity: 167 | needs: setup 168 | runs-on: ubuntu-latest 169 | steps: 170 | - uses: actions/checkout@v4 171 | - name: Install singularity 172 | uses: singularityhub/install-singularity@main 173 | - name: Install Nextflow 174 | run: curl -fsSL get.nextflow.io | bash 175 | - name: Restore exemplar-001 cache 176 | uses: actions/cache@v4 177 | with: 178 | path: ~/data/exemplar-001 179 | key: ex001-2022-02-24 180 | - name: Test exemplar-001 with singularity containers 181 | run: | 182 | ./nextflow main.nf --in ~/data/exemplar-001 --viz --segmentation ilastik -profile singularity 183 | ls -R ~/data/exemplar-001 184 | -------------------------------------------------------------------------------- /.github/workflows/env.yml: -------------------------------------------------------------------------------- 1 | name: env 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: 7 | - master 8 | paths: 9 | - 'env/*.Dockerfile' 10 | - .github/workflows/env.yml 11 | 12 | env: 13 | REGISTRY: ghcr.io 14 | IMAGE_NAME: ${{ github.repository }} 15 | 16 | jobs: 17 | build: 18 | strategy: 19 | matrix: 20 | environment: 21 | - dev 22 | - roadie 23 | - auto-minerva 24 | runs-on: ubuntu-latest 25 | permissions: 26 | contents: read 27 | packages: write 28 | 29 | steps: 30 | - name: Get current date 31 | id: date 32 | run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV 33 | 34 | - name: Checkout repository 35 | uses: actions/checkout@v3 36 | 37 | - name: Login to GitHub Container Registry 38 | uses: docker/login-action@v2 39 | with: 40 | registry: ${{ env.REGISTRY }} 41 | username: ${{ github.actor }} 42 | password: ${{ secrets.GITHUB_TOKEN }} 43 | 44 | # This step will resolve username capitalization 45 | - name: Extract metadata for Docker 46 | id: meta 47 | uses: docker/metadata-action@v4 48 | with: 49 | images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} 50 | tags: | 51 | ${{ matrix.environment }}-latest 52 | ${{ matrix.environment }}-${{ env.date }} 53 | - name: Build and push Docker image 54 | uses: docker/build-push-action@v3 55 | with: 56 | context: . 57 | file: env/${{ matrix.environment }}.Dockerfile 58 | push: true 59 | tags: ${{ steps.meta.outputs.tags }} 60 | provenance: false 61 | -------------------------------------------------------------------------------- /.github/workflows/o2.yml: -------------------------------------------------------------------------------- 1 | name: O2-CI 2 | 3 | # Controls when the action will run. 4 | on: 5 | 6 | # Allows you to run this workflow manually from the Actions tab 7 | workflow_dispatch: 8 | 9 | jobs: 10 | 11 | test: 12 | runs-on: O2 13 | steps: 14 | - uses: actions/checkout@v3 15 | - name: Test on exemplars 16 | run: | 17 | module load java 18 | nextflow main.nf --in ~/exemplar-001 -profile O2 19 | nextflow main.nf --in ~/exemplar-002 -profile O2 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Nextflow 2 | .nextflow* 3 | work/* 4 | 5 | # Jekyll 6 | _site/ 7 | *-cache/ 8 | .jekyll-metadata 9 | 10 | # Ruby 11 | .bundle/ 12 | .byebug_history 13 | .ruby-gemset 14 | .ruby-version 15 | *.gem 16 | Gemfile.lock 17 | 18 | # Files 19 | .DS_Store 20 | 21 | */.obsidian/ 22 | -------------------------------------------------------------------------------- /.gitpod.yml: -------------------------------------------------------------------------------- 1 | image: ghcr.io/labsyspharm/mcmicro:dev-latest 2 | tasks: 3 | - name: Download default containers 4 | init: | 5 | python -c "import yaml;\ 6 | mcp = yaml.safe_load(open('config/defaults.yml'));\ 7 | mods = mcp['modules'];\ 8 | [print(m['container'] + ':' + m['version']) for m in \ 9 | [mods['registration'], mods['segmentation'][3],\ 10 | mods['watershed'], mods['quantification']]]"\ 11 | | while read line ; do docker pull $line ; done 12 | - name: Default workdir 13 | init: cd /workspace 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Artem Sokolov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) ![Build Status](https://github.com/labsyspharm/mcmicro/actions/workflows/ci.yml/badge.svg) 2 | 3 | # MCMICRO: Multiple-choice microscopy pipeline 4 | 5 | MCMICRO is an end-to-end processing pipeline for multiplexed whole slide imaging and tissue microarrays developed at the [HMS Laboratory of Systems Pharmacology](https://hits.harvard.edu/the-program/laboratory-of-systems-pharmacology/about/). It comprises stitching and registration, segmentation, and single-cell feature extraction. Each step of the pipeline is containerized to enable portable deployment across an array of compute environments. 6 | 7 | The pipeline is described in [Nature Methods](https://www.nature.com/articles/s41592-021-01308-y). Please see [mcmicro.org](https://mcmicro.org/) for documentation, tutorials, benchmark datasets and more. 8 | 9 | ## Quick start 10 | 11 | 1. [Install](http://mcmicro.org/instructions/nextflow/installation.html) nextflow and Docker. Verify installation with `nextflow run hello` and `docker run hello-world` 12 | 1. [Download](http://mcmicro.org/datasets/) exemplar data: `nextflow run labsyspharm/mcmicro/exemplar.nf --name exemplar-001` 13 | 1. [Run](https://mcmicro.org/instructions/nextflow/) mcmicro on the exemplar: `nextflow run labsyspharm/mcmicro --in exemplar-001` 14 | 15 | ## Funding 16 | 17 | This work is supported by the following: 18 | 19 | * NCI grants U54-CA22508U2C-CA233262 and U2C-CA233280 20 | * *NIH grant 1U54CA225088: Systems Pharmacology of Therapeutic and Adverse Responses to Immune Checkpoint and Small Molecule Drugs* 21 | * Ludwig Center at Harvard Medical School and the Ludwig Cancer Research Foundation 22 | * Denis Schapiro was supported by the University of Zurich BioEntrepreneur-Fellowship (BIOEF-17-001) and a Swiss National Science Foundation Early Postdoc Mobility fellowship (P2ZHP3_181475). He is currently a [Damon Runyon Quantitative Biology Fellow](https://www.damonrunyon.org/news/entries/5551/Damon%20Runyon%20Cancer%20Research%20Foundation%20awards%20new%20Quantitative%20Biology%20Fellowships) 23 | * NCI grant [1U24CA274494-01](https://reporter.nih.gov/project-details/10525124): Multi-Consortia Coordinating Center (MC2 Center) for Cancer Biology: Building Interdisciplinary Scientific Communities, Coordinating Impactful Resource Sharing, and Advancing Cancer Research 24 | 25 | [Contributors](https://mcmicro.org/community/) 26 | -------------------------------------------------------------------------------- /config/defaults.yml: -------------------------------------------------------------------------------- 1 | workflow: 2 | start-at: registration 3 | stop-at: quantification 4 | qc-files: inherit 5 | tma: false 6 | viz: false 7 | illumination: true 8 | background: false 9 | background-method: backsub 10 | staging-method: phenoimager2mc 11 | multi-formats: '{.xdce,.nd,.scan,.htd}' 12 | single-formats: '{.ome.tiff,.ome.tif,.rcpnl,.btf,.nd2,.tif,.czi}' 13 | segmentation: unmicst 14 | segmentation-recyze: false 15 | segmentation-max-projection: false 16 | downstream: scimap 17 | options: 18 | ashlar: -m 30 19 | cypository: --model zeisscyto 20 | ilastik: --num_channels 1 21 | mcquant: --masks cell*.tif *_cp_masks*.tif 22 | naivestates: -p png 23 | imagej-rolling-ball: 100 -n=4 -j="-Xmx4g" 24 | modules: 25 | illumination: 26 | name: basic 27 | container: labsyspharm/basic-illumination 28 | version: 1.4.0 29 | registration: 30 | name: ashlar 31 | container: labsyspharm/ashlar 32 | version: 1.18.0 33 | dearray: 34 | name: coreograph 35 | container: labsyspharm/unetcoreograph 36 | version: 2.4.1 37 | cmd: python /app/UNetCoreograph.py --outputPath . 38 | input: --imagePath 39 | staging: 40 | - 41 | name: phenoimager2mc 42 | container: ghcr.io/schapirolabor/phenoimager2mc 43 | version: v0.1.1 44 | cmd: |- 45 | python /phenoimager2mc/scripts/phenoimager2mc.py \ 46 | --indir ${indir} \ 47 | -o ${cycle}.ome.tif 48 | background: 49 | - 50 | name: backsub 51 | container: ghcr.io/schapirolabor/background_subtraction 52 | version: v0.4.1 53 | cmd: |- 54 | python3 /background_subtraction/background_sub.py \ 55 | -r ${image} -m ${marker} \ 56 | -o ${image_id}_backsub.ome.tif -mo markers_bs.csv 57 | - 58 | name: imagej-rolling-ball 59 | container: yuanchen12/imagej-rolling-ball 60 | version: v2023.9.1 61 | cmd: |- 62 | cp ${marker} markers_bs.csv && \ 63 | rolling-ball --imagej_version="/home/mambauser/Fiji.app" ${image} 64 | segmentation: 65 | - 66 | name: unmicst 67 | container: labsyspharm/unmicst 68 | version: 2.7.7 69 | cmd: python /app/unmicstWrapper.py --stackOutput --outputPath . 70 | input: '' 71 | channel: --channel 72 | idxbase: 1 73 | watershed: 'yes' 74 | - 75 | name: cypository 76 | container: labsyspharm/cypository 77 | version: 1.1.5 78 | cmd: python /app/deployMaskRCNN.py --stackOutput --outputPath . 79 | input: '' 80 | channel: --channel 81 | idxbase: 1 82 | watershed: bypass 83 | - 84 | name: ilastik 85 | container: labsyspharm/mcmicro-ilastik 86 | version: 1.6.1 87 | cmd: python /app/mc-ilastik.py --output . 88 | input: --input 89 | model: --model 90 | channel: --channelIDs 91 | idxbase: 1 92 | watershed: 'yes' 93 | - 94 | name: mesmer 95 | container: vanvalenlab/deepcell-applications 96 | version: 0.4.0 97 | cmd: python /usr/src/app/run_app.py mesmer --squeeze --output-directory . --output-name cell.tif 98 | input: --nuclear-image 99 | membrane-input: --membrane-image 100 | channel: --nuclear-channel 101 | idxbase: 0 102 | watershed: 'no' 103 | - 104 | name: cellpose 105 | container: biocontainers/cellpose 106 | version: 2.1.1_cv2 107 | cmd: cellpose --channel_axis 0 --save_tif --savedir . --verbose 108 | input: --image_path 109 | model: --pretrained_model 110 | idxbase: 0 111 | watershed: 'no' 112 | watershed: 113 | name: s3seg 114 | container: labsyspharm/s3segmenter 115 | version: 1.5.6 116 | channel: --probMapChan 117 | idxbase: 1 118 | quantification: 119 | name: mcquant 120 | container: labsyspharm/quantification 121 | version: 1.6.0 122 | downstream: 123 | - 124 | name: naivestates 125 | container: labsyspharm/naivestates 126 | version: 1.7.0 127 | cmd: /app/main.R -o . 128 | input: -i 129 | model: --mct 130 | - 131 | name: scimap 132 | container: labsyspharm/scimap 133 | version: 0.17.7 134 | cmd: scimap-mcmicro -o . 135 | input: '' 136 | - 137 | name: fastpg 138 | container: labsyspharm/mc-fastpg 139 | version: 1.2.3 140 | cmd: python3 /app/cluster.py -c 141 | input: -i 142 | - 143 | name: scanpy 144 | container: labsyspharm/mc-scanpy 145 | version: 1.0.1 146 | cmd: python3 /app/cluster.py -c 147 | input: -i 148 | - 149 | name: flowsom 150 | container: labsyspharm/mc-flowsom 151 | version: 1.0.2 152 | cmd: python3 /app/cluster.py -c 153 | input: -i 154 | viz: 155 | name: autominerva 156 | container: ghcr.io/labsyspharm/mcmicro 157 | version: auto-minerva-2023-08-29 158 | -------------------------------------------------------------------------------- /config/nf/O2base.config: -------------------------------------------------------------------------------- 1 | includeConfig 'singularity.config' 2 | 3 | process{ 4 | executor = 'slurm' 5 | queue = 'short' 6 | cpus = 4 7 | time = '6h' 8 | memory = '64GB' 9 | } 10 | -------------------------------------------------------------------------------- /config/nf/aws.config: -------------------------------------------------------------------------------- 1 | includeConfig 'docker.config' 2 | 3 | process { 4 | executor = 'awsbatch' 5 | queue = 'mcmicro-queue' 6 | } 7 | aws { 8 | region = 'us-east-1' 9 | batch.cliPath = '/home/ec2-user/miniconda/bin/aws' 10 | } 11 | -------------------------------------------------------------------------------- /config/nf/docker.config: -------------------------------------------------------------------------------- 1 | docker.enabled = true 2 | docker.runOptions = '--cpus 0.000 -u root --entrypoint ""' 3 | 4 | params.contPfx = '' 5 | 6 | -------------------------------------------------------------------------------- /config/nf/sage.config: -------------------------------------------------------------------------------- 1 | // Config profile metadata 2 | params { 3 | config_profile_description = 'The Sage Bionetworks Nextflow Config Profile' 4 | config_profile_contact = 'Bruno Grande (@BrunoGrandePhD)' 5 | config_profile_url = 'https://github.com/Sage-Bionetworks-Workflows' 6 | } 7 | 8 | // Leverage us-east-1 mirror of select human and mouse genomes 9 | params { 10 | igenomes_base = 's3://sage-igenomes/igenomes' 11 | cpus = 4 12 | max_cpus = 32 13 | max_memory = 128.GB 14 | max_time = 240.h 15 | single_cpu_mem = 6.GB 16 | } 17 | 18 | // Enable retries globally for certain exit codes 19 | process { 20 | maxErrors = '-1' 21 | maxRetries = 5 22 | errorStrategy = { task.attempt <= 5 ? 'retry' : 'finish' } 23 | } 24 | 25 | // Increase time limit to allow file transfers to finish 26 | // The default is 12 hours, which results in timeouts 27 | threadPool.FileTransfer.maxAwait = '24 hour' 28 | 29 | // Configure Nextflow to be more reliable on AWS 30 | aws { 31 | region = "us-east-1" 32 | client { 33 | uploadMaxThreads = 4 34 | } 35 | batch { 36 | retryMode = 'built-in' 37 | maxParallelTransfers = 1 38 | maxTransferAttempts = 10 39 | delayBetweenAttempts = '60 sec' 40 | } 41 | } 42 | 43 | // Adjust default resource allocations (see `../docs/sage.md`) 44 | process { 45 | 46 | cpus = { check_max( 1 * factor(task, 2), 'cpus' ) } 47 | memory = { check_max( 6.GB * factor(task, 1), 'memory' ) } 48 | time = { check_max( 24.h * factor(task, 1), 'time' ) } 49 | 50 | // Process-specific resource requirements 51 | withLabel: 'process_single' { 52 | cpus = { check_max( 1 * factor(task, 2), 'cpus' ) } 53 | memory = { check_max( 6.GB * factor(task, 1), 'memory' ) } 54 | time = { check_max( 24.h * factor(task, 1), 'time' ) } 55 | } 56 | withLabel: 'process_low' { 57 | cpus = { check_max( 2 * factor(task, 2), 'cpus' ) } 58 | memory = { check_max( 12.GB * factor(task, 1), 'memory' ) } 59 | time = { check_max( 24.h * factor(task, 1), 'time' ) } 60 | } 61 | withLabel: 'process_medium' { 62 | cpus = { check_max( 8 * factor(task, 2), 'cpus' ) } 63 | memory = { check_max( 32.GB * factor(task, 1), 'memory' ) } 64 | time = { check_max( 48.h * factor(task, 1), 'time' ) } 65 | } 66 | withLabel: 'process_high' { 67 | cpus = { check_max( 16 * factor(task, 2), 'cpus' ) } 68 | memory = { check_max( 64.GB * factor(task, 1), 'memory' ) } 69 | time = { check_max( 96.h * factor(task, 1), 'time' ) } 70 | } 71 | withLabel: 'process_long' { 72 | time = { check_max( 96.h * factor(task, 1), 'time' ) } 73 | } 74 | withLabel: 'process_high_memory|memory_max' { 75 | memory = { check_max( 128.GB * factor(task, 1), 'memory' ) } 76 | } 77 | withLabel: 'cpus_max' { 78 | cpus = { check_max( 32 * factor(task, 2), 'cpus' ) } 79 | } 80 | 81 | } 82 | 83 | 84 | // Function to finely control the increase of the resource allocation 85 | def factor(task, slow_factor = 1) { 86 | if ( task.exitStatus in [143,137,104,134,139,247] ) { 87 | return Math.ceil( task.attempt / slow_factor) as int 88 | } else { 89 | return 1 as int 90 | } 91 | } 92 | 93 | 94 | // Function to ensure that resource requirements don't go 95 | // beyond a maximum limit (copied here for Sarek v2) 96 | def check_max(obj, type) { 97 | if (type == 'memory') { 98 | try { 99 | if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1) 100 | return params.max_memory as nextflow.util.MemoryUnit 101 | else 102 | return obj 103 | } catch (all) { 104 | println " ### ERROR ### Max memory '${params.max_memory}' is not valid! Using default value: $obj" 105 | return obj 106 | } 107 | } else if (type == 'time') { 108 | try { 109 | if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1) 110 | return params.max_time as nextflow.util.Duration 111 | else 112 | return obj 113 | } catch (all) { 114 | println " ### ERROR ### Max time '${params.max_time}' is not valid! Using default value: $obj" 115 | return obj 116 | } 117 | } else if (type == 'cpus') { 118 | try { 119 | return Math.min( obj, params.max_cpus as int ) 120 | } catch (all) { 121 | println " ### ERROR ### Max cpus '${params.max_cpus}' is not valid! Using default value: $obj" 122 | return obj 123 | } 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /config/nf/singularity.config: -------------------------------------------------------------------------------- 1 | singularity.enabled = true 2 | singularity.autoMounts = true 3 | singularity.runOptions = '-C -H "$PWD"' 4 | 5 | params.contPfx = 'docker://' 6 | 7 | -------------------------------------------------------------------------------- /config/nf/tma.config: -------------------------------------------------------------------------------- 1 | process { 2 | withName:illumination { 3 | time = '6h' 4 | memory = '64G' 5 | } 6 | withName:coreograph { 7 | time = '12h' 8 | memory = '128G' 9 | } 10 | withName:ashlar { 11 | time = '12h' 12 | memory = '64G' 13 | } 14 | withName:s3seg { 15 | time = '1h' 16 | memory = '4G' 17 | } 18 | withName:mcquant { 19 | time = '1h' 20 | memory = '16G' 21 | } 22 | withName:worker { 23 | cpus = 4 24 | time = '1h' 25 | memory = '32G' 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /config/nf/tower.config: -------------------------------------------------------------------------------- 1 | process { 2 | cpus = {4 * task.attempt} 3 | memory = {16.GB * task.attempt} 4 | maxRetries = 3 5 | errorStrategy = 'retry' 6 | } 7 | -------------------------------------------------------------------------------- /config/nf/wsi.config: -------------------------------------------------------------------------------- 1 | process { 2 | withName:illumination { 3 | time = '6h' 4 | memory = '64G' 5 | } 6 | withName:coreograph { 7 | time = '12h' 8 | memory = '128G' 9 | } 10 | withName:ashlar { 11 | time = '6h' 12 | memory = '64G' 13 | } 14 | withName:s3seg { 15 | cpus = 6 16 | time = '12h' 17 | memory = '230G' 18 | } 19 | withName:mcquant { 20 | time = '12h' 21 | memory = '128G' 22 | } 23 | withName:worker { 24 | cpus = 4 25 | time = '12h' 26 | memory = '230G' 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /config/schema.yml: -------------------------------------------------------------------------------- 1 | workflow: 2 | - sample-name 3 | - start-at 4 | - stop-at 5 | - qc-files 6 | - tma 7 | - viz 8 | - multi-formats 9 | - single-formats 10 | - illumination 11 | - segmentation 12 | - segmentation-channel 13 | - segmentation-recyze 14 | - segmentation-nuclear-channel 15 | - segmentation-membrane-channel 16 | - segmentation-max-projection 17 | - downstream 18 | - ilastik-model 19 | - mesmer-model 20 | - background 21 | - staging-method 22 | - cellpose-model 23 | - background-method 24 | - flowsom-model 25 | - scanpy-model 26 | - fastpg-model 27 | - naivestates-model 28 | deprecated: 29 | quantification-mask: "--quant-opts '--masks ...'" 30 | illum: --start-at illumination 31 | core-opts: --coreograph-opts 32 | mask-spatial: "--quant-opts '--masks ...'" 33 | mask-add: "--quant-opts '--masks ...'" 34 | nstates-opts: --naivestates-opts 35 | quant-opts: --mcquant-opts 36 | cell-states: --downstream 37 | probability-maps: --segmentation 38 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | _site 2 | Gemfile 3 | Gemfile.lock 4 | */.obsidian/ 5 | -------------------------------------------------------------------------------- /docs/404.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Page not found 4 | nav_exclude: true 5 | search_exclude: true 6 | permalink: /404.html 7 | --- 8 | 9 | # Page not found 10 | 11 | The requested page does not exist on this site. 12 | 13 | {% if site.search_enabled != false %} 14 | Please try the site search box above to find what you are looking for. 15 | {% endif %} 16 | -------------------------------------------------------------------------------- /docs/CNAME: -------------------------------------------------------------------------------- 1 | mcmicro.org -------------------------------------------------------------------------------- /docs/Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | # Hello! This is where you manage which Jekyll version is used to run. 3 | # When you want to use a different version, change it below, save the 4 | # file and run `bundle install`. Run Jekyll with `bundle exec`, like so: 5 | # 6 | # bundle exec jekyll serve 7 | # 8 | # This will help ensure the proper Jekyll version is running. 9 | # Happy Jekylling! 10 | # gem "jekyll", "~> 4.2.0" 11 | # This is the default theme for new Jekyll sites. You may change this to anything you like. 12 | gem "minima", "~> 2.5" 13 | # If you want to use GitHub Pages, remove the "gem "jekyll"" above and 14 | # uncomment the line below. To upgrade, run `bundle update github-pages`. 15 | gem "github-pages", "~>215", group: :jekyll_plugins 16 | # If you have any plugins, put them here! 17 | group :jekyll_plugins do 18 | gem "jekyll-feed", "~> 0.12" 19 | gem "jekyll-include-cache" 20 | end 21 | 22 | # Windows and JRuby does not include zoneinfo files, so bundle the tzinfo-data gem 23 | # and associated library. 24 | platforms :mingw, :x64_mingw, :mswin, :jruby do 25 | gem "tzinfo", "~> 1.2" 26 | gem "tzinfo-data" 27 | end 28 | 29 | # Performance-booster for watching directories on Windows 30 | gem "wdm", "~> 0.1.1", :platforms => [:mingw, :x64_mingw, :mswin] 31 | 32 | -------------------------------------------------------------------------------- /docs/_config.yml: -------------------------------------------------------------------------------- 1 | title: MCMICRO 2 | description: Multiple-choice microscopy pipeline 3 | logo: "/assets/images/mcmicro-logo.svg" 4 | hero_background: "/assets/images/hero_background.jpg" 5 | 6 | remote_theme: labsyspharm/just-the-docs-lsp 7 | color_scheme: mcmicro 8 | search_enabled: true 9 | heading_anchors: true 10 | 11 | # Banner links to include 12 | banner_links: 13 | lsp: true 14 | hits: true 15 | 16 | # Aux links for the upper right navigation 17 | aux_links: 18 | "mcmicro on GitHub": 19 | - "//github.com/labsyspharm/mcmicro" 20 | aux_links_new_tab: false 21 | 22 | 23 | # Footer content 24 | # appears at the bottom of every page's main content 25 | footer_content: 26 | 27 | # Back to top link 28 | back_to_top: false 29 | back_to_top_text: "Back to Top" 30 | 31 | # Provide license information for the project 32 | license: 33 | - description: "MCMICRO source code is licensed under" 34 | name: "MIT License" 35 | url: "https://github.com/labsyspharm/mcmicro/blob/master/LICENSE" 36 | - description: "Contents of this site is licensed under" 37 | name: Creative Commons License 38 | url: "https://creativecommons.org/licenses/by-nc/4.0/" 39 | 40 | # Linked logos 41 | footer_logos: 42 | - name: "Laboratory of Systems Pharmacology" 43 | image: "/assets/images/logo_lsp_white.svg" 44 | url: "https://labsyspharm.org/" 45 | - name: "Harvard Medical School" 46 | image: "/assets/images/logo_hms.svg" 47 | url: "https://hms.harvard.edu/" 48 | 49 | # Footer last edited timestamp 50 | last_edit_timestamp: true # show or hide edit time - page must have `last_modified_date` defined in the frontmatter 51 | last_edit_time_format: "%b %e %Y" # format: https://ruby-doc.org/stdlib-2.7.0/libdoc/time/rdoc/Time.html 52 | 53 | # Footer "Edit this page on GitHub" link text 54 | gh_edit_link: true # show or hide edit this page link 55 | gh_edit_link_text: "Edit this page on GitHub." 56 | gh_edit_repository: "https://github.com/labsyspharm/mcmicro" # the github URL for your repo 57 | gh_edit_branch: "master" # the branch that your docs is served from 58 | gh_edit_source: docs # the source that your files originate from 59 | gh_edit_view_mode: "tree" # "tree" or "edit" if you want the user to jump into the editor immediately 60 | 61 | # Google Analytics 62 | ga_tracking: G-EBP7DZSHEP 63 | ga_tracking_anonymize_ip: true # Use GDPR compliant Google Analytics settings 64 | 65 | plugins: 66 | - jekyll-seo-tag 67 | - jekyll-remote-theme 68 | - jekyll-include-cache 69 | - jekyll-redirect-from 70 | 71 | repository: labsyspharm/mcmicro 72 | 73 | exclude: ["node_modules/", "*.gemspec", "*.gem", "Gemfile", "Gemfile.lock", "package.json", "package-lock.json", "script/", "LICENSE.txt", "lib/", "bin/", "README.md", "Rakefile"] 74 | compress_html: 75 | clippings: all 76 | comments: all 77 | endings: all 78 | startings: [] 79 | blanklines: false 80 | profile: false 81 | -------------------------------------------------------------------------------- /docs/_includes/dev.md: -------------------------------------------------------------------------------- 1 | # For developers 2 | 3 | ## Updating modules with new versions 4 | 5 | The versions of individual modules are pinned for standard pipeline runs. When a new version of a particular module becomes available: 6 | 7 | 1. Increment the corresponding version in `nextflow.config`. 8 | 1. Submit a pull request (PR) and resolve any issues raised in the automated Travis-CI tests. 9 | -------------------------------------------------------------------------------- /docs/_includes/footer_custom.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/_includes/footer_custom.html -------------------------------------------------------------------------------- /docs/_includes/install.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | ## Install Nextflow 4 | 5 | If not already installed, install Java: [https://adoptopenjdk.net/](https://adoptopenjdk.net/) 6 | 7 | Install [Nextflow](https://www.nextflow.io/): `curl -s https://get.nextflow.io | bash` 8 | 9 | This command will create a `nextflow` executable in the current directory. To simplify usage, consider moving this executable to a directory that is available on `$PATH`. One common place for this is a `bin/` directory in your home folder: 10 | 11 | ``` bash 12 | mkdir -p ~/bin # Creates a bin directory in the home folder 13 | mv nextflow ~/bin # Moves nextflow to that directory 14 | echo 'export PATH="$HOME/bin:$PATH"' >> ~/.bashrc # Make the directory accessible on $PATH 15 | source ~/.bashrc # Reload the shell configuration 16 | ``` 17 | 18 | Verify that Nextflow is accessible by going to your home directory (`cd ~`) and typing `nextflow` on the command line. 19 | 20 | ## Install Docker 21 | 22 | Install [Docker](https://docs.docker.com/install/). Ensure that the Docker engine is running by typing `docker images`. If the engine is running, it should return a (possibly empty) list of container images currently downloaded to your system. 23 | -------------------------------------------------------------------------------- /docs/_includes/params.md: -------------------------------------------------------------------------------- 1 | # Parameter reference 2 | 3 | ## Parameters controlling the pipeline behavior 4 | 5 | The following parameters control the pipeline as a whole. These can be specified on the command line using the double-dash format (e.g., `--in`), or inside a YAML file as key-value pairs. Parameters that don't require an explicit value because their presence controls the behavior (e.g., `--tma`) should instead be assigned to `true` in the YAML file. For example, 6 | 7 | Example: `nextflow run labsyspharm/mcmicro --in /my/data --tma` 8 | 9 | or equivalently: `nextflow run labsyspharm/mcmicro -params-file myparams.yml`, where `myparams.yml` contains 10 | ``` 11 | in: /my/data 12 | tma: true 13 | ``` 14 | 15 | ### Mandatory parameters: 16 | 17 | | Parameter | Description | 18 | | --- | --- | 19 | | `--in /local/path` | Location of the data | 20 | 21 | ### Optional parameters: 22 | 23 | | Parameter | Default | Description | 24 | | --- | --- | --- | 25 | | `--sample-name ` | Directory name supplied to `--in` | The name of the experiment/specimen | 26 | | `--start-at ` | `registration` | Name of the first step to be executed by the pipeline. Must be one of `illumination`, `registration`, `dearray` (TMA only), `probability-maps`, `segmentation`, `quantification`, `cell-states` | 27 | | `--stop-at ` | `quantification` | Name of the final step to be executed by the pipeline. Spans the same vocabulary as `--start-at`. | 28 | | `--tma` | Omitted | If specified, mcmicro treats input data as a TMA. If omitted, the input is assumed to be a whole-slide image. | 29 | | `--background` | Omitted | If specified, mcmicro treats input data as if autofluorescence channels should be subtracted. | 30 | | `--ilastik-model ` | None | A custom `.ilp` file to be used as the classifier model for ilastik. | 31 | | `--probability-maps ` | `unmicst` | Which module(s) to use for probability map computation. Must be one of `unmicst`, `ilastik`, `all` (`unmicst` AND `ilastik`), and `cypository` for cytoplasm segmentation | 32 | 33 | ## Parameters for individual modules 34 | 35 | Module-specific parameters can be specified using the various `opts` arguments, followed by the parameters enclosed inside single quotes `'`: 36 | 37 | Example 1: `nextflow run labsyspharm/mcmicro --in /my/data --ashlar-opts '-m 30 --pyramid'` 38 | 39 | Example 2: `nextflow run labsyspharm/mcmicro --in /my/data --nstates-opts '--log no --plots pdf'` 40 | 41 | Example 3: `nextflow run labsyspharm/mcmicro --in /my/data --quant-opts '--masks cytoMask.tif nucleiMask.tif'` 42 | 43 | ### Arguments to ASHLAR (`--ashlar-opts`): 44 | 45 | Up-to-date list can be viewed at [https://github.com/labsyspharm/ashlar](https://github.com/labsyspharm/ashlar) 46 | 47 | ### Arguments to Coreograph(`--core-opts`): 48 | 49 | Up-to-date list can be viewed at [https://github.com/HMS-IDAC/UNetCoreograph](https://github.com/HMS-IDAC/UNetCoreograph) 50 | 51 | ### Arguments to UnMicst(`--unmicst-opts`): 52 | 53 | | Parameter | Default | Description | 54 | | --- | --- | --- | 55 | | `--tool ` | `unmicst-solo` | UnMicst version: *unmicst-legacy* is the old single channel model. *unmicst-solo* uses DAPI. *unmicst-duo* uses DAPI and lamin. | 56 | | `--model` | human nuclei from DAPI | The name of the UNet model. By default, this is the human nuclei model that identifies nuclei centers, nuclei contours, and background from a DAPI channel. Other models include mouse nuclei from DAPI, and cytoplasm from stains resembling WGA | 57 | | `--channel ` | `1` | The channel used to infer and generate probability maps from. If using UnMicst2, then specify 2 channels. If only 1 channel is specified, it will simply be duplicated. **NOTE: If not using default value, the 1st channel must be specified to S3segmenter as --probMapChan in --s3seg-opts**| 58 | | `--classOrder` | None | If your training data isn't in the order 1. background, 2. contours, 3. foreground, you can specify the order here. For example, if you had trained the class order backwards, specify `--classOrder 3 2 1`. If you only have background and contours, use `--classOrder 1 2 1`. | 59 | | `--mean ` | Extracted from the model | Override the trained model's mean intensity. Useful if your images are significantly dimmer or brighter. | 60 | | `--std ` | Extracted from the model | Override the trained model's standard deviation intensity. Useful if your images are significantly dimmer or brighter. | 61 | | `--scalingFactor ` | `1` | An upsample or downsample factor used to resize the image. Useful when the pixel sizes of your image differ from the model (ie. 0.65 microns/pixel for human nuclei model) | 62 | | `--stackOutput` | Specified | If selected, UnMicst will write all probability maps as a single multipage tiff file. Otherwise, UnMicst will write each class as a separate file. | 63 | | `--GPU ` | Automatic | Explicitly specify which GPU (1-based indexing) you want to use. Useful for running on local workstations with multiple GPUs. | 64 | 65 | ### Arguments to Ilastik(`--ilastik-opts`): 66 | 67 | | Parameter | Default | Description | 68 | | --- | --- | --- | 69 | | `--nonzero_fraction ` |`None` | Indicates fraction of pixels per crop above global threshold to ensure tissue and not only background is selected | 70 | | `--nuclei_index ` |`1` | Index of nuclei channel to use for nonzero_fraction argument | 71 | | `--crop` | Omitted | If specified, crop regions for ilastik training | 72 | | `--num_channels ` | `None`| Number of channels to export per image (Ex: 40 corresponds to a 40 channel ome.tif image) | 73 | | `--channelIDs ` |`None` | Integer indices specifying which channels to export (Ex: 1 2 4). **NOTE: You must specify a channel to use for filtering in S3segmenter as --probMapChan in --s3seg-opts**| 74 | | `--ring_mask`| Omitted | Specify if you have a ring mask in the same directory to use for reducing size of hdf5 image | 75 | | `--crop_amount `| `None`| Number of crops you would like to extract | 76 | 77 | Up-to-date list can be viewed at [https://github.com/labsyspharm/mcmicro-ilastik](https://github.com/labsyspharm/mcmicro-ilastik) 78 | 79 | ### Arguments to S3Segmenter(`--s3seg-opts`): 80 | 81 | | Parameter | Default | Description | 82 | | --- | --- | --- | 83 | | `--probMapChan ` | `1` | which channel is used for nuclei segmentation. **Coincides with the channel used in upstream semantic segmentation modules. Must specify when different from default.** | 84 | | `--crop ` | `noCrop` | Type of cropping: `interactiveCrop` - a window will appear for user input to crop a smaller region of the image; `plate` - this is for small fields of view such as from a multiwell plate; `noCrop`, the default, is to use the entire image | 85 | 86 | **Nuclei parameters:** 87 | 88 | | Parameter | Default | Description | 89 | | --- | --- | --- | 90 | | `--nucleiFilter ` | `IntPM` | Method to filter false positive nuclei: `IntPM` - filter based on probability intensity; `Int` - filted based on raw image intensity | 91 | | `--logSigma ` | `3 60` | A range of nuclei diameters to search for. | 92 | 93 | **Cytoplasm parameters:** 94 | 95 | | Parameter | Default | Description | 96 | | --- | --- | --- | 97 | | `--segmentCytoplasm ` | `ignoreCytoplasm` | Select whether to `segmentCytoplasm` or `ignoreCytoplasm` | 98 | | `--CytoMaskChan ` | `2` | One or more channels to use for segmenting cytoplasm, specified as 1-based indices (e.g., `2` is the 2nd channel). | 99 | | `--cytoMethod ` | `distanceTransform` | The method to segment cytoplasm: `distanceTransform` - take the distance transform outwards from each nucleus and mask with the tissue mask; `ring` - take an annulus of a certain pixel size around the nucleus (see `cytoDilation`); `hybrid` - uses a combination of greyscale intensity and distance transform to more accurately approximate the extent of the cytoplasm. Similar to Cellprofiler's implementation. | 100 | | `--cytoDilation ` | `5` | The number of pixels to expand from the nucleus to get the cytoplasm ring. | 101 | | `--TissueMaskChan ` | Union of `probMapChan` and `CytoMaskChan` | One or more channels to use for identifying the general tissue area for masking purposes. | 102 | 103 | ### Arguments to quantification(`--quant-opts`): 104 | 105 | Up-to-date list can be viewed at [https://github.com/labsyspharm/quantification](https://github.com/labsyspharm/quantification) 106 | 107 | ### Arguments to naivestates(`--nstates-opts`): 108 | 109 | Up-to-date list can be viewed at [https://github.com/labsyspharm/naivestates](https://github.com/labsyspharm/naivestates) 110 | -------------------------------------------------------------------------------- /docs/_sass/color_schemes/lsp.scss: -------------------------------------------------------------------------------- 1 | $link-color: $blue-000; 2 | $btn-primary-color: $blue-100; 3 | 4 | .align-items-center { 5 | align-items: center; 6 | } -------------------------------------------------------------------------------- /docs/_sass/color_schemes/mcmicro.scss: -------------------------------------------------------------------------------- 1 | 2 | $sidebar-color: #e8ecef; 3 | $footer-background-color: #00b0e9; 4 | $branding-background-color: #ffffff; 5 | $branding-background-opacity: .85; 6 | $hero-background-color: #00b0e9; 7 | $hero-background-opacity: .85; 8 | 9 | $body-text-color: #58595b; 10 | $body-heading-color: #1e506c; // Default heading color 11 | $body-heading-2-color: #00b0e9; // Target H2 heading color 12 | $nav-child-link-color: #58595b; 13 | $link-color: #006eb8; 14 | $arrow-btn-color: #006eb8; 15 | $btn-primary-color: #006eb8; 16 | $image-card-label-background-color: #00b0e9; 17 | -------------------------------------------------------------------------------- /docs/assets/css/style.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/assets/css/style.scss -------------------------------------------------------------------------------- /docs/assets/images/bg-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/assets/images/bg-2.png -------------------------------------------------------------------------------- /docs/assets/images/hero_background.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/assets/images/hero_background.jpg -------------------------------------------------------------------------------- /docs/assets/images/mcmicro-logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 11 | 12 | 15 | 16 | 17 | 20 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /docs/cite/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: How to cite? 4 | nav_order: 13 5 | has_children: true 6 | 7 | --- 8 | 9 | # Citing MCMICRO 10 | 11 | Schapiro, D., Sokolov, A., Yapp, C. _et al._ MCMICRO: a scalable, modular image-processing pipeline for multiplexed tissue imaging. _Nat Methods_ **19,** 311–315 (2022). [https://doi.org/10.1038/s41592-021-01308-y](https://doi.org/10.1038/s41592-021-01308-y){:target="_blank"} 12 | -------------------------------------------------------------------------------- /docs/cite/publications.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Publications related to MCMICRO 4 | nav_order: 90 5 | parent: How to cite? 6 | 7 | --- 8 | 9 | ## MCMICRO related publications 10 | 11 | The [Laboratory of Systems Pharmacology (LSP)](https://labsyspharm.org/){:target="_blank"} and Harvard Program in Therapeutic Sciences ([HiTS](https://hits.harvard.edu/){:target="_blank"}) at Harvard Medical School has a number of publications related to MCMICRO. 12 | 13 | 14 | * Hoffer, J., Rashid, R., Muhlich, J., Chen, Y.-A., Russell, D., Ruokonen, J., Krueger, R., Pfister, H., Santagata, S., & Sorger, P. (2020). Minerva: A light-weight, narrative image browser for multiplexed tissue images. _Journal of Open Source Software_, _5_(54), 2579. [https://doi.org/10.21105/joss.02579](https://doi.org/10.21105/joss.02579){:target="_blank"} 15 | 16 | * Lin, J.-R., Izar, B., Wang, S., Yapp, C., Mei, S., Shah, P. M., Santagata, S., & Sorger, P. K. (2018). Highly multiplexed immunofluorescence imaging of human tissues and tumors using t-CyCIF and conventional optical microscopes. _ELife_, _7_. [https://doi.org/10.7554/eLife.31657](https://doi.org/10.7554/eLife.31657){:target="_blank"} 17 | 18 | * Lin, J.-R., Wang, S., Coy, S., Chen, Y.-A., Yapp, C., Tyler, M., Nariya, M.K., Heiser, C.N., Lau, K.S., Santagata, S., Sorger, P.K. (2023). Multiplexed 3D atlas of state transitions and immune interaction in colorectal cancer. _Cell_, _186_, (2), 363. [https://doi.org/10.1016/j.cell.2022.12.028](https://doi.org/10.1016/j.cell.2022.12.028){:target="_blank"} 19 | 20 | * Muhlich, J., Chen, Y.-A., Yapp, C., Russell, D., Santagata, S., & Sorger, P. K. (2022). Stitching and registering highly multiplexed whole-slide images of tissues and tumors using ASHLAR. _Bioinformatics_, _38_ (19), 4613. [https://doi.org/10.1093/bioinformatics/btac544](https://doi.org/10.1093/bioinformatics/btac544){:target="_blank"} 21 | 22 | * Nirmal, A. J., Maliga, Z., Vallius, T., Quattrochi, B., Chen, A. A., Jacobson, C. A., Pelletier, R. J., Yapp, C., Arias-Camison, R., Chen, Y.-A., Lian, C. G., Murphy, G. F., Santagata, S., & Sorger, P. K. (2022). The spatial landscape of progression and immunoediting in primary melanoma at single cell resolution. _Cancer Discovery_. [https://doi.org/10.1158/2159-8290.CD-21-1357](https://doi.org/10.1158/2159-8290.CD-21-1357){:target="_blank"} 23 | 24 | * Rashid, R., Chen, Y.-A., Hoffer, J., Muhlich, J. L., Lin, J.-R., Krueger, R., Pfister, H., Mitchell, R., Santagata, S., & Sorger, P. K. (2021). Narrative online guides for the interpretation of digital-pathology images and tissue-atlas data. _Nature Biomedical Engineering_. [https://doi.org/10.1038/s41551-021-00789-8](https://doi.org/10.1038/s41551-021-00789-8){:target="_blank"} 25 | 26 | * Schapiro, D., Sokolov, A., Yapp, C., Chen, Y.-A., Muhlich, J. L., Hess, J., Creason, A. L., Nirmal, A. J., Baker, G. J., Nariya, M. K., Lin, J.-R., Maliga, Z., Jacobson, C. A., Hodgman, M. W., Ruokonen, J., Farhi, S. L., Abbondanza, D., McKinley, E. T., Persson, D., … Sorger, P. K. (2022). MCMICRO: A scalable, modular image-processing pipeline for multiplexed tissue imaging. _Nature Methods_. [https://doi.org/10.1038/s41592-021-01308-y](https://doi.org/10.1038/s41592-021-01308-y){:target="_blank"} 27 | 28 | * Schapiro, D., Yapp, C., Sokolov, A., Reynolds, S. M., Chen, Y.-A., Sudar, D., Xie, Y., Muhlich, J. L., Arias-Camison, R., Arena, S., Taylor, A. J., Nikolov, M., Tyler, M., Lin, J.-R., Burlingame, E. A., Human Tumor Atlas Network, Chang, Y. H., Farhi, S. L., Thorsson, V., … Sorger, P. K. (2022). MITI Minimum Information guidelines for highly multiplexed tissue images. _Nature Methods_. [https://doi.org/10.1038/s41592-022-01415-4](https://doi.org/10.1038/s41592-022-01415-4){:target="_blank"} 29 | 30 | * Yapp, C., Novikov, E., Jang, W.-D., Vallius, T., Chen, Y.-A., Cicconet, M., Maliga, Z., Jacobson, C. A., Wei, D., Santagata, S., Pfister, H., & Sorger, P. K. (2022). UnMICST: Deep learning with real augmentation for robust segmentation of highly multiplexed images of human tissues. _Commun Biol_ **5**, 1263 (2022). [https://doi.org/10.1038/s42003-022-04076-3](https://doi.org/10.1038/s42003-022-04076-3){:target="_blank"} 31 | 32 | * View the [HiTS bioRxiv channel](https://connect.biorxiv.org/relate/content/151){:target="_blank"} and the [LSP website](https://labsyspharm.org/publications/highlighted/){:target="_blank"} for the most up-to-date list of our work. 33 | 34 | -------------------------------------------------------------------------------- /docs/community/code_of_conduct.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Code of Conduct 4 | nav_order: 94 5 | parent: Community 6 | --- 7 | 8 | # Code of Conduct 9 | 10 | ## Introduction 11 | 12 | This code of conduct applies to all spaces managed by the MCMICRO project. 13 | 14 | This code of conduct should be honored by everyone who participates in the MCMICRO community formally or informally, or claims any affiliation with the project, in any project-related activities and especially when representing the project, in any role. 15 | 16 | This code is not exhaustive or complete. It serves to distill our common understanding of a collaborative, shared environment and goals. Please try to follow this code in spirit as much as in letter, to create a friendly and productive environment that enriches the surrounding community. 17 | 18 | ## Specific Guidelines 19 | 20 | We strive to: 21 | 22 | 1. Be open. We invite anyone to participate in our community. We prefer to use public methods of communication for project-related messages, unless discussing something sensitive. This applies to messages for help or project-related support, too; not only is a public support request much more likely to result in an answer to a question, it also ensures that any inadvertent mistakes in answering are more easily detected and corrected. 23 | 24 | 2. Be empathetic, welcoming, friendly, and patient. We work together to resolve conflict, and assume good intentions. We may all experience some frustration from time to time, but we do not allow frustration to turn into a personal attack. A community where people feel uncomfortable or threatened is not a productive one. 25 | 26 | 3. Be collaborative. Our work will be used by other people, and in turn we will depend on the work of others. When we make something for the benefit of the project, we are willing to explain to others how it works, so that they can build on the work to make it even better. Any decision we make will affect users and colleagues, and we take those consequences seriously when making decisions. 27 | 28 | 4. Be inquisitive. Nobody knows everything! Asking questions early avoids many problems later, so we encourage questions, although we may direct them to the appropriate forum. We will try hard to be responsive and helpful. 29 | 30 | 5. Be careful in the words that we choose. We are careful and respectful in our communication and we take responsibility for our own speech. Be kind to others. Do not insult or put down other participants. We will not accept harassment or other exclusionary behaviour, such as: 31 | 32 | > * Violent threats or language directed against another person. 33 | > * Sexist, racist, ableist, or otherwise discriminatory jokes and language. 34 | > * Posting sexually explicit or violent material. 35 | > * Posting (or threatening to post) other people’s personally identifying information (“doxing”). 36 | > * Personal insults, especially those using sexist, racist, or ableist terms. 37 | > * Intentional or repeated misgendering of participants who have explicitly requested to be addressed by specific pronouns. 38 | > * Unwelcome sexual attention. 39 | > * Excessive profanity. Please avoid swearwords; people differ greatly in their sensitivity to swearing. 40 | > * Repeated harassment of others. In general, if someone asks you to stop, then stop. 41 | > * Advocating for, or encouraging, any of the above behaviour. 42 | 43 | ## Diversity Statement 44 | 45 | The MCMICRO project welcomes and encourages participation by everyone. We are committed to being a community that everyone enjoys being part of. Although we may not always be able to accommodate each individual’s preferences, we try our best to treat everyone kindly. 46 | 47 | No matter how you identify yourself or how others perceive you: we welcome you. Though no list can hope to be comprehensive, we explicitly honour diversity in: age, culture, ethnicity, genotype, gender identity or expression, language, national origin, neurotype, phenotype, political beliefs, profession, race, religion, sexual orientation, socioeconomic status, subculture and technical ability, to the extent that these do not conflict with this code of conduct. 48 | 49 | Though we welcome people fluent in all languages, MCMICRO development is conducted in English. 50 | 51 | Standards for behaviour in the MCMICRO community are detailed in the Code of Conduct above. Participants in our community should uphold these standards in all their interactions and help others to do so as well (see next section). 52 | 53 | ## Reporting Guidelines 54 | 55 | We know that it is painfully common for internet communication to start at or devolve into obvious and flagrant abuse. We also recognize that sometimes people may have a bad day, or be unaware of some of the guidelines in this Code of Conduct. Please keep this in mind when deciding on how to respond to a breach of this Code. 56 | 57 | For clearly intentional breaches, report those to the Code of Conduct committee (see below). For possibly unintentional breaches, you may reply to the person and point out this code of conduct (either in public or in private, whatever is most appropriate). If you would prefer not to do that, please feel free to report to the Code of Conduct Committee directly, or ask the Committee for advice, in confidence. 58 | 59 | You can report issues to the MCMICRO Code of Conduct committee, at `MCMICRO-conduct@googlegroups.com`. Currently, the committee consists of: 60 | 61 | * [Denis Schapiro](https://github.com/DenisSch) (chair) 62 | * [Artem Sokolov](https://github.com/ArtemSokolov) 63 | * [Jeremy Muhlich](https://github.com/jmuhlich) 64 | 65 | If your report involves any members of the committee, or if they feel they have a conflict of interest in handling it, then they will recuse themselves from considering your report. 66 | 67 | ## Endnotes 68 | 69 | We are thankful to the groups behind the following documents, from which we drew content and inspiration: 70 | 71 | * [The napari Code of Conduct](https://github.com/napari) 72 | -------------------------------------------------------------------------------- /docs/community/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Community 4 | nav_order: 11 5 | --- 6 | 7 | # Contributors 8 | 9 | MCMICRO is a multi-institutional effort with contributions by the following developers: 10 | 11 | {: .text-center} 12 | |Institution| People| 13 | |---|---| 14 | |**Harvard Medical School**|[Artem Sokolov](https://scholar.harvard.edu/artem-sokolov)
[Clarence Yapp](https://scholar.harvard.edu/clarence/who-clarence)
[Jeremy Muhlich](https://github.com/jmuhlich)
[Yu-An Chen](https://github.com/Yu-AnChen)
[Clemens Hug](https://github.com/clemenshug)
[Greg Baker](https://github.com/gjbaker)
[Juha Ruokonen](https://github.com/Juha-Ruokonen)
[Edward Novikov](https://github.com/edn314)
[Robert Krueger](https://github.com/kruegert)| 15 | |**Heidelberg University**|[Denis Schapiro](https://twitter.com/denisschapiro)
[Florian Wünnemann](https://github.com/FloWuenne)
[Miguel Ibarra](https://github.com/migueLib)
[Krešimir Beštak](https://github.com/kbestak)
[Victor Perez](https://github.com/VictorDidier)| 16 | |**Oregon Health and Sciences University**|[Allison Creason](https://www.ohsu.edu/people/allison-creason-phd)
[Jeremy Goecks](https://www.ohsu.edu/people/jeremy-goecks-phd)
Daniel Persson
[Qiang Gu](https://github.com/qiagu)
[Luke Sargent](https://github.com/luke-c-sargent)
[Cameron Watson](https://github.com/CameronFRWatson)
Luke Strgar| 17 | |**Dana-Farber Cancer Institute**|[Ajit Johnson](https://ajitjohnson.com/)| 18 | |**Vanderbilt University**|[Darren Tyson](https://medschool.vanderbilt.edu/cancer-biology/person/darren-tyson-ph-d/)| 19 | |**Brigham and Women's Hospital**|[Giorgio Gaglia](https://connects.catalyst.harvard.edu/Profiles/display/Person/25408)| 20 | |**The Jackson Laboratory**|[Brian White](https://www.jax.org/people/brian-white)| 21 | |**Broad Institute**|[Huan Wang](https://www.linkedin.com/in/huan-wang-4b940473/)| 22 | |**Brigham Young University**|[Matthew Hodgman](https://www.linkedin.com/in/matthodgman/)| 23 | |**Indica Labs**|[Erik Burlingame](https://www.linkedin.com/in/erik-burlingame/)| 24 | |**University of Macau**|[Yimin Zheng](https://github.com/Mr-Milk)| 25 | |**Sage Bionetworks**|[Adam Taylor](https://github.com/adamjtaylor)| 26 | 27 | 28 | # Additional efforts 29 | Additional development is contributed by researchers in the laboratory of [Peter Sorger](https://orcid.org/0000-0002-3364-1838) and the HMS Laboratory of Systems Pharmacology. 30 | 31 | ## Website Developers 32 | * [Juliann Tefft](https://www.linkedin.com/in/juliann-tefft) 33 | * [Han Xu](https://www.linkedin.com/in/han-xu-16a0216b/) 34 | 35 | ## Early adopters 36 | * [Roxy Pelletier](https://github.com/rjp21) 37 | * Zoltan Maliga 38 | * Janae Davis 39 | * Hanisha Udhani 40 | * Tuulia Vallius 41 | * Connor Jacobson 42 | * Claire Ritch 43 | * Nomeda Girnius 44 | 45 | # Funding 46 | 47 | This work is supported by: 48 | 49 | * NCI grants U54-CA22508U2C-CA233262 and U2C-CA233280 as part of the [Human Tumor Atlas Network](https://humantumoratlas.org/). 50 | * NIH grant 1U54CA225088: Systems Pharmacology of Therapeutic and Adverse Responses to Immune Checkpoint and Small Molecule Drugs as part of the [Center For Cancer Systems Biology Consortium](https://csbconsortium.org/). 51 | * NIH grant [1U24CA274494-01](https://reporter.nih.gov/search/ZpOeZajIwE6SwG2skcgclQ/project-details/10525124): Multi-Consortia Coordinating Center (MC2 Center) for Cancer Biology: Building Interdisciplinary Scientific Communities, Coordinating Impactful Resource Sharing, and Advancing Cancer Research 52 | * Ludwig Center at Harvard Medical School and the Ludwig Cancer Research Foundation 53 | * Denis Schapiro was supported by the University of Zurich BioEntrepreneur-Fellowship (BIOEF-17-001) and a Swiss National Science Foundation Early Postdoc Mobility fellowship (P2ZHP3_181475). He is currently a [Damon Runyon Quantitative Biology Fellow](https://www.damonrunyon.org/news/entries/5551/Damon%20Runyon%20Cancer%20Research%20Foundation%20awards%20new%20Quantitative%20Biology%20Fellowships) 54 | 55 | -------------------------------------------------------------------------------- /docs/datasets/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Example datasets 4 | nav_order: 10 5 | has_children: false 6 | redirect_from: /datasets.html 7 | --- 8 | # Example datasets 9 | {: .no_toc } 10 | 11 |
12 | 13 | Table of contents 14 | 15 | {: .text-delta } 16 | 1. TOC 17 | {:toc} 18 |
19 | 20 | ## Exemplar data for testing MCMICRO 21 | 22 |
23 | 24 | {% assign imageUrl = site.baseurl | append: "/images/mcmicro-exemplar-001.jpg" %} 25 | {% include image-card.html 26 | image=imageUrl 27 | label="exemplar-001" 28 | %} 29 | 30 | {% assign imageUrl = site.baseurl | append: "/images/mcmicro-exemplar-002.jpg" %} 31 | {% include image-card.html 32 | image=imageUrl 33 | label="exemplar-002" 34 | %} 35 | 36 |
37 | *Note that the representative images above depict only a subset of the image channels present in each data set.* 38 | {: .fs-3} 39 | 40 |
41 | 42 | **Two exemplars are currently available for demonstration purposes:** 43 | * `exemplar-001` is a minimal reproducible example for running all modules of the pipeline, except the dearray step. 44 | >* The exemplar consists of a small lung adenocarcinoma specimen taken from a larger tissue microarray (TMA), imaged using CyCIF with three cycles. 45 | >* Each cycle consists of six four-channel image tiles, for a total of 12 channels. 46 | >* Because this exemplar is too small for automated illumination correction, illumination profiles were precomputed from the entire TMA and included with the raw images. 47 | 48 | * `exemplar-002` is a two-by-two segment of a TMA that is meant to test the dearray step, followed by processing of all four cores in parallel. . 49 | >* The four cores are two meningioma tumors, one GI stroma tumor, and one normal colon specimen. 50 | >* This data set has ten cycles with 40 total channels and also includes precomputed illumination profiles. 51 | 52 | **After [installing Nextflow]({{ site.baseurl }}/tutorial/installation.html), both exemplars can be downloaded using the following commands:** 53 | ``` bash 54 | nextflow run labsyspharm/mcmicro/exemplar.nf --name exemplar-001 --path /local/path/ 55 | nextflow run labsyspharm/mcmicro/exemplar.nf --name exemplar-002 --path /local/path/ 56 | ``` 57 | 58 | >(Content download to the directory indicated by `/local/path/`.) 59 | {: .fs-3} 60 | 61 |
62 | 63 | **Or, the examples can be downloaded as .zip files.** 64 | 65 | {: .fs-3} 66 | > **Note:** When downloading the example files from this link, your system will need enough space for both the .zip file and for the expanded contents (indicated above). Downloading via these links is slower than the command line download, but may be preferred by users that are not using Nextflow. 67 | 68 | * **Exemplar-001:** [https://mcmicro.s3.amazonaws.com/exemplars/exemplar-001.zip](https://mcmicro.s3.amazonaws.com/exemplars/exemplar-001.zip) - 240 MB (320 MB unzipped) 69 | * **Exemplar-002:** [https://mcmicro.s3.amazonaws.com/exemplars/exemplar-002.zip](https://mcmicro.s3.amazonaws.com/exemplars/exemplar-002.zip) - 2.5 GB (3.5 GB unzipped) 70 | 71 |
72 | 73 | ### Visual Guide: Processing Exemplar-002 with MCMICRO 74 | This detailed [visual guide]({{ site.baseurl }}/overview/pipeline-visual-guide.html){:target="_blank"} walks you through the MCMICRO pipeline steps as it processes `exemplar-002`. This guide was generated using the [Minerva]({{base.siteurl}}/parameters/core.html#minerva) software package. 75 | 76 | [View the visual guide]({{ site.baseurl }}/overview/pipeline-visual-guide.html){: .btn .btn-green .btn-outline .btn-arrow } 77 | 78 |
79 | 80 | {: .text-center} 81 | **Want to give it a try??** Process Exemplar-001 and -002 using our [tutorial]({{ site.baseurl }}/tutorial/tutorial.html)! 82 | 83 |
84 | 85 | ## Exemplar Microscopy Images of Tissues (EMIT) 86 | 87 | The EMIT (Exemplar Microscopy Images of Tissues) data set is reference data containing a variety of images that can be used to develop and benchmark computational methods for image processing. These data are expected to be useful for biological studies; however, annotation of this data set for biological interpretation ongoing. Users of these data should expect to see updates and additions - these will be versioned and made backwards compatible whenever possible. 88 | 89 | Presently, EMIT comprises one tissue microarray (TMA) data set and one whole-slide image (WSI) data set. Users of these resources should cite [Schapiro et al 2021](https://doi.org/10.1038/s41592-021-01308-y){:target="_blank"}. 90 | 91 | 92 | ### Tissue microarrays (TMAs) 93 | 94 | EMIT-TMA22 95 | 96 | The TMA contains cores from 34 cancer, non-neoplastic diseases, and normal tissues collected from clinical discards under an IRB-supervised protocol. The TMA was imaged using the [cyclic immunofluorescence (CyCIF) method](https://www.cycif.org/){:target="_blank"} described in [Lin et al 2018](https://elifesciences.org/articles/31657){:target="_blank"}. Data were collected with a 20X magnification, 0.75 NA objective with 2x2-pixel binning using two multiplex antibody panels (on section 11 “TMA11” and section 22 “TMA22"). 97 | 98 | * [Primary Data](https://www.synapse.org/#!Synapse:syn22345748/wiki/609239){:target="_blank"} 99 | 100 | ### Whole-slide images (WSIs) 101 | 102 | Data was collected from a tonsil specimen (4 year-old female, caucasian). The specimen was serially sectioned with each section processed using a different imaging techonology. The resulting images comprise a 100+GB dataset, which has been processed by MCMICRO with primary data and all intermediates available on [Synapse](https://www.synapse.org/#!Synapse:syn24849819/wiki/608441){:target="_blank"}. 103 | 104 | * [Primary Data](https://www.synapse.org/#!Synapse:syn24849819/wiki/608441){:target="_blank"} 105 | * [Additional information](https://labsyspharm.github.io/mcmicro-images/){:target="_blank"} 106 | * Preview: 107 | 108 |
109 | 110 | {% include image-card.html 111 | image="https://labsyspharm.github.io/mcmicro-images/images/thumbnail-WD-75684-01.jpg" 112 | link="https://labsyspharm.github.io/mcmicro-images/stories/WD-75684-01.html" 113 | label="H&E" 114 | %} 115 | {% include image-card.html 116 | image="https://labsyspharm.github.io/mcmicro-images/images/thumbnail-WD-75684-02.jpg" 117 | link="https://labsyspharm.github.io/mcmicro-images/stories/WD-75684-02.html" 118 | label="CyCIF" 119 | %} 120 | {% include image-card.html 121 | image="https://labsyspharm.github.io/mcmicro-images/images/thumbnail-WD-75684-12.jpg" 122 | link="https://labsyspharm.github.io/mcmicro-images/stories/WD-75684-12.html" 123 | label="mIHC" 124 | %} 125 | {% include image-card.html 126 | image="https://labsyspharm.github.io/mcmicro-images/images/thumbnail-WD-75684-05.jpg" 127 | link="https://labsyspharm.github.io/mcmicro-images/stories/WD-75684-05.html" 128 | label="CODEX" 129 | %} 130 | 131 |
132 | -------------------------------------------------------------------------------- /docs/how-to-use.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: How to use this website 4 | nav_order: 3 5 | parent: Home 6 | --- 7 | 8 | ## This website contains several types of documentation 9 | 10 | 11 | 1. [Overview](./overview/) provides relevant background information related to multiplexed tissue imaging and MCMICRO. 12 | 13 | 2. View a [visual guide]({{ site.baseurl }}/tutorial/pipeline-visual-guide.html){:target="_blank"} through the steps of MCMICRO, then download [example data](./datasets/) to try it yourself! 14 | 15 | 3. The [tutorial](./tutorial/tutorial.html)\* provides a step-by-step example workflow that allows beginners to process two example sample data sets into mosaic images. 16 | >\* *First time users should start here to verify that MCMICRO is working with their system* 17 | 18 | 4. [Inputs/Outputs](./io.html) sections provide detailed documentation on what the inputs and outputs expected by MCMICRO are. 19 | >\*View the [troubleshooting](./troubleshooting/) section when running into problems. 20 | 21 | 5. Go to [platforms](./platforms/) to learn how to deploy MCMICRO in your compute environment. 22 | 23 | 6. Go to [parameters](./parameters/) for reference information on the usage and parameters for the various MCMICRO modules. 24 | >\* View [parameter tuning](./troubleshooting/tuning/) for in-depth guides on how parameter values affect module output. 25 | 26 | 7. Visit [community](./community/) to learn more about the cross-institutional effort that has culminated in MCMICRO and to get involved! -------------------------------------------------------------------------------- /docs/images/EMIT_TMA22.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/EMIT_TMA22.png -------------------------------------------------------------------------------- /docs/images/FAQ-binning.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/FAQ-binning.png -------------------------------------------------------------------------------- /docs/images/Fig1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/Fig1.png -------------------------------------------------------------------------------- /docs/images/Segmentation_crop2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/Segmentation_crop2.png -------------------------------------------------------------------------------- /docs/images/addmod/Step1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/addmod/Step1.png -------------------------------------------------------------------------------- /docs/images/addmod/Step2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/addmod/Step2.png -------------------------------------------------------------------------------- /docs/images/addmod/Step3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/addmod/Step3.png -------------------------------------------------------------------------------- /docs/images/coreograph-crop.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/coreograph-crop.png -------------------------------------------------------------------------------- /docs/images/coreograph-probmap.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/coreograph-probmap.jpg -------------------------------------------------------------------------------- /docs/images/coreograph-raw.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/coreograph-raw.jpg -------------------------------------------------------------------------------- /docs/images/coreograph1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/coreograph1.png -------------------------------------------------------------------------------- /docs/images/coreograph1av2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/coreograph1av2.png -------------------------------------------------------------------------------- /docs/images/coreograph3a.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/coreograph3a.png -------------------------------------------------------------------------------- /docs/images/coreograph3b.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/coreograph3b.png -------------------------------------------------------------------------------- /docs/images/coreographIntro.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/coreographIntro.png -------------------------------------------------------------------------------- /docs/images/coreographbannerv10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/coreographbannerv10.png -------------------------------------------------------------------------------- /docs/images/cycifoverview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/cycifoverview.png -------------------------------------------------------------------------------- /docs/images/cycifoverview1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/cycifoverview1.png -------------------------------------------------------------------------------- /docs/images/cylinter_banner.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/cylinter_banner.png -------------------------------------------------------------------------------- /docs/images/galaxy-build-collection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/galaxy-build-collection.png -------------------------------------------------------------------------------- /docs/images/galaxy-collection-history.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/galaxy-collection-history.png -------------------------------------------------------------------------------- /docs/images/galaxy-inputs-select.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/galaxy-inputs-select.png -------------------------------------------------------------------------------- /docs/images/galaxy-inputs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/galaxy-inputs.png -------------------------------------------------------------------------------- /docs/images/galaxy-order-collection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/galaxy-order-collection.png -------------------------------------------------------------------------------- /docs/images/galaxy-run.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/galaxy-run.png -------------------------------------------------------------------------------- /docs/images/galaxy-success-invocation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/galaxy-success-invocation.png -------------------------------------------------------------------------------- /docs/images/galaxy-wf-select.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/galaxy-wf-select.png -------------------------------------------------------------------------------- /docs/images/galaxy-wf-upload.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/galaxy-wf-upload.png -------------------------------------------------------------------------------- /docs/images/galaxy-wf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/galaxy-wf.png -------------------------------------------------------------------------------- /docs/images/hande.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/hande.png -------------------------------------------------------------------------------- /docs/images/if.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/if.png -------------------------------------------------------------------------------- /docs/images/imgmap_modules.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/images/imgmap_overview.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/images/main-menu-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/main-menu-1.png -------------------------------------------------------------------------------- /docs/images/main-menu-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/main-menu-2.png -------------------------------------------------------------------------------- /docs/images/main-menu-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/main-menu-3.png -------------------------------------------------------------------------------- /docs/images/mcmicro-exemplar-001.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/mcmicro-exemplar-001.jpg -------------------------------------------------------------------------------- /docs/images/mcmicro-exemplar-002.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/mcmicro-exemplar-002.jpg -------------------------------------------------------------------------------- /docs/images/minerva-examp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/minerva-examp.png -------------------------------------------------------------------------------- /docs/images/modules/SCIMAP.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/modules/SCIMAP.png -------------------------------------------------------------------------------- /docs/images/modules/ashlar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/modules/ashlar.png -------------------------------------------------------------------------------- /docs/images/modules/basic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/modules/basic.png -------------------------------------------------------------------------------- /docs/images/modules/coreo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/modules/coreo.png -------------------------------------------------------------------------------- /docs/images/modules/cylinter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/modules/cylinter.png -------------------------------------------------------------------------------- /docs/images/modules/mcquant.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/modules/mcquant.png -------------------------------------------------------------------------------- /docs/images/modules/minerva.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/modules/minerva.png -------------------------------------------------------------------------------- /docs/images/modules/others.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/modules/others.png -------------------------------------------------------------------------------- /docs/images/modules/s3seg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/modules/s3seg.png -------------------------------------------------------------------------------- /docs/images/modules/unmicst.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/modules/unmicst.png -------------------------------------------------------------------------------- /docs/images/pipeline-no-microscope-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/pipeline-no-microscope-white.png -------------------------------------------------------------------------------- /docs/images/pipeline-no-microscope.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/pipeline-no-microscope.png -------------------------------------------------------------------------------- /docs/images/pipeline-two-rows-v3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/pipeline-two-rows-v3.png -------------------------------------------------------------------------------- /docs/images/scimap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/scimap.png -------------------------------------------------------------------------------- /docs/images/screenshot-after-run.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/screenshot-after-run.jpg -------------------------------------------------------------------------------- /docs/images/segbannerv7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/segbannerv7.png -------------------------------------------------------------------------------- /docs/images/segbannerv8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/segbannerv8.png -------------------------------------------------------------------------------- /docs/images/segmentation1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/segmentation1.jpg -------------------------------------------------------------------------------- /docs/images/segmentation1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/segmentation1.png -------------------------------------------------------------------------------- /docs/images/segmentation2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/segmentation2.png -------------------------------------------------------------------------------- /docs/images/segmentation3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/segmentation3.png -------------------------------------------------------------------------------- /docs/images/segmentation3b.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/segmentation3b.png -------------------------------------------------------------------------------- /docs/images/segmentation3c.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/segmentation3c.png -------------------------------------------------------------------------------- /docs/images/segmentation4aa.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/segmentation4aa.png -------------------------------------------------------------------------------- /docs/images/segmentation4ab.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/segmentation4ab.png -------------------------------------------------------------------------------- /docs/images/segmentation4bi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/segmentation4bi.png -------------------------------------------------------------------------------- /docs/images/segmentation4bii.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/segmentation4bii.png -------------------------------------------------------------------------------- /docs/images/segmentation4c.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/segmentation4c.png -------------------------------------------------------------------------------- /docs/images/segmentation5ci.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/segmentation5ci.png -------------------------------------------------------------------------------- /docs/images/segmentation5cii.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/segmentation5cii.png -------------------------------------------------------------------------------- /docs/images/segmentation5i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/segmentation5i.png -------------------------------------------------------------------------------- /docs/images/segmentation5ii.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/segmentation5ii.png -------------------------------------------------------------------------------- /docs/images/squareoutlines.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/squareoutlines.png -------------------------------------------------------------------------------- /docs/images/stitching-art.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/stitching-art.png -------------------------------------------------------------------------------- /docs/images/tutorials/exemplar-001-cycle-06-dfp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/tutorials/exemplar-001-cycle-06-dfp.png -------------------------------------------------------------------------------- /docs/images/tutorials/exemplar-001-cycle-06-ffp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/tutorials/exemplar-001-cycle-06-ffp.png -------------------------------------------------------------------------------- /docs/images/tutorials/exemplar-001-cycle-06.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/tutorials/exemplar-001-cycle-06.png -------------------------------------------------------------------------------- /docs/images/tutorials/exemplar-001-message.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/tutorials/exemplar-001-message.PNG -------------------------------------------------------------------------------- /docs/images/tutorials/exemplar-001-outlinemerge.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/tutorials/exemplar-001-outlinemerge.PNG -------------------------------------------------------------------------------- /docs/images/tutorials/exemplar-002-message.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/tutorials/exemplar-002-message.PNG -------------------------------------------------------------------------------- /docs/images/tutorials/tma-map.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/tutorials/tma-map.png -------------------------------------------------------------------------------- /docs/images/tutorials/vizguide.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/tutorials/vizguide.png -------------------------------------------------------------------------------- /docs/images/unmicst2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/unmicst2.png -------------------------------------------------------------------------------- /docs/images/unmicst3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/unmicst3.png -------------------------------------------------------------------------------- /docs/images/unmicst4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/unmicst4.png -------------------------------------------------------------------------------- /docs/images/unmicst5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/unmicst5.png -------------------------------------------------------------------------------- /docs/images/unmicst6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/unmicst6.png -------------------------------------------------------------------------------- /docs/images/unmicstbannerv2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/labsyspharm/mcmicro/6085b05586ffc7fabd8600a03de36f4a7c41c1f8/docs/images/unmicstbannerv2.png -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Home 4 | nav_order: 1 5 | has_children: true 6 | description: "" 7 | hero_heading: "Multiple-choice microscopy pipeline" 8 | hero_body: "An end-to-end processing pipeline that transforms multi-channel whole-slide images into single-cell data. This website is a consolidated source of information for when, why, and how to use MCMICRO." 9 | hero_ctas: 10 | - label: "OVERVIEW" 11 | link: "overview/" 12 | - label: "TUTORIAL" 13 | link: "tutorial/tutorial.html" 14 | - label: "GITHUB" 15 | link: "https://github.com/labsyspharm/mcmicro" 16 | # last_modified_date: 2022-03-28 17 | --- 18 | # Features 19 | {% include youtube.html id="DY_F-eG9nm4" autoplay=true mute=true controls=false loop=true related=false %} 20 | 21 |
22 | 23 |
24 | ## Images to insights 25 | Multiplexed tissue imaging provides deep insight into the composition, organization, and phenotype of normal and diseased tissues. MCMICRO converts these multiplexed images into single-cell data using state of the art algorithms. Single-cell resolution images provide spatial context of the cellular microenvironment and can be used alongside additional profiling methods like scRNA-Seq to make robust biological conclusions. 26 | 27 |
28 |
29 | ## Open source pipeline 30 | MCMICRO is an open source, community supported software that uses Docker and workflow software to create pipelines for analyzing microscopy-based images of tissues. MCMICRO processes data sequentially using algorithms (modules) developed in different research groups. 31 |
32 |
33 | ## Modular tools for a new field 34 | High-plex tissue imaging is a new interdisciplinary field involving a wide range of imaging technologies, and the best image analysis approach is not always clear. MCMICRO implements a “multiple choice” approach that allows users to select different modules for customized image processing. 35 |
36 |
37 | ## Flexible implementation 38 | MCMICRO is implemented in the workflow languages [Nextflow](https://www.nextflow.io/){:target="_blank"} and [Galaxy](https://galaxyproject.org/){:target="_blank"}. Both implementations can be run locally, on a compute cluster, or on the cloud. 39 |
40 |
41 | ## A growing community 42 | Modules are being added to MCMICRO incrementally by a diverse developer community seeded by the NCI [Human Tissue Atlas Network](https://humantumoratlas.org/){:target="_blank"}. See what modules we are currently [using](./parameters/core.html), view our growing [community](./community/), or [get help](./troubleshooting). 43 |
44 |
45 | ## Robust test data 46 | MCMICRO comes with a growing library of imaging data ([EMIT data](./datasets/#exemplar-microscopy-images-of-tissues-emit)) for testing your test run or for developing new algorithms. There is a lot of unexplored biology in the test data as well! 47 |
48 |
49 | ## Technology agnostic 50 | MCMICRO works with any image that meets the [BioFormats standard](https://www.openmicroscopy.org/bio-formats/){:target="_blank"}, most commonly OME-TIFF. These images can be acquired using a wide range of technologies- CODEX, CyCIF, mIHC, mxIF, IMC or MIBI. 51 |
52 |
53 | ## Evolving best practices 54 | The MCMICRO team is collaborating with the NCI to run hackathons and challenges to identify the best modules and pipelines for specific types of data. 55 |
56 | 57 |
58 | 59 | -------------------------------------------------------------------------------- /docs/overview/pipeline-visual-guide.html: -------------------------------------------------------------------------------- 1 | --- 2 | title: Pipeline visual guide 3 | nav_order: 3 4 | parent: Overview 5 | --- 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 |
18 | 19 | 20 | 21 | 22 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /docs/parameters/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Parameters 4 | nav_order: 5 5 | --- 6 | 7 | # Supplying parameters to MCMICRO 8 | 9 | MCMICRO uses [default values](https://github.com/labsyspharm/mcmicro/blob/master/config/defaults.yml){:target="_blank"} for any settings not specified in a parameter file. MCMICRO will resolve conflicting parameter values with a [priority list](./#parameter-value-prioritization). 10 | {: .fs-5} 11 | {: .fw-400} 12 | 13 | --- 14 | 15 | Parameters can be supplied in two ways, 16 | {: .fs-5} 17 | {: .fw-300} 18 | 19 | 1. by [providing `params.yml` as a file](./#option-1-parameter-files-recommended) (**RECOMMENDED**) 20 | 1. in the project [directory]({{site.baseurl}}/io.html#input){:target="_blank"} 21 | 2. or by [pointing to a `params.yml` file in another directory](./#specifying-an-external-parameter-file) 22 | 3. by [overwriting individual parameters](./#overwriting-individual-parameters) on the command line. 23 | 24 | On this page, you will find more information about how to supply parameters to the pipeline for each of these methods, how to [change where intermediate files are stored](./#specifying-path-for-intermediate-files). 25 | 26 | --- 27 | 28 | ## Option 1: Parameter files (RECOMMENDED) 29 | 30 | Parameters must be specified in standard YAML format using the following three namespaces: 31 | {: .fs-5} 32 | {: .fw-300} 33 | 34 | * [`workflow`](./workflow) - pipeline-level parameters controlling overall behavior of MCMICRO 35 |
36 | * [`options`](./core) - module-level parameters that will be forwarded to individual tools 37 | * the entries are matched against `name` fields in `modules` 38 |
39 | * [`modules`](./specs) - module specifications, such as container name and version, whether the module uses 0-based or 1-based indexing, etc. (**For advanced users!**) 40 | 41 |
42 | _Please see the subpages for more information about the parameters for the [workflow](./workflow.html), [options](./core.html), and [modules](./specs.html) namespaces._ 43 | {: .text-center } 44 | 45 |
46 | An example `params.yml` may look as follows: 47 | {: .fs-5} 48 | {: .fw-300} 49 | 50 | ``` yaml 51 | workflow: 52 | start-at: registration 53 | stop-at: quantification 54 | viz: true 55 | segmentation-channel: 1 5 56 | options: 57 | ashlar: --flip-y -c 5 58 | s3seg: --maxima-footprint-size 5 59 | modules: 60 | watershed: 61 | version: 1.4.0-large 62 | ``` 63 | 64 | Note: When supplying multiple `option` parameters for the same module, only use the module name ONCE. See the line for `ashlar` in the example above. 65 | {: .fs-5} 66 | {: .fw-300} 67 |
68 | 69 | ### Specifying an external parameter file 70 | 71 | **By default, MCMICRO will look for `params.yml` in the [project directory]({{site.baseurl}}/io.html#input).** If you want to use the same parameter values for multiple projects, you can instead create a single `myparams.yml` file (elsewhere) and supply the path to this file to the pipeline with `--params`: 72 | 73 | ``` bash 74 | ## External parameter files can be supplied with --params 75 | nextflow run labsyspharm/mcmicro --in /path/to/project1 --params /path/to/externalparams.yml 76 | 77 | ## The same parameter file can then be used in multiple projects 78 | nextflow run labsyspharm/mcmicro --in /path/to/project2 --params /path/to/externalparams.yml 79 | ``` 80 | 81 | Values specified in the external `externalparams.yml` will overwrite any values found in `params.yml` files of individual project directories. *See [parameter prioritization](./#parameter-value-prioritization) below.* 82 | 83 |
84 | 85 | ## Option 2: Overwriting individual parameters 86 | 87 | Individual parameters can be overwritten directly on the command line. This provides an opportunity for "one-off" runs without the need to modify existing parameter files. 88 | 89 | ``` bash 90 | ## Most workflow parameters can be specified as is 91 | nextflow run labsyspharm/mcmicro --in /path/to/project --start-at segmentation 92 | 93 | ## There is no need to explicitly specify "true" for binary parameters 94 | nextflow run labsyspharm/mcmicro --in /path/to/project --tma --viz 95 | 96 | ## Enclose multiple values inside single quotes 97 | nextflow run labsyspharm/mcmicro --in /path/to/project --segmentation-channel '1 5' 98 | 99 | ## Add -opts suffix for module specific options 100 | nextflow run labsyspharm/mcmicro --in /path/to/project --ashlar-opts '-m 50' 101 | ``` 102 | 103 |
104 | 105 | ## Parameter value prioritization 106 | 107 | If the pipeline is supplied with conflicting parameters, the conflicting values will be resolved according to the following prioritization: 108 | 109 | * **(Lowest)** Default parameter values in [config/defaults.yml](https://github.com/labsyspharm/mcmicro/blob/master/config/defaults.yml). 110 | * Parameter values in `params.yml` of the project [directory]({{site.baseurl}}/io.html#input) 111 | * _(We recommend supplying parameters within your project directory)_ 112 | * Parameter values in a YAML file provided via `--params`. 113 | * **(Highest)** Individual parameters provided as double-dashed command-line arguments (e.g., `--start-at`) 114 | 115 |
116 | 117 | ## Specifying path for intermediate files 118 | By default, Nextflow writes intermediate files to a `work/` directory inside whatever location you initiate a pipeline run from. Use `-w` flag to provide a different location. 119 | 120 | ``` bash 121 | nextflow run labsyspharm/mcmicro --in /path/to/my-data -w /path/to/work/ 122 | ``` 123 | 124 | 125 | -------------------------------------------------------------------------------- /docs/parameters/workflow.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Workflow 4 | parent: Parameters 5 | nav_order: 1 6 | --- 7 | 8 | # Workflow parameters 9 | Parameters under the namespace `workflow` regulate the function of the pipeline as a whole. 10 | {: .fs-5} 11 | {: .fw-400} 12 | 13 |
14 | 15 | Table of contents 16 | 17 | {: .text-delta } 18 | 1. TOC 19 | {:toc} 20 |
21 | 22 | --- 23 | 24 | 25 | 26 | By default, the pipeline starts from the registration step ([ASHLAR]({{site.baseurl}}/parameters/core.html#ashlar)), proceeds through [UnMICST]({{site.baseurl}}/parameters/core.html#unmicst), [S3segmenter]({{site.baseurl}}/parameters/core.html#s3segmenter), and stops after executing the quantification [MCQuant]({{site.baseurl}}/parameters/core.html#mcquant) step. 27 | {: .fs-5} 28 | {: .fw-500} 29 | 30 | 31 | ## `start-at` 32 | Name of the first step to be executed by the pipeline 33 | {: .fs-4} 34 | {: .fw-300} 35 | 36 | * **Valid values:** `illumination`, `registration`, `background` (background only), `dearray` (TMA only), `segmentation`, `watershed`, `quantification`, `downstream` 37 | * **Default:** `registration` 38 | * **Example:** 39 | 40 | ``` yaml 41 | workflow: 42 | start-at: segmentation 43 | ``` 44 | 45 | 46 | ## `stop-at` 47 | Name of the final step to be executed by the pipeline 48 | {: .fs-4} 49 | {: .fw-300} 50 | 51 | * **Valid values:** `illumination`, `registration`, `background` (background only), `dearray` (TMA only), `segmentation`, `quantification`, `downstream` 52 | * **Default:** `quantification` 53 | * **Example:** 54 | 55 | ``` yaml 56 | workflow: 57 | stop-at: downstream 58 | ``` 59 | 60 | 61 | ### Additional examples: Specifying start and stop modules 62 | 63 | Use `start-at` and `stop-at` workflow parameters to execute any contiguous section of the pipeline. Remember, by default the pipeline will run from ASHLAR through MCQuant. 64 | {: .fs-4} 65 | {: .fw-300} 66 | 67 | **Example 1: Running illumination correction and registration only** 68 | 69 | ``` yaml 70 | workflow: 71 | start-at: illumination 72 | stop-at: registration 73 | ``` 74 | 75 | **Example 2: Start by dearraying an already-registered TMA image** 76 | 77 | ``` yaml 78 | workflow: 79 | tma: true 80 | start-at: dearray 81 | ``` 82 | 83 | **Note:** Starting at any step beyond registration requires pre-computed output of the previous steps placed at the correct location in the project directory. 84 | {: .fs-3} 85 | 86 | 87 | ## `tma` 88 | If `true`, MCMICRO treats input data as a TMA. If `false`, the input is assumed to be a whole-slide image 89 | {: .fs-4} 90 | {: .fw-300} 91 | 92 | * **Valid values:** `true`, `false` 93 | * **Default:** `false` 94 | * **Example:** 95 | 96 | ``` yaml 97 | workflow: 98 | tma: true 99 | ``` 100 | 101 | ## `segmentation-channel` 102 | One or more channels to use for segmentation, specified using 1-based indexing; values will be forwarded to all segmentation modules 103 | {: .fs-4} 104 | {: .fw-300} 105 | 106 | * **Valid values:** one or more positive integers, each separated with a space 107 | * **Default:** `1` 108 | * **Example:** 109 | 110 | ``` yaml 111 | workflow: 112 | segmentation-channel: 1 5 113 | ``` 114 | 115 | ## `segmentation-recyze` 116 | Whether the image should be reduced to the channels specified in `segmentation-channel` prior to being provided to the segmentation modules. 117 | {: .fs-4} 118 | {: .fw-300} 119 | 120 | * **Valid values:** `true`, `false` 121 | * **Default:** `false` 122 | * **Example:** 123 | 124 | ``` yaml 125 | workflow: 126 | segmentation-channel: 5 127 | segmentation-recyze: true 128 | ``` 129 | 130 | ## `ilastik-model` 131 | 132 | A custom `.ilp` file to be used as the classifier model for ilastik 133 | {: .fs-4} 134 | {: .fw-300} 135 | 136 | * **Valid values:** A full path to any file 137 | * **Default:** None 138 | * **Example:** 139 | 140 | ``` yaml 141 | workflow: 142 | ilastik-model: /full/path/to/mymodel.ilp 143 | ``` 144 | 145 | ## `segmentation` 146 | 147 | A list of segmentation modules to run 148 | {: .fs-4} 149 | {: .fw-300} 150 | 151 | * **Valid values:** One or more of `unmicst`, `ilastik`, `mesmer`, `cypository`, `cellpose` specified as a YAML list 152 | * **Default:** `unmicst` 153 | * **Example:** 154 | 155 | ``` yaml 156 | workflow: 157 | segmentation: [unmicst, ilastik] 158 | ``` 159 | 160 | ## `downstream` 161 | 162 | A list of downstream modules to run 163 | {: .fs-4} 164 | {: .fw-300} 165 | 166 | * **Valid values:** One or more of `naivestates`, `scimap`, `fastpg`, `scanpy`, `flowsom` 167 | * **Default:** `scimap` 168 | * **Example:** 169 | 170 | ``` yaml 171 | workflow: 172 | downstream: [scanpy, flowsom] 173 | ``` 174 | 175 | ## `viz` 176 | 177 | Whether to generate a vizualization with Auto-Minerva 178 | {: .fs-4} 179 | {: .fw-300} 180 | 181 | * **Valid values:** `true`, `false` 182 | * **Default:** `false` 183 | * **Example:** 184 | 185 | ``` yaml 186 | workflow: 187 | viz: true 188 | ``` 189 | 190 | ## `qc-files` 191 | Whether QC files should be copied, moved, hard linked, or symbolically linked from work directories to the project directory. 'inherit' may be specified to use the value of the publish_dir_mode pipeline parameter. 192 | {: .fs-4} 193 | {: .fw-300} 194 | 195 | * **Valid values:** `copy`, `move`, `link`, `symlink`, `inherit` 196 | * **Default:** `inherit` 197 | * **Example:** 198 | 199 | ``` yaml 200 | workflow: 201 | qc-files: move 202 | ``` 203 | 204 | ## `background` 205 | 206 | Whether background subtraction should be performed, and the computed intermediates used in further processing 207 | {: .fs-4} 208 | {: .fw-300} 209 | 210 | * **Valid values:** `true`, `false` 211 | * **Default:** `false` 212 | * **Example:** 213 | 214 | ``` yaml 215 | workflow: 216 | background: true 217 | ``` 218 | 219 | ## `background-method` 220 | 221 | Which background subtraction module to use when `background: true`. 222 | {: .fs-4} 223 | {: .fw-300} 224 | 225 | * **Valid values:** `backsub`, `imagej-rolling-ball` 226 | * **Default:** `backsub` 227 | * **Example:** 228 | 229 | ``` yaml 230 | workflow: 231 | background-method: backsub 232 | ``` 233 | -------------------------------------------------------------------------------- /docs/platforms/galaxy/galaxy-running.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Running MCMICRO 4 | nav_order: 20 5 | parent: Galaxy 6 | --- 7 | 8 | # Running MCMICRO in Galaxy 9 | 10 | The Galaxy workflow provides a sensible set of [default parameters for individual modules]({{ site.baseurl }}/parameters/core.html). These parameters can be modified and saved in the workflow using the Workflow Editor or can be changed at runtime. 11 | 12 | 13 | ## Import input data 14 | 15 | [Upload](https://galaxyproject.org/tutorials/upload/) the input datasets from your local filesystem or export the input datasets from a Data Library to a Galaxy history. 16 | 17 | 18 | 19 | ## Create a collection 20 | Some workflows require a collection or list of datasets as input. To create a collection list, select the image datasets to include. 21 | 22 | 23 | Order the datasets, name, and create the collection. 24 | 25 | 26 | Now the image datasets will appear in the history as a single collection list. 27 | 28 | 29 | ## Invoke the Workflow 30 | 31 | From the Workflow page, select the workflow to run: 32 | 33 | 34 | Select the inputs for the workflow. Change any tool parameters, if desired. Run the workflow. 35 | 36 | 37 | The successfully invoked workflow will schedule jobs and populate the history with the results for each tool. 38 | 39 | 40 | ## Need Help? 41 | For help, take a look at the [Galaxy tutorials](https://galaxyproject.org/learn/), [FAQ](https://galaxyproject.org/support/), or ask a question on [GalaxyHelp](https://help.galaxyproject.org/). -------------------------------------------------------------------------------- /docs/platforms/galaxy/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Galaxy 4 | nav_order: 10 5 | has_children: true 6 | parent: Platforms 7 | --- 8 | 9 | # MCMICRO Galaxy Workflow 10 | 11 | The Galaxy platform provides users a web based interface for running computational tools and workflows, manipulating datasets, and viewing or interacting with images. The MCMICRO modules and workflows have been integrated into Galaxy allowing users to run the MCMICRO workflow, individual modules, and view images from their web browser. 12 | 13 | ## Want to learn more or contribute to Galaxy? 14 | Learn more about the [Galaxy Project](https://galaxyproject.org/) or how to contribute to [Galaxy Development](https://galaxyproject.org/develop/). 15 | 16 | ## Need Help? 17 | For help, take a look at the [Galaxy tutorials](https://galaxyproject.org/learn/), [FAQ](https://galaxyproject.org/support/), or ask a question on [GalaxyHelp](https://help.galaxyproject.org/). 18 | 19 | -------------------------------------------------------------------------------- /docs/platforms/galaxy/installation.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Installation 4 | nav_order: 10 5 | parent: Galaxy 6 | --- 7 | 8 | # Installation 9 | 10 | ## Install Galaxy 11 | 12 | To install [Galaxy](https://galaxyproject.org/): 13 | ``` 14 | git clone https://github.com/galaxyproject/galaxy.git 15 | cd galaxy 16 | ./run.sh 17 | ``` 18 | 19 | This command will clone the Galaxy Github repository, install all needed dependencies in a python virtual environment, and start up a local galaxy server. Navigate to [http://localhost:8080](http://localhost:8080) to access the server from your browser. 20 | 21 | 22 | ## Install the MCMICRO Tools 23 | Each of the MCMICRO modules can be installed as a [galaxy tool](https://galaxyproject.org/admin/tools/add-tool-from-toolshed-tutorial/) from the [Main Galaxy Toolshed](https://toolshed.g2.bx.psu.edu/). 24 | - [Basic Illumination](https://toolshed.g2.bx.psu.edu/view/perssond/basic_illumination/fd8dfd64f25e) 25 | - [ASHLAR](https://toolshed.g2.bx.psu.edu/view/perssond/ashlar/b3054f3d42b2) 26 | - [Coreograph](https://toolshed.g2.bx.psu.edu/view/perssond/coreograph/99308601eaa6) 27 | - [UnMicst](https://toolshed.g2.bx.psu.edu/view/perssond/unmicst/6bec4fef6b2e) 28 | - [S3Segmenter](https://toolshed.g2.bx.psu.edu/view/perssond/s3segmenter/37acf42a824b) 29 | - [quantification](https://toolshed.g2.bx.psu.edu/view/perssond/quantification/928db0f952e3) 30 | - [naivestates](https://toolshed.g2.bx.psu.edu/view/perssond/naivestates/a62b0c62270e) 31 | 32 | ## Upload the MCMICRO Galaxy workflow 33 | Two workflows are currently available for running MCMICRO in Galaxy. 34 | - [MCMICRO Tissue Microarray Workflow](https://github.com/ohsu-comp-bio/cycIF-galaxy/blob/master/workflows/Galaxy-Workflow-MCMICRO_TMA_v1.0.0.ga) 35 | - [MCMICRO Whole Slide Tissue Workflow](https://github.com/ohsu-comp-bio/cycIF-galaxy/blob/master/workflows/Galaxy-Workflow-MCMICRO_Tissue_v1.0.0.ga) 36 | 37 | To upload a workflow, naviate to the Workflow page, click Upload, select the workflow file (`.ga`), and Import workflow. 38 | 39 | 40 | ## Need Help? 41 | For more help installing galaxy, try the [Get Galaxy](https://galaxyproject.org/admin/get-galaxy/) docs. Interested in running Galaxy with Docker? Check out [Galaxy Docker Image](https://github.com/bgruening/docker-galaxy-stable) and [Tutorials](https://training.galaxyproject.org/training-material/topics/admin/tutorials/galaxy-docker/slides.html#1). 42 | -------------------------------------------------------------------------------- /docs/platforms/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Platforms 4 | nav_order: 7 5 | has_children: true 6 | 7 | --- 8 | 9 | # Deploying MCMICRO 10 | 11 | For small images (on the order of a few GB, like the exemplar dataset), MCMICRO can be run on your local computer. For larger images, MCMICRO may need to be deployed on infrastructure with high resource capacity. MCMICRO uses Docker containers for uniform pipeline execution across many compute environments. However, some environments require additional steps. Instructions for systems using Galaxy, AWS or the (HMS-specific) O2 Compute Cluster are detailed in the sub-pages. 12 | 13 | 1. Instructions for deploying MCMICRO in a [Galaxy environment](./galaxy/) 14 | 1. Instructions for deploying MCMICRO on [Amazon Web Services](./run-AWS.html) 15 | 1. Instructions for deploying MCMICRO on the [O2 Compute Cluster](./run-O2.html) (for users at **Harvard Medical School only**) 16 | 1. Instructions for deploying MCMICRO on [Nextflow Tower](./run_tower.html) 17 | 18 | -------------------------------------------------------------------------------- /docs/platforms/run-AWS.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: AWS configuration 4 | nav_order: 4 5 | parent: Platforms 6 | --- 7 | 8 | # AWS configuration 9 | 10 | ## Install AWS CLI tool 11 | The folder `cloudformation` contains the AWS CloudFormation template to create the needed AWS resources for running mcmicro pipeline in AWS. To create the stack, you need to have AWS [command line tools installed and configured](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html). 12 | 13 | ## Configure parameters-batch.json 14 | You also have to configure a few things in the parameters-batch.json file. Here is a list of the parameters, star denotes a parameter that must be changed from its default value. 15 | 16 | | ParameterKey | ParameterValue | 17 | | ------------ | -------------- | 18 | | VpcId* | VPC ID (Select Services->VPC from AWS console, look up the VPC ID) | 19 | | BatchSubnets* | Subnet IDs (comma separated) Select Services->VPC->Subnets from AWS console. Private subnets are more secure and preferred. | 20 | | NextflowAMI | AMI ID of the EC2 image (Select Services-EC2->AMIs from AWS console, look up the AMI ID) | 21 | | BucketPrefix | Prefix for S3 buckets Nextflow will use. If the prefix is "mcmicro", buckets will be named "mcmicro-in", "mcmicro-out", "mcmicro-work" | 22 | | ProjectTag | Adds a tag named "project" which tracking of AWS costs | 23 | | ComputeEnvironmentNamePrefix | Prefix for the AWS Batch Compute Environments | 24 | | BatchClusterEC2Min/Max/DesiredCpus | These parameters control what type of EC2 instances will be created for the pipeline jobs. Best to leave them as-is, and let Nextflow have the control. | 25 | | BatchClusterSpotBidPercentage | Bid percentage, i.e. how much you are prepared to pay for the Spot instances | 26 | 27 | ## Create CloudFormation stacks 28 | After everything is configured, run the bash scripts (in this order) 29 | - `create-s3.sh` 30 | - `create-batch.sh` 31 | 32 | If later on you need to change something, e.g. project tag, you need to run the bash script `update-s3.sh` and `update-batch.sh` 33 | (When updating AWS Batch resources you might need to delete the stack from AWS console first and then recreate it from scratch) 34 | 35 | You should be all set now to run mcmicro in AWS! 36 | 37 | ## Running mcmicro with an EC2 instance 38 | 39 | ### Create and configure EC2 instance 40 | When using AWS, it's recommended to run Nextflow within an EC2 instance. Network access to S3 work bucket will be much faster. Follow these steps to create and configure the EC2 instance: 41 | - Create a new EC2 instance. The instance is used only to run Nextflow and orchestrate the pipeline, it will not be used to run actual jobs. Hence a low performance type such as t3.small/medium should suffice. 42 | - The instance needs an IAM role which allows access to AWS Batch and S3. You can use the provided IAM role: mcmicro-nextflow-batch-McmicroEC2ClientProfile 43 | - Start the instance, and install AWS CLI, Nextflow and mcmicro 44 | 45 | ### Run mcmicro pipeline 46 | Input data can either be in local computer or in S3 bucket. If the data is local, Nextflow will 47 | upload it automatically to S3 anyway (staging). 48 | 49 | Log in EC2 instance 50 | 51 | Run mcmicro pipeline according to normal instructions, but with a few changes: 52 | - use one of the AWS profiles, e.g. AWSTMA 53 | - give the S3 work bucket with parameter -bucket-dir 54 | - give the S3 out bucket with parameter --out 55 | - give the S3 in bucket with parameter --in (if input data was copied to S3) 56 | `nextflow run labsyspharm/mcmicro -profile AWSTMA -bucket-dir s3://mcmicro-work/imagename --in s3://mcmicro-in/imagename --out s3://mcmicro-out/imagename` 57 | 58 | -------------------------------------------------------------------------------- /docs/platforms/run-O2.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: O2 cluster (HMS only) 4 | nav_order: 2 5 | parent: Platforms 6 | --- 7 | 8 | # Running mcmicro on the O2 compute cluster 9 | 10 | **O2 is a high-performance cluster at Harvard Medical School (HMS). Non-HMS users can safely ignore this page.** 11 | 12 | **Familiarity with basic O2 commands will make running MCMICRO much easier on O2. If this is your first time interacting with the platform, please check out HMS's [O2 documentation](https://harvardmed.atlassian.net/wiki/spaces/O2/overview).** 13 | 14 | ## Setting up for MCMICRO on O2 15 | 16 | 1. Follow [installation instructions]({{ site.baseurl }}/tutorial/installation.html) to install Nextflow. **Docker is not needed**, because O2 uses Singularity to execute module containers, and Singularity is already available on O2. 17 | 18 | 1. Please ensure that Java is available by running `module load java`. 19 | 20 | 1. If your account is not in the `HITS_lsp-analysis` group (type `groups` to check), then please run the following command to prepare your environment: `nextflow run labsyspharm/mcmicro/setup/O2ext.nf` 21 | 22 | 1. When working with [exemplars]({{ site.baseurl }}/datasets/), please download your own copy to `/n/scratch/users/.../$USER/` (where `$USER` is your eCommons ID and `...` is its first letter). 23 | 24 | ## O2 execution 25 | 26 | To run the pipeline on O2, the following additional steps are required: 27 | 1. If your account is in the `HITS_lsp-analysis` group, please add the flag `-profile O2`. Use `-profile O2,WSI` and `-profile O2,TMA` for very large whole-slide images (WSIs) and tissue microarrays (TMAs), respectively. The profiles differ in the amount of resources requested for each module. 28 | 1. If your account is not in the `HITS_lsp-analysis` group, please use `-profile O2ext`. Similarly, use `-profile O2ext,WSI` and `-profile O2ext,TMA` for WSIs and TMAs. 29 | 1. To avoid running over on your disk quota, it is also recommended to use `/n/scratch` for holding the `work/` directory. Furthermore, `/n/scratch` is faster than `/home` or `/n/groups`, so jobs will complete faster. 30 | 31 | Compose an `sbatch` script that encapsulates resource requests, module loading and the `nextflow` command into a single entity. Create a `submit_mcmicro.sh` file based on the following template: 32 | 33 | ``` 34 | #!/bin/sh 35 | #SBATCH -p short 36 | #SBATCH -J mcmicro 37 | #SBATCH -o mcmicro-%J.log 38 | #SBATCH -t 0-12:00 39 | #SBATCH --mem=1G 40 | #SBATCH --mail-type=END # Type of email notification- BEGIN,END,FAIL,ALL 41 | #SBATCH --mail-user=user@university.edu # Email to which notifications will be sent 42 | 43 | module purge 44 | module load java 45 | /home/$USER/bin/nextflow run labsyspharm/mcmicro --in /n/scratch/users/${USER:0:1}/$USER/exemplar-001 -profile O2 -w /n/scratch/users/${USER:0:1}/$USER/work 46 | ``` 47 | replacing relevant fields (e.g., `user@university.edu`) with your own values. 48 | 49 | The pipeline run can then be kicked off with `sbatch submit_mcmicro.sh`. 50 | 51 | ### Requesting O2 resources 52 | 53 | The default profiles `WSI` and `TMA` establish reasonable defaults for O2 resource requests. These will work for most scenarios, but individual projects may also have custom time and memory requirements. To overwrite the defaults, compose a new config file (e.g., `myproject.config`) specifying the desired custom requirements using [process selectors](https://www.nextflow.io/docs/latest/config.html#process-selectors). For example, to request 128GB of memory and 96 hours in the `medium` queue for ASHLAR, one would specify 54 | ``` 55 | process{ 56 | withName:ashlar { 57 | queue = 'medium' 58 | time = '96h' 59 | memory = '128G' 60 | } 61 | } 62 | ``` 63 | Use [existing profiles](https://github.com/labsyspharm/mcmicro/tree/master/config/nf) as examples. Once `myproject.config` is composed, it can be provided to a `nextflow run` command using the `-c` flag: 64 | 65 | ``` 66 | nextflow run labsyspharm/mcmicro --in /path/to/exemplar-001 -profile O2 -c myproject.config 67 | ``` 68 | 69 | Note that `-profile` is still needed because it defines additional configurations, such as where to find the container modules. `myproject.config` simply overrides a portion of the fields in the overall profile. 70 | -------------------------------------------------------------------------------- /docs/troubleshooting/O2.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: O2 cluster (HMS only) 4 | nav_order: 2 5 | parent: Troubleshooting 6 | --- 7 | 8 | # Troubleshooting MCMICRO on O2 9 | 10 | The following are common problems users encounter on O2. For more information, please review [the instructions for running MCMICRO on O2]({{ site.baseurl }}/platforms/run-O2.html). 11 | 12 | ### I am getting `java: command not found` 13 | 14 | Make sure Java is loaded by doing `module load java` 15 | 16 | ### I am getting `docker: command not found` 17 | 18 | Be sure to include `-profile O2` if you are in the `lsp` group on O2 and `-profile O2ext` if you are not. Type `groups` to check if you are in the `lsp` group. 19 | 20 | ### I am getting `FATAL: singularity image is not in an allowed configured path` 21 | 22 | Rerun `nextflow run labsyspharm/mcmicro/setup/O2ext.nf` 23 | -------------------------------------------------------------------------------- /docs/troubleshooting/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Troubleshooting 4 | nav_order: 8 5 | --- 6 | 7 | **Commonly-encountered issues** are described in [Frequently Asked Questions](faq.html) 8 | 9 | **Questions** about executing the pipeline can be posted to [image.sc forums](https://forum.image.sc/tag/mcmicro), under the `mcmicro` tag. 10 | 11 | **Bugs** can be reported by opening an issue in the [GitHub repository](https://github.com/labsyspharm/mcmicro/issues). 12 | 13 | -------------------------------------------------------------------------------- /docs/troubleshooting/tuning/coreograph.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Coreograph 4 | nav_order: 32 5 | parent: Parameter tuning 6 | --- 7 | 8 | ![]({{ site.baseurl }}/images/coreographbannerv10.png) 9 |

10 | Great....yet another TMA dearray program. What does this one do? 11 |

12 |
13 |
14 | 15 | [Coreograph](https://doi.org/10.1038/s41592-021-01308-y){:target="_blank"} [(code)](https://github.com/HMS-IDAC/Coreograph){:target="_blank"} uses UNet [(Ronneberger et al., 2015)](https://arxiv.org/abs/1505.04597){:target="_blank"}, a deep learning model, to identify complete/incomplete tissue cores on a tissue microarray, and export them individually for faster downstream processing. It has been trained on 9 TMA slides of different sizes and tissue types. 16 | 17 | Training sets were acquired at 0.65 microns/pixel resolution and downsampled 1/32 times, **or 25 times**, to speed up performance. Once the center of each core has been identifed, active contours is used to generate a tissue mask of each core that can aid downstream single cell segmentation. A GPU is not required but will reduce computation time. 18 | 19 | ## Troubleshooting scenarios 20 | 21 | {: .fs-3} 22 | **NOTE for HMS users:** When using Coreograph on O2, the O2tma profile should be used!!
23 | ### **1. Alright, let's get started!** 24 | When using MCMICRO, Coreograph does not require any additional input parameters to run. The DNA channel is assumed to be in the 1st channel. 25 | 26 | To enable Coreograph, add `tma: true` to the `workflow` namespace of your `params.yml`: 27 | 28 | ``` yaml 29 | workflow: 30 | tma: true 31 | ``` 32 | 33 |

34 | 35 |

36 | As one can see, each core is labelled with a single number implying that each core was found uniquely. Furthermore, each core has a thick white line to indicate the accuracy of segmenting each core. (Future versions will have a colored outlines for better visibility). 37 | 38 | ### **2. Well, my DNA channel is not in the 1st channel.** 39 | 40 | No problem! Specify `--channel` in the `coreograph` field of module options. The field uses 0-based indexing, i.e., the 1st channel has index 0. For example, to use the 4th channel, your `params.yml` may look as follows: 41 | 42 | ```yaml 43 | workflow: 44 | tma: true 45 | options: 46 | coreograph: --channel 3 47 | ``` 48 | 49 | ### **3. The cores aren't being found properly.** 50 | Coreograph is trained on various core sizes ranging from 500 microns to 2 mm acquired at a pixel size of 0.65 microns per pixel and then downsampled 5 times. If your core size or image resolution are significantly different, you will need to either upsample or downsample a different number of times using `--downSampleFactor`. See below for examples: 51 | 52 | #### 3a) If your pixel size is 0.325 microns per pixel, then your pixel size is double the training data by a factor of 2 (0.65/0.325). You should downsample more times. Use 6 instead of 5.
53 | ![map]({{ site.baseurl }}/images/coreograph3a.png)
54 | 55 | Corresponding `params.yml`: 56 | 57 | ```yaml 58 | workflow: 59 | tma: true 60 | options: 61 | coreograph: --downsampleFactor 6 62 | ``` 63 | 64 | --- 65 | 66 | #### 3b) If your pixel size is 1.3 microns per pixel, then your pixel size is half of the training data (0.65/1.3). Instead of downsampling by 5 times (default), you should downsample less. Try 4. 67 | ![map]({{ site.baseurl }}/images/coreograph3b.png)
68 | 69 | Corresponding `params.yml`: 70 | 71 | ```yaml 72 | workflow: 73 | tma: true 74 | options: 75 | coreograph: --downsampleFactor 4 76 | ``` 77 | 78 | ### **4. What do the square outlines indicate?** 79 | 80 | ![squareoutlines]({{ site.baseurl }}/images/squareoutlines.png)
81 | 82 | The white square indicate that a core was found but Coreograph could not detect the exact outlines. A possible cause is dim staining. Downsizing 4 times instead of 5(default) can be a potential solution. 83 | 84 | Corresponding `params.yml`: 85 | 86 | ```yaml 87 | workflow: 88 | tma: true 89 | options: 90 | coreograph: --downsampleFactor 4 91 | ``` 92 | 93 | *Note: The line segments inside the circle outlines are an artefact of how Coreograph draws the outlines. They do not indicate any issues.* 94 | 95 | 96 | -------------------------------------------------------------------------------- /docs/troubleshooting/tuning/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Parameter tuning 4 | nav_order: 60 5 | has_children: true 6 | parent: Troubleshooting 7 | --- 8 | 9 | # Parameter tuning 10 | *These pages describe how you can tune parameters on a few modules based on some common troubleshooting scenarios.* 11 | 12 | [Coreograph](coreograph){: .btn .btn-green .btn-outline .btn-arrow } 13 |
14 | 15 | [UnMICST](unmicst){: .btn .btn-green .btn-outline .btn-arrow } 16 |
17 | 18 | [S3segmenter](s3seg){: .btn .btn-green .btn-outline .btn-arrow } 19 |
20 | 21 |
22 | 23 | *Detailed tuning information for other modules can be found on their respective websites, go to the [main module page]({{site.baseurl}}/parameters/core.html) for more information.* -------------------------------------------------------------------------------- /docs/troubleshooting/tuning/s3seg.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: S3segmenter 4 | nav_order: 34 5 | parent: Parameter tuning 6 | --- 7 | 8 | ![]({{ site.baseurl }}/images/segbannerv8.png)
9 |

10 | "So... you want to do single-cell image segmentation?" 11 |

12 | 13 | ## Troubleshooting scenarios 14 | ### **1. I’m new to this whole segmentation thingy. And I have a deadline. Just get me started with finding nuclei!**
15 | In its simplest form, s3segmenter by default will identify primary objects only (usually nuclei) and assumes this is in channel 1 (the first channel). In this case, no settings need to be specified.
16 | 17 | ![]({{ site.baseurl }}/images/segmentation1.png)
18 | 19 | If you specified `segmentation-channel` in your workflow parameters, the channel index will be correctly propagated to s3seg and any preprocessing steps. For example, if your `params.yml` was as follows: 20 | 21 | ``` yaml 22 | workflow: 23 | segmentation: unmicst # Can be omitted, since this is default 24 | segmentation-channel: 5 25 | ``` 26 | 27 | then channel 5 will be passed to both UnMICST and S3Segmenter to ensure that the two steps remain in sync. 28 | 29 | ### **2. It’s a disaster. It’s not finding all the nuclei**
30 | Depending on the type of pre-processing that was done, you may need to use a different method of finding cells. Let’s add `--nucleiRegion localMax` to the options: 31 | 32 | ``` yaml 33 | options: 34 | s3seg: --nucleiRegion localMax 35 | ``` 36 | 37 | ![]({{ site.baseurl }}/images/segmentation2.png)
38 | 39 | ### **3. Looks good! I want to filter out some objects based on size**
40 | You can specify a range of nuclei diameters that you expect your nuclei to be. Using `--logSigma ` 41 | Ie. `--logSigma 10 50` will retain all nuclei that have diameters between 10 and 50 pixels. Default is 3 60 42 | 43 | **Examples:** 44 | a)
45 | 46 | ``` yaml 47 | options: 48 | s3seg: --nucleiRegion localMax --logSigma 3 10 49 | ``` 50 | 51 | ![]({{ site.baseurl }}/images/segmentation3.png)
52 | 53 | --- 54 | 55 | b)
56 | 57 | ``` yaml 58 | options: 59 | s3seg: --nucleiRegion localMax --logSigma 30 60 60 | ``` 61 | 62 | ![]({{ site.baseurl }}/images/segmentation3b.png)
63 | 64 | --- 65 | 66 | c)
67 | 68 | ``` yaml 69 | options: 70 | s3seg: --nucleiRegion localMax --logSigma 3 60 71 | ``` 72 | 73 | ![]({{ site.baseurl }}/images/segmentation3c.png)
74 | 75 | ### **4. a) How do I segment the cytoplasm as well?**
76 | 77 | To do this, you will need to: 78 | * look at your image and identify a suitable cytoplasm channel such as the example below.
79 | ![]({{ site.baseurl }}/images/segmentation4aa.png)
80 | Nuclei and cytoplasm stained with Hoechst (purple) and NaK ATPase (green) respectively. 81 | Notice how the plasma membrane is distinctive and separates one cell from another. It also has good signal-to-background (contrast) and is in-focus. 82 | 83 | Specify `--CytoMaskChan `. For example, to specify the 10th channel, use `--CytoMaskChan 10`. To combine and sum the 10th and 11th channels, use `--CytoMaskChan 10 11`. Doing this maximizes the ability to capture more cells. 84 | 85 | * Also, specify this to activate cytoplasm segmentation: 86 | `--segmentyCytoplasm segmentCytoplasm` 87 | 88 | Putting it together in a `params.yml` file may look as follows: 89 | 90 | ``` yaml 91 | options: 92 | s3seg: --nucleiRegion localMax --CytoMaskChan 10 --segmentCytoplasm segmentCytoplasm 93 | ``` 94 | 95 | ![]({{ site.baseurl }}/images/segmentation4ab.png)
96 | 97 | **4. b) I don’t have a suitable cytoplasm channel…..**
98 | That’s ok. Cytoplasm segmentation is hard because there isn’t a universal marker. It’s generally acceptable to sample some number of pixels around the nucleus to approximate the cytoplasm. 99 | 1. Choose `--cytoMethod ring` 100 | 2. Then, specify the width of this ring `--cytoDilation ` ie. `--cytoDilation 3` will surround the nuclei with a 3-pixel thick cytoplasmic ring. The default is 5 pixels. 101 | 102 | **Examples**
103 | i)
104 | 105 | ``` yaml 106 | options: 107 | s3seg: --nucleiRegion localMax --CytoMaskChan 10 --segmentCytoplasm segmentCytoplasm --cytoMethod ring --cytoDilation 15 108 | ``` 109 | 110 | ![]({{ site.baseurl }}/images/segmentation4bi.png)
111 | 112 | Cytoplasm spilling beyond cytoplasm stain. Possibly too large `--cytoDilation` parameter. 113 | 114 | --- 115 | 116 | ii)
117 | 118 | ``` yaml 119 | options: 120 | s3seg: --nucleiRegion localMax --CytoMaskChan 10 --segmentCytoplasm segmentCytoplasm --cytoMethod ring --cytoDilation 6 121 | ``` 122 | 123 | ![]({{ site.baseurl }}/images/segmentation4bii.png)
124 | Much better. Cytoplasm outlines now just within the marker stain. 125 | 126 | **4. c) Are there other ways to detect the cytopolasm?**
127 | There’s a hybrid approach that combines a cytoplasm channel and the ring around the nuclei to deal with tissues that have sporadic cytoplasm staining. 128 | Try changing --cytoMethod to ‘hybrid’.
129 | 130 | ``` yaml 131 | options: 132 | s3seg: --nucleiRegion localMax --CytoMaskChan 10 --segmentCytoplasm segmentCytoplasm --cytoMethod hybrid 133 | ``` 134 | 135 | ![]({{ site.baseurl }}/images/segmentation4c.png)
136 | This is still a very experimental technique and may not yield better results! 137 | 138 | ### **5. I have an instance segmentation model, which already produces a mask. How do I incorporate this in?.** 139 | S3segmenter can accept pre-made instance segmentation primary object masks and still run some of the later functions we talked about above. To bypass nuclei segmentation, specify `--nucleiRegion bypass`. Then, you can still use `--logSigma` to filter overly small/large objects. 140 | 141 | ``` yaml 142 | options: 143 | s3seg: --logSigma 45 300 --nucleiRegion bypass 144 | ``` 145 | 146 | ![]({{ site.baseurl }}/images/segmentation5ii.png) 147 | ![]({{ site.baseurl }}/images/segmentation5i.png) 148 | 149 | ### **6. Nuclei…. Cytoplasm… NOW GIVE ME INTRACELLULAR SPOTS** 150 | This is a very complexed operation and requires several parameters. 151 | 1. Detail which channels you want to run spot detection on. 152 | `--detectPuncta ` . ie. `--detectPuncta 1 2 3` will look for spots in the 1st, 2nd, and 3rd channels. 153 | 154 | 2. `--punctaSigma `. This is equivalent to the standard deviation of a fitted Gaussian curve through one of your spots. 155 | Select custom values for each channels. 156 | Ie. `--punctaSigma 1.5 2 1.75` 157 | If you specify one value, it will use that for all channels 158 | 159 | 3. Lastly, choose the sensitivity of the spot detector. 160 | `--punctaSD `. 161 | 162 | Lower numbers increase sensitivity at the expense of more false positives. Not advisable when images are very noisy. May need to work out SD empirically. 163 | 164 | Select custom values for each channels. 165 | i) ie. `--punctaSD 10 12 10` (more stringent) 166 | Note: If you specify one value, it will use that for all channels
167 | ![]({{ site.baseurl }}/images/segmentation5ci.png)
168 | Hmmm….only 4 puncta are being detected shown by the green dots, but there are clearly other spots not being picked up. 169 | 170 | ii) `--punctaSD 3 3 3` (more sensitive)
171 | ![]({{ site.baseurl }}/images/segmentation5cii.png)
172 | Perfect! All visible spots appear to be detected with the more sensitive option. 173 | 174 | ``` yaml 175 | options: 176 | s3seg: --logSigma 45 300 --detectPuncta 1 2 3 --punctaSigma 1.5 2 1.75 --punctaSD 3 177 | ``` 178 | -------------------------------------------------------------------------------- /docs/troubleshooting/tuning/unmicst.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: UnMICST 4 | nav_order: 33 5 | parent: Parameter tuning 6 | --- 7 | 8 | # UnMICST - Universal Models for Identifying Cells and Segmenting Tissue
9 | ![UnMICST banner image]({{ site.baseurl }}/images/unmicstbannerv2.png)
10 |

11 | (pronounced un-mixed) 12 |

13 | 14 | ## Introduction 15 | Nuclei segmentation, especially for tissues, is a challenging and unsolved problem. Convolutional neural networks are particularly well-suited for this task: separating the foreground class (nuclei pixels) from the background class. [UnMICST](https://labsyspharm.github.io/UnMICST-info/){:target="_blank"} generates probability maps where the intensity at each pixel defines how confident the pixel has been correctly classified to the aforementioned classes. UnMICST can utilize a nuclear envelope channel (lamin B and nucleoporin 98) alongside a DNA channel to improve segmentation accuracy, in addition to more standard single-channel segmentation. 16 | 17 | These probability maps can make downstream image binarization more accurate using tools such as [S3segmenter](https://github.com/HMS-IDAC/S3segmenter) [(Saka et al., 2019)](https://doi.org/10.1038/s41587-019-0207-y){:target="_blank"}. UnMICST currently uses the UNet architecture [(Ronneberger et al., 2015)](https://arxiv.org/abs/1505.04597){:target="_blank"} but Mask R-CNN and Pyramid Scene Parsing (PSP) Net are coming very soon! 18 | 19 | ## Training data 20 | Quality machine learning algorithms can only be generated from quality training data. 21 | 22 | *UnMICST has been trained on 6 issue types that encapsulate the different morphologies of the entire tissue microarray:* 23 | 1) lung adenocarcinoma, 24 | 2) non-neoplastic prostate 25 | 3) non-neoplastic small intestine, 26 | 4) tonsil, 27 | 5) glioblastoma, and 28 | 6) colon adenocarcinoma. 29 | 30 | UnMICST also trained on **real augmentations** - such as intentionally de-focused planes and saturated pixels - to further improve segmentation accuracy relative to real-world experimental artifacts. 31 | 32 | **For more information about accessing the training data visit:** [https://github.com/HMS-IDAC/UnMicst](https://github.com/HMS-IDAC/UnMicst){:target="_blank"} 33 |
34 | **or read the publication here:** [(Yapp et al., 2021)](https://doi.org/10.1101/2021.04.02.438285){:target="_blank"} 35 | 36 | ![Image of a single TMA core with two insets that show single cells that have been segmented]({{ site.baseurl }}/images/unmicst2.png) 37 | 38 | --- 39 | 40 | ## Troubleshooting Scenarios 41 | **1. I just wanted to get started.**
42 | Set `--tool unmicst-solo` in the `unmicst` field of module options, and choose a channel that has your DNA stain in the `segmentation-channel` field of workflow parameters. Channel specification uses 1-based indexing. An example `params.yml` may look as follows: 43 | 44 | ``` yaml 45 | workflow: 46 | segmentation-channel: 1 47 | options: 48 | unmicst: --tool unmicst-solo 49 | ``` 50 | ![]({{ site.baseurl }}/images/unmicst3.png)
51 | 52 | **2. My tissue images have very packed nuclei. What do I do??**
53 | unmicst-solo uses a single DNA channel whereas unmicst-duo uses a DNA channel and a nuclear envelope stain, which can help the model discriminate between tightly-packed nuclei. This additional stain can come from markers such as lamin B1, B2, nucleoporin 98 or some additive combination. 54 | Set `--tool unmicst-duo` and choose channels that have your DNA and nuclear envelope stains. If your DNA and envelope stains are in the 1st and 5th channel respectively, the corresponding `params.yml` may look as follows: 55 | 56 | ```yaml 57 | workflow: 58 | segmentation-channel: 1 5 59 | options: 60 | unmicst: --tool unmicst-duo 61 | ``` 62 | 63 | ![]({{ site.baseurl }}/images/unmicst4.png)
64 | 65 | **3. My tissue images are blurry. What do I do??**
66 | Again, consider using *unmicst-duo* with a nuclear envelope stain.
67 | *without nuclear envelope stain*
68 | ![]({{ site.baseurl }}/images/unmicst5.png)
69 |
70 |
71 | *with nuclear envelope stain*
72 | ![]({{ site.baseurl }}/images/unmicst6.png)
73 | 74 | **4. You said the training data is sampled at 0.65microns/pixel and acquired with a 20x/0.75NA objective lens. What do I do if my data was acquired with a 40x lens?**
75 | First of all, check what is your pixel size since that is more relevant. If your pixel size is about half of the training data (ie. 0.325 microns/pixel), use a `--scalingFactor` of 0.5. If your pixel size is double (ie. 1.3 microns/pixel), then set your `--scalingFactor` to 2. For example, 76 | 77 | ``` yaml 78 | options: 79 | unmicst: --scalingFactor 2 80 | ``` 81 | 82 | **5. I heard unmicst-legacy is spectacular.**
83 | You mean spectacularly **bad**. unmicst-legacy is deprecated. Use unmicst-solo or unmicst-duo if you have a nuclear envelope staining. 84 | 85 | -------------------------------------------------------------------------------- /docs/tutorial/installation.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Installation 4 | nav_order: 1 5 | parent: Tutorial 6 | --- 7 | 8 | # Installation 9 | 10 | ## Install Nextflow 11 | 12 | If not already installed, install Java: [https://adoptopenjdk.net/](https://adoptopenjdk.net/){:target="_blank"} 13 | 23 | 24 |
25 | 26 | Expand to see troubleshooting tips related to Java 27 | 28 |
29 | > * If nextflow has trouble interacting with your java, we recommend checking the version number with `java --version` 30 | 31 | > * Some errors have been occurring with version numbers with four components (i.e. 11.0.14.1). If your version has four components, consider downloading an archived version, such as "11.0.14+9", from [https://adoptium.net/archive.html?variant=openjdk11](https://adoptium.net/archive.html?variant=openjdk11){:target="_blank"} as a temporary solution until this issue is resolved. 32 | 33 |
34 |
35 | 36 | Install [Nextflow](https://www.nextflow.io/){:target="_blank"}: 37 | ``` 38 | curl -s https://get.nextflow.io | bash 39 | ``` 40 | 41 | >This command will create a `nextflow` executable in the current directory. To simplify usage, consider moving this executable to a directory that is available on `$PATH`. One common place for this is a `bin/` directory in your home folder: 42 | 43 | ``` bash 44 | mkdir -p ~/bin # Creates a bin directory in the home folder 45 | mv nextflow ~/bin # Moves nextflow to that directory 46 | echo $SHELL # Determine what shell is used by your terminal 47 | ``` 48 | 49 | > If your terminal uses `bash`, the following commands should work as is. 50 | > Replace `.bashrc` with `.zshrc` in these commands, if your terminal uses `zsh` instead (often the case on Mac OS X). 51 | 52 | ``` bash 53 | echo 'export PATH="$HOME/bin:$PATH"' >> ~/.bashrc # Make the directory accessible on $PATH 54 | source ~/.bashrc # Reload the shell configuration 55 | ``` 56 | 57 | Verify that Nextflow is accessible by going to your home directory (`cd ~`) and typing `nextflow` on the command line. This should automatically print the help menu. 58 | 59 | ## Install Docker* 60 | 61 | Install [Docker](https://docs.docker.com/install/){:target="_blank"}. Ensure that the Docker engine is running by typing `docker run hello-world`. If the engine is running, you should see "This message shows that your installation appears to be working correctly." in the output. 62 | 63 | {: .text-center } 64 | {: .fs-3 } 65 | {: .fw-300 } 66 | \* *Harvard Medical School users using the O2 Compute Cluster should not install Docker - learn more [here](../platforms/run-O2.html).* 67 | 68 |
69 | 70 | Ready to run?? 71 | {: .fw-500} 72 | {: .fs-7} 73 | {: .text-grey-dk-250} 74 | 75 | Beginners, start with the [tutorial]({{site.baseurl}}/tutorial/tutorial.html){: .btn .btn-outline .btn-arrow } 76 | 77 | Experienced users can go to the [Parameter reference]({{site.baseurl}}/parameters/){: .btn .btn-outline .btn-arrow } 78 | -------------------------------------------------------------------------------- /docs/updates/future.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Future development 4 | nav_order: 2 5 | parent: 6 | nav_exclude: true 7 | 8 | --- 9 | 10 | # Areas for future development 11 | ## 3D Image Processing 12 | Currently, high-plex tissue imaging of 3D samples remains rare. Most published 3D studies use stacks of images spaced along the Z axis (parallel to the objective lens). In live-cells studies, time is also captured by a series of images to create a movie. MCMICRO can manage 3D image stacks, however, specialized viewers are required to look at the data. In preclinical settings, there has been much effort to develop more effective ways to sample 3D data. The most common methods acquire data directly in 3D data without the need for physical sectioning, methods such as optical deconvolution microscopy [(Sibarita et al., 2005)](https://doi.org/10.1007/b102215){:target="_blank"}, confocal microscopy, and fluorescent light sheet microscopy (LSFM) [(Power et al., 2017)](https://doi.org/10.1038/nmeth.4224){:target="_blank"}. LSFM is particularly valuable for tissue imaging because samples can be up to several hundred microns thick. These 3D sample compatible high-resolution microscopes are not yet commonly used for high-plex tissue imaging, but we expect their rapid uptake over the next few years. For this reason, we are working to add true 3D capability to MCMICRO. 13 | 14 | ## Image segmentation 15 | Image segmentation is currently one of the most challenging steps in single-cell analysis of tissue images. Computationally, image segmentation assigns class labels to an image in a pixel-wise manner to optimally subdivide it. Extensive efforts have helped master cell segmentation for *in vitro*, 2-D cultures of cells, but segmenting cells within dense tissue samples is substantially more complex: cell sizes and shapes are more diverse in tissues, and cells are often closely packed. Deep learning methods have become standard in image segmentation, object detection, and synthetic image generation19, based on architectures such as ResNet [(He et al., 2015)](https://doi.org/10.48550/arXiv.1512.03385){:target="_blank"}, UNet [(Ronnenberger et al., 2015)](https://doi.org/10.48550/arXiv.1505.04597){:target="_blank"}, and Mask R-CNN [(He et al., 2018)](https://doi.org/10.48550/arXiv.1703.06870){:target="_blank"}. UNet in particular has become popular due to its ease of deployment on Graphical Processing Units (GPUs) and its superior performance. MCMICRO provides access to all of these architectures as a standard feature. It is always necessary to examine an overlay of primary image data and segmentation mask to make sure that images are not over or under segmented. 16 | 17 | 18 | -------------------------------------------------------------------------------- /docs/updates/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Updates 4 | nav_order: 15 5 | has_children: true 6 | --- 7 | 8 | # Software Updates 9 | 10 | The MCMICRO pipeline is actively being developed to improve functionality. To see a record of changes, view the [GitHub update notes](https://github.com/labsyspharm/mcmicro/blob/master/CHANGES.md){:target="_blank"}. 11 | 12 | 13 | # Website Updates 14 | 15 | This website was last updated on {{ site.time | date: "%Y-%m-%d" }}. 16 | -------------------------------------------------------------------------------- /env/auto-minerva.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ghcr.io/labsyspharm/mcmicro:roadie-latest 2 | 3 | RUN apt-get update && \ 4 | apt-get install -y openslide-tools && \ 5 | rm -rf /var/lib/apt/lists/* 6 | 7 | RUN python -m pip install --no-cache-dir \ 8 | altair \ 9 | imagecodecs \ 10 | matplotlib \ 11 | openslide-python \ 12 | waitress \ 13 | flask \ 14 | flask_cors \ 15 | scikit-image 16 | 17 | RUN git clone https://github.com/labsyspharm/minerva-author.git /app/minerva-author 18 | 19 | ADD modules/ext/story.py /app 20 | -------------------------------------------------------------------------------- /env/dev.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM gitpod/workspace-full-vnc:2023-08-09-14-26-44 2 | 3 | # Install Java, OpenCV, and QT5 4 | RUN sudo apt-get update && \ 5 | sudo apt-get install -y openjdk-19-jre \ 6 | python3-opencv \ 7 | qtbase5-dev \ 8 | qtchooser \ 9 | qt5-qmake \ 10 | qtbase5-dev-tools && \ 11 | sudo rm -rf /var/lib/apt/lists/* 12 | 13 | # Python packages for Napari and Roadie 14 | RUN pip install --no-cache-dir \ 15 | "napari[all]" \ 16 | threadpoolctl \ 17 | scikit-learn \ 18 | pandas \ 19 | tifffile \ 20 | zarr \ 21 | scikit-image \ 22 | ome_types \ 23 | palom 24 | 25 | # Let pyjnius find that JRE without the full JDK installed 26 | ENV JAVA_HOME=/usr/lib/jvm/java-19-openjdk-amd64 27 | 28 | # Install Nextflow 29 | RUN curl -s https://get.nextflow.io | bash && \ 30 | sudo mv nextflow /usr/bin 31 | 32 | # Download exemplar-001 33 | RUN sudo mkdir -p /data && \ 34 | sudo chown gitpod:gitpod /data && \ 35 | cd /workspace && \ 36 | nextflow run labsyspharm/mcmicro/exemplar.nf --name exemplar-001 --path /data && \ 37 | rm -rf work 38 | -------------------------------------------------------------------------------- /env/roadie.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9 2 | 3 | RUN apt-get update && \ 4 | apt-get install -y python3-opencv && \ 5 | rm -rf /var/lib/apt/lists/* 6 | 7 | RUN python -m pip install --no-cache-dir \ 8 | threadpoolctl \ 9 | scikit-learn \ 10 | pandas \ 11 | tifffile==2023.3.15 \ 12 | zarr \ 13 | scikit-image \ 14 | ome_types>=0.4.2 \ 15 | palom 16 | -------------------------------------------------------------------------------- /exemplar.nf: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env nextflow 2 | 3 | nextflow.enable.dsl=2 4 | 5 | if( params.containsKey('help') ) { 6 | println """ 7 | Download MCMICRO exemplar datasets 8 | 9 | Expected parameters 10 | --name - Name of the exemplar, e.g., "exemplar-001", "exemplar-002", etc. 11 | 12 | Optional parameters 13 | --help - Print this message and exit 14 | --path - Path to the destination folder (Default: . or current directory) 15 | --from-cycle - Index of the first cycle to download 16 | --to-cycle - Index of the final cycle to download 17 | --nc - Number of cycles to download (overrides --from-cycle and --to-cycle) 18 | 19 | Examples: 20 | 1. Download the first five cycles of exemplar-001 to the current directory 21 | 22 | nextflow run labsyspharm/mcmicro/exemplar.nf --name exemplar-001 --nc 5 23 | 24 | 2. Download cycle 3 through 9 (inclusively) of exemplar-002 to /data 25 | 26 | nextflow run labsyspharm/mcmicro/exemplar.nf \\ 27 | --name exemplar-002 --path /data \\ 28 | --from-cycle 3 --to-cycle 9 29 | """ 30 | exit 0 31 | } 32 | 33 | params.nc = 0 34 | params.path = '.' 35 | 36 | // Define remote URLs and default parameters 37 | switch( params.name ) { 38 | case "exemplar-001": 39 | url = 'https://mcmicro.s3.amazonaws.com/exemplars/001/exemplar-001' 40 | hasParams = false 41 | registered = false 42 | params.fromCycle = 6 43 | params.toCycle = 8 44 | break 45 | case "exemplar-002": 46 | url = 'https://mcmicro.s3.amazonaws.com/exemplars/002/exemplar-002' 47 | hasParams = true 48 | registered = false 49 | params.fromCycle = 1 50 | params.toCycle = 10 51 | break 52 | case "exemplar-003": 53 | url = 'https://mcmicro.s3.amazonaws.com/exemplars/003/exemplar-003' 54 | hasParams = true 55 | registered = true 56 | break 57 | default: 58 | error "Unknown exemplar name" 59 | } 60 | 61 | process getImages { 62 | publishDir "${params.path}/${params.name}", mode: 'move' 63 | 64 | input: 65 | val name 66 | val loc 67 | output: file '**' 68 | 69 | script: 70 | def img = "${loc}/${name}.ome.tiff" 71 | """ 72 | mkdir ${loc} 73 | curl -f -o ${img} ${url}/${img} 74 | """ 75 | } 76 | 77 | process getIllumination { 78 | publishDir "${params.path}/${params.name}", mode: 'move' 79 | 80 | input: val name 81 | output: file '**' 82 | 83 | script: 84 | def ilp = 'illumination' 85 | def dfp = "${ilp}/${name}-dfp.tif" 86 | def ffp = "${ilp}/${name}-ffp.tif" 87 | """ 88 | mkdir $ilp 89 | curl -f -o $dfp ${url}/$dfp 90 | curl -f -o $ffp ${url}/$ffp 91 | """ 92 | } 93 | 94 | process getMarkers { 95 | publishDir "${params.path}/${params.name}", mode: 'move' 96 | 97 | input: val post 98 | output: file 'markers.csv' 99 | 100 | """ 101 | curl -f "${url}/markers.csv" ${post} > markers.csv 102 | """ 103 | } 104 | 105 | process getParams { 106 | publishDir "${params.path}/${params.name}", mode: 'move' 107 | 108 | output: file 'params.yml' 109 | when: hasParams 110 | 111 | """ 112 | curl -f -o params.yml "${url}/params.yml" 113 | """ 114 | } 115 | 116 | workflow { 117 | 118 | // Is the exemplar pre-registered? 119 | if(registered) { 120 | 121 | // Write downloaded images directly to registration/ 122 | getImages(params.name, 'registration') 123 | 124 | // No post-processing of markers.csv 125 | getMarkers('') 126 | 127 | } else { 128 | 129 | // Determine the sequence of individual cycles to download 130 | if(params.nc > 0 ) { 131 | seq = Channel.of( 1..params.nc ) 132 | mFrom = 2 133 | mTo = params.nc * 4 + 1 // Four markers per channel, plus header 134 | } 135 | else { 136 | seq = Channel.of( params.fromCycle..params.toCycle ) 137 | mFrom = (params.fromCycle-1) * 4 + 2 138 | mTo = (params.toCycle) * 4 + 1 139 | } 140 | 141 | // Compose filenames and write downloads to raw/ 142 | fn = seq.map{it -> "${params.name}-cycle-${String.format("%02d", it)}"} 143 | getImages(fn, 'raw') 144 | 145 | // Fetch illumination profiles 146 | getIllumination(fn) 147 | 148 | // Cut the appropriate rows from markers.csv 149 | getMarkers("| sed -n \"1p;${mFrom},${mTo}p\"") 150 | } 151 | 152 | getParams() 153 | } 154 | -------------------------------------------------------------------------------- /lib/mcmicro/Flow.groovy: -------------------------------------------------------------------------------- 1 | package mcmicro 2 | 3 | /** 4 | * Constructs a path to QC directory from the project path 5 | * 6 | * @param pathIn path to the project directory 7 | * @param modName name of the module producing QC (can be '') 8 | */ 9 | static def QC(pathIn, modName) { 10 | pathIn + "/qc/" + modName 11 | } 12 | 13 | /** 14 | * Determines indices of the start and stop steps in the pipeline 15 | * 16 | * @param wfp workflow parameters 17 | */ 18 | static def flowSegment(wfp) { 19 | // Deprecation checks 20 | if(wfp['start-at'] == 'probability-maps' || 21 | wfp['stop-at'] == 'probability-maps') { 22 | String msg = "probability-maps is deprecated; please use " + 23 | "--start-at segmentation and --stop-at segmentation" 24 | throw new Exception(msg) 25 | } 26 | 27 | // Valid start/stop steps in the mcmicro pipeline 28 | List mcsteps = [ 29 | "staging", // Step 0 30 | "raw", // Step 1 31 | "illumination", // Step 2 32 | "registration", // Step 3 33 | "background", // Step 4 34 | "dearray", // Step 5 35 | "segmentation", // Step 6 36 | "watershed", // Step 7 37 | "quantification", // Step 8 38 | "downstream" // Step 9 39 | ] 40 | 41 | // Identify starting and stopping indices 42 | int idxStart = mcsteps.indexOf( wfp['start-at'] ) 43 | int idxStop = mcsteps.indexOf( wfp['stop-at'] ) 44 | if( idxStart < 0 ) 45 | throw new Exception("Unknown starting step ${wfp['start-at']}") 46 | if( idxStop < 0 ) 47 | throw new Exception("Unknown stopping step ${wfp['stop-at']}") 48 | 49 | // Advance segmentation -> watershed to ensure no dangling probability maps 50 | if( idxStop == 6 ) idxStop = 7 51 | 52 | return [idxStart, idxStop] 53 | } 54 | 55 | /** 56 | * Determines which precomputed intermediates are relevant 57 | * 58 | * @param wfp workflow parameters 59 | */ 60 | static def precomputed(wfp) { 61 | // Identify what segment of the pipeline to run 62 | def (idxStart, idxStop) = flowSegment(wfp) 63 | 64 | // Define whether a precomputed intermediate is relevant 65 | [ 66 | staging: idxStart == 0, 67 | raw: idxStart <= 3 && idxStart > 0, 68 | illumination: idxStart == 3, 69 | registration: idxStart == 4 || (idxStart == 5 && !wfp.background) || (idxStart > 5 && !wfp.tma && !wfp.background), // needed for background (3), tma if no background (4), everything else if both tma and background aren't specified 70 | background: idxStart > 4 && wfp.background, // if background specified, required 71 | dearray: idxStart > 5 && wfp.tma, // if tma specified, required 72 | 'probability-maps': idxStart == 7, 73 | segmentation: idxStart == 8, 74 | quantification: idxStart == 9 75 | ] 76 | } 77 | 78 | /** 79 | * Determines whether to run a given step 80 | * 81 | * @param step name of the step in the pipeline 82 | * @param wfp workflow parameters 83 | */ 84 | static def doirun(step, wfp) { 85 | // Identify what segment of the pipeline to run 86 | def (idxStart, idxStop) = flowSegment(wfp) 87 | 88 | switch(step) { 89 | case 'staging': 90 | return(idxStart == 0 && idxStop >= 0) 91 | case 'illumination': 92 | return(idxStart <= 2 && idxStop >= 2 && wfp.illumination) 93 | case 'registration': 94 | return(idxStart <= 3 && idxStop >= 3) 95 | case 'background': 96 | return(idxStart <= 4 && idxStop >= 4 && wfp.background) 97 | case 'dearray': 98 | return(idxStart <= 5 && idxStop >= 5 && wfp.tma) 99 | case 'segmentation': 100 | return(idxStart <= 6 && idxStop >= 6) 101 | case 'watershed': 102 | return(idxStart <= 7 && idxStop >= 7) 103 | case 'quantification': 104 | return(idxStart <= 8 && idxStop >= 8) 105 | case 'downstream': 106 | return(idxStart <= 9 && idxStop >= 9) 107 | case 'viz': 108 | return(wfp.viz) 109 | default: 110 | throw new Exception("Unknown step name ${step}") 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /lib/mcmicro/Util.groovy: -------------------------------------------------------------------------------- 1 | package mcmicro 2 | 3 | static def getSampleName(f, rawdir) { 4 | // Resolve paths relative to the input project directory 5 | String rel = rawdir.relativize(f).toString() 6 | rel.contains('/') ? rel.split('/').head() : 7 | rawdir.parent.getName() 8 | } 9 | static def getCycleNameFromDir(f, rawdir) { 10 | // Resolve paths relative to the input project directory 11 | String rel = rawdir.relativize(f).toString() 12 | rel.split('/').head() 13 | } 14 | 15 | /** 16 | * Extracts a file ID as the first token before delim in the filename 17 | * 18 | * @param f a file object 19 | * @param delim a string delimiter 20 | */ 21 | static def getFileID(f, delim) { 22 | f.getName().toString().split(delim).head() 23 | } 24 | 25 | /** 26 | * Extracts an image ID from a filename by dropping extension 27 | * 28 | * @param f a file object 29 | */ 30 | static def getImageID(f) { 31 | f.getBaseName().toString().replaceFirst(/\.ome$/, "") 32 | } 33 | 34 | /** 35 | * Apply escaping to make an arbitrary string usable as a shell argument. 36 | */ 37 | static def escapeForShell(p) { 38 | 39 | // Escape embedded single-quotes and wrap in single-quotes. 40 | "'" + p.toString().replace("'", "'\\''") + "'" 41 | } 42 | 43 | /** 44 | * Cleans a filename (string) by replacing spaces and punctuation with _ 45 | */ 46 | static def cleanFilename(s) { 47 | s.toString().replaceAll("[:() ]", "_") 48 | } 49 | 50 | /** 51 | * Extracts marker names from markers.csv 52 | */ 53 | static def getMarkerNames(mcsv) { 54 | List raw = mcsv.readLines() 55 | if(raw.size() < 2) 56 | throw new Exception(mcsv.getName() + " is not in correct format") 57 | 58 | // Find the marker_name column 59 | List cols = raw[0].split(',') 60 | int j = cols.indexOf('marker_name') 61 | if(j < 0) 62 | throw new Exception(mcsv.getName() + " missing marker_name column") 63 | 64 | // Extract the markers 65 | raw[1..-1].collect{line -> line.split(',')[j]} 66 | } 67 | -------------------------------------------------------------------------------- /lib/worker.nf: -------------------------------------------------------------------------------- 1 | import mcmicro.* 2 | 3 | // General worker process 4 | // 5 | // Inputs: 6 | // tag - used to match against other files at the pipeline level 7 | // the tag is assigned to the outputs without being modified by the worker 8 | // module - a list of module parameters (usually comes from config/modules.config) 9 | // .name - name of the module 10 | // .container - associated Docker container image 11 | // .version - Docker container image tag 12 | // .cmd - command to be executed inside the container 13 | // .input - how the input will be provided to the module 14 | // .model - (optional) how a custom model will be provided to the module 15 | // model - a custom model file that a user specifies through --${module.name}-model 16 | // inp - input file to process (e.g., .ome.tif for probability-map generation) 17 | // fnOut - (optional) if not '', outputs will be renamed to this upon publication 18 | // outfmt - a regular expression defining the outputs to capture 19 | // step - name of the pipeline step that is running the worker(s) 20 | // pubDir - directory to publish outputs to 21 | // 22 | // Outputs: 23 | // The process captures files matching outfmt and publishes them to pubDir 24 | // It also captures and publishes all files in the plots/ subdirectory 25 | // Lastly, it captures all files in the qc/ subdirectory and publishes them to project/qc/ 26 | process worker { 27 | container "${params.contPfx}${module.container}:${module.version}" 28 | tag "${module.name}-${task.index}" 29 | 30 | // Output files in the pre-configured output format (outfmt) and optional plots 31 | publishDir "${pubDir}", mode: "${params.publish_dir_mode}", 32 | pattern: "$outfmt", saveAs: {fn -> fnOut != '' ? fnOut : fn} 33 | publishDir "${pubDir}", mode: "${params.publish_dir_mode}", pattern: 'plots/**' 34 | 35 | // QC 36 | publishDir "${Flow.QC(params.in, module.name)}", mode: "${mcp.workflow['qc-files']}", 37 | pattern: 'qc/**', saveAs: { fn -> fn.replaceFirst("qc/","") } 38 | 39 | // Provenance 40 | publishDir "${Flow.QC(params.in, 'provenance')}", mode: 'copy', 41 | pattern: '.command.{sh,log}', 42 | saveAs: {fn -> fn.replace('.command', "${module.name}-${task.index}")} 43 | 44 | input: 45 | val(mcp) 46 | tuple val(tag), val(module), file(model), path(inp), val(pubDir), val(fnOut) 47 | val(outfmt) 48 | val(step) 49 | 50 | output: 51 | 52 | // Every worker emits a tuple (tag, module used, result) 53 | // The tag is used to match against files in other pipeline steps 54 | tuple val(tag), val("${module.name}"), path("$outfmt"), emit: res 55 | 56 | // Modules have the option of producing additional files in plots/ and qc/ 57 | // subdirectories. These are captured and published to the project directory. 58 | path('plots/**') optional true 59 | path('qc/**') optional true 60 | 61 | // Provenance 62 | tuple path('.command.sh'), path('.command.log') 63 | 64 | when: Flow.doirun(step, mcp.workflow) 65 | 66 | // We are creating a copy of the model file to deal with some tools locking files 67 | // Without this copying, the lock prevents parallel execution of multiple processes 68 | // if they all use the same model file. 69 | script: 70 | 71 | // Find module specific parameters 72 | def opts = "${Opts.moduleOpts(module, mcp)}" 73 | 74 | // Determine if we need to pass the input as a membrane image also 75 | def mmbr = (opts.indexOf('membrane') > -1 && module.containsKey('membrane-input')) ? 76 | "${module['membrane-input']} $inp" : "" 77 | 78 | // Compose the command 79 | def cmd = "${module.cmd} ${module.input} $inp $mmbr $opts" 80 | String m = "${module.name}-model" 81 | 82 | // Create a copy of the model file if one is provided 83 | if( mcp.workflow.containsKey(m) ) { 84 | def mdlcp = "cp-${model.name}" 85 | """ 86 | cp $model $mdlcp 87 | $cmd ${module.model} $mdlcp 88 | """ 89 | } else { 90 | """ 91 | $cmd 92 | """ 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /modules/background.nf: -------------------------------------------------------------------------------- 1 | // Import utility functions from lib/mcmicro/*.groovy 2 | import mcmicro.* 3 | 4 | // For nested string formatting 5 | import groovy.text.GStringTemplateEngine 6 | 7 | // Process name will appear in the the nextflow execution log 8 | // While not strictly required, it's a good idea to make the 9 | // process name match your tool name to avoid user confusion 10 | process backsub { 11 | 12 | // Use the container specification from the parameter file 13 | container "${params.contPfx}${module.container}:${module.version}" 14 | 15 | // Specify the project subdirectory for writing the outputs to 16 | // The pattern: specification must match the output: files below 17 | // Subdirectory: background 18 | publishDir "${params.in}/background", mode: "${params.publish_dir_mode}", 19 | pattern: "*.ome.tif" 20 | publishDir "${params.in}/background", mode: "${params.publish_dir_mode}", 21 | pattern:'markers_bs.csv' 22 | 23 | // Stores .command.sh and .command.log from the work directory 24 | // to the project provenance 25 | // No change to this line is required 26 | publishDir "${Flow.QC(params.in, 'provenance')}", mode: 'copy', 27 | pattern: '.command.{sh,log}', 28 | saveAs: {fn -> fn.replace('.command', "${module.name}-${task.index}")} 29 | 30 | // Inputs for the process 31 | // mcp - MCMICRO parameters (workflow, options, etc.) 32 | // module - module specifications (name, container, options, etc.) 33 | // tuple - image file ID, image file path, markers.csv file path 34 | input: 35 | val mcp 36 | val module 37 | tuple val(image_id), path(image), path(marker) 38 | 39 | // outputs are returned as results with appropriate patterns 40 | output: 41 | // Output background subtracted image and markers_bs.csv 42 | path("*.ome.tif"), emit: image_out 43 | path('markers_bs.csv'), emit: marker_out 44 | // Provenance files 45 | tuple path('.command.sh'), path('.command.log') 46 | 47 | // Specifies whether to run the process 48 | // Here, we simply take the flag from the workflow parameters 49 | when: mcp.workflow["background"] 50 | 51 | script: 52 | // String replacement Map for the syntax of ${variable_name} in 53 | // module.cmd 54 | def formatMap = [ 55 | marker: marker, 56 | image_id: image_id, 57 | image: image, 58 | ] 59 | def command = new GStringTemplateEngine() 60 | .createTemplate(module.cmd) 61 | .make(formatMap) 62 | .toString() 63 | 64 | """ 65 | $command ${Opts.moduleOpts(module, mcp)} 66 | """ 67 | } 68 | 69 | workflow background { 70 | 71 | // Inputs: 72 | take: 73 | mcp // MCMICRO parameters (workflow, options, etc.) 74 | imgs // images to apply background subtraction to 75 | marker // marker file 76 | 77 | main: 78 | // Assemble inputs from multiple channels 79 | inputs = imgs 80 | .map{ f -> tuple(Util.getImageID(f), f) } 81 | .combine(marker) 82 | backsub(mcp, mcp.modules['background'], inputs) 83 | 84 | // Return the outputs produced by the tool 85 | emit: 86 | image = backsub.out.image_out 87 | marker = backsub.out.marker_out 88 | } -------------------------------------------------------------------------------- /modules/dearray.nf: -------------------------------------------------------------------------------- 1 | import mcmicro.* 2 | 3 | process coreograph { 4 | container "${params.contPfx}${module.container}:${module.version}" 5 | 6 | // Output 7 | publishDir "${params.in}/dearray", mode: "${params.publish_dir_mode}", 8 | pattern: '{*.ome.tif,**mask.tif}' 9 | 10 | // QC 11 | publishDir "${Flow.QC(params.in, module.name)}", 12 | mode: "${mcp.workflow['qc-files']}", pattern: '{TMA_MAP.tif,centroidsY-X.txt}' 13 | 14 | // Provenance 15 | publishDir "${Flow.QC(params.in, 'provenance')}", mode: 'copy', 16 | pattern: '.command.{sh,log}', 17 | saveAs: {fn -> fn.replace('.command', "${module.name}")} 18 | 19 | input: 20 | val mcp 21 | val module 22 | path s 23 | 24 | output: 25 | path "*.ome.tif", emit: cores 26 | path "**_mask.tif", emit: masks 27 | path "TMA_MAP.tif" 28 | path "centroidsY-X.txt" 29 | tuple path('.command.sh'), path('.command.log') 30 | 31 | when: Flow.doirun('dearray', mcp.workflow) 32 | 33 | """ 34 | ${module.cmd} ${module.input} $s ${Opts.moduleOpts(module, mcp)} 35 | """ 36 | } 37 | 38 | workflow dearray { 39 | take: 40 | mcp // MCMICRO parameters (as returned by Opts.parseParams()) 41 | tma // Image of the entire TMA 42 | 43 | main: 44 | coreograph(mcp, mcp.modules['dearray'], tma) 45 | 46 | emit: 47 | cores = coreograph.out.cores.flatten() 48 | masks = coreograph.out.masks.flatten() 49 | } 50 | -------------------------------------------------------------------------------- /modules/downstream.nf: -------------------------------------------------------------------------------- 1 | include {worker} from "$projectDir/lib/worker" 2 | 3 | workflow downstream { 4 | take: 5 | mcp 6 | input 7 | 8 | main: 9 | 10 | // Determine if there are any custom models specified 11 | inp = Channel.of( mcp.modules['downstream'] ) 12 | .flatten() 13 | .map{ it -> String m = "${it.name}-model"; 14 | tuple(it, mcp.workflow.containsKey(m) ? 15 | file(mcp.workflow[m]) : 'built-in') } 16 | .combine(input) 17 | .map{ mod, _2, _3 -> 18 | tuple( '', mod, _2, _3, "${params.in}/downstream/${mod.name}", '') } 19 | worker( mcp, inp, '*.{csv,h5ad}', 'downstream' ) 20 | 21 | emit: 22 | worker.out.res 23 | } 24 | -------------------------------------------------------------------------------- /modules/ext/story.py: -------------------------------------------------------------------------------- 1 | import json 2 | import numpy as np 3 | import pandas as pd 4 | import ome_types 5 | import scipy.stats 6 | import sklearn.mixture 7 | import sys 8 | import threadpoolctl 9 | import tifffile 10 | import zarr 11 | import argparse 12 | import os 13 | 14 | def auto_threshold(img): 15 | 16 | assert img.ndim == 2 17 | 18 | yi, xi = np.floor(np.linspace(0, img.shape, 200, endpoint=False)).astype(int).T 19 | # Slice one dimension at a time. Should generally use less memory than a meshgrid. 20 | img = img[yi] 21 | img = img[:, xi] 22 | img_log = np.log(img[img > 0]) 23 | gmm = sklearn.mixture.GaussianMixture(3, max_iter=1000, tol=1e-6) 24 | gmm.fit(img_log.reshape((-1,1))) 25 | means = gmm.means_[:, 0] 26 | _, i1, i2 = np.argsort(means) 27 | mean1, mean2 = means[[i1, i2]] 28 | std1, std2 = gmm.covariances_[[i1, i2], 0, 0] ** 0.5 29 | 30 | x = np.linspace(mean1, mean2, 50) 31 | y1 = scipy.stats.norm(mean1, std1).pdf(x) * gmm.weights_[i1] 32 | y2 = scipy.stats.norm(mean2, std2).pdf(x) * gmm.weights_[i2] 33 | 34 | lmax = mean2 + 2 * std2 35 | lmin = x[np.argmin(np.abs(y1 - y2))] 36 | if lmin >= mean2: 37 | lmin = mean2 - 2 * std2 38 | vmin = max(np.exp(lmin), img.min(), 0) 39 | vmax = min(np.exp(lmax), img.max()) 40 | 41 | return vmin, vmax 42 | 43 | 44 | def main(in_path, out_path, markers_path): 45 | 46 | threadpoolctl.threadpool_limits(1) 47 | 48 | print(f"opening image: {in_path}", file=sys.stderr) 49 | tiff = tifffile.TiffFile(in_path) 50 | ndim = tiff.series[0].ndim 51 | if ndim == 2: 52 | # FIXME This can be handled easily (promote to 3D array), we just need a 53 | # test file to make sure we're doing it right. 54 | raise Exception("Can't handle 2-dimensional images (yet)") 55 | elif ndim == 3: 56 | pass 57 | else: 58 | raise Exception(f"Can't handle {ndim}-dimensional images") 59 | # Get smallest pyramid level that's at least 200 in both dimensions. 60 | level_series = next( 61 | level for level in reversed(tiff.series[0].levels) 62 | if all(d >= 200 for d in level.shape[1:]) 63 | ) 64 | zarray = zarr.open(level_series.aszarr()) 65 | signed = not np.issubdtype(zarray.dtype, np.unsignedinteger) 66 | 67 | print(f"reading metadata", file=sys.stderr) 68 | try: 69 | ome = ome_types.from_xml(tiff.pages[0].tags[270].value) 70 | ome_px = ome.images[0].pixels 71 | pixel_ratio = ome_px.physical_size_x_quantity / ome_px.physical_size_y_quantity 72 | if not np.isclose(pixel_ratio, 1): 73 | print( 74 | "WARNING: Non-square pixels detected. Using only X-size to set scale bar.", 75 | file=sys.stderr, 76 | ) 77 | pixels_per_micron = 1 / ome_px.physical_size_x_quantity.to("um").magnitude 78 | channel_names = [c.name for c in ome_px.channels] 79 | for i, n in enumerate(channel_names): 80 | if not n: 81 | channel_names[i] = f"Channel {i + 1}" 82 | except: 83 | print( 84 | "WARNING: Could not read OME metadata. Story will use generic channel names and\n" 85 | " the scale bar will be omitted.", 86 | file=sys.stderr, 87 | ) 88 | pixels_per_micron = None 89 | channel_names = [f"Channel {i + 1}" for i in range(zarray.shape[0])] 90 | 91 | # Override channel names with a markers.csv, if provided 92 | if markers_path is not None: 93 | print(f"Using marker names from {markers_path}") 94 | markers = pd.read_csv(markers_path) 95 | if markers.shape[0] != len(channel_names): 96 | raise Exception("The number of channels in markers.csv does not match the image") 97 | channel_names = markers['marker_name'].tolist() 98 | 99 | story = { 100 | "sample_info": { 101 | "name": "", 102 | "rotation": 0, 103 | "text": "", 104 | "pixels_per_micron": pixels_per_micron, 105 | }, 106 | "groups": [], 107 | "waypoints": [], 108 | } 109 | 110 | color_cycle = 'ffffff', 'ff0000', '00ff00', '0000ff' 111 | 112 | scale = np.iinfo(zarray.dtype).max if np.issubdtype(zarray.dtype, np.integer) else 1 113 | for gi, idx_start in enumerate(range(0, zarray.shape[0], 4), 1): 114 | idx_end = min(idx_start + 4, zarray.shape[0]) 115 | channel_numbers = range(idx_start, idx_end) 116 | channel_defs = [] 117 | for ci, color in zip(channel_numbers, color_cycle): 118 | print( 119 | f"analyzing channel {ci + 1}/{zarray.shape[0]}", file=sys.stderr 120 | ) 121 | img = zarray[ci] 122 | if signed and img.min() < 0: 123 | print(" WARNING: Ignoring negative pixel values", file=sys.stderr) 124 | vmin, vmax = auto_threshold(img) 125 | vmin /= scale 126 | vmax /= scale 127 | channel_defs.append({ 128 | "color": color, 129 | "id": ci, 130 | "label": channel_names[ci], 131 | "min": vmin, 132 | "max": vmax, 133 | }) 134 | story["groups"].append({ 135 | "label": f"Group {gi}", 136 | "channels": channel_defs, 137 | }) 138 | 139 | with open(out_path, 'w') as fout: 140 | json.dump(story, fout) 141 | 142 | if __name__ == "__main__": 143 | parser = argparse.ArgumentParser() 144 | parser.add_argument('--in', type=str, required=True, help="Input Image Path") 145 | parser.add_argument('--out', type=str, required=False, help="Output JSON Path") 146 | parser.add_argument('--m', type=str, required=False, default=None, help="Path to markers.csv") 147 | args = parser.parse_args() 148 | 149 | # Automatically infer the output filename, if not specified 150 | in_path = vars(args)['in'] 151 | out_path = args.out 152 | if out_path is None: 153 | tokens = os.path.basename(in_path).split(os.extsep) 154 | if len(tokens) < 2: stem = in_path 155 | elif tokens[-2] == "ome": stem = os.extsep.join(tokens[0:-2]) 156 | else: stem = os.extsep.join(tokens[0:-1]) 157 | out_path = stem + ".json" 158 | 159 | main(in_path, out_path, vars(args)['m']) 160 | -------------------------------------------------------------------------------- /modules/illumination.nf: -------------------------------------------------------------------------------- 1 | import mcmicro.* 2 | import java.nio.file.Paths 3 | 4 | def escapeForImagej(s) { 5 | // When passing an arbitrary string as an ImageJ macro parameter value, we 6 | // must backslash-escape backslashes and double-quotes and wrap the whole 7 | // thing with double-quotes. 8 | "\"" + s.toString().replace("\\", "\\\\").replace("\"", "\\\"") + "\"" 9 | } 10 | 11 | process illumination { 12 | container "${params.contPfx}${module.container}:${module.version}" 13 | 14 | // Output profiles 15 | publishDir "${params.in}/illumination/${sname}", mode: "${params.publish_dir_mode}", 16 | pattern: '*.tif' 17 | 18 | // Provenance 19 | publishDir "${Flow.QC(params.in, 'provenance')}", mode: 'copy', 20 | pattern: '.command.{sh,log}', 21 | saveAs: {fn -> fn.replace('.command', "${module.name}-${task.index}")} 22 | 23 | input: 24 | val wfp 25 | val module 26 | tuple val(sname), path(raw), val(relPath) // raw is only for staging, use relPath for paths 27 | output: 28 | tuple val(sname), path('*-dfp.tif'), emit: dfp 29 | tuple val(sname), path('*-ffp.tif'), emit: ffp 30 | tuple path('.command.sh'), path('.command.log') 31 | 32 | when: Flow.doirun('illumination', wfp) 33 | 34 | script: 35 | def relPath = Paths.get(relPath) 36 | def fn = escapeForImagej(relPath) 37 | def xpn = escapeForImagej(relPath.subpath(0, 1)) 38 | def macroParams = Util.escapeForShell( 39 | """filename=$fn,output_dir=".",experiment_name=$xpn""" 40 | ) 41 | """ 42 | /opt/fiji/Fiji.app/ImageJ-linux64 --ij2 --headless \ 43 | --run /opt/fiji/imagej_basic_ashlar.py \ 44 | $macroParams 45 | """ 46 | } 47 | -------------------------------------------------------------------------------- /modules/quantification.nf: -------------------------------------------------------------------------------- 1 | import mcmicro.* 2 | 3 | process mcquant { 4 | container "${params.contPfx}${module.container}:${module.version}" 5 | 6 | // Output 7 | publishDir "${params.in}/quantification", mode: "${params.publish_dir_mode}", 8 | pattern: '*.csv' 9 | 10 | // Provenance 11 | publishDir "${Flow.QC(params.in, 'provenance')}", mode: 'copy', 12 | pattern: '.command.{sh,log}', 13 | saveAs: {fn -> fn.replace('.command', "${module.name}-${task.index}")} 14 | 15 | input: 16 | val mcp 17 | val module 18 | tuple val(tag), path("$tag"), path(masks), path(ch) 19 | 20 | output: 21 | path '*.csv', emit: tables 22 | tuple path('.command.sh'), path('.command.log') 23 | 24 | when: Flow.doirun('quantification', mcp.workflow) 25 | 26 | """ 27 | shopt -s nullglob 28 | mcquant --image $tag \ 29 | ${Opts.moduleOpts(module, mcp)} --output . --channel_names $ch 30 | """ 31 | } 32 | 33 | workflow quantification { 34 | take: 35 | mcp 36 | imgs 37 | segmasks 38 | markers 39 | 40 | main: 41 | 42 | // Determine IDs of images 43 | id_imgs = imgs.map{ f -> tuple(Util.getImageID(f), f) } 44 | 45 | // Determine IDs of segmentation masks 46 | id_msks = segmasks.map{ id, msk -> x = id.split('-',2); tuple(x[1], x[0], msk) } 47 | 48 | // Combine everything based on IDs 49 | inputs = id_msks.combine(id_imgs, by:0) 50 | .map{ id, mtd, msk, img -> 51 | tuple("${Util.getImageID(img)}--${mtd}.ome.tif", img, msk) } 52 | .combine( markers ) 53 | mcquant(mcp, mcp.modules['quantification'], inputs) 54 | 55 | emit: 56 | mcquant.out.tables.flatten() 57 | } 58 | -------------------------------------------------------------------------------- /modules/registration.nf: -------------------------------------------------------------------------------- 1 | import mcmicro.* 2 | 3 | process ashlar { 4 | container "${params.contPfx}${module.container}:${module.version}" 5 | publishDir "${params.in}/registration", mode: "${params.publish_dir_mode}", 6 | pattern: '*.tif' 7 | 8 | // Provenance 9 | publishDir "${Flow.QC(params.in, 'provenance')}", mode: 'copy', 10 | pattern: '.command.{sh,log}', 11 | saveAs: {fn -> fn.replace('.command', "${module.name}")} 12 | 13 | input: 14 | val mcp 15 | val module 16 | tuple val(sampleName), path(lraw), val(lrelPath), path(lffp), path(ldfp) 17 | 18 | output: 19 | path "*.ome.tif", emit: img 20 | tuple path('.command.sh'), path('.command.log') 21 | 22 | when: Flow.doirun('registration', mcp.workflow) 23 | 24 | script: 25 | // Options 26 | def opts = Opts.moduleOpts(module, mcp) 27 | .replace('{samplename}', sampleName) 28 | 29 | // Images 30 | def imgs = opts.contains("filepattern|") || opts.contains("fileseries|") ? "" : 31 | lrelPath.collect{ Util.escapeForShell(it) }.join(" ") 32 | 33 | // Illumination profiles 34 | def ilp = "--ffp $lffp --dfp $ldfp" 35 | if (ilp == '--ffp --dfp ') ilp = '' // Don't supply empty --ffp --dfp 36 | 37 | """ 38 | ashlar $imgs $opts $ilp -o ${sampleName}.ome.tif 39 | """ 40 | } 41 | 42 | workflow registration { 43 | take: 44 | mcp // MCMICRO parameters as read by Opts.parseParams() 45 | raw // raw image tiles 46 | ffp // flat-field profiles 47 | dfp // dark-field profiles 48 | 49 | main: 50 | 51 | srt = {a, b -> file(a).getName() <=> file(b).getName()} 52 | 53 | rawg = raw.groupTuple(sort: srt) 54 | ffpg = ffp.groupTuple(sort: srt) 55 | dfpg = dfp.groupTuple(sort: srt) 56 | 57 | inputs = rawg.join(ffpg, remainder:true).join(dfpg, remainder:true) 58 | .map{tuple( 59 | it[0], it[1], it[2], 60 | it[3] == null ? [] : it[3], // Convert null to empty list 61 | it[4] == null ? [] : it[4] // Ditto 62 | )} 63 | 64 | ashlar(mcp, mcp.modules['registration'], inputs) 65 | 66 | emit: 67 | ashlar.out.img 68 | } 69 | -------------------------------------------------------------------------------- /modules/segmentation.nf: -------------------------------------------------------------------------------- 1 | import mcmicro.* 2 | 3 | process s3seg { 4 | container "${params.contPfx}${module.container}:${module.version}" 5 | 6 | // Output 7 | publishDir "${pubDir}/$tag", mode: "${params.publish_dir_mode}", 8 | pattern: '*/*.ome.tif', saveAs: {f -> file(f).name} 9 | 10 | // QC 11 | publishDir "${Flow.QC(params.in, '/s3seg/' + tag)}", 12 | mode: "${mcp.workflow['qc-files']}", 13 | pattern: '*/qc/**', saveAs: {f -> file(f).name} 14 | 15 | // Provenance 16 | publishDir "${Flow.QC(params.in, 'provenance')}", mode: 'copy', 17 | pattern: '.command.{sh,log}', 18 | saveAs: {fn -> fn.replace('.command', "${module.name}-${task.index}")} 19 | 20 | input: 21 | val mcp 22 | val module 23 | tuple val(tag), path(core), file('mask.tif'), path(probs), val(bypass) 24 | val pubDir 25 | 26 | output: 27 | // output for quantification 28 | tuple val(tag), path("*/*.ome.tif"), emit: segmasks 29 | 30 | // qc and provenance 31 | path('*/qc/**') optional true 32 | tuple path('.command.sh'), path('.command.log') 33 | 34 | when: Flow.doirun('watershed', mcp.workflow) 35 | 36 | script: 37 | def crop = mcp.workflow['tma'] ? 38 | '--crop dearray --maskPath mask.tif' : 39 | '' 40 | """ 41 | python /app/S3segmenter.py $crop \ 42 | --imagePath $core --stackProbPath $probs \ 43 | $bypass ${Opts.moduleOpts(module, mcp)} --outputPath . 44 | """ 45 | } 46 | 47 | include {worker} from "$projectDir/lib/worker" 48 | include {roadie} from "$projectDir/roadie" 49 | 50 | workflow segmentation { 51 | take: 52 | mcp // MCMICRO parameters (as returned by Opts.parseParams()) 53 | imgs // Input images 54 | tmamasks // TMA masks (if any) 55 | prepmaps // Pre-computed probability maps 56 | 57 | main: 58 | // A channel iterating over the segmentation modules 59 | moduleSeg = Channel.of( mcp.modules['segmentation'] ).flatten() 60 | 61 | // Define relevant paths 62 | pathPM = "${params.in}/probability-maps" 63 | pathSeg = "${params.in}/segmentation" 64 | 65 | // Compose a mapping for which modules need watershed 66 | needWS = moduleSeg.map{ it -> tuple(it.watershed, it.name) } 67 | 68 | // Cut out the segmentation channels if requested 69 | recyzeIn = imgs.branch{ 70 | toCut: mcp.workflow['segmentation-recyze'] 71 | noCut: !mcp.workflow['segmentation-recyze'] 72 | } 73 | 74 | // Account for 0-based indexing in recyze 75 | chan = mcp.workflow.containsKey('segmentation-channel') ? 76 | mcp.workflow['segmentation-channel'].toString() 77 | .tokenize().collect{"${(it as int)-1}"}.join(' ') : '0' 78 | 79 | nuc_chan = mcp.workflow.containsKey('segmentation-nuclear-channel') ? 80 | mcp.workflow['segmentation-nuclear-channel'].toString() 81 | .tokenize().collect{"${(it as int)-1}"}.join(' ') : '0' 82 | 83 | mem_chan = mcp.workflow.containsKey('segmentation-membrane-channel') ? 84 | mcp.workflow['segmentation-membrane-channel'].toString() 85 | .tokenize().collect{"${(it as int)-1}"}.join(' ') : '' 86 | 87 | recyzeOut = roadie('recyze', 88 | recyzeIn.toCut, 89 | ["--channels $chan", 90 | mcp.workflow.containsKey('segmentation-nuclear-channel') ? "--nuclear_channels $nuc_chan" : "", 91 | mcp.workflow.containsKey('segmentation-membrane-channel') ? "--membrane_channels $mem_chan" : "", 92 | mcp.workflow['segmentation-max-projection'] ? "--max_projection" : ""].join(" ").trim(), 93 | false, '', '' ) 94 | 95 | // Determine IDs of images 96 | id_cut = recyzeOut.map{ f -> tuple(Util.getFileID(f, '_crop.ome'), f) } 97 | id_uncut = recyzeIn.noCut.map{ f -> tuple(Util.getImageID(f), f) } 98 | id_imgs = id_cut.mix(id_uncut) 99 | 100 | // Determine if there are any custom models for each module 101 | // Overwrite output filenames with -pmap.tif for pmap generators 102 | // Publish instance segmentation outputs directly to segmentation/ 103 | inpPM = moduleSeg.map{ it -> String m = "${it.name}-model"; 104 | tuple(it, mcp.workflow.containsKey(m) ? 105 | file(mcp.workflow[m]) : 'built-in') } 106 | .combine(id_imgs) 107 | .map{ mod, _2, tag, f -> 108 | mod.watershed == 'no' ? 109 | tuple(tag, mod, _2, f, "${pathSeg}/${mod.name}-${tag}", '') : 110 | tuple(tag, mod, _2, f, "${pathPM}/${mod.name}", tag + '-pmap.tif') } 111 | 112 | // Run probability map generators and instance segmenters 113 | // All outputs will be published to probability-maps/ 114 | worker( mcp, inpPM, '*.{tif,tiff}', 'segmentation' ) 115 | 116 | // Merge against precomputed probability maps 117 | // and information about whether the module needs watershed 118 | allpmaps = prepmaps.map{ mtd, f -> 119 | tuple(Util.getFileID(f, '-pmap'), mtd, f) } 120 | .mix(worker.out.res) 121 | .combine( needWS, by:1 ) // changes order to (mtd, tag, f, ws) 122 | 123 | // Filter out any workers who published their files to segmentation/ 124 | // i.e., all the instance segmenters 125 | // Add nuclear segmentation bypass to those that require it 126 | id_pmaps = allpmaps.filter{ _1, _2, _3, ws -> ws != 'no' } 127 | .map{ mtd, tag, _3, ws -> ws == 'bypass' ? 128 | tuple(tag, mtd, _3, '--nucleiRegion bypass') : 129 | tuple(tag, mtd, _3, '') } 130 | 131 | // Determine IDs of TMA masks 132 | // Whole-slide images have no TMA masks 133 | id_wsi = id_imgs.map{ id, _2 -> tuple(id, 'NO_MASK') } 134 | .filter{ !mcp.workflow['tma'] } 135 | id_masks = tmamasks.map{ f -> tuple(Util.getFileID(f,'_mask'), f) } 136 | .mix(id_wsi) 137 | 138 | // Combine everything based on IDs 139 | inputs = id_imgs.join(id_masks).combine( id_pmaps, by:0 ) 140 | .map{ tag, img, msk, mtd, pm, bypass -> 141 | tuple("${mtd}-${tag}", img, msk, pm, bypass) } 142 | 143 | // Apply s3seg to probability-maps only 144 | s3seg( mcp, mcp.modules['watershed'], inputs, pathSeg ) 145 | 146 | // Merge against instance segmentation outputs 147 | instSeg = allpmaps.filter{ _1, _2, _3, ws -> ws == 'no' } 148 | .map{ mtd, tag, _3, _4 -> tuple("${mtd}-${tag}", _3) }.groupTuple() 149 | 150 | emit: 151 | s3seg.out.segmasks.mix(instSeg) 152 | } 153 | -------------------------------------------------------------------------------- /modules/staging.nf: -------------------------------------------------------------------------------- 1 | import mcmicro.* 2 | 3 | import groovy.text.GStringTemplateEngine 4 | 5 | process phenoimager2mc { 6 | // Use the container specification from the parameter file 7 | container "${params.contPfx}${module.container}:${module.version}" 8 | publishDir "${params.in}/raw", mode: "${params.publish_dir_mode}", 9 | pattern: '*.tif' 10 | 11 | // Provenance 12 | publishDir "${Flow.QC(params.in, 'provenance')}", mode: 'copy', 13 | pattern: '.command.{sh,log}', 14 | saveAs: {fn -> fn.replace('.command', "${module.name}")} 15 | 16 | input: 17 | val mcp 18 | val module 19 | tuple val(samplename), val(cycle), path(indir), path(marker) 20 | 21 | output: 22 | tuple val(samplename), val(cycle), path("*.tif"), emit: img 23 | tuple path('.command.sh'), path('.command.log') 24 | 25 | when: Flow.doirun('staging', mcp.workflow) 26 | 27 | script: 28 | def formatMap = [ 29 | marker: marker, 30 | cycle: cycle, 31 | indir: indir, 32 | ] 33 | 34 | def command = new GStringTemplateEngine() 35 | .createTemplate(module.cmd) 36 | .make(formatMap) 37 | .toString() 38 | 39 | """ 40 | ${command} ${Opts.moduleOpts(module, mcp)} 41 | """ 42 | } 43 | 44 | workflow staging { 45 | take: 46 | mcp // MCMICRO parameters as read by Opts.parseParams() 47 | indir // input directories 48 | marker // marker file 49 | 50 | main: 51 | inputs = indir 52 | .combine(marker) 53 | 54 | phenoimager2mc(mcp, mcp.modules['staging'], inputs) 55 | 56 | emit: 57 | phenoimager2mc.out.img 58 | } 59 | -------------------------------------------------------------------------------- /modules/template.nf: -------------------------------------------------------------------------------- 1 | /* 2 | A template for adding new modules to MCMICRO 3 | 4 | Step 1: Add module specs to config/defaults.yml 5 | 6 | a: Add a flag specifying whether the module should be run to workflow: 7 | b: Add default module options to options: 8 | c: Add module name and container specs to modules: 9 | 10 | For example, support we wanted to add a module that produces a QC report 11 | about the signal-to-noise ratio (snr). The three additions to defaults.yml 12 | may then look as follows: 13 | 14 | workflow: 15 | report: false 16 | options: 17 | snr: --cool-parameter 42 18 | modules: 19 | report: 20 | name: snr 21 | container: myorganization/snr 22 | version: 1.0.0 23 | 24 | Step 2: Modify the code below as needed 25 | 26 | Step 3: Run the module from the main workflow in main.nf 27 | 28 | a: add an include statement to import the relevant workflow. For example: 29 | 30 | ... 31 | include {downstream} from "$projectDir/modules/downstream" 32 | include {viz} from "$projectDir/modules/viz" 33 | include {report} from "$projectDir/modules/report" // <- importing the new module 34 | 35 | b: add a statement calling the module near the bottom of the main workflow: 36 | 37 | ... 38 | downstream(mcp, sft) 39 | 40 | report(mcp, allimg, sft) // <- calling the new module 41 | 42 | // Vizualization 43 | viz(mcp, allimg) 44 | ... 45 | 46 | */ 47 | 48 | // Import utility functions from lib/mcmicro/*.groovy 49 | import mcmicro.* 50 | 51 | // Process name will appear in the the nextflow execution log 52 | // While not strictly required, it's a good idea to make the 53 | // process name match your tool name to avoid user confusion 54 | process snr { 55 | 56 | // Use the container specification from the parameter file 57 | // No change to this line is required 58 | container "${params.contPfx}${module.container}:${module.version}" 59 | 60 | // Specify the project subdirectory for writing the outputs to 61 | // The pattern: specification must match the output: files below 62 | // TODO: replace report with the desired output directory 63 | // TODO: replace the pattern to match the output: clause below 64 | publishDir "${params.in}/report", mode: "${params.publish_dir_mode}", 65 | pattern: "*.html" 66 | 67 | // Stores .command.sh and .command.log from the work directory 68 | // to the project provenance 69 | // No change to this line is required 70 | publishDir "${Flow.QC(params.in, 'provenance')}", mode: 'copy', 71 | pattern: '.command.{sh,log}', 72 | saveAs: {fn -> fn.replace('.command', "${module.name}-${task.index}")} 73 | 74 | // Inputs for the process 75 | // mcp - MCMICRO parameters (workflow, options, etc.) 76 | // module - module specifications (name, container, options, etc.) 77 | // img/sft - pairs of images and their matching spatial feature tables 78 | input: 79 | val mcp 80 | val module 81 | tuple path(img), path(sft) 82 | 83 | // Process outputs that should be captured and 84 | // a) returned as results 85 | // b) published to the project directory 86 | // TODO: replace *.html with the pattern of the tool output files 87 | output: 88 | path("*.html"), emit: results 89 | 90 | // Provenance files -- no change is needed here 91 | tuple path('.command.sh'), path('.command.log') 92 | 93 | // Specifies whether to run the process 94 | // Here, we simply take the flag from the workflow parameters 95 | // TODO: change snr to match the true/false workflow parameter in defaults.yml 96 | when: mcp.workflow["report"] 97 | 98 | // The command to be executed inside the tool container 99 | // The command must write all outputs to the current working directory (.) 100 | // Opts.moduleOpts() will identify and return the appropriate module options 101 | """ 102 | python /app/mytool.py --image $img --features $sft ${Opts.moduleOpts(module, mcp)} 103 | """ 104 | } 105 | 106 | workflow report { 107 | 108 | // Inputs: 109 | // mcp - MCMICRO parameters (workflow, options, etc.) 110 | // imgs - images 111 | // sfts - spatial feature tables 112 | take: 113 | mcp 114 | imgs 115 | sfts 116 | 117 | main: 118 | 119 | // Match images against feature tables 120 | id_imgs = imgs.map{ it -> tuple(Util.getImageID(it), it) } 121 | id_sfts = sfts.map{ it -> tuple(Util.getFileID(it, '--'), it) } 122 | 123 | // Apply the process to each (image, sft) pair 124 | id_imgs.combine(id_sfts, by:0) 125 | .map{ tag, img, sft -> tuple(img, sft) } | snr 126 | 127 | // Return the outputs produced by the tool 128 | emit: 129 | snr.out.results 130 | } -------------------------------------------------------------------------------- /modules/viz.nf: -------------------------------------------------------------------------------- 1 | import mcmicro.* 2 | 3 | process autominerva { 4 | container "${params.contPfx}${module.container}:${module.version}" 5 | 6 | // Output 7 | publishDir "${params.in}/viz", mode: "${params.publish_dir_mode}", pattern: "$tag/**" 8 | 9 | // Provenance 10 | publishDir "${Flow.QC(params.in, 'provenance')}", mode: 'copy', 11 | pattern: '.command.{sh,log}', 12 | saveAs: {fn -> fn.replace('.command', "${module.name}-${task.index}")} 13 | 14 | input: 15 | val wfp 16 | val module 17 | tuple val(tag), path(img), path(markers) 18 | 19 | output: 20 | path("${tag}/**"), emit: viz 21 | 22 | // qc and provenance 23 | path('*/qc/**') optional true 24 | tuple path('.command.sh'), path('.command.log') 25 | 26 | when: Flow.doirun('viz', wfp) 27 | 28 | """ 29 | python /app/story.py --in $img --m $markers --out story.json 30 | python /app/minerva-author/src/save_exhibit_pyramid.py $img story.json $tag 31 | """ 32 | } 33 | 34 | workflow viz { 35 | take: 36 | mcp 37 | imgs 38 | markers 39 | 40 | main: 41 | 42 | inputs = imgs.map{ it -> tuple(Util.getImageID(it), it) }.combine( markers ) 43 | autominerva(mcp.workflow, mcp.modules['viz'], inputs) 44 | 45 | emit: 46 | autominerva.out.viz 47 | } -------------------------------------------------------------------------------- /nextflow.config: -------------------------------------------------------------------------------- 1 | // Execution environment for miscellaneous tasks 2 | params.roadie = 'ghcr.io/labsyspharm/mcmicro:roadie-2023-10-25' 3 | 4 | // The mode for transfering output between work and project directories 5 | params.publish_dir_mode = 'copy' 6 | 7 | // Platform-specific profiles 8 | profiles { 9 | 10 | /**************************** 11 | * Platform-specific configs * 12 | ****************************/ 13 | 14 | // Default profile, best suited for local runs 15 | standard { 16 | includeConfig 'config/nf/docker.config' 17 | } 18 | 19 | // Profile for using singularity to run module containers 20 | singularity { 21 | includeConfig 'config/nf/singularity.config' 22 | } 23 | 24 | // Profile specific to the HMS compute cluster 25 | O2 { 26 | includeConfig 'config/nf/O2base.config' 27 | singularity.cacheDir = '/n/groups/lsp/mcmicro/singularity' 28 | } 29 | 30 | // Same as O2, but for external users not in the `lsp` group 31 | O2ext { 32 | includeConfig 'config/nf/O2base.config' 33 | singularity.cacheDir = "$HOME/.mcmicro" 34 | } 35 | 36 | // Amazon Web Services 37 | AWS { 38 | includeConfig 'config/nf/aws.config' 39 | } 40 | 41 | // Nextflow tower 42 | tower { 43 | includeConfig 'config/nf/tower.config' 44 | } 45 | 46 | /**************************** 47 | * Resource-specific configs * 48 | ****************************/ 49 | 50 | WSI { 51 | includeConfig 'config/nf/wsi.config' 52 | } 53 | 54 | TMA { 55 | includeConfig 'config/nf/tma.config' 56 | } 57 | 58 | GPU { 59 | docker.runOptions = '--cpus 0.000 -u root --entrypoint "" --gpus all' 60 | singularity.runOptions = '-C -H "$PWD" --nv' 61 | 62 | process{ 63 | withName: 'segmentation:worker' { 64 | queue = 'gpu' 65 | clusterOptions = '--gres=gpu:1' 66 | } 67 | } 68 | } 69 | 70 | /************************** 71 | * Backwards compatibility * 72 | **************************/ 73 | 74 | // Profiles below this line are deprecated 75 | // Left here for backwards compatibility 76 | 77 | // Same as -profile O2,WSI 78 | O2WSI { 79 | includeConfig 'config/nf/O2base.config' 80 | includeConfig 'config/nf/wsi.config' 81 | } 82 | 83 | // Same as -profile O2,TMA 84 | O2TMA { 85 | includeConfig 'config/nf/O2base.config' 86 | includeConfig 'config/nf/tma.config' 87 | } 88 | 89 | // Same as -profile O2,WSI 90 | O2massive { 91 | includeConfig 'config/nf/O2base.config' 92 | includeConfig 'config/nf/wsi.config' 93 | } 94 | 95 | } 96 | -------------------------------------------------------------------------------- /roadie.nf: -------------------------------------------------------------------------------- 1 | nextflow.enable.dsl=2 2 | 3 | import org.yaml.snakeyaml.Yaml 4 | 5 | def roadieHelp() { 6 | println """ 7 | Roadie: miscellaneous MCMICRO-related tasks 8 | 9 | Usage: 10 | To run a task, 11 | nextflow run labsyspharm/mcmicro/roadie.nf --do 12 | 13 | To list available tasks, 14 | nextflow run labsyspharm/mcmicro/roadie.nf --list-tasks 15 | 16 | To get help about individual tasks, 17 | nextflow run labsyspharm/mcmicro/roadie.nf --help 18 | 19 | Examples: 20 | 21 | Show help for the recyze task: 22 | nextflow run labsyspharm/mcmicro/roadie.nf --help recyze 23 | 24 | Make a 1024x1024 crop from myimage.ome.tif: 25 | nextflow run labsyspharm/mcmicro/roadie.nf --do recyze \\ 26 | --in myimage.ome.tif --x 0 --y 0 --w 1024 --h 1024 27 | 28 | Derive an auto-minerva story and write the output to result/: 29 | nextflow run labsyspharm/mcmicro/roadie.nf --do story \\ 30 | --in myimage.ome.tif --out result/ 31 | """ 32 | } 33 | 34 | process showHelp { 35 | executor 'local' 36 | container "${params.contPfx}${params.roadie}" 37 | 38 | when: params.containsKey('help') 39 | input: path(code) 40 | output: stdout 41 | 42 | """ 43 | echo '' 44 | python $code --help 45 | """ 46 | } 47 | 48 | process runTask { 49 | container "${params.contPfx}${params.roadie}" 50 | publishDir "${specs.pubDir}", mode: "${specs.pubMode}", enabled: "${specs.pub}" 51 | 52 | input: each path(code); path(input); val(opts); val(specs) 53 | output: path("${specs.output}") 54 | 55 | """ 56 | python $code --in $input $opts 57 | """ 58 | } 59 | 60 | // Internal interface to be used by MCMICRO 61 | workflow roadie { 62 | take: 63 | task 64 | input 65 | opts 66 | 67 | pub // Whether to publish results 68 | pubDir // Where to publish results to 69 | pubMode // What type of publishing (copy, move, symlink) 70 | 71 | main: 72 | // Parse task specs 73 | tasks = new Yaml().load(file("$projectDir/roadie/tasks.yml")) 74 | specs = tasks.containsKey(task) ? tasks[task] : (error "Unknown task.") 75 | 76 | // Pad specs with the publication strategy 77 | specs.pub = pub 78 | specs.pubDir = pubDir 79 | specs.pubMode = pubMode 80 | 81 | // Identify and execute the script 82 | code = Channel.fromPath("$projectDir/roadie/scripts/${task}.py") 83 | runTask(code, input, opts, specs) 84 | 85 | emit: 86 | runTask.out 87 | } 88 | 89 | // Command-line interface 90 | workflow { 91 | // By default, write all output to the current directory 92 | params.out = '.' 93 | 94 | if((params.containsKey('help') && (params.help instanceof Boolean)) || 95 | (!params.containsKey('help') && 96 | !params.containsKey('do') && 97 | !params.containsKey('list-tasks') 98 | )) { 99 | roadieHelp() 100 | exit 0 101 | } 102 | 103 | // Parse task specs 104 | tasks = new Yaml().load(file("$projectDir/roadie/tasks.yml")) 105 | 106 | // List available tasks 107 | if(params.containsKey('list-tasks')) { 108 | println "Available tasks:" 109 | tasks.each{ key, val -> 110 | println " " + key.padRight(12, ' ') + "- " + val.description 111 | } 112 | exit 0 113 | } 114 | 115 | // Retrieve the appropriate specs 116 | task = params.containsKey('do') ? params.do : params.help 117 | specs = tasks.containsKey(task) ? tasks[task] : (error "Unknown task.") 118 | 119 | // Identify the script 120 | code = Channel.fromPath("$projectDir/roadie/scripts/${task}.py") 121 | 122 | // Display task help, if requested 123 | showHelp(code).view() 124 | 125 | // Verify the presence of input parameters 126 | if(params.containsKey('do')) { 127 | if(!params.containsKey('in')) 128 | error "Please provide input via --in" 129 | 130 | // Forward the appropriate parameters to the task script 131 | opts = params.inject('') { 132 | prev, key, val -> specs.params.indexOf(key) > -1 ? 133 | prev + ' --' + key + ' ' + val : prev + '' 134 | } 135 | 136 | // Split up the out argument into its directory/file components 137 | // Directory will be passed to the publishDir directive 138 | // Filename will be passed to the tool 139 | out = file(params.out) 140 | (outd, outf) = out.isDirectory() ? [params.out, ''] : 141 | [out.getParent(), out.getName()] 142 | opts = opts + ' ' + ((outf == '') ? outf : "--out " + outf) 143 | 144 | // Identify the input file and execute the task 145 | inp = Channel.fromPath(params.in) 146 | roadie(task, inp, opts, true, outd, 'move') 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /roadie/scripts/pyramidize.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import pathlib 3 | 4 | import ome_types 5 | import palom.pyramid 6 | import palom.reader 7 | 8 | 9 | def detect_pixel_size(img_path): 10 | try: 11 | metadata = ome_types.from_tiff(img_path) 12 | pixel_size = metadata.images[0].pixels.physical_size_x 13 | except Exception as err: 14 | print(err) 15 | print() 16 | print('Pixel size detection using ome-types failed') 17 | pixel_size = None 18 | return pixel_size 19 | 20 | 21 | def _file(path): 22 | path = pathlib.Path(path) 23 | if path.is_file(): return path 24 | else: raise FileNotFoundError(path.resolve()) 25 | 26 | 27 | if __name__ == "__main__": 28 | parser = argparse.ArgumentParser() 29 | parser.add_argument( 30 | '--in', 31 | nargs='+', 32 | type=_file, 33 | required=True, 34 | help="Input Image Paths" 35 | ) 36 | parser.add_argument('--out', type=str, required=False, help="Output Image Path") 37 | args = parser.parse_args() 38 | 39 | in_paths = vars(args)['in'] 40 | # Automatically infer the output filename, if not specified 41 | if args.out is None: 42 | in_path = in_paths[0] 43 | stem = in_path.stem 44 | out_path = in_path.parent / f"{stem}.ome.tif" 45 | else: 46 | out_path = pathlib.Path(args.out) 47 | # pixel data is read into RAM lazily, cannot overwrite input file 48 | assert out_path not in in_paths 49 | 50 | # Detect pixel size in ome-xml 51 | pixel_size = detect_pixel_size(in_path) 52 | if pixel_size is None: pixel_size = 1 53 | 54 | # Use palom to pyramidize the input image 55 | readers = [palom.reader.OmePyramidReader(in_path) for in_path in in_paths] 56 | mosaics = [reader.pyramid[0] for reader in readers] 57 | palom.pyramid.write_pyramid( 58 | mosaics, out_path, downscale_factor=2, pixel_size=pixel_size 59 | ) 60 | -------------------------------------------------------------------------------- /roadie/tasks.yml: -------------------------------------------------------------------------------- 1 | recyze: 2 | description: channel and pixel cropping 3 | output: '*.ome.tif*' 4 | params: ['x', 'x2', 'y', 'y2', 'w', 'h', 'channels', 'nuclear_channels', 'membrane_channels', 'max_projection'] 5 | pyramidize: 6 | description: generate a pyramid for the input image 7 | output: '*.ome.tif' 8 | params: [] 9 | -------------------------------------------------------------------------------- /setup/O2ext.nf: -------------------------------------------------------------------------------- 1 | process O2ext { 2 | executor 'local' 3 | 4 | ''' 5 | rm -rfv $HOME/.mcmicro 6 | mkdir -pv $HOME/.mcmicro 7 | ln -sv /n/groups/lsp/mcmicro/singularity/* $HOME/.mcmicro 8 | ''' 9 | } 10 | 11 | workflow { 12 | O2ext() 13 | } 14 | -------------------------------------------------------------------------------- /setup/cloudformation/create-batch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | aws cloudformation create-stack --stack-name mcmicro-nextflow-batch --template-body file://$PWD/mcmicro-nextflow-batch.yml --parameters file://$PWD/parameters-batch.json --capabilities CAPABILITY_NAMED_IAM 3 | 4 | retVal=$? 5 | if [ $retVal -ne 0 ]; then 6 | echo "* If the stack has been previously created, run update-stack.sh instead" 7 | fi -------------------------------------------------------------------------------- /setup/cloudformation/create-s3.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | aws cloudformation create-stack --stack-name mcmicro-nextflow-s3 --template-body file://$PWD/mcmicro-nextflow-s3.yml --parameters file://$PWD/parameters-s3.json 3 | 4 | retVal=$? 5 | if [ $retVal -ne 0 ]; then 6 | echo "* If the stack has been previously created, run update-stack.sh instead" 7 | fi -------------------------------------------------------------------------------- /setup/cloudformation/mcmicro-nextflow-s3.yml: -------------------------------------------------------------------------------- 1 | --- 2 | AWSTemplateFormatVersion: '2010-09-09' 3 | Description: >- 4 | S3 resources for running mcmicro pipeline in AWS Batch 5 | 6 | Parameters: 7 | BucketPrefix: 8 | Type: String 9 | Description: Prefix for the Nextflow S3 buckets 10 | ProjectTag: 11 | Type: String 12 | Description: Project tag 13 | 14 | Resources: 15 | McmicroNextflowWorkBucket: 16 | Type: AWS::S3::Bucket 17 | Properties: 18 | BucketName: !Sub ${BucketPrefix}-work 19 | Tags: 20 | - Key: project 21 | Value: !Ref ProjectTag 22 | 23 | McmicroNextflowInBucket: 24 | Type: AWS::S3::Bucket 25 | Properties: 26 | BucketName: !Sub ${BucketPrefix}-in 27 | Tags: 28 | - Key: project 29 | Value: !Ref ProjectTag 30 | 31 | McmicroNextflowOutBucket: 32 | Type: AWS::S3::Bucket 33 | Properties: 34 | BucketName: !Sub ${BucketPrefix}-out 35 | Tags: 36 | - Key: project 37 | Value: !Ref ProjectTag 38 | 39 | Outputs: 40 | McmicroNextflowWorkBucketArn: 41 | Description: S3 work bucket Arn 42 | Value: !GetAtt McmicroNextflowWorkBucket.Arn 43 | Export: 44 | Name: mcmicro-nextflow-work-bucket-arn 45 | McmicroNextflowInBucketArn: 46 | Description: S3 in bucket Arn 47 | Value: !GetAtt McmicroNextflowInBucket.Arn 48 | Export: 49 | Name: mcmicro-nextflow-in-bucket-arn 50 | McmicroNextflowOutBucketArn: 51 | Description: S3 out bucket Arn 52 | Value: !GetAtt McmicroNextflowOutBucket.Arn 53 | Export: 54 | Name: mcmicro-nextflow-out-bucket-arn 55 | -------------------------------------------------------------------------------- /setup/cloudformation/parameters-batch.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "ParameterKey":"VpcId", 4 | "ParameterValue":"vpc-xxxxxxxx" 5 | }, 6 | { 7 | "ParameterKey":"ComputeEnvironmentNamePrefix", 8 | "ParameterValue":"mcmicro-compute" 9 | }, 10 | { 11 | "ParameterKey":"ProjectTag", 12 | "ParameterValue":"mcmicro" 13 | }, 14 | { 15 | "ParameterKey":"NextflowAMI", 16 | "ParameterValue":"ami-08c1bd5176f3daa3e" 17 | }, 18 | { 19 | "ParameterKey":"NextflowAMIGPU", 20 | "ParameterValue":"ami-0e2c49a125d6cf87f" 21 | }, 22 | { 23 | "ParameterKey":"BatchClusterEC2MinCpus", 24 | "ParameterValue": "0" 25 | }, 26 | { 27 | "ParameterKey":"BatchClusterEC2MaxCpus", 28 | "ParameterValue": "256" 29 | }, 30 | { 31 | "ParameterKey":"BatchClusterEC2DesiredCpus", 32 | "ParameterValue": "0" 33 | }, 34 | { 35 | "ParameterKey":"BatchClusterSpotMinCpus", 36 | "ParameterValue": "0" 37 | }, 38 | { 39 | "ParameterKey":"BatchClusterSpotMaxCpus", 40 | "ParameterValue": "256" 41 | }, 42 | { 43 | "ParameterKey":"BatchClusterSpotDesiredCpus", 44 | "ParameterValue": "0" 45 | }, 46 | { 47 | "ParameterKey":"BatchClusterSpotDesiredCpus", 48 | "ParameterValue": "0" 49 | }, 50 | { 51 | "ParameterKey":"BatchClusterSpotBidPercentage", 52 | "ParameterValue":"50" 53 | }, 54 | { 55 | "ParameterKey":"BatchSubnets", 56 | "ParameterValue": "subnet-xxxxxx,subnet-xxxxxx" 57 | } 58 | ] -------------------------------------------------------------------------------- /setup/cloudformation/parameters-s3.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "ParameterKey":"BucketPrefix", 4 | "ParameterValue":"mcmicro" 5 | }, 6 | { 7 | "ParameterKey":"ProjectTag", 8 | "ParameterValue":"mcmicro" 9 | } 10 | ] -------------------------------------------------------------------------------- /setup/cloudformation/update-batch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | aws cloudformation update-stack --stack-name mcmicro-nextflow-batch --template-body file://$PWD/mcmicro-nextflow-batch.yml --parameters file://$PWD/parameters-batch.json --capabilities CAPABILITY_NAMED_IAM 3 | -------------------------------------------------------------------------------- /setup/cloudformation/update-s3.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | aws cloudformation update-stack --stack-name mcmicro-nextflow-s3 --template-body file://$PWD/mcmicro-nextflow-s3.yml --parameters file://$PWD/parameters-s3.json --------------------------------------------------------------------------------