├── .gitignore ├── .pre-commit-config.yaml ├── .travis.yml ├── LICENSE ├── README.md ├── bin ├── cellposeStaging.py ├── imgProcessing.py ├── main.py ├── pseudoSort.py ├── qcDriver.py ├── segmentationDriver.py ├── spaceTxConverter.py └── starfishDriver.py ├── docker ├── baysor │ └── Dockerfile ├── cellpose │ └── Dockerfile ├── starfish-custom │ └── Dockerfile ├── starfish-docker-runner │ └── Dockerfile └── starfish │ └── Dockerfile ├── docker_images.txt ├── flowchart.svg ├── input_schemas ├── cellpose.json ├── pipeline.json ├── processing.json ├── psortedDefaultParams.json ├── qc.json ├── segmentation.json ├── sorter.json ├── spaceTxConversion.json └── starfishRunner.json ├── logo.png ├── pipeline-manifest.json ├── pipeline.cwl ├── pyproject.toml ├── requirements.txt ├── requirements_test.txt ├── steps ├── baysor.cwl ├── baysorStaged.cwl ├── cellpose.cwl ├── fileSizer.cwl ├── inputParser.cwl ├── processing.cwl ├── psortedDefaultParams.cwl ├── qc.cwl ├── segmentation.cwl ├── sorter.cwl ├── spaceTxConversion.cwl ├── starfishRunner.cwl └── tmpdir.cwl └── test.sh /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | .*.kate-swp 3 | .*.swp 4 | .idea/ 5 | __pycache__/ 6 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/psf/black 3 | rev: 24.1.1 4 | hooks: 5 | - id: black 6 | language_version: python3 7 | - repo: https://github.com/pycqa/isort 8 | rev: 5.12.0 9 | hooks: 10 | - id: isort 11 | args: ["--profile", "black"] 12 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: focal 2 | language: python 3 | python: 3.8 4 | install: 5 | - pip install -r requirements_test.txt 6 | script: 7 | - ./test.sh 8 | -------------------------------------------------------------------------------- /bin/cellposeStaging.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import glob 4 | from argparse import ArgumentParser 5 | from copy import deepcopy 6 | from os import makedirs 7 | from pathlib import Path 8 | 9 | import numpy as np 10 | import skimage.measure 11 | import skimage.segmentation 12 | import tifffile 13 | from scipy.ndimage import gaussian_filter 14 | from starfish.core.intensity_table.decoded_intensity_table import DecodedIntensityTable 15 | 16 | 17 | def scale_img(image): 18 | image = image - np.min(image) 19 | image = image.astype("float32") / np.max(image) 20 | image = image * 2**16 21 | image[image == 2**16] = 2**16 - 1 22 | return np.rint(image).astype("uint16") 23 | 24 | 25 | def _clip_percentile_to_zero(image, p_min, p_max, min_coeff=1, max_coeff=1): 26 | v_min, v_max = np.percentile(image, [p_min, p_max]) 27 | v_min = min_coeff * v_min 28 | v_max = max_coeff * v_max 29 | return np.clip(image, v_min, v_max) - np.float32(v_min) 30 | 31 | 32 | def cellpose_format(output_dir, input_dir, aux_ch_names, mRNA_dir, selected_fovs): 33 | # Get all fov names 34 | if selected_fovs is None: 35 | primary_jsons = glob.glob(f"{input_dir}/primary-*.json") 36 | fovs = [ 37 | primary_json.split("/")[-1].split("-")[-1].split(".")[0] 38 | for primary_json in primary_jsons 39 | ] 40 | else: 41 | fovs = ["fov_{:05}".format(int(f)) for f in selected_fovs] 42 | 43 | # Get number of z slices by looking at the fov_000 files 44 | fov0_files = glob.glob(f"{input_dir}/primary-{fovs[0]}*.tiff") 45 | shape = tifffile.imread(fov0_files[0]).shape # Get image xy shape (will need later) 46 | fov0_files = [file.split("/")[-1] for file in fov0_files] 47 | zs = np.max([int(file.split("-")[-1][1]) for file in fov0_files]) 48 | 49 | # Make folder for cellpose inputs if it doesn't exist 50 | makedirs(output_dir, exist_ok=True) 51 | 52 | # Create cellpose inputs 53 | # Images have to be 16 bit and have dimension order z, ch, y, x 54 | for fov in fovs: 55 | mRNA_ch = 1 if mRNA_dir else 0 56 | empty_ch = 1 if len(aux_ch_names) == 1 and not mRNA_ch else 0 57 | new_img = np.zeros( 58 | [zs + 1, len(aux_ch_names) + mRNA_ch + empty_ch] + list(shape), dtype="uint16" 59 | ) 60 | for ch, aux_ch_name in enumerate(aux_ch_names): 61 | files = sorted(glob.glob(f"{input_dir}/{aux_ch_name}-{fov}-c0-r0-z*.tiff")) 62 | for z, file in enumerate(files): 63 | img = tifffile.imread(file) 64 | if np.max(img) <= 1: 65 | img = np.rint(img * 2**16).astype("uint16") 66 | new_img[z, ch] = deepcopy(img) 67 | new_img[:, ch] = _clip_percentile_to_zero(new_img[:, ch], p_min=0, p_max=99.9) 68 | new_img[:, ch] = scale_img(new_img[:, ch]) 69 | 70 | # Add mRNA density channel if specified 71 | # Each mRNA is plotted as a single point of maximum intensity and the the resulting image is then 72 | # blurred with a guassian filter. 73 | if mRNA_dir: 74 | dit = DecodedIntensityTable.open_netcdf(f"{mRNA_dir}/cdf/{fov}_decoded.cdf") 75 | coords = np.array( 76 | [[z, y, x] for z, y, x in zip(dit["z"].data, dit["y"].data, dit["x"].data)] 77 | ) 78 | mRNAs = np.zeros([zs + 1] + list(shape), dtype="uint16") 79 | mRNAs[tuple(coords.T)] = 2**16 - 1 80 | for z in range(zs + 1): 81 | mRNAs[z] = gaussian_filter( 82 | mRNAs[z], sigma=10, cval=0, truncate=4.0, mode="nearest" 83 | ) 84 | new_img[:, -1] = scale_img(mRNAs) 85 | 86 | # Save result, squeeze out any size 1 dimensions 87 | tifffile.imsave(f"{output_dir}/{fov}_image.tiff", np.squeeze(new_img)) 88 | 89 | 90 | def filter_cellpose( 91 | output_dir, input_dir, border_buffer=None, label_exp_size=None, min_size=None, max_size=None 92 | ): 93 | # Make folder if it doesn't exist 94 | makedirs(output_dir, exist_ok=True) 95 | 96 | # For each file, check for each function if it should be run then run it if yes 97 | files = glob.glob(f"{input_dir}/*cp_masks*") 98 | for file in files: 99 | # print(f"found {file}") 100 | mask = tifffile.imread(file) 101 | 102 | # Clear border objects 103 | if border_buffer is not None: 104 | # print(f"\tclearing border of size {border_buffer}") 105 | if mask.ndim == 3: 106 | for z in range(mask.shape[0]): 107 | mask[z] = skimage.segmentation.clear_border(mask[z], buffer_size=border_buffer) 108 | else: 109 | mask = skimage.segmentation.clear_border(mask, buffer_size=border_buffer) 110 | mask = skimage.segmentation.relabel_sequential(mask)[0] 111 | 112 | # Expand labels 113 | if label_exp_size is not None: 114 | # print(f"\texpanding labels by size {label_exp_size}") 115 | if mask.ndim == 3: 116 | for z in range(mask.shape[0]): 117 | mask[z] = skimage.segmentation.expand_labels(mask[z], distance=label_exp_size) 118 | else: 119 | mask = skimage.segmentation.expand_labels(mask, distance=label_exp_size) 120 | 121 | # Remove labels below a minimum size threshold 122 | if min_size is not None: 123 | # print(f"\tcells beneath size {min_size} being removed") 124 | props = skimage.measure.regionprops(mask) 125 | small_labels = np.where([p.area < min_size for p in props])[0] + 1 126 | mask[np.isin(mask, small_labels)] = 0 127 | mask = skimage.segmentation.relabel_sequential(mask)[0] 128 | 129 | # Remove labels above a maximum size threshold 130 | if max_size is not None: 131 | # print(f"\tcells above size {max_size} being removed") 132 | props = skimage.measure.regionprops(mask) 133 | big_labels = np.where([p.area > max_size for p in props])[0] + 1 134 | mask[np.isin(mask, big_labels)] = 0 135 | mask = skimage.segmentation.relabel_sequential(mask)[0] 136 | 137 | # Save result 138 | tifffile.imsave(f'{output_dir}/fov_{file.split("fov_")[-1][:5]}_masks.tiff', mask) 139 | 140 | 141 | if __name__ == "__main__": 142 | p = ArgumentParser() 143 | 144 | p.add_argument("--input-dir", type=Path) 145 | p.add_argument("--tmp-prefix", type=str) 146 | p.add_argument("--selected-fovs", nargs="+", const=None) 147 | 148 | p.add_argument("--format", dest="format", action="store_true") 149 | p.add_argument("--aux-views", type=str, nargs="+") 150 | p.add_argument("--decoded-dir", type=Path, nargs="?") 151 | 152 | p.add_argument("--filter", dest="filter", action="store_true") 153 | p.add_argument("--border-buffer", type=int, nargs="?") 154 | p.add_argument("--label-exp-size", type=int, nargs="?") 155 | p.add_argument("--min-size", type=int, nargs="?") 156 | p.add_argument("--max-size", type=int, nargs="?") 157 | 158 | args = p.parse_args() 159 | 160 | if not (args.format ^ args.filter): 161 | raise ValueError("Script must be run with --format xor --filter. Terminating.") 162 | 163 | if args.format: 164 | cellpose_format( 165 | output_dir=f"tmp/{args.tmp_prefix}/5A_cellpose_input", 166 | input_dir=args.input_dir, 167 | aux_ch_names=args.aux_views, 168 | mRNA_dir=args.decoded_dir, 169 | selected_fovs=args.selected_fovs, 170 | ) 171 | else: 172 | filter_cellpose( 173 | output_dir=f"tmp/{args.tmp_prefix}/5C_cellpose_filtered", 174 | input_dir=args.input_dir, 175 | border_buffer=args.border_buffer, 176 | label_exp_size=args.label_exp_size, 177 | min_size=args.min_size, 178 | max_size=args.max_size, 179 | ) 180 | # because we know these two will be called as a part of the same cwl, 181 | # we don't need to re-clarify selected_fovs on the filter step 182 | -------------------------------------------------------------------------------- /bin/imgProcessing.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import hashlib 4 | import json 5 | import os 6 | import shutil 7 | import sys 8 | from argparse import ArgumentParser 9 | from concurrent.futures.process import ProcessPoolExecutor 10 | from copy import deepcopy 11 | from datetime import datetime 12 | from functools import partial, partialmethod 13 | from os import path 14 | from pathlib import Path 15 | from time import time 16 | from typing import List 17 | 18 | import cv2 19 | import numpy as np 20 | import starfish 21 | import tifffile as tiff 22 | from scipy import ndimage 23 | from skimage import exposure, morphology, restoration 24 | from skimage.morphology import disk 25 | from skimage.registration import phase_cross_correlation 26 | from starfish import Experiment, ImageStack 27 | from starfish.types import Levels 28 | from tqdm import tqdm 29 | 30 | 31 | def saveImg(loc: str, prefix: str, img: ImageStack): 32 | # save the individual slices of an image in the same format starfish does 33 | for r in range(img.num_rounds): 34 | for c in range(img.num_chs): 35 | for z in range(img.num_zplanes): 36 | tiff.imsave( 37 | "{}{}-c{}-r{}-z{}.tiff".format(loc, prefix, c, r, z), img._data[r, c, z, :, :] 38 | ) 39 | 40 | 41 | def saveExp( 42 | source_dir: str, save_dir: str, exp: Experiment = None, selected_fovs: List[str] = None 43 | ): 44 | # go through and save all images, if an experiment is provided 45 | if exp: 46 | for fov in exp.keys(): 47 | for view in exp[fov].image_types: 48 | img = exp[fov].get_image(view) 49 | prefix = f"{view}-{fov}" 50 | saveImg(save_dir, prefix, img) 51 | 52 | # copy the non-tiff files to the new directory 53 | cp_files = [x for x in os.listdir(source_dir) if x[-5:] != ".tiff" and x[-4:] != ".log"] 54 | for file in cp_files: 55 | print(f"looking at {file}.") 56 | if "fov" in file: 57 | # images were only updated if we looked at that fov 58 | if (selected_fovs is None) or (True in [f in file for f in selected_fovs]): 59 | # if file contains images, we need to update sha's 60 | data = json.load(open(str(source_dir) + "/" + file)) 61 | for i in range(len(data["tiles"])): 62 | abspath = str(save_dir) + "/" + data["tiles"][i]["file"] 63 | with open(os.fspath(abspath), "rb") as fh: 64 | hsh = hashlib.sha256(fh.read()).hexdigest() 65 | data["tiles"][i]["sha256"] = hsh 66 | print(f"\tupdated hash for {data['tiles'][i]['file']}") 67 | with open(str(save_dir) + "/" + file, "w") as f: 68 | json.dump(data, f) 69 | print(f"saved {file} with modified hashes") 70 | else: 71 | print(f"\tskipping file {file}") 72 | else: 73 | # if we're using a subset of fovs, we'll need to modify view files 74 | if (selected_fovs is not None) and ( 75 | "json" in file and "codebook" not in file and "experiment" not in file 76 | ): 77 | data = json.load(open(str(source_dir) + "/" + file)) 78 | new_data = {} 79 | for k, v in data.items(): 80 | if k == "contents": 81 | # this is where we select only fovs that we care about 82 | new_data["contents"] = {f: v[f] for f in selected_fovs} 83 | else: 84 | new_data[k] = v 85 | with open(str(save_dir) + "/" + file, "w") as f: 86 | json.dump(new_data, f) 87 | print(f"\tsaved {file} with used FOVs.") 88 | else: 89 | # we can just copy the rest of the files 90 | shutil.copyfile(f"{source_dir}/{file}", f"{save_dir}/{file}") 91 | print(f"\tcopied {file}") 92 | 93 | 94 | def register_primary_aux(img, reg_img, chs_per_reg): 95 | """ 96 | Register primary images using provided registration images. 97 | chs_per_reg is number of primary image channels 98 | associated with each registration image. 99 | 100 | Calculates shifts between auxillary images using the first 101 | auxillary image as a reference, then applies shifts to primary images. 102 | """ 103 | # Calculate registration shifts from registration images 104 | shifts = {} 105 | # Reference is set arbitrarily to first round/channel 106 | reference = reg_img.xarray.data[0, 0] 107 | for r in range(reg_img.num_rounds): 108 | for ch in range(reg_img.num_chs): 109 | shift, error, diffphase = phase_cross_correlation( 110 | reference, reg_img.xarray.data[r, ch], upsample_factor=100 111 | ) 112 | shifts[(r, ch)] = shift 113 | 114 | # Create transformation matrices 115 | shape = img.raw_shape 116 | tforms = {} 117 | for r, ch in shifts: 118 | tform = np.diag([1.0] * 4) 119 | # Start from 1 because we don't want to shift in the z direction (if there is one) 120 | for i in range(1, 3): 121 | tform[i, 3] = shifts[(r, ch)][i] 122 | tforms[(r, ch)] = tform 123 | 124 | # Register primary images 125 | for r in range(img.num_rounds): 126 | for ch in range(img.num_chs): 127 | img.xarray.data[r, ch] = ndimage.affine_transform( 128 | img.xarray.data[r, ch], 129 | np.linalg.inv(tforms[(r, ch // chs_per_reg)]), 130 | output_shape=shape[2:], 131 | ) 132 | return img 133 | 134 | 135 | def calc_reg_shift(images, reg_img): 136 | """ 137 | Helper function for calculating registration shifts in parallel 138 | """ 139 | 140 | # Calculate shifts from each primary image to the registration image 141 | shifts = [] 142 | for image in images: 143 | shift, error, diffphase = phase_cross_correlation(reg_img, image, upsample_factor=100) 144 | shifts.append(shift) 145 | return shifts 146 | 147 | 148 | def apply_reg_shift(image_tforms, shape): 149 | """ 150 | Helper function for applying registration shifts in parallel 151 | """ 152 | 153 | align_imgs = [] 154 | for i in range(len(image_tforms[0])): 155 | image = image_tforms[0][i] 156 | tform = image_tforms[1][i] 157 | align_img = ndimage.affine_transform( 158 | image, 159 | np.linalg.inv(tform), 160 | output_shape=shape[2:], 161 | ) 162 | align_imgs.append(align_img) 163 | return align_imgs 164 | 165 | 166 | def register_primary_primary_parallel(img, reg_img, num_threads): 167 | """ 168 | Register primary images using provided registration images. 169 | 170 | Calculates shifts between primary images and the single auxillary image 171 | and then applies shifts to primary images. 172 | """ 173 | # Chunk round/channel combinations for parallel run 174 | rchs = [(r, ch) for r in range(img.num_rounds) for ch in range(img.num_chs)] 175 | 176 | # Calculates index ranges to chunk data by 177 | ranges = [0] 178 | for i in range(1, num_threads + 1): 179 | ranges.append(int((len(rchs) / num_threads) * i)) 180 | chunked_rchs = [rchs[ranges[i] : ranges[i + 1]] for i in range(len(ranges[:-1]))] 181 | 182 | # Create list of lists of images corresponding to the above chunking 183 | chunked_imgs = [] 184 | for rch_chunk in chunked_rchs: 185 | chunk = [] 186 | for rch in rch_chunk: 187 | chunk.append(img.xarray.data[rch[0], rch[1]]) 188 | chunked_imgs.append(chunk) 189 | 190 | # Calculate registration shifts in parallel 191 | part = partial(calc_reg_shift, reg_img=reg_img.xarray.data[0, 0]) 192 | with ProcessPoolExecutor() as pool: 193 | poolMap = pool.map(part, [img_chunk for img_chunk in chunked_imgs]) 194 | results = [x for x in poolMap] 195 | 196 | # Create transformation matrices 197 | tforms = {} 198 | for i in range(len(chunked_rchs)): 199 | for j in range(len(chunked_rchs[i])): 200 | r, ch = chunked_rchs[i][j] 201 | tform = np.diag([1.0] * 4) 202 | # Start from 1 because we don't want to shift in the z direction (if there is one) 203 | for k in range(1, 3): 204 | tform[k, 3] = results[i][j][k] 205 | tforms[(r, ch)] = tform 206 | 207 | # Create list of lists of tforms corresponding to the above chunking 208 | chunked_tforms = [] 209 | for rch_chunk in chunked_rchs: 210 | chunk = [] 211 | for rch in rch_chunk: 212 | chunk.append(tforms[(rch[0], rch[1])]) 213 | chunked_tforms.append(chunk) 214 | 215 | # Apply registration shifts in parallel 216 | shape = img.raw_shape 217 | part = partial(apply_reg_shift, shape=shape) 218 | with ProcessPoolExecutor() as pool: 219 | poolMap = pool.map( 220 | part, [img_tform_chunk for img_tform_chunk in zip(chunked_imgs, chunked_tforms)] 221 | ) 222 | results = [x for x in poolMap] 223 | 224 | # Compile results 225 | for i in range(len(chunked_rchs)): 226 | for j in range(len(chunked_rchs[i])): 227 | r, ch = chunked_rchs[i][j] 228 | img.xarray.data[r, ch] = deepcopy(results[i][j]) 229 | 230 | return img 231 | 232 | 233 | def subtract_background(img, background): 234 | """ 235 | Subtract real background image from primary images. Will register to same reference as primary images were 236 | aligned to if reg_img is provided, assumes background is of same round/channel dimensions as reference. 237 | """ 238 | 239 | # Subtract background images from primary 240 | bg_dat = background.xarray.data 241 | num_chs = background.num_chs 242 | for r in range(img.num_rounds): 243 | for ch in range(img.num_chs): 244 | for z in range(img.num_zplanes): 245 | data = img.xarray.data[r, ch, z].astype("float32") 246 | data -= bg_dat[r, ch % num_chs, z].astype("float32") 247 | data[data < 0] = 0 248 | img.xarray.data[r, ch, z] = data.astype("uint16") 249 | 250 | return img 251 | 252 | 253 | def morph_open(images): 254 | """ 255 | Multiprocessing helper function to run morphological openings in parallel. 256 | """ 257 | size = 100 258 | morphed = [] 259 | for image in images: 260 | background = np.zeros_like(image) 261 | for z in range(image.shape[0]): 262 | background[z] = cv2.morphologyEx(image[z], cv2.MORPH_OPEN, disk(size)) 263 | morphed.append(background) 264 | return morphed 265 | 266 | 267 | def subtract_background_estimate(img, num_threads): 268 | """ 269 | Estimate background using large morphological opening (radis = 100px) and subtract from image. 270 | """ 271 | # Chunk round/channel combinations for parallel run 272 | rchs = [(r, ch) for r in range(img.num_rounds) for ch in range(img.num_chs)] 273 | 274 | # Calculates index ranges to chunk data by 275 | ranges = [0] 276 | for i in range(1, num_threads + 1): 277 | ranges.append(int((len(rchs) / num_threads) * i)) 278 | chunked_rchs = [rchs[ranges[i] : ranges[i + 1]] for i in range(len(ranges[:-1]))] 279 | 280 | # Create list of lists of images corresponding to the above chunking 281 | chunked_imgs = [] 282 | for rch_chunk in chunked_rchs: 283 | chunk = [] 284 | for rch in rch_chunk: 285 | chunk.append(img.xarray.data[rch[0], rch[1]]) 286 | chunked_imgs.append(chunk) 287 | 288 | # Run morph open in parallel 289 | with ProcessPoolExecutor() as pool: 290 | poolMap = pool.map(morph_open, [img_chunk for img_chunk in chunked_imgs]) 291 | results = [x for x in poolMap] 292 | 293 | # Subtract background estimates from img 294 | for i in range(len(chunked_rchs)): 295 | for j in range(len(chunked_rchs[i])): 296 | r, ch = chunked_rchs[i][j] 297 | img.xarray.data[r, ch] = img.xarray.data[r, ch] - results[i][j] 298 | img.xarray.data[r, ch][img.xarray.data[r, ch] < 0] = 0 299 | return img 300 | 301 | 302 | def rolling_ball(img, rolling_rad=3, num_threads=1): 303 | """ 304 | Peform rolling ball background subtraction. 305 | """ 306 | for r in range(img.num_rounds): 307 | for ch in range(img.num_chs): 308 | for z in range(img.num_zplanes): 309 | background = restoration.rolling_ball( 310 | img.xarray.data[r, ch, z], radius=rolling_rad, num_threads=num_threads 311 | ) 312 | img.xarray.data[r, ch, z] -= background 313 | return img 314 | 315 | 316 | def match_hist_2_min(img): 317 | """ 318 | Calculate the lowest average intensity image in stack and use as reference to match histograms for all 319 | other rounds/channels. 320 | """ 321 | # Calculate image means to find min 322 | meds = {} 323 | for r in range(img.num_rounds): 324 | for ch in range(img.num_chs): 325 | meds[(r, ch)] = np.mean(img.xarray.data[r, ch]) 326 | min_rch = sorted(meds.items(), key=lambda item: item[1])[0][0] 327 | 328 | # Use min image as reference for histogram matching 329 | reference = img.xarray.data[min_rch[0], min_rch[1]] 330 | for r in range(img.num_rounds): 331 | for ch in range(img.num_chs): 332 | img.xarray.data[r, ch] = np.rint( 333 | exposure.match_histograms(img.xarray.data[r, ch], reference) 334 | ) 335 | return img 336 | 337 | 338 | def white_top_hat(img, wth_rad): 339 | """ 340 | Perform white top hat filter on image. 341 | """ 342 | footprint = morphology.disk(wth_rad) 343 | for r in range(img.num_rounds): 344 | for ch in range(img.num_chs): 345 | for z in range(img.num_zplanes): 346 | img.xarray.data[r, ch, z] = cv2.morphologyEx( 347 | img.xarray.data[r, ch, z], cv2.MORPH_TOPHAT, footprint 348 | ) 349 | return img 350 | 351 | 352 | def cli( 353 | input_dir: Path, 354 | output_dir: str, 355 | n_processes: int, 356 | clip_min: float = 0, 357 | clip_max: float = 99.9, 358 | level_method: str = "", 359 | is_volume: bool = False, 360 | register_aux_view: str = None, 361 | register_to_primary: bool = False, 362 | ch_per_reg: int = 1, 363 | background_name: str = None, 364 | register_background: bool = False, 365 | anchor_name: str = None, 366 | high_sigma: int = None, 367 | decon_iter: int = 15, 368 | decon_sigma: int = None, 369 | low_sigma: int = None, 370 | rolling_rad: int = None, 371 | match_hist: bool = False, 372 | wth_rad: int = None, 373 | rescale: bool = False, 374 | selected_fovs: List[int] = None, 375 | ): 376 | """ 377 | n_processes: If provided, the number of threads to use for processing. Otherwise, the max number of 378 | available CPUs will be used. 379 | 380 | clip_min: minimum value for ClipPercentileToZero 381 | 382 | is_volume: whether to treat the z-planes as a 3D image. 383 | 384 | level_method: Which level method to be applied to the Clip filter. 385 | 386 | aux_name: name of the aux view to align registration to 387 | 388 | chs_per_reg: Number of images/channels associated with each registration image. 389 | If registration images are duplicated so that the dimensions of primary and 390 | registration images match then keep set to 1 for 1-to-1 registration. 391 | 392 | background_name: name of the background view that will be subtracted, if provided. 393 | 394 | register_background: if true, the background image will be registered to 'aux_name' 395 | 396 | anchor_name: name of the aux view anchor round to perform processing on, if provided. 397 | 398 | high_sigma: Sigma value for high pass filter. High values remove less autofluorescence 399 | while lower values remove more. Won't need to change between data sets unless you 400 | had lots of autofluorescence. 401 | 402 | decon_iter: Number of iterations for deconvolution. High values remove more noise while 403 | lower values remove less. Won't need to change between data sets unless image is very noisy. 404 | 405 | decon_sigma: Sigma value for deconvolution. Should be approximately the expected spot size. 406 | 407 | low_sigma: Sigma value for lowpass filtering. Larger values result in stronger 408 | blurring. This should be low so can remain constant. 409 | 410 | rolling_rad: Radius for rolling ball background subtraction. Larger values lead to 411 | increased intensity evening effect. Likely doesn't need changed from 3. 412 | 413 | match_hist: If true, will perform histogram matching. 414 | 415 | wth_rad: Radius for white top hat filter. Should be slightly larger than the expected spot radius. 416 | 417 | rescale: If true, will not run final clip and scale on image, because it is expected to rescale 418 | the images in the following decoding step. 419 | 420 | selected_fovs: If provided, only FOVs with the provided indicies will be run. 421 | """ 422 | 423 | os.makedirs(output_dir, exist_ok=True) 424 | 425 | reporter = open( 426 | path.join(output_dir, datetime.now().strftime("%Y%m%d_%H%M%S.%f_img_processing.log")), "w" 427 | ) 428 | sys.stdout = reporter 429 | sys.stderr = reporter 430 | 431 | print(locals()) 432 | 433 | tqdm.__init__ = partialmethod(tqdm.__init__, disable=True) 434 | 435 | if level_method and level_method.upper() == "SCALE_BY_CHUNK": 436 | level_method = Levels.SCALE_BY_CHUNK 437 | elif level_method and level_method.upper() == "SCALE_BY_IMAGE": 438 | level_method = Levels.SCALE_BY_IMAGE 439 | elif level_method and level_method.upper() == "SCALE_SATURATED_BY_CHUNK": 440 | level_method = Levels.SCALE_SATURATED_BY_CHUNK 441 | elif level_method and level_method.upper() == "SCALE_SATURATED_BY_IMAGE": 442 | level_method = Levels.SCALE_SATURATED_BY_IMAGE 443 | else: 444 | level_method = Levels.SCALE_BY_IMAGE 445 | 446 | t0 = time() 447 | exp = starfish.core.experiment.experiment.Experiment.from_json( 448 | str(input_dir / "experiment.json") 449 | ) 450 | if selected_fovs is not None: 451 | fovs = ["fov_{:05}".format(int(f)) for f in selected_fovs] 452 | else: 453 | fovs = list(exp.keys()) 454 | 455 | for fov in fovs: 456 | img = exp[fov].get_image("primary") 457 | t1 = time() 458 | print("Fetched view " + fov) 459 | 460 | anchor = None 461 | if anchor_name: 462 | anchor = exp[fov].get_image(anchor_name) 463 | print("\tanchor image retrieved") 464 | 465 | if background_name: 466 | # If a background image is provided, subtract it from the primary image. 467 | bg = exp[fov].get_image(background_name) 468 | print("\tremoving existing backgound...") 469 | img = subtract_background(img, bg) 470 | if anchor_name: 471 | print("\tremoving existing background from anchor image...") 472 | anchor = subtract_background(anchor, bg) 473 | else: 474 | # If no background image is provided, estimate background using a large morphological 475 | # opening to subtract from primary images 476 | print("\tremoving estimated background...") 477 | img = subtract_background_estimate(img, n_processes) 478 | if anchor_name: 479 | print("\tremoving estimated background from anchor image...") 480 | anchor = subtract_background_estimate(anchor, n_processes) 481 | 482 | if high_sigma: 483 | # Remove cellular autofluorescence w/ gaussian high-pass filter 484 | print("\trunning high pass filter...") 485 | ghp = starfish.image.Filter.GaussianHighPass(sigma=high_sigma) 486 | # ghp.run(img, verbose=False, in_place=True) 487 | ghp.run(img, verbose=False, in_place=True, n_processes=n_processes) 488 | if anchor_name: 489 | print("\trunning high pass filter on anchor image...") 490 | ghp.run(anchor, verbose=False, in_place=True, n_processes=n_processes) 491 | 492 | if decon_sigma: 493 | # Increase resolution by deconvolving w/ point spread function 494 | print("\tdeconvolving point spread function...") 495 | dpsf = starfish.image.Filter.DeconvolvePSF(num_iter=decon_iter, sigma=decon_sigma) 496 | # dpsf.run(img, verbose=False, in_place=True) 497 | dpsf.run(img, verbose=False, in_place=True, n_processes=n_processes) 498 | if anchor_name: 499 | print("\tdeconvolving point spread function on anchor image...") 500 | dpsf.run(anchor, verbose=False, in_place=True, n_processes=n_processes) 501 | 502 | if low_sigma: 503 | # Blur image with lowpass filter 504 | print("\trunning low pass filter...") 505 | glp = starfish.image.Filter.GaussianLowPass(sigma=low_sigma) 506 | # glp.run(img, verbose=False, in_place=True) 507 | glp.run(img, verbose=False, in_place=True, n_processes=n_processes) 508 | 509 | if wth_rad: 510 | print("\trunning white tophat filter...") 511 | img = white_top_hat(img, wth_rad) 512 | if anchor_name: 513 | print("\trunning white tophat filter on anchor image...") 514 | anchor = white_top_hat(anchor, wth_rad) 515 | 516 | if rolling_rad: 517 | # Apply rolling ball background subtraction method to even out intensities through each 2D image 518 | print("\tapplying rolling ball background subtraction...") 519 | img = rolling_ball(img, rolling_rad=rolling_rad, num_threads=n_processes) 520 | if anchor_name: 521 | print("\tapplying rolling ball background subtraction to anchor image...") 522 | anchor = rolling_ball(anchor, rolling_rad=rolling_rad, num_threads=n_processes) 523 | 524 | if match_hist: 525 | # Use histogram matching to lower the intensities of each 3D image down to the same 526 | # intensity range as the least bright image. This is done so spot finding can be done. 527 | # BlobDetector doesn't do well when the intensities are in different ranges and c 528 | # lipping the values is not sufficient. 529 | print("\tapplying histogram matching...") 530 | img = match_hist_2_min(img) 531 | if anchor_name: 532 | print("\tapplying histogram matching to anchor image...") 533 | anchor = match_hist_2_min(anchor) 534 | 535 | if register_to_primary: 536 | # If register_to_primary calculate registration shifts between primary images and the single aux image and 537 | # apply to primary images 538 | register = exp[fov].get_image(register_aux_view) 539 | if register.shape["r"] != 1: 540 | raise Exception( 541 | "If --register-primary-view is used, auxillary images must have only a single round/channel (use the --aux-single-round option)" 542 | ) 543 | else: 544 | print("\taligning to " + register_aux_view) 545 | img = register_primary_primary_parallel(img, register, n_processes) 546 | elif register_aux_view: 547 | # If not register_to_primary but still registering, calculate registration shifts between specified aux images and apply to primary images 548 | register = exp[fov].get_image(register_aux_view) 549 | if register.shape["r"] != img.shape["r"]: 550 | raise Exception( 551 | "If --register-aux-view is used, auxillary image dimensions must match primary image dimensions" 552 | ) 553 | else: 554 | print("\taligning to " + register_aux_view) 555 | img = register_primary_aux(img, register, ch_per_reg) 556 | 557 | if not rescale and not (clip_min == 0 and clip_max == 0): 558 | print("\tclip and scaling...") 559 | # Scale image, clipping all but the highest intensities to zero 560 | clip = starfish.image.Filter.ClipPercentileToZero( 561 | p_min=clip_min, p_max=clip_max, is_volume=is_volume, level_method=level_method 562 | ) 563 | clip.run(img, in_place=True) 564 | if anchor_name: 565 | print("\tapplying clip and scale to anchor image...") 566 | clip = starfish.image.Filter.ClipPercentileToZero( 567 | p_min=90, p_max=99.9, is_volume=is_volume, level_method=level_method 568 | ) 569 | clip.run(anchor, in_place=True) 570 | 571 | else: 572 | print("\tskipping clip and scale.") 573 | # Clip values below 0 and greater than 1 (prevents errors in decoding) 574 | clip = starfish.image.Filter.ClipPercentileToZero( 575 | p_min=0, p_max=100, is_volume=is_volume, level_method=Levels.CLIP 576 | ) 577 | clip.run(img, in_place=True) 578 | 579 | print(f"\tView {fov} complete") 580 | # save modified image 581 | saveImg(output_dir, f"primary-{fov}", img) 582 | 583 | # save all aux views while we're here 584 | for view in exp[fov].image_types: 585 | if view != "primary" and view != anchor_name: 586 | aux_img = exp[fov].get_image(view) 587 | saveImg(output_dir, f"{view}-{fov}", aux_img) 588 | elif view == anchor_name: 589 | saveImg(output_dir, f"{view}-{fov}", anchor) 590 | 591 | print(f"View {fov} saved") 592 | print(f"Time for {fov}: {time() - t1}") 593 | 594 | print(f"Saving updated .jsons for {fovs}, copying other jsons\n") 595 | saveExp(input_dir, output_dir, exp=None, selected_fovs=fovs) 596 | print(f"\n\nTotal time elapsed for processing: {time() - t0}") 597 | 598 | 599 | if __name__ == "__main__": 600 | p = ArgumentParser() 601 | 602 | p.add_argument("--tmp-prefix", type=str) 603 | p.add_argument("--input-dir", type=Path) 604 | p.add_argument("--clip-min", type=float, default=0) 605 | p.add_argument("--clip-max", type=float, default=99.9) 606 | p.add_argument("--level-method", type=str, nargs="?") 607 | p.add_argument("--is-volume", dest="is_volume", action="store_true") 608 | p.add_argument("--register-aux-view", type=str, nargs="?") 609 | p.add_argument("--register-to-primary", dest="register_to_primary", action="store_true") 610 | p.add_argument("--ch-per-reg", type=int, nargs="?") 611 | p.add_argument("--background-view", type=str, nargs="?") 612 | p.add_argument("--register-background", dest="register_background", action="store_true") 613 | p.add_argument("--anchor-view", type=str, nargs="?") 614 | p.add_argument("--high-sigma", type=int, nargs="?") 615 | p.add_argument("--decon-iter", type=int, nargs="?") 616 | p.add_argument("--decon-sigma", type=int, nargs="?") 617 | p.add_argument("--low-sigma", type=int, nargs="?") 618 | p.add_argument("--rolling-radius", type=int, nargs="?") 619 | p.add_argument("--match-histogram", dest="match_histogram", action="store_true") 620 | p.add_argument("--tophat-radius", type=int, nargs="?") 621 | p.add_argument("--rescale", dest="rescale", action="store_true") 622 | p.add_argument("--n-processes", type=int, nargs="?") 623 | p.add_argument("--selected-fovs", nargs="+", const=None) 624 | 625 | args = p.parse_args() 626 | 627 | output_dir = f"tmp/{args.tmp_prefix}/3_processed/" 628 | 629 | if args.n_processes: 630 | n_processes = args.n_processes 631 | else: 632 | try: 633 | # the following line is not guaranteed to work on non-linux machines. 634 | n_processes = len(os.sched_getaffinity(os.getpid())) 635 | except Exception: 636 | n_processes = 1 637 | 638 | cli( 639 | input_dir=args.input_dir, 640 | output_dir=output_dir, 641 | clip_min=args.clip_min, 642 | clip_max=args.clip_max, 643 | level_method=args.level_method, 644 | is_volume=args.is_volume, 645 | register_aux_view=args.register_aux_view, 646 | register_to_primary=args.register_to_primary, 647 | ch_per_reg=args.ch_per_reg, 648 | background_name=args.background_view, 649 | register_background=args.register_background, 650 | anchor_name=args.anchor_view, 651 | high_sigma=args.high_sigma, 652 | decon_iter=args.decon_iter, 653 | decon_sigma=args.decon_sigma, 654 | low_sigma=args.low_sigma, 655 | rolling_rad=args.rolling_radius, 656 | match_hist=args.match_histogram, 657 | wth_rad=args.tophat_radius, 658 | rescale=args.rescale, 659 | n_processes=n_processes, 660 | selected_fovs=args.selected_fovs, 661 | ) 662 | -------------------------------------------------------------------------------- /bin/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from argparse import ArgumentParser 3 | from pathlib import Path 4 | 5 | import starfish 6 | 7 | 8 | def main(data_dir: Path): 9 | print("hello world") 10 | pass 11 | 12 | 13 | if __name__ == "__main__": 14 | p = ArgumentParser() 15 | p.add_argument("data_dir", type=Path) 16 | args = p.parse_args() 17 | 18 | main(args.data_dir) 19 | -------------------------------------------------------------------------------- /bin/pseudoSort.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | from argparse import ArgumentParser 5 | from datetime import datetime 6 | from os import makedirs, path 7 | from pathlib import Path 8 | from typing import Dict, List 9 | 10 | import numpy as np 11 | import pandas as pd 12 | import skimage.io 13 | import yaml 14 | from starfish import Codebook 15 | 16 | 17 | def parse_codebook(codebook_csv: str) -> Codebook: 18 | csv: pd.DataFrame = pd.read_csv(codebook_csv, index_col=0) 19 | genes = csv.index.values 20 | data_raw = csv.values 21 | rounds = csv.shape[1] 22 | channels = data_raw.max() 23 | 24 | # convert data_raw -> data, where data is genes x channels x rounds 25 | data = np.zeros((len(data_raw), rounds, channels)) 26 | for b in range(len(data_raw)): 27 | for i in range(len(data_raw[b])): 28 | if data_raw[b][i] != 0: 29 | data[b][i][data_raw[b][i] - 1] = 1 30 | 31 | return Codebook.from_numpy(genes, rounds, channels, data) 32 | 33 | 34 | def convert_codebook( 35 | oldbook: Codebook, cycles_conv: Dict[int, int], channels_conv: List[Dict[int, int]] 36 | ) -> Codebook: 37 | raw = oldbook.data 38 | targets = np.shape(raw)[0] 39 | rounds = len(cycles_conv) 40 | channels = len(channels_conv[0]) 41 | new_data = np.empty((targets, rounds, channels), dtype=int) 42 | for t in range(targets): 43 | for pr in range(len(raw[t])): 44 | # annoying math because dicts are saved for the other direction 45 | pchannel = np.argmax(raw[t][pr]) 46 | subChannel = [ 47 | [tch for tch, pch in subchannel.items() if pch == pchannel] 48 | for subchannel in channels_conv 49 | ] 50 | subRound = np.argmax([len(per_round) for per_round in subChannel]) 51 | tchannel = subChannel[subRound][0] 52 | tround = [tr for tr, pround in cycles_conv.items() if pround == pr][subRound] 53 | # print("channel {}->{} round {}->{}".format(pchannel,tchannel,pr,tround)) 54 | new_data[t][tround][tchannel] = 1 55 | 56 | return Codebook.from_numpy(oldbook.coords["target"].data, rounds, channels, new_data) 57 | 58 | 59 | def reformatter( 60 | cycles_conv: Dict[int, int], 61 | channels_conv: List[Dict[int, int]], 62 | input_dir: str, 63 | file_format: str = "", 64 | output_format: str = "", 65 | output_vars: List[str] = [], 66 | output_dir: str = "", 67 | file_vars: str = "", 68 | fov_count: int = 1, 69 | cache_read_order: List[str] = [], 70 | channel_slope: float = 1, 71 | channel_intercept: int = 0, 72 | fov_offset: int = 0, 73 | round_offset: int = 0, 74 | channel_offset: int = 0, 75 | aux_file_formats: List[str] = [], 76 | aux_file_vars: List[List[str]] = [], 77 | aux_names: List[str] = [], 78 | aux_cache_read_order: List[List[str]] = [], 79 | aux_channel_count: List[int] = [], 80 | aux_channel_slope: List[float] = [], 81 | aux_channel_intercept: List[int] = [], 82 | ): 83 | reportFile = path.join(output_dir, datetime.now().strftime("%Y%d%m_%H%M_psorting.log")) 84 | sys.stdout = open(reportFile, "w") 85 | 86 | combined_file_format = [file_format] + aux_file_formats 87 | combined_file_vars = [file_vars] + aux_file_vars 88 | combined_names = [""] + aux_names 89 | combined_cache_read_order = [cache_read_order] + aux_cache_read_order 90 | combined_channel_count = [len(channels_conv[0])] + aux_channel_count 91 | channel_slope = [1] + aux_channel_slope 92 | channel_intercept = [0] + aux_channel_intercept 93 | views = len(combined_names) 94 | 95 | for r in cycles_conv.keys(): 96 | for c in range(max(combined_channel_count)): 97 | for fov in range(fov_count): 98 | varTable = { 99 | "channel": c, 100 | "offset_channel": c + channel_offset, 101 | "round": r, 102 | "offset_round": r + round_offset, 103 | "fov": fov, 104 | "offset_fov": fov + fov_offset, 105 | } 106 | for target in range(views): 107 | if c < combined_channel_count[target]: 108 | varTable["input_channel"] = int( 109 | int(c * channel_slope[target]) + channel_intercept[target] 110 | ) 111 | file_path = path.join( 112 | input_dir, 113 | combined_file_format[target].format( 114 | *[varTable[arg] for arg in combined_file_vars[target]] 115 | ), 116 | ) 117 | print(varTable) 118 | img = skimage.io.imread(file_path) 119 | 120 | # Convert to uint16 if not already 121 | if np.max(img) <= 1: 122 | img = np.rint(img * 2**16).astype("uint16") 123 | if img.dtype != "uint16": 124 | img = img.astype("uint16") 125 | # img_out = img 126 | 127 | # figure out what slice to take. 128 | slices = [] 129 | for i in range(len(combined_cache_read_order[target])): 130 | axis = combined_cache_read_order[target][i] 131 | if axis.lower() == "ch": 132 | c_adj = int(channel_slope[target] * c) + channel_intercept[target] 133 | slices.append(int(c_adj)) 134 | elif axis.lower() == "round": 135 | slices.append(r) 136 | else: 137 | slices.append(slice(0, img.shape[i])) 138 | 139 | # take slices out of image and reduce unneeded dims 140 | slices = tuple(slices) 141 | print(slices) 142 | img_out = np.squeeze(img[slices]) 143 | 144 | # convert to new rounds/channels 145 | pr = cycles_conv[r] 146 | pc = channels_conv[r % len(channels_conv)][c] 147 | 148 | # get output string 149 | varTableConv = { 150 | "channel": pc, 151 | "offset_channel": pc + channel_offset, 152 | "round": pr, 153 | "offset_round": pr + round_offset, 154 | "fov": fov, 155 | "offset_fov": fov + fov_offset, 156 | "aux_name": combined_names[target], 157 | } 158 | output_path = path.join( 159 | output_dir, 160 | output_format.format(*[varTableConv[arg] for arg in output_vars]), 161 | ) 162 | print("{}\n->{}".format(file_path, output_path)) 163 | print(np.shape(img_out)) 164 | skimage.io.imsave(output_path, img_out) 165 | 166 | sys.stdout = sys.__stdout__ 167 | return True 168 | 169 | 170 | if __name__ == "__main__": 171 | p = ArgumentParser() 172 | p.add_argument("--tmp-prefix", type=str) 173 | p.add_argument("--input-dir", type=Path) 174 | p.add_argument("--codebook-csv", type=Path, nargs="?") 175 | p.add_argument("--codebook-json", type=Path, nargs="?") 176 | p.add_argument("--channel-yml", type=Path) 177 | p.add_argument("--cycle-yml", type=Path) 178 | p.add_argument("--file-format", type=str) 179 | p.add_argument("--file-vars", type=str, nargs="+") 180 | p.add_argument("--cache-read-order", type=str, nargs="+") 181 | p.add_argument("--z-plane-offset", type=int) 182 | p.add_argument("--fov-offset", type=int) 183 | p.add_argument("--round-offset", type=int) 184 | p.add_argument("--channel-offset", type=int) 185 | p.add_argument("--fov-count", type=int) 186 | p.add_argument("--channel-slope", type=float) 187 | p.add_argument("--channel-intercept", type=int) 188 | p.add_argument("--aux-file-formats", type=str, nargs="+", const=None) 189 | p.add_argument("--aux-file-vars", type=str, nargs="+", const=None) 190 | p.add_argument("--aux-names", type=str, nargs="+", const=None) 191 | p.add_argument("--aux-cache-read-order", type=str, nargs="+", const=None) 192 | p.add_argument("--aux-channel-count", type=int, nargs="+", const=None) 193 | p.add_argument("--aux-channel-slope", type=float, nargs="+", const=None) 194 | p.add_argument("--aux-channel-intercept", type=float, nargs="+", const=None) 195 | 196 | args = p.parse_args() 197 | 198 | aux_lens = [] 199 | aux_vars = [ 200 | args.aux_file_formats, 201 | args.aux_file_vars, 202 | args.aux_names, 203 | args.aux_cache_read_order, 204 | args.aux_channel_count, 205 | args.aux_channel_slope, 206 | args.aux_channel_intercept, 207 | ] 208 | 209 | for item in aux_vars: 210 | if isinstance(item, list): 211 | aux_lens.append(len(item)) 212 | elif item is not None: 213 | aux_lens.append(1) 214 | else: 215 | aux_lens.append(0) 216 | 217 | if len(set(aux_lens)) > 1: 218 | print(aux_vars) 219 | print(aux_lens) 220 | raise Exception("Dimensions of all aux parameters must match.") 221 | 222 | output_dir = f"tmp/{args.tmp_prefix}/1_pseudosort/" 223 | output_format = "PseudoCycle{}/MMStack_Pos{}_{}ch{}.ome.tif" 224 | output_vars = ["round", "fov", "aux_name", "channel"] 225 | 226 | with open(args.channel_yml, "r") as fl: 227 | channels_conv: List[Dict[int, int]] = yaml.load(fl, Loader=yaml.FullLoader) 228 | 229 | with open(args.cycle_yml, "r") as fl: 230 | cycles_conv: Dict[int, int] = yaml.load(fl, Loader=yaml.FullLoader) 231 | 232 | for i in range(len(set(cycles_conv.values()))): 233 | makedirs("{}PseudoCycle{}".format(output_dir, i)) 234 | 235 | aux_file_vars = [item.split(";") for item in args.aux_file_vars] 236 | aux_cache_read_order = [item.split(";") for item in args.aux_cache_read_order] 237 | 238 | reformatter( 239 | cycles_conv=cycles_conv, 240 | channels_conv=channels_conv, 241 | input_dir=args.input_dir, 242 | file_format=args.file_format, 243 | output_format=output_format, 244 | output_vars=output_vars, 245 | output_dir=output_dir, 246 | file_vars=args.file_vars, 247 | fov_count=args.fov_count, 248 | cache_read_order=args.cache_read_order, 249 | channel_slope=args.channel_slope, 250 | channel_intercept=args.channel_intercept, 251 | fov_offset=args.fov_offset, 252 | round_offset=args.round_offset, 253 | channel_offset=args.channel_offset, 254 | aux_file_formats=args.aux_file_formats, 255 | aux_file_vars=aux_file_vars, 256 | aux_names=args.aux_names, 257 | aux_cache_read_order=aux_cache_read_order, 258 | aux_channel_count=args.aux_channel_count, 259 | aux_channel_slope=args.aux_channel_slope, 260 | aux_channel_intercept=args.aux_channel_intercept, 261 | ) 262 | 263 | if args.codebook_csv: 264 | codebook = parse_codebook(args.codebook_csv) 265 | elif args.codebook_json: 266 | codebook = Codebook.open_json(args.codebook_json) 267 | else: 268 | print("Can't convert notebook, none provided.") 269 | 270 | conv_codebook = convert_codebook(codebook, cycles_conv, channels_conv) 271 | codebook.to_json(output_dir + "pround_codebook.json") 272 | conv_codebook.to_json(output_dir + "codebook.json") 273 | -------------------------------------------------------------------------------- /docker/baysor/Dockerfile: -------------------------------------------------------------------------------- 1 | # based on https://github.com/kharchenkolab/Baysor/blob/master/Dockerfile 2 | # retrieved 2023.01.18 3 | # version of this image in dockerhub breaks in cwltool due to included CMD line. 4 | 5 | FROM julia:latest 6 | 7 | RUN apt-get update && apt-get install -y build-essential 8 | 9 | ## Jupyter 10 | 11 | RUN apt-get install -y python3 python3-pip vim 12 | 13 | RUN pip3 install jupyterlab numpy scipy matplotlib seaborn pandas sklearn scikit-image 14 | 15 | RUN pip3 install -Iv six==1.12.0 16 | 17 | RUN julia -e 'using Pkg; Pkg.add("IJulia"); Pkg.build(); using IJulia;' 18 | 19 | ### jupyter notebook --no-browser --port=8989 --ip=0.0.0.0 --allow-root ./ 20 | 21 | ## Julia Baysor envitonment 22 | ### Ignore cache (https://stackoverflow.com/questions/35134713/disable-cache-for-specific-run-commands) 23 | ARG CACHEBUST=1 24 | RUN julia -e 'using Pkg; Pkg.add(PackageSpec(url="https://github.com/kharchenkolab/Baysor.git"));' 25 | 26 | ENV LazyModules_lazyload false 27 | 28 | RUN julia -e 'import Baysor, Pkg; Pkg.activate(dirname(dirname(pathof(Baysor)))); Pkg.instantiate(); Pkg.build();' 29 | RUN echo "export PATH=/root/.julia/bin/:$PATH" >> ~/.bashrc 30 | RUN echo "alias julia='/usr/local/julia/bin/julia --sysimage=/root/.julia/scratchspaces/cc9f9468-1fbe-11e9-0acf-e9460511877c/sysimg/libbaysor.so'" >> ~/.bashrc 31 | RUN ln -s /root/.julia/bin/baysor /usr/local/bin/baysor && chmod +x /root/.julia/bin/baysor 32 | 33 | CMD ["/bin/bash"] 34 | -------------------------------------------------------------------------------- /docker/cellpose/Dockerfile: -------------------------------------------------------------------------------- 1 | #FROM ubuntu:18.04 2 | FROM continuumio/miniconda3:23.3.1-0 3 | 4 | RUN conda create -y -n cellpose -c andfoy python=3.8 pyqt 5 | RUN conda init bash; . /root/.bashrc; conda activate cellpose 6 | 7 | RUN apt update 8 | RUN apt -y upgrade 9 | RUN apt install -y make gcc build-essential libgtk-3-dev wget git 10 | #RUN apt install -y python3.8-dev python3.8-venv python3-pip 11 | 12 | #ENV VIRTUAL_ENV=/opt/venv 13 | #RUN python3.8 -m venv $VIRTUAL_ENV 14 | #ENV PATH="$VIRTUAL_ENV/bin:$PATH" 15 | RUN pip install --upgrade pip 16 | RUN pip install wheel numpy cython 17 | RUN conda install -c conda-forge imagecodecs 18 | 19 | # Can't run on GPU inside a cwl, no sense installing this. 20 | #RUN pip install torch cuda-python==11.7 21 | 22 | RUN pip install git+https://www.github.com/nickeener/cellpose.git 23 | 24 | CMD ["/bin/bash"] 25 | -------------------------------------------------------------------------------- /docker/starfish-custom/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-buster 2 | 3 | COPY requirements.txt /opt 4 | 5 | # alt installs needed for ARM build 6 | ARG TARGETPLATFORM 7 | RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ 8 | apt-get update && apt-get install -y libhdf5-dev && rm -rf /var/lib/apt/lists/*; \ 9 | else \ 10 | echo "Using pip install."; \ 11 | fi 12 | 13 | RUN python3 -m pip install -r /opt/requirements.txt \ 14 | && rm -rf /root/cache/.pip /opt/requirements.txt 15 | 16 | RUN git clone --branch ctcisar-hubmap https://github.com/ctcisar/starfish.git 17 | RUN cd /starfish;make install-dev 18 | 19 | COPY bin /opt 20 | COPY input_schemas /opt 21 | RUN chmod +x /opt/*.* 22 | 23 | CMD ["/bin/bash"] 24 | -------------------------------------------------------------------------------- /docker/starfish-docker-runner/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM quay.io/ucsc_cgl/toil:5.11.0a1-f6dda143d2bbd1e0a28138263cb34e2deca04377-py3.9 2 | 3 | ENV DEBIAN_FRONTEND=noninteractive 4 | RUN apt-get update && apt-get install -y subversion && rm -rf /var/lib/apt/lists/* 5 | 6 | RUN git clone --branch release https://github.com/hubmapconsortium/spatial-transcriptomics-pipeline.git 7 | -------------------------------------------------------------------------------- /docker/starfish/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7-stretch 2 | 3 | COPY requirements.txt /opt 4 | 5 | RUN python3 -m pip install -r /opt/requirements.txt \ 6 | && python3 -m pip install starfish \ 7 | && rm -rf /root/cache/.pip /opt/requirements.txt 8 | 9 | COPY bin /opt 10 | COPY input_schemas /opt 11 | RUN chmod +x /opt/*.py 12 | 13 | CMD ["/bin/bash"] 14 | -------------------------------------------------------------------------------- /docker_images.txt: -------------------------------------------------------------------------------- 1 | #hubmap/starfish docker/starfish/Dockerfile base_directory_build 2 | hubmap/starfish-custom docker/starfish-custom/Dockerfile base_directory_build,platforms=linux/amd64&linux/arm64 3 | hubmap/starfish-docker-runner docker/starfish-docker-runner/Dockerfile base_directory_build,platforms=linux/amd64&linux/arm64 4 | # As of v0.6.0 the build for the baysor docker image is broken. 5 | # Check up on it again later to see if it's working (last update 2023.04.13) 6 | #hubmap/baysor docker/baysor/Dockerfile base_directory_build 7 | hubmap/cellpose docker/cellpose/Dockerfile base_directory_build,platforms=linux/amd64&linux/arm64 8 | -------------------------------------------------------------------------------- /input_schemas/cellpose.json: -------------------------------------------------------------------------------- 1 | [ 2 | "zplane_count?", 3 | "selected_fovs?", 4 | "use_mrna?", 5 | "pretrained_model_str?", 6 | "diameter?", 7 | "flow_threshold?", 8 | "stitch_threshold?", 9 | "cellprob_threshold?", 10 | "border_buffer?", 11 | "label_exp_size?", 12 | "min_allowed_size?", 13 | "max_allowed_size?", 14 | "aux_views" 15 | ] 16 | -------------------------------------------------------------------------------- /input_schemas/pipeline.json: -------------------------------------------------------------------------------- 1 | [ 2 | "run_baysor", 3 | "aux_views", 4 | "skip_formatting", 5 | "skip_processing", 6 | "register_aux_view", 7 | "run_cellpose", 8 | { 9 | "fov_positioning":[ 10 | [ 11 | "x_locs?", 12 | "x_shape?", 13 | "x_voxel?", 14 | "y_locs?", 15 | "y_shape?", 16 | "y_voxel?", 17 | "z_locs?", 18 | "z_shape?", 19 | "z_voxel?" 20 | ] 21 | ] 22 | }, 23 | "add_blanks", 24 | "skip_seg", 25 | "skip_qc" 26 | ] 27 | -------------------------------------------------------------------------------- /input_schemas/processing.json: -------------------------------------------------------------------------------- 1 | [ 2 | "fov_count", 3 | "selected_fovs", 4 | "clip_min?", 5 | "clip_max?", 6 | "level_method?", 7 | "rescale?", 8 | "is_volume?", 9 | "register_aux_view?", 10 | "register_to_primary?", 11 | "channels_per_reg?", 12 | "background_view?", 13 | "register_background?", 14 | "anchor_view?", 15 | "high_sigma?", 16 | "deconvolve_iter?", 17 | "deconvolve_sigma?", 18 | "low_sigma?", 19 | "rolling_radius?", 20 | "match_histogram?", 21 | "tophat_radius?", 22 | "channel_count", 23 | "n_processes?", 24 | { 25 | "aux_tilesets":[ 26 | [ 27 | "aux_names?", 28 | "aux_channel_count?" 29 | ] 30 | ] 31 | } 32 | ] 33 | -------------------------------------------------------------------------------- /input_schemas/psortedDefaultParams.json: -------------------------------------------------------------------------------- 1 | [ 2 | "round_count", 3 | "fov_count", 4 | "channel_count", 5 | "zplane_count", 6 | "cache_read_order", 7 | { 8 | "aux_tilesets": [ 9 | [ 10 | "aux_names?", 11 | "aux_cache_read_order?", 12 | "aux_channel_count" 13 | ] 14 | ] 15 | } 16 | ] 17 | -------------------------------------------------------------------------------- /input_schemas/qc.json: -------------------------------------------------------------------------------- 1 | [ 2 | "selected_fovs?", 3 | "find_ripley", 4 | "save_pdf", 5 | { 6 | "fov_positioning" : [ 7 | [], 8 | [ 9 | "x_shape", 10 | "y_shape", 11 | "z_shape" 12 | ] 13 | ] 14 | }, 15 | { 16 | "decoding":[ 17 | [], 18 | [ 19 | "decode_method?", 20 | "magnitude_threshold?", 21 | { 22 | "decoder": [ 23 | [], 24 | [ 25 | "min_intensity" 26 | ] 27 | ] 28 | } 29 | ] 30 | ] 31 | } 32 | ] 33 | -------------------------------------------------------------------------------- /input_schemas/segmentation.json: -------------------------------------------------------------------------------- 1 | [ 2 | "selected_fovs?", 3 | "aux_name", 4 | { 5 | "binary_mask":[ 6 | [], 7 | [ 8 | "img_threshold", 9 | "min_dist", 10 | "min_allowed_size", 11 | "max_allowed_size", 12 | "masking_radius" 13 | ], 14 | [ 15 | "nuclei_view", 16 | "cyto_seg", 17 | "correct_seg", 18 | "border_buffer", 19 | "area_thresh", 20 | "thresh_block_size", 21 | "watershed_footprint_size", 22 | "label_exp_size" 23 | ] 24 | ] 25 | } 26 | ] 27 | -------------------------------------------------------------------------------- /input_schemas/sorter.json: -------------------------------------------------------------------------------- 1 | [ 2 | "round_count", 3 | "fov_count", 4 | "round_offset?", 5 | "fov_offset?", 6 | "channel_offset?", 7 | "channel_slope?", 8 | "file_format", 9 | "file_vars", 10 | "cache_read_order", 11 | { 12 | "aux_tilesets":[ 13 | [ 14 | "aux_names?", 15 | "aux_file_formats?", 16 | "aux_file_vars?", 17 | "aux_cache_read_order?", 18 | "aux_channel_count?", 19 | "aux_channel_slope?", 20 | "aux_channel_intercept?" 21 | ] 22 | ] 23 | } 24 | ] 25 | -------------------------------------------------------------------------------- /input_schemas/spaceTxConversion.json: -------------------------------------------------------------------------------- 1 | [ 2 | "round_count", 3 | "fov_count", 4 | "zplane_count", 5 | "channel_count", 6 | "round_offset?", 7 | "fov_offset?", 8 | "zplane_offset?", 9 | "channel_offset?", 10 | "file_format", 11 | "file_vars", 12 | "cache_read_order", 13 | { 14 | "aux_tilesets":[ 15 | [ 16 | "aux_names?", 17 | "aux_file_formats?", 18 | "aux_file_vars?", 19 | "aux_cache_read_order?", 20 | "aux_single_round?", 21 | "aux_channel_count?", 22 | "aux_channel_slope?", 23 | "aux_channel_intercept?" 24 | ] 25 | ] 26 | }, 27 | { 28 | "fov_positioning":[ 29 | [ 30 | ], 31 | [ 32 | "x_locs?", 33 | "x_shape?", 34 | "x_voxel?", 35 | "y_locs?", 36 | "y_shape?", 37 | "y_voxel?", 38 | "z_locs?", 39 | "z_shape?", 40 | "z_voxel?" 41 | ] 42 | ] 43 | }, 44 | "add_blanks?" 45 | ] 46 | -------------------------------------------------------------------------------- /input_schemas/starfishRunner.json: -------------------------------------------------------------------------------- 1 | [ 2 | "fov_count", 3 | "selected_fovs?", 4 | "level_method?", 5 | "use_ref_img?", 6 | "is_volume?", 7 | "rescale?", 8 | "anchor_view?", 9 | "not_filtered_results?", 10 | "n_processes?", 11 | "scatter_into_n?", 12 | { 13 | "decoding":[ 14 | [ 15 | "min_sigma?", 16 | "max_sigma?", 17 | "num_sigma?", 18 | "threshold?", 19 | "overlap?", 20 | "decode_method?", 21 | "pnorm?", 22 | "distance_threshold?", 23 | "magnitude_threshold?", 24 | "min_area?", 25 | "max_area?", 26 | "norm_order?", 27 | "composite_decode?", 28 | "composite_pmin?", 29 | "composite_pmax?", 30 | { 31 | "decoder":[ 32 | [ 33 | 34 | ], 35 | [ 36 | "trace_building_strategy", 37 | "max_distance", 38 | "min_intensity", 39 | "pnorm?", 40 | "norm_order?", 41 | "anchor_round?", 42 | "search_radius?", 43 | "return_original_intensities?" 44 | ], 45 | [ 46 | "search_radius?", 47 | "anchor_round?", 48 | "trace_building_strategy" 49 | ], 50 | [ 51 | "search_radius?", 52 | "error_rounds?", 53 | "mode", 54 | "physical_coords?" 55 | ] 56 | ] 57 | } 58 | ] 59 | ] 60 | } 61 | ] 62 | -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hubmapconsortium/spatial-transcriptomics-pipeline/2e31b8ddd4d509c7bbcb983ad41e401687623ddb/logo.png -------------------------------------------------------------------------------- /pipeline-manifest.json: -------------------------------------------------------------------------------- 1 | [ 2 | "//TODO": "Baysor folders, cellpose folders.", 3 | { 4 | "pattern": "1_pseudosort/PseudoCycle(?P)/MMStack_Pos(?P)_(?P)ch(?P).ome.tif", 5 | "description": "Input images, rearranged into consistent round and channel counts.", 6 | "edam_ontology_term": "EDAM_1.24.format_3591" 7 | }, 8 | { 9 | "pattern": "1_pseudosort/(?P)_psorting.log", 10 | "description": "Log of completed operations while round sorting.", 11 | "edam_ontology_term": "EDAM_1.24.data_3671" 12 | }, 13 | { 14 | "pattern": "1_pseudosort/codebook.json", 15 | "description": "Codebook that represents the truerounds and truechannels for barcodes.", 16 | "edam_ontology_term": "EDAM_1.24.format_3464" 17 | }, 18 | { 19 | "pattern": "1_pseudosort/pround_codebook.json", 20 | "description": "Codebook that represents the pseudorounds and pseudochannels that will be used for decoding.", 21 | "edam_ontology_term": "EDAM_1.24.format_3464" 22 | }, 23 | { 24 | "pattern": "2_tx_converted/(?P)-fov_(?P)-c(?P)-r(?P)-z(?P).tiff", 25 | "description": "Images saved in spacetx format.", 26 | "edam_ontology_term": "EDAM_1.24.format_3591" 27 | }, 28 | { 29 | "pattern": "2_tx_converted/(?P)-fov_(?P).json", 30 | "description": "spacetx metadata files describing each fov, view combination.", 31 | "edam_ontology_term": "EDAM_1.24.format_3591" 32 | }, 33 | { 34 | "pattern": "2_tx_converted/(?P).json", 35 | "description": "spacetx metadata files describing each view.", 36 | "edam_ontology_term": "EDAM_1.24.format_3591" 37 | }, 38 | { 39 | "pattern": "2_tx_converted/codebook.json", 40 | "description": "Codebook that contains round and channel information for barcodes.", 41 | "edam_ontology_term": "EDAM_1.24.format_3591" 42 | }, 43 | { 44 | "pattern": "2_tx_converted/experiment.json", 45 | "description": "spacetx metadata files describing the entirety of the experiment.", 46 | "edam_ontology_term": "EDAM_1.24.format_3591" 47 | }, 48 | { 49 | "pattern": "2_tx_converted/(?P)_img_processing.log", 50 | "description": "Log of completed operations while converting images.", 51 | "edam_ontology_term": "EDAM_1.24.data_3671" 52 | }, 53 | { 54 | "pattern": "3_processed/(?P)-fov_(?P)-c(?P)-r(?P)-z(?P).tiff", 55 | "description": "Images saved in spacetx format.", 56 | "edam_ontology_term": "EDAM_1.24.format_3591" 57 | }, 58 | { 59 | "pattern": "3_processed/(?P)-fov_(?P).json", 60 | "description": "spacetx metadata files describing each fov, view combination.", 61 | "edam_ontology_term": "EDAM_1.24.format_3591" 62 | }, 63 | { 64 | "pattern": "3_processed/(?P).json", 65 | "description": "spacetx metadata files describing each view.", 66 | "edam_ontology_term": "EDAM_1.24.format_3591" 67 | }, 68 | { 69 | "pattern": "3_processed/codebook.json", 70 | "description": "Codebook that contains round and channel information for barcodes.", 71 | "edam_ontology_term": "EDAM_1.24.format_3591" 72 | }, 73 | { 74 | "pattern": "3_processed/experiment.json", 75 | "description": "spacetx metadata files describing the entirety of the experiment.", 76 | "edam_ontology_term": "EDAM_1.24.format_3591" 77 | }, 78 | { 79 | "pattern": "3_processed/(?P)_TXconversion.log", 80 | "description": "Log of completed operations while modifying images.", 81 | "edam_ontology_term": "EDAM_1.24.data_3671" 82 | }, 83 | { 84 | "pattern": "4_Decoded/cdf/fov_(?P)_decoded.cdf", 85 | "description": "netCDF formatted gene table with transcript locations.", 86 | "edam_ontology_term": "EDAM_1.24.format_3650" 87 | }, 88 | { 89 | "pattern": "4_Decoded/csv/fov_(?P)_decoded.csv", 90 | "description": "csv formatted gene table with transcript locations.", 91 | "edam_ontology_term": "EDAM_1.24.format_3752" 92 | }, 93 | { 94 | "pattern": "4_Decoded/spots/fov_(?P)_coords_(?P).nc", 95 | "description": "FOV positioning information for decoded spots.", 96 | "edam_ontology_term": "EDAM_1.24.format_3650" 97 | }, 98 | { 99 | "pattern": "4_Decoded/spots/fov_(?P)_SpotFindingResults.json", 100 | "description": "Metadata for saved spots.", 101 | "edam_ontology_term": "EDAM_1.24.format_3591" 102 | }, 103 | { 104 | "pattern": "4_Decoded/spots/fov_(?P)_log.arr", 105 | "description": "Log of operation history for this FOV's spots.", 106 | "edam_ontology_term": "EDAM_1.24.format_3591" 107 | }, 108 | { 109 | "pattern": "4_Decoded/spots/fov_(?P)_spots_(?P)_(?P).nc", 110 | "description": "Spot data for one round, channel combination.", 111 | "edam_ontology_term": "EDAM_1.24.format_3650" 112 | }, 113 | { 114 | "pattern": "4_Decoded/(?p)_starfish_runner.log", 115 | "description": "log of completed operations while decoding the experiment.", 116 | "edam_ontology_term": "edam_1.24.data_3671" 117 | }, 118 | { 119 | "pattern": "5_Segmented/fov_(?P)/segmentation.csv", 120 | "description": "CSV table of transcripts, annotated with location and cell IDs.", 121 | "edam_ontology_term": "EDAM_1.24.format_3752" 122 | }, 123 | { 124 | "pattern": "5_Segmented/fov_(?P)/df_segmented.cdf", 125 | "description": "NetCDF table of transcripts, annotated with location and cell IDs.", 126 | "edam_ontology_term": "EDAM_1.24.format_3650" 127 | }, 128 | { 129 | "pattern": "5_Segmented/fov_(?P)/exp_segmented.cdf", 130 | "description": "NetCDF Cell x Gene table.", 131 | "edam_ontology_term": "EDAM_1.24.format_3650" 132 | }, 133 | { 134 | "pattern": "5_Segmented/fov_(?P)/exp_segmented.csv", 135 | "description": "CSV Cell x Gene table.", 136 | "edam_ontology_term": "EDAM_1.24.format_3752" 137 | }, 138 | { 139 | "pattern": "5_Segmented/fov_(?P)/exp_segmented.h5ad", 140 | "description": "Anndata Cell x Gene table.", 141 | "edam_ontology_term": "FIXME" 142 | }, 143 | { 144 | "pattern": "5_Segmented/fov_(?P)/mask.tiff", 145 | "description": "A copy of the segmentation mask applied to this FOV", 146 | "edam_ontology_term": "EDAM_1.24.format_3591" 147 | }, 148 | { 149 | "pattern": "5_Segmented/(?p)_starfish_segmenter.log", 150 | "description": "log of completed operations while segmenting the experiment.", 151 | "edam_ontology_term": "edam_1.24.data_3671" 152 | }, 153 | { 154 | "pattern": "7_QC/(?p)_QC_metrics.log", 155 | "description": "log of completed operations while calculating QC values.", 156 | "edam_ontology_term": "edam_1.24.data_3671" 157 | }, 158 | { 159 | "pattern": "7_QC/QC_results.yml", 160 | "description": "Complete human-readable set of QC values.", 161 | "edam_ontology_term": "edam_1.24.data_3750" 162 | }, 163 | { 164 | "pattern": "7_QC/fov_(?P)_graph_output.pdf", 165 | "description": "Visual plots of QC metrics.", 166 | "edam_ontology_term": "edam_1.24.data_3508" 167 | }, 168 | { 169 | "pattern": "7_QC/fov_combined_graph_output.pdf", 170 | "description": "Visual plots of QC metrics, combined across all FOVs.", 171 | "edam_ontology_term": "edam_1.24.data_3508" 172 | } 173 | ] 174 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 99 3 | 4 | [tool.isort] 5 | profile = "black" 6 | multi_line_output = 3 7 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | scikit-image==0.18.3 3 | tqdm 4 | anndata 5 | astropy 6 | tifffile 7 | matplotlib 8 | opencv-python-headless==4.6.0.66 9 | pyro-ppl 10 | -------------------------------------------------------------------------------- /requirements_test.txt: -------------------------------------------------------------------------------- 1 | black>=22.3.0 2 | isort 3 | -------------------------------------------------------------------------------- /steps/baysor.cwl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cwl-runner 2 | 3 | class: CommandLineTool 4 | cwlVersion: v1.2 5 | baseCommand: ["baysor","run"] 6 | 7 | requirements: 8 | DockerRequirement: 9 | dockerPull: vpetukhov/baysor@sha256:ce58af2bbd81ca29f7382497223afe9dbfbcc674e810155964722b447b676087 10 | #dockerPull: hubmap/baysor:latest 11 | 12 | inputs: 13 | csv: 14 | type: File 15 | inputBinding: 16 | position: 5 17 | doc: csv with transcript information 18 | priors: 19 | type: File? 20 | inputBinding: 21 | position: 6 22 | doc: Binary Mask image with prior segmentation. 23 | scale: 24 | type: int? 25 | inputBinding: 26 | position: 1 27 | prefix: -s 28 | doc: Expected scale equal to cell radius in the same units as x, y, and z. 29 | x_col: 30 | type: string? 31 | inputBinding: 32 | position: 2 33 | prefix: -x 34 | doc: Name of the column with x information 35 | default: x 36 | y_col: 37 | type: string? 38 | inputBinding: 39 | position: 3 40 | prefix: -y 41 | default: y 42 | doc: Name of the column with y information 43 | gene_col: 44 | type: string? 45 | inputBinding: 46 | position: 4 47 | prefix: --gene 48 | default: target 49 | doc: Name of the column with gene names 50 | 51 | outputs: 52 | segmented: 53 | type: File[] 54 | outputBinding: 55 | glob: "segmentation*" 56 | 57 | stdout: baysor_stdout.log 58 | -------------------------------------------------------------------------------- /steps/baysorStaged.cwl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cwl-runner 2 | 3 | class: Workflow 4 | cwlVersion: v1.2 5 | requirements: 6 | ScatterFeatureRequirement: {} 7 | inputs: 8 | segmented: Directory 9 | outputs: 10 | baysor: 11 | type: Directory 12 | outputSource: restage/pool_dir 13 | 14 | steps: 15 | stage: 16 | run: 17 | class: CommandLineTool 18 | baseCommand: [ls] 19 | requirements: 20 | DockerRequirement: 21 | dockerPull: ubuntu:latest 22 | InitialWorkDirRequirement: 23 | listing: 24 | - $(inputs.segDir) 25 | inputs: 26 | segDir: 27 | type: Directory 28 | doc: Directory with output from starfish segmentation step. 29 | outputs: 30 | csvs: 31 | type: 32 | type: array 33 | items: File 34 | outputBinding: 35 | glob: "**/**/segmentation.csv" 36 | priors: 37 | type: 38 | type: array 39 | items: File 40 | outputBinding: 41 | glob: "**/**/mask.tiff" 42 | in: 43 | segDir: segmented 44 | out: [csvs, priors] 45 | baysor_run: 46 | run: baysor.cwl 47 | in: 48 | csv: stage/csvs 49 | priors: stage/priors 50 | scatter: [csv, priors] 51 | scatterMethod: dotproduct 52 | out: [segmented] 53 | restage: 54 | run: 55 | class: ExpressionTool 56 | requirements: 57 | InlineJavascriptRequirement: {} 58 | inputs: 59 | file_array: 60 | type: 61 | type: array 62 | items: 63 | type: array 64 | items: File 65 | outputs: 66 | pool_dir: Directory 67 | expression: | 68 | ${ var dir = []; 69 | for(var i=0;i 1){ 384 | return 0; 385 | } else { 386 | return null; 387 | } 388 | } else { 389 | if(self[1] > 1){ 390 | return 0; 391 | } else { 392 | return null; 393 | } 394 | } 395 | } 396 | channel_axis: 397 | source: [stage_cellpose/zplane_count, zplane_count] 398 | valueFrom: | 399 | ${ 400 | if(self[0]){ 401 | if(self[0] > 1){ 402 | return 1; 403 | } else { 404 | return 0; 405 | } 406 | } else { 407 | if(self[1] > 1){ 408 | return 1; 409 | } else { 410 | return 0; 411 | } 412 | } 413 | } 414 | pretrained_model_str: 415 | source: [stage_cellpose/pretrained_model_str, pretrained_model_str] 416 | valueFrom: | 417 | ${ 418 | if(self[0]){ 419 | return self[0]; 420 | } else if(self[1]) { 421 | return self[1]; 422 | } else { 423 | return null; 424 | } 425 | } 426 | pretrained_model_dir: pretrained_model_dir 427 | diameter: 428 | source: [stage_cellpose/diameter, diameter, pretrained_model_dir] 429 | valueFrom: | 430 | ${ 431 | if(self[2]){ 432 | return 0; 433 | } else if(self[0]){ 434 | return self[0]; 435 | } else if(self[1]) { 436 | return self[1]; 437 | } else { 438 | return null; 439 | } 440 | } 441 | flow_threshold: 442 | source: [stage_cellpose/flow_threshold, flow_threshold] 443 | valueFrom: | 444 | ${ 445 | if(self[0]){ 446 | return self[0]; 447 | } else if(self[1]) { 448 | return self[1]; 449 | } else { 450 | return null; 451 | } 452 | } 453 | stitch_threshold: 454 | source: [stage_cellpose/stitch_threshold, stitch_threshold, stage_cellpose/zplane_count, zplane_count] 455 | valueFrom: | 456 | ${ 457 | if(self[2] == 1 || self[3] == 1){ 458 | return null; 459 | } 460 | if(self[0]){ 461 | return self[0]; 462 | } else if(self[1]) { 463 | return self[1]; 464 | } else { 465 | return null; 466 | } 467 | } 468 | cellprob_threshold: 469 | source: [stage_cellpose/cellprob_threshold, cellprob_threshold] 470 | valueFrom: | 471 | ${ 472 | if(self[0]){ 473 | return self[0]; 474 | } else if(self[1]) { 475 | return self[1]; 476 | } else { 477 | return null; 478 | } 479 | } 480 | out: [cellpose_output] 481 | 482 | execute_filtering: 483 | run: 484 | class: CommandLineTool 485 | baseCommand: /opt/cellposeStaging.py 486 | 487 | requirements: 488 | DockerRequirement: 489 | dockerPull: hubmap/starfish-custom:latest 490 | ResourceRequirement: 491 | tmpdirMin: | 492 | ${ 493 | if(inputs.dir_size === null) { 494 | return null; 495 | } else { 496 | return inputs.dir_size * 4; 497 | } 498 | } 499 | outdirMin: | 500 | ${ 501 | if(inputs.dir_size === null) { 502 | return null; 503 | } else { 504 | return inputs.dir_size * 4; 505 | } 506 | } 507 | 508 | inputs: 509 | dir_size: 510 | type: long? 511 | 512 | tmp_prefix: 513 | type: string 514 | inputBinding: 515 | prefix: --tmp-prefix 516 | 517 | input_loc: 518 | type: Directory 519 | doc: Output from cellpose. 520 | inputBinding: 521 | prefix: --input-dir 522 | 523 | selected_fovs: 524 | type: int[]? 525 | doc: If provided, segmentation will only be run on FOVs with these indices. 526 | inputBinding: 527 | prefix: --selected-fovs 528 | 529 | border_buffer: 530 | type: int? 531 | doc: If not None, removes cytoplasms whose nuclei lie within the given distance from the border. 532 | inputBinding: 533 | prefix: --border-buffer 534 | 535 | label_exp_size: 536 | type: int? 537 | doc: Pixel size labels are dilated by in final step. Helpful for closing small holes that are common from thresholding but can also cause cell boundaries to exceed their true boundaries if set too high. Label dilation respects label borders and does not mix labels. 538 | inputBinding: 539 | prefix: --label-exp-size 540 | 541 | max_allowed_size: 542 | type: int? 543 | doc: maximum size for a cell (in pixels) 544 | inputBinding: 545 | prefix: --max-size 546 | 547 | min_allowed_size: 548 | type: int? 549 | doc: minimum size for a cell (in pixels) 550 | inputBinding: 551 | prefix: --min-size 552 | 553 | filter: 554 | type: boolean 555 | doc: Used to specify method in python script 556 | default: true 557 | inputBinding: 558 | prefix: --filter 559 | 560 | outputs: 561 | cellpose_filtered: 562 | type: Directory 563 | outputBinding: 564 | glob: $("tmp/" + inputs.tmp_prefix + "/5C_cellpose_filtered") 565 | 566 | in: 567 | dir_size: dir_size 568 | tmp_prefix: tmpname/tmp 569 | input_loc: execute_cellpose/cellpose_output 570 | selected_fovs: 571 | source: [stage_cellpose/selected_fovs, selected_fovs] 572 | valueFrom: | 573 | ${ 574 | if(self[0]){ 575 | return self[0]; 576 | } else if(self[1]) { 577 | return self[1]; 578 | } else { 579 | return null; 580 | } 581 | } 582 | border_buffer: 583 | source: [stage_cellpose/border_buffer, border_buffer] 584 | valueFrom: | 585 | ${ 586 | if(self[0]){ 587 | return self[0]; 588 | } else if(self[1]) { 589 | return self[1]; 590 | } else { 591 | return null; 592 | } 593 | } 594 | label_exp_size: 595 | source: [stage_cellpose/label_exp_size, label_exp_size] 596 | valueFrom: | 597 | ${ 598 | if(self[0]){ 599 | return self[0]; 600 | } else if(self[1]) { 601 | return self[1]; 602 | } else { 603 | return null; 604 | } 605 | } 606 | min_allowed_size: 607 | source: [stage_cellpose/min_allowed_size, min_allowed_size] 608 | valueFrom: | 609 | ${ 610 | if(self[0]){ 611 | return self[0]; 612 | } else if(self[1]) { 613 | return self[1]; 614 | } else { 615 | return null; 616 | } 617 | } 618 | max_allowed_size: 619 | source: [stage_cellpose/max_allowed_size, max_allowed_size] 620 | valueFrom: | 621 | ${ 622 | if(self[0]){ 623 | return self[0]; 624 | } else if(self[1]) { 625 | return self[1]; 626 | } else { 627 | return null; 628 | } 629 | } 630 | out: [cellpose_filtered] 631 | -------------------------------------------------------------------------------- /steps/fileSizer.cwl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cwl-runner 2 | cwlVersion: v1.2 3 | class: Workflow 4 | requirements: 5 | MultipleInputFeatureRequirement: {} 6 | InlineJavascriptRequirement: {} 7 | 8 | inputs: 9 | example_dir: 10 | type: Directory 11 | 12 | outputs: 13 | dir_size: 14 | type: long 15 | outputSource: formatter/dir_size 16 | 17 | steps: 18 | size_dir: 19 | run: 20 | class: CommandLineTool 21 | requirements: 22 | DockerRequirement: 23 | dockerPull: hubmap/starfish-custom:latest 24 | InitialWorkDirRequirement: 25 | listing: 26 | - $(inputs.example_dir) 27 | baseCommand: ["du", "-s", "--block-size=1MiB"] 28 | inputs: 29 | example_dir: 30 | type: Directory 31 | inputBinding: 32 | position: 0 33 | outputs: 34 | dir_size: 35 | type: stdout 36 | in: 37 | example_dir: example_dir 38 | out: [dir_size] 39 | 40 | formatter: 41 | run: 42 | class: ExpressionTool 43 | requirements: 44 | InlineJavascriptRequirement: {} 45 | expression: | 46 | ${ 47 | return {dir_size: Number(inputs.len_str.contents.split("\t")[0])} 48 | } 49 | inputs: 50 | len_str: 51 | type: File 52 | loadContents: true 53 | outputs: 54 | dir_size: 55 | type: long 56 | in: 57 | len_str: size_dir/dir_size 58 | out: [dir_size] 59 | -------------------------------------------------------------------------------- /steps/inputParser.cwl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cwl-runner 2 | cwlVersion: v1.0 3 | class: ExpressionTool 4 | 5 | requirements: 6 | InlineJavascriptRequirement: {} 7 | ResourceRequirement: 8 | ramMin: 1000 9 | tmpdirMin: 1000 10 | outdirMin: 1000 11 | 12 | inputs: 13 | datafile: 14 | type: File 15 | inputBinding: 16 | loadContents: true 17 | 18 | schema: 19 | type: File 20 | inputBinding: 21 | loadContents: true 22 | 23 | # all possible outputs from tool must be listed here. 24 | # not every output needs to be present in workflows that call this. 25 | outputs: 26 | round_count: int 27 | zplane_count: int 28 | channel_count: int 29 | fov_count: int 30 | round_offset: int 31 | fov_offset: int 32 | zplane_offset: int 33 | channel_offset: int 34 | channel_slope: float 35 | file_format: string 36 | file_vars: string[] 37 | cache_read_order: string[] 38 | aux_tilesets_aux_names: string[] 39 | aux_tilesets_aux_file_formats: string[] 40 | aux_tilesets_aux_file_vars: string[] 41 | aux_tilesets_aux_cache_read_order: string[] 42 | aux_tilesets_aux_single_round: string[] 43 | aux_tilesets_aux_channel_count: float[] 44 | aux_tilesets_aux_channel_slope: float[] 45 | aux_tilesets_aux_channel_intercept: int[] 46 | fov_positioning_x_locs: string 47 | fov_positioning_x_shape: int 48 | fov_positioning_x_voxel: float 49 | fov_positioning_y_locs: string 50 | fov_positioning_y_shape: int 51 | fov_positioning_y_voxel: float 52 | fov_positioning_z_locs: string 53 | fov_positioning_z_shape: int 54 | fov_positioning_z_voxel: float 55 | add_blanks: boolean 56 | skip_formatting: boolean 57 | skip_processing: boolean 58 | selected_fovs: int[] 59 | clip_min: float 60 | clip_max: float 61 | level_method: string 62 | register_aux_view: string 63 | register_to_primary: boolean 64 | channels_per_reg: int 65 | background_view: string 66 | register_background: boolean 67 | anchor_view: string 68 | high_sigma: int 69 | deconvolve_iter: int 70 | deconvolve_sigma: int 71 | low_sigma: int 72 | rolling_radius: int 73 | match_histogram: boolean 74 | tophat_radius: int 75 | use_ref_img: boolean 76 | is_volume: boolean 77 | rescale: boolean 78 | not_filtered_results: boolean 79 | n_processes: int 80 | scatter_into_n: int 81 | decoding_min_sigma: float[] 82 | decoding_max_sigma: float[] 83 | decoding_num_sigma: int 84 | decoding_threshold: float 85 | decoding_overlap: float 86 | decoding_decode_method: string 87 | decoding_decoder_trace_building_strategy: string 88 | decoding_decoder_max_distance: float 89 | decoding_decoder_min_intensity: float 90 | decoding_decoder_pnorm: int 91 | decoding_decoder_norm_order: int 92 | decoding_decoder_anchor_round: int 93 | decoding_decoder_search_radius: int 94 | decoding_decoder_return_original_intensities: boolean 95 | decoding_decoder_error_rounds: int 96 | decoding_decoder_mode: string 97 | decoding_decoder_physical_coords: boolean 98 | decoding_pnorm: int 99 | decoding_distance_threshold: float 100 | decoding_magnitude_threshold: float 101 | decoding_min_area: int 102 | decoding_max_area: int 103 | decoding_norm_order: int 104 | decoding_composite_decode: boolean 105 | decoding_composite_pmin: float 106 | decoding_composite_pmax: float 107 | skip_seg: boolean 108 | run_cellpose: boolean 109 | use_mrna: boolean 110 | pretrained_model_str: string 111 | diameter: float 112 | flow_threshold: float 113 | stitch_threshold: float 114 | cellprob_threshold: float 115 | border_buffer: int 116 | label_exp_size: int 117 | min_allowed_size: int 118 | max_allowed_size: int 119 | aux_views: string[] 120 | aux_name: string 121 | binary_mask_img_threshold: float 122 | binary_mask_min_dist: int 123 | binary_mask_min_allowed_size: int 124 | binary_mask_max_allowed_size: int 125 | binary_mask_masking_radius: int 126 | binary_mask_nuclei_view: string 127 | binary_mask_cyto_seg: string 128 | binary_mask_correct_seg: boolean 129 | binary_mask_border_buffer: int 130 | binary_mask_area_thresh: float 131 | binary_mask_thresh_block_size: int 132 | binary_mask_watershed_footprint_size: int 133 | binary_mask_label_exp_size: int 134 | run_baysor: boolean 135 | skip_qc: boolean 136 | find_ripley: boolean 137 | save_pdf: boolean 138 | 139 | # input schema describes the expected layout of variables in json format. 140 | # inputs are stored in an array. 141 | # any items that are treated as records for cwl input are stored in an object, where the key is the prefix on all items in the object. 142 | # the value in an object is an array or an array of arrays. 143 | # if there are two nested arrays, the sub-array with the closest match is used, ie the sub-arrays are mutually exclusive. 144 | # all items in an object's array must be included in the json file, unless the item ends with a question mark. 145 | # objects can be nested inside other objects, and all of their prefixes will apply to all items. 146 | expression: | 147 | ${ var data = JSON.parse(inputs.datafile.contents); 148 | var schema = JSON.parse(inputs.schema.contents); 149 | function enforce_record(data, key, items, output_dict){ 150 | if(Array.isArray(items[0])){ 151 | // Record where one of a mutually exclusive set of sublists is defined 152 | // Find closest match and enforce that 153 | var subind = 0; 154 | var coverage = 0.0; 155 | for(var i=0;i coverage){ 180 | coverage = new_coverage; 181 | subind = i; 182 | } 183 | } 184 | enforce_record(data, key, items[subind], output_dict); 185 | } else { 186 | for(var i=0;i -1){ 159 | cache.splice(ind, 1); 160 | } 161 | for(var i=0; i -1){ 171 | aux_cache.splice(aux_ind, 1); 172 | } 173 | aux_cache = aux_cache.join(";"); 174 | aux_names.push(inputs.aux_names[i]); 175 | aux_file_formats.push("PseudoCycle{}/MMStack_Pos{}_"+inputs.aux_names[i]+"ch{}.ome.tif"); 176 | aux_file_vars.push("round;fov;channel"); 177 | aux_cache_read_order.push(aux_cache); 178 | aux_channel_count.push(inputs.channel_count); 179 | aux_channel_slope.push(1); 180 | aux_channel_intercept.push(0); 181 | } 182 | return {"codebook": cb, 183 | "round_offset": 0, 184 | "fov_offset": 0, 185 | "channel_offset": 0, 186 | "zplane_offset": 0, 187 | "file_format": "PseudoCycle{}/MMStack_Pos{}_ch{}.ome.tif", 188 | "file_vars": ["round", "fov", "channel"], 189 | "cache_read_order": cache, 190 | "aux_names": aux_names, 191 | "aux_file_formats": aux_file_formats, 192 | "aux_file_vars": aux_file_vars, 193 | "aux_cache_read_order": aux_cache_read_order, 194 | "aux_channel_slope": aux_channel_slope, 195 | "aux_channel_intercept": aux_channel_intercept, 196 | "aux_channel_count": aux_channel_count 197 | }; 198 | } 199 | 200 | inputs: 201 | exp_dir: 202 | type: Directory 203 | aux_names: 204 | type: string[]? 205 | cache_read_order: 206 | type: string[] 207 | aux_cache_read_order: 208 | type: string[]? 209 | channel_count: 210 | type: int? 211 | 212 | outputs: 213 | codebook: 214 | type: File 215 | round_offset: 216 | type: int 217 | fov_offset: 218 | type: int 219 | channel_offset: 220 | type: int 221 | zplane_offset: 222 | type: int 223 | file_format: 224 | type: string 225 | file_vars: 226 | type: string[] 227 | cache_read_order: 228 | type: string[] 229 | aux_names: 230 | type: string[] 231 | aux_file_formats: 232 | type: string[] 233 | aux_file_vars: 234 | type: string[] 235 | aux_cache_read_order: 236 | type: string[] 237 | aux_channel_slope: 238 | type: string[] 239 | aux_channel_intercept: 240 | type: string[] 241 | aux_channel_count: 242 | type: int[] 243 | in: 244 | exp_dir: exp_dir 245 | aux_names: 246 | source: [stage_defaults/aux_tilesets_aux_names, aux_names] 247 | valueFrom: | 248 | ${ 249 | if(self[0]){ 250 | return self[0]; 251 | } else if(self[1]){ 252 | return self[1]; 253 | } else { 254 | return null; 255 | } 256 | } 257 | cache_read_order: 258 | source: [stage_defaults/cache_read_order, cache_read_order] 259 | pickValue: first_non_null 260 | channel_count: 261 | source: [stage_defaults/channel_count, channel_count] 262 | pickValue: first_non_null 263 | aux_cache_read_order: 264 | source: [stage_defaults/aux_tilesets_aux_cache_read_order, aux_cache_read_order] 265 | valueFrom: | 266 | ${ 267 | if(self[0]){ 268 | return self[0]; 269 | } else if (self[1]){ 270 | return self[1]; 271 | } else { 272 | return null; 273 | } 274 | } 275 | 276 | out: [codebook, round_offset, fov_offset, channel_offset, zplane_offset, file_format, file_vars, cache_read_order, aux_names, aux_file_formats, aux_file_vars, aux_cache_read_order, aux_channel_slope, aux_channel_intercept, aux_channel_count] 277 | -------------------------------------------------------------------------------- /steps/qc.cwl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cwl-runner 2 | 3 | class: Workflow 4 | cwlVersion: v1.2 5 | 6 | requirements: 7 | - class: SubworkflowFeatureRequirement 8 | - class: InlineJavascriptRequirement 9 | - class: StepInputExpressionRequirement 10 | - class: MultipleInputFeatureRequirement 11 | 12 | inputs: 13 | 14 | codebook_exp: 15 | type: Directory? 16 | doc: Flattened codebook input, refer to record entry. 17 | 18 | codebook_pkl: 19 | type: File? 20 | doc: Flattened codebook input, refer to record entry. 21 | 22 | locs_json: 23 | type: File? 24 | doc: Flattened json input, refer to record entry. 25 | 26 | codebook: 27 | type: 28 | - 'null' 29 | - type: record 30 | name: pkl 31 | fields: 32 | pkl: 33 | type: File 34 | doc: A codebook for this experiment, saved in a python pickle. 35 | - type: record 36 | name: exp 37 | fields: 38 | exp: 39 | type: Directory 40 | doc: The location of an experiment.json file, which has the corresponding codebook for this experiment. 41 | segmentation_loc: 42 | type: Directory? 43 | doc: The location of the output from the segmentation step, if it was performed. 44 | 45 | data_pkl_spots: 46 | type: File? 47 | doc: Flattened data input, refer to record entry. 48 | 49 | data_pkl_transcripts: 50 | type: File? 51 | doc: Flattened data input, refer to record entry. 52 | 53 | data_exp: 54 | type: Directory? 55 | doc: Flattened data input, refer to record entry. 56 | 57 | data: 58 | type: 59 | - 'null' 60 | - type: record 61 | name: pkl 62 | fields: 63 | spots: 64 | type: File? 65 | doc: Spots found in this experiment, saved in a python pickle. 66 | transcripts: 67 | type: File 68 | doc: The output DecodedIntensityTable, saved in a python pickle. 69 | - type: record 70 | name: exp 71 | fields: 72 | exp: 73 | type: Directory 74 | doc: The location of output of starfish runner step, 4_Decoded. Contains spots (if applicable) and netcdfs containing the DecodedIntensityTable. 75 | 76 | selected_fovs: 77 | type: int[]? 78 | doc: If provided, QC will only be run on FOVs with these indices. 79 | 80 | has_spots: 81 | type: boolean? 82 | doc: If true, will look for spots within the experiment field. 83 | 84 | roi: 85 | type: File? 86 | doc: The location of the RoiSet.zip, if applicable. 87 | 88 | parameter_json: 89 | type: File? 90 | doc: The json with parameters to be read in for the following variables. 91 | 92 | imagesize: 93 | type: 94 | - 'null' 95 | - type: record 96 | name: locs 97 | fields: 98 | locs: 99 | type: File? 100 | doc: Input locations as a json file, using the same records as below. 101 | - type: record 102 | name: fov_positioning 103 | fields: 104 | - name: x_size 105 | type: int 106 | doc: x-dimension of image 107 | - name: y_size 108 | type: int 109 | doc: y-dimension of image 110 | - name: z_size 111 | type: int 112 | doc: number of z-stacks 113 | 114 | spot_threshold: 115 | type: float? 116 | doc: If has_spots is true and this is provided, spots with an intensity lower than this will not be included in qc metrics 117 | 118 | find_ripley: 119 | type: boolean? 120 | doc: If true, will run ripley K estimates to find spatial density measures. Can be slow. 121 | default: False 122 | 123 | save_pdf: 124 | type: boolean? 125 | doc: If true, will save graphical output to a pdf. 126 | default: True 127 | 128 | outputs: 129 | qc_metrics: 130 | type: Directory 131 | outputSource: execute_qc/qc_metrics 132 | 133 | steps: 134 | 135 | tmpname: 136 | run: tmpdir.cwl 137 | in: [] 138 | out: [tmp] 139 | 140 | read_schema: 141 | run: 142 | class: CommandLineTool 143 | baseCommand: cat 144 | 145 | requirements: 146 | DockerRequirement: 147 | dockerPull: hubmap/starfish-custom:latest 148 | ResourceRequirement: 149 | ramMin: 1000 150 | tmpdirMin: 1000 151 | outdirMin: 1000 152 | 153 | inputs: 154 | schema: 155 | type: string 156 | inputBinding: 157 | position: 1 158 | 159 | outputs: 160 | data: 161 | type: stdout 162 | 163 | in: 164 | schema: 165 | valueFrom: "/opt/qc.json" 166 | out: [data] 167 | 168 | stage_qc: 169 | run: inputParser.cwl 170 | in: 171 | datafile: parameter_json 172 | schema: read_schema/data 173 | out: [selected_fovs, find_ripley, save_pdf, fov_positioning_x_shape, fov_positioning_y_shape, fov_positioning_z_shape, decoding_decode_method, decoding_magnitude_threshold, decoding_decoder_min_intensity] 174 | when: $(inputs.datafile != null) 175 | 176 | codebook_grabber: 177 | run: 178 | class: ExpressionTool 179 | requirements: 180 | - class: InlineJavascriptRequirement 181 | - class: LoadListingRequirement 182 | 183 | inputs: 184 | experiment: 185 | type: Directory? 186 | doc: A directory containing a spaceTx-formatted experiment 187 | 188 | outputs: 189 | codebook: 190 | type: File? 191 | 192 | expression: | 193 | ${ 194 | for(var i=0;i 0){ 832 | listing.push({"class":"Directory","basename":"spots","listing":spots}); 833 | } 834 | return {"pool_dir": { 835 | "class": "Directory", 836 | "basename": "4_Decoded", 837 | "listing": listing, 838 | }}; 839 | } 840 | inputs: 841 | dir_size: 842 | type: long 843 | 844 | file_array: 845 | type: 846 | type: array 847 | items: Directory 848 | 849 | outputs: 850 | pool_dir: 851 | type: Directory 852 | 853 | in: 854 | file_array: execute_runner/decoded 855 | dir_size: dir_size 856 | out: [pool_dir] 857 | -------------------------------------------------------------------------------- /steps/tmpdir.cwl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cwl-runner 2 | class: ExpressionTool 3 | cwlVersion: v1.2 4 | 5 | requirements: 6 | InlineJavascriptRequirement: {} 7 | ResourceRequirement: 8 | ramMin: 1000 9 | tmpdirMin: 1000 10 | outdirMin: 1000 11 | 12 | expression: | 13 | ${ 14 | const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; 15 | const charactersLength = characters.length; 16 | let counter = 0; 17 | let result = ""; 18 | while (counter < 10) { 19 | result += characters.charAt(Math.floor(Math.random() * charactersLength)); 20 | counter += 1; 21 | } 22 | return {"tmp": result}; 23 | } 24 | inputs: [] 25 | outputs: 26 | tmp: string 27 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | set -o pipefail 4 | 5 | start() { echo travis_fold':'start:$1; echo $1; } 6 | end() { set +v; echo travis_fold':'end:$1; echo; echo; } 7 | die() { set +v; echo "$*" 1>&2 ; exit 1; } 8 | 9 | start black 10 | black --check . 11 | end black 12 | 13 | start isort 14 | isort --check-only . 15 | end isort 16 | --------------------------------------------------------------------------------