| ATAC |
| Total reads | " ~{atac_total_reads} " |
" > output.txt
54 | echo "| Aligned uniquely | " ~{atac_aligned_uniquely} " |
" >> output.txt
55 | echo "| Unaligned | " ~{atac_unaligned} " |
" >> output.txt
56 | echo "| Unique Reads | " ~{atac_feature_reads} " |
" >> output.txt
57 | echo "| Duplicate Reads | " ~{atac_duplicate_reads} " |
" >> output.txt
58 | echo "| Percent Duplicates | " ~{atac_percent_duplicates} " |
" >> output.txt
59 | echo "RNA | | Total reads | " ~{rna_total_reads} " |
" >> output.txt
60 | echo "| Aligned uniquely | " ~{rna_aligned_uniquely} " |
" >> output.txt
61 | echo "| Aligned multimap | " ~{rna_aligned_multimap} " |
" >> output.txt
62 | echo "| Unaligned | " ~{rna_unaligned} " |
" >> output.txt
63 | echo "| Filtered (feature) Reads | " ~{rna_feature_reads} " |
" >> output.txt
64 | echo "| Duplicate Reads | " ~{rna_duplicate_reads} " |
" >> output.txt
65 | percent=$(( ~{default=0 rna_duplicate_reads}*100/~{default=1 rna_feature_reads} ))
66 | echo "| Percent Duplicates | " $percent " |
" >> output.txt
67 | PYTHONIOENCODING=utf-8 python3 /software/write_html.py ~{output_file} image_list.txt log_list.txt --input_file_name output.txt
68 | >>>
69 | output {
70 | File html_report_file = "~{output_file}"
71 | }
72 |
73 | runtime {
74 | docker: 'us.gcr.io/buenrostro-share-seq/share_task_html_report:v1.0.0'
75 | }
76 | }
77 |
--------------------------------------------------------------------------------
/src/python/filter_mito_reads.py:
--------------------------------------------------------------------------------
1 | # From Kundaje lab
2 | # https://github.com/kundajelab/ENCODE_scatac/blob/master/workflow/scripts/filter_mito.py
3 | #
4 | # Modified by: Eugenio Mattei
5 | # Affiliation: The Broad InstituteOf MIT and Harvard
6 | #
7 | # Changelog:
8 | # 2023/01/20: Now it returns the statistics per barcode
9 | #
10 |
11 | import argparse
12 | import pysam
13 | from collections import defaultdict
14 |
15 |
16 |
17 | def filter_mito(in_path, out_path, barcode_tag, cutoff, prefix, threads=1):
18 | """
19 | Removes mitochondrial alignments from BAM
20 | Calculates number of mapped mitochondrial and non-mitochondrial reads (not alignments)
21 | Assumes mitochondrial chromosome is "chrM"
22 | """
23 |
24 | infile = pysam.AlignmentFile(in_path, "rb", threads=threads)
25 | outfile = pysam.AlignmentFile(out_path, "wb", template=infile, threads=threads)
26 | outfile_bulk_metrics = f"{prefix}.mito.bulk-metrics.tsv"
27 | outfile_barcode_metrics = f"{prefix}.mito.bc-metrics.tsv"
28 |
29 | number_mito = 0
30 | number_non_mito = 0
31 |
32 | # Initializing the dictionary setting the counts for non-mito and mito.
33 | barcode_metrics = defaultdict(lambda: [0,0])
34 |
35 | for read in infile.fetch(until_eof=True,multiple_iterators=True):
36 | if read.reference_name == "chrM":
37 | if read.flag & 260 == 0: # Alignment is mapped and is primary
38 | number_mito += 1
39 | barcode_metrics[read.get_tag(barcode_tag)][1] += 1
40 |
41 | else:
42 | if read.flag & 260 == 0:
43 | number_non_mito += 1
44 | barcode_metrics[read.get_tag(barcode_tag)][0] += 1
45 | #outfile.write(read)
46 |
47 | # Write the summary metrics
48 | with open(outfile_bulk_metrics, "w") as fh:
49 | print("raw_reads_nonmito\traw_reads_mito", file = fh)
50 | print(f"{number_non_mito}\t{number_mito}", file = fh)
51 |
52 | # Write the metrics per barcode
53 | with open(outfile_barcode_metrics, "w") as fh:
54 | # Print header
55 | print("barcode\traw_reads_nonmito\traw_reads_mito", file = fh)
56 | for barcode,counts in barcode_metrics.items():
57 | print(f"{barcode}\t{counts[0]}\t{counts[1]}", file = fh)
58 |
59 | # Write a filtered bam
60 | for read in infile:
61 | if read.flag & 260 == 0 and read.reference_name != "chrM" and barcode_metrics[read.get_tag(barcode_tag)][0] > cutoff*2:
62 | outfile.write(read)
63 |
64 | outfile.close()
65 | return
66 |
67 |
68 |
69 | if __name__ == '__main__':
70 |
71 | msg = "Add the description"
72 | parser = argparse.ArgumentParser(description = msg)
73 |
74 | # Adding optional argument
75 | parser.add_argument("bam", help = "Path to the coordinate-sorted bam file.")
76 | parser.add_argument("-o", "--output", help = "Path to the mitochondrial-free bam file.")
77 | parser.add_argument("-p", help = "Number of threads to use.", type=int, default=1)
78 | parser.add_argument("--prefix", help = "Prefix for the metrics output file.")
79 | parser.add_argument("--cutoff", help = "Remove barcodes with a number of fragments less than the cutoff.", type=int, default=1)
80 | parser.add_argument("--bc_tag", help = "Specify the tag containing the cell barcode.", default="CB")
81 |
82 | # Read arguments from command line
83 | args = parser.parse_args()
84 |
85 | if args.prefix:
86 | prefix = args.prefix
87 | else:
88 | prefix = args.bam[:-4]
89 |
90 | if args.output:
91 | out_path = args.output
92 | else:
93 | out_path = f"{prefix}.no_mito.bam"
94 |
95 | bc_tag = args.bc_tag
96 |
97 | filter_mito(args.bam, out_path, bc_tag, args.cutoff, prefix, threads=args.p)
98 |
--------------------------------------------------------------------------------
/src/R/cell_annotation_helper_functions.R:
--------------------------------------------------------------------------------
1 | #!/usr/bin/Rscript
2 |
3 | ## ---------------------------
4 | ## Helper functions for cell annotation
5 | ## Author: Zhijian Li
6 | ## Date Created: 2023-05-29
7 | ## Email: lzj1769@gmail.com
8 | ## ---------------------------
9 | library(reticulate)
10 | use_python("/usr/bin/python3")
11 |
12 | read_h5ad <- function(
13 | filename,
14 | backed = NULL
15 | ) {
16 | python_anndata <- reticulate::import("anndata", convert = FALSE)
17 | filename <- normalizePath(filename, mustWork = FALSE)
18 | py_to_r_ifneedbe(python_anndata$read_h5ad(
19 | filename = filename,
20 | backed = backed
21 | ))
22 | }
23 |
24 | py_to_r_ifneedbe <- function(x) {
25 | if (inherits(x, "python.builtin.object")) {
26 | py_to_r(x)
27 | } else {
28 | x
29 | }
30 | }
31 |
32 | #' @name r-py-conversion
33 | #' @export
34 | py_to_r.pandas.core.indexes.base.Index <- function(x) {
35 | python_builtins <- reticulate::import_builtins()
36 | out <- python_builtins$list(x)
37 | attr(out, "name") <- py_to_r_ifneedbe(x$name)
38 | out
39 | }
40 |
41 | #' Convert between Python and R objects
42 | #'
43 | #' @param x A Python object.
44 | #' @param name A name
45 | #' @param value A value
46 | #'
47 | #' @return An \R object, as converted from the Python object.
48 | #'
49 | #' @name r-py-conversion
50 | #' @export
51 | `[[<-.collections.abc.MutableMapping` <- function(x, name, value) {
52 | if (!is.null(value)) {
53 | reticulate::py_set_item(x, name, value)
54 | } else if (name %in% x$keys()) {
55 | reticulate::py_del_item(x, name)
56 | }
57 | }
58 |
59 | #' @name r-py-conversion
60 | #' @export
61 | `[[.collections.abc.Mapping` <- function(x, name) {
62 | if (name %in% x$keys()) {
63 | py_to_r_ifneedbe(reticulate::py_get_item(x, name))
64 | } else {
65 | NULL
66 | }
67 | }
68 |
69 | #' @name r-py-conversion
70 | #' @export
71 | `[<-.collections.abc.MutableMapping` <- `[[<-.collections.abc.MutableMapping`
72 | #
73 | #' @name r-py-conversion
74 | #' @export
75 | `[.collections.abc.Mapping` <- `[[.collections.abc.Mapping`
76 | #
77 | #' @name r-py-conversion
78 | #' @export
79 | `names.collections.abc.Mapping` <- function(x) {
80 | python_builtins <- reticulate::import_builtins()
81 | python_builtins$list(x$keys())
82 | }
83 |
84 | #' @name r-py-conversion
85 | #' @export
86 | `py_to_r.collections.abc.Set` <- function(x) {
87 | python_builtins <- reticulate::import_builtins()
88 | python_builtins$list(x)
89 | }
90 |
91 | #' @name r-py-conversion
92 | #' @export
93 | py_to_r.pandas.core.indexes.base.Index <- function(x) {
94 | python_builtins <- reticulate::import_builtins()
95 | out <- python_builtins$list(x)
96 | attr(out, "name") <- py_to_r_ifneedbe(x$name)
97 | out
98 | }
99 |
100 | #' @name r-py-conversion
101 | #' @export
102 | py_to_r.collections.abc.KeysView <- function(x) {
103 | python_builtins <- reticulate::import_builtins()
104 | python_builtins$list(x)
105 | }
106 |
107 | #' @name r-py-conversion
108 | #' @export
109 | `py_to_r.collections.abc.Mapping` <- function(x) {
110 | python_builtins <- reticulate::import_builtins()
111 |
112 | x_list <- python_builtins$dict(x)
113 |
114 | # convert members of x_list if need be
115 | for (i in seq_along(x_list)) {
116 | if (inherits(x_list[[i]], "python.builtin.object")) {
117 | x_list[[i]] <- py_to_r_ifneedbe(x_list[[i]])
118 | }
119 | }
120 |
121 | x_list
122 | }
123 |
124 |
125 | #' @importFrom Matrix sparseMatrix
126 | py_to_r.scipy.sparse.csc.csc_matrix <- function(x) {
127 | Matrix::sparseMatrix(
128 | i = as.integer(py_to_r_ifneedbe(x$indices))+1,
129 | p = as.integer(py_to_r_ifneedbe(x$indptr)),
130 | x = as.vector(py_to_r_ifneedbe(x$data)),
131 | dims = as.integer(dim(x))
132 | )
133 | }
134 |
--------------------------------------------------------------------------------
/workflows/subwf-find-dorcs.wdl:
--------------------------------------------------------------------------------
1 | version 1.0
2 |
3 |
4 | # Import the tasks called by the pipeline
5 | import "../tasks/dorcs_task_find_dorcs.wdl" as find_dorcs
6 |
7 | workflow wf_dorcs {
8 |
9 | meta {
10 | version: 'v0.1'
11 | author: 'Siddarth Wekhande (swekhand@broadinstitute.org)'
12 | description: 'Broad Institute of MIT and Harvard SHARE-Seq pipeline: Sub-workflow to find DORCs from SHARE-seq data.'
13 | }
14 |
15 | input {
16 | File? rna_matrix
17 | File? atac_fragments
18 | File peak_file
19 |
20 | String genome
21 | Int n_cores = 4
22 | String save_plots_to_dir = "TRUE"
23 | String? output_filename
24 |
25 | Int minFeature_RNA = 200
26 | Int maxFeature_RNA = 2500
27 | Float percentMT_RNA = 5
28 | Int minCells_RNA = 3
29 |
30 | Int dorcGeneCutOff = 10
31 | Float fripCutOff = 0.3
32 | Float corrPVal = 0.05
33 | Int topNGene = 20
34 | Int windowPadSize = 50000
35 |
36 | Int numNearestNeighbor = 30
37 | Float numBackgroundPairs = 100000
38 | Float chunkSize = 50000
39 |
40 | String? prefix
41 | Int mem_gb = 64
42 | Int disk_gb = 100
43 | String? docker
44 | }
45 |
46 | File rna_matrix_ = select_first([rna_matrix])
47 | File atac_fragments_ = select_first([atac_fragments])
48 |
49 | if ( !defined(rna_matrix) || !defined(atac_fragments) ){
50 | call raise_exception as missing_input {
51 | input:
52 | msg = "The genes-by-cell matrix or the dna fragments file are missing."
53 | }
54 | }
55 |
56 | call find_dorcs.find_dorcs as find_dorcs{
57 | input:
58 | rna_matrix = rna_matrix_,
59 | atac_fragments = atac_fragments_,
60 | peak_file = peak_file,
61 | genome = genome,
62 | n_cores = n_cores,
63 | save_plots_to_dir = save_plots_to_dir,
64 | output_filename = output_filename,
65 | minFeature_RNA = minFeature_RNA,
66 | maxFeature_RNA = maxFeature_RNA,
67 | percentMT_RNA = percentMT_RNA,
68 | minCells_RNA = minCells_RNA,
69 | dorcGeneCutOff = dorcGeneCutOff,
70 | fripCutOff = fripCutOff,
71 | corrPVal = corrPVal,
72 | topNGene = topNGene,
73 | windowPadSize = windowPadSize,
74 | numNearestNeighbor = numNearestNeighbor,
75 | numBackgroundPairs = numBackgroundPairs,
76 | chunkSize = chunkSize,
77 | mem_gb = mem_gb,
78 | disk_gb = disk_gb,
79 | docker_image = docker,
80 | prefix = prefix
81 | }
82 |
83 | output {
84 | File dorcs_notebook_output = find_dorcs.notebook_output
85 | File dorcs_notebook_log = find_dorcs.notebook_log
86 | File? seurat_violin_plot = find_dorcs.seurat_violin_plot
87 | File? j_plot = find_dorcs.j_plot
88 | File? plots_zip = find_dorcs.plots_zip
89 | File? dorcs_genes_summary = find_dorcs.dorcs_genes_summary
90 | File? dorcs_regions_summary = find_dorcs.dorcs_regions_summary
91 | }
92 |
93 | }
94 |
95 | # Task to report errors to user.
96 | # From https://github.com/ENCODE-DCC/chip-seq-pipeline2/blob/master/chip.wdl
97 | task raise_exception {
98 | input {
99 | String msg
100 | Array[String]? vals
101 | }
102 | command {
103 | echo -e "\n* Error: ${msg}\n" >&2
104 | echo -e "* Vals: ${sep=',' vals}\n" >&2
105 | exit 2
106 | }
107 | output {
108 | String error_msg = '${msg}'
109 | }
110 | runtime {
111 | maxRetries : 0
112 | cpu : 1
113 | memory : '2 GB'
114 | time : 1
115 | disks : 'local-disk 10 SSD'
116 | docker : 'encodedcc/chip-seq-pipeline:v2.2.1'
117 | }
118 | }
119 |
--------------------------------------------------------------------------------
/dockerfiles/share_task_filter_atac.dockerfile:
--------------------------------------------------------------------------------
1 | ############################################################
2 | # Dockerfile for BROAD GRO share-seq-pipeline
3 | # Based on Debian slim
4 | ############################################################
5 |
6 | FROM debian:buster-slim as builder
7 |
8 | ENV BEDTOOLS_VERSION v2.29.0
9 | ENV PICARD_VERSION 2.27.5
10 | ENV SAMTOOLS_VERSION 1.16
11 | ENV SAMBAMBA_VERSION 0.6.6
12 |
13 | # To prevent time zone prompt
14 | ENV DEBIAN_FRONTEND=noninteractive
15 |
16 | # Install softwares from apt repo
17 | RUN apt-get update && apt-get install -y \
18 | autoconf \
19 | automake \
20 | build-essential \
21 | git \
22 | libcurl4-openssl-dev \
23 | liblz4-dev \
24 | liblzma-dev \
25 | libncurses5-dev \
26 | libncursesw5-dev \
27 | libbz2-dev \
28 | perl \
29 | python \
30 | unzip \
31 | xz-utils \
32 | wget \
33 | zlib1g-dev &&\
34 | rm -rf /var/lib/apt/lists/*
35 |
36 | # Make directory for all softwares
37 | RUN mkdir /software
38 | WORKDIR /software
39 | ENV PATH="/software:${PATH}"
40 |
41 | # Install bedtools 2.29.0
42 | RUN git clone --branch ${BEDTOOLS_VERSION} --single-branch https://github.com/arq5x/bedtools2.git && \
43 | cd bedtools2 && make && make install && cd ../ && rm -rf bedtools2*
44 |
45 | # Install sambamba 0.6.6
46 | RUN wget https://github.com/lomereiter/sambamba/releases/download/v${SAMBAMBA_VERSION}/sambamba_v${SAMBAMBA_VERSION}_linux.tar.bz2 && \
47 | tar -xvjf sambamba_v${SAMBAMBA_VERSION}_linux.tar.bz2 && \
48 | mv sambamba_v${SAMBAMBA_VERSION} /usr/local/bin/sambamba && \
49 | rm -rf sambamba_*
50 |
51 | # Install samtools 1.16
52 | RUN git clone --branch ${SAMTOOLS_VERSION} --single-branch https://github.com/samtools/htslib.git && \
53 | cd htslib && git submodule update --init --recursive && autoreconf -i && make && make install && cd ../ && \
54 | git clone --branch ${SAMTOOLS_VERSION} --single-branch https://github.com/samtools/samtools.git && \
55 | cd samtools && make && make install && cd ../ && rm -rf samtools* && rm -rf htslib*
56 |
57 |
58 | # Install Picard 2.20.7
59 | RUN wget https://github.com/broadinstitute/picard/releases/download/${PICARD_VERSION}/picard.jar && chmod +x picard.jar && mv picard.jar /usr/local/bin
60 |
61 |
62 |
63 | FROM debian:buster-slim
64 |
65 | LABEL maintainer = "Eugenio Mattei"
66 | LABEL software = "Share-seq pipeline"
67 | LABEL software.version="1.0.0"
68 | LABEL software.organization="Broad Institute of MIT and Harvard"
69 | LABEL software.version.is-production="Yes"
70 | LABEL software.task="filter"
71 |
72 | RUN apt-get update && apt-get install -y \
73 | gcc \
74 | libcurl4-openssl-dev \
75 | libbz2-dev \
76 | liblzma-dev \
77 | python3 \
78 | python3-dev \
79 | python3-pip \
80 | openjdk-11-jre \
81 | zlib1g-dev &&\
82 | rm -rf /var/lib/apt/lists/*
83 |
84 | # Install packages for python3 scripts
85 | RUN python3 -m pip install --upgrade pip
86 | RUN python3 -m pip install --no-cache-dir --ignore-installed pysam
87 |
88 | # Create and setup new user
89 | ENV USER=shareseq
90 | WORKDIR /home/$USER
91 |
92 | RUN groupadd -r $USER &&\
93 | useradd -r -g $USER --home /home/$USER -s /sbin/nologin -c "Docker image user" $USER &&\
94 | chown $USER:$USER /home/$USER
95 |
96 | # Add folder with software to the path
97 | ENV PATH="/software:${PATH}"
98 |
99 | # Copy the compiled software from the builder
100 | COPY --from=builder --chown=$USER:$USER /usr/local/bin/* /usr/local/bin/
101 | COPY --from=builder --chown=$USER:$USER /lib/x86_64-linux-gnu/* /lib/x86_64-linux-gnu/
102 | COPY --chown=$USER:$USER src/bash/monitor_script.sh /usr/local/bin
103 | COPY --chown=$USER:$USER src/python/filter_mito_reads.py /usr/local/bin
104 | COPY --chown=$USER:$USER src/python/bam_to_fragments.py /usr/local/bin
105 | COPY --chown=$USER:$USER src/python/assign_multimappers.py /usr/local/bin
106 |
107 |
108 | USER ${USER}
109 |
--------------------------------------------------------------------------------
/dockerfiles/share_task_qc_atac.dockerfile:
--------------------------------------------------------------------------------
1 | ############################################################
2 | # Dockerfile for BROAD GRO share-seq-pipeline
3 | # Based on Debian slim
4 | ############################################################
5 |
6 | FROM debian:buster-slim as builder
7 |
8 | ENV SAMTOOLS_VERSION 1.9
9 | ENV BEDTOOLS_VERSION v2.29.0
10 | ENV PICARD_VERSION 2.27.5
11 |
12 | # To prevent time zone prompt
13 | ENV DEBIAN_FRONTEND=noninteractive
14 |
15 | # Install softwares from apt repo
16 | RUN apt-get update && apt-get install -y \
17 | autoconf \
18 | build-essential \
19 | git \
20 | libcurl4-openssl-dev \
21 | liblz4-dev \
22 | liblzma-dev \
23 | libncurses5-dev \
24 | libbz2-dev \
25 | python \
26 | unzip \
27 | wget \
28 | zlib1g-dev &&\
29 | rm -rf /var/lib/apt/lists/*
30 |
31 |
32 | # Make directory for all softwares
33 | RUN mkdir /software
34 | WORKDIR /software
35 | ENV PATH="/software:${PATH}"
36 |
37 | # Install bedtools 2.29.0
38 | RUN git clone --branch ${BEDTOOLS_VERSION} --single-branch https://github.com/arq5x/bedtools2.git && \
39 | cd bedtools2 && make && make install && cd ../ && rm -rf bedtools2*
40 |
41 | # Install samtools 1.9
42 | RUN git clone --branch ${SAMTOOLS_VERSION} --single-branch https://github.com/samtools/samtools.git && \
43 | git clone --branch ${SAMTOOLS_VERSION} --single-branch https://github.com/samtools/htslib.git && \
44 | cd samtools && make && make install && cd ../ && rm -rf samtools* && \
45 | cd htslib && autoreconf -i && make && make install && cd ../ && rm -rf htslib*
46 |
47 | # Install Picard 2.20.7
48 | RUN wget https://github.com/broadinstitute/picard/releases/download/${PICARD_VERSION}/picard.jar && chmod +x picard.jar && mv picard.jar /usr/local/bin
49 |
50 |
51 |
52 | FROM debian:buster-slim
53 |
54 | LABEL maintainer = "Eugenio Mattei"
55 | LABEL software = "Share-seq pipeline"
56 | LABEL software.version="1.0.0"
57 | LABEL software.organization="Broad Institute of MIT and Harvard"
58 | LABEL software.version.is-production="Yes"
59 | LABEL software.task="qc-atac"
60 |
61 | RUN apt-get update && apt-get install -y \
62 | gcc \
63 | git \
64 | python3 \
65 | python3-dev \
66 | python3-pip \
67 | openjdk-11-jre \
68 | r-base \
69 | zlib1g-dev &&\
70 | rm -rf /var/lib/apt/lists/*
71 |
72 | # Install packages for python3 scripts (pysam, SAMstats)
73 | RUN python3 -m pip install --upgrade pip
74 | RUN python3 -m pip install --no-cache-dir --ignore-installed numpy matplotlib pandas plotnine pysam --editable=git+https://github.com/kundajelab/SAMstats@75e60f1e67c6d5d066371a0b53729e4b1f6f76c5#egg=SAMstats
75 |
76 | # Create and setup new user
77 | ENV USER=shareseq
78 | WORKDIR /home/$USER
79 |
80 | RUN groupadd -r $USER &&\
81 | useradd -r -g $USER --home /home/$USER -s /sbin/nologin -c "Docker image user" $USER &&\
82 | chown $USER:$USER /home/$USER
83 |
84 | # Add folder with software to the path
85 | ENV PATH="/software:${PATH}"
86 |
87 | # Copy the compiled software from the builder
88 | COPY --from=builder --chown=$USER:$USER /usr/local/bin/* /usr/local/bin/
89 | COPY --from=builder --chown=$USER:$USER /lib/x86_64-linux-gnu/* /lib/x86_64-linux-gnu/
90 | COPY --chown=$USER:$USER src/bash/monitor_script.sh /usr/local/bin
91 | COPY --chown=$USER:$USER src/python/pbc_stats.py /usr/local/bin
92 | COPY --chown=$USER:$USER src/python/qc_atac_compute_tss_enrichment.py /usr/local/bin
93 | COPY --chown=$USER:$USER src/python/qc_atac_count_duplicates_per_barcode.py /usr/local/bin
94 | COPY --chown=$USER:$USER src/python/qc_atac_compute_reads_in_peaks.py /usr/local/bin
95 | COPY --chown=$USER:$USER src/python/plot_insert_size_hist.py /usr/local/bin
96 | COPY --chown=$USER:$USER src/R/barcode_rank_functions.R /usr/local/bin
97 | COPY --chown=$USER:$USER src/R/atac_qc_plots.R /usr/local/bin
98 | COPY --chown=$USER:$USER src/bash/monitor_script.sh /usr/local/bin
99 |
100 |
101 | USER ${USER}
102 |
103 |
104 |
--------------------------------------------------------------------------------
/tasks/dorcs_task_find_dorcs.wdl:
--------------------------------------------------------------------------------
1 | version 1.0
2 |
3 | task find_dorcs {
4 | meta {
5 | version: 'v0.1'
6 | author: 'Siddarth Wekhande (swekhand@broadinstitute.org) at Broad Institute of MIT and Harvard'
7 | description: 'Broad Institute of MIT and Harvard SHARE-Seq pipeline: find DORCs task'
8 | }
9 |
10 | input {
11 | #This task takes in the RNA and ATAC files and finds the DORCs based on the cut-off criteria provided
12 |
13 | #DORCs parameters
14 | File rna_matrix
15 | File atac_fragments
16 | File? peak_file
17 | String genome
18 | Int n_cores = 4
19 | String save_plots_to_dir = "TRUE"
20 | String prefix = "prefix"
21 |
22 | #RNA QC parameters
23 | Int minFeature_RNA = 200
24 | Int maxFeature_RNA = 2500
25 | Float percentMT_RNA = 5
26 | Int minCells_RNA = 3
27 |
28 | #ATAC QC parameter
29 | Float fripCutOff = 0.3
30 | Float chunkSize = 50000
31 |
32 | #Background correlation parameters
33 | Int numNearestNeighbor = 100
34 | Float numBackgroundPairs = 100000
35 |
36 | #DORC genes parameter
37 | # Regulatory region around TSS. Default is +/- 50Kb
38 | Int windowPadSize = 50000
39 | Int dorcGeneCutOff = 10
40 | Float corrPVal = 0.05
41 | Int topNGene = 20
42 |
43 | String output_filename = "${prefix}.dorcs.notebook.${genome}.ipynb"
44 | String docker_image = "us.gcr.io/buenrostro-share-seq/dorcs_task_find_dorcs:v1.0.0"
45 | #String docker_image = "swekhande/shareseq-prod:share-task-dorcs"
46 | Int mem_gb = 64
47 | Int disk_gb = 100
48 | }
49 |
50 | #Output filepaths
51 |
52 | String violin_plot = '${prefix}.dorcs.plots.${genome}/${prefix}.dorcs.rna_violin_plot.${genome}.png'
53 | String jplot = '${prefix}.dorcs.plots.${genome}/${prefix}.dorcs.jplot.${genome}.png'
54 | String dorc_genes_summ = '${prefix}.dorcs.dorc_genes_summary.${genome}.csv'
55 | String all_regions_summ = '${prefix}.dorcs.all_regions_summary.${genome}.csv'
56 | String plots_zip_dir = '${prefix}.dorcs.plots.${genome}.zip'
57 | #String papermill_log_filename = 'papermill.logfile.txt'
58 | String log_filename = "log/${prefix}.dorcs.logfile.${genome}.txt"
59 |
60 | command {
61 | gzip -dc ${atac_fragments} > tmp_fragments.bedpe
62 |
63 | papermill $(which dorcs_jplot_notebook.ipynb) ${output_filename} \
64 | -p rnaCountMatrix ${rna_matrix} \
65 | -p atacFragFile tmp_fragments.bedpe \
66 | -p peakFile ${peak_file} \
67 | -p savePlotsToDir ${save_plots_to_dir} \
68 | -p nCores ${n_cores} \
69 | -p genome ${genome} \
70 | -p minFeature_RNA ${minFeature_RNA} \
71 | -p maxFeature_RNA ${maxFeature_RNA} \
72 | -p percentMT_RNA ${percentMT_RNA} \
73 | -p minCells_RNA ${minCells_RNA} \
74 | -p dorcGeneCutOff ${dorcGeneCutOff} \
75 | -p fripCutOff ${fripCutOff} \
76 | -p corrPVal ${corrPVal} \
77 | -p topNGene ${topNGene} \
78 | -p windowPadSize ${windowPadSize} \
79 | -p numNearestNeighbor ${numNearestNeighbor} \
80 | -p numBackgroundPairs ${numBackgroundPairs} \
81 | -p chunkSize ${chunkSize} \
82 | -p prefix ${prefix}
83 | }
84 |
85 | output {
86 | File notebook_output = output_filename
87 | File notebook_log = log_filename
88 | #File papermill_log = papermill_log_filename
89 |
90 | File? seurat_violin_plot = violin_plot
91 | File? j_plot = jplot
92 | File? plots_zip = plots_zip_dir
93 |
94 | File? dorcs_genes_summary = dorc_genes_summ
95 | File? dorcs_regions_summary = all_regions_summ
96 |
97 |
98 | }
99 |
100 | runtime {
101 | cpu : 4
102 | memory : mem_gb+'G'
103 | docker : docker_image
104 | disks : 'local-disk ${disk_gb} LOCAL'
105 | maxRetries : 0
106 | }
107 | }
108 |
109 |
110 |
--------------------------------------------------------------------------------
/tasks/share_task_star.wdl:
--------------------------------------------------------------------------------
1 | version 1.0
2 |
3 | # TASK
4 | # SHARE-atac-STAR
5 |
6 | task share_rna_align {
7 | meta {
8 | version: 'v0.1'
9 | author: 'Eugenio Mattei (emattei@broadinstitute.org) at Broad Institute of MIT and Harvard'
10 | description: 'Broad Institute of MIT and Harvard SHARE-Seq pipeline: align RNA task'
11 | }
12 |
13 | input {
14 | # This function takes in input the pre-processed fastq and align it to the genome
15 | # using STAR.
16 |
17 | Array[File] fastq_R1
18 | Array[File]? fastq_R2
19 | File? genome_index_tar
20 | String genome_name
21 | String? prefix
22 | String docker_image = "docker.io/nchernia/share_task_star:1"
23 | Int cpus = 16
24 | }
25 | #Float input_file_size_gb = size(input[0], "G")
26 | Int samtools_cpus = 6
27 | Int samtools_mem_gb = 8
28 | Int mem_gb = 64
29 | Int disk_gb = 850
30 | #Int disk_gb = round(20.0 + 4 * input_file_size_gb)
31 |
32 | # Define the output names
33 | String sorted_bam = "${default="share-seq" prefix}.rna.align.${genome_name}.sorted.bam"
34 | String sorted_bai = "${default="share-seq" prefix}.rna.align.${genome_name}.sorted.bam.bai"
35 | String alignment_log = "${default="share-seq" prefix}.rna.align.${genome_name}.log"
36 |
37 | command {
38 | set -e
39 | # Untar the genome
40 | tar xvzf ${genome_index_tar} --no-same-owner -C ./
41 |
42 | mkdir out
43 |
44 | $(which STAR) \
45 | --runThreadN ${cpus} \
46 | --chimOutType WithinBAM \
47 | --genomeDir ./ \
48 | --readFilesIn ${sep=',' fastq_R1} ${sep=',' fastq_R2} \
49 | --outFileNamePrefix out/${default="share-seq" prefix}.rna.align.${genome_name}. \
50 | --outFilterMultimapNmax 20 \
51 | --outFilterScoreMinOverLread 0.3 \
52 | --outFilterMatchNminOverLread 0.3 \
53 | --outSAMattributes NH HI AS nM MD \
54 | --limitOutSJcollapsed 2000000 \
55 | --outSAMtype BAM Unsorted \
56 | --limitIObufferSize 400000000 400000000 \
57 | --outReadsUnmapped Fastx \
58 | --readFilesCommand zcat
59 |
60 | $(which samtools) sort \
61 | -@ ${samtools_cpus} \
62 | -m ${samtools_mem_gb}G \
63 | -o out/${sorted_bam} \
64 | out/${default="share-seq" prefix}.rna.align.${genome_name}.Aligned.out.bam
65 |
66 | $(which samtools) index \
67 | -@ ${cpus} \
68 | out/${sorted_bam}
69 | }
70 |
71 | output {
72 | File rna_alignment = "out/${sorted_bam}"
73 | File rna_alignment_index = "out/${sorted_bai}"
74 | File rna_alignment_log = glob('out/*.Log.final.out')[0]
75 | }
76 |
77 | runtime {
78 | cpu : cpus
79 | memory : mem_gb+'G'
80 | disks : 'local-disk ${disk_gb} SSD'
81 | maxRetries: 0
82 | docker: docker_image
83 | }
84 |
85 | parameter_meta {
86 | fastq_R1: {
87 | description: 'Read1 fastq',
88 | help: 'Processed fastq for read1.',
89 | example: 'processed.atac.R1.fq.gz'
90 | }
91 | genome_index_tar: {
92 | description: 'STAR indexes',
93 | help: 'Index files for STAR to use during alignment in tar.gz.',
94 | example: ['']
95 | }
96 | genome_name: {
97 | description: 'Reference name',
98 | help: 'The name of the reference genome used by the aligner.',
99 | example: ['hg38', 'mm10', 'both']
100 | }
101 | prefix: {
102 | description: 'Prefix for output files',
103 | help: 'Prefix that will be used to name the output files',
104 | example: 'MyExperiment'
105 | }
106 | cpus: {
107 | description: 'Number of cpus',
108 | help: 'Set the number of cpus useb by bowtie2',
109 | example: '4'
110 | }
111 | docker_image: {
112 | description: 'Docker image.',
113 | help: 'Docker image for preprocessing step. Dependencies: STAR',
114 | example: ['put link to gcr or dockerhub']
115 | }
116 | }
117 | }
118 |
--------------------------------------------------------------------------------
/src/python/qc_atac_compute_reads_in_peaks.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 |
3 | # Author: Eugenio Mattei, Broad Institute of MIT and Harvard
4 | # modified from Jason Buenrostro's tool
5 |
6 | import argparse
7 | import os
8 | import pysam
9 | from collections import Counter
10 | from collections import defaultdict
11 | import numpy as np
12 |
13 | #import os
14 | #import sys
15 | import matplotlib
16 | matplotlib.use('Agg')
17 | import matplotlib.pyplot as plt
18 | #from multiprocessing import Pool
19 |
20 |
21 | ##### DEFINE FUNCTIONS #####
22 | def count_fragments_in_peaks(tabix_filename,
23 | peaks_list,
24 | mapq_threshold = 30):
25 | """
26 | This funtion counts the per-barcode number of reads in the peak region.
27 |
28 | Parameters
29 | ----------
30 | tabix_filename : str
31 | Path to the tabix file containing the fragments.
32 | File needs to be coordinate-sorted and indexed.
33 | peaks_list : array
34 | Array containing the list of peaks to be included.
35 | Each member of the array contains the following four elements:
36 | Chr, Start, End, Strand
37 | barcode_tag : str
38 | Which tag in the BAM file contains the barcode id.
39 | mapq_threshold : int
40 | Keep only the reads with mapq score greater or equal.
41 | default: 30
42 |
43 | Returns
44 | -------
45 |
46 | Dictionary
47 | Key: Barcode
48 | Value: Number of fragments in peaks.
49 | """
50 | # To count the number of fragments in peaks
51 | reads_in_peaks_counter = defaultdict(set)
52 | fragments_in_peaks_counter = defaultdict(set)
53 |
54 | tabixfile = pysam.TabixFile(tabix_filename)
55 |
56 | for peak in peaks_list:
57 | peak_chr = str(peak[0])
58 | peak_start = int(peak[1])
59 | peak_end = int(peak[2])
60 |
61 | # Find all the fragments overlapping the promoter.
62 | for fragment in tabixfile.fetch(peak_chr, peak_start, peak_end):
63 | fragment_fields = fragment.split("\t")
64 |
65 | fragment_contig = fragment_fields[0]
66 | fragment_start = int(fragment_fields[1])
67 | fragment_end = int(fragment_fields[2])
68 | barcode = fragment_fields[3]
69 |
70 | fragment_id = "-".join(fragment_fields)
71 | fragments_in_peaks_counter[barcode].add(fragment_id)
72 |
73 | # Increment the counter for the specific barcode.
74 | if fragment_start >= peak_start and fragment_start <= peak_end-1:
75 | reads_in_peaks_counter[barcode].add(fragment_id+"start")
76 |
77 | if fragment_end >= peak_start and fragment_end <= peak_end-1:
78 | reads_in_peaks_counter[barcode].add(fragment_id+"end")
79 |
80 | return reads_in_peaks_counter, fragments_in_peaks_counter
81 |
82 |
83 | if __name__ == '__main__':
84 |
85 | #args = _parse_sanitize_cmdline_arguments()
86 |
87 | msg = "Add the description"
88 | parser = argparse.ArgumentParser(description = msg)
89 |
90 | # Adding optional argument
91 | parser.add_argument("tabix", help= "Fragments file in tabix format and indexed.")
92 | parser.add_argument("--prefix", help = "Prefix for the metrics output fil.")
93 | parser.add_argument("--peaks", help= "Peaks bed file")
94 |
95 | # Read arguments from command line
96 | args = parser.parse_args()
97 |
98 | if args.prefix:
99 | prefix = args.prefix
100 | else:
101 | prefix = args.bam[:-4]
102 |
103 | # It is extremely fast. Don't think we need parallel processing.
104 | #cpus = len(os.sched_getaffinity(0))/2
105 | # Using column chr, start, end and what user input contains the strand information.
106 | peaks_list = np.loadtxt(args.peaks, 'str', usecols = (0,1,2))
107 |
108 | reads_in_peaks, fragments_in_peaks = count_fragments_in_peaks(args.tabix,
109 | peaks_list
110 | )
111 | output_fnp = f"{prefix}.reads.in.peak.tsv"
112 |
113 | with open(output_fnp,"w") as out_file:
114 | print(f"barcode\treads_peaks\tfragment_peaks", file=out_file)
115 | for barcode,fragments_in_peak in fragments_in_peaks.items():
116 | print(f"{barcode}\t{len(reads_in_peaks[barcode])}\t{len(fragments_in_peak)}", file=out_file)
117 |
--------------------------------------------------------------------------------
/src/python/infer_barcodes.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 |
3 | # This script is used to infer molecular barcodes
4 | # from raw sequencing BCL data.
5 | #
6 | # It requires running Picard ExtractIlluminaBarcodes with BARCODE=N,
7 | # to extract all barcodes into *_barcode.txt.gz files first.
8 |
9 | import glob
10 | import gzip
11 | import sys
12 |
13 | [
14 | _name,
15 | multiplex_params_file,
16 | candidate_molecular_barcodes_file,
17 | barcode_matches_file,
18 | ] = sys.argv
19 |
20 | YIELD_THRESHOLD = 0.1
21 | MIN_READ_COUNT = 1e6
22 |
23 |
24 | def parse_barcodes(file_path):
25 | with open(file_path) as f:
26 | barcodes = {}
27 | for row in f.readlines():
28 | row = row.strip().split('\t')
29 | copa = row[0]
30 | barcode = ''.join(row[1:])
31 | barcodes[barcode] = copa
32 | return barcodes
33 |
34 |
35 | copa_barcodes = parse_barcodes(multiplex_params_file)
36 | molecular_barcodes = parse_barcodes(candidate_molecular_barcodes_file)
37 |
38 | # count each unique barcode combination
39 | counts = {}
40 | for extracted in glob.glob('*_barcode.txt.gz'):
41 | with gzip.open(extracted, 'rt') as f:
42 | for row in f.readlines():
43 | barcode = row.split('\t')[0]
44 | if barcode in counts:
45 | counts[barcode] += 1
46 | else:
47 | counts[barcode] = 1
48 |
49 | # add any missing barcodes from the list of CoPAs
50 | for barcode in copa_barcodes.keys():
51 | if barcode not in counts:
52 | counts[barcode] = 0
53 |
54 |
55 | def distance(b1, b2):
56 | return sum(c1 != c2 for c1, c2 in zip(b1, b2))
57 |
58 |
59 | COPA_UNDEFINED = 'UNDEFINED'
60 |
61 | # match barcodes to candidates
62 | results = {}
63 | molecular_barcode_len = len(next(iter(molecular_barcodes)))
64 | for barcode, count in counts.items():
65 | molecular_barcode_matched = False
66 | molecular_barcode_match = barcode[:molecular_barcode_len]
67 | molecular_barcode_match_name = molecular_barcode_match
68 |
69 | if molecular_barcode_match in molecular_barcodes:
70 | molecular_barcode_matched = True
71 | molecular_barcode_match_name = molecular_barcodes[molecular_barcode_match]
72 |
73 | barcode_match = molecular_barcode_match
74 | copa = copa_barcodes[barcode_match] if barcode_match in copa_barcodes else COPA_UNDEFINED
75 | if barcode_match in results:
76 | results[barcode_match]['Count'] += count
77 | else:
78 | results[barcode_match] = {
79 | 'CoPA': copa,
80 | 'Molecular Barcode': molecular_barcode_match_name,
81 | 'Count': count,
82 | 'Matched': molecular_barcode_matched
83 | }
84 |
85 | # show barcodes that correspond to a CoPA or have a matched
86 | # barcode at the top of the output file, otherwise
87 | # sort by count
88 | results = sorted(
89 | results.values(),
90 | key=lambda r: (r['CoPA'], int(not r['Matched']), -r['Count'])
91 | )
92 |
93 | # calculate % of average yield
94 | total_yield = 0
95 | copa_count = 0
96 | for r in results:
97 | if r['CoPA'] != COPA_UNDEFINED:
98 | total_yield += r['Count']
99 | copa_count += 1
100 | avg_yield = total_yield / copa_count if copa_count else None
101 | for r in results:
102 | percent_avg_yield = ''
103 | if r['CoPA'] != COPA_UNDEFINED:
104 | percent_avg_yield = '{:.2f}%'.format(
105 | 100 * r['Count'] / avg_yield) if avg_yield else 0
106 | r['Percent of average'] = percent_avg_yield
107 |
108 | # report results as a TSV
109 | with open(barcode_matches_file, 'w') as f:
110 | header = (
111 | 'CoPA', 'Molecular Barcode',
112 | 'Count', 'Percent of average',
113 | )
114 | print('\t'.join(header), file=f)
115 |
116 | # print CoPA matches, barcode matches, and the top barcodes
117 | # with the highest read count
118 | for r in results:
119 | if (
120 | r['CoPA'] != COPA_UNDEFINED or
121 | r['Matched'] or
122 | r['Count'] >= MIN_READ_COUNT
123 | ):
124 | print('\t'.join((str(r[col]) for col in header)), file=f)
125 |
126 | # fail the task (and the workflow) for low yield
127 | if not avg_yield:
128 | raise Exception('None of the candidate barcodes matched any CoPAs!')
129 | failed_copas = []
130 | for r in results:
131 | if (r['CoPA'] != COPA_UNDEFINED and
132 | float(r['Percent of average'].replace('%', '')) < YIELD_THRESHOLD):
133 | failed_copas.append(r['CoPA'])
134 | failed_copas = ', '.join(failed_copas)
135 | if failed_copas:
136 | raise Exception(
137 | f'Found CoPA(s) with < {YIELD_THRESHOLD}% yield: {failed_copas}')
--------------------------------------------------------------------------------
/tasks/share_task_cell_annotation.wdl:
--------------------------------------------------------------------------------
1 | version 1.0
2 |
3 | task cell_annotation {
4 | meta {
5 | version: 'v0.1'
6 | author: 'Zhijian Li'
7 | affiliation: 'Broad Institute of MIT and Harvard'
8 | email: 'lizhijia@broadinstitute.org'
9 | description: 'SHARE-Seq pipeline: cell type annotation using RNA-seq data.'
10 | }
11 |
12 | input {
13 | # Sample or project name
14 | String? prefix = "prefix"
15 |
16 | # Reference genome
17 | String genome
18 |
19 | # Reference data name and id
20 | String reference_data_id
21 | String reference_data_name
22 | String reference_label
23 |
24 | # Query data
25 | File query_data
26 |
27 | String? gene_id_to_symbol
28 |
29 | # Docker image
30 | String? docker_image
31 |
32 | # Runtime parameter
33 | Float? memory_factor
34 | Float? disk_factor
35 | }
36 |
37 | # Determine the size of the input
38 | Float input_file_size_mb = size(query_data, "G")
39 |
40 | # Determining memory size base on the size of the input files.
41 | Float mem_gb = 64.0 + memory_factor * input_file_size_mb
42 |
43 | # Determining disk size base on the size of the input files.
44 | Int disk_gb = round(disk_factor * input_file_size_mb)
45 |
46 | # Determining disk type base on the size of disk.
47 | String disk_type = if disk_gb > 375 then "SSD" else "LOCAL"
48 |
49 | #Output files
50 | String reference_h5ad = "${reference_data_name}.h5ad"
51 | String monitor_log = "cell_annotation_monitor.log"
52 | String notebook_log = "log/${prefix}.cell.annotation.logfile.${genome}.txt"
53 | String prediction = "${prefix}.cell.annotation.prediction.${genome}.csv"
54 | String prediction_labels = "${prefix}.cell.annotation.labels.${genome}.png"
55 | String prediction_scores = "${prefix}.cell.annotation.scores.${genome}.pdf"
56 |
57 | command {
58 | set -e
59 |
60 | bash $(which monitor_script.sh) | tee ~{monitor_log} 1>&2 &
61 |
62 | # Download data from cellxgene
63 | python3 $(which get_cellxgene_data.py) \
64 | --id ${reference_data_id} \
65 | --out ${reference_data_name}
66 |
67 |
68 | # Perform cell annotation
69 | Rscript $(which cell_annotation.R) \
70 | --prefix ${prefix} \
71 | --reference_data_name ${reference_data_name} \
72 | --reference_label ${reference_label} \
73 | --query_data ${query_data} \
74 | --genome ${genome} \
75 | --gene_id_to_symbol ${gene_id_to_symbol}
76 |
77 | }
78 |
79 | output {
80 | File reference_h5ad = "${reference_h5ad}"
81 | File monitor_log = "${monitor_log}"
82 | File notebook_log = "${notebook_log}"
83 | File prediction = '${prediction}'
84 | File prediction_labels = '${prediction_labels}'
85 | File prediction_scores = '${prediction_scores}'
86 | }
87 |
88 | runtime {
89 | memory : "${mem_gb} GB"
90 | memory_retry_multiplier: 2
91 | disks: "local-disk ${disk_gb} ${disk_type}"
92 | docker : "${docker_image}"
93 | maxRetries:1
94 | }
95 |
96 | parameter_meta {
97 | reference_data_id: {
98 | description: 'Reference dataset id',
99 | help: 'The dataset id from cellxgene server.',
100 | examples: ['3bbb6cf9-72b9-41be-b568-656de6eb18b5']
101 | }
102 |
103 | reference_data_name: {
104 | description: 'Reference data',
105 | help: 'This file will be used as reference',
106 | examples: ['reference.h5ad']
107 | }
108 |
109 | query_data: {
110 | description: 'Query data',
111 | help: 'scRNA-seq data used as query',
112 | examples: ['put link to gcr']
113 | }
114 |
115 | genome: {
116 | description: 'Reference name',
117 | help: 'Reference genome.',
118 | examples: ['hg38', 'mm10', 'hg19', 'mm9']
119 | }
120 |
121 | prefix: {
122 | description: 'Project name',
123 | help: 'String used to name your project and associated file names',
124 | example: "shareseq"
125 | }
126 |
127 | docker_image: {
128 | description: 'Docker image.',
129 | help: 'Docker image for preprocessing step.',
130 | example: ['put link to gcr or dockerhub']
131 | }
132 |
133 | disk_factor: {
134 | description: 'Disk factor',
135 | help: 'Multiply this value to input .h5 file size (MB) to determine disk space (GB)',
136 | example: 16.0
137 | }
138 |
139 | memory_factor: {
140 | description: 'Memory factor',
141 | help: 'Multiply this value to input .h5 file size (MB) and add to default 32GB memory to determine RAM (GB)',
142 | example: 1.0
143 | }
144 | }
145 | }
146 |
--------------------------------------------------------------------------------
/src/R/barcode_rank_functions.R:
--------------------------------------------------------------------------------
1 | #!/usr/bin/Rscript
2 |
3 | ## Define functions needed for plotting barcode rank
4 |
5 | # Helper function to get vectors on which to call the elbow_knee_finder.
6 | # Takes in xy values of the curve, outputs appropriate xy vectors to be passed to elbow_knee_finder.
7 | #
8 | # Function computes the second derivative of the curve, and uses the shape of the second
9 | # derivative curve to determine whether the curve has multiple "joints" (i.e. if knee should be found).
10 | # If the second derivative is uniformly positive or uniformly negative, the curve has a single "joint",
11 | # and so elbow_knee_finder can be called on the original input vectors.
12 | # Otherwise (multiple "joints"), find the zeroes of the second derivative to the left and right of the
13 | # absolute minimum of the second derivative.
14 | # These will be the endpoints of the elbow_knee_finder, so return the slices of the xy vectors
15 | # between these zeroes.
16 | get_vectors <- function(x, y){
17 | smooth_spline <- smooth.spline(x, y, spar=1)
18 | second_deriv <- predict(smooth_spline, x, deriv=2)
19 |
20 | # Second derivative values can be noisy at beginning and end of graph; exclude first 10% and last 10%
21 | # of values when establishing uniformity of second derivative sign
22 | ten_percent <- round(length(second_deriv$x)*0.1)
23 | mid_second_deriv <- second_deriv$y[(ten_percent+1):(length(second_deriv$y)-ten_percent)]
24 |
25 | if (all(mid_second_deriv >= 0) | all(mid_second_deriv <= 0)){
26 | print("Returning original vectors")
27 | return(list(x,y)) }
28 | else {
29 | # Find absolute minimum
30 | abs_min_idx <- second_deriv$x[which.min(second_deriv$y)]
31 | # Find last non-negative value before absolute minimum
32 | left_vect <- second_deriv$y[0:abs_min_idx]
33 | endpt_1_idx <- tail(which(left_vect >= 0), n=1)
34 | # Find first non-positive value after absolute minimum
35 | right_vect <- second_deriv$y[abs_min_idx:length(second_deriv$y)]
36 | endpt_2_idx <- abs_min_idx + which(right_vect >= 0)[1] - 1
37 |
38 | # Error cases: revert to elbow finder
39 | # Used when second derivative curve has both positive and negative values,
40 | # but doesn't match positive-negative-positive shape expected of a knee's second derivative
41 | if (length(endpt_1_idx)==0 | length(endpt_2_idx)==0){
42 | print("Returning original vectors")
43 | return(list(x,y))
44 | } else if (is.na(endpt_1_idx) | is.na(endpt_2_idx)){
45 | print("Returning original vectors")
46 | return(list(x,y))
47 | } else {
48 | print("Returning sliced vectors")
49 | return(list(x[endpt_1_idx:endpt_2_idx], y[endpt_1_idx:endpt_2_idx]))
50 | }
51 | }
52 | }
53 |
54 | # Function to find the elbow or knee of a plot.
55 | # Takes in set of xy coordinates of the plot and mode, returns point which is farthest
56 | # from the line formed by the endpoints.
57 | # Basic mode (default) is used when the plot is known to have only one "joint",
58 | # whereas advanced mode is used when it is not known whether the function needs to find an
59 | # elbow or a knee.
60 | elbow_knee_finder <- function(x, y, mode="basic") {
61 | # With advanced mode, use helper function to determine which vectors to perform calculation on
62 | if (mode == "advanced") {
63 | # smooth.spline() function used in get_vectors() requires at least 4 unique
64 | # x values; preempt this error
65 | if (length(unique(x)) < 4) {
66 | return(NULL)
67 | } else {
68 | xy_vects <- get_vectors(x, y)
69 | x <- xy_vects[[1]]
70 | y <- xy_vects[[2]]
71 | }
72 | }
73 | # Error case: return null if vectors have length 0
74 | if (length(x)==0 | length(y)==0) {
75 | return(NULL)
76 | }
77 | # Get endpoints (point with smallest x value, point with largest x value)
78 | endpts_df <- data.frame(x_coords=c(x[1], x[length(x)]),
79 | y_coords=c(y[1], y[length(y)]))
80 | # Fit line between endpoints
81 | fit <- lm(endpts_df$y_coords ~ endpts_df$x_coords)
82 | # For each point, get distance from line
83 | distances <- numeric(length(x))
84 | for(i in 1:length(x)) {
85 | distances[i] <- abs(coef(fit)[2]*x[i] - y[i] + coef(fit)[1]) / sqrt(coef(fit)[2]^2 + 1^2)
86 | }
87 |
88 | # Get point farthest from line
89 | x_max_dist <- x[which.max(distances)]
90 | y_max_dist <- y[which.max(distances)]
91 |
92 | return(c(x_max_dist, y_max_dist))
93 | }
94 |
95 | # Function to find the elbow/knee of a plot, and the elbow/knee of the points
96 | # before the first elbow/knee (i.e. elbow/knee of all barcodes, and elbow/knee
97 | # of top-ranked barcodes).
98 | # Takes in xy coordinates of the plot and returns vector of four coordinates:
99 | # xy coordinates of first elbow/knee, and xy coordinates of second elbow/knee.
100 | get_elbow_knee_points <- function(x, y) {
101 | point_1 <- elbow_knee_finder(x, y, mode="basic")
102 | if (!is.null(point_1)) {
103 | point_2 <- elbow_knee_finder(x[1:point_1[1]], y[1:point_1[1]], mode="advanced")
104 | }
105 | return(c(point_1, point_2))
106 | }
107 |
--------------------------------------------------------------------------------
/src/python/generate_h5_rna.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | # coding=utf8
3 |
4 | """
5 | This script takes in the STARsolo barcodes tsv file, features tsv file,
6 | and raw count matrix mtx file, and generates an h5 file containing the
7 | genes x barcodes count matrix.
8 | """
9 |
10 | import argparse
11 | from collections import defaultdict
12 | import gzip
13 | import h5py
14 | import logging
15 | from scipy.sparse import csc_matrix
16 |
17 | def parse_arguments():
18 | parser = argparse.ArgumentParser(description="Generate an h5 count matrix of genes x barcodes")
19 | parser.add_argument("matrix_file", help="Filename for STARsolo raw matrix mtx file")
20 | parser.add_argument("features_file", help="Filename for STARsolo features tsv file")
21 | parser.add_argument("barcodes_file", help="Filename for STARsolo barcodes tsv file")
22 | parser.add_argument("output_file", help="Filename for output h5 file")
23 | parser.add_argument("pkr", help="Experiment prefix", nargs = '?')
24 | parser.add_argument("--ensembl", help="Flag for outputting genes using ENSEMBL ID, rather than gene name", action="store_true")
25 |
26 | return parser.parse_args()
27 |
28 | def get_split_lines(file_name, delimiter, skip=0):
29 | """Read file contents and yield generator with line entries"""
30 | opener = gzip.open if file_name.endswith('.gz') else open
31 |
32 | with opener(file_name, "rt") as f:
33 | for i in range(skip):
34 | next(f)
35 | for line in f:
36 | yield line.rstrip().split(sep=delimiter)
37 |
38 | def rename_duplicates(duplicate_list):
39 | """Rename duplicate entries as entry, entry.1, entry.2, etc."""
40 | seen = defaultdict(int)
41 | renamed_list = []
42 |
43 | for entry in duplicate_list:
44 | renamed_list.append(f"{entry}.{seen[entry]}" if entry in seen else entry)
45 | seen[entry] += 1
46 |
47 | return renamed_list
48 |
49 | def build_count_matrix(matrix):
50 | """Convert contents of mtx file to csc matrix"""
51 | # first line of matrix contains dimensions
52 | dimensions = next(matrix)
53 | n_rows = int(dimensions[0])
54 | n_cols = int(dimensions[1])
55 |
56 | gene_indices = []
57 | barcode_indices = []
58 | counts = []
59 |
60 | for line in matrix:
61 | # subtract 1 from indices to convert to zero-based indexing
62 | gene_indices.append(int(line[0])-1)
63 | barcode_indices.append(int(line[1])-1)
64 | counts.append(int(line[2]))
65 |
66 | count_matrix = csc_matrix((counts, (gene_indices,barcode_indices)), shape=(n_rows,n_cols))
67 |
68 | return count_matrix
69 |
70 | def write_h5(output_file, count_matrix, barcode_list, gene_list):
71 | h5_file = h5py.File(output_file, "w")
72 |
73 | # create datasets expected for Seurat import
74 | g = h5_file.create_group("group")
75 | g.create_dataset("barcodes", data=barcode_list)
76 | g.create_dataset("data", data=count_matrix.data)
77 | g.create_dataset("gene_names", data=gene_list)
78 | g.create_dataset("genes", data=gene_list)
79 | g.create_dataset("indices", data=count_matrix.indices)
80 | g.create_dataset("indptr", data=count_matrix.indptr)
81 | g.create_dataset("shape", data=count_matrix.shape)
82 |
83 | h5_file.close()
84 |
85 | def main():
86 | # create log file
87 | logging.basicConfig(filename="generate_h5_rna.log", level=logging.INFO)
88 |
89 | # get arguments
90 | args = parse_arguments()
91 | matrix_file = getattr(args, "matrix_file")
92 | features_file = getattr(args, "features_file")
93 | barcodes_file = getattr(args, "barcodes_file")
94 | pkr = getattr(args, "pkr", None)
95 | output_file = getattr(args, "output_file")
96 | ensembl = getattr(args, "ensembl")
97 |
98 | # read input files
99 | logging.info("Reading input files\n")
100 |
101 | # get indices and counts from matrix file; skip first two lines of matrix file (header)
102 | matrix = get_split_lines(matrix_file, delimiter=" ", skip=2)
103 |
104 | # get genes from features file
105 | features = get_split_lines(features_file, delimiter="\t")
106 | if ensembl:
107 | gene_list = [line[0] for line in features]
108 | else:
109 | gene_list_duplicated = [line[1] for line in features]
110 | # append .1, .2, etc. for duplicated genes
111 | gene_list = rename_duplicates(gene_list_duplicated)
112 |
113 | # get barcodes from barcodes file, reformat as R1R2R3_PKR
114 | barcodes = get_split_lines(barcodes_file, delimiter="\t")
115 | barcode_list = [line[0] for line in barcodes]
116 | if pkr is None:
117 | formatted_barcode_list = barcode_list
118 | else:
119 | formatted_barcode_list = [barcode + "_" + pkr for barcode in barcode_list]
120 |
121 | # generate count matrix
122 | logging.info("Generating count matrix\n")
123 | count_matrix = build_count_matrix(matrix)
124 |
125 | # write h5 file
126 | logging.info(f"Writing to {output_file}.h5\n")
127 | write_h5(output_file, count_matrix, formatted_barcode_list, gene_list)
128 | logging.info("Finished writing h5 file\n")
129 |
130 | if __name__ == "__main__":
131 | main()
132 |
--------------------------------------------------------------------------------
/src/python/rna_barcode_metadata.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 |
3 | """
4 | This script takes in a bam file, and outputs a txt file containing the number of
5 | total reads, duplicate reads, UMIs, genes, and percent mitochondrial reads for each barcode.
6 | """
7 |
8 | import argparse
9 | import logging
10 | import pysam
11 | from collections import defaultdict
12 |
13 | logging.basicConfig(filename='barcode_metadata.log', encoding='utf-8', level=logging.DEBUG)
14 | logging.debug('Creating the barcode metadata for RNA from bam.')
15 |
16 | def parse_arguments():
17 | parser = argparse.ArgumentParser(description="Get total reads, duplicate reads, UMIs, genes, and percent mitochondrial reads for each barcode from bam file")
18 | parser.add_argument("bam_file", help="Filename for input bam file")
19 | parser.add_argument("bai_file", help="Filename for bam index file")
20 | parser.add_argument("barcode_metadata_file", help="Filename for output barcode metadata txt file")
21 | parser.add_argument("pkr", help="PKR id for shareseq", default = None, nargs='?')
22 | parser.add_argument("--barcode_tag", help="PKR id for shareseq", default="CB")
23 |
24 | return parser.parse_args()
25 |
26 | def get_metrics(bam, barcode_tag="CB", pkr=None):
27 | """
28 | Get barcode metrics from bam file; all counts are only for reads overlapping genes.
29 | Reported metrics are total counts, UMIs (one UMI counted per unique UMI-gene mapping),
30 | duplicate counts, genes, percent mitochondrial reads
31 | """
32 | total_counts = defaultdict(int)
33 | genes = defaultdict(set)
34 | umi_gene = defaultdict(set)
35 | mitochondrial_counts = defaultdict(int)
36 | barcodes = set()
37 | formatted_barcodes = {}
38 |
39 | for read in bam:
40 | try:
41 | # get barcode; skip read if not present
42 | barcode = read.get_tag(barcode_tag)
43 | if barcode == "-":
44 | #logging.warning(f"Skipping {read.qname} because the {barcode_tag} tag is empty") slowing down
45 | continue
46 |
47 | # get gene id; skip read if not present
48 | gene_id = read.get_tag("GX")
49 | if gene_id == "-":
50 | #logging.warning(f"Skipping {read.qname} because the GX tag is empty")
51 | continue
52 |
53 | # get UMI; skip read if not present
54 | umi = read.get_tag("UB")
55 | if umi == "-":
56 | #logging.warning(f"Skipping {read.qname} because the UB tag is empty")
57 | continue
58 |
59 | barcodes.add(barcode)
60 |
61 | total_counts[barcode] += 1
62 |
63 | genes[barcode].add(gene_id)
64 |
65 | umi_gene[barcode].add(umi + gene_id)
66 |
67 | if read.reference_name == "chrM":
68 | mitochondrial_counts[barcode] += 1
69 | except KeyError:
70 | logging.error(f"Skipping {read.qname} because one of the tags {barcode_tag},GX, or UB is missing.")
71 |
72 | # count unique genes per barcode
73 | genes_per_barcode = {barcode:len(gene_set) for (barcode, gene_set) in genes.items()}
74 |
75 | # count unique umi-gene mappings per barcode
76 | umis_per_barcode = {barcode:len(umi_gene_set) for (barcode, umi_gene_set) in umi_gene.items()}
77 |
78 | # create list with barcodes and associated metrics
79 | barcode_metadata = []
80 | for barcode in barcodes:
81 | total_val = str(total_counts[barcode])
82 | umi_val = str(umis_per_barcode.get(barcode, 0))
83 | duplicate_val = str(total_counts[barcode] - umis_per_barcode.get(barcode, 0))
84 | gene_val = str(genes_per_barcode.get(barcode, 0))
85 | mitochondrial_val = str(round(mitochondrial_counts.get(barcode, 0) / total_counts[barcode] * 100, 2))
86 | out_barcode = barcode + "_" + pkr if pkr else barcode
87 |
88 | metrics = [out_barcode, total_val, duplicate_val, umi_val, gene_val, mitochondrial_val]
89 |
90 | barcode_metadata.append(metrics)
91 |
92 | return barcode_metadata
93 |
94 | def write_metadata_file(barcode_metadata, output_file):
95 | fields = ["barcode", "total_counts", "duplicate_counts", "umis", "genes", "percent_mitochondrial"]
96 |
97 | with open(output_file, "w") as f:
98 | # write header
99 | f.write("\t".join(fields) + "\n")
100 | # write rows
101 | for metrics_list in barcode_metadata:
102 | f.write("\t".join(metrics_list[:]) + "\n")
103 |
104 | def main():
105 | # get arguments
106 | args = parse_arguments()
107 | bam_file = getattr(args, "bam_file")
108 | bai_file = getattr(args, "bai_file")
109 |
110 | pkr = getattr(args, "pkr")
111 | barcode_tag = getattr(args, "barcode_tag")
112 |
113 | barcode_metadata_file = getattr(args, "barcode_metadata_file")
114 |
115 | # load bam file
116 | bam = pysam.AlignmentFile(bam_file, "rb", index_filename=bai_file)
117 |
118 | # get metrics for each barcode
119 | barcode_metadata = get_metrics(bam, barcode_tag, pkr)
120 |
121 | # write txt file
122 | write_metadata_file(barcode_metadata, barcode_metadata_file)
123 |
124 | if __name__ == "__main__":
125 |
126 | main()
127 |
--------------------------------------------------------------------------------
/src/bash/monitor_script.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | declare -a TEMP=$(mktemp temp_monitoring.XXXXXXXX)
4 |
5 | if [[ -z "${BACKEND}" ]]; then
6 | backend=""
7 | else
8 | backend=${BACKEND}
9 | fi
10 |
11 | function get_disk_info() {
12 | # df command and cromwell root field
13 | if [ "$backend" = "aws" ]; then
14 | df | grep '/$'
15 | else
16 | df | grep cromwell_root
17 | fi
18 | }
19 |
20 | function get_disk_usage() {
21 | # get disk usage field
22 | get_disk_info | awk '{ print $5 }'
23 | }
24 |
25 | function get_mem_info() {
26 | # /proc/meminfo
27 | cat /proc/meminfo
28 | }
29 |
30 | function get_mem_available() {
31 | # mem unused from /proc/meminfo
32 | get_mem_info | grep MemAvailable | awk 'BEGIN { FS=" " } ; { print $2 }'
33 | }
34 |
35 | function get_mem_total() {
36 | # mem total from /proc/meminfo
37 | get_mem_info | grep MemTotal | awk 'BEGIN { FS=" " } ; { print $2 }'
38 | }
39 |
40 | function get_mem_usage() {
41 | # memTotal and memAvailable
42 | local -r mem_total=$(get_mem_total)
43 | local -r mem_available=$(get_mem_available)
44 |
45 | # usage = 100 * mem_used / mem_total
46 | local -r mem_used=$(($mem_total-$mem_available))
47 | echo "$mem_used" "$mem_total" "%"| awk '{ print 100*($1/$2)$3 }'
48 | }
49 |
50 | function get_cpu_info() {
51 | # cpu info from /proc/stat
52 | cat /proc/stat | grep "cpu "
53 | }
54 |
55 | function get_cpu_total() {
56 | # get the total cpu usage since a given time (including idle and iowait)
57 | # user+nice+system+idle+iowait+irq+softirq+steal
58 | get_cpu_info | awk 'BEGIN { FS=" " } ; { print $2+$3+$4+$5+$6+$7+$8+$9 }'
59 | }
60 |
61 | function get_cpu_used() {
62 | # get the cpu usage since a given time (w/o idle or iowait)
63 | # user+nice+system+irq+softirq+steal
64 | get_cpu_info | awk 'BEGIN { FS=" " } ; { print $2+$3+$4+$7+$8+$9 }'
65 | }
66 |
67 | function get_cpu_usage() {
68 | # get the cpu usage since a given time (w/o idle or iowait)
69 | # user+nice+system+irq+softirq+steal
70 | local -r cpu_used_cur=$(get_cpu_used)
71 |
72 | # get the total cpu usage since a given time (including idle and iowait)
73 | # user+nice+system+idle+iowait+irq+softirq+steal
74 | local -r cpu_total_cur=$(get_cpu_total)
75 |
76 | # read in previous cpu usage values
77 | read -r -a cpu_prev < ${TEMP}
78 | local -r cpu_used_prev=${cpu_prev[0]}
79 | local -r cpu_total_prev=${cpu_prev[1]}
80 |
81 | # save current values as prev values for next iteration
82 | cpu_prev[0]=$cpu_used_cur
83 | cpu_prev[1]=$cpu_total_cur
84 | echo "${cpu_prev[@]}" > ${TEMP}
85 |
86 | # usage = 100 * (cpu_used_cur - cpu_used_prev) / (cpu_total_cur-cpu_total_prev)
87 | echo "$cpu_used_cur" "$cpu_used_prev" "$cpu_total_cur" "$cpu_total_prev" "%"| awk 'BEGIN {FS=" "} ; { print 100*(($1-$2)/($3-$4))$5 }'
88 |
89 | }
90 |
91 | function print_usage() {
92 | echo [$(date)]
93 | echo \* CPU usage: "$(get_cpu_usage)"
94 | echo \* Memory usage: "$(get_mem_usage)"
95 | echo \* Disk usage: $(get_disk_usage)
96 | }
97 |
98 | function print_summary() {
99 | # display header information
100 | echo ==================================
101 | echo =========== MONITORING ===========
102 | echo ==================================
103 |
104 | # summary info
105 | echo --- General Information ---
106 | # number of cores
107 | echo \#CPU: $(nproc)
108 | # multiply by 10^-6 to convert KB to GB
109 | echo Total Memory: $(echo $(get_mem_total) 1000000 | awk '{ print $1/$2 }')G
110 |
111 | if [ "$backend" = "aws" ]; then
112 | echo Total Disk space: $(df -h | grep '/$' | awk '{ print $2 }')
113 | else
114 | echo Total Disk space: $(df -h | grep cromwell_root | awk '{ print $2}')
115 | fi
116 | }
117 |
118 | function main() {
119 | # disk, mem and cpu general statisitcs
120 | print_summary
121 |
122 | # create variable to store cpu being used (cpu_prev[0]) and total cpu total (cpu_prev[1])
123 | # save variable to a temp file to allow passing in values to a function
124 | declare -a cpu_prev
125 | cpu_prev[0]=$(get_cpu_used)
126 | cpu_prev[1]=$(get_cpu_total)
127 | # save global values to temp file to allow passing in values to a function
128 | echo "${cpu_prev[@]}" > ${TEMP}
129 |
130 | # sleep b/w getting usage and intially storing the cpu_previous usage values
131 | # this is b/c cpu usage values are time dependent
132 | # to calculate cpu usage, values must be determined from 2 diff time stamps
133 | if [ -z "$MONITOR_SCRIPT_SLEEP" ]; then
134 | MONITOR_SCRIPT_SLEEP=30
135 | fi
136 | # get usage of disk, cpu and mem every MONITOR_SCRIPT_SLEEP sec
137 | echo
138 | echo --- Runtime Information ---
139 |
140 | sleep "$MONITOR_SCRIPT_SLEEP";
141 | while true; do print_usage; sleep "$MONITOR_SCRIPT_SLEEP"; done
142 | }
143 |
144 | main
145 |
--------------------------------------------------------------------------------
/tasks/share_task_qc_rna.wdl:
--------------------------------------------------------------------------------
1 | version 1.0
2 |
3 | # TASK
4 | # SHARE-qc-rna
5 |
6 | task qc_rna {
7 | meta {
8 | version: 'v0.1'
9 | author: 'Mei Knudson (mknudson@broadinstitute.org) at Broad Institute of MIT and Harvard'
10 | description: 'Broad Institute of MIT and Harvard SHARE-Seq pipeline: QC RNA task'
11 | }
12 |
13 | input {
14 | # This function takes in input the sorted bam file produced by STARsolo
15 | File bam
16 | Int? umi_cutoff = 100
17 | Int? gene_cutoff = 100
18 | String genome_name
19 | String? barcode_tag = "CB"
20 | String? pkr
21 | String? prefix
22 |
23 | Int? cpus = 16
24 | Float? disk_factor = 1.0
25 | Float? memory_factor = 1.5
26 | String docker_image = "us.gcr.io/buenrostro-share-seq/share_task_qc_rna:v1.0.0"
27 | }
28 |
29 | # Determine the size of the input
30 | Float input_file_size_gb = size(bam, "G")
31 |
32 | # Determining memory size based on the size of the input files.
33 | Float mem_gb = 5.0 + memory_factor * input_file_size_gb
34 |
35 | # Determining disk size based on the size of the input files.
36 | Int disk_gb = round(40.0 + disk_factor * input_file_size_gb)
37 |
38 | # Determining disk type based on the size of disk.
39 | String disk_type = if disk_gb > 375 then "SSD" else "LOCAL"
40 |
41 | String assay = "RNA"
42 | String bai = "~{default="share-seq" prefix}.qc.rna.~{genome_name}.bam.bai"
43 | String barcode_metadata = "~{default="share-seq" prefix}.qc.rna.~{genome_name}.barcode.metadata.tsv"
44 | String duplicates_log = "~{default="share-seq" prefix}.qc.rna.~{genome_name}.duplicates.log.txt"
45 | String umi_barcode_rank_plot = "~{default="share-seq" prefix}.qc.rna.~{genome_name}.umi.barcode.rank.plot.png"
46 | String gene_barcode_rank_plot = "~{default="share-seq" prefix}.qc.rna.~{genome_name}.gene.barcode.rank.plot.png"
47 | String gene_umi_scatter_plot = "~{default="share-seq" prefix}.qc.rna.~{genome_name}.gene.umi.scatter.plot.png"
48 | String monitor_log = "monitor.log"
49 |
50 | command <<<
51 | set -e
52 |
53 | bash $(which monitor_script.sh) | tee ~{monitor_log} 1>&2 &
54 |
55 | # Index bam file
56 | samtools index -@ ~{cpus} ~{bam} ~{bai}
57 |
58 | # Extract barcode metadata (total counts, unique counts, duplicate counts, genes, percent mitochondrial) from bam file
59 | python3 $(which rna_barcode_metadata.py) ~{bam} \
60 | ~{bai} \
61 | ~{barcode_metadata} \
62 | ~{pkr} ~{"--barcode_tag " + barcode_tag}
63 |
64 | awk '{total+=$2; duplicate+=$3; unique+=$4} END {print "total reads:", total; print "unique reads:", unique; print "duplicate reads:", duplicate}' ~{barcode_metadata} > ~{duplicates_log}
65 |
66 | # Make QC plots
67 | Rscript $(which rna_qc_plots.R) ~{barcode_metadata} ~{umi_cutoff} ~{gene_cutoff} ~{umi_barcode_rank_plot} ~{gene_barcode_rank_plot} ~{gene_umi_scatter_plot}
68 | >>>
69 |
70 | output {
71 | File rna_barcode_metadata = "~{barcode_metadata}"
72 | File rna_duplicates_log = "~{duplicates_log}"
73 | File rna_barcode_metadata_log = "barcode_metadata.log"
74 | File? rna_umi_barcode_rank_plot = "~{umi_barcode_rank_plot}"
75 | File? rna_gene_barcode_rank_plot = "~{gene_barcode_rank_plot}"
76 | File? rna_gene_umi_scatter_plot = "~{gene_umi_scatter_plot}"
77 | }
78 |
79 | runtime {
80 | cpu : cpus
81 | memory : "~{mem_gb} GB"
82 | disks: "local-disk ~{disk_gb} ~{disk_type}"
83 | docker : "${docker_image}"
84 | }
85 |
86 | parameter_meta {
87 | bam: {
88 | description: 'Alignment bam file',
89 | help: 'Aligned reads in bam format.',
90 | example: 'hg38.aligned.bam'
91 | }
92 | umi_cutoff: {
93 | description: 'UMI cutoff',
94 | help: 'Cutoff for number of UMIs required when making UMI barcode rank plot.',
95 | example: 10
96 | }
97 | gene_cutoff: {
98 | description: 'Gene cutoff',
99 | help: 'Cutoff for number of genes required when making gene barcode rank plot.',
100 | example: 10
101 | }
102 | pkr: {
103 | description: 'Experiment pkr',
104 | help: 'Id of the sample pkr (share-seq specific).',
105 | examples: ['SS-PKR-000']
106 | }
107 | genome_name: {
108 | description: 'Reference name',
109 | help: 'The name genome reference used to align.',
110 | example: ['hg38', 'mm10', 'hg19', 'mm9']
111 | }
112 | prefix: {
113 | description: 'Prefix for output files',
114 | help: 'Prefix that will be used to name the output files',
115 | example: 'MyExperiment'
116 | }
117 | docker_image: {
118 | description: 'Docker image.',
119 | help: 'Docker image for preprocessing step. Dependencies: samtools',
120 | example: ['put link to gcr or dockerhub']
121 | }
122 | }
123 | }
124 |
--------------------------------------------------------------------------------
/tasks/share_task_joint_qc.wdl:
--------------------------------------------------------------------------------
1 | version 1.0
2 |
3 | # TASK
4 | # SHARE-joint-qc-plotting
5 |
6 |
7 | task joint_qc_plotting {
8 | meta {
9 | version: 'v0.1'
10 | author: 'Mei Knudson (mknudson@broadinstitute.org) at Broad Institute of MIT and Harvard'
11 | description: 'Broad Institute of MIT and Harvard SHARE-Seq pipeline: Joint QC plot'
12 | }
13 |
14 | input {
15 | # This task generates a plot of barcodes QC'd jointly by RNA and ATAC metrics, as well as a
16 | # density plot of all barcodes passing at least one filter.
17 | File? atac_barcode_metadata
18 | File? rna_barcode_metadata
19 | Int remove_low_yielding_cells = 10
20 | Int min_umis = 100
21 | Int min_genes = 200
22 | Int min_tss = 4
23 | Int min_frags = 100
24 |
25 | Float? disk_factor = 8.0
26 | Float? memory_factor = 2.0
27 |
28 | String? prefix
29 | String genome_name
30 |
31 | String docker_image = "us.gcr.io/buenrostro-share-seq/share_task_joint_qc:v1.0.0"
32 | }
33 |
34 | # Determine the size of the input
35 | Float input_file_size_gb = size(atac_barcode_metadata, "G") + size(rna_barcode_metadata, "G")
36 |
37 | # Determine memory size based on the size of the input files
38 | Float mem_gb = 5.0 + memory_factor * input_file_size_gb
39 |
40 | # Determine disk size based on the size of the input files
41 | Int disk_gb = round(40.0 + disk_factor * input_file_size_gb)
42 |
43 | # Determining disk type base on the size of disk.
44 | String disk_type = if disk_gb > 375 then "SSD" else "LOCAL"
45 |
46 | String joint_qc_plot = '${default="share-seq" prefix}.${genome_name}.joint.qc.plot.png'
47 | String joint_density_plot = '${default="share-seq" prefix}.${genome_name}.joint.density.plot.png'
48 | String joint_barcode_metadata = '${default="share-seq" prefix}.joint.barcode.metadata.${genome_name}.csv'
49 |
50 | command {
51 | set -e
52 |
53 | bash $(which monitor_script.sh) > monitoring.log &
54 |
55 | # Make joint qc plot
56 | python3 $(which joint_cell_plotting.py) ${rna_barcode_metadata} ${atac_barcode_metadata} ${remove_low_yielding_cells} ${min_umis} ${min_genes} ${min_tss} ${min_frags} ${joint_qc_plot} ${joint_barcode_metadata} ${default="share-seq" prefix}
57 |
58 | # Make joint density plot
59 | Rscript $(which joint_cell_plotting_density.R) ${default="share-seq" prefix} ${joint_barcode_metadata} ${joint_density_plot}
60 | }
61 |
62 | output {
63 | File joint_calling_monitor = "monitoring.log"
64 | File joint_calling_log = "joint_cell_plotting.log"
65 | File? joint_qc_plot = "${joint_qc_plot}"
66 | File? joint_density_plot = "${joint_density_plot}"
67 | File joint_barcode_metadata = "${joint_barcode_metadata}"
68 | }
69 |
70 | runtime {
71 | memory : "${mem_gb} GB"
72 | disks: "local-disk ${disk_gb} ${disk_type}"
73 | docker : "${docker_image}"
74 | }
75 |
76 | parameter_meta {
77 | atac_barcode_metadata: {
78 | description: 'File containing ATAC barcode metrics.',
79 | help: 'tsv file with ATAC barcode (R1,R2,R3,PKR), fragments, TSS enrichment.',
80 | example: 'qc.atac.barcode.metadata.tsv'
81 | }
82 | rna_barcode_metadata: {
83 | description: 'File containing RNA barcode metrics.',
84 | help: 'tsv file with RNA barcode (R1,R2,R3,PKR), UMIs, genes.',
85 | example: 'qc.rna.barcode.metadata.tsv'
86 | }
87 | remove_low_yielding_cells: {
88 | description: 'UMI and fragments cutoff for plotting.',
89 | help: 'Minimum number of UMIs/fragments required for barcode to be plotted.',
90 | example: 10
91 | }
92 | min_umis: {
93 | description: 'UMI cutoff for RNA QC.',
94 | help: 'Minimum number of UMIs required for barcode to pass RNA QC.',
95 | example: 100
96 | }
97 | min_genes: {
98 | description: 'Gene cutoff for RNA QC.',
99 | help: 'Minimum number of genes required for barcode to pass RNA QC.',
100 | example: 200
101 | }
102 | min_tss: {
103 | description: 'TSS cutoff for ATAC QC.',
104 | help: 'Minimum TSS score required for barcode to pass ATAC QC.',
105 | example: 4
106 | }
107 | min_frags: {
108 | description: 'Fragments cutoff for ATAC QC.',
109 | help: 'Minimum number of fragments required for barcode to pass ATAC QC.',
110 | example: 100
111 | }
112 | prefix: {
113 | description: 'Prefix for output files',
114 | help: 'Prefix that will be used to name the output files',
115 | examples: 'MyExperiment'
116 | }
117 | genome_name: {
118 | description: 'Reference name',
119 | help: 'The name genome reference used to align.',
120 | example: ['hg38', 'mm10', 'hg19', 'mm9']
121 | }
122 | docker_image: {
123 | description: 'Docker image.',
124 | help: 'Docker image for preprocessing step.',
125 | example: ['put link to gcr or dockerhub']
126 | }
127 | }
128 | }
129 |
--------------------------------------------------------------------------------
/tasks/10x_task_preprocess.wdl:
--------------------------------------------------------------------------------
1 | version 1.0
2 |
3 | # TASK
4 | # 10x_task_preprocess
5 |
6 | task preprocess_tenx {
7 | meta {
8 | version: 'v0.1'
9 | author: 'Eugenio Mattei (emattei@broadinstitute.org) at Broad Institute of MIT and Harvard'
10 | description: 'Broad Institute of MIT and Harvard SHARE-Seq pipeline: preprocess 10x ATAC data.'
11 | }
12 |
13 | input {
14 | # This task takes in input the 3 fastqs coming out from cellranger mkfastqs and preprocess them.
15 | File fastq_R1 # Pair 1 reads
16 | File fastq_R3 # Pair 2 reads
17 | File fastq_R2 # Barcode fastq
18 | File? whitelist # Barcode whitelist (chemistry specific)
19 | Int? barcode_dist = 2
20 | Float? threshold_pct_barcode_matching = 0.60
21 | String chemistry
22 | String? prefix
23 | Int? cpus = 16
24 | Float? disk_factor = 8.0
25 | Float? memory_factor = 0.15
26 | String docker_image = "us.gcr.io/buenrostro-share-seq/10x_task_preprocess:v1.0.0"
27 | }
28 |
29 | # Determine the size of the input
30 | Float input_file_size_gb = size(fastq_R1, "G") + size(fastq_R2, "G") + size(fastq_R3, "G")
31 |
32 | # Determining memory size base on the size of the input files.
33 | Float mem_gb = 5.0 + memory_factor * input_file_size_gb
34 |
35 | # Determining disk size base on the size of the input files.
36 | Int disk_gb = round(40.0 + disk_factor * input_file_size_gb)
37 |
38 | # Determining disk type base on the size of disk.
39 | String disk_type = if disk_gb > 375 then "SSD" else "LOCAL"
40 |
41 | # auto-detect barcode complementation outfiles
42 | String barcode_complementation_qc = "${default="10x" prefix}.atac.preprocess.complementation.qc.txt"
43 | String barcode_complementation_out = "${default="10x" prefix}.atac.preprocess.complementation.out.txt"
44 |
45 | # barcode correction and filtering outfiles
46 | String barcode_correction_qc = "${default="10x" prefix}.atac.preprocess.barcode.correction.qc.txt"
47 | String cleaned_fastq_R1 = "${default="10x" prefix}.atac.preprocess.cleaned.R1.fastq.gz"
48 | String cleaned_fastq_R2 = "${default="10x" prefix}.atac.preprocess.cleaned.R2.fastq.gz"
49 |
50 | # read trimming outfiles
51 | String final_fastq_R1 = "${default="10x" prefix}.atac.preprocess.cleaned.trimmed.R1.fastq.gz"
52 | String final_fastq_R2 = "${default="10x" prefix}.atac.preprocess.cleaned.trimmed.R2.fastq.gz"
53 | String trimming_log_json = "${default="10x" prefix}.atac.preprocess.trimming.log.json"
54 | String trimming_log_html = "${default="10x" prefix}.atac.preprocess.trimming.log.html"
55 | String trimming_stats = "${default="10x" prefix}.atac.preprocess.trimming.adapter.stats.txt"
56 |
57 | String barcode_conversion_dict = "barcode_conversion_dict.csv"
58 |
59 | String monitor_log = 'monitor_10x_preprocessing.log.txt'
60 |
61 | command <<<
62 | set -e
63 |
64 | bash $(which monitor_script.sh) | tee ~{monitor_log} 1>&2 &
65 |
66 | # Strip read description
67 | zcat ~{fastq_R1} | sed 's/ .*//' | gzip > stripped_R1.fastq.gz
68 | zcat ~{fastq_R3} | sed 's/ .*//' | gzip > stripped_R2.fastq.gz
69 | zcat ~{fastq_R2} | sed 's/ .*//' | gzip > stripped_barcode.fastq.gz
70 |
71 | if [[ '~{whitelist}' == *.gz ]]; then
72 | gunzip -c ~{whitelist} > whitelist.txt
73 | else
74 | ln -s ~{whitelist} whitelist.txt
75 | fi
76 |
77 | # auto-detect barcode complementation
78 | # python3 barcode_revcomp_detect.py barcode_fastq chemistry whitelist qc_out out threshold
79 |
80 | python3 $(which barcode_revcomp_detect.py) stripped_barcode.fastq.gz ~{chemistry} whitelist.txt ~{barcode_complementation_qc} ~{barcode_complementation_out} ~{threshold_pct_barcode_matching}
81 |
82 | # barcode correction and filtering
83 | # python3 match_barcodes.py
84 |
85 | python3 $(which match_barcodes.py) stripped_R1.fastq.gz stripped_R2.fastq.gz stripped_barcode.fastq.gz ~{chemistry} ~{barcode_dist} ~{barcode_complementation_out} whitelist.txt ~{cleaned_fastq_R1} ~{cleaned_fastq_R2} ~{barcode_correction_qc} ~{cpus}
86 |
87 | # Cleaned old files
88 | rm stripped_R1.fastq.gz stripped_R2.fastq.gz stripped_barcode.fastq.gz
89 | >>>
90 |
91 | output {
92 | File fastq_R1_preprocessed = cleaned_fastq_R1
93 | File fastq_R2_preprocessed = cleaned_fastq_R2
94 | File tenx_barcode_complementation_qc = barcode_complementation_qc
95 | File tenx_barcode_correction_qc = barcode_correction_qc
96 | File? tenx_barcode_conversion_dict = barcode_conversion_dict
97 | #File tenx_trimming_log_json = trimming_log_json
98 | #File trimming_log_html = trimming_log_html
99 | #File tenx_trimming_stats = trimming_stats
100 | }
101 |
102 | runtime {
103 | cpu: cpus
104 | docker: "${docker_image}"
105 | disks: "local-disk ${disk_gb} ${disk_type}"
106 | memory: "${mem_gb} GB"
107 | }
108 |
109 | parameter_meta {
110 | fastq_R1: {
111 | description: 'Pairs 1 fastq',
112 | help: 'Pairs 1 fastq',
113 | }
114 | fastq_R2: {
115 | description: 'Barcode fastq',
116 | help: 'Barcode fastq',
117 | }
118 | fastq_R3: {
119 | description: 'Pairs 2 fastq',
120 | help: 'Pairs 2 fastq',
121 | }
122 | }
123 |
124 | }
125 |
--------------------------------------------------------------------------------
/src/R/rna_qc_plots.R:
--------------------------------------------------------------------------------
1 | #!/usr/bin/Rscript
2 |
3 | ### Takes RNA barcode metadata tsv file, and outputs QC plots as png files.
4 | ### QC plots include barcode rank by number of UMIs (all barcodes and top-ranked barcodes),
5 | ### barcode rank by number of genes (all barcodes and top-ranked barcodes),
6 | ### and genes vs UMIs scatter plot.
7 |
8 | ## Import helper functions
9 | source("/usr/local/bin/barcode_rank_functions.R")
10 |
11 | ## Get arguments, read input
12 | args <- commandArgs()
13 |
14 | barcode_metadata_file <- args[6]
15 | umi_cutoff <- as.integer(args[7])
16 | gene_cutoff <- as.integer(args[8])
17 | umi_rank_plot_file <- args[9]
18 | gene_rank_plot_file <- args[10]
19 | gene_umi_plot_file <- args[11]
20 |
21 | barcode_metadata <- read.table(barcode_metadata_file, header=T)
22 |
23 | ## Get plot inputs
24 |
25 | # Impose UMI cutoff, sort in decreasing order, assign rank
26 | umi_filtered <- barcode_metadata$umis[barcode_metadata$umis >= umi_cutoff]
27 | umi_filtered_sort <- sort(umi_filtered, decreasing=T)
28 | umi_rank <- 1:length(umi_filtered_sort)
29 |
30 | # Find elbow/knee of UMI barcode rank plot and top-ranked UMI barcode rank plot
31 | umi_points <- get_elbow_knee_points(x=umi_rank, y=log10(umi_filtered_sort))
32 | # For each valid plot, make factor for coloring plot points
33 | if (length(umi_points) > 0) { # Elbow found in first plot
34 | umi_plot1 <- TRUE
35 | is_top_ranked_umi <- factor(ifelse(umi_rank <= umi_points[1], 1, 0))
36 | if (length(umi_points) > 2) { # Elbow/knee found in second plot
37 | umi_plot2 <- TRUE
38 | umi_top_rank <- umi_rank[1:umi_points[1]]
39 | umi_top_umi <- umi_filtered_sort[1:umi_points[1]]
40 | is_top_top_ranked_umi <- factor(ifelse(umi_top_rank <= umi_points[3], 1, 0))
41 | } else {
42 | umi_plot2 <- FALSE
43 | }
44 | } else {
45 | umi_plot1 <- FALSE
46 | }
47 |
48 | # Impose gene cutoff, sort in decreasing order, assign rank
49 | gene_filtered <- barcode_metadata$genes[barcode_metadata$genes >= gene_cutoff]
50 | gene_filtered_sort <- sort(gene_filtered, decreasing=T)
51 | gene_rank <- 1:length(gene_filtered_sort)
52 |
53 | # Find elbow/knee of gene barcode rank plot and top-ranked gene barcode rank plot
54 | gene_points <- get_elbow_knee_points(x=gene_rank, y=log10(gene_filtered_sort))
55 | # For each valid plot, make factor for coloring plot points
56 | if (length(gene_points) > 0) { # Elbow found in first plot
57 | gene_plot1 <- TRUE
58 | is_top_ranked_gene <- factor(ifelse(gene_rank <= gene_points[1], 1, 0))
59 | if (length(gene_points) > 2) { # Elbow/knee found in second plot
60 | gene_plot2 <- TRUE
61 | gene_top_rank <- gene_rank[1:gene_points[1]]
62 | gene_top_gene <- gene_filtered_sort[1:gene_points[1]]
63 | is_top_top_ranked_gene <- factor(ifelse(gene_top_rank <= gene_points[3], 1, 0))
64 | } else {
65 | gene_plot2 <- FALSE
66 | }
67 | } else {
68 | gene_plot1 <- FALSE
69 | }
70 |
71 | ## Generate plots
72 |
73 | options(scipen=999)
74 |
75 | # Make UMI barcode rank plots
76 | png(umi_rank_plot_file, width=8, height=8, units='in', res=300)
77 | par(mfrow = c(2,1))
78 |
79 | # Plot 1 (all barcodes passing UMI filter vs log10(UMIs))
80 | if (umi_plot1) {
81 | plot(x=umi_rank,
82 | y=umi_filtered_sort,
83 | log="y",
84 | xlab=paste0(" Barcode rank (", length(umi_rank)-umi_points[1], " low quality cells)"),
85 | ylab="log10(UMIs)",
86 | main="RNA UMIs per Barcode",
87 | col=c("dimgrey","darkblue")[is_top_ranked_umi],
88 | pch=16,
89 | ylim=c(1,100000))
90 | abline(v=umi_points[1], h=10^(umi_points[2]))
91 | text(umi_points[1], 10^(umi_points[2]),
92 | paste0("(", umi_points[1], ", ", 10^(umi_points[2]), ")"),
93 | adj=c(-0.1,-0.5))
94 | }
95 |
96 | # Plot 2 (top ranked barcodes vs log10(UMIs))
97 | if (umi_plot2) {
98 | plot(x=umi_top_rank,
99 | y=umi_top_umi,
100 | log="y",
101 | xlab="Barcode rank",
102 | ylab="log10(UMIs)",
103 | main="RNA UMIs per Top-Ranked Barcode",
104 | col=c("dimgrey","darkblue")[is_top_top_ranked_umi],
105 | pch=16,
106 | ylim=c(1,100000))
107 | abline(v=umi_points[3], h=10^(umi_points[4]))
108 | text(umi_points[3], 10^(umi_points[4]),
109 | paste("(", umi_points[3], ", ", 10^(umi_points[4]), ")", sep=""),
110 | adj=c(-0.1,-0.5))
111 | }
112 | dev.off()
113 |
114 |
115 | # Make gene barcode rank plots
116 | png(gene_rank_plot_file, width=8, height=8, units='in', res=300)
117 | par(mfrow = c(2,1))
118 |
119 | # Plot 1 (all barcodes passing gene filter vs log10(genes))
120 | if (gene_plot1) {
121 | plot(x=gene_rank,
122 | y=gene_filtered_sort,
123 | log="y",
124 | xlab=paste0(" Barcode rank (", length(gene_rank)-gene_points[1], " low quality cells)"),
125 | ylab="log10(genes)",
126 | main="RNA Genes per Barcode",
127 | col=c("dimgrey","darkblue")[is_top_ranked_gene],
128 | pch=16,
129 | ylim=c(1,10000))
130 | abline(v=gene_points[1], h=10^(gene_points[2]))
131 | text(gene_points[1], 10^(gene_points[2]),
132 | paste0("(", gene_points[1], ", ", 10^(gene_points[2]), ")"),
133 | adj=c(-0.1,-0.5))
134 | }
135 |
136 | # Plot 2 (top ranked barcodes vs log10(genes))
137 | if (gene_plot2) {
138 | plot(x=gene_top_rank,
139 | y=gene_top_gene,
140 | log="y",
141 | xlab="Barcode rank",
142 | ylab="log10(genes)",
143 | main="RNA Genes per Top-Ranked Barcode",
144 | col=c("dimgrey","darkblue")[is_top_top_ranked_gene],
145 | pch=16,
146 | ylim=c(1,10000))
147 | abline(v=gene_points[3], h=10^(gene_points[4]))
148 | text(gene_points[3], 10^(gene_points[4]),
149 | paste("(", gene_points[3], ", ", 10^(gene_points[4]), ")", sep=""),
150 | adj=c(-0.1,-0.5))
151 | }
152 | dev.off()
153 |
154 | # Make genes vs UMIs scatter plot
155 | png(gene_umi_plot_file, width=8, height=8, units='in', res=300)
156 |
157 | plot(x=barcode_metadata$umis,
158 | y=barcode_metadata$genes,
159 | xlab="UMIs",
160 | ylab="Genes",
161 | main="RNA Genes vs UMIs",
162 | col="darkblue",
163 | pch=16)
164 |
165 | dev.off()
166 |
--------------------------------------------------------------------------------
/tasks/share_task_merge_bams.wdl:
--------------------------------------------------------------------------------
1 | version 1.0
2 |
3 | # TASK
4 | # SHARE-atac-merge_bams
5 |
6 | task share_atac_merge_bams {
7 | meta {
8 | version: 'v0.1'
9 | author: 'Eugenio Mattei (emattei@broadinstitute.org) at Broad Institute of MIT and Harvard'
10 | description: 'Broad Institute of MIT and Harvard SHARE-Seq pipeline: merge the individual bams together'
11 | }
12 |
13 | input {
14 | # This task takes in input the preprocessed ATAC fastqs and align them to the genome.
15 | Array[File] bams
16 | Array[File] logs
17 | String genome_name
18 | String prefix = "sample-share"
19 | Int? multimappers # = 5
20 | Int? cpus = 16
21 | Float? disk_factor = 8.0
22 | Float? memory_factor = 0.15
23 | String? docker_image = "us.gcr.io/buenrostro-share-seq/share_task_merge_bams:v1.0.0"
24 | }
25 |
26 | # Determine the size of the input
27 | Float input_file_size_gb = size(bams, "G")
28 |
29 | # Determining memory size base on the size of the input files.
30 | Float mem_gb = 16.0 + memory_factor * input_file_size_gb
31 |
32 | # Determining disk size base on the size of the input files.
33 | Int disk_gb = round(20.0 + disk_factor * input_file_size_gb)
34 |
35 | # Determining disk type base on the size of disk.
36 | String disk_type = if disk_gb > 375 then "SSD" else "LOCAL"
37 |
38 | # Determining memory for samtools.
39 | Float samtools_memory_gb = 0.8 * mem_gb # Samtools has overheads so reducing the memory to 80% of the total.
40 |
41 | # Number of threads to beable to use 4GB of memory per thread seems to be the fastest way
42 | Int samtools_threads_ = floor(samtools_memory_gb / 4)
43 | Int samtools_threads = if samtools_threads_ == 0 then 1 else samtools_threads_
44 |
45 | Int sambamba_threads = floor(cpus/2)
46 |
47 | # Now that we know how many threads we can use to assure 4GB of memory per thread
48 | # we assign any remaining memory to the threads.
49 | Int samtools_memory_per_thread_ = floor(samtools_memory_gb * 1024 / samtools_threads) # Computing the memory per thread for samtools in MB.
50 | Int samtools_memory_per_thread = if samtools_memory_per_thread_ < 768 then 768 else samtools_memory_per_thread_
51 |
52 | # Tim parameters
53 | Int machine_mem_mb = 18150
54 | Int cpu = 1
55 | Int compression_level = 5
56 | # default to 500GiB of space
57 | Int disk = 500
58 | Int command_mem_mb = machine_mem_mb - 500
59 |
60 | # Define tmp file name
61 | String unsorted_bam = "${prefix}.atac.merge.${genome_name}.bam"
62 |
63 | # Define the output names
64 | String merged_bam = "${prefix}.atac.merged.k${multimappers}.${genome_name}.sorted.bam"
65 | String merged_bai = "${prefix}.atac.merged.k${multimappers}.${genome_name}.sorted.bam.bai"
66 | String alignment_log = "${prefix}.atac.merged.k${multimappers}.${genome_name}.log"
67 |
68 | String monitor_log = "atac_merge_monitor.log"
69 |
70 | command <<<
71 | set -e
72 |
73 | bash $(which monitor_script.sh) 2>&1 &
74 |
75 | #sambamba merge -t ~{cpus} ~{unsorted_bam} ~{sep=" " bams}
76 |
77 | #sambamba sort -t ~{cpus} -m ~{command_mem_mb}M -o ~{merged_bam} ~{unsorted_bam}
78 |
79 | #sambamba index -t ~{cpus} ~{merged_bam}
80 |
81 | # Trying picard
82 |
83 | java -Dsamjdk.compression_level=~{compression_level} -Xms~{command_mem_mb}m -Xmx~{command_mem_mb}m -jar /usr/local/bin/picard.jar \
84 | MergeSamFiles \
85 | USE_THREADING=true \
86 | SORT_ORDER="coordinate" \
87 | INPUT=~{sep=' INPUT=' bams} \
88 | OUTPUT=~{merged_bam}
89 |
90 | sambamba index -t ~{cpus} ~{merged_bam}
91 |
92 | sed 's/^[[:space:]]*//g' ~{sep=" " logs} | cut -f 1 -d ' ' | awk '{ sum[FNR%15]+=$1 } END {n_total=length(sum);for (idx=1; idx <= n_total; idx++){print sum[idx]}}' > ~{alignment_log}
93 |
94 | >>>
95 |
96 | output {
97 | File atac_merged_alignment = merged_bam
98 | File atac_merged_alignment_index = merged_bai
99 | File atac_merged_alignment_log = alignment_log
100 | }
101 |
102 | runtime {
103 | cpu: cpu
104 | docker: "${docker_image}"
105 | disks: "local-disk ${disk} HDD"
106 | disk: disk + " GB" # TES
107 | #disks: "local-disk ${disk_gb} ${disk_type}"
108 | maxRetries:1
109 | memory: "${machine_mem_mb} MiB"
110 | #memory: "${mem_gb} GB"
111 | memory_retry_multiplier: 2
112 | }
113 |
114 | parameter_meta {
115 | bams: {
116 | description: 'Individuals bams from the scatter alignment task',
117 | help: 'Individuals bams from the scatter alignment task',
118 | example: 'align.raw.L1.bam',
119 | }
120 | cpus: {
121 | description: 'Number of cpus.',
122 | help: 'Set the number of cpus used by bowtie2',
123 | default: 16
124 | }
125 | disk_factor: {
126 | description: 'Multiplication factor to determine disk required for task align.',
127 | help: 'This factor will be multiplied to the size of FASTQs to determine required disk of instance (GCP/AWS) or job (HPCs).',
128 | default: 8.0
129 | }
130 | memory_factor: {
131 | description: 'Multiplication factor to determine memory required for task align.',
132 | help: 'This factor will be multiplied to the size of FASTQs to determine required memory of instance (GCP/AWS) or job (HPCs).',
133 | default: 0.15
134 | }
135 | prefix: {
136 | description: 'Prefix for output files.',
137 | help: 'Prefix that will be used to name the output files',
138 | examples: 'my-experiment'
139 | }
140 | docker_image: {
141 | description: 'Docker image.',
142 | help: 'Docker image for the alignment step.',
143 | example: ["us.gcr.io/buenrostro-share-seq/share_task_bowtie2"]
144 | }
145 | }
146 |
147 |
148 | }
149 |
--------------------------------------------------------------------------------
/src/python/trim_fastq.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | # -*- coding: utf-8 -*-
3 | """
4 | Trim fastq
5 | Removes dovetail (overlap) between R1 and R2
6 | """
7 |
8 | import argparse
9 | import Levenshtein
10 | import xopen
11 | from collections import deque
12 |
13 | def parse_arguments():
14 | parser = argparse.ArgumentParser(description="Trim dovetail (overlap) between read1 and read2")
15 | parser.add_argument("input_read1_fastq_file", help="Filename for untrimmed input read 1 FASTQ file")
16 | parser.add_argument("input_read2_fastq_file", help="Filename for untrimmed input read 2 FASTQ file")
17 | parser.add_argument("output_read1_fastq_file", help="Filename for corrected output read 1 FASTQ file")
18 | parser.add_argument("output_read2_fastq_file", help="Filename for corrected output read 2 FASTQ file")
19 | parser.add_argument("trimming_stats_file", help="Filename for txt file containing trimming statistics")
20 |
21 | return parser.parse_args()
22 |
23 | REV_COMP = str.maketrans("ATGC", "TACG")
24 | def reverse_complement(seq):
25 | return str.translate(seq, REV_COMP)[::-1]
26 |
27 | def trim_fastqs(input_read1_fastq_file, input_read2_fastq_file,
28 | output_read1_fastq_file, output_read2_fastq_file,
29 | trimming_stats_file):
30 | """
31 | Trim reads if overlapping, write reads to output FASTQ files.
32 | Produces file enumerating how many reads were processed and trimmed.
33 | """
34 | # counters
35 | total = trimmed = 0
36 |
37 | read1_out_writer = xopen.xopen(output_read1_fastq_file, mode="w")
38 | read2_out_writer = xopen.xopen(output_read2_fastq_file, mode="w")
39 |
40 | buffer1 = deque()
41 | buffer2 = deque()
42 | buffer_counter = 0
43 |
44 | # process FASTQs together
45 | with xopen.xopen(input_read1_fastq_file, mode= "r", threads= 8) as read1_fh, xopen.xopen(input_read2_fastq_file, mode= "r", threads= 8) as read2_fh:
46 | for readline1, readline2 in zip(read1_fh, read2_fh):
47 | total += 2
48 |
49 | name1 = readline1.strip()
50 | name2 = readline2.strip()
51 |
52 | readline1 = next(read1_fh)
53 | readline2 = next(read2_fh)
54 |
55 | sequence1 = readline1.strip()
56 | sequence2 = readline2.strip()
57 |
58 | next(read1_fh)
59 | next(read2_fh)
60 |
61 | readline1 = next(read1_fh)
62 | readline2 = next(read2_fh)
63 |
64 | quality1 = readline1.strip()
65 | quality2 = readline2.strip()
66 |
67 | # trim adapters for ATAC
68 | where = trim(sequence1, sequence2)
69 |
70 | if where > -1:
71 | trimmed += 2
72 |
73 | # add trimmed read 1 to buffer
74 | trimmed_read1 = f"{name1}\n{sequence1[:where]}\n+\n{quality1[:where]}\n"
75 | buffer1.append(trimmed_read1)
76 |
77 | # add trimmed read 2 to buffer
78 | trimmed_read2 = f"{name2}\n{sequence2[:where]}\n+\n{quality2[:where]}\n"
79 | buffer2.append(trimmed_read2)
80 |
81 | else:
82 | # add original read 1 to buffer
83 | read1 = f"{name1}\n{sequence1}\n+\n{quality1}\n"
84 | buffer1.append(read1)
85 |
86 | # add original read 1 to buffer
87 | read2 = f"{name2}\n{sequence2}\n+\n{quality2}\n"
88 | buffer2.append(read2)
89 |
90 | buffer_counter += 1
91 |
92 | # write reads to trimmed FASTQ files
93 | if buffer_counter == 10000000:
94 | read1_out_writer.write("".join(buffer1))
95 | buffer1.clear()
96 | read2_out_writer.write("".join(buffer2))
97 | buffer2.clear()
98 | buffer_counter = 0
99 |
100 | # write out remaining reads
101 | if buffer_counter > 0:
102 | read1_out_writer.write("".join(buffer1))
103 | buffer1.clear()
104 | read2_out_writer.write("".join(buffer2))
105 | buffer2.clear()
106 | buffer_counter = 0
107 |
108 | # write trimming statistics output file
109 | with open(trimming_stats_file, "w") as f:
110 | fields = ["total_reads", "untrimmed_reads", "trimmed_reads", "%trimmed"]
111 | f.write("\t".join(fields) + "\n")
112 | f.write("%i\t%i\t%i\t%0.1f" % (total, total-trimmed, trimmed, trimmed/total*100 if total > 0 else 0))
113 |
114 | def trim(seq1, seq2):
115 | """
116 | Find overlap between read1 and read2 and return location
117 | """
118 | query = reverse_complement(seq2[0:20])
119 | idx = seq1.rfind(query) # look for perfect match
120 | if idx == -1:
121 | idx = fuzz_align(query,seq1)
122 |
123 | # found it, return everything through match
124 | if idx > -1:
125 | idx = idx+20
126 | else:
127 | idx = -1
128 | return idx
129 |
130 | def fuzz_align(s_seq, l_seq):
131 | """
132 | Align allowing Levenshtein distance of 1
133 | This iteration should go from the right end of l_seq
134 | since we want to do a rfind
135 | """
136 | for i, base in enumerate(l_seq): # loop through equal size windows
137 | l_subset = l_seq[i:i+len(s_seq)]
138 | dist = Levenshtein.distance(l_subset, s_seq, score_cutoff= 1)
139 | if dist <= 1: # find first then break
140 | return i
141 | return -1
142 |
143 | def main():
144 | args = parse_arguments()
145 | input_read1_fastq_file = getattr(args, "input_read1_fastq_file")
146 | input_read2_fastq_file = getattr(args, "input_read2_fastq_file")
147 | output_read1_fastq_file = getattr(args, "output_read1_fastq_file")
148 | output_read2_fastq_file = getattr(args, "output_read2_fastq_file")
149 | trimming_stats_file = getattr(args, "trimming_stats_file")
150 |
151 | trim_fastqs(input_read1_fastq_file, input_read2_fastq_file,
152 | output_read1_fastq_file, output_read2_fastq_file,
153 | trimming_stats_file)
154 |
155 |
156 | if __name__ == "__main__":
157 | main()
158 |
--------------------------------------------------------------------------------
/src/python/match_barcodes.py:
--------------------------------------------------------------------------------
1 | import gzip
2 |
3 | import numpy as np
4 | # import pandas as pd
5 |
6 | import matcha
7 | import sys
8 |
9 | REV_COMP = str.maketrans("ATGC", "TACG")
10 | def reverse_complement(seq):
11 | return str.translate(seq, REV_COMP)[::-1]
12 |
13 | def get_open_fn(path):
14 | with open(path, "rb") as f:
15 | is_gzipped = (f.read(2) == b'\x1f\x8b')
16 | return gzip.open if is_gzipped else open
17 |
18 | def read_barcodes(path, revcomp):
19 | # if path.endswith(".tsv"):
20 | # bc = pd.read_csv(path, sep="\t")["sequence"]
21 | # else:
22 | open_fn = get_open_fn(path)
23 | with open_fn(path, 'rt') as file:
24 | bc = [b.strip() for b in file]
25 | if revcomp:
26 | valid = [reverse_complement(b) for b in bc]
27 | else:
28 | valid = bc
29 |
30 | return valid
31 |
32 | def match_one_bc(fastqs, whitelists, revcomp, max_barcode_dist, offsets, fastq1_out_path, fastq2_out_path, qc_path, threads):
33 | f = matcha.FastqReader(threads = threads)
34 | f.add_sequence("R1", fastqs["R1"], output_path=fastq1_out_path)
35 | f.add_sequence("R2", fastqs["R2"])
36 | f.add_sequence("R3", fastqs["R3"], output_path=fastq2_out_path)
37 |
38 | with open(revcomp["R2"]) as rf:
39 | rc = (int(rf.read().strip()) == 1)
40 |
41 | barcode_sequences = read_barcodes(whitelists["R2"], rc)
42 | cell_barcode = matcha.HashMatcher(
43 | sequences = barcode_sequences,
44 | labels = barcode_sequences,
45 | max_mismatches=max_barcode_dist,
46 | subsequence_count=2
47 | )
48 | f.add_barcode("cell", cell_barcode, "R2", match_start=offsets["R2"])
49 | f.set_output_names("{read_name} CB:Z:{cell}")
50 |
51 | barcode_counts = np.zeros(max_barcode_dist + 2, int)
52 |
53 | total_reads = 0
54 | total_pass = 0
55 |
56 | # print("start read") ####
57 | chunk_size = 10000
58 | while f.read_chunk(chunk_size):
59 | pass_filter = (f.get_match_result("cell", "dist") <=max_barcode_dist) & \
60 | (f.get_match_result("cell", "second_best_dist") > f.get_match_result("cell", "dist"))
61 |
62 | total_reads += len(pass_filter)
63 | total_pass += pass_filter.sum()
64 | values, counts = np.unique(f.get_match_result("cell", "dist"), return_counts=True)
65 | barcode_counts[np.minimum(values, max_barcode_dist + 1)] += counts
66 |
67 | f.write_chunk(pass_filter)
68 |
69 | with open(qc_path, "w") as stats_output:
70 | print(f"{total_pass}/{total_reads} reads passing, ({total_pass/total_reads*100:.2f}%)\n", file=stats_output)
71 | print("mismatches\treads", file=stats_output)
72 | for dist in range(max_barcode_dist + 2):
73 | print(
74 | dist if dist <= max_barcode_dist else f">{max_barcode_dist}",
75 | barcode_counts[dist],
76 | sep = "\t",
77 | file=stats_output
78 | )
79 |
80 |
81 | # def match_two_bc(fastqs, whitelists, revcomp, max_barcode_dist, offsets, fastq1_out_path, fastq2_out_path, qc_path, threads):
82 | # f = matcha.FastqReader(threads = threads)
83 | # f.add_sequence("R1", fastqs["R1"], output_path=fastq1_out_path)
84 | # f.add_sequence("R2", fastqs["R2"], output_path=fastq2_out_path)
85 | # f.add_sequence("I1", fastqs["I1"])
86 | # f.add_sequence("I2", fastqs["I2"])
87 |
88 | # i5_sequences, i5_maybe_rc = read_barcodes(whitelists["I2"], revcomp["I2"])
89 | # T7_sequences, T7_maybe_rc = read_barcodes(whitelists["I1"], revcomp["I1"])
90 |
91 | # i5_barcode = matcha.HashMatcher(
92 | # sequences = i5_maybe_rc,
93 | # labels = i5_sequences,
94 | # max_mismatches=max_barcode_dist,
95 | # subsequence_count=2
96 | # )
97 |
98 | # T7_barcode = matcha.HashMatcher(
99 | # sequences = T7_maybe_rc,
100 | # labels = T7_sequences,
101 | # max_mismatches=max_barcode_dist,
102 | # subsequence_count=2
103 | # )
104 |
105 | # f.add_barcode("i5", i5_barcode, "I2", match_start=offsets["I2"])
106 | # f.add_barcode("T7", T7_barcode, "I1", match_start=offsets["I1"])
107 |
108 | # f.set_output_names("{read_name} CB:Z:{i5}{T7}")
109 |
110 | # barcode_counts = np.zeros((max_barcode_dist + 2, max_barcode_dist + 2), int)
111 |
112 | # total_reads = 0
113 | # total_pass = 0
114 |
115 | # chunk_size = 10000
116 |
117 | # dists = [None, None]
118 | # second_dists = [None, None]
119 | # while f.read_chunk(chunk_size):
120 | # dists[0] = f.get_match_result("i5", "dist")
121 | # second_dists[0] = f.get_match_result("i5", "second_best_dist")
122 | # dists[1] = f.get_match_result("T7", "dist")
123 | # second_dists[1] = f.get_match_result("T7", "second_best_dist")
124 |
125 | # pass_filter = (dists[0] < max_barcode_dist) & \
126 | # (dists[1] < max_barcode_dist) & \
127 | # (dists[0] + dists[1] < second_dists[0] + second_dists[1])
128 |
129 | # total_reads += len(pass_filter)
130 | # total_pass += pass_filter.sum()
131 |
132 | # values, counts = np.unique(dists, axis = 1, return_counts=True)
133 | # indices = np.minimum(values, max_barcode_dist+1)
134 | # barcode_counts[(indices[0], indices[1])] += counts
135 |
136 | # f.write_chunk(pass_filter)
137 |
138 | # with open(qc_path, "w") as stats_output:
139 | # print(f"{total_pass}/{total_reads} reads passing, ({total_pass/total_reads*100:.2f}%)\n", file=stats_output)
140 | # print("mismatches_i5\tmismatches_T7\treads", file=stats_output)
141 | # for i5_dist in range(max_barcode_dist + 2):
142 | # for T7_dist in range(max_barcode_dist + 2):
143 | # print(
144 | # i5_dist if i5_dist <= max_barcode_dist else f">{max_barcode_dist}",
145 | # T7_dist if T7_dist <= max_barcode_dist else f">{max_barcode_dist}",
146 | # barcode_counts[i5_dist, T7_dist],
147 | # sep = "\t",
148 | # file=stats_output
149 | # )
150 |
151 | modality = sys.argv[4]
152 | whitelist = sys.argv[7]
153 | fastq1_out_path = sys.argv[8]
154 | fastq2_out_path = sys.argv[9]
155 | qc_path = sys.argv[10]
156 | threads = int(sys.argv[11])
157 | max_barcode_dist = int(sys.argv[5])
158 | fastqs = {
159 | "R1": sys.argv[1],
160 | "R2": sys.argv[3],
161 | "R3": sys.argv[2],
162 | }
163 | revcomp = {
164 | "R2": sys.argv[6],
165 | }
166 | if modality == "10x":
167 | whitelists = {
168 | "R2": whitelist,
169 | }
170 | offsets = {
171 | "R2": 0,
172 | }
173 | match_one_bc(fastqs, whitelists, revcomp, max_barcode_dist, offsets, fastq1_out_path, fastq2_out_path, qc_path, threads)
174 |
175 | elif modality == "10x_multiome":
176 | whitelists = {
177 | "R2": whitelist,
178 | }
179 | offsets = {
180 | "R2": 8,
181 | }
182 | match_one_bc(fastqs, whitelists, revcomp, max_barcode_dist, offsets, fastq1_out_path, fastq2_out_path, qc_path, threads)
183 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Broad Institute of MIT and Harvard Single-Cell/Nucleus Multiomic Processing Pipeline
2 |
3 | Pipeline specifications can be found [here](https://docs.google.com/document/d/1J-NWpDLkEGLsLjVe6h6-Rx4nxzTdgy1TJZvuMnYiiyg/edit?usp=sharing).
4 |
5 | Pipeline main page on [dockstore](https://dockstore.org/workflows/github.com/broadinstitute/epi-SHARE-seq-pipeline/SHARE-seq:release?tab=info).
6 |
7 | 
9 | 