├── .flake8 ├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── LICENSE.txt ├── Makefile ├── README.md ├── bin ├── remove-repeated-headers.py ├── sbatch.py ├── slurm-pipeline-status-plot.py ├── slurm-pipeline-status.py ├── slurm-pipeline-version.py └── slurm-pipeline.py ├── examples ├── blast-with-force-and-skipping │ ├── 0-start-log.sh │ ├── 1-split-fasta.sh │ ├── 2-run-blast.sh │ ├── 3-collect.sh │ ├── 4-end-log.sh │ ├── Makefile │ ├── README.md │ ├── blast.sh │ ├── specification.json │ ├── specification.toml │ └── viruses.fasta ├── blast │ ├── 0-start-log.sh │ ├── 1-split-fasta.sh │ ├── 2-run-blast.sh │ ├── 3-collect.sh │ ├── 4-end-log.sh │ ├── Makefile │ ├── README.md │ ├── blast.sh │ ├── specification.json │ ├── specification.toml │ └── viruses.fasta ├── double-collect │ ├── 0-start.sh │ ├── 1-species-count.sh │ ├── 2-category-emit.sh │ ├── 3-category-count.sh │ ├── 4-summarize.sh │ ├── Makefile │ ├── README.md │ ├── categories │ ├── data │ │ ├── cat │ │ ├── cow │ │ ├── dog │ │ ├── mosquito │ │ └── tick │ ├── specification.json │ └── specification.toml ├── word-count-with-skipping │ ├── Makefile │ ├── README.md │ ├── scripts │ │ ├── long-words-only.sh │ │ ├── one-word-per-line.sh │ │ └── summarize.sh │ ├── specification.json │ ├── specification.toml │ └── texts │ │ ├── 1-karamazov.txt │ │ ├── 2-trial.txt │ │ └── 3-ulysses.txt └── word-count │ ├── Makefile │ ├── README.md │ ├── scripts │ ├── long-words-only.sh │ ├── one-word-per-line.sh │ └── summarize.sh │ ├── specification.json │ ├── specification.toml │ └── texts │ ├── 1-karamazov.txt │ ├── 2-trial.txt │ └── 3-ulysses.txt ├── requirements-dev.txt ├── requirements.txt ├── setup.py ├── slurm_pipeline ├── __init__.py ├── base.py ├── error.py ├── pipeline.py ├── sacct.py ├── status.py └── utils.py ├── talk-slides.pdf ├── test ├── __init__.py ├── test_base.py ├── test_pipeline.py ├── test_sacct.py ├── test_status.py └── test_utils.py └── tox.ini /.flake8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/.flake8 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/.gitignore -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/.travis.yml -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/CHANGELOG.md -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/LICENSE.txt -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/README.md -------------------------------------------------------------------------------- /bin/remove-repeated-headers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/bin/remove-repeated-headers.py -------------------------------------------------------------------------------- /bin/sbatch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/bin/sbatch.py -------------------------------------------------------------------------------- /bin/slurm-pipeline-status-plot.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/bin/slurm-pipeline-status-plot.py -------------------------------------------------------------------------------- /bin/slurm-pipeline-status.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/bin/slurm-pipeline-status.py -------------------------------------------------------------------------------- /bin/slurm-pipeline-version.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/bin/slurm-pipeline-version.py -------------------------------------------------------------------------------- /bin/slurm-pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/bin/slurm-pipeline.py -------------------------------------------------------------------------------- /examples/blast-with-force-and-skipping/0-start-log.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/blast-with-force-and-skipping/0-start-log.sh -------------------------------------------------------------------------------- /examples/blast-with-force-and-skipping/1-split-fasta.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/blast-with-force-and-skipping/1-split-fasta.sh -------------------------------------------------------------------------------- /examples/blast-with-force-and-skipping/2-run-blast.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/blast-with-force-and-skipping/2-run-blast.sh -------------------------------------------------------------------------------- /examples/blast-with-force-and-skipping/3-collect.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/blast-with-force-and-skipping/3-collect.sh -------------------------------------------------------------------------------- /examples/blast-with-force-and-skipping/4-end-log.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/blast-with-force-and-skipping/4-end-log.sh -------------------------------------------------------------------------------- /examples/blast-with-force-and-skipping/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/blast-with-force-and-skipping/Makefile -------------------------------------------------------------------------------- /examples/blast-with-force-and-skipping/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/blast-with-force-and-skipping/README.md -------------------------------------------------------------------------------- /examples/blast-with-force-and-skipping/blast.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/blast-with-force-and-skipping/blast.sh -------------------------------------------------------------------------------- /examples/blast-with-force-and-skipping/specification.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/blast-with-force-and-skipping/specification.json -------------------------------------------------------------------------------- /examples/blast-with-force-and-skipping/specification.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/blast-with-force-and-skipping/specification.toml -------------------------------------------------------------------------------- /examples/blast-with-force-and-skipping/viruses.fasta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/blast-with-force-and-skipping/viruses.fasta -------------------------------------------------------------------------------- /examples/blast/0-start-log.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/blast/0-start-log.sh -------------------------------------------------------------------------------- /examples/blast/1-split-fasta.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/blast/1-split-fasta.sh -------------------------------------------------------------------------------- /examples/blast/2-run-blast.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/blast/2-run-blast.sh -------------------------------------------------------------------------------- /examples/blast/3-collect.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/blast/3-collect.sh -------------------------------------------------------------------------------- /examples/blast/4-end-log.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/blast/4-end-log.sh -------------------------------------------------------------------------------- /examples/blast/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/blast/Makefile -------------------------------------------------------------------------------- /examples/blast/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/blast/README.md -------------------------------------------------------------------------------- /examples/blast/blast.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/blast/blast.sh -------------------------------------------------------------------------------- /examples/blast/specification.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/blast/specification.json -------------------------------------------------------------------------------- /examples/blast/specification.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/blast/specification.toml -------------------------------------------------------------------------------- /examples/blast/viruses.fasta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/blast/viruses.fasta -------------------------------------------------------------------------------- /examples/double-collect/0-start.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/double-collect/0-start.sh -------------------------------------------------------------------------------- /examples/double-collect/1-species-count.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/double-collect/1-species-count.sh -------------------------------------------------------------------------------- /examples/double-collect/2-category-emit.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/double-collect/2-category-emit.sh -------------------------------------------------------------------------------- /examples/double-collect/3-category-count.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/double-collect/3-category-count.sh -------------------------------------------------------------------------------- /examples/double-collect/4-summarize.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/double-collect/4-summarize.sh -------------------------------------------------------------------------------- /examples/double-collect/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/double-collect/Makefile -------------------------------------------------------------------------------- /examples/double-collect/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/double-collect/README.md -------------------------------------------------------------------------------- /examples/double-collect/categories: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/double-collect/categories -------------------------------------------------------------------------------- /examples/double-collect/data/cat: -------------------------------------------------------------------------------- 1 | 10 2 | 20 3 | 30 4 | -------------------------------------------------------------------------------- /examples/double-collect/data/cow: -------------------------------------------------------------------------------- 1 | 40 2 | 50 3 | -------------------------------------------------------------------------------- /examples/double-collect/data/dog: -------------------------------------------------------------------------------- 1 | 60 2 | 70 3 | 80 4 | 90 5 | 100 6 | -------------------------------------------------------------------------------- /examples/double-collect/data/mosquito: -------------------------------------------------------------------------------- 1 | 5 2 | 10 3 | -------------------------------------------------------------------------------- /examples/double-collect/data/tick: -------------------------------------------------------------------------------- 1 | 15 2 | 20 3 | 25 4 | -------------------------------------------------------------------------------- /examples/double-collect/specification.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/double-collect/specification.json -------------------------------------------------------------------------------- /examples/double-collect/specification.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/double-collect/specification.toml -------------------------------------------------------------------------------- /examples/word-count-with-skipping/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/word-count-with-skipping/Makefile -------------------------------------------------------------------------------- /examples/word-count-with-skipping/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/word-count-with-skipping/README.md -------------------------------------------------------------------------------- /examples/word-count-with-skipping/scripts/long-words-only.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/word-count-with-skipping/scripts/long-words-only.sh -------------------------------------------------------------------------------- /examples/word-count-with-skipping/scripts/one-word-per-line.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/word-count-with-skipping/scripts/one-word-per-line.sh -------------------------------------------------------------------------------- /examples/word-count-with-skipping/scripts/summarize.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/word-count-with-skipping/scripts/summarize.sh -------------------------------------------------------------------------------- /examples/word-count-with-skipping/specification.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/word-count-with-skipping/specification.json -------------------------------------------------------------------------------- /examples/word-count-with-skipping/specification.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/word-count-with-skipping/specification.toml -------------------------------------------------------------------------------- /examples/word-count-with-skipping/texts/1-karamazov.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/word-count-with-skipping/texts/1-karamazov.txt -------------------------------------------------------------------------------- /examples/word-count-with-skipping/texts/2-trial.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/word-count-with-skipping/texts/2-trial.txt -------------------------------------------------------------------------------- /examples/word-count-with-skipping/texts/3-ulysses.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/word-count-with-skipping/texts/3-ulysses.txt -------------------------------------------------------------------------------- /examples/word-count/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/word-count/Makefile -------------------------------------------------------------------------------- /examples/word-count/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/word-count/README.md -------------------------------------------------------------------------------- /examples/word-count/scripts/long-words-only.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/word-count/scripts/long-words-only.sh -------------------------------------------------------------------------------- /examples/word-count/scripts/one-word-per-line.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/word-count/scripts/one-word-per-line.sh -------------------------------------------------------------------------------- /examples/word-count/scripts/summarize.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/word-count/scripts/summarize.sh -------------------------------------------------------------------------------- /examples/word-count/specification.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/word-count/specification.json -------------------------------------------------------------------------------- /examples/word-count/specification.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/word-count/specification.toml -------------------------------------------------------------------------------- /examples/word-count/texts/1-karamazov.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/word-count/texts/1-karamazov.txt -------------------------------------------------------------------------------- /examples/word-count/texts/2-trial.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/word-count/texts/2-trial.txt -------------------------------------------------------------------------------- /examples/word-count/texts/3-ulysses.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/examples/word-count/texts/3-ulysses.txt -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | pycodestyle>=2.6.0 2 | flake8>=3.8.4 3 | pytest>=6.2.2 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pandas 2 | plotly 3 | pytest>=6.2.2 4 | toml 5 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/setup.py -------------------------------------------------------------------------------- /slurm_pipeline/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/slurm_pipeline/__init__.py -------------------------------------------------------------------------------- /slurm_pipeline/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/slurm_pipeline/base.py -------------------------------------------------------------------------------- /slurm_pipeline/error.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/slurm_pipeline/error.py -------------------------------------------------------------------------------- /slurm_pipeline/pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/slurm_pipeline/pipeline.py -------------------------------------------------------------------------------- /slurm_pipeline/sacct.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/slurm_pipeline/sacct.py -------------------------------------------------------------------------------- /slurm_pipeline/status.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/slurm_pipeline/status.py -------------------------------------------------------------------------------- /slurm_pipeline/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/slurm_pipeline/utils.py -------------------------------------------------------------------------------- /talk-slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/talk-slides.pdf -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/test_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/test/test_base.py -------------------------------------------------------------------------------- /test/test_pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/test/test_pipeline.py -------------------------------------------------------------------------------- /test/test_sacct.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/test/test_sacct.py -------------------------------------------------------------------------------- /test/test_status.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/test/test_status.py -------------------------------------------------------------------------------- /test/test_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/test/test_utils.py -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acorg/slurm-pipeline/HEAD/tox.ini --------------------------------------------------------------------------------