├── .gitignore ├── BioQueue ├── __init__.py ├── settings-example.py ├── urls.py └── wsgi.py ├── LICENSE ├── QueueDB ├── __init__.py ├── admin.py ├── apps.py ├── extmodels │ ├── GoogleDrive.py │ └── __init__.py ├── migrations │ ├── 0001_initial.py │ └── __init__.py ├── models.py ├── tests.py └── views.py ├── README.md ├── _config.yml ├── accounts ├── __init__.py ├── admin.py ├── apps.py ├── forms.py ├── migrations │ └── __init__.py ├── models.py ├── templates │ └── accounts │ │ ├── login.html │ │ └── register.html ├── tests.py ├── urls.py └── views.py ├── config ├── bioqueue.conf ├── custom.conf └── file_support.conf ├── deploy ├── 000-default.conf.tpl ├── mysql.tpl ├── postgresql.tpl ├── prerequisites.txt ├── python_pip_non_root.sh ├── sqlite.tpl ├── status_code.md └── switch_from_MySQLdb_to_PyMySQL.py ├── init_resource.json ├── install.py ├── manage.py ├── protocol_dumps └── __init__.py ├── status_page.png ├── templates ├── admin │ └── login.html └── registration │ └── logged_out.html ├── ui ├── __init__.py ├── admin.py ├── apps.py ├── cron_jobs.py ├── cron_runner.py ├── ena.py ├── extmodels │ ├── GoogleDrive.py │ └── __init__.py ├── forms.py ├── maintenance_protocols │ ├── __init__.py │ ├── bam_index.py │ ├── bam_sort.py │ ├── compress.py │ ├── configure_make_install.py │ ├── decompress.py │ ├── download.py │ ├── fastqc.py │ ├── git.py │ ├── gunzip.py │ ├── make.py │ ├── svn.py │ ├── tarbz2.py │ ├── targz.py │ └── zip.py ├── plugins │ ├── __init__.py │ └── google_drive.py ├── static │ ├── admin │ │ ├── css │ │ │ ├── base.css │ │ │ ├── changelists.css │ │ │ ├── dashboard.css │ │ │ ├── fonts.css │ │ │ ├── forms.css │ │ │ ├── login.css │ │ │ ├── rtl.css │ │ │ └── widgets.css │ │ ├── fonts │ │ │ ├── LICENSE.txt │ │ │ ├── README.txt │ │ │ ├── Roboto-Bold-webfont.woff │ │ │ ├── Roboto-Light-webfont.woff │ │ │ └── Roboto-Regular-webfont.woff │ │ ├── img │ │ │ ├── LICENSE │ │ │ ├── README.txt │ │ │ ├── calendar-icons.svg │ │ │ ├── gis │ │ │ │ ├── move_vertex_off.svg │ │ │ │ └── move_vertex_on.svg │ │ │ ├── icon-addlink.svg │ │ │ ├── icon-alert.svg │ │ │ ├── icon-calendar.svg │ │ │ ├── icon-changelink.svg │ │ │ ├── icon-clock.svg │ │ │ ├── icon-deletelink.svg │ │ │ ├── icon-no.svg │ │ │ ├── icon-unknown-alt.svg │ │ │ ├── icon-unknown.svg │ │ │ ├── icon-yes.svg │ │ │ ├── inline-delete.svg │ │ │ ├── search.svg │ │ │ ├── selector-icons.svg │ │ │ ├── sorting-icons.svg │ │ │ ├── tooltag-add.svg │ │ │ └── tooltag-arrowright.svg │ │ └── js │ │ │ ├── SelectBox.js │ │ │ ├── SelectFilter2.js │ │ │ ├── actions.js │ │ │ ├── actions.min.js │ │ │ ├── admin │ │ │ ├── DateTimeShortcuts.js │ │ │ └── RelatedObjectLookups.js │ │ │ ├── calendar.js │ │ │ ├── cancel.js │ │ │ ├── change_form.js │ │ │ ├── collapse.js │ │ │ ├── collapse.min.js │ │ │ ├── core.js │ │ │ ├── inlines.js │ │ │ ├── inlines.min.js │ │ │ ├── jquery.init.js │ │ │ ├── popup_response.js │ │ │ ├── prepopulate.js │ │ │ ├── prepopulate.min.js │ │ │ ├── prepopulate_init.js │ │ │ ├── timeparse.js │ │ │ ├── urlify.js │ │ │ └── vendor │ │ │ ├── jquery │ │ │ ├── LICENSE-JQUERY.txt │ │ │ ├── jquery.js │ │ │ └── jquery.min.js │ │ │ └── xregexp │ │ │ ├── LICENSE-XREGEXP.txt │ │ │ ├── xregexp.js │ │ │ └── xregexp.min.js │ └── ui │ │ ├── css │ │ ├── bootstrap-grid.css │ │ ├── bootstrap-grid.css.map │ │ ├── bootstrap-grid.min.css │ │ ├── bootstrap-grid.min.css.map │ │ ├── bootstrap-reboot.css │ │ ├── bootstrap-reboot.css.map │ │ ├── bootstrap-reboot.min.css │ │ ├── bootstrap-reboot.min.css.map │ │ ├── bootstrap-select.css.map │ │ ├── bootstrap-select.min.css │ │ ├── bootstrap.min.css │ │ ├── bootstrap.min.css.map │ │ ├── index.html │ │ └── site.css │ │ ├── fontawesome │ │ ├── LICENSE.txt │ │ ├── css │ │ │ ├── all.min.css │ │ │ ├── brands.min.css │ │ │ ├── fontawesome.min.css │ │ │ ├── regular.min.css │ │ │ ├── solid.min.css │ │ │ ├── svg-with-js.min.css │ │ │ └── v4-shims.min.css │ │ └── webfonts │ │ │ ├── fa-brands-400.eot │ │ │ ├── fa-brands-400.svg │ │ │ ├── fa-brands-400.ttf │ │ │ ├── fa-brands-400.woff │ │ │ ├── fa-brands-400.woff2 │ │ │ ├── fa-regular-400.eot │ │ │ ├── fa-regular-400.svg │ │ │ ├── fa-regular-400.ttf │ │ │ ├── fa-regular-400.woff │ │ │ ├── fa-regular-400.woff2 │ │ │ ├── fa-solid-900.eot │ │ │ ├── fa-solid-900.svg │ │ │ ├── fa-solid-900.ttf │ │ │ ├── fa-solid-900.woff │ │ │ └── fa-solid-900.woff2 │ │ ├── img │ │ ├── index.html │ │ ├── logo.png │ │ ├── logoc.png │ │ └── mls │ │ │ └── index.html │ │ ├── index.html │ │ └── js │ │ ├── auto.micro.js │ │ ├── bootstrap-select.min.js │ │ ├── bootstrap-select.min.js.map │ │ ├── bootstrap.bundle.js │ │ ├── bootstrap.bundle.js.map │ │ ├── bootstrap.bundle.min.js │ │ ├── bootstrap.bundle.min.js.map │ │ ├── index.html │ │ ├── job-complete.min.js │ │ ├── jquery-3.6.0.min.js │ │ ├── jquery.cookie.js │ │ └── site.js ├── templates │ └── ui │ │ ├── 403.html │ │ ├── add_job.html │ │ ├── add_protocol.html │ │ ├── add_step.html │ │ ├── archive_table.html │ │ ├── base.html │ │ ├── create_simple_models.html │ │ ├── error.html │ │ ├── fetch_data.html │ │ ├── fetch_learning.html │ │ ├── foot.html │ │ ├── get_learning_result.html │ │ ├── import_protocol.html │ │ ├── index.html │ │ ├── install_ref.html │ │ ├── install_tool.html │ │ ├── list_protocol.html │ │ ├── manage_reference.html │ │ ├── navigation.html │ │ ├── query_job.html │ │ ├── register_sample.html │ │ ├── registered_sample_cards.html │ │ ├── settings.html │ │ ├── show_environment.html │ │ ├── show_job_folder.html │ │ ├── show_learning.html │ │ ├── show_learning_steps.html │ │ ├── show_reference.html │ │ ├── show_samples.html │ │ ├── show_steps.html │ │ ├── show_uploads.html │ │ ├── show_workspace.html │ │ ├── single.html │ │ ├── step_atom.html │ │ ├── tar_files.html │ │ ├── update_simple_models.html │ │ └── variant_hint.html ├── templatetags │ ├── __init__.py │ └── job_tags.py ├── tests.py ├── tools.py ├── urls.py ├── views.py └── views │ ├── Job.py │ ├── Misc.py │ ├── Plugins.py │ ├── Protocol.py │ ├── References.py │ ├── Samples.py │ ├── Shield.py │ └── __init__.py ├── version └── worker ├── __init__.py ├── _step.py ├── bases.py ├── bioqueue.py ├── cluster_models ├── HTCondor.py ├── HTCondor.tpl ├── LSF.py ├── LSF.tpl ├── TorquePBS.py ├── TorquePBS.tpl ├── __init__.py └── cluster_model.py ├── cluster_support.py ├── compile_tool.py ├── django_initial.py ├── feedback.py ├── integrityTools.py ├── maintenance_models ├── __init__.py └── make.py ├── ml_collector.py └── ml_container.py /.gitignore: -------------------------------------------------------------------------------- 1 | .project 2 | *.log 3 | *.lock 4 | .pydevproject 5 | *.pyc 6 | workspace/* 7 | /.idea/* 8 | worker/.idea/* 9 | BioQueue/settings.py 10 | deploy/000-default.conf 11 | deploy/prerequisites.txt 12 | worker/config.conf 13 | 14 | config/custom.conf 15 | -------------------------------------------------------------------------------- /BioQueue/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/BioQueue/__init__.py -------------------------------------------------------------------------------- /BioQueue/settings-example.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for BioQueue project. 3 | 4 | Generated by 'django-admin startproject' using Django 2.2. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/2.2/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/2.2/ref/settings/ 11 | """ 12 | 13 | import os 14 | 15 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 16 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 17 | 18 | 19 | # Quick-start development settings - unsuitable for production 20 | # See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/ 21 | 22 | # SECURITY WARNING: keep the secret key used in production secret! 23 | SECRET_KEY = '{SECRET_KEY}' 24 | 25 | # SECURITY WARNING: don't run with debug turned on in production! 26 | DEBUG = True 27 | 28 | ALLOWED_HOSTS = ['*', ] 29 | 30 | 31 | # Application definition 32 | 33 | INSTALLED_APPS = [ 34 | 'django.contrib.admin', 35 | 'django.contrib.auth', 36 | 'django.contrib.contenttypes', 37 | 'django.contrib.sessions', 38 | 'django.contrib.messages', 39 | 'django.contrib.staticfiles', 40 | 'QueueDB', 41 | 'accounts', 42 | 'ui', 43 | 'worker', 44 | #'django_ftpserver', 45 | ] 46 | 47 | MIDDLEWARE = [ 48 | 'django.middleware.security.SecurityMiddleware', 49 | 'django.contrib.sessions.middleware.SessionMiddleware', 50 | 'django.middleware.common.CommonMiddleware', 51 | 'django.middleware.csrf.CsrfViewMiddleware', 52 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 53 | 'django.contrib.messages.middleware.MessageMiddleware', 54 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 55 | ] 56 | 57 | ROOT_URLCONF = 'BioQueue.urls' 58 | 59 | TEMPLATES = [ 60 | { 61 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 62 | 'DIRS': [os.path.join(BASE_DIR, 'templates')], 63 | 'APP_DIRS': True, 64 | 'OPTIONS': { 65 | 'context_processors': [ 66 | 'django.template.context_processors.debug', 67 | 'django.template.context_processors.request', 68 | 'django.contrib.auth.context_processors.auth', 69 | 'django.contrib.messages.context_processors.messages', 70 | ], 71 | }, 72 | }, 73 | ] 74 | 75 | WSGI_APPLICATION = 'BioQueue.wsgi.application' 76 | 77 | 78 | # Database 79 | # https://docs.djangoproject.com/en/2.2/ref/settings/#databases 80 | 81 | DATABASES = { 82 | {DATABASE_BACKEND} 83 | } 84 | 85 | 86 | # Password validation 87 | # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators 88 | 89 | AUTH_PASSWORD_VALIDATORS = [ 90 | { 91 | 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 92 | }, 93 | { 94 | 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 95 | }, 96 | { 97 | 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', 98 | }, 99 | { 100 | 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', 101 | }, 102 | ] 103 | 104 | 105 | # Internationalization 106 | # https://docs.djangoproject.com/en/2.2/topics/i18n/ 107 | 108 | LANGUAGE_CODE = 'en-us' 109 | 110 | TIME_ZONE = 'UTC' 111 | 112 | USE_I18N = True 113 | 114 | USE_L10N = True 115 | 116 | USE_TZ = True 117 | 118 | 119 | # Static files (CSS, JavaScript, Images) 120 | # https://docs.djangoproject.com/en/2.2/howto/static-files/ 121 | 122 | STATIC_URL = '/static/' 123 | -------------------------------------------------------------------------------- /BioQueue/urls.py: -------------------------------------------------------------------------------- 1 | """BioQueue URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/2.2/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.urls import include, path 14 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 15 | """ 16 | from django.contrib import admin 17 | from django.urls import path, include 18 | from ui.views import index 19 | 20 | urlpatterns = [ 21 | path('', index), 22 | path('admin/', admin.site.urls), 23 | path('accounts/', include('accounts.urls')), 24 | path('ui/', include('ui.urls')), 25 | ] 26 | -------------------------------------------------------------------------------- /BioQueue/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for BioQueue project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/2.2/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'BioQueue.settings') 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /QueueDB/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/QueueDB/__init__.py -------------------------------------------------------------------------------- /QueueDB/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | -------------------------------------------------------------------------------- /QueueDB/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class QueuedbConfig(AppConfig): 5 | default_auto_field = 'django.db.models.AutoField' 6 | name = 'QueueDB' 7 | -------------------------------------------------------------------------------- /QueueDB/extmodels/GoogleDrive.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Author: Li Yao 4 | # @Date: 12/29/20 5 | # 6 | # BioQueue is free for personal use and is licensed under 7 | # the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, 9 | # or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 14 | 15 | from django.db import models 16 | from QueueDB.models import _OwnerModel 17 | 18 | 19 | class GoogleDriveConnection(_OwnerModel): 20 | credential_pickle = models.CharField(max_length=255) 21 | folder_id = models.CharField(max_length=255) 22 | -------------------------------------------------------------------------------- /QueueDB/extmodels/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | # Created by: Li Yao (yaoli.gm@gmail.com) 4 | # Created on: 12/27/20 5 | # from .RunOnQC import * 6 | -------------------------------------------------------------------------------- /QueueDB/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/QueueDB/migrations/__init__.py -------------------------------------------------------------------------------- /QueueDB/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /QueueDB/views.py: -------------------------------------------------------------------------------- 1 | from django.shortcuts import render 2 | 3 | # Create your views here. 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # BioQueue 2 | [![document](https://readthedocs.org/projects/bioqueue/badge/?version=latest "document")](https://bioqueue.readthedocs.io/en/latest/?badge=latest) 3 | 4 | BioQueue is a researcher-facing bioinformatic platform preferentially to improve the efficiency and robustness of analysis in bioinformatics research by estimating the system resources required by a particular job. At the same time, BioQueue also aims to promote the accessibility and reproducibility of data analysis in biomedical research. Implemented by Python **3.x**, BioQueue can work in both POSIX compatible systems (Linux, Solaris, OS X, etc.) and Windows. 5 | ## Get started 6 | 7 | `pip` is the default package manager for BioQueue, so before installing BioQueue, please make sure that you have [pip](https://pip.pypa.io/en/stable/) installed. 8 | 9 | ### Prerequisites 10 | 11 | BioQueue can store data on SQLite, which means users can set up BioQueue without an extra database software. However, to achieve a higher performance, **we suggest users to install MySQL or PostgreSQL**. If you want to set up MySQL or PostgreSQL on your machine, you can visit the wiki page. 12 | ### 1. Download and setup the BioQueue project 13 | 14 | First of all, you will need to clone the project from Github (Or you can download BioQueue by open [this link](https://github.com/liyao001/BioQueue/zipball/master)). 15 | ``` 16 | git clone https://github.com/liyao001/BioQueue.git 17 | Or 18 | wget https://github.com/liyao001/BioQueue/zipball/master 19 | ``` 20 | Then navigate to the project's directory, and run `install.py` script (All dependent python packages will be automatically installed): 21 | ``` 22 | cd BioQueue 23 | python install.py 24 | ``` 25 | When running `install.py`, this script will ask you a few questions include: 26 | 1. CPU cores: The amount of CPU to use. Default value: all cores on that machine. 27 | 2. Memory (Gb): The amount of memory to use. Default value: all physical memory on that machine. 28 | 3. Disk quota for each user(Gb, default value: all disk space on that machine). 29 | 30 | If you decide to run BioQueue with MySQL or PostgreSQL, the script will ask a few more questions: 31 | 1. Database host: If you install MySQL server on your own machine, enter `localhost` or `127.0.0.1`. 32 | 2. Database user: user name of the database. 33 | 3. Database password: password of the database. 34 | 4. Database name: Name of the data table. 35 | 5. Database port: `3306` by default for MySQL 36 | 37 | Then the script will interact with you and create a super user account for the platform. 38 | 39 | ### 2. Start the queue 40 | 41 | Run the `bioqueue.py` script in the `BioQueue/worker` folder 42 | ``` 43 | python worker/bioqueue.py 44 | ``` 45 | 46 | ### 3. Start webserver 47 | 48 | ``` 49 | python manage.py runserver 0.0.0.0:8000 50 | ``` 51 | This will start up the server on `0.0.0.0` and port `8000`, so BioQueue can be accessed over the network. If you want access BioQueue only in local environment, remove `0.0.0.0:8000`. 52 | 53 | ## Useful information 54 | 55 | * To stop the queue, the webserver or the ftp server, just hit `Ctrl-c` in the terminal from which BioQueue is running. 56 | * To get a better performance, moving the webserver to [Apache](http://bioqueue.readthedocs.io/en/latest/faq.html#use-bioqueue-with-apache-in-production-environment) or [nginx](https://nginx.org) is a good idea. 57 | 58 | ## Screenshot 59 | 60 | ![](status_page.png) 61 | 62 | ## Citation 63 | 64 | 1. Yao, L., Wang, H., Song, Y. & Sui, G. BioQueue: a novel pipeline framework to accelerate bioinformatics analysis. *Bioinformatics* 33, 3286–3288 (2017). [doi:10.1093/bioinformatics/btx403](https://doi.org/doi:10.1093/bioinformatics/btx403) -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-cayman -------------------------------------------------------------------------------- /accounts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/accounts/__init__.py -------------------------------------------------------------------------------- /accounts/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | -------------------------------------------------------------------------------- /accounts/apps.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from django.apps import AppConfig 4 | 5 | 6 | class AccountsConfig(AppConfig): 7 | name = 'accounts' 8 | -------------------------------------------------------------------------------- /accounts/forms.py: -------------------------------------------------------------------------------- 1 | from django import forms 2 | from django.contrib.auth.models import User 3 | 4 | 5 | class LoginForm(forms.Form): 6 | username = forms.CharField( 7 | required=True, 8 | label=False, 9 | error_messages={"required": "Please type username"}, 10 | widget=forms.TextInput( 11 | attrs={ 12 | "placeholder": "Username / Project name", 13 | "class": "form-control", 14 | } 15 | ), 16 | ) 17 | password = forms.CharField( 18 | required=True, 19 | label=False, 20 | error_messages={"required": "Please type your password"}, 21 | widget=forms.PasswordInput( 22 | attrs={ 23 | "placeholder": "Password", 24 | "class": "form-control", 25 | } 26 | ) 27 | ) 28 | 29 | 30 | class PasswordChangeForm(forms.Form): 31 | old_password = forms.CharField( 32 | required=True, 33 | label="Raw Password", 34 | error_messages={"required": "Please type your raw password"}, 35 | widget=forms.PasswordInput( 36 | attrs={ 37 | "placeholder": "Raw password", 38 | } 39 | ) 40 | ) 41 | new_password_1 = forms.CharField( 42 | required=True, 43 | label="New Password", 44 | error_messages={"required": "Please type your new password"}, 45 | widget=forms.PasswordInput( 46 | attrs={ 47 | "placeholder": "New password", 48 | } 49 | ) 50 | ) 51 | new_password_2 = forms.CharField( 52 | required=True, 53 | label="Confirm Password", 54 | error_messages={"required": "Please confirm new password"}, 55 | widget=forms.PasswordInput( 56 | attrs={ 57 | "placeholder": "Confirm new password", 58 | } 59 | ) 60 | ) 61 | 62 | 63 | class UserRegisterForm(forms.ModelForm): 64 | password = forms.CharField( 65 | required=True, 66 | label=False, 67 | widget=forms.PasswordInput( 68 | attrs={ 69 | "class": "form-control", 70 | "placeholder": "Password", 71 | } 72 | ) 73 | ) 74 | password_2 = forms.CharField( 75 | required=True, 76 | label=False, 77 | widget=forms.PasswordInput( 78 | attrs={ 79 | "class": 'form-control', 80 | "placeholder": "Confirm password", 81 | } 82 | ) 83 | ) 84 | 85 | class Meta: 86 | model = User 87 | fields = ("username", "first_name", "last_name", "email") 88 | widgets = { 89 | "username": forms.TextInput(attrs={"class": "form-control", "placeholder": "Username / Project name"}), 90 | "first_name": forms.TextInput(attrs={"class": "form-control", "placeholder": "First name"}), 91 | "last_name": forms.TextInput(attrs={"class": "form-control", "placeholder": "Last name"}), 92 | "email": forms.EmailInput(attrs={"class": "form-control", "placeholder": "E-mail"}), 93 | } 94 | labels = { 95 | "username": False, 96 | "first_name": False, 97 | "last_name": False, 98 | "email": False, 99 | } 100 | help_texts = { 101 | "username": False, 102 | "first_name": False, 103 | "last_name": False, 104 | } 105 | 106 | def clean_password2(self): 107 | cd = self.cleaned_data 108 | if cd["password"] != cd["password_2"]: 109 | raise forms.ValidationError('Password doesn\'t match.') 110 | return cd["password_2"] 111 | -------------------------------------------------------------------------------- /accounts/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/accounts/migrations/__init__.py -------------------------------------------------------------------------------- /accounts/models.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from django.db import models 4 | 5 | # Create your models here. 6 | -------------------------------------------------------------------------------- /accounts/templates/accounts/login.html: -------------------------------------------------------------------------------- 1 | {% extends "ui/single.html" %} 2 | {% block title %}Sign In | BioQueue{% endblock %} 3 | {% block content %} 4 |
5 |
6 |

Login

7 | 23 |
24 |
25 | {% endblock %} 26 | {% block script %} 27 | $(document) 28 | .ajaxStart(function(){ 29 | $("button:submit").addClass("log-in").attr("disabled", true); 30 | }) 31 | .ajaxStop(function(){ 32 | $("button:submit").removeClass("log-in").attr("disabled", false); 33 | }); 34 | $("form").submit(function(){ 35 | var self = $(this); 36 | self.find(".Validform_checktip").hide(); 37 | $.post(self.attr("action"), self.serialize(), success, "json"); 38 | return false; 39 | 40 | function success(data){ 41 | if(data.status){ 42 | window.location.href = data.url; 43 | } else { 44 | $("#id_password").val(''); 45 | self.find(".Validform_checktip").html(data.info).show(); 46 | } 47 | } 48 | }); 49 | {% endblock %} -------------------------------------------------------------------------------- /accounts/templates/accounts/register.html: -------------------------------------------------------------------------------- 1 | {% extends "ui/single.html" %} 2 | {% block title %}Sign Up | BioQueue{% endblock %} 3 | {% block content %} 4 |
5 |
6 |

Register

7 | 24 |
25 |
26 | {% endblock %} 27 | {% block script %} 28 | $(document) 29 | .ajaxStart(function(){ 30 | $("button:submit").attr("disabled", true); 31 | }) 32 | .ajaxStop(function(){ 33 | $("button:submit").attr("disabled", false); 34 | }); 35 | $("form").submit(function(){ 36 | var self = $(this); 37 | self.find(".Validform_checktip").hide(); 38 | $.post(self.attr("action"), self.serialize(), success, "json"); 39 | return false; 40 | 41 | function success(data){ 42 | if(data.status){ 43 | window.location.href = data.url; 44 | } else { 45 | $('form')[0].reset(); 46 | self.find(".Validform_checktip").html(data.info).show(); 47 | } 48 | } 49 | }); 50 | {% endblock %} -------------------------------------------------------------------------------- /accounts/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /accounts/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url 2 | from . import views 3 | import django.contrib.auth.views 4 | 5 | app_name = "accounts" 6 | 7 | urlpatterns = [ 8 | url(r'^login/$', views.user_login, name='login'), 9 | url(r'^logout/$', django.contrib.auth.views.logout_then_login, name='logout'), 10 | url(r'^password-change/$', views.change_password, name='password_change'), 11 | url(r'^register/$', views.register, name='register'), 12 | ] 13 | -------------------------------------------------------------------------------- /accounts/views.py: -------------------------------------------------------------------------------- 1 | from django.shortcuts import render 2 | from .forms import LoginForm, PasswordChangeForm, UserRegisterForm 3 | from django.contrib.auth import authenticate, login, update_session_auth_hash 4 | from ui.tools import success, error 5 | from django.contrib.auth.decorators import login_required 6 | from django.contrib.auth.models import Group 7 | 8 | 9 | def user_login(request): 10 | if request.method == 'POST': 11 | form = LoginForm(request.POST) 12 | if form.is_valid(): 13 | cd = form.cleaned_data 14 | user = authenticate(username=cd['username'], password=cd['password']) 15 | 16 | if user is not None: 17 | if user.is_active: 18 | login(request, user) 19 | if "next" in request.POST.keys(): 20 | red_url = request.POST["next"] 21 | if red_url == "": 22 | red_url = "/ui" 23 | else: 24 | red_url = "/ui" 25 | return success('Authenticated successfully', red_url) 26 | else: 27 | return error('Disabled account') 28 | else: 29 | return error('Wrong username or password') 30 | else: 31 | return error(str(form.errors)) 32 | else: 33 | form = LoginForm() 34 | if "next" in request.GET.keys(): 35 | return render(request, 'accounts/login.html', {'form': form, 'next': request.GET["next"]}) 36 | else: 37 | return render(request, 'accounts/login.html', {'form': form}) 38 | 39 | 40 | def register(request): 41 | if request.method == 'POST': 42 | user_form = UserRegisterForm(request.POST) 43 | if user_form.is_valid(): 44 | new_user = user_form.save(commit=False) 45 | new_user.set_password( 46 | user_form.cleaned_data['password'] 47 | ) 48 | new_user.is_active = False 49 | new_user.save() 50 | try: 51 | group = Group.objects.get(name="normal") 52 | new_user.groups.add(group) 53 | except Exception as e: 54 | return error(e) 55 | return success('Your account has been successfully created.', '/accounts/login') 56 | else: 57 | return error(str(user_form.errors)) 58 | else: 59 | user_form = UserRegisterForm() 60 | return render(request, 'accounts/register.html', {'form': user_form}) 61 | 62 | 63 | @login_required 64 | def change_password(request): 65 | if request.method == 'POST': 66 | form = PasswordChangeForm(request.POST) 67 | if form.is_valid(): 68 | username = request.user.username 69 | cd = form.cleaned_data 70 | user = authenticate(username=username, password=cd['old_password']) 71 | if user is not None and user.is_active: 72 | new_password = cd['new_password_1'] 73 | user.set_password(new_password) 74 | user.save() 75 | update_session_auth_hash(request, username) 76 | return success('Your password had been updated.', 'accounts:login') 77 | else: 78 | return error('Password doesn\'t match.') 79 | else: 80 | return error('Your form is illegal.') 81 | else: 82 | return error('Please confirm your approaching method.') 83 | -------------------------------------------------------------------------------- /config/bioqueue.conf: -------------------------------------------------------------------------------- 1 | [program] 2 | latest_version = https://raw.githubusercontent.com/liyao001/BioQueue/master/version 3 | api = http://open.bioqueue.org/index.php/Api 4 | tool_repo = https://raw.githubusercontent.com/BioQueue/tools/master/{query} 5 | tool_repo_search_api = https://api.github.com/search/code?q={query}+repo:BioQueue/tools 6 | ref_repo = https://raw.githubusercontent.com/BioQueue/references/master/{query} 7 | ref_repo_search_api = https://api.github.com/search/code?q={query}+repo:BioQueue/references -------------------------------------------------------------------------------- /config/custom.conf: -------------------------------------------------------------------------------- 1 | [env] 2 | workspace = 3 | log = 4 | outputs = 5 | batch_job = 6 | cpu = 7 | memory = 8 | disk_quota = 9 | secret_key = 10 | feedback = 11 | max_job = 12 | 13 | [ml] 14 | trainstore = 15 | imgstore = 16 | confidence_weight_disk = 1 17 | confidence_weight_mem = 1 18 | confidence_weight_cpu = 0.8 19 | threshold = 0.5 20 | 21 | [cluster] 22 | type = 23 | cpu = 24 | queue = 25 | mem = 26 | vrt = 27 | walltime = 28 | 29 | [mail] 30 | notify = off 31 | sender = 32 | mail_host = 33 | mail_port = 34 | mail_user = 35 | mail_password = 36 | 37 | -------------------------------------------------------------------------------- /config/file_support.conf: -------------------------------------------------------------------------------- 1 | [generic] 2 | Compress/Decompress=gg 3 | 4 | [fastq] 5 | Quality Report=fastqc 6 | 7 | [bam] 8 | Sort=bam_sort 9 | Index=bam_index -------------------------------------------------------------------------------- /deploy/000-default.conf.tpl: -------------------------------------------------------------------------------- 1 | 2 | # The ServerName directive sets the request scheme, hostname and port that 3 | # the server uses to identify itself. This is used when creating 4 | # redirection URLs. In the context of virtual hosts, the ServerName 5 | # specifies what hostname must appear in the request's Host: header to 6 | # match this virtual host. For the default virtual host (this file) this 7 | # value is not decisive as it is used as a last resort host regardless. 8 | # However, you must set it for any further virtual host explicitly. 9 | #ServerName www.example.com 10 | 11 | ServerAdmin webmaster@localhost 12 | 13 | # Available loglevels: trace8, ..., trace1, debug, info, notice, warn, 14 | # error, crit, alert, emerg. 15 | # It is also possible to configure the loglevel for particular 16 | # modules, e.g. 17 | #LogLevel info ssl:warn 18 | Alias /static {APP_ROOT}/ui/static 19 | 20 | Require all granted 21 | 22 | ErrorLog ${APACHE_LOG_DIR}/error.log 23 | CustomLog ${APACHE_LOG_DIR}/access.log combined 24 | 25 | # For most configuration files from conf-available/, which are 26 | # enabled or disabled at a global level, it is possible to 27 | # include a line for only one particular virtual host. For example the 28 | # following line enables the CGI configuration for this host only 29 | # after it has been globally disabled with "a2disconf". 30 | #Include conf-available/serve-cgi-bin.conf 31 | 32 | 33 | Require all granted 34 | 35 | 36 | WSGIDaemonProcess BioQueue python-path=/usr/bin/python:/usr/lib/python2.7/dist-packages 37 | WSGIProcessGroup BioQueue 38 | WSGIScriptAlias / {APP_ROOT}/BioQueue/wsgi.py 39 | 40 | 41 | # vim: syntax=apache ts=4 sw=4 sts=4 sr noet 42 | -------------------------------------------------------------------------------- /deploy/mysql.tpl: -------------------------------------------------------------------------------- 1 | 'default': { 2 | 'ENGINE': 'django.db.backends.mysql', 3 | 'NAME': '{DB_NAME}', 4 | 'USER': '{DB_USER}', 5 | 'PASSWORD': '{DB_PASSWORD}', 6 | 'HOST': '{DB_HOST}', 7 | 'PORT': '{DB_PORT}', 8 | }, -------------------------------------------------------------------------------- /deploy/postgresql.tpl: -------------------------------------------------------------------------------- 1 | 'default': { 2 | 'ENGINE': 'django.db.backends.postgresql_psycopg2', 3 | 'NAME': '{DB_NAME}', 4 | 'USER': '{DB_USER}', 5 | 'PASSWORD': '{DB_PASSWORD}', 6 | 'HOST': '{DB_HOST}', 7 | 'PORT': '{DB_PORT}', 8 | }, -------------------------------------------------------------------------------- /deploy/prerequisites.txt: -------------------------------------------------------------------------------- 1 | Django==3.2.4 2 | numpy 3 | psutil 4 | pytz 5 | scipy 6 | sqlparse 7 | python-magic 8 | prettydiff 9 | 10 | -------------------------------------------------------------------------------- /deploy/python_pip_non_root.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | echo "Downloading source code" 3 | wget https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tgz 4 | echo "Decompressing" 5 | tar -xzf Python-2.7.13.tgz 6 | cd Python-2.7.13 7 | mkdir -p $HOME/bin 8 | echo "Configuring" 9 | ./configure --prefix="$HOME" 10 | echo "Making" 11 | make 12 | echo "Installing" 13 | make install 14 | export PATH=$HOME/bin:$PATH 15 | echo "Downloading pip" 16 | wget https://bootstrap.pypa.io/get-pip.py 17 | if [ $? -ne 0 ]; then 18 | echo "Cannot download get-pip.py, now try another source" 19 | wget --no-check-certificate https://raw.githubusercontent.com/BioQueue/dependent-packages-backup/master/get-pip.py 20 | if [ $? -ne 0 ]; then 21 | echo "Cannot download get-pip.py, now try another source" 22 | wget http://bioqueue.nefu.edu.cn/get-pip.py 23 | if [ $? -ne 0 ]; then 24 | echo "Cannot download get-pip.py, please download it manually" 25 | exit 26 | fi 27 | fi 28 | fi 29 | $HOME/bin/python get-pip.py 30 | echo "" 31 | echo "Now please add the bin directory to your PATH environment variable for quicker access." 32 | echo "For example, if you use the Bash shell on Unix, you could place this in your " 33 | echo "~/.bash_profile file (assuming you installed into /home/your_name/bin):" 34 | echo "" 35 | echo "export PATH=/home/your_name/bin:\$PATH" 36 | echo "" 37 | echo "then save the .bash_profile file and run" 38 | echo "" 39 | echo "source ~/.bash_profile" 40 | echo "" -------------------------------------------------------------------------------- /deploy/sqlite.tpl: -------------------------------------------------------------------------------- 1 | 'default': { 2 | 'ENGINE': 'django.db.backends.sqlite3', 3 | 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 4 | }, -------------------------------------------------------------------------------- /deploy/status_code.md: -------------------------------------------------------------------------------- 1 | ## BioQueue Status Code 2 | 3 | This file describes status code used in BioQueue. 4 | 5 | | status code| code description | 6 | |:----------:|:-----------------------------------:| 7 | | 0 | Waiting, not in job queue | 8 | | >0 | Running, the number is the n-th step| 9 | | -1 | Completed | 10 | | -2 | Waiting at check point | 11 | | -3 | Error occured | 12 | 13 | | learning code | code description | 14 | |:-------------:|:----------------:| 15 | | 1 | Disk usage | 16 | | 2 | Memory usage | 17 | | 3 | CPU usage | -------------------------------------------------------------------------------- /deploy/switch_from_MySQLdb_to_PyMySQL.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2017/4/26 10:04 4 | # @Project : BioQueue 5 | # @Author : Li Yao 6 | # @File : switch_from_MySQLdb_to_PyMySQL.py 7 | import os 8 | 9 | # remove from prerequisites 10 | common_suffix = os.path.split(os.path.realpath(__file__))[0] 11 | pip_import_path = common_suffix + '/prerequisites.txt' 12 | pip_import_fallback_path = common_suffix + '/prerequisites.fallback.txt' 13 | pip_list_handler = open(pip_import_path, 'r') 14 | pip_list = pip_list_handler.read() 15 | pip_list = pip_list.replace('MySQL-python==1.2.5', 'PyMySQL') 16 | pip_list_handler.close() 17 | pip_list_handler = open(pip_import_fallback_path, 'w') 18 | pip_list_handler.write(pip_list) 19 | pip_list_handler.close() 20 | 21 | # switch to PyMySQL 22 | import_string = """ 23 | try: 24 | import pymysql 25 | pymysql.install_as_MySQLdb() 26 | except ImportError: 27 | pass 28 | """ 29 | manage_py_path = os.path.split(os.path.split(os.path.realpath(__file__))[0])[0] + '/manage.py' 30 | mpp_handler = open(manage_py_path, 'r') 31 | mpp = mpp_handler.read() 32 | mpp = mpp.replace('import sys\n', 'import sys\n' + import_string) 33 | mpp_handler.close() 34 | mpp_handler = open(manage_py_path, 'w') 35 | mpp_handler.write(mpp) 36 | mpp_handler.close() 37 | 38 | worker_init_py_path = os.path.split(os.path.split(os.path.realpath(__file__))[0])[0] + '/worker/__init__.py' 39 | wi_handler = open(worker_init_py_path, 'w') 40 | wi_handler.write(import_string) 41 | wi_handler.close() -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "BioQueue.settings") 7 | try: 8 | from django.core.management import execute_from_command_line 9 | except ImportError: 10 | # The above import may fail for some other reason. Ensure that the 11 | # issue is really that Django is missing to avoid masking other 12 | # exceptions on Python 2. 13 | try: 14 | import django 15 | except ImportError: 16 | raise ImportError( 17 | "Couldn't import Django. Are you sure it's installed and " 18 | "available on your PYTHONPATH environment variable? Did you " 19 | "forget to activate a virtual environment?" 20 | ) 21 | raise 22 | execute_from_command_line(sys.argv) 23 | -------------------------------------------------------------------------------- /protocol_dumps/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/protocol_dumps/__init__.py -------------------------------------------------------------------------------- /status_page.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/status_page.png -------------------------------------------------------------------------------- /templates/admin/login.html: -------------------------------------------------------------------------------- 1 | {% extends "ui/single.html" %} 2 | {% block title %}Sign In | BioQueue{% endblock %} 3 | {% block content %} 4 | 5 |
6 |
7 |

Login

8 | 21 |
22 |
23 | {% endblock %} 24 | {% block script %} 25 | $(document) 26 | .ajaxStart(function(){ 27 | $("button:submit").addClass("log-in").attr("disabled", true); 28 | }) 29 | .ajaxStop(function(){ 30 | $("button:submit").removeClass("log-in").attr("disabled", false); 31 | }); 32 | {% endblock %} -------------------------------------------------------------------------------- /templates/registration/logged_out.html: -------------------------------------------------------------------------------- 1 | {% extends "ui/single.html" %} 2 | {% load i18n %} 3 | 4 | {% block breadcrumbs %}{% endblock %} 5 | 6 | {% block content %} 7 | 8 |

{% trans "Thanks for spending some quality time with the Web site today." %}

9 | 10 |

{% trans 'Log in again' %}

11 | 12 | {% endblock %} 13 | -------------------------------------------------------------------------------- /ui/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/ui/__init__.py -------------------------------------------------------------------------------- /ui/admin.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Author: Li Yao 4 | # @Date: 12/01/16 5 | from django.contrib import admin 6 | from QueueDB.models import Job, Training, Prediction, Step, ProtocolList, VirtualEnvironment, Experiment, \ 7 | Audition, Sample, Profile, FileArchive, Reference, Workspace #, GoogleDriveConnection 8 | 9 | ''' 10 | class QueueInline(admin.StackedInline): 11 | model = ProtocolList 12 | ''' 13 | 14 | 15 | class QueueAdmin(admin.ModelAdmin): 16 | # inlines = [QueueInline] 17 | list_display = ('id', 'job_name', 'protocol', 'status', 'result', 'create_time') 18 | search_fields = ('result', 'job_name',) 19 | 20 | 21 | class TrainingAdmin(admin.ModelAdmin): 22 | list_display = ('id', 'step_name', 'input', 'output', 'mem_in_gb', 'vrt_mem_in_gb', 'cpu', 'create_time') 23 | 24 | 25 | class PredictionAdmin(admin.ModelAdmin): 26 | list_display = ('id', 'step_name', 'step_hash', 'a', 'b', 'r', 'type') 27 | 28 | 29 | class ResourceAdmin(admin.ModelAdmin): 30 | list_display = ('cpu', 'mem', 'disk', 'lock') 31 | 32 | 33 | class ProtocolAdmin(admin.ModelAdmin): 34 | list_display = ('software', 'parameter', 'specify_output', 'parent') 35 | 36 | 37 | class ProtocolListAdmin(admin.ModelAdmin): 38 | list_display = ('name', 'user_id') 39 | 40 | 41 | class FileArchiveAdmin(admin.ModelAdmin): 42 | list_display = ('protocol', 'protocol_ver', 'create_time', 'shared_with') 43 | 44 | 45 | class ReferenceAdmin(admin.ModelAdmin): 46 | list_display = ('name', 'user_id') 47 | 48 | 49 | class AuditionAdmin(admin.ModelAdmin): 50 | list_display = ('id', 'operation', 'job_name', 'create_time') 51 | readonly_fields = ['id', 'operation', 'related_job', 'job_name', 'prev_par', 52 | 'new_par', 'prev_input', 'current_input', 'protocol', 53 | 'protocol_ver', 'resume_point', 'create_time', 'user'] 54 | 55 | 56 | admin.site.register(Job, QueueAdmin) 57 | admin.site.register(Prediction, PredictionAdmin) 58 | admin.site.register(Training, TrainingAdmin) 59 | admin.site.register(ProtocolList, ProtocolListAdmin) 60 | admin.site.register(Step, ProtocolAdmin) 61 | admin.site.register(VirtualEnvironment) 62 | admin.site.register(Experiment) 63 | admin.site.register(Sample) 64 | admin.site.register(Profile) 65 | admin.site.register(FileArchive, FileArchiveAdmin) 66 | admin.site.register(Reference, ReferenceAdmin) 67 | admin.site.register(Workspace) 68 | # admin.site.register(GoogleDriveConnection) 69 | admin.site.register(Audition, AuditionAdmin) 70 | -------------------------------------------------------------------------------- /ui/apps.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from django.apps import AppConfig 4 | 5 | 6 | class UiConfig(AppConfig): 7 | name = 'ui' 8 | -------------------------------------------------------------------------------- /ui/cron_runner.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | # @Author: Li Yao 4 | # @Date: 01/28/20 5 | import time 6 | import threading 7 | import logging 8 | import os 9 | import sys 10 | from django.core.wsgi import get_wsgi_application 11 | 12 | sys.path.append(os.path.split(os.path.split(os.path.realpath(__file__))[0])[0]) 13 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "BioQueue.settings") 14 | 15 | application = get_wsgi_application() 16 | from ui.cron_jobs import archive_job_manager, audit_job_manager, audit_archiving_file_manager 17 | 18 | DEFAULT_PREFIX = str(os.getpid()) 19 | logging.basicConfig(format='%(name)s - %(asctime)s - %(levelname)s: %(message)s', 20 | datefmt='%d-%b-%y %H:%M:%S', 21 | level=logging.INFO, 22 | handlers=[ 23 | logging.FileHandler(os.path.join(os.getcwd(), 'cron_%s.log' % DEFAULT_PREFIX)), 24 | logging.StreamHandler() 25 | ]) 26 | logger = logging.getLogger("BioQueue - Cron") 27 | 28 | 29 | def aj_cron(logger): 30 | while True: 31 | logger.info("Cron - File archive") 32 | archive_job_manager(logger) 33 | logger.info("Cron - Done - File archive") 34 | time.sleep(300) 35 | 36 | 37 | def pv_cron(logger): 38 | while True: 39 | logger.info("Cron - Job audit") 40 | audit_job_manager(logger) 41 | logger.info("Cron - Done - Job audit") 42 | time.sleep(3000) 43 | 44 | 45 | def af_cron(logger): 46 | while True: 47 | logger.info("Cron - Archived file audit") 48 | audit_archiving_file_manager(logger) 49 | logger.info("Cron - Done - Archived file audit") 50 | time.sleep(3000) 51 | 52 | 53 | def init_cron(function, args): 54 | t = threading.Thread(target=function, args=args) 55 | t.setDaemon(True) 56 | t.start() 57 | 58 | 59 | if __name__ == "__main__": 60 | init_cron(aj_cron, (logger, )) 61 | init_cron(pv_cron, (logger, )) 62 | init_cron(af_cron, (logger, )) 63 | 64 | while True: 65 | time.sleep(30) 66 | -------------------------------------------------------------------------------- /ui/ena.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | # @Author: Li Yao 4 | # @Date: 05/01/20 5 | import requests 6 | import json 7 | 8 | 9 | def get_download_link(acc, field="fastq_ftp"): 10 | """ 11 | Get download link from EBI ENA 12 | :param acc: str 13 | :param field: str 14 | :return: 15 | """ 16 | links = list() 17 | # https://www.ebi.ac.uk/ena/submit/read-data-format 18 | if acc[2] == "A": 19 | keyword = "submission_accession" 20 | elif acc[2] == "S": 21 | keyword = "sample_accession" 22 | elif acc[2] == "P": 23 | keyword = "study_accession" 24 | elif acc[2] == "X": 25 | keyword = "experiment_accession" 26 | elif acc[2] == "R": 27 | keyword = "run_accession" 28 | else: 29 | print("Unsupported accession type %s" % acc) 30 | return [] 31 | api_bus = 'https://www.ebi.ac.uk/ena/portal/api/search?result=read_run&query="%s=%s"&fields=%s&format=json' % (keyword, acc, field) 32 | # res_data = urlopen(api_bus) 33 | # ret = res_data.read() 34 | try: 35 | response = requests.get(api_bus) 36 | if response.ok: 37 | res = response.json() 38 | for run in res: 39 | links.extend(run[field].split(";")) 40 | return links 41 | else: 42 | return [] 43 | except: 44 | return [] 45 | 46 | 47 | def get_accession(query): 48 | """ 49 | Get accession ID from EBI Search 50 | :param query: 51 | :return: 52 | """ 53 | api_bus = "http://www.ebi.ac.uk/ebisearch/ws/rest/nucleotideSequences?query=%s&fieldurl=true&viewurl=true&format=json" % query 54 | sra_list = list() 55 | try: 56 | response = requests.get(api_bus) 57 | if response.ok: 58 | res = response.json() 59 | else: 60 | return 0 61 | if res["hitCount"] == 0: 62 | return 0 63 | for entry in res["entries"]: 64 | sra_list.append(entry["id"]) 65 | return sra_list 66 | except Exception as e: 67 | print(e) 68 | return 0 69 | 70 | 71 | def query_download_link_from_ebi(query): 72 | """ 73 | Query download link from ebi 74 | :param query: str 75 | :return: list 76 | """ 77 | all_links = list() 78 | fl = get_accession(query) 79 | if type(fl) != int: 80 | for r in fl: 81 | tmp = get_download_link(r) 82 | if len(tmp) > 0: 83 | all_links.extend(tmp) 84 | ret_links = ["ftp://"+link for link in all_links] 85 | return sorted(list(set(ret_links))) 86 | else: 87 | return [] 88 | 89 | 90 | # if __name__ == "__main__": 91 | # print(query_download_link_from_ebi("GSM3318211")) 92 | -------------------------------------------------------------------------------- /ui/extmodels/GoogleDrive.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Author: Li Yao 4 | # @Date: 12/29/20 5 | # 6 | # BioQueue is free for personal use and is licensed under 7 | # the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, 9 | # or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 14 | 15 | from django.db import models 16 | from django.contrib.auth.models import User 17 | 18 | 19 | class GoogleDriveConnection(models.Model): 20 | user = models.ForeignKey(User, on_delete=models.PROTECT) 21 | credential_pickle = models.CharField(max_length=255) 22 | folder_id = models.CharField(max_length=255) 23 | -------------------------------------------------------------------------------- /ui/extmodels/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | # Created by: Li Yao (yaoli.gm@gmail.com) 4 | # Created on: 12/27/20 5 | from .RunOnQC import * 6 | -------------------------------------------------------------------------------- /ui/maintenance_protocols/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/ui/maintenance_protocols/__init__.py -------------------------------------------------------------------------------- /ui/maintenance_protocols/bam_index.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 14/01/2018 01:04 AM 4 | # @Project : BioQueue 5 | # @Author : Li Yao 6 | # @File : bam_index.py 7 | 8 | 9 | def get_sub_protocol(db_obj, protocol_parent, step_order_start=1): 10 | steps = list() 11 | steps.append(db_obj(software='samtools', 12 | parameter='index {{InputFile}}.sorted.bam', 13 | parent=protocol_parent, 14 | user_id=0, 15 | hash='030fdbd05e08e64d7b5df6db763a3687', 16 | step_order=step_order_start)) 17 | return step_order_start+len(steps), steps 18 | -------------------------------------------------------------------------------- /ui/maintenance_protocols/bam_sort.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 14/01/2018 01:04 AM 4 | # @Project : BioQueue 5 | # @Author : Li Yao 6 | # @File : bam_sort.py 7 | 8 | 9 | def get_sub_protocol(db_obj, protocol_parent, step_order_start=1): 10 | steps = list() 11 | steps.append(db_obj(software='samtools', 12 | parameter='sort -@ {{ThreadN}} -o sorted.bam {{InputFile}}', 13 | parent=protocol_parent, 14 | user_id=0, 15 | hash='1c676531136f2dbd035894eb8a36e2e1', 16 | step_order=step_order_start)) 17 | steps.append(db_obj(software='samtools', 18 | parameter='index sorted.bam', 19 | parent=protocol_parent, 20 | user_id=0, 21 | hash='cb09b58adf333beb914e95e3a271fac3', 22 | step_order=step_order_start + 1)) 23 | return step_order_start+len(steps), steps 24 | -------------------------------------------------------------------------------- /ui/maintenance_protocols/compress.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 14/01/2018 01:04 AM 4 | # @Project : BioQueue 5 | # @Author : Li Yao 6 | # @File : gunzip.py 7 | 8 | 9 | def get_sub_protocol(db_obj, protocol_parent, step_order_start=1): 10 | steps = list() 11 | steps.append(db_obj(software='gzip', 12 | parameter='{{InputFile}}', 13 | parent=protocol_parent, 14 | user_id=0, 15 | hash='fe5d52008fdd08800b24354df032b07c', 16 | step_order=step_order_start)) 17 | return step_order_start+len(steps), steps 18 | -------------------------------------------------------------------------------- /ui/maintenance_protocols/configure_make_install.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 29/12/2017 10:05 AM 4 | # @Project : main 5 | # @Author : Li Yao 6 | # @File : configure_make_install.py 7 | 8 | 9 | def get_sub_protocol(db_obj, protocol_parent, step_order_start=1): 10 | steps = list() 11 | steps.append(db_obj(software='./configure', 12 | parameter='--prefix {{UserBin}}', 13 | parent=protocol_parent, 14 | user_id=0, 15 | hash='dfca5277f71c6782e3351f6ed9ac7fcb', 16 | step_order=step_order_start)) 17 | steps.append(db_obj(software='make', 18 | parameter='', 19 | parent=protocol_parent, 20 | user_id=0, 21 | hash='099dafc678df7d266c25f95ccf6cde22', 22 | step_order=step_order_start+1)) 23 | steps.append(db_obj(software='make', 24 | parameter='install', 25 | parent=protocol_parent, 26 | user_id=0, 27 | hash='12b64827119f4815ca8d43608d228f36', 28 | step_order=step_order_start+2)) 29 | return step_order_start+len(steps), steps -------------------------------------------------------------------------------- /ui/maintenance_protocols/decompress.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 14/01/2018 01:04 AM 4 | # @Project : BioQueue 5 | # @Author : Li Yao 6 | # @File : gunzip.py 7 | 8 | 9 | def get_sub_protocol(db_obj, protocol_parent, step_order_start=1): 10 | steps = list() 11 | steps.append(db_obj(software='gunzip', 12 | parameter='{{InputFile}}', 13 | parent=protocol_parent, 14 | user_id=0, 15 | hash='541df26aff8e4d054a57c7e3717e91ca', 16 | step_order=step_order_start)) 17 | return step_order_start+len(steps), steps 18 | -------------------------------------------------------------------------------- /ui/maintenance_protocols/download.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 02/12/2017 8:47 PM 4 | # @Project : BioQueue 5 | # @Author : Li Yao 6 | # @File : download.py 7 | 8 | 9 | def get_sub_protocol(db_obj, protocol_parent, step_order_start=1): 10 | steps = list() 11 | steps.append(db_obj(software='wget', 12 | parameter='{{InputFile}}', 13 | parent=protocol_parent, 14 | user_id=0, 15 | hash='3efb64d0fa1144993ee287d3233dde06', 16 | step_order=step_order_start, 17 | force_local=1)) 18 | return step_order_start+len(steps), steps 19 | -------------------------------------------------------------------------------- /ui/maintenance_protocols/fastqc.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 19/08/2018 03:04 PM 4 | # @Project : BioQueue 5 | # @Author : Li Yao 6 | # @File : fastqc.py 7 | 8 | 9 | def get_sub_protocol(db_obj, protocol_parent, step_order_start=1): 10 | steps = list() 11 | steps.append(db_obj(software='fastqc', 12 | parameter='-o {{Workspace}} {{InputFile}}', 13 | parent=protocol_parent, 14 | user_id=protocol_parent.user_id, 15 | hash='3a24f9a8e2f78d22e6e310d2694c08c2', 16 | step_order=step_order_start)) 17 | return step_order_start+len(steps), steps 18 | -------------------------------------------------------------------------------- /ui/maintenance_protocols/git.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 02/12/2017 8:56 PM 4 | # @Project : BioQueue 5 | # @Author : Li Yao 6 | # @File : git.py 7 | 8 | 9 | def get_sub_protocol(db_obj, protocol_parent, step_order_start=1): 10 | steps = list() 11 | steps.append(db_obj(software='git', 12 | parameter='clone {{InputFile}}', 13 | parent=protocol_parent, 14 | user_id=0, 15 | hash='f95d6d36a0d0cae61a03704b46f72892', 16 | step_order=step_order_start)) 17 | return step_order_start+len(steps), steps 18 | -------------------------------------------------------------------------------- /ui/maintenance_protocols/gunzip.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 14/01/2018 01:04 AM 4 | # @Project : BioQueue 5 | # @Author : Li Yao 6 | # @File : gunzip.py 7 | 8 | 9 | def get_sub_protocol(db_obj, protocol_parent, step_order_start=1): 10 | steps = list() 11 | steps.append(db_obj(software='gunzip', 12 | parameter='{{LastOutput}}', 13 | parent=protocol_parent, 14 | user_id=0, 15 | hash='0603ff9d39724ecd6d62be4618901b54', 16 | step_order=step_order_start)) 17 | return step_order_start+len(steps), steps 18 | -------------------------------------------------------------------------------- /ui/maintenance_protocols/make.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 02/12/2017 11:59 PM 4 | # @Project : BioQueue 5 | # @Author : Li Yao 6 | # @File : zip.py 7 | 8 | 9 | def get_sub_protocol(db_obj, protocol_parent, step_order_start=1): 10 | steps = list() 11 | steps.append(db_obj(software='make', 12 | parameter='', 13 | parent=protocol_parent, 14 | user_id=0, 15 | hash='099dafc678df7d266c25f95ccf6cde22', 16 | step_order=step_order_start)) 17 | return step_order_start+len(steps), steps 18 | -------------------------------------------------------------------------------- /ui/maintenance_protocols/svn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 02/12/2017 8:57 PM 4 | # @Project : BioQueue 5 | # @Author : Li Yao 6 | # @File : svn.py 7 | 8 | 9 | def get_sub_protocol(db_obj, protocol_parent, step_order_start=1): 10 | steps = list() 11 | steps.append(db_obj(software='svn', 12 | parameter='checkout {{InputFile}}', 13 | parent=protocol_parent, 14 | user_id=0, 15 | hash='c025d53644388a50fb3704b4a81d5a93', 16 | step_order=step_order_start)) 17 | return step_order_start+len(steps), steps 18 | -------------------------------------------------------------------------------- /ui/maintenance_protocols/tarbz2.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 04/12/2017 1:39 PM 4 | # @Project : main 5 | # @Author : Li Yao 6 | # @File : tarbz2.py 7 | def get_sub_protocol(db_obj, protocol_parent, step_order_start=1): 8 | steps = list() 9 | steps.append(db_obj(software='tar', 10 | parameter='-jxvf {{LastOutput}}', 11 | parent=protocol_parent, 12 | user_id=0, 13 | hash='670fb21245414e6d79b43dbd59ae1fb9', 14 | step_order=step_order_start)) 15 | return step_order_start+len(steps), steps -------------------------------------------------------------------------------- /ui/maintenance_protocols/targz.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 04/12/2017 1:39 PM 4 | # @Project : main 5 | # @Author : Li Yao 6 | # @File : targz.py 7 | def get_sub_protocol(db_obj, protocol_parent, step_order_start=1): 8 | steps = list() 9 | steps.append(db_obj(software='tar', 10 | parameter='-zxvf {{LastOutput}}', 11 | parent=protocol_parent, 12 | user_id=0, 13 | hash='ea8e630bc7eccdbe04343351fb6ba886', 14 | step_order=step_order_start)) 15 | return step_order_start+len(steps), steps -------------------------------------------------------------------------------- /ui/maintenance_protocols/zip.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 02/12/2017 11:40 PM 4 | # @Project : BioQueue 5 | # @Author : Li Yao 6 | # @File : zip.py 7 | 8 | 9 | def get_sub_protocol(db_obj, protocol_parent, step_order_start=1): 10 | steps = list() 11 | steps.append(db_obj(software='unzip', 12 | parameter='{{LastOutput}}', 13 | parent=protocol_parent, 14 | user_id=0, 15 | hash='728ab46516121c0215887cd60bcbb8bd', 16 | step_order=step_order_start)) 17 | return step_order_start+len(steps), steps 18 | -------------------------------------------------------------------------------- /ui/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | # Created by: Li Yao 4 | # Created on: 1/28/20 -------------------------------------------------------------------------------- /ui/plugins/google_drive.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | # Created by: Li Yao 4 | # Created on: 1/28/20 5 | import pickle 6 | import os 7 | from googleapiclient.discovery import build 8 | from google_auth_oauthlib.flow import InstalledAppFlow 9 | from google.auth.transport.requests import Request 10 | from googleapiclient.http import MediaFileUpload 11 | 12 | 13 | class GoogleDrive(object): 14 | SCOPES = ['https://www.googleapis.com/auth/drive.file', 'https://www.googleapis.com/auth/drive.metadata.readonly'] 15 | 16 | def __init__(self, token_file="token.pickle", app_credential="client_secret.json"): 17 | creds = None 18 | # The file token.pickle stores the user's access and refresh tokens, and is 19 | # created automatically when the authorization flow completes for the first 20 | # time. 21 | if os.path.exists(token_file): 22 | with open(token_file, 'rb') as token: 23 | creds = pickle.load(token) 24 | # If there are no (valid) credentials available, let the user log in. 25 | if not creds or not creds.valid: 26 | if creds and creds.expired and creds.refresh_token: 27 | creds.refresh(Request()) 28 | else: 29 | flow = InstalledAppFlow.from_client_secrets_file( 30 | app_credential, self.SCOPES) 31 | creds = flow.run_local_server(port=0) 32 | # Save the credentials for the next run 33 | with open(token_file, 'wb') as token: 34 | pickle.dump(creds, token) 35 | self.service = build('drive', 'v3', credentials=creds) 36 | 37 | def create_folder(self, folder_name): 38 | file_metadata = { 39 | 'name': folder_name, 40 | 'mimeType': 'application/vnd.google-apps.folder', 41 | } 42 | file = self.service.files().create(body=file_metadata, fields='id').execute() 43 | folder_id = file.get('id') 44 | return folder_id 45 | 46 | def already_folder(self, folder_name): 47 | results = self.service.files().list( 48 | q="mimeType='application/vnd.google-apps.folder' and name='" + folder_name + "'", 49 | fields="nextPageToken, files(id, name)").execute() 50 | items = results.get('files', []) 51 | if items: 52 | return True, items[-1]['id'] 53 | else: 54 | return False, "" 55 | 56 | def upload_file(self, file_name, file_path, folder_id, description='', mimetype='application/x-compressed'): 57 | file_metadata = { 58 | 'name': file_name, 59 | 'parents': [folder_id], 60 | 'description': description, 61 | 62 | } 63 | media = MediaFileUpload(file_path, 64 | mimetype=mimetype, 65 | resumable=True) 66 | file = self.service.files().create(body=file_metadata, 67 | media_body=media, 68 | fields='id').execute() 69 | file_id = file.get('id') 70 | return file_id 71 | 72 | def update_file(self, file_id, new_file, new_mimetype='application/x-compressed'): 73 | try: 74 | file = self.service.files().get(fileId=file_id).execute() 75 | file['mimeType'] = new_mimetype 76 | 77 | media_body = MediaFileUpload(new_file, mimetype=new_mimetype, resumable=True) 78 | updated_file = self.service.files().update( 79 | fileId=file_id, 80 | body=file, 81 | new_revision=True, 82 | media_body=media_body 83 | ).execute() 84 | return updated_file.get('id') 85 | except Exception as e: 86 | print(e) 87 | return None 88 | 89 | def share_with_person(self, file_id, share_with, msg='', permission='reader'): 90 | def callback(request_id, response, exception): 91 | if exception: 92 | # Handle error 93 | print(exception) 94 | else: 95 | print("Permission Id: %s" % response.get('id')) 96 | 97 | batch = self.service.new_batch_http_request(callback=callback) 98 | for user in share_with: 99 | user_permission = { 100 | 'type': 'user', 101 | 'role': permission, 102 | 'emailAddress': user 103 | } 104 | batch.add(self.service.permissions().create( 105 | fileId=file_id, 106 | body=user_permission, 107 | fields='id', 108 | emailMessage=msg, 109 | )) 110 | res = batch.execute() 111 | return res.get('id') 112 | -------------------------------------------------------------------------------- /ui/static/admin/css/dashboard.css: -------------------------------------------------------------------------------- 1 | /* DASHBOARD */ 2 | 3 | .dashboard .module table th { 4 | width: 100%; 5 | } 6 | 7 | .dashboard .module table td { 8 | white-space: nowrap; 9 | } 10 | 11 | .dashboard .module table td a { 12 | display: block; 13 | padding-right: .6em; 14 | } 15 | 16 | /* RECENT ACTIONS MODULE */ 17 | 18 | .module ul.actionlist { 19 | margin-left: 0; 20 | } 21 | 22 | ul.actionlist li { 23 | list-style-type: none; 24 | } 25 | 26 | ul.actionlist li { 27 | overflow: hidden; 28 | text-overflow: ellipsis; 29 | -o-text-overflow: ellipsis; 30 | } 31 | -------------------------------------------------------------------------------- /ui/static/admin/css/fonts.css: -------------------------------------------------------------------------------- 1 | @font-face { 2 | font-family: 'Roboto'; 3 | src: url('../fonts/Roboto-Bold-webfont.woff'); 4 | font-weight: 700; 5 | font-style: normal; 6 | } 7 | 8 | @font-face { 9 | font-family: 'Roboto'; 10 | src: url('../fonts/Roboto-Regular-webfont.woff'); 11 | font-weight: 400; 12 | font-style: normal; 13 | } 14 | 15 | @font-face { 16 | font-family: 'Roboto'; 17 | src: url('../fonts/Roboto-Light-webfont.woff'); 18 | font-weight: 300; 19 | font-style: normal; 20 | } 21 | -------------------------------------------------------------------------------- /ui/static/admin/css/login.css: -------------------------------------------------------------------------------- 1 | /* LOGIN FORM */ 2 | 3 | body.login { 4 | background: #f8f8f8; 5 | } 6 | 7 | .login #header { 8 | height: auto; 9 | padding: 5px 16px; 10 | } 11 | 12 | .login #header h1 { 13 | font-size: 18px; 14 | } 15 | 16 | .login #header h1 a { 17 | color: #fff; 18 | } 19 | 20 | .login #content { 21 | padding: 20px 20px 0; 22 | } 23 | 24 | .login #container { 25 | background: #fff; 26 | border: 1px solid #eaeaea; 27 | border-radius: 4px; 28 | overflow: hidden; 29 | width: 28em; 30 | min-width: 300px; 31 | margin: 100px auto; 32 | } 33 | 34 | .login #content-main { 35 | width: 100%; 36 | } 37 | 38 | .login .form-row { 39 | padding: 4px 0; 40 | float: left; 41 | width: 100%; 42 | border-bottom: none; 43 | } 44 | 45 | .login .form-row label { 46 | padding-right: 0.5em; 47 | line-height: 2em; 48 | font-size: 1em; 49 | clear: both; 50 | color: #333; 51 | } 52 | 53 | .login .form-row #id_username, .login .form-row #id_password { 54 | clear: both; 55 | padding: 8px; 56 | width: 100%; 57 | -webkit-box-sizing: border-box; 58 | -moz-box-sizing: border-box; 59 | box-sizing: border-box; 60 | } 61 | 62 | .login span.help { 63 | font-size: 10px; 64 | display: block; 65 | } 66 | 67 | .login .submit-row { 68 | clear: both; 69 | padding: 1em 0 0 9.4em; 70 | margin: 0; 71 | border: none; 72 | background: none; 73 | text-align: left; 74 | } 75 | 76 | .login .password-reset-link { 77 | text-align: center; 78 | } 79 | -------------------------------------------------------------------------------- /ui/static/admin/css/rtl.css: -------------------------------------------------------------------------------- 1 | body { 2 | direction: rtl; 3 | } 4 | 5 | /* LOGIN */ 6 | 7 | .login .form-row { 8 | float: right; 9 | } 10 | 11 | .login .form-row label { 12 | float: right; 13 | padding-left: 0.5em; 14 | padding-right: 0; 15 | text-align: left; 16 | } 17 | 18 | .login .submit-row { 19 | clear: both; 20 | padding: 1em 9.4em 0 0; 21 | } 22 | 23 | /* GLOBAL */ 24 | 25 | th { 26 | text-align: right; 27 | } 28 | 29 | .module h2, .module caption { 30 | text-align: right; 31 | } 32 | 33 | .module ul, .module ol { 34 | margin-left: 0; 35 | margin-right: 1.5em; 36 | } 37 | 38 | .addlink, .changelink { 39 | padding-left: 0; 40 | padding-right: 16px; 41 | background-position: 100% 1px; 42 | } 43 | 44 | .deletelink { 45 | padding-left: 0; 46 | padding-right: 16px; 47 | background-position: 100% 1px; 48 | } 49 | 50 | .object-tools { 51 | float: left; 52 | } 53 | 54 | thead th:first-child, 55 | tfoot td:first-child { 56 | border-left: none; 57 | } 58 | 59 | /* LAYOUT */ 60 | 61 | #user-tools { 62 | right: auto; 63 | left: 0; 64 | text-align: left; 65 | } 66 | 67 | div.breadcrumbs { 68 | text-align: right; 69 | } 70 | 71 | #content-main { 72 | float: right; 73 | } 74 | 75 | #content-related { 76 | float: left; 77 | margin-left: -300px; 78 | margin-right: auto; 79 | } 80 | 81 | .colMS { 82 | margin-left: 300px; 83 | margin-right: 0; 84 | } 85 | 86 | /* SORTABLE TABLES */ 87 | 88 | table thead th.sorted .sortoptions { 89 | float: left; 90 | } 91 | 92 | thead th.sorted .text { 93 | padding-right: 0; 94 | padding-left: 42px; 95 | } 96 | 97 | /* dashboard styles */ 98 | 99 | .dashboard .module table td a { 100 | padding-left: .6em; 101 | padding-right: 16px; 102 | } 103 | 104 | /* changelists styles */ 105 | 106 | .change-list .filtered table { 107 | border-left: none; 108 | border-right: 0px none; 109 | } 110 | 111 | #changelist-filter { 112 | right: auto; 113 | left: 0; 114 | border-left: none; 115 | border-right: none; 116 | } 117 | 118 | .change-list .filtered .results, .change-list .filtered .paginator, .filtered #toolbar, .filtered div.xfull { 119 | margin-right: 0; 120 | margin-left: 280px; 121 | } 122 | 123 | #changelist-filter li.selected { 124 | border-left: none; 125 | padding-left: 10px; 126 | margin-left: 0; 127 | border-right: 5px solid #eaeaea; 128 | padding-right: 10px; 129 | margin-right: -15px; 130 | } 131 | 132 | .filtered .actions { 133 | margin-left: 280px; 134 | margin-right: 0; 135 | } 136 | 137 | #changelist table tbody td:first-child, #changelist table tbody th:first-child { 138 | border-right: none; 139 | border-left: none; 140 | } 141 | 142 | /* FORMS */ 143 | 144 | .aligned label { 145 | padding: 0 0 3px 1em; 146 | float: right; 147 | } 148 | 149 | .submit-row { 150 | text-align: left 151 | } 152 | 153 | .submit-row p.deletelink-box { 154 | float: right; 155 | } 156 | 157 | .submit-row input.default { 158 | margin-left: 0; 159 | } 160 | 161 | .vDateField, .vTimeField { 162 | margin-left: 2px; 163 | } 164 | 165 | .aligned .form-row input { 166 | margin-left: 5px; 167 | } 168 | 169 | form ul.inline li { 170 | float: right; 171 | padding-right: 0; 172 | padding-left: 7px; 173 | } 174 | 175 | input[type=submit].default, .submit-row input.default { 176 | float: left; 177 | } 178 | 179 | fieldset .field-box { 180 | float: right; 181 | margin-left: 20px; 182 | margin-right: 0; 183 | } 184 | 185 | .errorlist li { 186 | background-position: 100% 12px; 187 | padding: 0; 188 | } 189 | 190 | .errornote { 191 | background-position: 100% 12px; 192 | padding: 10px 12px; 193 | } 194 | 195 | /* WIDGETS */ 196 | 197 | .calendarnav-previous { 198 | top: 0; 199 | left: auto; 200 | right: 10px; 201 | } 202 | 203 | .calendarnav-next { 204 | top: 0; 205 | right: auto; 206 | left: 10px; 207 | } 208 | 209 | .calendar caption, .calendarbox h2 { 210 | text-align: center; 211 | } 212 | 213 | .selector { 214 | float: right; 215 | } 216 | 217 | .selector .selector-filter { 218 | text-align: right; 219 | } 220 | 221 | .inline-deletelink { 222 | float: left; 223 | } 224 | 225 | form .form-row p.datetime { 226 | overflow: hidden; 227 | } 228 | 229 | /* MISC */ 230 | 231 | .inline-related h2, .inline-group h2 { 232 | text-align: right 233 | } 234 | 235 | .inline-related h3 span.delete { 236 | padding-right: 20px; 237 | padding-left: inherit; 238 | left: 10px; 239 | right: inherit; 240 | float:left; 241 | } 242 | 243 | .inline-related h3 span.delete label { 244 | margin-left: inherit; 245 | margin-right: 2px; 246 | } 247 | 248 | /* IE7 specific bug fixes */ 249 | 250 | div.colM { 251 | position: relative; 252 | } 253 | 254 | .submit-row input { 255 | float: left; 256 | } 257 | -------------------------------------------------------------------------------- /ui/static/admin/fonts/README.txt: -------------------------------------------------------------------------------- 1 | Roboto webfont source: https://www.google.com/fonts/specimen/Roboto 2 | Weights used in this project: Light (300), Regular (400), Bold (700) 3 | -------------------------------------------------------------------------------- /ui/static/admin/fonts/Roboto-Bold-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/ui/static/admin/fonts/Roboto-Bold-webfont.woff -------------------------------------------------------------------------------- /ui/static/admin/fonts/Roboto-Light-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/ui/static/admin/fonts/Roboto-Light-webfont.woff -------------------------------------------------------------------------------- /ui/static/admin/fonts/Roboto-Regular-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/ui/static/admin/fonts/Roboto-Regular-webfont.woff -------------------------------------------------------------------------------- /ui/static/admin/img/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Code Charm Ltd 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /ui/static/admin/img/README.txt: -------------------------------------------------------------------------------- 1 | All icons are taken from Font Awesome (http://fontawesome.io/) project. 2 | The Font Awesome font is licensed under the SIL OFL 1.1: 3 | - http://scripts.sil.org/OFL 4 | 5 | SVG icons source: https://github.com/encharm/Font-Awesome-SVG-PNG 6 | Font-Awesome-SVG-PNG is licensed under the MIT license (see file license 7 | in current folder). 8 | -------------------------------------------------------------------------------- /ui/static/admin/img/calendar-icons.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /ui/static/admin/img/gis/move_vertex_off.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ui/static/admin/img/gis/move_vertex_on.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ui/static/admin/img/icon-addlink.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /ui/static/admin/img/icon-alert.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /ui/static/admin/img/icon-calendar.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /ui/static/admin/img/icon-changelink.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /ui/static/admin/img/icon-clock.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /ui/static/admin/img/icon-deletelink.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /ui/static/admin/img/icon-no.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /ui/static/admin/img/icon-unknown-alt.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /ui/static/admin/img/icon-unknown.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /ui/static/admin/img/icon-yes.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /ui/static/admin/img/inline-delete.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /ui/static/admin/img/search.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /ui/static/admin/img/selector-icons.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /ui/static/admin/img/sorting-icons.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /ui/static/admin/img/tooltag-add.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /ui/static/admin/img/tooltag-arrowright.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /ui/static/admin/js/actions.min.js: -------------------------------------------------------------------------------- 1 | (function(a){var f;a.fn.actions=function(e){var b=a.extend({},a.fn.actions.defaults,e),g=a(this),k=!1,l=function(){a(b.acrossClears).hide();a(b.acrossQuestions).show();a(b.allContainer).hide()},m=function(){a(b.acrossClears).show();a(b.acrossQuestions).hide();a(b.actionContainer).toggleClass(b.selectedClass);a(b.allContainer).show();a(b.counterContainer).hide()},n=function(){a(b.acrossClears).hide();a(b.acrossQuestions).hide();a(b.allContainer).hide();a(b.counterContainer).show()},p=function(){n(); 2 | a(b.acrossInput).val(0);a(b.actionContainer).removeClass(b.selectedClass)},q=function(c){c?l():n();a(g).prop("checked",c).parent().parent().toggleClass(b.selectedClass,c)},h=function(){var c=a(g).filter(":checked").length,d=a(".action-counter").data("actionsIcnt");a(b.counterContainer).html(interpolate(ngettext("%(sel)s of %(cnt)s selected","%(sel)s of %(cnt)s selected",c),{sel:c,cnt:d},!0));a(b.allToggle).prop("checked",function(){var a;c===g.length?(a=!0,l()):(a=!1,p());return a})};a(b.counterContainer).show(); 3 | a(this).filter(":checked").each(function(c){a(this).parent().parent().toggleClass(b.selectedClass);h();1===a(b.acrossInput).val()&&m()});a(b.allToggle).show().click(function(){q(a(this).prop("checked"));h()});a("a",b.acrossQuestions).click(function(c){c.preventDefault();a(b.acrossInput).val(1);m()});a("a",b.acrossClears).click(function(c){c.preventDefault();a(b.allToggle).prop("checked",!1);p();q(0);h()});f=null;a(g).click(function(c){c||(c=window.event);var d=c.target?c.target:c.srcElement;if(f&& 4 | a.data(f)!==a.data(d)&&!0===c.shiftKey){var e=!1;a(f).prop("checked",d.checked).parent().parent().toggleClass(b.selectedClass,d.checked);a(g).each(function(){if(a.data(this)===a.data(f)||a.data(this)===a.data(d))e=e?!1:!0;e&&a(this).prop("checked",d.checked).parent().parent().toggleClass(b.selectedClass,d.checked)})}a(d).parent().parent().toggleClass(b.selectedClass,d.checked);f=d;h()});a("form#changelist-form table#result_list tr").find("td:gt(0) :input").change(function(){k=!0});a('form#changelist-form button[name="index"]').click(function(a){if(k)return confirm(gettext("You have unsaved changes on individual editable fields. If you run an action, your unsaved changes will be lost."))}); 5 | a('form#changelist-form input[name="_save"]').click(function(c){var d=!1;a("select option:selected",b.actionContainer).each(function(){a(this).val()&&(d=!0)});if(d)return k?confirm(gettext("You have selected an action, but you haven't saved your changes to individual fields yet. Please click OK to save. You'll need to re-run the action.")):confirm(gettext("You have selected an action, and you haven't made any changes on individual fields. You're probably looking for the Go button rather than the Save button."))})}; 6 | a.fn.actions.defaults={actionContainer:"div.actions",counterContainer:"span.action-counter",allContainer:"div.actions span.all",acrossInput:"div.actions input.select-across",acrossQuestions:"div.actions span.question",acrossClears:"div.actions span.clear",allToggle:"#action-toggle",selectedClass:"selected"};a(document).ready(function(){var e=a("tr input.action-select");0' + gettext("Show") + 11 | ')'); 12 | } 13 | }); 14 | // Add toggle to anchor tag 15 | $("fieldset.collapse a.collapse-toggle").click(function(ev) { 16 | if ($(this).closest("fieldset").hasClass("collapsed")) { 17 | // Show 18 | $(this).text(gettext("Hide")).closest("fieldset").removeClass("collapsed").trigger("show.fieldset", [$(this).attr("id")]); 19 | } else { 20 | // Hide 21 | $(this).text(gettext("Show")).closest("fieldset").addClass("collapsed").trigger("hide.fieldset", [$(this).attr("id")]); 22 | } 23 | return false; 24 | }); 25 | }); 26 | })(django.jQuery); 27 | -------------------------------------------------------------------------------- /ui/static/admin/js/collapse.min.js: -------------------------------------------------------------------------------- 1 | (function(a){a(document).ready(function(){a("fieldset.collapse").each(function(b,c){0===a(c).find("div.errors").length&&a(c).addClass("collapsed").find("h2").first().append(' ('+gettext("Show")+")")});a("fieldset.collapse a.collapse-toggle").click(function(b){a(this).closest("fieldset").hasClass("collapsed")?a(this).text(gettext("Hide")).closest("fieldset").removeClass("collapsed").trigger("show.fieldset",[a(this).attr("id")]):a(this).text(gettext("Show")).closest("fieldset").addClass("collapsed").trigger("hide.fieldset", 2 | [a(this).attr("id")]);return!1})})})(django.jQuery); 3 | -------------------------------------------------------------------------------- /ui/static/admin/js/inlines.min.js: -------------------------------------------------------------------------------- 1 | (function(c){c.fn.formset=function(b){var a=c.extend({},c.fn.formset.defaults,b),d=c(this);b=d.parent();var k=function(a,g,l){var b=new RegExp("("+g+"-(\\d+|__prefix__))");g=g+"-"+l;c(a).prop("for")&&c(a).prop("for",c(a).prop("for").replace(b,g));a.id&&(a.id=a.id.replace(b,g));a.name&&(a.name=a.name.replace(b,g))},e=c("#id_"+a.prefix+"-TOTAL_FORMS").prop("autocomplete","off"),l=parseInt(e.val(),10),g=c("#id_"+a.prefix+"-MAX_NUM_FORMS").prop("autocomplete","off"),h=""===g.val()||0'+a.addText+""),m=b.find("tr:last a")):(d.filter(":last").after('"),m=d.filter(":last").next().find("a"));m.click(function(b){b.preventDefault();b=c("#"+a.prefix+"-empty");var f=b.clone(!0);f.removeClass(a.emptyCssClass).addClass(a.formCssClass).attr("id", 3 | a.prefix+"-"+l);f.is("tr")?f.children(":last").append('
'+a.deleteText+"
"):f.is("ul")||f.is("ol")?f.append('
  • '+a.deleteText+"
  • "):f.children(":first").append(''+a.deleteText+"");f.find("*").each(function(){k(this,a.prefix,e.val())});f.insertBefore(c(b));c(e).val(parseInt(e.val(),10)+1);l+=1;""!==g.val()&&0>=g.val()-e.val()&&m.parent().hide(); 4 | f.find("a."+a.deleteCssClass).click(function(b){b.preventDefault();f.remove();--l;a.removed&&a.removed(f);c(document).trigger("formset:removed",[f,a.prefix]);b=c("."+a.formCssClass);c("#id_"+a.prefix+"-TOTAL_FORMS").val(b.length);(""===g.val()||0 0) { 26 | values.push(field.val()); 27 | } 28 | }); 29 | prepopulatedField.val(URLify(values.join(' '), maxLength, allowUnicode)); 30 | }; 31 | 32 | prepopulatedField.data('_changed', false); 33 | prepopulatedField.change(function() { 34 | prepopulatedField.data('_changed', true); 35 | }); 36 | 37 | if (!prepopulatedField.val()) { 38 | $(dependencies.join(',')).keyup(populate).change(populate).focus(populate); 39 | } 40 | }); 41 | }; 42 | })(django.jQuery); 43 | -------------------------------------------------------------------------------- /ui/static/admin/js/prepopulate.min.js: -------------------------------------------------------------------------------- 1 | (function(c){c.fn.prepopulate=function(e,f,g){return this.each(function(){var a=c(this),b=function(){if(!a.data("_changed")){var b=[];c.each(e,function(a,d){d=c(d);0 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /ui/static/ui/css/bootstrap-reboot.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Bootstrap Reboot v4.6.0 (https://getbootstrap.com/) 3 | * Copyright 2011-2021 The Bootstrap Authors 4 | * Copyright 2011-2021 Twitter, Inc. 5 | * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE) 6 | * Forked from Normalize.css, licensed MIT (https://github.com/necolas/normalize.css/blob/master/LICENSE.md) 7 | */*,::after,::before{box-sizing:border-box}html{font-family:sans-serif;line-height:1.15;-webkit-text-size-adjust:100%;-webkit-tap-highlight-color:transparent}article,aside,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}body{margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans","Liberation Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-size:1rem;font-weight:400;line-height:1.5;color:#212529;text-align:left;background-color:#fff}[tabindex="-1"]:focus:not(:focus-visible){outline:0!important}hr{box-sizing:content-box;height:0;overflow:visible}h1,h2,h3,h4,h5,h6{margin-top:0;margin-bottom:.5rem}p{margin-top:0;margin-bottom:1rem}abbr[data-original-title],abbr[title]{text-decoration:underline;-webkit-text-decoration:underline dotted;text-decoration:underline dotted;cursor:help;border-bottom:0;-webkit-text-decoration-skip-ink:none;text-decoration-skip-ink:none}address{margin-bottom:1rem;font-style:normal;line-height:inherit}dl,ol,ul{margin-top:0;margin-bottom:1rem}ol ol,ol ul,ul ol,ul ul{margin-bottom:0}dt{font-weight:700}dd{margin-bottom:.5rem;margin-left:0}blockquote{margin:0 0 1rem}b,strong{font-weight:bolder}small{font-size:80%}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}a{color:#007bff;text-decoration:none;background-color:transparent}a:hover{color:#0056b3;text-decoration:underline}a:not([href]):not([class]){color:inherit;text-decoration:none}a:not([href]):not([class]):hover{color:inherit;text-decoration:none}code,kbd,pre,samp{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;font-size:1em}pre{margin-top:0;margin-bottom:1rem;overflow:auto;-ms-overflow-style:scrollbar}figure{margin:0 0 1rem}img{vertical-align:middle;border-style:none}svg{overflow:hidden;vertical-align:middle}table{border-collapse:collapse}caption{padding-top:.75rem;padding-bottom:.75rem;color:#6c757d;text-align:left;caption-side:bottom}th{text-align:inherit;text-align:-webkit-match-parent}label{display:inline-block;margin-bottom:.5rem}button{border-radius:0}button:focus:not(:focus-visible){outline:0}button,input,optgroup,select,textarea{margin:0;font-family:inherit;font-size:inherit;line-height:inherit}button,input{overflow:visible}button,select{text-transform:none}[role=button]{cursor:pointer}select{word-wrap:normal}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button}[type=button]:not(:disabled),[type=reset]:not(:disabled),[type=submit]:not(:disabled),button:not(:disabled){cursor:pointer}[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner,button::-moz-focus-inner{padding:0;border-style:none}input[type=checkbox],input[type=radio]{box-sizing:border-box;padding:0}textarea{overflow:auto;resize:vertical}fieldset{min-width:0;padding:0;margin:0;border:0}legend{display:block;width:100%;max-width:100%;padding:0;margin-bottom:.5rem;font-size:1.5rem;line-height:inherit;color:inherit;white-space:normal}progress{vertical-align:baseline}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{outline-offset:-2px;-webkit-appearance:none}[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{font:inherit;-webkit-appearance:button}output{display:inline-block}summary{display:list-item;cursor:pointer}template{display:none}[hidden]{display:none!important} 8 | /*# sourceMappingURL=bootstrap-reboot.min.css.map */ -------------------------------------------------------------------------------- /ui/static/ui/css/index.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ui/static/ui/fontawesome/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Font Awesome Free License 2 | ------------------------- 3 | 4 | Font Awesome Free is free, open source, and GPL friendly. You can use it for 5 | commercial projects, open source projects, or really almost whatever you want. 6 | Full Font Awesome Free license: https://fontawesome.com/license/free. 7 | 8 | # Icons: CC BY 4.0 License (https://creativecommons.org/licenses/by/4.0/) 9 | In the Font Awesome Free download, the CC BY 4.0 license applies to all icons 10 | packaged as SVG and JS file types. 11 | 12 | # Fonts: SIL OFL 1.1 License (https://scripts.sil.org/OFL) 13 | In the Font Awesome Free download, the SIL OFL license applies to all icons 14 | packaged as web and desktop font files. 15 | 16 | # Code: MIT License (https://opensource.org/licenses/MIT) 17 | In the Font Awesome Free download, the MIT license applies to all non-font and 18 | non-icon files. 19 | 20 | # Attribution 21 | Attribution is required by MIT, SIL OFL, and CC BY licenses. Downloaded Font 22 | Awesome Free files already contain embedded comments with sufficient 23 | attribution, so you shouldn't need to do anything additional when using these 24 | files normally. 25 | 26 | We've kept attribution comments terse, so we ask that you do not actively work 27 | to remove them from files, especially code. They're a great way for folks to 28 | learn about Font Awesome. 29 | 30 | # Brand Icons 31 | All brand icons are trademarks of their respective owners. The use of these 32 | trademarks does not indicate endorsement of the trademark holder by Font 33 | Awesome, nor vice versa. **Please do not use brand logos for any purpose except 34 | to represent the company, product, or service to which they refer.** 35 | -------------------------------------------------------------------------------- /ui/static/ui/fontawesome/css/brands.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome Free 5.14.0 by @fontawesome - https://fontawesome.com 3 | * License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) 4 | */ 5 | @font-face{font-family:"Font Awesome 5 Brands";font-style:normal;font-weight:400;font-display:block;src:url(../webfonts/fa-brands-400.eot);src:url(../webfonts/fa-brands-400.eot?#iefix) format("embedded-opentype"),url(../webfonts/fa-brands-400.woff2) format("woff2"),url(../webfonts/fa-brands-400.woff) format("woff"),url(../webfonts/fa-brands-400.ttf) format("truetype"),url(../webfonts/fa-brands-400.svg#fontawesome) format("svg")}.fab{font-family:"Font Awesome 5 Brands";font-weight:400} -------------------------------------------------------------------------------- /ui/static/ui/fontawesome/css/regular.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome Free 5.14.0 by @fontawesome - https://fontawesome.com 3 | * License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) 4 | */ 5 | @font-face{font-family:"Font Awesome 5 Free";font-style:normal;font-weight:400;font-display:block;src:url(../webfonts/fa-regular-400.eot);src:url(../webfonts/fa-regular-400.eot?#iefix) format("embedded-opentype"),url(../webfonts/fa-regular-400.woff2) format("woff2"),url(../webfonts/fa-regular-400.woff) format("woff"),url(../webfonts/fa-regular-400.ttf) format("truetype"),url(../webfonts/fa-regular-400.svg#fontawesome) format("svg")}.far{font-family:"Font Awesome 5 Free";font-weight:400} -------------------------------------------------------------------------------- /ui/static/ui/fontawesome/css/solid.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome Free 5.14.0 by @fontawesome - https://fontawesome.com 3 | * License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) 4 | */ 5 | @font-face{font-family:"Font Awesome 5 Free";font-style:normal;font-weight:900;font-display:block;src:url(../webfonts/fa-solid-900.eot);src:url(../webfonts/fa-solid-900.eot?#iefix) format("embedded-opentype"),url(../webfonts/fa-solid-900.woff2) format("woff2"),url(../webfonts/fa-solid-900.woff) format("woff"),url(../webfonts/fa-solid-900.ttf) format("truetype"),url(../webfonts/fa-solid-900.svg#fontawesome) format("svg")}.fa,.fas{font-family:"Font Awesome 5 Free";font-weight:900} -------------------------------------------------------------------------------- /ui/static/ui/fontawesome/webfonts/fa-brands-400.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/ui/static/ui/fontawesome/webfonts/fa-brands-400.eot -------------------------------------------------------------------------------- /ui/static/ui/fontawesome/webfonts/fa-brands-400.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/ui/static/ui/fontawesome/webfonts/fa-brands-400.ttf -------------------------------------------------------------------------------- /ui/static/ui/fontawesome/webfonts/fa-brands-400.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/ui/static/ui/fontawesome/webfonts/fa-brands-400.woff -------------------------------------------------------------------------------- /ui/static/ui/fontawesome/webfonts/fa-brands-400.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/ui/static/ui/fontawesome/webfonts/fa-brands-400.woff2 -------------------------------------------------------------------------------- /ui/static/ui/fontawesome/webfonts/fa-regular-400.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/ui/static/ui/fontawesome/webfonts/fa-regular-400.eot -------------------------------------------------------------------------------- /ui/static/ui/fontawesome/webfonts/fa-regular-400.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/ui/static/ui/fontawesome/webfonts/fa-regular-400.ttf -------------------------------------------------------------------------------- /ui/static/ui/fontawesome/webfonts/fa-regular-400.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/ui/static/ui/fontawesome/webfonts/fa-regular-400.woff -------------------------------------------------------------------------------- /ui/static/ui/fontawesome/webfonts/fa-regular-400.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/ui/static/ui/fontawesome/webfonts/fa-regular-400.woff2 -------------------------------------------------------------------------------- /ui/static/ui/fontawesome/webfonts/fa-solid-900.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/ui/static/ui/fontawesome/webfonts/fa-solid-900.eot -------------------------------------------------------------------------------- /ui/static/ui/fontawesome/webfonts/fa-solid-900.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/ui/static/ui/fontawesome/webfonts/fa-solid-900.ttf -------------------------------------------------------------------------------- /ui/static/ui/fontawesome/webfonts/fa-solid-900.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/ui/static/ui/fontawesome/webfonts/fa-solid-900.woff -------------------------------------------------------------------------------- /ui/static/ui/fontawesome/webfonts/fa-solid-900.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/ui/static/ui/fontawesome/webfonts/fa-solid-900.woff2 -------------------------------------------------------------------------------- /ui/static/ui/img/index.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ui/static/ui/img/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/ui/static/ui/img/logo.png -------------------------------------------------------------------------------- /ui/static/ui/img/logoc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/ui/static/ui/img/logoc.png -------------------------------------------------------------------------------- /ui/static/ui/img/mls/index.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ui/static/ui/index.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ui/static/ui/js/auto.micro.js: -------------------------------------------------------------------------------- 1 | function AutoSuggestControl(id_or_element){this.provider=new wordSuggestions();this.textbox=typeof id_or_element=="string"?document.getElementById(id_or_element):id_or_element;this.init()}AutoSuggestControl.prototype.autosuggest=function(aSuggestions){if(aSuggestions.length>0){this.typeAhead(aSuggestions[0])}};AutoSuggestControl.prototype.handleKeyUp=function(oEvent){var iKeyCode=oEvent.keyCode;var evtobj=oEvent;window.eventobj=evtobj;if((iKeyCode!=16&&iKeyCode<32)||(iKeyCode>=33&&iKeyCode<=46)||(iKeyCode>=112&&iKeyCode<=123)||(iKeyCode==65&&evtobj.ctrlKey)||(iKeyCode==90&&evtobj.ctrlKey)){if(iKeyCode==90&&evtobj.ctrlKey){}}else{this.provider.requestSuggestions(this)}};AutoSuggestControl.prototype.init=function(){var oThis=this;lastDate=new Date();oThis.textbox.onkeyup=function(oEvent){if(!oEvent){oEvent=window.event}newDate=new Date();if(newDate.getTime()>lastDate.getTime()+200){oThis.handleKeyUp(oEvent);lastDate=newDate}}};AutoSuggestControl.prototype.selectRange=function(iStart,iLength){if(this.textbox.createTextRange){var oRange=this.textbox.createTextRange();oRange.moveStart("character",iStart);oRange.moveEnd("character",iLength);oRange.select()}else{if(this.textbox.setSelectionRange){this.textbox.setSelectionRange(iStart,iLength)}}this.textbox.focus()};AutoSuggestControl.prototype.typeAhead=function(sSuggestion){if(this.textbox.createTextRange||this.textbox.setSelectionRange){var lastSpace=this.textbox.value.lastIndexOf(" ");var lastQuote=this.textbox.value.lastIndexOf("'");var lastHypen=this.textbox.value.lastIndexOf("-");var lastDoubleQuote=this.textbox.value.lastIndexOf('"');var lastEnter=this.textbox.value.lastIndexOf("\n");var lastIndex=Math.max(lastSpace,lastEnter,lastQuote,lastHypen,lastDoubleQuote)+1;var contentStripped=this.textbox.value.substring(0,lastIndex);var lastWord=this.textbox.value.substring(lastIndex,this.textbox.value.length);this.textbox.value=contentStripped+sSuggestion;var start=this.textbox.value.length-sSuggestion.replace(lastWord,"").length;var end=this.textbox.value.length;this.selectRange(start,end)}};wordSuggestions.prototype.requestSuggestions=function(oAutoSuggestControl){var aSuggestions=[];var sTextbox=oAutoSuggestControl.textbox.value;var sTextboxSplit=sTextbox.split(/[\s,]+/);var sTextboxLast=sTextboxSplit[sTextboxSplit.length-1];var sTextboxValue=sTextboxLast;if(sTextboxValue.length>0){for(var i=0;i0){ 13 | $(".verifyimg").attr("src", verifyimg+'&random='+Math.random()); 14 | }else{ 15 | $(".verifyimg").attr("src", verifyimg.replace(/\?.*$/,'')+'?'+Math.random()); 16 | } 17 | }); 18 | }); -------------------------------------------------------------------------------- /ui/templates/ui/403.html: -------------------------------------------------------------------------------- 1 | {% extends "ui/base.html" %} 2 | {% load i18n %} 3 | 4 | {% block breadcrumbs %} 5 | 9 | {% endblock %} 10 | 11 | {% block title %}{% trans 'Server error (500)' %}{% endblock %} 12 | 13 | {% block content %} 14 |

    {% trans 'Server Error (500)' %}

    15 |

    {{ error_msg }}

    16 | 17 | {% endblock %} 18 | -------------------------------------------------------------------------------- /ui/templates/ui/add_step.html: -------------------------------------------------------------------------------- 1 |
    2 |
    3 | 4 | 5 |
    6 |
    7 | 8 | 9 |
    10 |
    11 | 12 | 13 |
    14 |
    15 | 16 | 22 |
    23 |
    24 | 25 | 26 |
    27 |
    28 | 29 | 33 | It will work only in cluster mode. When you run BioQueue in clusters, the computing nodes may not connect to the internet, so software, like wget, fastq-dump may not work. Under this circumstance, you can force BioQueue to run this step in local. 34 |
    35 |
    36 | 37 | 38 |
    39 | 40 |
    41 |
    42 | {% include 'ui/variant_hint.html' %} 43 | -------------------------------------------------------------------------------- /ui/templates/ui/archive_table.html: -------------------------------------------------------------------------------- 1 |
      2 |
    • Protocol: {{ protocol }}
    • 3 |
    • Protocol version: {{ ver }}
    • 4 |
    • Inputs: {{ inputs }}
    • 5 |
    6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | {% for file, md5 in files %} 14 | 15 | 16 | 17 | 18 | {% endfor %} 19 |
    FileMD5
    {{ file }}{{ md5 }}
    20 | -------------------------------------------------------------------------------- /ui/templates/ui/create_simple_models.html: -------------------------------------------------------------------------------- 1 | {% extends 'ui/base.html' %} 2 | {% block title %}{{ model_name }} | BioQueue{% endblock %} 3 | {% block content %} 4 |
    5 | 9 |
    10 |
    11 |
    12 |

    13 | 16 |

    17 |
    18 |
    19 |
    20 |
    21 | {% for field in form %} 22 |
    23 | {{ field.errors }} 24 | {{ field.label_tag }} {{field}} 25 | {% if field.help_text %} 26 | {{ field.help_text|safe }} 27 | {% endif %} 28 |
    29 | {% endfor %} 30 | {% csrf_token %} 31 | 32 |
    33 |
    34 |
    35 |
    36 |
    37 | 44 |
    45 |
    46 |
    47 | {% endblock %} 48 | {% block script %} 49 | $("#topNav{{model_name}}").addClass("active"); 50 | if ("{{model_name}}"=="Environment" || "{{model_name}}" == "Workspace"){ 51 | $(".scaffold-nav").addClass("active"); 52 | } 53 | var csrftoken = $.cookie('csrftoken'); 54 | $.ajaxSetup({ 55 | beforeSend: function(xhr, settings){ 56 | xhr.setRequestHeader("X-CSRFToken", csrftoken); 57 | } 58 | }); 59 | $("form").submit(function(){ 60 | var self = $(this); 61 | $.post(self.attr("action"), self.serialize(), success, "json"); 62 | return false; 63 | 64 | function success(data){ 65 | if(data.status){ 66 | post_msg(data.info); 67 | self[0].reset(); 68 | } else { 69 | post_msg(data.info, 1); 70 | } 71 | } 72 | }); 73 | {% endblock %} -------------------------------------------------------------------------------- /ui/templates/ui/error.html: -------------------------------------------------------------------------------- 1 | {% extends "ui/base.html" %} 2 | {% load i18n %} 3 | 4 | {% block breadcrumbs %} 5 | 9 | {% endblock %} 10 | 11 | {% block title %}{% trans 'Server error (500)' %}{% endblock %} 12 | 13 | {% block content %} 14 |

    {% trans 'Server Error (500)' %}

    15 |

    {{ error_msg }}

    16 | 17 | {% endblock %} 18 | -------------------------------------------------------------------------------- /ui/templates/ui/fetch_data.html: -------------------------------------------------------------------------------- 1 | {% extends "ui/base.html" %} 2 | {% block title %}Fetch Data | BioQueue{% endblock %} 3 | {% block content %} 4 | 7 |
    8 |
    9 | 16 |
    17 |
    18 |
    19 |
    20 | 21 | 22 | This function provides support for GEO ID, like GSM1014192, GSE42815, and EBI accssion ID, like ERA000092 (submission), ERP000016 (study), ERR003990 (run), ERS000081 (sample), ERX000398 (experiment). 23 |
    24 |
    25 | 26 |
    27 | {% csrf_token %} 28 |
    29 | 33 |
    34 |
    35 |
    36 |
    37 | {% endblock %} 38 | {% block script %} 39 | $("#fetch-data").addClass("active"); 40 | $(".scaffold-nav").addClass("active"); 41 | var csrftoken = $.cookie('csrftoken'); 42 | $("#fetch-data-form").submit(function(){ 43 | $("#link-container").html(""); 44 | $(".search-result").hide("slow"); 45 | var self = $(this); 46 | $.post(self.attr("action"), self.serialize(), success, "json"); 47 | return false; 48 | 49 | function success(data){ 50 | if(data.status){ 51 | $("#link-container").html(data.info); 52 | $(".search-result").show("slow"); 53 | self[0].reset(); 54 | }else{ 55 | Messenger().post({ 56 | message: data.info, 57 | type: 'error', 58 | showCloseButton: true 59 | }); 60 | } 61 | } 62 | }); 63 | {% endblock %} -------------------------------------------------------------------------------- /ui/templates/ui/fetch_learning.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 |
    abraverage
    {{ step.a }}{{ step.b }}{{ step.r }}{{ step.av }}
    17 |

    18 | Import 19 |

    20 | -------------------------------------------------------------------------------- /ui/templates/ui/foot.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ui/templates/ui/get_learning_result.html: -------------------------------------------------------------------------------- 1 |
    Prior
    2 |
    3 | {% csrf_token %} 4 |
    5 |
    6 | 7 | 8 |
    9 |
    10 | 11 | 12 |
    13 |
    14 | 15 | 16 |
    17 |
    18 | 19 | 20 |
    21 |
    Predicted resource usage
    22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | {% for hit in hits %} 33 | 34 | 35 | 36 | 37 | 38 | 39 | {% endfor %} 40 | 41 |
    ResourceSlopeInterceptr
    {{ hit.get_type_display }}{{ hit.a }}{{ hit.b }}{{ hit.r }}
    42 | -------------------------------------------------------------------------------- /ui/templates/ui/import_protocol.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | {% for ref in ref_list %} 11 | 12 | 13 | 14 | 15 | {% endfor %} 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 |
    ReferenceDescription
    {{ ref.name }}{{ ref.description }}
    This color indicates you need to create the reference manually.
    This color indicates a reference in the same name has existed. So it may causing conflict.
    -------------------------------------------------------------------------------- /ui/templates/ui/registered_sample_cards.html: -------------------------------------------------------------------------------- 1 | {% load tz %} 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | {% for file_item in samples %} 13 | 14 | 15 | 16 | 17 | 32 | 33 | 34 | 37 | 38 | {% endfor %} 39 | 40 |
    Sample NameAssayCreate TimeOperation
    {{ file_item.name }}{{ file_item.exp }}{{ file_item.create_time | localtime }} 18 |
    19 | 20 |
    21 | 24 | 29 |
    30 |
    31 |
    File name: {{ file_item.raw }} 35 |

    {{ file_item.attribute|linebreaksbr }}

    41 | -------------------------------------------------------------------------------- /ui/templates/ui/show_job_folder.html: -------------------------------------------------------------------------------- 1 | {% if user_files %} 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | {% for file_item in user_files %} 13 | 14 | 15 | 16 | 17 | 24 | 25 | {% endfor %} 26 | 27 |
    File NameFile SizeCreate TimeOperation
    {% if file_item.is_link %}{% endif %} {{ file_item.name }}{{ file_item.file_size | filesizeformat }}{{ file_item.file_create }} 18 |
    19 | 20 | 21 | 22 |
    23 |
    28 | 35 | {% endif %} -------------------------------------------------------------------------------- /ui/templates/ui/show_learning.html: -------------------------------------------------------------------------------- 1 | {% extends 'ui/base.html' %} 2 | {% block title %}Import Check Point | BioQueue{% endblock %} 3 | {% block content %} 4 |
    5 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | {% for protocol in protocol_list %} 17 | 18 | 19 | 20 | 21 | {% endfor %} 22 | 23 |
    IDProtocol
    {{ protocol.id }}{{ protocol.name }}
    24 | 33 | 34 |
    35 | 49 | {% endblock %} 50 | {% block script %} 51 | function showML(h, t){ 52 | $.get("{% url 'ui:fetch_learning' %}", {hash: h, type: t}, function(result){ 53 | $('#ml .modal-body').html(result.info); 54 | $('#ml').modal(); 55 | }); 56 | } 57 | function showSteps(recId){ 58 | $.get("{% url 'ui:show_learning_steps' %}", {parent: recId}, function(result){ 59 | if ($('#stepsContainer').css("display") != 'none'){ 60 | $('#stepsContainer').hide("fast"); 61 | } 62 | $('#stepsContainer').html(result.info); 63 | $('#stepsContainer').show("slow"); 64 | }); 65 | } 66 | {% endblock %} -------------------------------------------------------------------------------- /ui/templates/ui/show_learning_steps.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | {% for step in step_list %} 12 | 13 | 14 | 15 | 20 | 21 | {% endfor %} 22 | 23 |
    IDSoftwareImport Check point
    {{ step.id }}{{ step.software }} 16 | CPU 17 | Memory 18 | Disk 19 |
    -------------------------------------------------------------------------------- /ui/templates/ui/show_steps.html: -------------------------------------------------------------------------------- 1 |
    2 |

    {{protocol.description}}

    3 |
    4 |
    5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | {% for step in step_list %} 15 | 16 | 23 | 24 | 25 | 26 | 27 | 47 | 48 | {% endfor %} 49 | 50 |
    OrderSoftwareParameter
    17 | 22 | {{ step.software }}{{ step.parameter }}
    28 |
    29 |
    30 | 31 |
    32 | 38 |
    39 |
    40 | 41 | 42 | 43 |
    44 |
    45 |
    46 |
    51 |
    52 | 53 | -------------------------------------------------------------------------------- /ui/templates/ui/show_uploads.html: -------------------------------------------------------------------------------- 1 | {% if user_files %} 2 | {% for file_item in user_files %} 3 |
    4 | 7 |
    8 | {% endfor %} 9 | {% endif %} -------------------------------------------------------------------------------- /ui/templates/ui/show_workspace.html: -------------------------------------------------------------------------------- 1 | {% extends 'ui/base.html' %} 2 | {% block title %}Workspaces | BioQueue{% endblock %} 3 | {% block content %} 4 |
    5 | 9 |
    10 |
    11 |
    12 |

    13 | 14 | Add new workspace 15 | 16 |

    17 |
    18 |
    19 |
    20 |
    21 |

    22 | 25 |

    26 |
    27 |
    28 |
    29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | {% for obj in objects %} 39 | 40 | 41 | 42 | 50 | 51 | {% endfor %} 52 | 53 |
    IDWorkspaceOperation
    {{ obj.id }}{{ obj.name }} 43 | {% if obj.user is not None %} 44 |
    45 | 46 | 47 |
    48 | {% endif %} 49 |
    54 | 77 |
    78 |
    79 |
    80 |
    81 |
    82 | 83 | {% endblock %} 84 | {% block script %} 85 | $("#topNavWorkspace").addClass("active"); 86 | $(".scaffold-nav").addClass("active"); 87 | var csrftoken = $.cookie('csrftoken'); 88 | $.ajaxSetup({ 89 | beforeSend: function(xhr, settings){ 90 | xhr.setRequestHeader("X-CSRFToken", csrftoken); 91 | } 92 | }); 93 | $("form").submit(function(){ 94 | var self = $(this); 95 | $.post(self.attr("action"), self.serialize(), success, "json"); 96 | return false; 97 | 98 | function success(data){ 99 | if(data.status){ 100 | addNew(data.info, $("#name").val(), $("#ve-value").val()); 101 | self[0].reset(); 102 | } else { 103 | post_msg(data.info, 1); 104 | } 105 | } 106 | }); 107 | function delWorkspace(obj){ 108 | let self = $(obj); 109 | generic_get(obj, false); 110 | $("#recRow"+self.data("id")).hide("slow"); 111 | } 112 | function updateWP(obj){ 113 | generic_get_modal(obj); 114 | } 115 | {% endblock %} -------------------------------------------------------------------------------- /ui/templates/ui/single.html: -------------------------------------------------------------------------------- 1 | {% load static %} 2 | 3 | 4 | 5 | 6 | Sign In | BioQueue 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | {% include 'ui/navigation.html' %} 16 |
    17 |
    18 |
    19 |
    20 | {% block content %}{% endblock %} 21 |
    22 |
    23 |
    24 |
    25 |
    26 | {% include 'ui/foot.html' %} 27 | 28 | 29 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /ui/templates/ui/step_atom.html: -------------------------------------------------------------------------------- 1 |
    2 | 3 |
    4 | 5 |
    6 | 7 |
    8 |
    9 | 14 |
    15 |
    16 | 17 | 18 |
    19 |
    20 | 21 | 22 |
    23 |
    24 |
    25 | 26 | Having expectations of resources this step takes? 27 | 28 |
    29 |
    30 | 31 | 32 | Percent of expected CPU usage. For example, if you specify a software to work with two parallel threads/subprocesses, then you can specify the value here as 200. Leave it as -1 if no info about it is available. 33 |
    34 |
    35 | 36 | 37 | Expected peak memory usage in the unit of bytes. Leave it as -1 if no info about it is available. 38 |
    39 |
    40 | 41 | 42 | Expected disk usage in the unit of bytes. Leave it as -1 if no info about it is available. 43 |
    44 |
    45 |
    46 |
    47 |
    48 | 49 | 55 |
    56 |
    57 | 58 | 62 | It will work only in cluster mode. When you run BioQueue in clusters, the computing nodes may not connect to the internet, so software, like wget, fastq-dump may not work. Under this circumstance, you can force BioQueue to run this step in local. 63 |
    -------------------------------------------------------------------------------- /ui/templates/ui/update_simple_models.html: -------------------------------------------------------------------------------- 1 |
    2 | {% for field in form %} 3 |
    4 | {{ field.errors }} 5 | {{ field.label_tag }} {{field}} 6 | {% if field.help_text %} 7 | {{ field.help_text|safe }} 8 | {% endif %} 9 |
    10 | 11 | {% endfor %} 12 | {% csrf_token %} 13 | 14 |
    -------------------------------------------------------------------------------- /ui/templates/ui/variant_hint.html: -------------------------------------------------------------------------------- 1 |
    2 | You can use variables or references when defining the parameters. Below are some of the system level variable: 3 |
      4 |
    • AllOutputBefore: All output files before this step.
    • 5 |
    • History:N-X: Output file "X" in the n-th job.
    • 6 |
    • InputFile: The initial input files which maps to files you provide when you create a job.
    • 7 |
    • InputFile:N: The n-th file in input files.
    • 8 |
    • Job: Job ID.
    • 9 |
    • JobName: Job name.
    • 10 |
    • LastOutput: Output files of last step.
    • 11 |
    • LastOutput:N: The n-th output file of last step (in alphabetical order).
    • 12 |
    • Output:N-M: The output file M produced by the n-th step (in alphabetical order).
    • 13 |
    • Suffix:X: Output file(s) with a "X" suffix in last step.
    • 14 |
    • Suffix:N-X: Output file(s) with a "X" suffix in the n-th step.
    • 15 |
    • Suffix:N-X-M: The m-th output file with a "X" suffix in the n-th step.
    • 16 |
    • ThreadN: The number of CPUs in the running node.
    • 17 |
    • UserBin: Path where BioQueue will install software to for a specific user.
    • 18 |
    • Workspace: Local storage path for the job.
    • 19 |
    20 | All variables should be surrounded by curly-braces, like {% verbatim %}{{ThreadN}}{% endverbatim %}. 21 |
    22 | -------------------------------------------------------------------------------- /ui/templatetags/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | # Created by: Li Yao 4 | # Created on: 5/10/20 -------------------------------------------------------------------------------- /ui/templatetags/job_tags.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | # Created by: Li Yao 4 | # Created on: 5/10/20 5 | from django import template 6 | 7 | register = template.Library() 8 | 9 | 10 | @register.simple_tag(takes_context=True) 11 | def keep_par_paginator(context, **kwargs): 12 | query = context["request"].GET.copy() 13 | for k, v in kwargs.items(): 14 | query[k] = v 15 | return query.urlencode() 16 | -------------------------------------------------------------------------------- /ui/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /ui/tools.py: -------------------------------------------------------------------------------- 1 | from django.http import JsonResponse, StreamingHttpResponse 2 | from worker.bases import get_config, rand_sig, get_user_folder_size 3 | from django.core.paginator import EmptyPage, PageNotAnInteger 4 | import os 5 | 6 | 7 | def build_json_protocol(protocol): 8 | import json 9 | """ 10 | response = StreamingHttpResponse(json.dumps(protocol)) 11 | response['Content-Type'] = 'application/octet-stream' 12 | response['Content-Disposition'] = 'attachment;filename="{0}"'.format(protocol['name']+'.txt') 13 | return response 14 | """ 15 | return json.dumps(protocol) 16 | 17 | 18 | def build_json_reference(ref): 19 | result = list() 20 | for value in ref: 21 | result.append('{'+value+'}') 22 | return JsonResponse(','.join(result), safe=False) 23 | 24 | 25 | def check_user_existence(username): 26 | from django.contrib.auth.models import User 27 | try: 28 | u = User.objects.get(username=username) 29 | return u.id 30 | except Exception as e: 31 | return 0 32 | 33 | 34 | def check_disk_quota_lock(user): 35 | disk_limit = get_config('env', 'disk_quota') 36 | if disk_limit: 37 | if get_user_folder_size(user) < int(disk_limit): 38 | return 1 39 | else: 40 | return 0 41 | else: 42 | return 1 43 | 44 | 45 | def delete_file(file_path): 46 | import os 47 | try: 48 | if os.path.exists(file_path): 49 | os.remove(file_path) 50 | return success('Deleted') 51 | else: 52 | return error('File can not be found.') 53 | except Exception as e: 54 | return error(e) 55 | 56 | 57 | def get_disk_quota_info(user): 58 | try: 59 | disk_pool = int(get_config('env', 'disk_quota')) 60 | disk_used = get_user_folder_size(user) 61 | disk_perc = int(round(disk_used / disk_pool * 100)) 62 | except Exception as e: 63 | print(e) 64 | disk_pool = disk_used = disk_perc = 0 65 | return disk_pool, disk_used, disk_perc 66 | 67 | 68 | def get_maintenance_protocols(): 69 | """ 70 | Get maintenance protocols 71 | :return: list, module names 72 | """ 73 | protocols = [] 74 | protocols_path = os.path.join(os.path.split(os.path.realpath(__file__))[0], 'maintenance_protocols') 75 | for model_name in os.listdir(protocols_path): 76 | if not model_name.endswith('.py') or model_name.startswith('_') or model_name.startswith('maintenance'): 77 | continue 78 | protocols.append(model_name.replace('.py', '')) 79 | return protocols 80 | 81 | 82 | def error(message, jump_url='.', msg_title="error", status=0, wait_second=3): 83 | json_data = dict() 84 | json_data['msg_title'] = msg_title 85 | json_data['info'] = str(message) 86 | json_data['url'] = jump_url 87 | json_data['status'] = status 88 | json_data['wait_second'] = wait_second 89 | return JsonResponse(json_data) 90 | 91 | 92 | def handle_uploaded_file(f): 93 | import os 94 | file_name = os.path.join(get_config('env', 'batch_job'), rand_sig()+'.txt') 95 | with open(file_name, 'wb+') as destination: 96 | for chunk in f.chunks(): 97 | destination.write(chunk) 98 | return file_name 99 | 100 | 101 | def os_to_int(): 102 | import platform 103 | if platform.system() == 'Linux': 104 | return 1 105 | elif platform.system() == 'Darwin': 106 | return 3 107 | else: 108 | return 2 109 | 110 | 111 | def page_info(page_model, page): 112 | try: 113 | items = page_model.page(page) 114 | except PageNotAnInteger: 115 | items = page_model.page(1) 116 | except EmptyPage: 117 | items = page_model.page(page_model.num_pages) 118 | return items 119 | 120 | 121 | def success(message, jump_url='.', msg_title="success", status=1, wait_second=1): 122 | json_data = dict() 123 | json_data['msg_title'] = msg_title 124 | json_data['info'] = message 125 | json_data['url'] = jump_url 126 | json_data['status'] = status 127 | json_data['wait_second'] = wait_second 128 | return JsonResponse(json_data) 129 | 130 | -------------------------------------------------------------------------------- /ui/urls.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Author: Li Yao 4 | # @Date: 12/01/16 5 | from django.conf.urls import url 6 | from django.urls import path 7 | from . import views 8 | 9 | app_name = "ui" 10 | 11 | urlpatterns = [ 12 | url(r'^$', views.index, name='index'), 13 | url(r'^add-job/$', views.add_job, name='add_job'), 14 | url(r'^add-step/$', views.add_step, name='add_step'), 15 | url(r'^audit-check/$', views.audit_jobs, name='audit_check'), 16 | url(r'^batch-job/$', views.batch_job, name='batch_job'), 17 | url(r'^batch-operation/$', views.batch_operation, name='batch_operation'), 18 | url(r'^clean-dead-folder/$', views.clean_dead_folder, name='clean_dead_folder'), 19 | url(r'^clean-dead-lock/$', views.clean_dead_lock, name='clean_dead_lock'), 20 | url(r'^create-protocol/$', views.create_protocol, name='create_protocol'), 21 | url(r'^create-reference-shortcut/$', views.create_reference_shortcut, name='create_reference_shortcut'), 22 | url(r'^delete-job/$', views.delete_job, name='delete_job'), 23 | url(r'^delete-job-file/(?P[a-z,A-Z,0-9,/,+,=]*)/$', views.delete_job_file, name='delete_job_file'), 24 | url(r'^delete-protocol/$', views.delete_protocol, name='delete_protocol'), 25 | url(r'^delete-sample/(?P[a-z,A-Z,0-9,/,+,=]*)/$', views.delete_sample, name='delete_sample'), 26 | url(r'^delete-step/$', views.delete_step, name='delete_step'), 27 | url(r'^delete-upload-file/(?P[a-z,A-Z,0-9,/,+,=]*)/$', views.delete_upload_file, name='delete_upload_file'), 28 | path('download-file///', views.download_file, name='download_file'), 29 | url(r'^export-protocol', views.export_protocol, name='export_protocol'), 30 | url(r'^file-support', views.file_support, name='file_support'), 31 | url(r'^fetch-data', views.fetch_data, name='fetch_data'), 32 | url(r'^fetch-learning', views.fetch_learning, name='fetch_learning'), 33 | url(r'^get-learning-result/$', views.get_learning_result, name='get_learning_result'), 34 | url(r'^get-job-list/$', views.get_job_list, name='get_job_list'), 35 | url(r'^get-job-file-list/$', views.get_job_file_list, name='get_job_file_list'), 36 | url(r'^get-version-check/$', views.show_step_vc, name='get_version_check'), 37 | url(r'^import-learning/$', views.import_learning, name='import_learning'), 38 | url(r'^import-protocol-by-fetch/$', views.import_protocol_by_fetch, name='import_protocol_by_fetch'), 39 | url(r'^import-protocol/$', views.import_protocol, name='import_protocol'), 40 | url(r'^install-ref', views.install_reference, name='install_reference'), 41 | url(r'^lock-job', views.lock_job, name='lock_job'), 42 | url(r'^mark-audit', views.mark_wrong_job, name='mark_wrong_job'), 43 | url(r'^user-reference', views.print_user_reference, name='print_user_reference'), 44 | url(r'^plugin-post/$', views.plugin_post, name='plugin_post'), 45 | url(r'^query-job/$', views.query_job, name='query_job'), 46 | url(r'^query-job-parameter/$', views.query_job_parameter, name='query_job_parameter'), 47 | url(r'^query-job-sample/$', views.query_job_samples, name='query_job_samples'), 48 | url(r'^query-protocol', views.query_protocol, name='query_protocol'), 49 | url(r'^query-running-jobs', views.query_running_jobs, name='query_running_jobs'), 50 | url(r'^query-usage', views.query_usage, name='query_usage'), 51 | url(r'^register-sample/', views.register_sample, name='register_sample'), 52 | url(r'^rerun-job/$', views.rerun_job, name='rerun_job'), 53 | url(r'^resume-job/$', views.resume_job, name='resume_job'), 54 | path('simple-models///', views.manage_simple_models, name='manage_simple_models'), 55 | url(r'^send-file-as-reference/(?P[a-z,A-Z,0-9,/,+,=]*)/$', views.send_file_as_reference, name='send_file_as_reference'), 56 | url(r'^settings/$', views.settings, name='settings'), 57 | url(r'^set-workspace/$', views.set_workspace, name='set_workspace'), 58 | url(r'^share-with-peer/$', views.share_with_peer, name='share_with_peer'), 59 | url(r'^show-job-log/$', views.show_job_log, name='show_job_log'), 60 | url(r'^show-job-folder/$', views.show_job_folder, name='show_job_folder'), 61 | url(r'^show-learning/$', views.show_learning, name='show_learning'), 62 | url(r'^show-learning-steps', views.show_learning_steps, name='show_learning_steps'), 63 | url(r'^show-step/$', views.show_step, name='show_step'), 64 | url(r'^show-upload/$', views.show_upload_files, name='show_upload'), 65 | url(r'^show-workspace/$', views.show_workspace, name='show_workspace'), 66 | url(r'^tar-files/$', views.tar_files, name='tar_files'), 67 | url(r'^terminate-job/$', views.terminate_job, name='terminate_job'), 68 | url(r'^update-bioqueue/$', views.update_bioqueue, name='update_bioqueue'), 69 | url(r'^update-job-inputs/$', views.update_job_inputs, name='update_job_inputs'), 70 | url(r'^update-job-parameter/$', views.update_job_parameter, name='update_job_parameter'), 71 | url(r'^update-parameter/$', views.update_parameter, name='update_parameter'), 72 | url(r'^update-protocol-description/$', views.update_protocol_description, name='update_protocol_description'), 73 | url(r'^upload-protocol/$', views.upload_protocol, name='upload_protocol'), 74 | url(r'^update-sample-attr/$', views.update_sample_attr, name='update_sample_attr'), 75 | url(r'^update-step-environment/$', views.update_step_environment, name='update_step_environment'), 76 | url(r'^update-step-order/$', views.update_step_order, name='update_step_order'), 77 | url(r'^update-step-prior/$', views.update_step_prior, name='update_step_prior'), 78 | url(r'^update-version-check/$', views.update_version_check, name='update_step_vc'), 79 | url(r'^update-workspace/$', views.update_workspace, name='update_workspace'), 80 | ] 81 | -------------------------------------------------------------------------------- /ui/views.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | # @Author: Li Yao 4 | # @Date: 7/24/20 5 | from django.contrib.auth.decorators import login_required 6 | from QueueDB.models import Job 7 | from django.shortcuts import render 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /ui/views/Plugins.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | # Created by: Li Yao 4 | # Created on: 8/2/20 5 | from django.http import JsonResponse 6 | from django.views.decorators.csrf import csrf_exempt 7 | import json 8 | import hashlib 9 | from django.apps import apps 10 | from QueueDB.models import Profile 11 | 12 | 13 | class Api(object): 14 | @staticmethod 15 | def ret_info(valid, results): 16 | json_data = dict() 17 | json_data['valid'] = valid 18 | json_data['results'] = results 19 | return JsonResponse(json_data) 20 | 21 | def _valid(self): 22 | """ 23 | Validate API request 24 | 25 | :return: bool 26 | """ 27 | try: 28 | user_profile = Profile.objects.get(api_key=self.api_key) 29 | api_secret = user_profile.api_secret 30 | m = hashlib.md5() 31 | par_str = json.dumps(self.pars, sort_keys=True) + api_secret 32 | m.update(par_str.encode()) 33 | if self.check_sum == m.hexdigest(): 34 | self.user = user_profile.delegate 35 | return True 36 | else: 37 | return False 38 | except Profile.DoesNotExist: 39 | return False 40 | 41 | def __init__(self, target_model, api_key, pars, check_sum): 42 | self.api_key = api_key 43 | self.check_sum = check_sum 44 | self._protected_models = {"Job", "Protocol", "ProtocolList", "Profile", "VirtualEnvironment", "Prediction", 45 | "Reference", "Resource", "Training", "Experiment", "Sample", "FileArchives", 46 | "Workspace"} 47 | self.is_valid = 1 48 | del pars["check_sum"] 49 | del pars["model"] 50 | self.pars = pars 51 | self.user = None 52 | if target_model in self._protected_models: 53 | self.is_valid = 0 54 | if not self._valid(): 55 | self.is_valid = 0 56 | 57 | @property 58 | def protected_models(self): 59 | return self._protected_models 60 | 61 | def create_data(self): 62 | form_mod = __import__("..plugins.{model}", fromlist=["create_obj", ]) 63 | if form_mod.create_obj(self.pars, self.user): 64 | return Api.ret_info(valid=1, results="Saved") 65 | else: 66 | return Api.ret_info(valid=0, results="Failed") 67 | 68 | 69 | @csrf_exempt 70 | def plugin_post(request): 71 | models = set(model.__name__ for model in apps.get_models()) 72 | if "model" in request.POST and "API_key" in request.POST and "check_sum" in request.POST: 73 | if request.POST["model"] in models and request.POST["API_key"] is not None: 74 | api_obj = Api(request.POST["model"], request.POST["API_key"], 75 | request.POST.copy(), request.POST["check_sum"]) 76 | return api_obj.create_data() 77 | else: 78 | return JsonResponse() 79 | 80 | -------------------------------------------------------------------------------- /ui/views/Shield.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | # Created by: Li Yao 4 | # Created on: 5/25/20 5 | from django.contrib.auth.decorators import login_required 6 | from django.contrib.admin.views.decorators import staff_member_required 7 | from ..tools import error, success 8 | from worker.bases import get_config 9 | from QueueDB.models import Job 10 | import os 11 | 12 | 13 | def audit_job_atom(): 14 | a_counts = 0 15 | try: 16 | for q in Job.objects.filter(): 17 | if q.protocol_ver != q.protocol.ver and q.audit != 1: 18 | q.audit = 1 19 | q.save() 20 | a_counts += 1 21 | else: 22 | # otherwise release tag 23 | if q.audit > 0 and q.protocol_ver == q.protocol.ver: 24 | q.audit = 0 25 | q.save() 26 | return a_counts, "" 27 | except Exception as e: 28 | return -1, e 29 | 30 | 31 | @staff_member_required 32 | def audit_jobs(request): 33 | counts, msg = audit_job_atom() 34 | if counts == -1: 35 | return error(msg) 36 | else: 37 | return success("Checking finished, %d tags were put." % counts) 38 | 39 | 40 | def check_job_integrity(job_path): 41 | from configparser import ConfigParser 42 | snapshot_file = os.path.join(job_path, ".snapshot.ini") 43 | if not os.path.exists(snapshot_file): 44 | return 0 # compatible to old jobs 45 | else: 46 | snapshot = ConfigParser() 47 | snapshot.read(snapshot_file) 48 | for input_file, wow in snapshot['input'].items(): 49 | cctime = int(os.path.getctime(input_file)) 50 | cmtime = int(os.path.getmtime(input_file)) 51 | 52 | pctime, pmtime, pdigest = wow.split(";") 53 | pctime = int(pctime) 54 | pmtime = int(pmtime) 55 | if cctime == pctime and cmtime == pmtime: 56 | return 0 57 | else: 58 | return 1 59 | 60 | 61 | @login_required 62 | def clean_dead_folder(request): 63 | from shutil import rmtree 64 | jobs_containers = os.path.join(get_config('env', 'workspace'), str(request.user.queuedb_profile_related.delegate.id)) 65 | job_results = set( 66 | [re["result"] for re in Job.objects.filter(user_id=str(request.user.queuedb_profile_related.delegate.id)).values('result')]) 67 | protected_folders = set(["refs", "bin", "uploads", "archives", "OVERRIDE_UPLOAD"]) 68 | death_counter = 0 69 | failed = 0 70 | if os.path.exists(jobs_containers): 71 | for job_result in os.listdir(jobs_containers): 72 | if job_result not in job_results and job_result not in protected_folders: 73 | death_counter += 1 74 | try: 75 | abs_path = os.path.join(jobs_containers, job_result) 76 | if os.path.isdir(abs_path): 77 | rmtree(abs_path) 78 | except: 79 | failed += 1 80 | return success("%d/%d dead folder detected. %d of them failed to be cleaned." % (death_counter, 81 | len(job_results), 82 | failed)) 83 | 84 | 85 | @staff_member_required 86 | def clean_dead_lock(request): 87 | Job.objects.filter(status__gte=0).update(status=-3) 88 | Job.objects.filter(status=-2).update(status=-3) 89 | return success("Updated") 90 | 91 | 92 | @staff_member_required 93 | def update_bioqueue(request): 94 | try: 95 | update_py_path = os.path.split(os.path.split(os.path.realpath(__file__))[0])[0] + '/worker/update.py' 96 | update_command = 'python %s' % update_py_path 97 | if not os.system(update_command): 98 | return success('Your instance has been updated, please restart ' 99 | 'the ui1 server and queue service to apply the changes.') 100 | else: 101 | return error('An error occurred during the update process, please run update.py manually.') 102 | except Exception as e: 103 | return error(str(e)) 104 | -------------------------------------------------------------------------------- /ui/views/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | # Created by: Li Yao 4 | # Created on: 5/25/20 5 | from .Job import * 6 | from .Misc import * 7 | from .Protocol import * 8 | from .Plugins import * 9 | from .References import * 10 | from .Samples import * 11 | from .Shield import * 12 | 13 | -------------------------------------------------------------------------------- /version: -------------------------------------------------------------------------------- 1 | 2.0.0 -------------------------------------------------------------------------------- /worker/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | # Created by: Li Yao 4 | # Created on: 6/23/20 -------------------------------------------------------------------------------- /worker/cluster_models/HTCondor.tpl: -------------------------------------------------------------------------------- 1 | # {JOBNAME} 2 | executable = {PROTOCOL} 3 | log = {STDOUT} 4 | output = {STDOUT} 5 | error = {STDERR} 6 | initialdir = {WORKSPACE} 7 | request_cpus = {GLOBAL_MAX_CPU_FOR_CLUSTER} 8 | should_transfer_files = Yes 9 | when_to_transfer_output = ON_EXIT 10 | {MEM} 11 | queue -------------------------------------------------------------------------------- /worker/cluster_models/LSF.tpl: -------------------------------------------------------------------------------- 1 | #BSUB -q {DEFAULT_QUEUE} 2 | #BSUB -n {GLOBAL_MAX_CPU_FOR_CLUSTER} 3 | {MEM} 4 | #BSUB -o {STDOUT} 5 | #BSUB -outdir {WORKSPACE} 6 | #BSUB -cwd {WORKSPACE} 7 | #BSUB -e {STDERR} 8 | #BSUB -J {JOBNAME} 9 | {WALLTIME} 10 | {PROTOCOL} -------------------------------------------------------------------------------- /worker/cluster_models/TorquePBS.tpl: -------------------------------------------------------------------------------- 1 | #PBS -N {JOBNAME} 2 | #PBS -l {MEM}nodes=1:ppn={GLOBAL_MAX_CPU_FOR_CLUSTER} 3 | #PBS -e {STDERR} 4 | #PBS -o {STDOUT} 5 | #PBS -j oe 6 | {WALLTIME} 7 | #PBS -q {DEFAULT_QUEUE} 8 | 9 | cd {WORKSPACE} 10 | {PROTOCOL} -------------------------------------------------------------------------------- /worker/cluster_models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liyao001/BioQueue/4a364aa745fb1fb2e14d4ad60b5e9e9966898604/worker/cluster_models/__init__.py -------------------------------------------------------------------------------- /worker/cluster_models/cluster_model.py: -------------------------------------------------------------------------------- 1 | class ClusterModel: 2 | def alter_attribute(self, attribute): 3 | raise NotImplementedError 4 | 5 | def cancel_job(self): 6 | raise NotImplementedError 7 | 8 | def get_cluster_status(self): 9 | raise NotImplementedError 10 | 11 | def hold_job(self): 12 | raise NotImplementedError 13 | 14 | def load_template(self): 15 | raise NotImplementedError 16 | 17 | def query_job_status(self): 18 | raise NotImplementedError 19 | 20 | def release_job(self): 21 | raise NotImplementedError 22 | 23 | def submit_job(self, protocol, job_id, job_step, cpu=0, queue='', workspace=''): 24 | raise NotImplementedError 25 | -------------------------------------------------------------------------------- /worker/cluster_support.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import print_function 3 | import time 4 | import os 5 | from . import django_initial 6 | from QueueDB.models import Job, Training 7 | 8 | 9 | def if_terminate(job_id): 10 | """ 11 | If terminate a job 12 | :param job_id: int 13 | :return: 14 | """ 15 | try: 16 | job = Job.objects.get(id=job_id) 17 | return job.ter 18 | except: 19 | return 1 20 | 21 | 22 | def get_cluster_models(): 23 | """ 24 | Get cluster modules 25 | :return: list, module names 26 | """ 27 | models = [] 28 | models_path = os.path.join(os.path.split(os.path.realpath(__file__))[0], 'cluster_models') 29 | for model_name in os.listdir(models_path): 30 | if not model_name.endswith('.py') or model_name.startswith('_') or model_name.startswith('cluster'): 31 | continue 32 | models.append(model_name.replace('.py', '')) 33 | return models 34 | 35 | 36 | def dispatch(cluster_type): 37 | """ 38 | Load cluster module 39 | :param cluster_type: string 40 | :return: mixed, module or None 41 | """ 42 | models = get_cluster_models() 43 | if cluster_type in models: 44 | try: 45 | model = __import__("cluster_models." + cluster_type, fromlist=[cluster_type]) 46 | return model 47 | except: 48 | return None 49 | else: 50 | return None 51 | 52 | 53 | def main(cluster_type, parameter, job_id, step_id, cpu, mem, vrt_mem, queue, workspace, log_path, wall_time='', learning=0, trace_id=0): 54 | """ 55 | Cluster support function 56 | :param cluster_type: string, cluster type, like TorquePBS 57 | :param parameter: string, job parameter 58 | :param job_id: int, job id 59 | :param step_id: int, step order 60 | :param cpu: int, cpu cores 61 | :param mem: string, allocate memory 62 | :param queue: string, queue name 63 | :param workspace: string, job path 64 | :param log_path: string, path to store job logs 65 | :param wall_time: string, CPU time limit for a job 66 | :param learning: int 67 | :param trace_id: int 68 | :return: int 69 | """ 70 | cluster_model = dispatch(cluster_type) 71 | base_name = str(job_id) + '_' + str(step_id) 72 | ml_file_name = os.path.join(workspace, base_name + ".mlc") 73 | pending_tag = 0 74 | if cluster_model: 75 | if learning == 0: 76 | cluster_id = cluster_model.submit_job(parameter, job_id, step_id, cpu, mem, vrt_mem, queue, 77 | log_path, wall_time, workspace) 78 | else: 79 | tmp_filename = os.path.join(workspace, base_name + ".tmp") 80 | tmp_file = open(tmp_filename, mode='w') 81 | tmp_file.write(parameter) 82 | tmp_file.close() 83 | 84 | ml_parameter = "python %s -j %s -w %s -o %s" % \ 85 | (os.path.join(os.path.split(os.path.realpath(__file__))[0], "ml_container.py"), 86 | os.path.join(os.path.split(os.path.realpath(__file__))[0], tmp_filename), 87 | workspace, ml_file_name) 88 | 89 | cluster_id = cluster_model.submit_job(ml_parameter, job_id, step_id, cpu, mem, vrt_mem, queue, 90 | log_path, wall_time, workspace) 91 | if cluster_id == 0: 92 | # fail to submit 93 | return 1 94 | while True: 95 | status_code = cluster_model.query_job_status(cluster_id) 96 | if status_code == 1 or status_code == 2: 97 | # running or queueing 98 | if status_code == 2 and pending_tag == 0: 99 | # queueing 100 | pending_tag = 1 101 | try: 102 | job = Job.objects.get(id=job_id) 103 | job.status = job.set_wait(5) 104 | except: 105 | pass 106 | 107 | if status_code == 1 and pending_tag == 1: 108 | pending_tag = 0 109 | try: 110 | job = Job.objects.get(id=job_id) 111 | job.set_status(step_id + 1) 112 | except: 113 | pass 114 | 115 | if if_terminate(job_id): 116 | cluster_model.cancel_job(cluster_id) 117 | break 118 | time.sleep(30) 119 | elif status_code == 0: 120 | try: 121 | os.remove(tmp_filename) 122 | except: 123 | pass 124 | if learning == 1: 125 | # load learning results 126 | try: 127 | import pickle 128 | with open(ml_file_name, "rb") as handler: 129 | res = pickle.load(handler) 130 | training_item = Training.objects.get(id=trace_id) 131 | training_item.update_cpu_mem(res['cpu'], res['mem'], res['vrt_mem']) 132 | os.remove(ml_file_name) 133 | except: 134 | pass 135 | return 0 136 | else: 137 | return 1 138 | else: 139 | print('Unknown Cluster') 140 | -------------------------------------------------------------------------------- /worker/compile_tool.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 03/12/2017 10:16 AM 4 | # @Project : BioQueue 5 | # @Author : Li Yao 6 | # @File : compile_tool.py 7 | from __future__ import print_function 8 | import getopt 9 | import os 10 | import subprocess 11 | import sys 12 | 13 | 14 | def get_maintenance_models(): 15 | """ 16 | Get maintenance models 17 | :return: list, module names 18 | """ 19 | protocols = [] 20 | protocols_path = os.path.join(os.path.split(os.path.realpath(__file__))[0], 'maintenance_models') 21 | for model_name in os.listdir(protocols_path): 22 | if not model_name.endswith('.py') or model_name.startswith('_') or model_name.startswith('maintenance'): 23 | continue 24 | protocols.append(model_name.replace('.py', '')) 25 | return protocols 26 | 27 | 28 | def main(compile_method, workspace, user_bin, enter_dir=""): 29 | if compile_method in get_maintenance_models(): 30 | if enter_dir != "": 31 | workspace = os.path.join(workspace, enter_dir) 32 | model = __import__("maintenance_models." + compile_method, fromlist=[compile_method]) 33 | steps = model.get_method() 34 | for step in steps: 35 | command = step['software']+' '+step['parameter'] 36 | command.replace('{UserBin}', user_bin) 37 | step_process = subprocess.Popen(command, shell=True, cwd=workspace) 38 | process_id = step_process.pid 39 | step_process.wait() 40 | if step_process.returncode != 0: 41 | sys.exit(1) 42 | else: 43 | sys.exit(1) 44 | 45 | 46 | if __name__ == '__main__': 47 | try: 48 | opts, args = getopt.getopt(sys.argv[1:], "c:w:u:s:", ["compile=", "workspace=", "user_bin=", "enter_dir="]) 49 | except getopt.GetoptError as err: 50 | print(str(err)) 51 | sys.exit() 52 | 53 | if len(opts) == 0: 54 | sys.exit() 55 | 56 | cm = '' 57 | w = '' 58 | ub = '' 59 | ed = '' 60 | 61 | for o, a in opts: 62 | if o in ("-c", "--compile"): 63 | cm = a 64 | elif o in ("-w", "--workspace"): 65 | w = a 66 | elif o in ("-u", "--user_bin"): 67 | ub = a 68 | elif o in ("-s", "--enter_dir"): 69 | ed = a 70 | main(cm, w, ub, ed) 71 | -------------------------------------------------------------------------------- /worker/django_initial.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | from django.core.wsgi import get_wsgi_application 5 | 6 | sys.path.append(os.path.split(os.path.split(os.path.realpath(__file__))[0])[0]) 7 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "BioQueue.settings") 8 | 9 | application = get_wsgi_application() 10 | -------------------------------------------------------------------------------- /worker/feedback.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time: 2017/2/27 12:24 4 | # @Author: Li Yao 5 | # @File: feedback.py 6 | # @License: Apache 7 | # @Bitbutcket: https://bitbucket.org/li_yao/ 8 | # @Github: https://github.com/liyao001 9 | import requests 10 | import base64 11 | from .bases import get_config, get_bioqueue_version 12 | 13 | 14 | def feedback(software, parameter, mail): 15 | """ 16 | Usage feedback 17 | :param software: 18 | :param parameter: 19 | :param hash: 20 | :return: 21 | """ 22 | ver = base64.b64encode(get_bioqueue_version()) 23 | software = base64.b64encode(software) 24 | parameter = base64.b64encode(parameter) 25 | get_data_dict = {'ver': base64.b64encode(get_bioqueue_version()), 26 | 'software': software, 27 | 'parameter': parameter, 28 | 'email': mail, 29 | } 30 | try: 31 | fb_url = get_config('program', 'api', 1) + '/Gate/feedback' 32 | _ = requests.get(fb_url, params=get_data_dict, timeout=3) 33 | except: 34 | pass 35 | 36 | 37 | def get_error_log(file_path): 38 | """ 39 | Get error log of a job 40 | :param file_path: string, path to the log 41 | :return: string, log 42 | """ 43 | error_log = '' 44 | try: 45 | from bases import get_job_log 46 | error_log = get_job_log(file_path) 47 | except: 48 | pass 49 | return error_log 50 | 51 | 52 | def feedback_protocol(user_mail, formatted_protocol): 53 | """ 54 | Feedback protocol 55 | :param user_mail: 56 | :param formatted_protocol: 57 | :return: 58 | """ 59 | import json 60 | post_data_dict = { 61 | 'email': user_mail, 62 | 'pro': formatted_protocol, 63 | } 64 | 65 | try: 66 | fb_url = get_config('program', 'api', 1) + '/Protocol/protocol_feedback' 67 | # _ = requests.post(fb_url, data=post_data_dict, timeout=5) 68 | data = requests.post(fb_url, data=post_data_dict, timeout=5) 69 | protocol_json = json.loads(data.text) 70 | return protocol_json 71 | except Exception as e: 72 | print(e) 73 | return None 74 | 75 | 76 | def feedback_error(software, parameter, error_message, mail=''): 77 | """ 78 | Feedback error 79 | :param software: string, software name 80 | :param parameter: string, parameter 81 | :param error_message: string, error message 82 | :return: 83 | """ 84 | post_data_dict = { 85 | 'ver': get_bioqueue_version(), 86 | 'software': software, 87 | 'parameter': parameter, 88 | 'error': error_message, 89 | 'email': mail, 90 | } 91 | try: 92 | fb_url = get_config('program', 'api', 1) + '/Gate/error_feedback' 93 | _ = requests.post(fb_url, data=post_data_dict, timeout=3) 94 | except: 95 | pass 96 | 97 | 98 | def feedback_checkpoint(software, parameter, hash, cpu_a, cpu_b, cpu_r, mem_a, mem_b, mem_r, disk_a, disk_b, disk_r, mail=''): 99 | """ 100 | Feedback checkpoint 101 | :param software: string, software name 102 | :param parameter: string, parameter 103 | :param hash: string, hash 104 | :param cpu_a: float, cpu a 105 | :param cpu_b: float, cpu b 106 | :param cpu_r: float, cpu r 107 | :param mem_a: float, memory a 108 | :param mem_b: float, memory b 109 | :param mem_r: float, memory r 110 | :param disk_a: float, disk a 111 | :param disk_b: float, disk b 112 | :param disk_r: float, disk r 113 | :return: 114 | """ 115 | from ui.tools import os_to_int 116 | post_data_dict = dict() 117 | post_data_dict['cpu'] = get_config('env', 'cpu') 118 | post_data_dict['mem'] = get_config('env', 'memory') 119 | post_data_dict['os'] = os_to_int() 120 | post_data_dict['software'] = software 121 | post_data_dict['parameter'] = parameter 122 | post_data_dict['hash'] = hash 123 | post_data_dict['parent'] = 1 124 | if abs(float(cpu_r)) > 0.5: 125 | post_data_dict['cpu_a'] = cpu_a 126 | post_data_dict['cpu_b'] = cpu_b 127 | post_data_dict['cpu_r'] = cpu_r 128 | if abs(float(mem_r)) > 0.5: 129 | post_data_dict['mem_a'] = mem_a 130 | post_data_dict['mem_b'] = mem_b 131 | post_data_dict['mem_r'] = mem_r 132 | if abs(float(disk_r)) > 0.5: 133 | post_data_dict['disk_a'] = disk_a 134 | post_data_dict['disk_b'] = disk_b 135 | post_data_dict['disk_r'] = disk_r 136 | try: 137 | fb_url = get_config('program', 'api', 1) + '/Gate/cb_feedback' 138 | t = requests.post(fb_url, data=post_data_dict, timeout=3) 139 | except: 140 | pass 141 | -------------------------------------------------------------------------------- /worker/integrityTools.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | # Created by: Li Yao 4 | # Created on: 5/25/20 5 | import hashlib 6 | import os 7 | from configparser import ConfigParser 8 | from .parameterParser import upload_file_map, history_map 9 | 10 | 11 | def digest_check(file_path): 12 | """ 13 | Generate hash digest for a file 14 | 15 | :param file_path: 16 | :return: 17 | """ 18 | h = hashlib.blake2b() 19 | with open(file_path, "rb") as fh: 20 | for chunk in iter(lambda: fh.read(4096), b""): 21 | h.update(chunk) 22 | return h.hexdigest() 23 | 24 | 25 | def snapshot_a_job(job_path, input_files, db): 26 | """ 27 | Create snapshot for a job 28 | 29 | :param job_path: 30 | :param input_files: 31 | :param db: 32 | :return: 33 | """ 34 | snapshot = ConfigParser() 35 | snapshot.optionxform = str 36 | snapshot['input'] = dict() 37 | snapshot['output'] = dict() 38 | user_dir, _ = os.path.split(job_path) 39 | _, user_id = os.path.split(user_dir) 40 | 41 | if os.path.exists(job_path): 42 | for f in os.listdir(job_path): 43 | if f == ".snapshot.ini": 44 | continue 45 | full_path = os.path.join(job_path, f) 46 | if os.path.isfile(full_path): 47 | ctime = os.path.getctime(full_path) 48 | mtime = os.path.getmtime(full_path) 49 | digest = digest_check(full_path) 50 | snapshot['output'][f] = "%d;%d;%s" % (ctime, mtime, digest) 51 | 52 | parsed_uploaded_files, _ = upload_file_map(input_files, user_dir) 53 | parsed_history_files, _, _ = history_map(parsed_uploaded_files, user_id, db) 54 | parsed_inputs = parsed_history_files.split(";") 55 | for input_file in parsed_inputs: 56 | if os.path.exists(input_file) and os.path.isfile(input_file): 57 | ctime = os.path.getctime(input_file) 58 | mtime = os.path.getmtime(input_file) 59 | digest = digest_check(input_file) 60 | snapshot['input'][input_file] = "%d;%d;%s" % (ctime, mtime, digest) 61 | 62 | with open(os.path.join(job_path, ".snapshot.ini"), 'w') as configfile: 63 | snapshot.write(configfile) 64 | -------------------------------------------------------------------------------- /worker/maintenance_models/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 03/12/2017 11:20 AM 4 | # @Project : BioQueue 5 | # @Author : Li Yao 6 | # @File : __init__.py -------------------------------------------------------------------------------- /worker/maintenance_models/make.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 03/12/2017 11:22 AM 4 | # @Project : BioQueue 5 | # @Author : Li Yao 6 | # @File : make.py 7 | def get_method(): 8 | steps = ({ 9 | 'software': 'make', 10 | 'parameter': '', 11 | },) 12 | return steps 13 | -------------------------------------------------------------------------------- /worker/ml_collector.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin python 2 | from __future__ import print_function 3 | import psutil 4 | import time 5 | import getopt 6 | import sys 7 | import django_initial 8 | from QueueDB.models import Training 9 | 10 | vrt_mem_list = [] 11 | mem_list = [] 12 | cpu_list = [] 13 | read_list = [] 14 | write_list = [] 15 | 16 | 17 | def get_mem(pid): 18 | running_process = psutil.Process(pid) 19 | if running_process.is_running(): 20 | mem = running_process.memory_info() 21 | return mem[0], mem[1] 22 | else: 23 | return 0, 0 24 | 25 | 26 | def get_cpu(pid): 27 | running_process = psutil.Process(pid) 28 | if running_process.is_running(): 29 | cpu = running_process.cpu_percent(interval=1) 30 | return cpu 31 | else: 32 | return 0 33 | 34 | 35 | def get_io(pid): 36 | running_process = psutil.Process(pid) 37 | if running_process.is_running(): 38 | io = running_process.io_counters() 39 | return io[2], io[3] 40 | else: 41 | return 0 42 | 43 | 44 | def get_cpu_mem(cpu_list, mem_list, virt_mem_list): 45 | """ 46 | Get CPU and memory usage 47 | :param cpu_list: list, cpu usage info 48 | :param mem_list: list, memory usage info 49 | :return: tuple, cpu_usage and mem_usage 50 | """ 51 | if len(mem_list) > 0: 52 | mem_usage = max(mem_list) 53 | else: 54 | mem_usage = -1 55 | if len(virt_mem_list) > 0: 56 | virt_mem_usage = max(virt_mem_list) 57 | else: 58 | virt_mem_usage = -1 59 | if len(cpu_list) > 2: 60 | samples = int(round(len(cpu_list) * 0.5)) 61 | cpu_list.sort(reverse=True) 62 | cpu_usage = sum(cpu_list[0:samples]) / samples 63 | elif len(cpu_list) > 0: 64 | cpu_usage = sum(cpu_list) / len(cpu_list) 65 | else: 66 | cpu_usage = -1 67 | return cpu_usage, mem_usage, virt_mem_usage 68 | 69 | 70 | def main(): 71 | 72 | try: 73 | opts, args = getopt.getopt(sys.argv[1:], "n:p:j:", ["protocolStep=", "pid=", "job_id="]) 74 | except getopt.GetoptError as err: 75 | print(str(err)) 76 | sys.exit() 77 | 78 | if len(opts) == 0: 79 | sys.exit() 80 | 81 | step_hash = '' 82 | process_id = 0 83 | job_id = 0 84 | 85 | for o, a in opts: 86 | if o in ("-n", "--protocolStep"): 87 | step_hash = a 88 | elif o in ("-p", "--pid"): 89 | process_id = int(a) 90 | elif o in ("-j", "--job_id"): 91 | job_id = int(a) 92 | 93 | if step_hash != '' and process_id != 0: 94 | while True: 95 | 96 | if process_id in psutil.pids(): 97 | process_info = psutil.Process(process_id) 98 | 99 | if process_info.is_running(): 100 | try: 101 | total_memory_usage, vrt = get_mem(process_id) 102 | total_cpu_usage = get_cpu(process_id) 103 | children = process_info.children() 104 | for child in children: 105 | t1, t2 = get_mem(child.pid) 106 | total_memory_usage += t1 107 | vrt += t2 108 | total_cpu_usage += get_cpu(child.pid) 109 | mem_list.append(total_memory_usage) 110 | vrt_mem_list.append(vrt) 111 | cpu_list.append(total_cpu_usage) 112 | time.sleep(30) 113 | except Exception as e: 114 | print(e) 115 | break 116 | else: 117 | break 118 | else: 119 | break 120 | 121 | cpu_usage, mem_usage, vrt_mem_usage = get_cpu_mem(cpu_list, mem_list, vrt_mem_list) 122 | 123 | try: 124 | training_item = Training.objects.get(id=job_id) 125 | training_item.mem = mem_usage 126 | training_item.vrt_mem = vrt_mem_usage 127 | training_item.cpu = cpu_usage 128 | training_item.save() 129 | except: 130 | pass 131 | 132 | else: 133 | sys.exit() 134 | 135 | if __name__ == '__main__': 136 | main() 137 | -------------------------------------------------------------------------------- /worker/ml_container.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin python 2 | from __future__ import print_function 3 | import getopt 4 | import time 5 | import subprocess 6 | import psutil 7 | import sys 8 | from ml_collector import get_cpu, get_mem, get_cpu_mem 9 | from bases import check_shell_sig 10 | import django_initial 11 | from QueueDB.models import Training 12 | import pickle 13 | 14 | 15 | def get_protocol(fn): 16 | pf = open(fn) 17 | tmp = pf.readlines() 18 | return tmp 19 | 20 | 21 | def main(pf, wd, output_file): 22 | protocol = get_protocol(pf) 23 | for step in protocol: 24 | vrt_mem_list = [] 25 | mem_list = [] 26 | cpu_list = [] 27 | from parameterParser import parameter_string_to_list 28 | parameters = parameter_string_to_list(step) 29 | 30 | true_shell = check_shell_sig(parameters) 31 | 32 | if true_shell: 33 | proc = subprocess.Popen(step, shell=True, cwd=wd) 34 | else: 35 | proc = subprocess.Popen(parameters, shell=False, stdout=None, stderr=None, cwd=wd) 36 | 37 | process_id = proc.pid 38 | 39 | while proc.poll() is None: 40 | if process_id in psutil.pids(): 41 | proc_info = psutil.Process(process_id) 42 | 43 | if proc_info.is_running(): 44 | try: 45 | total_memory_usage, vrt = get_mem(process_id) 46 | total_cpu_usage = get_cpu(process_id) 47 | children = proc_info.children() 48 | for child in children: 49 | t1, t2 = get_mem(child.pid) 50 | total_memory_usage += t1 51 | vrt += t2 52 | # total_memory_usage += get_mem(child.pid) 53 | total_cpu_usage += get_cpu(child.pid) 54 | mem_list.append(total_memory_usage) 55 | vrt_mem_list.append(vrt) 56 | cpu_list.append(total_cpu_usage) 57 | except Exception as e: 58 | print(e) 59 | time.sleep(10) 60 | 61 | cpu_usage, mem_usage, vrt_mem_usage = get_cpu_mem(cpu_list, mem_list, vrt_mem_list) 62 | # save results to local file 63 | result = {'cpu': cpu_usage, 'mem': mem_usage, 'vrt_mem': vrt_mem_usage} 64 | with open(output_file, 'wb') as handler: 65 | pickle.dump(result, handler) 66 | 67 | if proc.returncode != 0: 68 | import sys 69 | sys.exit(1) 70 | 71 | 72 | if __name__ == '__main__': 73 | try: 74 | opts, args = getopt.getopt(sys.argv[1:], "j:w:o:", ["job=", "workdir=", "output="]) 75 | except getopt.GetoptError as err: 76 | print(str(err)) 77 | sys.exit() 78 | if len(opts) == 0: 79 | sys.exit() 80 | job = '' 81 | work_dir = '' 82 | trace_id = 0 83 | output_file = '' 84 | for o, a in opts: 85 | if o in ("-j", "--job"): 86 | job = a 87 | elif o in ("-w", "--workdir"): 88 | work_dir = a 89 | elif o in ("-o", "--output"): 90 | output_file = a 91 | main(job, work_dir, output_file) 92 | --------------------------------------------------------------------------------