{{ form.non_field_errors }}
6 | {% endif %}
7 |
52 | {% endblock center_content %}
--------------------------------------------------------------------------------
/djangui/tests/test_models.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 | from django.test import TestCase, Client
4 |
5 | from . import factories, config, mixins
6 | from .. import version
7 |
8 | class ScriptTestCase(mixins.ScriptFactoryMixin, TestCase):
9 |
10 | def test_script_creation(self):
11 | script = factories.TranslateScriptFactory()
12 |
13 |
14 | class ScriptGroupTestCase(TestCase):
15 |
16 | def test_script_group_creation(self):
17 | group = factories.ScriptGroupFactory()
18 |
19 |
20 | class TestJob(mixins.ScriptFactoryMixin, mixins.FileCleanupMixin, TestCase):
21 | urls = 'djangui.test_urls'
22 |
23 | def test_jobs(self):
24 | script = factories.TranslateScriptFactory()
25 | from ..backend import utils
26 | from .. import settings
27 | # the test server doesn't have celery running
28 | settings.DJANGUI_CELERY = False
29 | job = utils.create_djangui_job(script_pk=script.pk, data={'job_name': 'abc', 'sequence': 'aaa', 'out': 'abc'})
30 | job = job.submit_to_celery()
31 | old_pk = job.pk
32 | new_job = job.submit_to_celery(resubmit=True)
33 | self.assertNotEqual(old_pk, new_job.pk)
34 | # test rerunning, our output should be removed
35 | from ..models import DjanguiFile
36 | old_output = sorted([i.pk for i in DjanguiFile.objects.filter(job=new_job)])
37 | job.submit_to_celery(rerun=True)
38 | # check that we overwrite our output
39 | new_output = sorted([i.pk for i in DjanguiFile.objects.filter(job=new_job)])
40 | # Django 1.6 has a bug where they are reusing pk numbers
41 | if version.DJANGO_VERSION >= version.DJ17:
42 | self.assertNotEqual(old_output, new_output)
43 | self.assertEqual(len(old_output), len(new_output))
44 | # check the old entries are gone
45 | if version.DJANGO_VERSION >= version.DJ17:
46 | # Django 1.6 has a bug where they are reusing pk numbers, so once again we cannot use this check
47 | self.assertEqual([], list(DjanguiFile.objects.filter(pk__in=old_output)))
48 |
49 | # check our download links are ok
50 | file_previews = utils.get_file_previews(job)
51 | for group, files in file_previews.items():
52 | for fileinfo in files:
53 | response = Client().get(fileinfo.get('url'))
54 | self.assertEqual(response.status_code, 200)
55 |
56 | job = utils.create_djangui_job(script_pk=script.pk,
57 | data={'fasta': open(os.path.join(config.DJANGUI_TEST_DATA, 'fasta.fasta')),
58 | 'out': 'abc', 'job_name': 'abc'})
59 |
60 | # check our upload link is ok
61 | file_previews = utils.get_file_previews(job)
62 | for group, files in file_previews.items():
63 | for fileinfo in files:
64 | response = Client().get(fileinfo.get('url'))
65 | self.assertEqual(response.status_code, 200)
--------------------------------------------------------------------------------
/djangui/views/authentication.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 | from django.views.generic import CreateView
3 | from django.http import HttpResponseRedirect
4 | from django.core.urlresolvers import reverse
5 | from django.forms.models import modelform_factory
6 | from django.contrib.auth import login, authenticate, get_user_model
7 | from django.utils.translation import gettext_lazy as _
8 | from django.utils.encoding import force_text
9 |
10 | from .. import settings as djangui_settings
11 | from ..django_compat import JsonResponse
12 |
13 | class DjanguiRegister(CreateView):
14 | template_name = 'djangui/registration/register.html'
15 | model = get_user_model()
16 | fields = ('username', 'email', 'password')
17 |
18 | def dispatch(self, request, *args, **kwargs):
19 | if djangui_settings.DJANGUI_AUTH is False:
20 | return HttpResponseRedirect(djangui_settings.DJANGUI_REGISTER_URL)
21 | return super(DjanguiRegister, self).dispatch(request, *args, **kwargs)
22 |
23 | def post(self, request, *args, **kwargs):
24 | self.object = None
25 | form = self.get_form_class()
26 | post = request.POST.copy()
27 | post['username'] = post['username'].lower()
28 | form = form(post)
29 | if request.POST['password'] != request.POST['password2']:
30 | form.add_error('password', _('Passwords do not match.'))
31 | if request.POST['username'].lower() == 'admin':
32 | form.add_error('username', _('Reserved username.'))
33 | if not request.POST['email']:
34 | form.add_error('email', _('Please enter your email address.'))
35 | if form.is_valid():
36 | return self.form_valid(form)
37 | else:
38 | return self.form_invalid(form)
39 |
40 | def get_success_url(self):
41 | next_url = self.request.POST.get('next')
42 | # for some bizarre reason the password isn't setting by the modelform
43 | self.object.set_password(self.request.POST['password'])
44 | self.object.save()
45 | auser = authenticate(username=self.object.username, password=self.request.POST['password'])
46 | login(self.request, auser)
47 | return reverse(next_url) if next_url else reverse('djangui:djangui_home')
48 |
49 | def djangui_login(request):
50 | if djangui_settings.DJANGUI_AUTH is False:
51 | return HttpResponseRedirect(djangui_settings.DJANGUI_LOGIN_URL)
52 | User = get_user_model()
53 | form = modelform_factory(User, fields=('username', 'password'))
54 | user = User.objects.filter(username=request.POST.get('username'))
55 | if user:
56 | user = user[0]
57 | else:
58 | user = None
59 | form = form(request.POST, instance=user)
60 | if form.is_valid():
61 | data = form.cleaned_data
62 | user = authenticate(username=data['username'], password=data['password'])
63 | if user is None:
64 | return JsonResponse({'valid': False, 'errors': {'__all__': [force_text(_('You have entered an invalid username or password.'))]}})
65 | login(request, user)
66 | return JsonResponse({'valid': True, 'redirect': request.POST['next']})
67 | else:
68 | return JsonResponse({'valid': False, 'errors': form.errors})
--------------------------------------------------------------------------------
/djangui/tests/scripts/translate.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | __author__ = 'chris'
3 | import argparse
4 | import sys
5 |
6 | BASE_PAIR_COMPLEMENTS = {'a': 't', 't': 'a', 'c': 'g', 'g': 'c', 'A': 'T', 'T': 'A', 'C': 'G', 'G': 'C', 'n': 'n', 'N': 'N'}
7 |
8 | CODON_TABLE = {'AAA': 'K', 'AAC': 'N', 'AAG': 'K', 'AAT': 'N', 'ACA': 'T', 'ACC': 'T', 'ACG': 'T', 'ACT': 'T',
9 | 'AGA': 'R', 'AGC': 'S', 'AGG': 'R', 'AGT': 'S', 'ATA': 'I', 'ATC': 'I', 'ATG': 'M', 'ATT': 'I',
10 | 'CAA': 'Q', 'CAC': 'H', 'CAG': 'Q', 'CAT': 'H', 'CCA': 'P', 'CCC': 'P', 'CCG': 'P', 'CCT': 'P',
11 | 'CGA': 'R', 'CGC': 'R', 'CGG': 'R', 'CGT': 'R', 'CTA': 'L', 'CTC': 'L', 'CTG': 'L', 'CTT': 'L',
12 | 'GAA': 'E', 'GAC': 'D', 'GAG': 'E', 'GAT': 'D', 'GCA': 'A', 'GCC': 'A', 'GCG': 'A', 'GCT': 'A',
13 | 'GGA': 'G', 'GGC': 'G', 'GGG': 'G', 'GGT': 'G', 'GTA': 'V', 'GTC': 'V', 'GTG': 'V', 'GTT': 'V',
14 | 'TAA': '*', 'TAC': 'Y', 'TAG': '*', 'TAT': 'Y', 'TCA': 'S', 'TCC': 'S', 'TCG': 'S', 'TCT': 'S',
15 | 'TGA': '*', 'TGC': 'C', 'TGG': 'W', 'TGT': 'C', 'TTA': 'L', 'TTC': 'F', 'TTG': 'L', 'TTT': 'F',
16 | 'NNN': 'X'}
17 |
18 | for i in 'ACTG':
19 | for j in 'ACTG':
20 | CODON_TABLE['%s%sN' % (i,j)] = 'X'
21 | CODON_TABLE['%sN%s' % (i,j)] = 'X'
22 | CODON_TABLE['N%s%s' % (i,j)] = 'X'
23 | CODON_TABLE['%sNN' % i] = 'X'
24 | CODON_TABLE['N%sN' % i] = 'X'
25 | CODON_TABLE['NN%s' % i] = 'X'
26 |
27 | parser = argparse.ArgumentParser(description="This will translate a given DNA sequence to protein.")
28 | group = parser.add_mutually_exclusive_group(required=True)
29 | group.add_argument('--sequence', help='The sequence to translate.', type=str)
30 | group.add_argument('--fasta', help='The fasta file to translate.', type=argparse.FileType('rb'))
31 | parser.add_argument('--frame', help='The frame to translate in.', type=str, choices=['+1', '+2', '+3', '-1', '-2', '-3'], default='+1')
32 | parser.add_argument('--out', help='The file to save translations to.', type=argparse.FileType('wb'))
33 |
34 | def main():
35 | args = parser.parse_args()
36 | seq = args.sequence
37 | fasta = args.fasta
38 |
39 | def translate(seq=None, frame=None):
40 | if frame.startswith('-'):
41 | seq = ''.join([BASE_PAIR_COMPLEMENTS.get(i, 'N') for i in seq])
42 | frame = int(frame[1])-1
43 | return ''.join([CODON_TABLE.get(seq[i:i+3], 'X') for i in xrange(frame, len(seq), 3) if i+3<=len(seq)])
44 |
45 | frame = args.frame
46 | with args.out as fasta_out:
47 | if fasta:
48 | with args.fasta as fasta_in:
49 | header = ''
50 | seq = ''
51 | for row in fasta_in:
52 | if row[0] == '>':
53 | if seq:
54 | fasta_out.write('{}\n{}\n'.format(header, translate(seq, frame)))
55 | header = row
56 | seq = ''
57 | else:
58 | seq += row.strip()
59 | if seq:
60 | fasta_out.write('{}\n{}\n'.format(header, translate(seq, frame)))
61 | else:
62 | fasta_out.write('{}\n{}\n'.format('>1', translate(seq.upper(), frame)))
63 |
64 | if __name__ == "__main__":
65 | sys.exit(main())
66 |
--------------------------------------------------------------------------------
/djangui/tests/scripts/nested_heatmap.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 |
3 | __author__ = 'chris'
4 | import argparse
5 | import os
6 | import sys
7 |
8 |
9 | parser = argparse.ArgumentParser(description="Create a nested heatmap from a delimited file.")
10 | parser.add_argument('--tsv', help='The delimited file to plot.', type=argparse.FileType('r'), required=True)
11 | parser.add_argument('--delimiter', help='The delimiter for fields. Default: tab', type=str, default='\t')
12 | parser.add_argument('--major-index', help='The first column to group by.', type=str, required=True)
13 | parser.add_argument('--minor-index', help='The second column to group by.', type=str, required=True)
14 | parser.add_argument('--minor-cutoff', help='The minimum number of minor entries grouped to be considered.', type=int, default=2)
15 | parser.add_argument('--log-normalize', help='Whether to log normalize data.', action='store_true')
16 | parser.add_argument('--translate', help='Whether to translate data so the minimum value is zero.', action='store_true')
17 |
18 | def main():
19 | args = parser.parse_args()
20 | import numpy as np
21 | import pandas as pd
22 | import seaborn as sns
23 | major_index = args.major_index
24 | minor_index = args.minor_index
25 | df = pd.read_table(args.tsv, index_col=[major_index, minor_index], sep=args.delimiter)
26 | df = np.log2(df) if args.log_normalize else df
27 | # set our undected samples to our lowest detection
28 | df[df==-1*np.inf] = df[df!=-1*np.inf].min().min()
29 | # translate our data so we have no negatives (which would screw up our addition and makes no biological sense)
30 | if args.translate:
31 | df+=abs(df.min().min())
32 | major_counts = df.groupby(level=[major_index]).count()
33 | # we only want to plot samples with multiple values in the minor index
34 | cutoff = args.minor_cutoff
35 | multi = df[df.index.get_level_values(major_index).isin(major_counts[major_counts>=cutoff].dropna().index)]
36 |
37 | # Let's select the most variable minor axis elements
38 | most_variable = multi.groupby(level=major_index).var().mean(axis=1).order(ascending=False)
39 | # and group by 20s
40 | for i in xrange(11):
41 | dat = multi[multi.index.get_level_values(major_index).isin(most_variable.index[10*i:10*(i+1)])]
42 | # we want to cluster by our major index, and then under these plot the values of our minor index
43 | major_dat = dat.groupby(level=major_index).sum()
44 | seaborn_map = sns.clustermap(major_dat, row_cluster=True, col_cluster=True)
45 | # now we keep this clustering, but recreate our data to fit the above clustering, with our minor
46 | # index below the major index (you can think of transcript levels under gene levels if you are
47 | # a biologist)
48 | merged_dat = pd.DataFrame(columns=[seaborn_map.data2d.columns])
49 | for major_val in seaborn_map.data2d.index:
50 | minor_rows = multi[multi.index.get_level_values(major_index)==major_val][seaborn_map.data2d.columns]
51 | major_row = major_dat.loc[major_val,][seaborn_map.data2d.columns]
52 | merged_dat.append(major_row)
53 | merged_dat = merged_dat.append(major_row).append(minor_rows)
54 | merged_map = sns.clustermap(merged_dat, row_cluster=False, col_cluster=False)
55 |
56 | # recreate our dendrogram, this is undocumented and probably a hack but it works
57 | seaborn_map.dendrogram_col.plot(merged_map.ax_col_dendrogram)
58 |
59 | # for rows, I imagine at some point it will fail to fall within the major axis but fortunately
60 | # for this dataset it is not true
61 | seaborn_map.dendrogram_row.plot(merged_map.ax_row_dendrogram)
62 | merged_map.savefig('{}_heatmap_{}.png'.format(os.path.split(args.tsv.name)[1], i))
63 |
64 | if __name__ == "__main__":
65 | sys.exit(main())
--------------------------------------------------------------------------------
/djangui/tests/test_views.py:
--------------------------------------------------------------------------------
1 | # TODO: Test for viewing a user's job as an anonymous user (fail case)
2 |
3 | import json
4 |
5 | from django.test import TestCase, RequestFactory, Client
6 | from django.core.urlresolvers import reverse
7 | from django.contrib.auth.models import AnonymousUser
8 |
9 | from . import factories, mixins
10 | from ..views import djangui_celery
11 |
12 | class CeleryViews(mixins.ScriptFactoryMixin, mixins.FileCleanupMixin, TestCase):
13 | def setUp(self):
14 | self.factory = RequestFactory()
15 |
16 | def test_celery_results(self):
17 | request = self.factory.get(reverse('djangui:celery_results'))
18 | user = factories.UserFactory()
19 | request.user = user
20 | response = djangui_celery.celery_status(request)
21 | d = response.content.decode("utf-8")
22 | self.assertEqual({'anon': [], 'user': []}, json.loads(d))
23 | job = factories.JobFactory()
24 | job.save()
25 | response = djangui_celery.celery_status(request)
26 | d = json.loads(response.content.decode("utf-8"))
27 | self.assertEqual(1, len(d['anon']))
28 | job.user = user
29 | job.save()
30 | response = djangui_celery.celery_status(request)
31 | d = json.loads(response.content.decode("utf-8"))
32 | # we now are logged in, make sure the job appears under the user jobs
33 | self.assertEqual(1, len(d['user']))
34 | user = AnonymousUser()
35 | request.user = user
36 | response = djangui_celery.celery_status(request)
37 | d = json.loads(response.content.decode("utf-8"))
38 | # test empty response since anonymous users should not see users jobs
39 | self.assertEqual({'anon': [], 'user': []}, d)
40 |
41 | def test_celery_commands(self):
42 | user = factories.UserFactory()
43 | job = factories.JobFactory()
44 | job.user = user
45 | job.save()
46 | celery_command = {'celery-command': ['delete'], 'job-id': [job.pk]}
47 | # test that we cannot modify a users script
48 | request = self.factory.post(reverse('djangui:celery_task_command'),
49 | celery_command)
50 | anon = AnonymousUser()
51 | request.user = anon
52 | response = djangui_celery.celery_task_command(request)
53 | d = response.content.decode("utf-8")
54 | self.assertFalse(json.loads(d).get('valid'))
55 |
56 | # test a nonsense command
57 | celery_command.update({'celery-command': ['thisshouldfail']})
58 | response = djangui_celery.celery_task_command(request)
59 | d = response.content.decode("utf-8")
60 | self.assertFalse(json.loads(d).get('valid'))
61 |
62 | # test that the user can interact with it
63 | request.user = user
64 | for i in ['resubmit', 'rerun', 'clone', 'stop', 'delete']:
65 | celery_command.update({'celery-command': [i]})
66 | response = djangui_celery.celery_task_command(request)
67 | d = response.content.decode("utf-8")
68 | self.assertTrue(json.loads(d).get('valid'))
69 |
70 | def test_celery_task_view(self):
71 | user = factories.UserFactory()
72 | job = factories.JobFactory()
73 | job.user = user
74 | job.save()
75 | # test that an anonymous user cannot view a user's job
76 | view = djangui_celery.CeleryTaskView.as_view()
77 | request = self.factory.get(reverse('djangui:celery_results_info', kwargs={'job_id': job.pk}))
78 | request.user = AnonymousUser()
79 | response = view(request, job_id=job.pk)
80 | self.assertIn('task_error', response.context_data)
81 | self.assertNotIn('task_info', response.context_data)
82 |
83 | # test the user can view the job
84 | request.user = user
85 | response = view(request, job_id=job.pk)
86 | self.assertNotIn('task_error', response.context_data)
87 | self.assertIn('task_info', response.context_data)
88 |
89 | # test that jobs that don't exist don't fail horridly
90 | request = self.factory.get(reverse('djangui:celery_results_info', kwargs={'job_id': '-1'}))
91 | response = view(request, job_id=-1)
92 | self.assertIn('task_error', response.context_data)
93 | self.assertNotIn('task_info', response.context_data)
--------------------------------------------------------------------------------
/djangui/backend/command_line.py:
--------------------------------------------------------------------------------
1 | import sys
2 | import os
3 | import subprocess
4 | import shutil
5 | from argparse import ArgumentParser
6 | from django.template import Context
7 | import djangui
8 | from .. import django_compat
9 |
10 | def which(pgm):
11 | # from http://stackoverflow.com/questions/9877462/is-there-a-python-equivalent-to-the-which-command
12 | path=os.getenv('PATH')
13 | for p in path.split(os.path.pathsep):
14 | p=os.path.join(p,pgm)
15 | if os.path.exists(p) and os.access(p, os.X_OK):
16 | return p
17 |
18 | def walk_dir(templates, dest, filter=None):
19 | l = []
20 | for root, folders, files in os.walk(templates):
21 | for filename in files:
22 | if filename.endswith('.pyc') or (filter and filename not in filter):
23 | continue
24 | relative_dir = '.{0}'.format(os.path.split(os.path.join(root, filename).replace(templates, ''))[0])
25 | l.append((os.path.join(root, filename), os.path.join(dest, relative_dir)))
26 | return l
27 |
28 | def bootstrap(env=None, cwd=None):
29 | if env is None:
30 | env = os.environ
31 | parser = ArgumentParser(description="Create a Django app with Djangui setup.")
32 | parser.add_argument('-p', '--project', help='The name of the django project to create.', type=str, required=True)
33 | args = parser.parse_args()
34 |
35 | project_name = args.project
36 | new_project = not os.path.exists(project_name)
37 | if not new_project:
38 | sys.stderr.write('Project {0} already exists.\n'.format(project_name))
39 | sys.exit(1)
40 | env['DJANGO_SETTINGS_MODULE'] = ''
41 | admin_command = [sys.executable] if sys.executable else []
42 | admin_path = which('django-admin.py')
43 | admin_command.extend([admin_path, 'startproject', project_name])
44 | admin_kwargs = {'env': env}
45 | if cwd is not None:
46 | admin_kwargs.update({'cwd': cwd})
47 | subprocess.call(admin_command, **admin_kwargs)
48 | project_root = project_name
49 | project_base_dir = os.path.join(os.path.realpath(os.path.curdir), project_root, project_name)
50 |
51 | djanguify_folder = os.path.split(os.path.realpath(djangui.__file__))[0]
52 | project_template_dir = os.path.join(djanguify_folder, 'conf', 'project_template')
53 |
54 | context = Context(
55 | dict({
56 | 'project_name': project_name,
57 | },
58 | autoescape=False
59 | ))
60 |
61 | template_files = []
62 |
63 | template_files += walk_dir(project_template_dir, project_base_dir)
64 |
65 | for template_file, dest_dir in template_files:
66 | template_file = open(template_file)
67 | content = template_file.read()
68 | template = django_compat.Engine().from_string(content)
69 | content = template.render(context)
70 | content = content.encode('utf-8')
71 | to_name = os.path.join(dest_dir, os.path.split(template_file.name)[1])
72 | try:
73 | os.mkdir(dest_dir)
74 | except:
75 | pass
76 | with open(to_name, 'wb') as new_file:
77 | new_file.write(content)
78 |
79 | # move the django settings to the settings path so we don't have to chase Django changes.
80 | shutil.move(os.path.join(project_base_dir, 'settings.py'), os.path.join(project_base_dir, 'settings', 'django_settings.py'))
81 | # do the same with urls
82 | shutil.move(os.path.join(project_base_dir, 'urls.py'), os.path.join(project_base_dir, 'urls', 'django_urls.py'))
83 | env['DJANGO_SETTINGS_MODULE'] = '.'.join([project_name, 'settings', 'user_settings'])
84 | if django_compat.DJANGO_VERSION >= django_compat.DJ17:
85 | subprocess.call(['python', os.path.join(project_root, 'manage.py'), 'makemigrations'], env=env)
86 | subprocess.call(['python', os.path.join(project_root, 'manage.py'), 'migrate'], env=env)
87 | else:
88 | subprocess.call(['python', os.path.join(project_root, 'manage.py'), 'syncdb', '--noinput'], env=env)
89 | subprocess.call(['python', os.path.join(project_root, 'manage.py'), 'collectstatic', '--noinput'], env=env)
90 | sys.stdout.write("Please enter the project directory {0}, and run python manage.py createsuperuser and"
91 | " python manage.py runserver to start. The admin can be found at localhost:8000/admin. You may also want to set your "
92 | "DJANGO_SETTINGS_MODULE environment variable to {0}.settings \n".format(project_name))
93 |
--------------------------------------------------------------------------------
/djangui/views/views.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 | from django.views.generic import DetailView, TemplateView
3 | from django.conf import settings
4 | from django.forms import FileField
5 | from django.utils.translation import gettext_lazy as _
6 | from django.utils.encoding import force_text
7 |
8 | from ..backend import utils
9 | from ..models import DjanguiJob, Script
10 | from .. import settings as djangui_settings
11 | from ..django_compat import JsonResponse
12 |
13 |
14 | class DjanguiScriptJSON(DetailView):
15 | model = Script
16 | slug_field = 'slug'
17 | slug_url_kwarg = 'script_name'
18 |
19 | def render_to_response(self, context, **response_kwargs):
20 | # returns the models required and optional fields as html
21 | job_id = self.kwargs.get('job_id')
22 | initial = None
23 | if job_id:
24 | job = DjanguiJob.objects.get(pk=job_id)
25 | if job.user is None or (self.request.user.is_authenticated() and job.user == self.request.user):
26 | initial = {}
27 | for i in job.get_parameters():
28 | value = i.value
29 | if value is not None:
30 | initial[i.parameter.slug] = value
31 | d = utils.get_form_groups(model=self.object, initial=initial)
32 | return JsonResponse(d)
33 |
34 | def post(self, request, *args, **kwargs):
35 | post = request.POST.copy()
36 | user = request.user if request.user.is_authenticated() else None
37 | if not djangui_settings.DJANGUI_ALLOW_ANONYMOUS and user is None:
38 | return JsonResponse({'valid': False, 'errors': {'__all__': [force_text(_('You are not permitted to access this script.'))]}})
39 | form = utils.get_master_form(pk=post['djangui_type'])
40 | # TODO: Check with people who know more if there's a smarter way to do this
41 | utils.validate_form(form=form, data=post, files=request.FILES)
42 | # for cloned jobs, we don't have the files in input fields, they'll be in a list like ['', filename]
43 | # This will cause issues.
44 | to_delete = []
45 | for i in post:
46 | if isinstance(form.fields.get(i), FileField):
47 | # if we have a value set, reassert this
48 | new_value = post.get(i)
49 | if i not in request.FILES and (i not in form.cleaned_data or (not form.cleaned_data[i] and new_value)):
50 | # this is a previously set field, so a cloned job
51 | if new_value is not None:
52 | form.cleaned_data[i] = utils.get_storage(local=False).open(new_value)
53 | to_delete.append(i)
54 | for i in to_delete:
55 | if i in form.errors:
56 | del form.errors[i]
57 |
58 | if not form.errors:
59 | # data = form.cleaned_data
60 | script_pk = form.cleaned_data.get('djangui_type')
61 | script = Script.objects.get(pk=script_pk)
62 | valid = utils.valid_user(script, request.user).get('valid')
63 | if valid is True:
64 | group_valid = utils.valid_user(script.script_group, request.user).get('valid')
65 | if valid is True and group_valid is True:
66 | job = utils.create_djangui_job(script_pk=script_pk, user=user, data=form.cleaned_data)
67 | job.submit_to_celery()
68 | return JsonResponse({'valid': True})
69 | return JsonResponse({'valid': False, 'errors': {'__all__': [force_text(_('You are not permitted to access this script.'))]}})
70 | return JsonResponse({'valid': False, 'errors': form.errors})
71 |
72 |
73 | class DjanguiHomeView(TemplateView):
74 | template_name = 'djangui/djangui_home.html'
75 |
76 | def get_context_data(self, **kwargs):
77 | job_id = self.request.GET.get('job_id')
78 | ctx = super(DjanguiHomeView, self).get_context_data(**kwargs)
79 | ctx['djangui_scripts'] = getattr(settings, 'DJANGUI_SCRIPTS', {})
80 | if job_id:
81 | job = DjanguiJob.objects.get(pk=job_id)
82 | if job.user is None or (self.request.user.is_authenticated() and job.user == self.request.user):
83 | ctx['clone_job'] = {'job_id': job_id, 'url': job.get_resubmit_url(), 'data_url': job.script.get_url()}
84 | return ctx
85 |
86 | class DjanguiProfileView(TemplateView):
87 | template_name = 'djangui/profile/profile_base.html'
--------------------------------------------------------------------------------
/djangui/conf/project_template/settings/user_settings.py:
--------------------------------------------------------------------------------
1 | from os import environ
2 | from .djangui_settings import *
3 |
4 | # This file is where the user can override and customize their installation of djangui
5 |
6 | # Djangui Apps - add additional apps here after the initial install (remember to follow everything by a comma)
7 |
8 | INSTALLED_APPS += (
9 | )
10 |
11 | # Whether to allow anonymous job submissions, set False to disallow 'guest' job submissions
12 | DJANGUI_ALLOW_ANONYMOUS = True
13 |
14 | ## Celery related options
15 | INSTALLED_APPS += (
16 | 'djcelery',
17 | 'kombu.transport.django',
18 | )
19 |
20 | CELERY_RESULT_BACKEND='djcelery.backends.database:DatabaseBackend'
21 | BROKER_URL = 'django://'
22 | CELERY_TRACK_STARTED = True
23 | DJANGUI_CELERY = True
24 | CELERY_SEND_EVENTS = True
25 |
26 | # Things you most likely do not need to change
27 |
28 | # the directory for uploads (physical directory)
29 | MEDIA_ROOT = os.path.join(BASE_DIR, 'user_uploads')
30 | # the url mapping
31 | MEDIA_URL = '/uploads/'
32 |
33 | # the directory to store our webpage assets (images, javascript, etc.)
34 | STATIC_ROOT = os.path.join(BASE_DIR, 'static')
35 | # the url mapping
36 | STATIC_URL = '/static/'
37 |
38 | ## Here is a setup example for production servers
39 |
40 | ## A postgres database -- for multiple users a sqlite based database is asking for trouble
41 |
42 | # DATABASES = {
43 | # 'default': {
44 | # 'ENGINE': 'django.db.backends.postgresql_psycopg2',
45 | # # for production environments, these should be stored as environment variables
46 | # # I also recommend the django-heroku-postgresify package for a super simple setup
47 | # 'NAME': os.environ.get('DATABASE_NAME', 'djangui'),
48 | # 'USER': os.environ.get('DATABASE_USER', 'djangui'),
49 | # 'PASSWORD': os.environ.get('DATABASE_PASSWORD', 'djangui'),
50 | # 'HOST': os.environ.get('DATABASE_URL', 'localhost'),
51 | # 'PORT': os.environ.get('DATABASE_PORT', '5432')
52 | # }
53 | # }
54 |
55 | ## A better celery backend -- using RabbitMQ (these defaults are from two free rabbitmq Heroku providers)
56 | # CELERY_RESULT_BACKEND = 'amqp'
57 | # BROKER_URL = os.environ.get('AMQP_URL') or \
58 | # os.environ.get('RABBITMQ_BIGWIG_TX_URL') or \
59 | # os.environ.get('CLOUDAMQP_URL', 'amqp://guest:guest@localhost:5672/')
60 | # BROKER_POOL_LIMIT = 1
61 | # CELERYD_CONCURRENCY = 1
62 | # CELERY_TASK_SERIALIZER = 'json'
63 | # ACKS_LATE = True
64 | #
65 |
66 | ## for production environments, django-storages abstracts away much of the difficulty of various storage engines.
67 | ## Here is an example for hosting static and user generated content with S3
68 |
69 | # from boto.s3.connection import VHostCallingFormat
70 | #
71 | # INSTALLED_APPS += (
72 | # 'storages',
73 | # 'collectfast',
74 | # )
75 |
76 | ## We have user authentication -- we need to use https (django-sslify)
77 | # if not DEBUG:
78 | # MIDDLEWARE_CLASSES = ['sslify.middleware.SSLifyMiddleware']+list(MIDDLEWARE_CLASSES)
79 | # SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
80 | #
81 | # ALLOWED_HOSTS = (
82 | # 'localhost',
83 | # '127.0.0.1',
84 | # "djangui.herokuapp.com",# put your site here
85 | # )
86 | #
87 | # AWS_CALLING_FORMAT = VHostCallingFormat
88 | #
89 | # AWS_ACCESS_KEY_ID = environ.get('AWS_ACCESS_KEY_ID', '')
90 | # AWS_SECRET_ACCESS_KEY = environ.get('AWS_SECRET_ACCESS_KEY', '')
91 | # AWS_STORAGE_BUCKET_NAME = environ.get('AWS_STORAGE_BUCKET_NAME', '')
92 | # AWS_AUTO_CREATE_BUCKET = True
93 | # AWS_QUERYSTRING_AUTH = False
94 | # AWS_S3_SECURE_URLS = True
95 | # AWS_FILE_OVERWRITE = False
96 | # AWS_PRELOAD_METADATA = True
97 | # AWS_S3_CUSTOM_DOMAIN = environ.get('AWS_S3_CUSTOM_DOMAIN', '')
98 | #
99 | # GZIP_CONTENT_TYPES = (
100 | # 'text/css',
101 | # 'application/javascript',
102 | # 'application/x-javascript',
103 | # 'text/javascript',
104 | # )
105 | #
106 | # AWS_EXPIREY = 60 * 60 * 7
107 | # AWS_HEADERS = {
108 | # 'Cache-Control': 'max-age=%d, s-maxage=%d, must-revalidate' % (AWS_EXPIREY,
109 | # AWS_EXPIREY)
110 | # }
111 | #
112 | # STATIC_URL = 'http://%s.s3.amazonaws.com/' % AWS_STORAGE_BUCKET_NAME
113 | # MEDIA_URL = '/user-uploads/'
114 | #
115 | # STATICFILES_STORAGE = DEFAULT_FILE_STORAGE = 'djangui.djanguistorage.CachedS3BotoStorage'
116 | # DJANGUI_EPHEMERAL_FILES = True
117 |
118 |
119 | AUTHENTICATION_BACKEND = 'django.contrib.auth.backends.ModelBackend'
--------------------------------------------------------------------------------
/djangui/forms/factory.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 | __author__ = 'chris'
3 | import copy
4 | import json
5 | import six
6 | from collections import OrderedDict
7 |
8 | from django import forms
9 |
10 | from .scripts import DjanguiForm
11 | from ..backend import utils
12 | from ..models import ScriptParameter
13 |
14 |
15 | class DjanguiFormFactory(object):
16 | djangui_forms = {}
17 |
18 | @staticmethod
19 | def get_field(param, initial=None):
20 | field = param.form_field
21 | choices = json.loads(param.choices)
22 | field_kwargs = {'label': param.script_param.title(),
23 | 'required': param.required,
24 | 'help_text': param.param_help,
25 | }
26 | if choices:
27 | field = 'ChoiceField'
28 | base_choices = [(None, '----')] if not param.required else []
29 | field_kwargs['choices'] = base_choices+[(str(i), str(i).title()) for i in choices]
30 | if field == 'FileField':
31 | if param.is_output:
32 | field = 'CharField'
33 | initial = None
34 | else:
35 | if initial is not None:
36 | initial = utils.get_storage_object(initial) if not hasattr(initial, 'path') else initial
37 | field_kwargs['widget'] = forms.ClearableFileInput()
38 | if initial is not None:
39 | field_kwargs['initial'] = initial
40 | field = getattr(forms, field)
41 | return field(**field_kwargs)
42 |
43 | def get_group_forms(self, model=None, pk=None, initial=None):
44 | pk = int(pk) if pk is not None else pk
45 | if pk is not None and pk in self.djangui_forms:
46 | if 'groups' in self.djangui_forms[pk]:
47 | return copy.deepcopy(self.djangui_forms[pk]['groups'])
48 | params = ScriptParameter.objects.filter(script=model).order_by('pk')
49 | # set a reference to the object type for POST methods to use
50 | script_id_field = forms.CharField(widget=forms.HiddenInput)
51 | group_map = {}
52 | for param in params:
53 | field = self.get_field(param, initial=initial.get(param.slug) if initial else None)
54 | group_id = -1 if param.required else param.parameter_group.pk
55 | group_name = 'Required' if param.required else param.parameter_group.group_name
56 | group = group_map.get(group_id, {
57 | 'group': group_name,
58 | 'fields': OrderedDict()
59 | })
60 | group['fields'][param.slug] = field
61 | group_map[group_id] = group
62 | # create individual forms for each group
63 | group_map = OrderedDict([(i, group_map[i]) for i in sorted(group_map.keys())])
64 | d = {'action': model.get_url()}
65 | d['groups'] = []
66 | pk = model.pk
67 | for group_index, group in enumerate(six.iteritems(group_map)):
68 | group_pk, group_info = group
69 | form = DjanguiForm()
70 | if group_index == 0:
71 | form.fields['djangui_type'] = script_id_field
72 | form.fields['djangui_type'].initial = pk
73 | for field_pk, field in six.iteritems(group_info['fields']):
74 | form.fields[field_pk] = field
75 | d['groups'].append({'group_name': group_info['group'], 'form': str(form)})
76 | try:
77 | self.djangui_forms[pk]['groups'] = d
78 | except KeyError:
79 | self.djangui_forms[pk] = {'groups': d}
80 | # if the master form doesn't exist, create it while we have the model
81 | if 'master' not in self.djangui_forms[pk]:
82 | self.get_master_form(model=model, pk=pk)
83 | return d
84 |
85 | def get_master_form(self, model=None, pk=None):
86 | pk = int(pk) if pk is not None else pk
87 | if pk is not None and pk in self.djangui_forms:
88 | if 'master' in self.djangui_forms[pk]:
89 | return copy.deepcopy(self.djangui_forms[pk]['master'])
90 | master_form = DjanguiForm()
91 | params = ScriptParameter.objects.filter(script=model).order_by('pk')
92 | # set a reference to the object type for POST methods to use
93 | pk = model.pk
94 | script_id_field = forms.CharField(widget=forms.HiddenInput)
95 | master_form.fields['djangui_type'] = script_id_field
96 | master_form.fields['djangui_type'].initial = pk
97 |
98 | for param in params:
99 | field = self.get_field(param)
100 | master_form.fields[param.slug] = field
101 | try:
102 | self.djangui_forms[pk]['master'] = master_form
103 | except KeyError:
104 | self.djangui_forms[pk] = {'master': master_form}
105 | # create the group forms while we have the model
106 | if 'groups' not in self.djangui_forms[pk]:
107 | self.get_group_forms(model=model, pk=pk)
108 | return master_form
109 |
110 | DJ_FORM_FACTORY = DjanguiFormFactory()
--------------------------------------------------------------------------------
/djangui/backend/ast/source_parser.py:
--------------------------------------------------------------------------------
1 | '''
2 | Created on Dec 11, 2013
3 |
4 | @author: Chris
5 |
6 | Collection of functions for extracting argparse related statements from the
7 | client code.
8 | '''
9 |
10 | import ast
11 | import _ast
12 | from itertools import *
13 |
14 | from . import codegen
15 |
16 |
17 | def parse_source_file(file_name):
18 | """
19 | Parses the AST of Python file for lines containing
20 | references to the argparse module.
21 |
22 | returns the collection of ast objects found.
23 |
24 | Example client code:
25 |
26 | 1. parser = ArgumentParser(desc="My help Message")
27 | 2. parser.add_argument('filename', help="Name of the file to load")
28 | 3. parser.add_argument('-f', '--format', help='Format of output \nOptions: ['md', 'html']
29 | 4. args = parser.parse_args()
30 |
31 | Variables:
32 | * nodes Primary syntax tree object
33 | * argparse_assignments The assignment of the ArgumentParser (line 1 in example code)
34 | * add_arg_assignments Calls to add_argument() (lines 2-3 in example code)
35 | * parser_var_name The instance variable of the ArgumentParser (line 1 in example code)
36 | * ast_source The curated collection of all parser related nodes in the client code
37 | """
38 | with open(file_name, 'r') as f:
39 | s = f.read()
40 |
41 | nodes = ast.parse(s)
42 |
43 | module_imports = get_nodes_by_instance_type(nodes, _ast.Import)
44 | specific_imports = get_nodes_by_instance_type(nodes, _ast.ImportFrom)
45 |
46 | assignment_objs = get_nodes_by_instance_type(nodes, _ast.Assign)
47 | call_objects = get_nodes_by_instance_type(nodes, _ast.Call)
48 |
49 | argparse_assignments = get_nodes_by_containing_attr(assignment_objs, 'ArgumentParser')
50 | group_arg_assignments = get_nodes_by_containing_attr(assignment_objs, 'add_argument_group')
51 | add_arg_assignments = get_nodes_by_containing_attr(call_objects, 'add_argument')
52 | parse_args_assignment = get_nodes_by_containing_attr(call_objects, 'parse_args')
53 | # there are cases where we have custom argparsers, such as subclassing ArgumentParser. The above
54 | # will fail on this. However, we can use the methods known to ArgumentParser to do a duck-type like
55 | # approach to finding what is the arg parser
56 | if not argparse_assignments:
57 | aa_references = set([i.func.value.id for i in chain(add_arg_assignments, parse_args_assignment)])
58 | argparse_like_objects = [getattr(i.value.func, 'id', None) for p_ref in aa_references for i in get_nodes_by_containing_attr(assignment_objs, p_ref)]
59 | argparse_like_objects = filter(None, argparse_like_objects)
60 | argparse_assignments = [get_nodes_by_containing_attr(assignment_objs, i) for i in argparse_like_objects]
61 | # for now, we just choose one
62 | try:
63 | argparse_assignments = argparse_assignments[0]
64 | except IndexError:
65 | pass
66 |
67 |
68 | # get things that are assigned inside ArgumentParser or its methods
69 | argparse_assigned_variables = get_node_args_and_keywords(assignment_objs, argparse_assignments, 'ArgumentParser')
70 | add_arg_assigned_variables = get_node_args_and_keywords(assignment_objs, add_arg_assignments, 'add_argument')
71 | parse_args_assigned_variables = get_node_args_and_keywords(assignment_objs, parse_args_assignment, 'parse_args')
72 |
73 | ast_argparse_source = chain(
74 | module_imports,
75 | specific_imports,
76 | argparse_assigned_variables,
77 | add_arg_assigned_variables,
78 | parse_args_assigned_variables,
79 | argparse_assignments,
80 | group_arg_assignments,
81 | add_arg_assignments,
82 | )
83 | return ast_argparse_source
84 |
85 |
86 | def read_client_module(filename):
87 | with open(filename, 'r') as f:
88 | return f.readlines()
89 |
90 |
91 | def get_node_args_and_keywords(assigned_objs, assignments, selector=None):
92 | referenced_nodes = set([])
93 | selector_line = -1
94 | assignment_nodes = []
95 | for node in assignments:
96 | for i in walk_tree(node):
97 | if i and isinstance(i, (_ast.keyword, _ast.Name)) and 'id' in i.__dict__:
98 | if i.id == selector:
99 | selector_line = i.lineno
100 | elif i.lineno == selector_line:
101 | referenced_nodes.add(i.id)
102 | for node in assigned_objs:
103 | for target in node.targets:
104 | if getattr(target, 'id', None) in referenced_nodes:
105 | assignment_nodes.append(node)
106 | return assignment_nodes
107 |
108 |
109 | def get_nodes_by_instance_type(nodes, object_type):
110 | return [node for node in walk_tree(nodes) if isinstance(node, object_type)]
111 |
112 |
113 | def get_nodes_by_containing_attr(nodes, attr):
114 | return [node for node in nodes if attr in walk_tree(node)]
115 |
116 |
117 | def walk_tree(node):
118 | try:
119 | d = node.__dict__
120 | except AttributeError:
121 | d = {}
122 | yield node
123 | for key, value in d.items():
124 | if isinstance(value, list):
125 | for val in value:
126 | for _ in ast.walk(val):
127 | yield _
128 | elif issubclass(type(value), ast.AST):
129 | for _ in walk_tree(value):
130 | yield _
131 | else:
132 | yield value
133 |
134 |
135 | def convert_to_python(ast_source):
136 | """
137 | Converts the ast objects back into human readable Python code
138 | """
139 | return map(codegen.to_source, ast_source)
--------------------------------------------------------------------------------
/djangui/views/djangui_celery.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 | import os
3 | import six
4 |
5 | from django.core.urlresolvers import reverse
6 | from django.views.generic import TemplateView
7 | from django.conf import settings
8 | from django.utils.translation import gettext_lazy as _
9 | from django.utils.encoding import force_text
10 | from django.db.models import Q
11 | from django.template.defaultfilters import escape
12 |
13 | from djcelery.models import TaskMeta
14 | from celery import app, states
15 |
16 | celery_app = app.app_or_default()
17 |
18 | from ..models import DjanguiJob
19 | from .. import settings as djangui_settings
20 | from ..backend.utils import valid_user, get_file_previews
21 | from ..django_compat import JsonResponse
22 |
23 | def celery_status(request):
24 | # TODO: This function can use some sprucing up, design a better data structure for returning jobs
25 | spanbase = ""
26 | STATE_MAPPER = {
27 | DjanguiJob.COMPLETED: spanbase.format('glyphicon-ok', _('Success')),
28 | DjanguiJob.RUNNING: spanbase.format('glyphicon-refresh spinning', _('Executing')),
29 | states.PENDING: spanbase.format('glyphicon-time', _('In queue')),
30 | states.REVOKED: spanbase.format('glyphicon-stop', _('Halted')),
31 | DjanguiJob.SUBMITTED: spanbase.format('glyphicon-hourglass', _('Waiting to be queued'))
32 | }
33 | user = request.user
34 | if user.is_superuser:
35 | jobs = DjanguiJob.objects.all()
36 | else:
37 | jobs = DjanguiJob.objects.filter(Q(user=None) | Q(user=user) if request.user.is_authenticated() else Q(user=None))
38 | jobs = jobs.exclude(status=DjanguiJob.DELETED)
39 | # divide into user and anon jobs
40 | def get_job_list(job_query):
41 | return [{'job_name': escape(job.job_name), 'job_status': STATE_MAPPER.get(job.status, job.status),
42 | 'job_submitted': job.created_date.strftime('%b %d %Y, %H:%M:%S'),
43 | 'job_id': job.pk,
44 | 'job_description': escape(six.u('Script: {}\n{}').format(job.script.script_name, job.job_description)),
45 | 'job_url': reverse('djangui:celery_results_info', kwargs={'job_id': job.pk})} for job in job_query]
46 | d = {'user': get_job_list([i for i in jobs if i.user == user]),
47 | 'anon': get_job_list([i for i in jobs if i.user == None or (user.is_superuser and i.user != user)])}
48 | return JsonResponse(d, safe=False)
49 |
50 |
51 | def celery_task_command(request):
52 |
53 | command = request.POST.get('celery-command')
54 | job_id = request.POST.get('job-id')
55 | job = DjanguiJob.objects.get(pk=job_id)
56 | response = {'valid': False,}
57 | valid = valid_user(job.script, request.user)
58 | if valid.get('valid') is True:
59 | user = request.user if request.user.is_authenticated() else None
60 | if user == job.user or job.user == None:
61 | if command == 'resubmit':
62 | new_job = job.submit_to_celery(resubmit=True, user=request.user)
63 | response.update({'valid': True, 'extra': {'task_url': reverse('djangui:celery_results_info', kwargs={'job_id': new_job.pk})}})
64 | elif command == 'rerun':
65 | job.submit_to_celery(user=request.user, rerun=True)
66 | response.update({'valid': True, 'redirect': reverse('djangui:celery_results_info', kwargs={'job_id': job_id})})
67 | elif command == 'clone':
68 | response.update({'valid': True, 'redirect': '{0}?job_id={1}'.format(reverse('djangui:djangui_task_launcher'), job_id)})
69 | elif command == 'delete':
70 | job.status = DjanguiJob.DELETED
71 | job.save()
72 | response.update({'valid': True, 'redirect': reverse('djangui:djangui_home')})
73 | elif command == 'stop':
74 | celery_app.control.revoke(job.celery_id, signal='SIGKILL', terminate=True)
75 | job.status = states.REVOKED
76 | job.save()
77 | response.update({'valid': True, 'redirect': reverse('djangui:celery_results_info', kwargs={'job_id': job_id})})
78 | else:
79 | response.update({'errors': {'__all__': [force_text(_("Unknown Command"))]}})
80 | else:
81 | response.update({'errors': {'__all__': [force_text(valid.get('error'))]}})
82 | return JsonResponse(response)
83 |
84 |
85 | class CeleryTaskView(TemplateView):
86 | template_name = 'djangui/tasks/task_view.html'
87 |
88 | def get_context_data(self, **kwargs):
89 | ctx = super(CeleryTaskView, self).get_context_data(**kwargs)
90 | job_id = ctx.get('job_id')
91 | try:
92 | djangui_job = DjanguiJob.objects.get(pk=job_id)
93 | except DjanguiJob.DoesNotExist:
94 | ctx['task_error'] = _('This task does not exist.')
95 | else:
96 | user = self.request.user
97 | user = None if not user.is_authenticated() and djangui_settings.DJANGUI_ALLOW_ANONYMOUS else user
98 | job_user = djangui_job.user
99 | if job_user == None or job_user == user or (user != None and user.is_superuser):
100 | out_files = get_file_previews(djangui_job)
101 | all = out_files.pop('all', [])
102 | archives = out_files.pop('archives', [])
103 | ctx['task_info'] = {
104 | 'all_files': all,
105 | 'archives': archives,
106 | 'file_groups': out_files,
107 | 'status': djangui_job.status,
108 | 'last_modified': djangui_job.modified_date,
109 | 'job': djangui_job
110 | }
111 | else:
112 | ctx['task_error'] = _('You are not authenticated to view this job.')
113 | return ctx
114 |
115 |
--------------------------------------------------------------------------------
/djangui/tasks.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 | import subprocess
3 | import tarfile
4 | import os
5 | import zipfile
6 | import six
7 |
8 | from django.utils.text import get_valid_filename
9 | from django.core.files.storage import default_storage
10 | from django.core.files import File
11 | from django.conf import settings
12 | from django.db.transaction import atomic
13 |
14 |
15 | from celery import Task
16 | from celery import states
17 | from celery import app
18 | from celery.contrib import rdb
19 |
20 | from . import settings as djangui_settings
21 |
22 | celery_app = app.app_or_default()
23 |
24 | class DjanguiTask(Task):
25 | pass
26 | # def after_return(self, status, retval, task_id, args, kwargs, einfo):
27 | # job, created = DjanguiJob.objects.get_or_create(djangui_celery_id=task_id)
28 | # job.content_type.djangui_celery_state = status
29 | # job.save()
30 |
31 | @celery_app.task(base=DjanguiTask)
32 | def submit_script(**kwargs):
33 | job_id = kwargs.pop('djangui_job')
34 | resubmit = kwargs.pop('djangui_resubmit', False)
35 | rerun = kwargs.pop('rerun', False)
36 | from .backend import utils
37 | from .models import DjanguiJob, DjanguiFile
38 | job = DjanguiJob.objects.get(pk=job_id)
39 |
40 | command = utils.get_job_commands(job=job)
41 | if resubmit:
42 | # clone ourselves, setting pk=None seems hackish but it works
43 | job.pk = None
44 |
45 | # This is where the script works from -- it is what is after the media_root since that may change between
46 | # setups/where our user uploads are stored.
47 | cwd = job.get_output_path()
48 |
49 | abscwd = os.path.abspath(os.path.join(settings.MEDIA_ROOT, cwd))
50 | job.command = ' '.join(command)
51 | job.save_path = cwd
52 |
53 | if rerun:
54 | # cleanup the old files, we need to be somewhat aggressive here.
55 | local_storage = utils.get_storage(local=True)
56 | remote_storage = utils.get_storage(local=False)
57 | to_delete = []
58 | with atomic():
59 | for dj_file in DjanguiFile.objects.filter(job=job):
60 | if dj_file.parameter is None or dj_file.parameter.parameter.is_output:
61 | to_delete.append(dj_file)
62 | path = local_storage.path(dj_file.filepath.name)
63 | dj_file.filepath.delete(False)
64 | if local_storage.exists(path):
65 | local_storage.delete(path)
66 | # TODO: This needs to be tested to make sure it's being nuked
67 | if remote_storage.exists(path):
68 | remote_storage.delete(path)
69 | [i.delete() for i in to_delete]
70 |
71 | utils.mkdirs(abscwd)
72 | # make sure we have the script, otherwise download it. This can happen if we have an ephemeral file system or are
73 | # executing jobs on a worker node.
74 | script_path = job.script.script_path
75 | if not utils.get_storage(local=True).exists(script_path.path):
76 | utils.get_storage(local=True).save(script_path.path, script_path.file)
77 |
78 | job.status = DjanguiJob.RUNNING
79 | job.save()
80 |
81 | proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=abscwd)
82 |
83 | stdout, stderr = proc.communicate()
84 | # tar/zip up the generated content for bulk downloads
85 | def get_valid_file(cwd, name, ext):
86 | out = os.path.join(cwd, name)
87 | index = 0
88 | while os.path.exists(six.u('{}.{}').format(out, ext)):
89 | index += 1
90 | out = os.path.join(cwd, six.u('{}_{}').format(name, index))
91 | return six.u('{}.{}').format(out, ext)
92 |
93 | # fetch the job again in case the database connection was lost during the job or something else changed.
94 | job = DjanguiJob.objects.get(pk=job_id)
95 |
96 | # if there are files generated, make zip/tar files for download
97 | if len(os.listdir(abscwd)):
98 | tar_out = get_valid_file(abscwd, get_valid_filename(job.job_name), 'tar.gz')
99 | tar = tarfile.open(tar_out, "w:gz")
100 | tar_name = os.path.splitext(os.path.splitext(os.path.split(tar_out)[1])[0])[0]
101 | tar.add(abscwd, arcname=tar_name)
102 | tar.close()
103 |
104 | zip_out = get_valid_file(abscwd, get_valid_filename(job.job_name), 'zip')
105 | zip = zipfile.ZipFile(zip_out, "w")
106 | arcname = os.path.splitext(os.path.split(zip_out)[1])[0]
107 | zip.write(abscwd, arcname=arcname)
108 | for root, folders, filenames in os.walk(os.path.split(zip_out)[0]):
109 | for filename in filenames:
110 | path = os.path.join(root, filename)
111 | if path == tar_out:
112 | continue
113 | if path == zip_out:
114 | continue
115 | zip.write(path, arcname=os.path.join(arcname, filename))
116 | zip.close()
117 |
118 | # save all the files generated as well to our default storage for ephemeral storage setups
119 | if djangui_settings.DJANGUI_EPHEMERAL_FILES:
120 | for root, folders, files in os.walk(abscwd):
121 | for filename in files:
122 | filepath = os.path.join(root, filename)
123 | s3path = os.path.join(root[root.find(cwd):], filename)
124 | remote = utils.get_storage(local=False)
125 | exists = remote.exists(s3path)
126 | filesize = remote.size(s3path)
127 | if not exists or (exists and filesize == 0):
128 | if exists:
129 | remote.delete(s3path)
130 | remote.save(s3path, File(open(filepath, 'rb')))
131 |
132 | utils.create_job_fileinfo(job)
133 |
134 |
135 | job.stdout = stdout
136 | job.stderr = stderr
137 | job.status = DjanguiJob.COMPLETED
138 | job.save()
139 |
140 | return (stdout, stderr)
--------------------------------------------------------------------------------
/djangui/migrations/0001_initial.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | from __future__ import unicode_literals
3 |
4 | from django.db import models, migrations
5 | import autoslug.fields
6 | from django.conf import settings
7 | import djangui.models.mixins
8 |
9 |
10 | class Migration(migrations.Migration):
11 |
12 | dependencies = [
13 | migrations.swappable_dependency(settings.AUTH_USER_MODEL),
14 | ]
15 |
16 | operations = [
17 | migrations.CreateModel(
18 | name='DjanguiFile',
19 | fields=[
20 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
21 | ('filepath', models.FileField(max_length=500, upload_to=b'')),
22 | ('filepreview', models.TextField(null=True, blank=True)),
23 | ('filetype', models.CharField(max_length=255, null=True, blank=True)),
24 | ],
25 | ),
26 | migrations.CreateModel(
27 | name='DjanguiJob',
28 | fields=[
29 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
30 | ('celery_id', models.CharField(max_length=255, null=True)),
31 | ('job_name', models.CharField(max_length=255)),
32 | ('job_description', models.TextField(null=True, blank=True)),
33 | ('stdout', models.TextField(null=True, blank=True)),
34 | ('stderr', models.TextField(null=True, blank=True)),
35 | ('status', models.CharField(default=b'submitted', max_length=255, choices=[(b'submitted', 'Submitted'), (b'running', 'Running'), (b'completed', 'Completed'), (b'deleted', 'Deleted')])),
36 | ('save_path', models.CharField(max_length=255, null=True, blank=True)),
37 | ('command', models.TextField()),
38 | ('created_date', models.DateTimeField(auto_now_add=True)),
39 | ('modified_date', models.DateTimeField(auto_now=True)),
40 | ],
41 | ),
42 | migrations.CreateModel(
43 | name='Script',
44 | fields=[
45 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
46 | ('script_name', models.CharField(max_length=255)),
47 | ('slug', autoslug.fields.AutoSlugField(unique=True, editable=False)),
48 | ('script_description', models.TextField(null=True, blank=True)),
49 | ('script_order', models.PositiveSmallIntegerField(default=1)),
50 | ('is_active', models.BooleanField(default=True)),
51 | ('script_path', models.FileField(upload_to=b'')),
52 | ('execute_full_path', models.BooleanField(default=True)),
53 | ('save_path', models.CharField(help_text=b'By default save to the script name, this will change the output folder.', max_length=255, null=True, blank=True)),
54 | ('script_version', models.PositiveSmallIntegerField(default=0)),
55 | ('created_date', models.DateTimeField(auto_now_add=True)),
56 | ('modified_date', models.DateTimeField(auto_now=True)),
57 | ],
58 | bases=(djangui.models.mixins.ModelDiffMixin, models.Model),
59 | ),
60 | migrations.CreateModel(
61 | name='ScriptGroup',
62 | fields=[
63 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
64 | ('group_name', models.TextField()),
65 | ('slug', autoslug.fields.AutoSlugField(unique=True, editable=False)),
66 | ('group_description', models.TextField(null=True, blank=True)),
67 | ('group_order', models.SmallIntegerField(default=1)),
68 | ('is_active', models.BooleanField(default=True)),
69 | ('user_groups', models.ManyToManyField(to='auth.Group', blank=True)),
70 | ],
71 | bases=(djangui.models.mixins.UpdateScriptsMixin, models.Model),
72 | ),
73 | migrations.CreateModel(
74 | name='ScriptParameter',
75 | fields=[
76 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
77 | ('short_param', models.CharField(max_length=255)),
78 | ('script_param', models.CharField(max_length=255)),
79 | ('slug', autoslug.fields.AutoSlugField(unique=True, editable=False)),
80 | ('is_output', models.BooleanField(default=None)),
81 | ('required', models.BooleanField(default=False)),
82 | ('output_path', models.FilePathField(path=b'', max_length=255, allow_files=False, recursive=True, allow_folders=True)),
83 | ('choices', models.CharField(max_length=255, null=True, blank=True)),
84 | ('choice_limit', models.PositiveSmallIntegerField(null=True, blank=True)),
85 | ('form_field', models.CharField(max_length=255)),
86 | ('default', models.CharField(max_length=255, null=True, blank=True)),
87 | ('input_type', models.CharField(max_length=255)),
88 | ('param_help', models.TextField(null=True, verbose_name=b'help', blank=True)),
89 | ('is_checked', models.BooleanField(default=False)),
90 | ],
91 | bases=(djangui.models.mixins.UpdateScriptsMixin, models.Model),
92 | ),
93 | migrations.CreateModel(
94 | name='ScriptParameterGroup',
95 | fields=[
96 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
97 | ('group_name', models.TextField()),
98 | ('script', models.ForeignKey(to='djangui.Script')),
99 | ],
100 | bases=(djangui.models.mixins.UpdateScriptsMixin, models.Model),
101 | ),
102 | migrations.CreateModel(
103 | name='ScriptParameters',
104 | fields=[
105 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
106 | ('_value', models.TextField(db_column='value')),
107 | ('job', models.ForeignKey(to='djangui.DjanguiJob')),
108 | ('parameter', models.ForeignKey(to='djangui.ScriptParameter')),
109 | ],
110 | ),
111 | migrations.AddField(
112 | model_name='scriptparameter',
113 | name='parameter_group',
114 | field=models.ForeignKey(to='djangui.ScriptParameterGroup'),
115 | ),
116 | migrations.AddField(
117 | model_name='scriptparameter',
118 | name='script',
119 | field=models.ForeignKey(to='djangui.Script'),
120 | ),
121 | migrations.AddField(
122 | model_name='script',
123 | name='script_group',
124 | field=models.ForeignKey(to='djangui.ScriptGroup'),
125 | ),
126 | migrations.AddField(
127 | model_name='script',
128 | name='user_groups',
129 | field=models.ManyToManyField(to='auth.Group', blank=True),
130 | ),
131 | migrations.AddField(
132 | model_name='djanguijob',
133 | name='script',
134 | field=models.ForeignKey(to='djangui.Script'),
135 | ),
136 | migrations.AddField(
137 | model_name='djanguijob',
138 | name='user',
139 | field=models.ForeignKey(blank=True, to=settings.AUTH_USER_MODEL, null=True),
140 | ),
141 | migrations.AddField(
142 | model_name='djanguifile',
143 | name='job',
144 | field=models.ForeignKey(to='djangui.DjanguiJob'),
145 | ),
146 | migrations.AddField(
147 | model_name='djanguifile',
148 | name='parameter',
149 | field=models.ForeignKey(blank=True, to='djangui.ScriptParameters', null=True),
150 | ),
151 | ]
152 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | Djangui has been merged with Wooey! You can get the latest and greatest at the organization [Wooey](http://www.github.com/wooey)
2 |
3 | # Djangui
4 |
5 | [](https://travis-ci.org/Chris7/django-djangui) [](https://coveralls.io/r/Chris7/django-djangui?branch=master)
6 |
7 | 1. [Installation](#install)
8 | 1. [A Djangui Only Project](#djonly)
9 | 2. [Adding Djangui to Existing Projects](#existing)
10 | 2. [Running Djangui](#running)
11 | 1. [A Procfile](#procfile)
12 | 2. [Two processes](#two-procs)
13 | 3. [Adding Scripts](#adding)
14 | 4. [Script Organization](#organization)
15 | 5. [Script Permissions](#permissions)
16 | 6. [Configuration](#config)
17 | 7. [Usage with S3/remote file systems](#s3)
18 | 8. [Script Guidelines](#script-guide)
19 |
20 | Djangui is designed to take scripts implemented with a python command line argument parser (such as argparse), and convert them into a web interface.
21 |
22 | This project was inspired by how simply and powerfully [sandman](https://github.com/jeffknupp/sandman) could expose users to a database.
23 | It was also based on my own needs as a data scientist to have a system that could:
24 |
25 | 1. Autodocument my workflows for data analysis
26 | (simple model saving).
27 | 2. Enable fellow lab members with no command line
28 | experience to utilize python scripts.
29 | 3. Enable the easy wrapping of any program in simple
30 | python instead of having to use language specific
31 | to existing tools such as Galaxy.
32 |
33 | # Installation
34 |
35 | pip install django-djangui
36 |
37 | ## A Djangui only project
38 |
39 | There is a bootstrapper included with djangui, which will create a Django project and setup most of the needed settings automagically for you.
40 |
41 | 1. djanguify -p ProjectName
42 | 2. Follow the instructions at the end of the bootstrapper
43 | to create the admin user and access the admin page
44 | 3. Login to the admin page wherever the project is
45 | being hosted (locally this would be localhost:8000/admin)
46 |
47 | ## Installation with existing Django Projects
48 |
49 | 1. Add 'djangui' to INSTALLED_APPS in settings.py (and optionally, djcelery unless you wish to tie into an existing celery instance)
50 | 2. Add the following to your urls.py:
51 | `url(r'^', include('djangui.urls')),`
52 | (Note: it does not need to be rooted at your site base,
53 | you can have r'^djangui/'... as your router):
54 |
55 | 3. Migrate your database:
56 | # Django 1.6 and below:
57 | `./manage.py syncdb`
58 |
59 | # Django 1.7 and above
60 | `./manage.py makemigrations`
61 | `./manage.py migrate`
62 |
63 | 4. Ensure the following are in your TEMPLATE_CONTEXT_PROCSSORS variable:
64 | TEMPLATE_CONTEXT_PROCESSORS = [
65 | ...
66 | 'django.contrib.auth.context_processors.auth',
67 | 'django.core.context_processors.request'
68 | ...]
69 |
70 | # Running Djangui
71 |
72 | Djangui depends on a distributed worker to handle tasks, you can disable this by setting **DJANGUI_CELERY** to False in your settings, which will allow you to run Djangui through the simple command:
73 |
74 | ```
75 | python manage.py runserver
76 | ```
77 |
78 | However, this will cause the server to execute tasks, which will block the site.
79 |
80 | The recommended ways to run Djangui are:
81 |
82 | ## Through a Procfile
83 |
84 | The simplest way to run Djangui is to use a Procfile with [honcho](https://github.com/nickstenning/honcho), which can be installed via pip. Make a file, called Procfile in the root of your project (the same place as manage.py) with the following contents:
85 |
86 | ```
87 | web: python manage.py runserver
88 | worker: python manage.py celery worker -c 1 --beat -l info
89 | EOM
90 | ```
91 |
92 | Your server can then be run by the simple command:
93 | ```
94 | honcho start
95 | ```
96 |
97 | ## Through two separate processes
98 |
99 | You can also run djangui by invoking two commands (you will need a separate process for each):
100 |
101 | ```
102 | python manage.py celery worker -c 1 --beat -l info
103 | python manage.py runserver
104 | ```
105 |
106 |
107 | # Adding & Managing Scripts
108 |
109 | Scripts may be added in two ways, through the Django admin interface as well as through the *addscript* command in manage.py.
110 |
111 | ### The admin Interface
112 |
113 | Within the django admin interface, scripts may be added to through the 'scripts' model. Here, the user permissions may be set, as
114 | well as cosmetic features such as the script's display name, description (if provided, otherwise the script name and description
115 | will be automatically populated by the description from argparse if available).
116 |
117 | ### The command line
118 |
119 | `./manage.py addscript`
120 |
121 | This will add a script or a folder of scripts to Djangui (if a folder is passed instead of a file).
122 | By default, scripts will be created in the 'Djangui Scripts' group.
123 |
124 | # Script Organization
125 |
126 | Scripts can be viewed at the root url of Djangui. The ordering of scripts, and groupings of scripts can be altered by
127 | changing the 'Script order' or 'Group order' options within the admin.
128 |
129 | # Script Permissions
130 |
131 | Scripts and script groups can be relegated to certain groups of users. The 'user groups' option, if set, will restrict script usage
132 | to users within selected groups.
133 |
134 | Scripts and groups may also be shutoff to all users by unchecked the 'script/group active' option.
135 |
136 | # Configuration
137 |
138 | **DJANGUI_FILE_DIR**: String, where the files uploaded by the user will be saved (Default: djangui_files)
139 |
140 | **DJANGUI_CELERY**: Boolean, whether or not celery is enabled. If disabled, tasks will run locally and block execution. (Default: True)
141 |
142 | **DJANGUI_CELERY_TASKS**: String, the name of the celery tasks for Djangui. (Default: 'djangui.tasks')
143 |
144 | **DJANGUI_ALLOW_ANONYMOUS**: Boolean, whether to allow submission of jobs by anonymous users. (Default: True)
145 |
146 | By default, Djangui has a basic user account system. It is very basic, and doesn't confirm registrations via email.
147 |
148 | **DJANGUI_AUTH**: Boolean, whether to use the authorization system of Djangui for simple login/registration. (Default: True)
149 |
150 | **DJANGUI_LOGIN_URL**: String, if you have an existing authorization system, the login url: (Default: settings.LOGIN_URL
151 |
152 | **DJANGUI_REGISTER_URL**: String, if you have an existing authorization system, the registration url: (Default: /accounts/register/)
153 |
154 | **DJANGUI_EPHEMERAL_FILES**: Boolean, if your file system changes with each restart. (Default: False)
155 |
156 | **DJANGUI_SHOW_LOCKED_SCRIPTS**: Boolean, whether to show locked scripts as disabled or hide them entirely. (Defalt: True -- show as disabled)
157 |
158 | # Remote File Systems
159 |
160 | Djangui has been tested on heroku with S3 as a file storage system. Settings for this can be seen in the user_settings.py, which give you a starting point for a non-local server. In short, you need to change your storage settings like such:
161 |
162 |
163 |
164 | STATICFILES_STORAGE = DEFAULT_FILE_STORAGE = 'djangui.djanguistorage.CachedS3BotoStorage'
165 | DJANGUI_EPHEMERAL_FILES = True
166 |
167 |
168 |
169 | # Script Guidelines
170 |
171 | The easiest way to make your scripts compatible with Djangui is to define your ArgParse class in the global scope. For instance:
172 |
173 | ```
174 |
175 | import argparse
176 | import sys
177 |
178 | parser = argparse.ArgumentParser(description="https://projecteuler.net/problem=1 -- Find the sum of all the multiples of 3 or 5 below a number.")
179 | parser.add_argument('--below', help='The number to find the sum of multiples below.', type=int, default=1000)
180 |
181 | def main():
182 | args = parser.parse_args()
183 | ...
184 |
185 | if __name__ == "__main__":
186 | sys.exit(main())
187 |
188 | ```
189 |
190 | If you have failing scripts, please open an issue with their contents so I can handle cases as they appear and try to make this as all-encompasing as possible. One known area which fails currently is defining your argparse instance inside the `if __name__ == "__main__" block`
191 |
--------------------------------------------------------------------------------
/djangui/templates/djangui/base.html:
--------------------------------------------------------------------------------
1 | {% load i18n %}
2 | {% load staticfiles %}
3 |
4 |
5 |
6 |
7 |
115 | {% block left_sidebar_content %}
116 | {% endblock left_sidebar_content %}
117 |
118 | {% endblock left_sidebar %}
119 |
120 | {% block center %}
121 |
122 |
123 | {# #}
124 | {% block center_content %}
125 | {% endblock center_content %}
126 | {#
#}
127 |
128 | {% endblock center %}
129 |
130 | {% block right_sidebar %}
131 |
132 | {% block right_sidebar_content %}
133 | {% endblock right_sidebar_content %}
134 |
135 | {% endblock right_sidebar %}
136 |
137 |
138 | {% block js %}
139 |
140 |
141 |
142 |
143 |
144 | {% endblock js %}
145 | {% block extra_js %}
146 | {% endblock extra_js %}
147 | {% block inline_js %}
148 |
149 |
225 | {% endblock inline_js %}
226 |
--------------------------------------------------------------------------------
/djangui/backend/argparse_specs.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 | import argparse
3 | import sys
4 | import json
5 | import imp
6 | import traceback
7 | import tempfile
8 | import six
9 | import copy
10 | from collections import OrderedDict
11 | from .ast import source_parser
12 | from itertools import chain
13 |
14 | def is_upload(action):
15 | """Checks if this should be a user upload
16 |
17 | :param action:
18 | :return: True if this is a file we intend to upload from the user
19 | """
20 | return 'r' in action.type._mode and (action.default is None or
21 | getattr(action.default, 'name') not in (sys.stderr.name, sys.stdout.name))
22 |
23 | # input attributes we try to set:
24 | # checked, name, type, value
25 | # extra information we want to append:
26 | # help,
27 | # required,
28 | # param (for executing the script and knowing if we need - or --),
29 | # upload (boolean providing info on whether it's a file are we uploading or saving)
30 | # choices (for selections)
31 | # choice_limit (for multi selection)
32 |
33 | CHOICE_LIMIT_MAP = {'?': '1', '+': '>=1', '*': '>=0'}
34 |
35 | # We want to map to model fields as well as html input types we encounter in argparse
36 | # keys are known variable types, as defined in __builtins__
37 | # the model is a Django based model, which can be fitted in the future for other frameworks.
38 | # The type is the HTML input type
39 | # nullcheck is a function to determine if the default value should be checked (for cases like default='' for strings)
40 | # the attr_kwargs is a mapping of the action attributes to its related html input type. It is a dict
41 | # of the form: {'name_for_html_input': {
42 | # and either one or both of:
43 | # 'action_name': 'attribute_name_on_action', 'callback': 'function_to_evaluate_action_and_return_value'} }
44 |
45 | GLOBAL_ATTRS = ['model', 'type']
46 |
47 | GLOBAL_ATTR_KWARGS = {
48 | 'name': {'action_name': 'dest'},
49 | 'value': {'action_name': 'default'},
50 | 'required': {'action_name': 'required'},
51 | 'help': {'action_name': 'help'},
52 | 'param': {'callback': lambda x: x.option_strings[0] if x.option_strings else ''},
53 | 'choices': {'callback': lambda x: x.choices},
54 | 'choice_limit': {'callback': lambda x: CHOICE_LIMIT_MAP.get(x.nargs, x.nargs)}
55 | }
56 |
57 | TYPE_FIELDS = {
58 | # Python Builtins
59 | bool: {'model': 'BooleanField', 'type': 'checkbox', 'nullcheck': lambda x: x.default is None,
60 | 'attr_kwargs': GLOBAL_ATTR_KWARGS},
61 | float: {'model': 'FloatField', 'type': 'text', 'html5-type': 'number', 'nullcheck': lambda x: x.default is None,
62 | 'attr_kwargs': GLOBAL_ATTR_KWARGS},
63 | int: {'model': 'IntegerField', 'type': 'text', 'nullcheck': lambda x: x.default is None,
64 | 'attr_kwargs': GLOBAL_ATTR_KWARGS},
65 | None: {'model': 'CharField', 'type': 'text', 'nullcheck': lambda x: x.default is None,
66 | 'attr_kwargs': GLOBAL_ATTR_KWARGS},
67 | str: {'model': 'CharField', 'type': 'text', 'nullcheck': lambda x: x.default == '' or x.default is None,
68 | 'attr_kwargs': GLOBAL_ATTR_KWARGS},
69 |
70 | # argparse Types
71 | argparse.FileType: {'model': 'FileField', 'type': 'file', 'nullcheck': lambda x: False,
72 | 'attr_kwargs': dict(GLOBAL_ATTR_KWARGS, **{
73 | 'value': None,
74 | 'required': {'callback': lambda x: x.required or x.default in (sys.stdout, sys.stdin)},
75 | 'upload': {'callback': is_upload}
76 | })},
77 | }
78 | if six.PY2:
79 | TYPE_FIELDS.update({
80 | file: {'model': 'FileField', 'type': 'file', 'nullcheck': lambda x: False,
81 | 'attr_kwargs': GLOBAL_ATTR_KWARGS},
82 | unicode: {'model': 'CharField', 'type': 'text', 'nullcheck': lambda x: x.default == '' or x.default is None,
83 | 'attr_kwargs': GLOBAL_ATTR_KWARGS},
84 | })
85 | elif six.PY3:
86 | import io
87 | TYPE_FIELDS.update({
88 | io.IOBase: {'model': 'FileField', 'type': 'file', 'nullcheck': lambda x: False,
89 | 'attr_kwargs': GLOBAL_ATTR_KWARGS},
90 | })
91 |
92 | def update_dict_copy(a, b):
93 | temp = copy.deepcopy(a)
94 | temp.update(b)
95 | return temp
96 |
97 | # There are cases where we can glean additional information about the form structure, e.g.
98 | # a StoreAction with default=True can be different than a StoreTrueAction with default=False
99 | ACTION_CLASS_TO_TYPE_FIELD = {
100 | argparse._StoreAction: update_dict_copy(TYPE_FIELDS, {}),
101 | argparse._StoreConstAction: update_dict_copy(TYPE_FIELDS, {}),
102 | argparse._StoreTrueAction: update_dict_copy(TYPE_FIELDS, {
103 | None: {'model': 'BooleanField', 'type': 'checkbox', 'nullcheck': lambda x: x.default is None,
104 | 'attr_kwargs': update_dict_copy(GLOBAL_ATTR_KWARGS, {
105 | 'checked': {'callback': lambda x: x.default},
106 | 'value': None,
107 | })
108 | },
109 | }),
110 | argparse._StoreFalseAction: update_dict_copy(TYPE_FIELDS, {
111 | None: {'model': 'BooleanField', 'type': 'checkbox', 'nullcheck': lambda x: x.default is None,
112 | 'attr_kwargs': update_dict_copy(GLOBAL_ATTR_KWARGS, {
113 | 'checked': {'callback': lambda x: x.default},
114 | 'value': None,
115 | })
116 | },
117 | })
118 | }
119 |
120 | class ArgParseNode(object):
121 | """
122 | This class takes an argument parser entry and assigns it to a Build spec
123 | """
124 | def __init__(self, action=None):
125 | fields = ACTION_CLASS_TO_TYPE_FIELD.get(type(action), TYPE_FIELDS)
126 | field_type = fields.get(action.type)
127 | if field_type is None:
128 | field_types = [i for i in fields.keys() if i is not None and issubclass(type(action.type), i)]
129 | if len(field_types) > 1:
130 | field_types = [i for i in fields.keys() if i is not None and isinstance(action.type, i)]
131 | if len(field_types) == 1:
132 | field_type = fields[field_types[0]]
133 | self.node_attrs = dict([(i, field_type[i]) for i in GLOBAL_ATTRS])
134 | null_check = field_type['nullcheck'](action)
135 | for attr, attr_dict in six.iteritems(field_type['attr_kwargs']):
136 | if attr_dict is None:
137 | continue
138 | if attr == 'value' and null_check:
139 | continue
140 | if 'action_name' in attr_dict:
141 | self.node_attrs[attr] = getattr(action, attr_dict['action_name'])
142 | elif 'callback' in attr_dict:
143 | self.node_attrs[attr] = attr_dict['callback'](action)
144 |
145 | @property
146 | def name(self):
147 | return self.node_attrs.get('name')
148 |
149 | def __str__(self):
150 | return json.dumps(self.node_attrs)
151 |
152 | def to_django(self):
153 | """
154 | This is a debug function to see what equivalent django models are being generated
155 | """
156 | exclude = {'name', 'model'}
157 | field_module = 'models'
158 | django_kwargs = {}
159 | if self.node_attrs['model'] == 'CharField':
160 | django_kwargs['max_length'] = 255
161 | django_kwargs['blank'] = not self.node_attrs['required']
162 | try:
163 | django_kwargs['default'] = self.node_attrs['value']
164 | except KeyError:
165 | pass
166 | return u'{0} = {1}.{2}({3})'.format(self.node_attrs['name'], field_module, self.node_attrs['model'],
167 | ', '.join(['{0}={1}'.format(i,v) for i,v in six.iteritems(django_kwargs)]),)
168 |
169 |
170 | class ArgParseNodeBuilder(object):
171 | def __init__(self, script_path=None, script_name=None):
172 | self.valid = False
173 | self.error = ''
174 | parsers = []
175 | try:
176 | module = imp.load_source(script_name, script_path)
177 | except:
178 | sys.stderr.write('Error while loading {0}:\n'.format(script_path))
179 | self.error = '{0}\n'.format(traceback.format_exc())
180 | sys.stderr.write(self.error)
181 | else:
182 | main_module = module.main.__globals__ if hasattr(module, 'main') else globals()
183 | parsers = [v for i, v in chain(six.iteritems(main_module), six.iteritems(vars(module)))
184 | if issubclass(type(v), argparse.ArgumentParser)]
185 | if not parsers:
186 | f = tempfile.NamedTemporaryFile()
187 | ast_source = source_parser.parse_source_file(script_path)
188 | python_code = source_parser.convert_to_python(list(ast_source))
189 | f.write(six.b('\n'.join(python_code)))
190 | f.seek(0)
191 | module = imp.load_source(script_name, f.name)
192 | main_module = module.main.__globals__ if hasattr(module, 'main') else globals()
193 | parsers = [v for i, v in chain(six.iteritems(main_module), six.iteritems(vars(module)))
194 | if issubclass(type(v), argparse.ArgumentParser)]
195 | if not parsers:
196 | sys.stderr.write('Unable to identify ArgParser for {0}:\n'.format(script_path))
197 | return
198 | self.valid = True
199 | parser = parsers[0]
200 | self.class_name = script_name
201 | self.script_path = script_path
202 | self.script_description = getattr(parser, 'description', None)
203 | self.script_groups = []
204 | self.nodes = OrderedDict()
205 | self.script_groups = []
206 | non_req = set([i.dest for i in parser._get_optional_actions()])
207 | self.optional_nodes = set([])
208 | self.containers = OrderedDict()
209 | for action in parser._actions:
210 | # This is the help message of argparse
211 | if action.default == argparse.SUPPRESS:
212 | continue
213 | node = ArgParseNode(action=action)
214 | container = action.container.title
215 | container_node = self.containers.get(container, None)
216 | if container_node is None:
217 | container_node = []
218 | self.containers[container] = container_node
219 | self.nodes[node.name] = node
220 | container_node.append(node.name)
221 | if action.dest in non_req:
222 | self.optional_nodes.add(node.name)
223 |
224 | def get_script_description(self):
225 | return {'name': self.class_name, 'path': self.script_path,
226 | 'description': self.script_description,
227 | 'inputs': [{'group': container_name, 'nodes': [self.nodes[node].node_attrs for node in nodes]}
228 | for container_name, nodes in six.iteritems(self.containers)]}
229 |
230 | @property
231 | def json(self):
232 | return json.dumps(self.get_script_description())
--------------------------------------------------------------------------------
/djangui/templates/djangui/tasks/task_view.html:
--------------------------------------------------------------------------------
1 | {% extends "djangui/djangui_home.html" %}
2 | {% load i18n %}
3 | {% load djangui_tags %}
4 | {% block extra_js %}
5 | {{ js.super }}
6 |
7 | {% endblock extra_js %}
8 | {% block extra_css %}
9 | {{ extra_css.super }}
10 |
11 | {% endblock extra_css %}
12 | {% block extra_style %}
13 | {{ extra_style.super }}
14 | #djangui-task-submit {
15 | float: left;
16 | margin-left: 3px;
17 | }
18 | {% endblock extra_style %}
19 | {% block left_sidebar %}{% endblock left_sidebar %}
20 | {% block center_content_class %}col-md-9 col-xs-12{% endblock center_content_class %}
21 | {% block center_content %}
22 | {% if task_error %}
23 | {{ task_error }}
24 | {% else %} 25 | {% include "djangui/modals/resubmit_modal.html" %} 26 |{{ task_info.job.job_name }}
27 |
28 |
29 |
30 |
76 |
77 |
78 |
79 |
232 |
233 |
234 | {% endif %}
235 | {% endblock center_content %}
236 |
237 | {% block inline_js %}
238 | {{ block.super }}
239 |
277 | {% endblock %}
--------------------------------------------------------------------------------
/djangui/templates/djangui/djangui_home.html:
--------------------------------------------------------------------------------
1 | {% extends "djangui/base.html" %}
2 | {% load i18n %}
3 | {% load djangui_tags %}
4 |
5 | {% block extra_style %}
6 |
7 | {% endblock extra_style %}
8 |
9 | {% block left_sidebar_content %}
10 |
80 |
81 |
231 | 81 |
82 |
83 |
121 |
84 |
96 |
85 |
87 | {% trans "Job Information" %}
86 |
88 |
95 | -
89 |
- {% trans "Job Status" %} 90 |
- {{ task_info.job.status }} 91 |
- {% trans "Job Description" %} 92 |
- {{ task_info.job.job_description }} 93 |
97 |
104 |
98 |
100 | {% trans "Command Sent" %}
99 |
101 |
103 | {{ task_info.job.command }}
102 |
105 |
112 |
106 |
108 | {% trans "Job Output" %}
107 |
109 |
111 | {{ task_info.job.stdout|linebreaks }}
110 |
113 |
120 |
114 |
116 | {% trans "Job Errors" %}
115 |
117 |
119 | {{ task_info.job.stderr|linebreaks }}
118 |
122 |
230 |
123 |
124 |
130 |
131 |
228 | {% endfor %}
229 |
132 |
133 |
215 | {% endif %}
216 | {% elif output_group == 'fasta' %}
217 | {% for output_file_content in output_files %}
218 |
134 |
141 | {% for output_group, output_files in task_info.file_groups.items %}
142 | -
135 | {% for file in task_info.all_files %}
136 |
- {% if file.slug %}{{ file.slug }}{% endif %} 137 |
- {{ file.name }} 138 | {% endfor %} 139 |
143 | {% if output_group == 'tabular' %}
144 | {% for output_file_content in output_files %}
145 |
204 |
205 |
206 |
207 |
208 | Previous
209 |
210 |
211 |
212 | Next
213 |
214 |
146 |
147 |
148 |
149 |
150 |
174 | {% endfor %}
175 |
176 | {% elif output_group == 'images' %}
177 | {% if output_files %}
178 |
194 | {% endif %}
195 |
151 |
152 | {% for entry in output_file_content.preview %}
153 | {% if forloop.first %}
154 |
155 | {% for item in entry %}
156 |
172 |
173 | | {{ item }} | 157 | {% endfor %} 158 | 159 | 160 | {% else %} 161 |
|---|
| {{ item }} | 164 | {% endfor %} 165 |
196 |
201 | {% endif %}
202 | {% endfor %}
203 |
197 | {% endif %}
198 | 
199 | {% if not forloop.empty and forloop.last %}
200 |
219 |
220 |
225 | {% endfor %}
226 | {% endif %}
227 |
221 | {% for entry in output_file_content.preview %}
222 | {{ entry }}
223 | {% endfor %} 224 |
223 | {% endfor %} 224 |
11 | {% for group_id, group in djangui_scripts.items %}
12 | {# check the user has access to the group#}
13 | {% with group_show=group.group|valid_user:request.user %}
14 | {% if group_show != 'hide' %}
15 |
45 | {% endblock %}
46 |
47 | {% block center_content %}
48 |
16 |
41 | {% endif %}
42 | {% endwith %}
43 | {% endfor %}
44 |
17 |
23 | 18 | 19 | {{ group.group.group_name }} 20 | 21 |
22 |
24 |
40 |
25 |
39 | {{ group.group.group_description }}
26 |
27 | {% for script in group.scripts %}
28 | {% with script_show=script|valid_user:request.user %}
29 | {% if script_show != 'hide' %}
30 |
31 |
38 | {{ script.script_name }}
32 |{{ script.script_description }}
33 | 34 | {% endif %} 35 | {% endwith %} 36 | {% endfor %} 37 |
49 |
50 |
51 |
52 |
79 | {% endblock center_content %}
80 |
81 | {% block right_sidebar_content %}
82 |
83 |
95 |
129 | {% endblock right_sidebar_content %}
130 |
131 | {% block inline_js %}
132 | {{ block.super }}
133 |
351 | {% endblock inline_js %}
--------------------------------------------------------------------------------
/djangui/backend/utils.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 | __author__ = 'chris'
3 | import json
4 | import errno
5 | import os
6 | import sys
7 | import six
8 | import traceback
9 | from operator import itemgetter
10 | from collections import OrderedDict
11 |
12 | from django.conf import settings
13 | from django.db import transaction
14 | from django.db.utils import OperationalError
15 | from django.core.files.storage import default_storage
16 | from django.core.files import File
17 | from django.utils.translation import gettext_lazy as _
18 | from celery.contrib import rdb
19 |
20 | from .argparse_specs import ArgParseNodeBuilder
21 |
22 | from .. import settings as djangui_settings
23 |
24 | def sanitize_name(name):
25 | return name.replace(' ', '_').replace('-', '_')
26 |
27 |
28 | def sanitize_string(value):
29 | return value.replace('"', '\\"')
30 |
31 | def get_storage(local=True):
32 | if djangui_settings.DJANGUI_EPHEMERAL_FILES:
33 | storage = default_storage.local_storage if local else default_storage
34 | else:
35 | storage = default_storage
36 | return storage
37 |
38 | def get_job_commands(job=None):
39 | script = job.script
40 | com = ['python', script.get_script_path()]
41 | parameters = job.get_parameters()
42 | for param in parameters:
43 | com.extend(param.get_subprocess_value())
44 | return com
45 |
46 | @transaction.atomic
47 | def create_djangui_job(user=None, script_pk=None, data=None):
48 | from ..models import Script, DjanguiJob, ScriptParameter, ScriptParameters
49 | script = Script.objects.get(pk=script_pk)
50 | if data is None:
51 | data = {}
52 | job = DjanguiJob(user=user, job_name=data.pop('job_name', None), job_description=data.pop('job_description', None),
53 | script=script)
54 | job.save()
55 | parameters = OrderedDict([(i.slug, i) for i in ScriptParameter.objects.filter(slug__in=data.keys()).order_by('pk')])
56 | for slug, param in six.iteritems(parameters):
57 | new_param = ScriptParameters(job=job, parameter=param)
58 | new_param.value = data.get(slug)
59 | new_param.save()
60 | return job
61 |
62 |
63 | def get_master_form(model=None, pk=None):
64 | from ..forms.factory import DJ_FORM_FACTORY
65 | return DJ_FORM_FACTORY.get_master_form(model=model, pk=pk)
66 |
67 |
68 | def get_form_groups(model=None, pk=None, initial=None):
69 | from ..forms.factory import DJ_FORM_FACTORY
70 | return DJ_FORM_FACTORY.get_group_forms(model=model, pk=pk, initial=initial)
71 |
72 | def validate_form(form=None, data=None, files=None):
73 | form.add_djangui_fields()
74 | form.data = data if data is not None else {}
75 | form.files = files if files is not None else {}
76 | form.is_bound = True
77 | form.full_clean()
78 |
79 | def load_scripts():
80 | from ..models import Script
81 | # select all the scripts we have, then divide them into groups
82 | dj_scripts = {}
83 | try:
84 | scripts = Script.objects.count()
85 | except OperationalError:
86 | # database not initialized yet
87 | return
88 | if scripts:
89 | scripts = Script.objects.all()
90 | for script in scripts:
91 | key = (script.script_group.group_order, script.script_group.pk)
92 | group = dj_scripts.get(key, {
93 | # 'url': reverse_lazy('script_group', kwargs={'script_group', script.script_group.slug}),
94 | 'group': script.script_group, 'scripts': []
95 | })
96 | dj_scripts[key] = group
97 | group['scripts'].append((script.script_order, script))
98 |
99 | for group_pk, group_info in six.iteritems(dj_scripts):
100 | # order scripts
101 | group_info['scripts'].sort(key=itemgetter(0))
102 | latest_scripts = OrderedDict()
103 | for i in group_info['scripts']:
104 | script = i[1]
105 | script_name = script.script_name
106 | if script_name not in latest_scripts or latest_scripts[script_name].script_version < script.script_version:
107 | latest_scripts[script_name] = script
108 | valid_scripts = []
109 | for i in latest_scripts.values():
110 | try:
111 | # make sure we can load the form
112 | get_master_form(script)
113 | except:
114 | sys.stdout.write('Traceback while loading {0}:\n {1}\n'.format(script, traceback.format_exc()))
115 | else:
116 | valid_scripts.append(i)
117 | group_info['scripts'] = valid_scripts
118 | # keep only the latest version of scripts
119 | # order groups
120 | ordered_scripts = OrderedDict()
121 | for key in sorted(dj_scripts.keys(), key=itemgetter(0)):
122 | ordered_scripts[key[1]] = dj_scripts[key]
123 | settings.DJANGUI_SCRIPTS = ordered_scripts
124 |
125 |
126 | def get_storage_object(path, local=False):
127 | storage = get_storage(local=local)
128 | obj = storage.open(path)
129 | obj.url = storage.url(path)
130 | obj.path = storage.path(path)
131 | return obj
132 |
133 | def add_djangui_script(script=None, group=None):
134 | from ..models import Script, ScriptGroup, ScriptParameter, ScriptParameterGroup
135 | # if we have a script, it will at this point be saved in the model pointing to our file system, which may be
136 | # ephemeral. So the path attribute may not be implemented
137 | if not isinstance(script, six.string_types):
138 | try:
139 | script_path = script.script_path.path
140 | except NotImplementedError:
141 | script_path = script.script_path.name
142 |
143 | script_obj, script = (script, get_storage_object(script_path, local=True).path) if isinstance(script, Script) else (False, script)
144 | if isinstance(group, ScriptGroup):
145 | group = group.group_name
146 | if group is None:
147 | group = 'Djangui Scripts'
148 | basename, extension = os.path.splitext(script)
149 | filename = os.path.split(basename)[1]
150 |
151 | parser = ArgParseNodeBuilder(script_name=filename, script_path=script)
152 | if not parser.valid:
153 | return (False, parser.error)
154 | # make our script
155 | d = parser.get_script_description()
156 | script_group, created = ScriptGroup.objects.get_or_create(group_name=group)
157 | if script_obj is False:
158 | djangui_script, created = Script.objects.get_or_create(script_group=script_group, script_description=d['description'],
159 | script_path=script, script_name=d['name'])
160 | else:
161 | created = False
162 | if not script_obj.script_description:
163 | script_obj.script_description = d['description']
164 | if not script_obj.script_name:
165 | script_obj.script_name = d['name']
166 | # probably a much better way to avoid this recursion
167 | script_obj._add_script = False
168 | script_obj.save()
169 | if not created:
170 | if script_obj is False:
171 | djangui_script.script_version += 1
172 | djangui_script.save()
173 | if script_obj:
174 | djangui_script = script_obj
175 | # make our parameters
176 | CHOICE_MAPPING = {
177 |
178 | }
179 | for param_group_info in d['inputs']:
180 | param_group, created = ScriptParameterGroup.objects.get_or_create(group_name=param_group_info.get('group'), script=djangui_script)
181 | for param in param_group_info.get('nodes'):
182 | # TODO: fix choice limits
183 | #choice_limit = CHOICE_MAPPING[param.get('choice_limit')]
184 | # TODO: fix 'file' to be global in argparse
185 | is_out = True if param.get('upload', None) is False and param.get('type') == 'file' else not param.get('upload', False)
186 | script_param, created = ScriptParameter.objects.get_or_create(script=djangui_script, short_param=param['param'], script_param=param['name'],
187 | is_output=is_out, required=param.get('required', False),
188 | form_field=param['model'], default=param.get('default'), input_type=param.get('type'),
189 | choices=json.dumps(param.get('choices')), choice_limit=None,
190 | param_help=param.get('help'), is_checked=param.get('checked', False),
191 | parameter_group=param_group)
192 | # update our loaded scripts
193 | load_scripts()
194 | return (True, '')
195 |
196 | def valid_user(obj, user):
197 | groups = obj.user_groups.all()
198 | from ..models import Script
199 | ret = {'valid': False, 'error': '', 'display': ''}
200 | if djangui_settings.DJANGUI_ALLOW_ANONYMOUS or user.is_authenticated():
201 | if isinstance(obj, Script):
202 | from itertools import chain
203 | groups = list(chain(groups, obj.script_group.user_groups.all()))
204 | if not user.is_authenticated() and djangui_settings.DJANGUI_ALLOW_ANONYMOUS and len(groups) == 0:
205 | ret['valid'] = True
206 | elif groups:
207 | ret['error'] = _('You are not permitted to use this script')
208 | if not groups and obj.is_active:
209 | ret['valid'] = True
210 | if obj.is_active is True:
211 | if set(list(user.groups.all())) & set(list(groups)):
212 | ret['valid'] = True
213 | ret['display'] = 'disabled' if djangui_settings.DJANGUI_SHOW_LOCKED_SCRIPTS else 'hide'
214 | return ret
215 |
216 | def mkdirs(path):
217 | try:
218 | os.makedirs(path)
219 | except OSError as exc:
220 | if exc.errno == errno.EEXIST and os.path.isdir(path):
221 | pass
222 | else:
223 | raise
224 |
225 |
226 | def get_file_info(filepath):
227 | # returns info about the file
228 | filetype, preview = False, None
229 | tests = [('tabular', test_delimited), ('fasta', test_fastx)]
230 | while filetype is False and tests:
231 | ptype, pmethod = tests.pop()
232 | filetype, preview = pmethod(filepath)
233 | filetype = ptype if filetype else filetype
234 | preview = None if filetype is False else preview
235 | filetype = None if filetype is False else filetype
236 | try:
237 | json_preview = json.dumps(preview)
238 | except:
239 | sys.stderr.write('Error encountered in file preview:\n {}\n'.format(traceback.format_exc()))
240 | json_preview = json.dumps(None)
241 | return {'type': filetype, 'preview': json_preview}
242 |
243 |
244 | def test_delimited(filepath):
245 | import csv
246 | if six.PY3:
247 | handle = open(filepath, 'r', newline='')
248 | else:
249 | handle = open(filepath, 'rb')
250 | with handle as csv_file:
251 | try:
252 | dialect = csv.Sniffer().sniff(csv_file.read(1024*16), delimiters=',\t')
253 | except Exception as e:
254 | return False, None
255 | csv_file.seek(0)
256 | reader = csv.reader(csv_file, dialect)
257 | rows = []
258 | try:
259 | for index, entry in enumerate(reader):
260 | if index == 5:
261 | break
262 | rows.append(entry)
263 | except Exception as e:
264 | return False, None
265 | return True, rows
266 |
267 | def test_fastx(filepath):
268 | # if we can be delimited by + or > we're maybe a fasta/q
269 | with open(filepath) as fastx_file:
270 | sequences = OrderedDict()
271 | seq = []
272 | header = ''
273 | for row_index, row in enumerate(fastx_file, 1):
274 | if row_index > 30:
275 | break
276 | if row and row[0] == '>':
277 | if seq:
278 | sequences[header] = ''.join(seq)
279 | seq = []
280 | header = row
281 | elif row:
282 | # we bundle the fastq stuff in here since it's just a visual
283 | seq.append(row)
284 | if seq and header:
285 | sequences[header] = ''.join(seq)
286 | if sequences:
287 | rows = []
288 | [rows.extend([i, v]) for i,v in six.iteritems(sequences)]
289 | return True, rows
290 | return False, None
291 |
292 | @transaction.atomic
293 | def create_job_fileinfo(job):
294 | parameters = job.get_parameters()
295 | from ..models import DjanguiFile
296 | # first, create a reference to things the script explicitly created that is a parameter
297 | files = []
298 | for field in parameters:
299 | try:
300 | if field.parameter.form_field == 'FileField':
301 | value = field.value
302 | if value is None:
303 | continue
304 | if isinstance(value, six.string_types):
305 | # check if this was ever created and make a fileobject if so
306 | if get_storage(local=True).exists(value):
307 | if not get_storage(local=False).exists(value):
308 | get_storage(local=False).save(value, File(get_storage(local=True).open(value)))
309 | value = field.value
310 | else:
311 | field.force_value(None)
312 | field.save()
313 | continue
314 | d = {'parameter': field, 'file': value}
315 | files.append(d)
316 | except ValueError:
317 | continue
318 |
319 | known_files = {i['file'].name for i in files}
320 | # add the user_output files, these are things which may be missed by the model fields because the script
321 | # generated them without an explicit arguments reference in the script
322 | file_groups = {'archives': []}
323 | absbase = os.path.join(settings.MEDIA_ROOT, job.save_path)
324 | for filename in os.listdir(absbase):
325 | new_name = os.path.join(job.save_path, filename)
326 | if any([i.endswith(new_name) for i in known_files]):
327 | continue
328 | try:
329 | filepath = os.path.join(absbase, filename)
330 | if os.path.isdir(filepath):
331 | continue
332 | d = {'name': filename, 'file': get_storage_object(os.path.join(job.save_path, filename))}
333 | if filename.endswith('.tar.gz') or filename.endswith('.zip'):
334 | file_groups['archives'].append(d)
335 | else:
336 | files.append(d)
337 | except IOError:
338 | sys.stderr.format('{}'.format(traceback.format_exc()))
339 | continue
340 |
341 | # establish grouping by inferring common things
342 | file_groups['all'] = files
343 | import imghdr
344 | file_groups['images'] = []
345 | for filemodel in files:
346 | if imghdr.what(filemodel['file'].path):
347 | file_groups['images'].append(filemodel)
348 | file_groups['tabular'] = []
349 | file_groups['fasta'] = []
350 |
351 | for filemodel in files:
352 | fileinfo = get_file_info(filemodel['file'].path)
353 | filetype = fileinfo.get('type')
354 | if filetype is not None:
355 | file_groups[filetype].append(dict(filemodel, **{'preview': fileinfo.get('preview')}))
356 | else:
357 | filemodel['preview'] = json.dumps(None)
358 |
359 | # Create our DjanguiFile models
360 |
361 | # mark things that are in groups so we don't add this to the 'all' category too to reduce redundancy
362 | grouped = set([i['file'].path for file_type, groups in six.iteritems(file_groups) for i in groups if file_type != 'all'])
363 | for file_type, group_files in six.iteritems(file_groups):
364 | for group_file in group_files:
365 | if file_type == 'all' and group_file['file'].path in grouped:
366 | continue
367 | try:
368 | preview = group_file.get('preview')
369 | dj_file = DjanguiFile(job=job, filetype=file_type, filepreview=preview,
370 | parameter=group_file.get('parameter'))
371 | filepath = group_file['file'].path
372 | save_path = job.get_relative_path(filepath)
373 | dj_file.filepath.name = save_path
374 | dj_file.save()
375 | except:
376 | sys.stderr.write('Error in saving DJFile: {}\n'.format(traceback.format_exc()))
377 | continue
378 |
379 |
380 | def get_file_previews(job):
381 | from ..models import DjanguiFile
382 | files = DjanguiFile.objects.filter(job=job)
383 | groups = {'all': []}
384 | for file_info in files:
385 | filedict = {'name': file_info.filepath.name, 'preview': json.loads(file_info.filepreview) if file_info.filepreview else None,
386 | 'url': get_storage(local=False).url(file_info.filepath.name),
387 | 'slug': file_info.parameter.parameter.script_param if file_info.parameter else None}
388 | try:
389 | groups[file_info.filetype].append(filedict)
390 | except KeyError:
391 | groups[file_info.filetype] = [filedict]
392 | if file_info.filetype != 'all':
393 | groups['all'].append(filedict)
394 | return groups
--------------------------------------------------------------------------------
/djangui/backend/ast/codegen.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | codegen
4 | ~~~~~~~
5 |
6 | Extension to ast that allow ast -> python code generation.
7 |
8 | :copyright: Copyright 2008 by Armin Ronacher.
9 | :license: BSD.
10 | """
11 | from ast import *
12 |
13 | BOOLOP_SYMBOLS = {
14 | And: 'and',
15 | Or: 'or'
16 | }
17 |
18 | BINOP_SYMBOLS = {
19 | Add: '+',
20 | Sub: '-',
21 | Mult: '*',
22 | Div: '/',
23 | FloorDiv: '//',
24 | Mod: '%',
25 | LShift: '<<',
26 | RShift: '>>',
27 | BitOr: '|',
28 | BitAnd: '&',
29 | BitXor: '^'
30 | }
31 |
32 | CMPOP_SYMBOLS = {
33 | Eq: '==',
34 | Gt: '>',
35 | GtE: '>=',
36 | In: 'in',
37 | Is: 'is',
38 | IsNot: 'is not',
39 | Lt: '<',
40 | LtE: '<=',
41 | NotEq: '!=',
42 | NotIn: 'not in'
43 | }
44 |
45 | UNARYOP_SYMBOLS = {
46 | Invert: '~',
47 | Not: 'not',
48 | UAdd: '+',
49 | USub: '-'
50 | }
51 |
52 | ALL_SYMBOLS = {}
53 | ALL_SYMBOLS.update(BOOLOP_SYMBOLS)
54 | ALL_SYMBOLS.update(BINOP_SYMBOLS)
55 | ALL_SYMBOLS.update(CMPOP_SYMBOLS)
56 | ALL_SYMBOLS.update(UNARYOP_SYMBOLS)
57 |
58 |
59 | def to_source(node, indent_with=' ' * 4, add_line_information=False):
60 | """This function can convert a node tree back into python sourcecode.
61 | This is useful for debugging purposes, especially if you're dealing with
62 | custom asts not generated by python itself.
63 |
64 | It could be that the sourcecode is evaluable when the AST itself is not
65 | compilable / evaluable. The reason for this is that the AST contains some
66 | more data than regular sourcecode does, which is dropped during
67 | conversion.
68 |
69 | Each level of indentation is replaced with `indent_with`. Per default this
70 | parameter is equal to four spaces as suggested by PEP 8, but it might be
71 | adjusted to match the application's styleguide.
72 |
73 | If `add_line_information` is set to `True` comments for the line numbers
74 | of the nodes are added to the output. This can be used to spot wrong line
75 | number information of statement nodes.
76 | """
77 | generator = SourceGenerator(indent_with, add_line_information)
78 | generator.visit(node)
79 | return ''.join(str(s) for s in generator.result)
80 |
81 |
82 | class SourceGenerator(NodeVisitor):
83 | """This visitor is able to transform a well formed syntax tree into python
84 | sourcecode. For more details have a look at the docstring of the
85 | `node_to_source` function.
86 | """
87 |
88 | def __init__(self, indent_with, add_line_information=False):
89 | self.result = []
90 | self.indent_with = indent_with
91 | self.add_line_information = add_line_information
92 | self.indentation = 0
93 | self.new_lines = 0
94 |
95 | def write(self, x):
96 | if self.new_lines:
97 | if self.result:
98 | self.result.append('\n' * self.new_lines)
99 | self.result.append(self.indent_with * self.indentation)
100 | self.new_lines = 0
101 | self.result.append(x)
102 |
103 | def newline(self, node=None, extra=0):
104 | self.new_lines = max(self.new_lines, 1 + extra)
105 | if node is not None and self.add_line_information:
106 | self.write('# line: %s' % node.lineno)
107 | self.new_lines = 1
108 |
109 | def body(self, statements):
110 | self.new_line = True
111 | self.indentation += 1
112 | for stmt in statements:
113 | self.visit(stmt)
114 | self.indentation -= 1
115 |
116 | def body_or_else(self, node):
117 | self.body(node.body)
118 | if node.orelse:
119 | self.newline()
120 | self.write('else:')
121 | self.body(node.orelse)
122 |
123 | def signature(self, node):
124 | want_comma = []
125 |
126 | def write_comma():
127 | if want_comma:
128 | self.write(', ')
129 | else:
130 | want_comma.append(True)
131 |
132 | padding = [None] * (len(node.args) - len(node.defaults))
133 | for arg, default in zip(node.args, padding + node.defaults):
134 | write_comma()
135 | self.visit(arg)
136 | if default is not None:
137 | self.write('=')
138 | self.visit(default)
139 | if node.vararg is not None:
140 | write_comma()
141 | self.write('*' + node.vararg)
142 | if node.kwarg is not None:
143 | write_comma()
144 | self.write('**' + node.kwarg)
145 |
146 | def decorators(self, node):
147 | for decorator in node.decorator_list:
148 | self.newline(decorator)
149 | self.write('@')
150 | self.visit(decorator)
151 | # Statements
152 |
153 | def visit_Assign(self, node):
154 | self.newline(node)
155 | for idx, target in enumerate(node.targets):
156 | if idx:
157 | self.write(', ')
158 | self.visit(target)
159 | self.write(' = ')
160 | self.visit(node.value)
161 |
162 | def visit_AugAssign(self, node):
163 | self.newline(node)
164 | self.visit(node.target)
165 | self.write(BINOP_SYMBOLS[type(node.op)] + '=')
166 | self.visit(node.value)
167 |
168 | def visit_ImportFrom(self, node):
169 | self.newline(node)
170 | self.write('from %s%s import ' % ('.' * node.level, node.module))
171 | for idx, item in enumerate(node.names):
172 | if idx:
173 | self.write(', ')
174 | self.write(item.name)
175 |
176 | def visit_Import(self, node):
177 | self.newline(node)
178 | if node.names:
179 | self.write('import ')
180 | for item_index, item in enumerate(node.names, 1):
181 | self.visit(item)
182 | if item_index != len(node.names):
183 | self.write(', ')
184 |
185 | def visit_Expr(self, node):
186 | self.newline(node)
187 | self.generic_visit(node)
188 |
189 | def visit_FunctionDef(self, node):
190 | self.newline(extra=1)
191 | self.decorators(node)
192 | self.newline(node)
193 | self.write('def %s(' % node.name)
194 | self.signature(node.args)
195 | self.write('):')
196 | self.body(node.body)
197 |
198 | def visit_ClassDef(self, node):
199 | have_args = []
200 |
201 | def paren_or_comma():
202 | if have_args:
203 | self.write(', ')
204 | else:
205 | have_args.append(True)
206 | self.write('(')
207 |
208 | self.newline(extra=2)
209 | self.decorators(node)
210 | self.newline(node)
211 | self.write('class %s' % node.name)
212 | for base in node.bases:
213 | paren_or_comma()
214 | self.visit(base)
215 | # XXX: the if here is used to keep this module compatible
216 | # with python 2.6.
217 | if hasattr(node, 'keywords'):
218 | for keyword in node.keywords:
219 | paren_or_comma()
220 | self.write(keyword.arg + '=')
221 | self.visit(keyword.value)
222 | if node.starargs is not None:
223 | paren_or_comma()
224 | self.write('*')
225 | self.visit(node.starargs)
226 | if node.kwargs is not None:
227 | paren_or_comma()
228 | self.write('**')
229 | self.visit(node.kwargs)
230 | self.write(have_args and '):' or ':')
231 | self.body(node.body)
232 |
233 | def visit_If(self, node):
234 | self.newline(node)
235 | self.write('if ')
236 | self.visit(node.test)
237 | self.write(':')
238 | self.body(node.body)
239 | while True:
240 | else_ = node.orelse
241 | if len(else_) == 1 and isinstance(else_[0], If):
242 | node = else_[0]
243 | self.newline()
244 | self.write('elif ')
245 | self.visit(node.test)
246 | self.write(':')
247 | self.body(node.body)
248 | else:
249 | self.newline()
250 | self.write('else:')
251 | self.body(else_)
252 | break
253 |
254 | def visit_For(self, node):
255 | self.newline(node)
256 | self.write('for ')
257 | self.visit(node.target)
258 | self.write(' in ')
259 | self.visit(node.iter)
260 | self.write(':')
261 | self.body_or_else(node)
262 |
263 | def visit_While(self, node):
264 | self.newline(node)
265 | self.write('while ')
266 | self.visit(node.test)
267 | self.write(':')
268 | self.body_or_else(node)
269 |
270 | def visit_With(self, node):
271 | self.newline(node)
272 | self.write('with ')
273 | self.visit(node.context_expr)
274 | if node.optional_vars is not None:
275 | self.write(' as ')
276 | self.visit(node.optional_vars)
277 | self.write(':')
278 | self.body(node.body)
279 |
280 | def visit_Pass(self, node):
281 | self.newline(node)
282 | self.write('pass')
283 |
284 | def visit_Print(self, node):
285 | # XXX: python 2.6 only
286 | self.newline(node)
287 | self.write('print ')
288 | want_comma = False
289 | if node.dest is not None:
290 | self.write(' >> ')
291 | self.visit(node.dest)
292 | want_comma = True
293 | for value in node.values:
294 | if want_comma:
295 | self.write(', ')
296 | self.visit(value)
297 | want_comma = True
298 | if not node.nl:
299 | self.write(',')
300 |
301 | def visit_Delete(self, node):
302 | self.newline(node)
303 | self.write('del ')
304 | for idx, target in enumerate(node):
305 | if idx:
306 | self.write(', ')
307 | self.visit(target)
308 |
309 | def visit_TryExcept(self, node):
310 | self.newline(node)
311 | self.write('try:')
312 | self.body(node.body)
313 | for handler in node.handlers:
314 | self.visit(handler)
315 |
316 | def visit_TryFinally(self, node):
317 | self.newline(node)
318 | self.write('try:')
319 | self.body(node.body)
320 | self.newline(node)
321 | self.write('finally:')
322 | self.body(node.finalbody)
323 |
324 | def visit_Global(self, node):
325 | self.newline(node)
326 | self.write('global ' + ', '.join(node.names))
327 |
328 | def visit_Nonlocal(self, node):
329 | self.newline(node)
330 | self.write('nonlocal ' + ', '.join(node.names))
331 |
332 | def visit_Return(self, node):
333 | self.newline(node)
334 | if node.value:
335 | self.write('return ')
336 | self.visit(node.value)
337 | else:
338 | self.write('return')
339 |
340 | def visit_Break(self, node):
341 | self.newline(node)
342 | self.write('break')
343 |
344 | def visit_Continue(self, node):
345 | self.newline(node)
346 | self.write('continue')
347 |
348 | def visit_Raise(self, node):
349 | # XXX: Python 2.6 / 3.0 compatibility
350 | self.newline(node)
351 | self.write('raise')
352 | if hasattr(node, 'exc') and node.exc is not None:
353 | self.write(' ')
354 | self.visit(node.exc)
355 | if node.cause is not None:
356 | self.write(' from ')
357 | self.visit(node.cause)
358 | elif hasattr(node, 'type') and node.type is not None:
359 | self.visit(node.type)
360 | if node.inst is not None:
361 | self.write(', ')
362 | self.visit(node.inst)
363 | if node.tback is not None:
364 | self.write(', ')
365 | self.visit(node.tback)
366 | # Expressions
367 |
368 | def visit_Attribute(self, node):
369 | self.visit(node.value)
370 | self.write('.' + node.attr)
371 |
372 | def visit_Call(self, node):
373 | want_comma = []
374 |
375 | def write_comma():
376 | if want_comma:
377 | self.write(', ')
378 | else:
379 | want_comma.append(True)
380 |
381 | self.visit(node.func)
382 | self.write('(')
383 | for arg in node.args:
384 | write_comma()
385 | self.visit(arg)
386 | for keyword in node.keywords:
387 | write_comma()
388 | self.write(keyword.arg + '=')
389 | self.visit(keyword.value)
390 | if node.starargs is not None:
391 | write_comma()
392 | self.write('*')
393 | self.visit(node.starargs)
394 | if node.kwargs is not None:
395 | write_comma()
396 | self.write('**')
397 | self.visit(node.kwargs)
398 | self.write(')')
399 |
400 | def visit_Name(self, node):
401 | self.write(node.id)
402 |
403 | def visit_Str(self, node):
404 | self.write(repr(node.s))
405 |
406 | def visit_Bytes(self, node):
407 | self.write(repr(node.s))
408 |
409 | def visit_Num(self, node):
410 | self.write(repr(node.n))
411 |
412 | def visit_Tuple(self, node):
413 | self.write('(')
414 | idx = -1
415 | for idx, item in enumerate(node.elts):
416 | if idx:
417 | self.write(', ')
418 | self.visit(item)
419 | self.write(idx and ')' or ',)')
420 |
421 | def sequence_visit(left, right):
422 | def visit(self, node):
423 | self.write(left)
424 | for idx, item in enumerate(node.elts):
425 | if idx:
426 | self.write(', ')
427 | self.visit(item)
428 | self.write(right)
429 | return visit
430 |
431 | visit_List = sequence_visit('[', ']')
432 | visit_Set = sequence_visit('{', '}')
433 | del sequence_visit
434 |
435 | def visit_Dict(self, node):
436 | self.write('{')
437 | for idx, (key, value) in enumerate(zip(node.keys, node.values)):
438 | if idx:
439 | self.write(', ')
440 | self.visit(key)
441 | self.write(': ')
442 | self.visit(value)
443 | self.write('}')
444 |
445 | def visit_BinOp(self, node):
446 | self.visit(node.left)
447 | self.write(' %s ' % BINOP_SYMBOLS[type(node.op)])
448 | self.visit(node.right)
449 |
450 | def visit_BoolOp(self, node):
451 | self.write('(')
452 | for idx, value in enumerate(node.values):
453 | if idx:
454 | self.write(' %s ' % BOOLOP_SYMBOLS[type(node.op)])
455 | self.visit(value)
456 | self.write(')')
457 |
458 | def visit_Compare(self, node):
459 | self.write('(')
460 | self.write(node.left)
461 | for op, right in zip(node.ops, node.comparators):
462 | self.write(' %s %%' % CMPOP_SYMBOLS[type(op)])
463 | self.visit(right)
464 | self.write(')')
465 |
466 | def visit_UnaryOp(self, node):
467 | self.write('(')
468 | op = UNARYOP_SYMBOLS[type(node.op)]
469 | self.write(op)
470 | if op == 'not':
471 | self.write(' ')
472 | self.visit(node.operand)
473 | self.write(')')
474 |
475 | def visit_Subscript(self, node):
476 | self.visit(node.value)
477 | self.write('[')
478 | self.visit(node.slice)
479 | self.write(']')
480 |
481 | def visit_Slice(self, node):
482 | if node.lower is not None:
483 | self.visit(node.lower)
484 | self.write(':')
485 | if node.upper is not None:
486 | self.visit(node.upper)
487 | if node.step is not None:
488 | self.write(':')
489 | if not (isinstance(node.step, Name) and node.step.id == 'None'):
490 | self.visit(node.step)
491 |
492 | def visit_ExtSlice(self, node):
493 | for idx, item in node.dims:
494 | if idx:
495 | self.write(', ')
496 | self.visit(item)
497 |
498 | def visit_Yield(self, node):
499 | self.write('yield ')
500 | self.visit(node.value)
501 |
502 | def visit_Lambda(self, node):
503 | self.write('lambda ')
504 | self.signature(node.args)
505 | self.write(': ')
506 | self.visit(node.body)
507 |
508 | def visit_Ellipsis(self, node):
509 | self.write('Ellipsis')
510 |
511 | def generator_visit(left, right):
512 | def visit(self, node):
513 | self.write(left)
514 | self.visit(node.elt)
515 | for comprehension in node.generators:
516 | self.visit(comprehension)
517 | self.write(right)
518 | return visit
519 |
520 | visit_ListComp = generator_visit('[', ']')
521 | visit_GeneratorExp = generator_visit('(', ')')
522 | visit_SetComp = generator_visit('{', '}')
523 | del generator_visit
524 |
525 | def visit_DictComp(self, node):
526 | self.write('{')
527 | self.visit(node.key)
528 | self.write(': ')
529 | self.visit(node.value)
530 | for comprehension in node.generators:
531 | self.visit(comprehension)
532 | self.write('}')
533 |
534 | def visit_IfExp(self, node):
535 | self.visit(node.body)
536 | self.write(' if ')
537 | self.visit(node.test)
538 | self.write(' else ')
539 | self.visit(node.orelse)
540 |
541 | def visit_Starred(self, node):
542 | self.write('*')
543 | self.visit(node.value)
544 |
545 | def visit_Repr(self, node):
546 | # XXX: python 2.6 only
547 | self.write('`')
548 | self.visit(node.value)
549 | self.write('`')
550 | # Helper Nodes
551 |
552 | def visit_alias(self, node):
553 | self.write(node.name)
554 | if node.asname is not None:
555 | self.write(' as ' + node.asname)
556 |
557 | def visit_comprehension(self, node):
558 | self.write(' for ')
559 | self.visit(node.target)
560 | self.write(' in ')
561 | self.visit(node.iter)
562 | if node.ifs:
563 | for if_ in node.ifs:
564 | self.write(' if ')
565 | self.visit(if_)
566 |
567 | def visit_excepthandler(self, node):
568 | self.newline(node)
569 | self.write('except')
570 | if node.type is not None:
571 | self.write(' ')
572 | self.visit(node.type)
573 | if node.name is not None:
574 | self.write(' as ')
575 | self.visit(node.name)
576 | self.write(':')
577 | self.body(node.body)
--------------------------------------------------------------------------------
96 | {% if request.user.is_authenticated %}
97 |
128 |
98 |
99 |
100 |
110 |
111 | {% endif %}
112 |
113 | | {% trans "Id" %} | 101 |{% trans "Name" %} | 102 |{% trans "Status" %} | 103 |{% trans "Submitted" %} | 104 | 105 | 106 | 107 | 108 | 109 |
|---|
114 |
115 |
116 |
126 |
127 | | {% trans "Id" %} | 117 |{% trans "Name" %} | 118 |{% trans "Status" %} | 119 |{% trans "Submitted" %} | 120 | 121 | 122 | 123 | 124 | 125 |
|---|