";
2 |
3 | var load_examples = function(page, endtime) {
4 | // First, destroy the more button if there is one.
5 | $('#more-button').remove();
6 |
7 | // Then, get the next page of data from the API (relative url)..
8 | $.ajax("ex/" + page + "/" + endtime, {
9 | success: examples_success,
10 | error: examples_error,
11 | });
12 | }
13 |
14 | var examples_success = function(data, status, jqXHR) {
15 | var stopping = false;
16 | if (data.results.length == 0) {
17 | load_examples(data.next_page, data.endtime);
18 | } else {
19 | $('#examples-container .lead').html(
20 | "The following messages would have matched this filter");
21 | stopping = true;
22 | }
23 |
24 | // Put our results on the page.
25 | $.each(data.results, function(i, meta) {
26 | var content = "' + data.furthermore + '
'); 75 | } 76 | $('#examples-container p').addClass('text-danger'); 77 | } 78 | 79 | $(document).ready(function() { 80 | // Kick it off, but only if we're on the right page and there are rules. 81 | var rules = $("#rules"); 82 | var container = $('#examples-container'); 83 | if (container.length > 0 && rules.children().length > 0) { 84 | $('#examples-container .lead').html(examples_loading_message); 85 | var now = Math.floor(new Date().getTime() / 1000.0); 86 | load_examples(1, now); 87 | } 88 | 89 | function autocloud_message() { 90 | var message = " _ _ _ \n\ 91 | __ _ _ _| |_ ___ ___| | ___ _ _ __| |\n\ 92 | / _` | | | | __/ _ \\ / __| |/ _ \\| | | |/ _` |\n\ 93 | | (_| | |_| | || (_) | (__| | (_) | |_| | (_| |\n\ 94 | \\__,_|\\__,_|\\__\\___/ \\___|_|\\___/ \\__,_|\\__,_|\n\ 95 | \n\ 96 | =======================================================\n\ 97 | Looking forward to contribute to the project?\n\ 98 | Head over to https://github.com/kushaldas/autocloud \n\ 99 | =======================================================\n\ 100 | "; 101 | console.log(message); 102 | } 103 | setTimeout(autocloud_message, 1000); 104 | }); 105 | -------------------------------------------------------------------------------- /autocloud/web/templates/404.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 |
17 | _ _ ___ _ _
18 | | || | / _ \| || |
19 | | || |_| | | | || |_
20 | |__ __| |_| |__ _|
21 | |_| \___/ |_|
22 |
23 |
24 |
25 |
26 | _\|/_
27 | (o o)
28 | +------------------------------------------------------------oOO-{_}-OOo---------------------------------------------------------+
29 | | _ _ _ _ _ _ |
30 | || | | | ___ _ _ ___| |_ ___ _ __ __ _____ | |__ __ ___ _____ __ _ _ __ _ __ ___ | |__ | | ___ _ __ ___ |
31 | || |_| |/ _ \| | | / __| __/ _ \| '_ \ \ \ /\ / / _ \ | '_ \ / _` \ \ / / _ \ / _` | | '_ \| '__/ _ \| '_ \| |/ _ \ '_ ` _ \ |
32 | || _ | (_) | |_| \__ \ || (_) | | | |_ \ V V / __/ | | | | (_| |\ V / __/ | (_| | | |_) | | | (_) | |_) | | __/ | | | | ||
33 | ||_| |_|\___/ \__,_|___/\__\___/|_| |_( ) \_/\_/ \___| |_| |_|\__,_| \_/ \___| \__,_| | .__/|_| \___/|_.__/|_|\___|_| |_| |_||
34 | | |/ |_| |
35 | | |
36 | +-------------------------------------------------------------------------------------------------------------------------------*/
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
--------------------------------------------------------------------------------
/autocloud/web/templates/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/kushaldas/autocloud/651272df9c24e8b8e2f11970cdfca61554a3e323/autocloud/web/templates/__init__.py
--------------------------------------------------------------------------------
/autocloud/web/templates/compose_details.html:
--------------------------------------------------------------------------------
1 | {% extends "master.html" %}
2 |
3 | {% block body %}
4 |
5 |
19 |
20 |
21 |
70 |
71 | {% endblock %}
72 |
--------------------------------------------------------------------------------
/autocloud/web/templates/index.html:
--------------------------------------------------------------------------------
1 | {% extends "master.html" %}
2 |
3 | {% block body %}
4 | | Compose ID | 23 |Status | 24 |Date | 25 |Type | 26 |Respin | 27 |Overview | 28 |29 | |
|---|---|---|---|---|---|---|
| 35 | {{ compose_detail.compose_id }} 36 | | 37 |38 | {% if compose_detail.status == 'c' %} 39 | 40 | {% elif compose_detail.status == 'r' %} 41 | 42 | {% elif compose_detail.status == 'q' %} 43 | 44 | {% else %} 45 | 46 | {% endif %} 47 | {{ compose_detail.status }} 48 | 49 | | 50 |51 | {{ compose_detail.date.strftime('%Y-%m-%d') }} 52 | | 53 |54 | {{ compose_detail.type }} 55 | | 56 |57 | {{ compose_detail.respin }} 58 | | 59 |60 | {{ compose_detail.passed }} / 61 | {{ compose_detail.failed }} 62 | | 63 ||
| No results to display! |
5 |
8 | {% endblock %}
9 |
--------------------------------------------------------------------------------
/autocloud/web/templates/job_details.html:
--------------------------------------------------------------------------------
1 | {% extends "master.html" %}
2 |
3 | {% block body %}
4 | Autocloud
6 | Go to compose details 7 |
5 |
19 |
41 |
42 |
43 |
97 |
98 | {% endblock %}
99 |
--------------------------------------------------------------------------------
/autocloud/web/templates/job_output.html:
--------------------------------------------------------------------------------
1 | {% extends "master.html" %}
2 |
3 | {% block body %}
4 | | Compose ID | 45 |Family | 46 |Architecture | 47 |Release | 48 |Type | 49 |Status | 50 |Created | 51 |Last updated | 52 |Output | 53 |
|---|---|---|---|---|---|---|---|---|
| 59 | {{ job_detail.compose_id }} 60 | | 61 |62 | {{ job_detail.family }} 63 | | 64 |65 | {{ job_detail.arch }} 66 | | 67 |68 | {{ job_detail.release }} 69 | | 70 |71 | {{ job_detail.image_type }} 72 | | 73 |74 | {% if job_detail.status == 's' %} 75 | 76 | {% elif job_detail.status in ('f', 'a') %} 77 | 78 | {% elif job_detail.status == 'r' %} 79 | 80 | {% elif job_detail.status == 'q' %} 81 | 82 | {% else %} 83 | 84 | {% endif %} 85 | {{ job_detail.status }} 86 | 87 | | 88 |{{ job_detail.created_on.strftime('%Y-%m-%d %H:%M:%S') }} | 89 |{{ job_detail.last_updated.strftime('%Y-%m-%d %H:%M:%S') }} | 90 ||
| No results to display! |
5 |
12 |
13 |
14 |
64 |
80 |
81 | {% endblock %}
82 |
--------------------------------------------------------------------------------
/autocloud/web/templates/login.html:
--------------------------------------------------------------------------------
1 | {% extends "master.html" %}
2 |
3 | {% block body %}
4 | | Compose ID | 16 |Family | 17 |Architecture | 18 |Release | 19 |Type | 20 |Status | 21 |Created | 22 |Last updated | 23 |Link | 24 |
|---|---|---|---|---|---|---|---|---|
| 29 | {{ job_detail.compose_id }} 30 | | 31 |32 | {{ job_detail.family }} 33 | | 34 |35 | {{ job_detail.arch }} 36 | | 37 |38 | {{ job_detail.release }} 39 | | 40 |41 | {{ job_detail.image_type }} 42 | | 43 |44 | {% if job_detail.status == 's' %} 45 | 46 | {% elif job_detail.status in ('f', 'a') %} 47 | 48 | {% elif job_detail.status == 'r' %} 49 | 50 | {% elif job_detail.status == 'q' %} 51 | 52 | {% else %} 53 | 54 | {% endif %} 55 | {{ job_detail.status }} 56 | 57 | | 58 |{{ job_detail.created_on.strftime('%Y-%m-%d %H:%M:%S') }} | 59 |{{ job_detail.last_updated.strftime('%Y-%m-%d %H:%M:%S') }} | 60 |
5 |
25 |
26 | {% endblock %}
27 |
--------------------------------------------------------------------------------
/autocloud/web/templates/master.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
6 |
7 |
24 |
8 | {% if config['FMN_ALLOW_GENERIC_OPENID'] %}
9 |
10 |
23 | OpenId Login
11 | 19 | 20 | 21 | {% endif %} 22 |
25 |
26 |
27 |
41 |
42 | {% block body %}{% endblock %}
43 |
44 |
45 |
46 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
79 |
80 | {% if config['FEDMENU_URL'] %}
81 |
82 |
83 |
93 | {% endif %}
94 |
95 |
96 |
97 |
--------------------------------------------------------------------------------
/autocloud/web/utils.py:
--------------------------------------------------------------------------------
1 | from sqlalchemy.orm import exc
2 | from werkzeug.exceptions import abort
3 |
4 |
5 | def get_object_or_404(session, model, *criterion):
6 | try:
7 | return session.query(model).filter(*criterion).one()
8 | except exc.NoResultFound, exc.MultipleResultsFound:
9 | abort(404)
10 |
--------------------------------------------------------------------------------
/autocloud_job:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 |
3 | python -tt /usr/share/autocloud/autocloud_job.py "$@"
4 |
--------------------------------------------------------------------------------
/autocloud_job.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | import copy
3 | import datetime
4 | import json
5 | import os
6 | import subprocess
7 |
8 | import fedfind.release
9 |
10 | from retask.queue import Queue
11 |
12 | from autocloud.constants import SUCCESS, FAILED, RUNNING
13 | from autocloud.models import init_model, ComposeJobDetails, ComposeDetails
14 | from autocloud.producer import publish_to_fedmsg
15 |
16 | import logging
17 |
18 | logging.basicConfig(level=logging.DEBUG)
19 | log = logging.getLogger(__name__)
20 |
21 |
22 | def handle_err(session, data, out, err):
23 | """
24 | Prints the details and exits.
25 | :param out:
26 | :param err:
27 | :return: None
28 | """
29 | # Update DB first.
30 | data.status = u'f'
31 | data.output = "%s: %s" % (out, err)
32 | timestamp = datetime.datetime.now()
33 | data.last_updated = timestamp
34 | session.commit()
35 | log.debug("%s: %s", out, err)
36 |
37 |
38 | def system(cmd):
39 | """
40 | Runs a shell command, and returns the output, err, returncode
41 |
42 | :param cmd: The command to run.
43 | :return: Tuple with (output, err, returncode).
44 | """
45 | ret = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
46 | stdout=subprocess.PIPE,
47 | stderr=subprocess.PIPE, close_fds=True)
48 | out, err = ret.communicate()
49 | returncode = ret.returncode
50 | return out, err, returncode
51 |
52 |
53 | def refresh_storage_pool():
54 | '''Refreshes libvirt storage pool.
55 |
56 | http://kushaldas.in/posts/storage-volume-error-in-libvirt-with-vagrant.html
57 | '''
58 | out, err, retcode = system('virsh pool-list')
59 | lines = out.split('\n')
60 | if len(lines) > 2:
61 | for line in lines[2:]:
62 | words = line.split()
63 | if len(words) == 3:
64 | if words[1] == 'active':
65 | system('virsh pool-refresh {0}'.format(words[0]))
66 |
67 |
68 | def image_cleanup(image_path):
69 | """
70 | Delete the image if it is processed or if there is any exception occur
71 |
72 | :param basename: Absoulte path for image
73 | """
74 | if os.path.exists(image_path):
75 | try:
76 | os.remove(image_path)
77 | except OSError as e:
78 | log.error('Error: %s - %s.', e.filename, e.strerror)
79 |
80 |
81 | def create_dirs():
82 | """
83 | Creates the runtime dirs
84 | """
85 | system('mkdir -p /var/run/tunir')
86 | system('mkdir -p /var/run/autocloud')
87 |
88 |
89 | def create_result_text(out):
90 | """
91 | :param out: Output text from the command.
92 | """
93 | result_filename = None
94 | lines = out.splitlines()
95 | for line in lines:
96 | if line.startswith('Result file at:'):
97 | result_filename = line.split(' ')[1]
98 |
99 | if not result_filename: # In case no result file.
100 | return "NO RESULT FILE."
101 | result_filename = result_filename.strip()
102 | if os.path.exists(result_filename):
103 | new_content = ''
104 | with open(result_filename) as fobj:
105 | new_content = fobj.read()
106 | job_status_index = out.find('Job status:')
107 | if job_status_index == -1:
108 | return out # No job status in the output.
109 | new_line_index = out[job_status_index:].find('\n')
110 | out = out[:job_status_index + new_line_index]
111 | out = out + '\n\n' + new_content
112 | system('rm -f {0}'.format(result_filename))
113 | return out
114 | return out
115 |
116 |
117 | def auto_job(task_data):
118 | """
119 | This fuction queues the job, and then executes the tests,
120 | updates the db as required.
121 |
122 | :param taskid: Koji taskid.
123 | :param image_url: URL to download the fedora image.
124 | :return:
125 | """
126 | # TODO:
127 | # We will have to update the job information on DB, rather
128 | # than creating it. But we will do it afterwards.
129 |
130 | compose_image_url = task_data['absolute_path']
131 | compose_id = task_data['compose']['id']
132 | release = task_data['compose']['release']
133 | job_id = task_data['job_id']
134 | image_type = task_data['type']
135 |
136 | job_type = 'vm'
137 |
138 | # Just to make sure that we have runtime dirs
139 | create_dirs()
140 |
141 | session = init_model()
142 | timestamp = datetime.datetime.now()
143 | data = None
144 | try:
145 | data = session.query(ComposeJobDetails).get(str(job_id))
146 | data.status = u'r'
147 | data.last_updated = timestamp
148 | except Exception as err:
149 | log.error("%s" % err)
150 | log.error("%s: %s", compose_id, compose_image_url)
151 | session.commit()
152 |
153 | params = {
154 | 'compose_url': compose_image_url,
155 | 'compose_id': compose_id,
156 | 'status': RUNNING,
157 | 'job_id': job_id,
158 | 'release': release,
159 | 'family': data.family.value,
160 | 'type': image_type,
161 | 'image_name': data.image_name,
162 | }
163 | publish_to_fedmsg(topic='image.running', **params)
164 |
165 | # Now we have job queued, let us start the job.
166 | # Step 1: Download the image
167 | image_url = compose_image_url
168 | basename = os.path.basename(image_url)
169 | image_path = '/var/run/autocloud/%s' % basename
170 | log.debug("Going to download {0}".format(image_url))
171 | out, err, ret_code = system('wget %s -O %s' % (image_url, image_path))
172 | if ret_code:
173 | image_cleanup(image_path)
174 | handle_err(session, data, out, err)
175 | log.debug("Return code: %d" % ret_code)
176 |
177 | params.update({'status': FAILED})
178 | publish_to_fedmsg(topic='image.failed', **params)
179 | check_status_of_compose_image(compose_id)
180 | return FAILED
181 |
182 | # Step 2: Create the conf file with correct image path.
183 | if basename.find('vagrant') == -1:
184 | conf = {"image": "/var/run/autocloud/%s" % basename,
185 | "name": "fedora",
186 | "password": "passw0rd",
187 | "ram": 2048,
188 | "type": "vm",
189 | "user": "fedora"}
190 |
191 | else: # We now have a Vagrant job.
192 | conf = {
193 | "name": "fedora",
194 | "type": "vagrant",
195 | "image": "/var/run/autocloud/%s" % basename,
196 | "ram": 2048,
197 | "user": "vagrant",
198 | "port": "22"
199 | }
200 | if basename.find('virtualbox') != -1:
201 | conf['provider'] = 'virtualbox'
202 | job_type = 'vagrant'
203 |
204 | # Now let us refresh the storage pool
205 | refresh_storage_pool()
206 |
207 | with open('/var/run/autocloud/fedora.json', 'w') as fobj:
208 | fobj.write(json.dumps(conf))
209 |
210 | system('/usr/bin/cp -f /etc/autocloud/fedora.txt '
211 | '/var/run/autocloud/fedora.txt')
212 |
213 | cmd = 'tunir --job fedora --config-dir /var/run/autocloud/'
214 | # Now run tunir
215 | out, err, ret_code = system(cmd)
216 | if ret_code:
217 | image_cleanup(image_path)
218 | handle_err(session, data, create_result_text(out), err)
219 | log.debug("Return code: %d" % ret_code)
220 | params.update({'status': FAILED})
221 | publish_to_fedmsg(topic='image.failed', **params)
222 | check_status_of_compose_image(compose_id)
223 | return FAILED
224 | else:
225 | image_cleanup(image_path)
226 |
227 | # Enabling direct stdout as output of the command
228 | out = create_result_text(out)
229 | if job_type == 'vm':
230 | com_text = out[out.find('/usr/bin/qemu-kvm'):]
231 | else:
232 | com_text = out
233 |
234 | data.status = u's'
235 | timestamp = datetime.datetime.now()
236 | data.last_updated = timestamp
237 | data.output = com_text
238 | session.commit()
239 | session.close()
240 |
241 | params.update({'status': SUCCESS})
242 | publish_to_fedmsg(topic='image.success', **params)
243 | check_status_of_compose_image(compose_id)
244 | return SUCCESS
245 |
246 |
247 | def check_status_of_compose_image(compose_id):
248 | session = init_model()
249 | compose_job_objs = session.query(ComposeJobDetails).filter_by(
250 | compose_id=compose_id).all()
251 | compose_obj = session.query(ComposeDetails).filter_by(
252 | compose_id=compose_id).first()
253 |
254 | results = {
255 | SUCCESS: 0,
256 | FAILED: 0,
257 | 'artifacts': {}
258 | }
259 |
260 | for compose_job_obj in compose_job_objs:
261 | status = compose_job_obj.status.code
262 | if status in ('r', 'q'):
263 | # Abort, since there's still jobs not finished
264 | return False
265 |
266 | elif status in ('s',):
267 | results[SUCCESS] = results[SUCCESS] + 1
268 |
269 | elif status in ('f', 'a'):
270 | results[FAILED] = results[FAILED] + 1
271 |
272 | artifact = {
273 | 'architecture': compose_job_obj.arch.value,
274 | 'family': compose_job_obj.family.value,
275 | 'image_url': compose_job_obj.image_url,
276 | 'release': compose_job_obj.release,
277 | 'subvariant': compose_job_obj.subvariant,
278 | 'format': compose_job_obj.image_format,
279 | 'type': compose_job_obj.image_type,
280 | 'name': compose_job_obj.image_name,
281 | 'status': compose_job_obj.status.value
282 | }
283 | results['artifacts'][str(compose_job_obj.id)] = artifact
284 |
285 | compose_obj.passed = results[SUCCESS]
286 | compose_obj.failed = results[FAILED]
287 | compose_obj.status = u'c'
288 |
289 | session.commit()
290 |
291 | compose_id = compose_obj.compose_id
292 | rel = fedfind.release.get_release(cid=compose_id)
293 | release = rel.release
294 |
295 | params = {
296 | 'id': compose_obj.compose_id,
297 | 'respin': compose_obj.respin,
298 | 'type': compose_obj.type,
299 | 'date': datetime.datetime.strftime(compose_obj.date, '%Y%m%d'),
300 | 'results': results,
301 | 'release': release,
302 | 'status': 'completed',
303 | 'compose_job_id': compose_obj.id
304 | }
305 |
306 | publish_to_fedmsg(topic='compose.complete', **params)
307 |
308 | session.close()
309 |
310 | return True
311 |
312 |
313 | def main():
314 | jobqueue = Queue('jobqueue')
315 | jobqueue.connect()
316 |
317 | while True:
318 | task = jobqueue.wait()
319 |
320 | task_data = task.data
321 | pos, num_images = task_data['pos']
322 |
323 | compose_details = task_data['compose']
324 |
325 | if pos == 1:
326 | session = init_model()
327 |
328 | compose_id = compose_details['id']
329 | compose_obj = session.query(ComposeDetails).filter_by(
330 | compose_id=compose_id).first()
331 |
332 | compose_status = compose_obj.status.code
333 |
334 | # Here the check if the compose_status has completed 'c' is for
335 | # failsafe. This condition is never to be hit. This is to avoid
336 | # sending message to fedmsg.
337 | if compose_status in ('r', 'c'):
338 | log.info("Compose %s already running. Skipping sending to \
339 | fedmsg" % compose_id)
340 | else:
341 | compose_obj.status = u'r'
342 | session.commit()
343 |
344 | params = copy.deepcopy(compose_details)
345 | params.update({'status': 'running'})
346 | publish_to_fedmsg(topic='compose.running', **params)
347 |
348 | session.close()
349 |
350 | result = auto_job(task_data)
351 |
352 |
353 | if __name__ == '__main__':
354 | main()
355 |
--------------------------------------------------------------------------------
/config/autocloud.cfg:
--------------------------------------------------------------------------------
1 | [autocloud]
2 | koji_server_url = http://koji.fedoraproject.org/kojihub/
3 | base_koji_task_url = https://kojipkgs.fedoraproject.org//work/
4 | virtualbox = false
5 | host = 0.0.0.0
6 | port = 5000
7 | debug = false
8 |
9 | [sqlalchemy]
10 | uri = sqlite:///autocloud.db
11 |
--------------------------------------------------------------------------------
/createdb.py:
--------------------------------------------------------------------------------
1 | from autocloud import models
2 |
3 | models.create_tables()
4 |
--------------------------------------------------------------------------------
/docs/Makefile:
--------------------------------------------------------------------------------
1 | # Makefile for Sphinx documentation
2 | #
3 |
4 | # You can set these variables from the command line.
5 | SPHINXOPTS =
6 | SPHINXBUILD = sphinx-build
7 | PAPER =
8 | BUILDDIR = _build
9 |
10 | # User-friendly check for sphinx-build
11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
13 | endif
14 |
15 | # Internal variables.
16 | PAPEROPT_a4 = -D latex_paper_size=a4
17 | PAPEROPT_letter = -D latex_paper_size=letter
18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
19 | # the i18n builder cannot share the environment and doctrees with the others
20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
21 |
22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
23 |
24 | help:
25 | @echo "Please use \`make