├── fedimg ├── services │ ├── ec2 │ │ ├── __init__.py │ │ ├── ec2base.py │ │ ├── ec2copy.py │ │ ├── ec2initiate.py │ │ ├── ec2imgpublisher.py │ │ └── ec2imguploader.py │ └── __init__.py ├── exceptions.py ├── __init__.py ├── messenger.py ├── config.py ├── uploader.py ├── consumers.py └── utils.py ├── setup.cfg ├── MANIFEST.in ├── mkdocs.yml ├── tox.ini ├── .travis.yml ├── fedimg-conf.toml.example ├── docs ├── architecture │ ├── consumer.rst │ └── configuration.rst ├── development │ ├── contribution.rst │ ├── installation.rst │ └── testing.rst ├── index.rst ├── release_notes.rst ├── services │ └── ec2.md ├── Makefile └── conf.py ├── .gitignore ├── tests ├── utils.py ├── test_uploader.py ├── test_consumers.py ├── test_utils.py └── vcr-request-data │ └── test_consumers.TestFedimgConsumer.test_incompatible_images ├── fedmsg.d ├── logging.py └── fedimg.py ├── bin ├── trigger_upload.py ├── partial_upload.py └── list-the-amis.py ├── README.md ├── setup.py ├── devel └── Vagrantfile.example └── CHANGELOG.rst /fedimg/services/ec2/__init__.py: -------------------------------------------------------------------------------- 1 | class EC2Service(): 2 | pass 3 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | test=pytest 3 | 4 | [flake8] 5 | show_source = True 6 | exclude = .git,.tox,dist,*egg,build 7 | application_import_names = fedimg 8 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.md 3 | include CHANGELOG.rst 4 | include fedimg-conf.toml.example 5 | include mkdocs.yml 6 | recursive-include fedmsg.d *.py 7 | recursive-include docs * 8 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: Fedimg 2 | pages: 3 | - ['index.md', 'About'] 4 | - ['installation.md', 'Installation'] 5 | - ['configuration.md', 'Configuration'] 6 | - ['consumer.md', 'Consumer'] 7 | - ['messaging.md', 'Messaging'] 8 | - ['contributing.md', 'Contributing'] 9 | - ['services/ec2.md', 'Services', 'EC2'] 10 | - ['development/testing.md', 'Development', 'Testing'] 11 | theme: readthedocs 12 | -------------------------------------------------------------------------------- /fedimg/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Global fedimg exception and warning classes 3 | """ 4 | 5 | 6 | class SourceNotFound(Exception): 7 | """ The requested source was not found""" 8 | pass 9 | 10 | 11 | class CommandRunFailed(Exception): 12 | """ The request command failed while running""" 13 | pass 14 | 15 | 16 | class UnCompressFailed(Exception): 17 | """ The uncompress operation of the raw image failed""" 18 | pass 19 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27,lint 3 | 4 | [testenv] 5 | deps = 6 | psutil 7 | moksha 8 | pytest 9 | pytest-cov 10 | mock 11 | vcrpy 12 | # Substitute your test runner of choice 13 | commands = 14 | py.test 15 | 16 | [testenv:lint] 17 | deps = 18 | flake8 19 | commands = 20 | python -m flake8 {posargs} 21 | 22 | [flake8] 23 | show-source = True 24 | max-line-length = 90 25 | exclude = .git,.tox,dist,*egg,tests -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.7" 4 | 5 | install: 6 | - pip install flake8 7 | - pip install psutil # hack: fedmsg depends on psutil but it doesn't get installed properly 8 | - pip install moksha # hack: fedmsg messes up its moksha dependency too 9 | - python setup.py -q install 10 | 11 | before_script: 12 | - sudo mkdir -p /etc/fedimg/ 13 | - sudo cp fedimg-conf.toml.example /etc/fedimg/fedimg-conf.toml 14 | 15 | script: 16 | - python setup.py -q test 17 | - flake8 fedimg 18 | -------------------------------------------------------------------------------- /fedimg-conf.toml.example: -------------------------------------------------------------------------------- 1 | [general] 2 | clean_up_on_failure = true 3 | delete_images_on_failure = true 4 | process_count = 1 5 | active_services = ['aws'] 6 | 7 | [aws] 8 | volume_size = 7 9 | access_id = "secretaccessID" 10 | secret_key = "supersecretsecretkey" 11 | base_region = 'us-east-1' 12 | bucket_name = 'fedora-s3-bucket-fedimg' 13 | volume_types = ['standard', 'gp2'] 14 | regions = ['ap-northeast-2', 'us-east-2', 'ap-southeast-1', 'ap-southeast-2', 15 | 'ap-south-1', 'eu-west-1', 'sa-east-1', 'us-east-1', 'us-west-2', 16 | 'us-west-1', 'eu-central-1', 'ap-northeast-1', 'ca-central-1', 17 | 'eu-west-2', 'eu-west-3'] 18 | -------------------------------------------------------------------------------- /docs/architecture/consumer.rst: -------------------------------------------------------------------------------- 1 | ============== 2 | FedimgConsumer 3 | ============== 4 | 5 | Fedimg at it's core uses FedimgConsumer which is a Fedmsg consumer. 6 | FedimgConsumer listions to the fedmsg bus for the completed `Pungi`_ composes. 7 | 8 | FedimgConsumer listens to ``pungi.compose.status.change`` topic. On receiving a 9 | message from fedmsg it: 10 | 11 | - Checks if the `status` of the compose is either ``FINISHED_INCOMPLETE`` or 12 | ``FINISHED``. 13 | - Then, it proceeds to strips the fedmsg-blanket to get the core message. 14 | - It parses the image location and starts the upload process. 15 | - For every event, Fedimg sends out a fedimg notification with the ``fedimg`` 16 | category. Read more about the `list of topics here`_ 17 | 18 | .. _Pungi: https://pagure.io/pungi 19 | .. _list of topics here: http://fedora-fedmsg.readthedocs.io/en/latest/topics.html#fedimg 20 | -------------------------------------------------------------------------------- /fedimg/services/__init__.py: -------------------------------------------------------------------------------- 1 | # This file is part of fedimg. 2 | # Copyright (C) 2014 Red Hat, Inc. 3 | # 4 | # fedimg is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as 6 | # published by the Free Software Foundation, either version 3 of the 7 | # License, or (at your option) any later version. 8 | # 9 | # fedimg is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | # Affero General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU Affero General Public 15 | # License along with fedimg; if not, see http://www.gnu.org/licenses, 16 | # or write to the Free Software Foundation, Inc., 17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 18 | # 19 | # Authors: David Gay 20 | # 21 | -------------------------------------------------------------------------------- /fedimg/__init__.py: -------------------------------------------------------------------------------- 1 | # This file is part of fedimg. 2 | # Copyright (C) 2018 Red Hat, Inc. 3 | # 4 | # fedimg is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as 6 | # published by the Free Software Foundation, either version 3 of the 7 | # License, or (at your option) any later version. 8 | # 9 | # fedimg is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | # Affero General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU Affero General Public 15 | # License along with fedimg; if not, see http://www.gnu.org/licenses, 16 | # or write to the Free Software Foundation, Inc., 17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 18 | # 19 | # Authors: Sayan Chowdhury 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | eggs/ 15 | lib/ 16 | lib64/ 17 | parts/ 18 | sdist/ 19 | var/ 20 | *.egg-info/ 21 | .installed.cfg 22 | *.egg 23 | 24 | # Installer logs 25 | pip-log.txt 26 | pip-delete-this-directory.txt 27 | 28 | # Unit test / coverage reports 29 | htmlcov/ 30 | .tox/ 31 | .coverage 32 | .cache 33 | nosetests.xml 34 | coverage.xml 35 | 36 | # Translations 37 | *.mo 38 | 39 | # Mr Developer 40 | .mr.developer.cfg 41 | .project 42 | .pydevproject 43 | 44 | # Rope 45 | .ropeproject 46 | 47 | # Django stuff: 48 | *.log 49 | *.pot 50 | 51 | # Sphinx documentation 52 | docs/_build/ 53 | 54 | # Swap files 55 | *~* 56 | *.swp 57 | 58 | # Config file (containing secret keys and whatnot) 59 | fedimg.cfg 60 | .eggs 61 | 62 | # pytest 63 | *pytest_cache 64 | 65 | # vagrant 66 | .vagrant 67 | Vagrantfile -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- 1 | # This file is part of fedimg. 2 | # Copyright (C) 2017 Red Hat, Inc. 3 | # 4 | # fedimg is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as 6 | # published by the Free Software Foundation, either version 3 of the 7 | # License, or (at your option) any later version. 8 | # 9 | # fedimg is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | # Affero General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU Affero General Public 15 | # License along with fedimg; if not, see http://www.gnu.org/licenses, 16 | # or write to the Free Software Foundation, Inc., 17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 18 | # 19 | # Authors: Sayan Chowdhury 20 | # 21 | 22 | 23 | class MockHub(object): 24 | config = { 25 | "fedimgconsumer.prod.enabled": True, 26 | "validate_signatures": False, 27 | } 28 | 29 | def subscribe(self, *args, **kwargs): 30 | pass 31 | -------------------------------------------------------------------------------- /fedmsg.d/logging.py: -------------------------------------------------------------------------------- 1 | # Setup fedmsg logging. 2 | # See the following for constraints on this format https://bit.ly/Xn1WDn 3 | bare_format = "[%(asctime)s][%(name)s][%(levelname)s](%(threadName)s) %(message)s" 4 | 5 | config = dict( 6 | logging=dict( 7 | version=1, 8 | formatters=dict( 9 | bare={ 10 | "datefmt": "%Y-%m-%d %H:%M:%S", 11 | "format": bare_format 12 | }, 13 | ), 14 | handlers=dict( 15 | console={ 16 | "class": "logging.StreamHandler", 17 | "formatter": "bare", 18 | "level": "INFO", 19 | "stream": "ext://sys.stdout", 20 | } 21 | ), 22 | loggers=dict( 23 | fedimg={ 24 | "level": "DEBUG", 25 | "propagate": False, 26 | "handlers": ["console"], 27 | }, 28 | fedmsg={ 29 | "level": "INFO", 30 | "propagate": False, 31 | "handlers": ["console"], 32 | }, 33 | moksha={ 34 | "level": "INFO", 35 | "propagate": False, 36 | "handlers": ["console"], 37 | }, 38 | ), 39 | ), 40 | ) 41 | -------------------------------------------------------------------------------- /fedmsg.d/fedimg.py: -------------------------------------------------------------------------------- 1 | # This file is part of fedimg. 2 | # Copyright (C) 2014-2017 Red Hat, Inc. 3 | # 4 | # fedimg is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as 6 | # published by the Free Software Foundation, either version 3 of the 7 | # License, or (at your option) any later version. 8 | # 9 | # fedimg is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | # Affero General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU Affero General Public 15 | # License along with fedimg; if not, see http://www.gnu.org/licenses, 16 | # or write to the Free Software Foundation, Inc., 17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 18 | # 19 | # Authors: David Gay 20 | # 21 | 22 | import socket 23 | hostname = socket.gethostname() 24 | 25 | NUM_BASE_THREADS = 4 26 | NUM_ATOMIC_THREADS = 2 27 | NUM_PORTS = 2 * ((NUM_BASE_THREADS + NUM_ATOMIC_THREADS) + 1) 28 | 29 | config = { 30 | 'fedimgconsumer.dev.enabled': True, 31 | 'fedimgconsumer.prod.enabled': False, 32 | 'fedimgconsumer.stg.enabled': False, 33 | 'endpoints': { 34 | "fedimg.%s" % hostname: [ 35 | "tcp://127.0.0.1:60%0.2i" % (i) 36 | for i in range(NUM_PORTS) 37 | ], 38 | }, 39 | } 40 | -------------------------------------------------------------------------------- /docs/development/contribution.rst: -------------------------------------------------------------------------------- 1 | Contribution Guidelines 2 | ======================= 3 | 4 | Fedimg welcomes contributors of all levels. Still, it can be a tricky project 5 | to jump into because of its relative youth and the challenge of testing a 6 | project that utilizes 3rd party services like AWS. This document hopes to 7 | alleviate confusion and encourage potential contributors. 8 | 9 | Communicating with other contributors 10 | ------------------------------------- 11 | 12 | Communication with other contributors is crucial. The best way to discuss 13 | Fedimg development is by joining ``#fedora-cloud`` or ``#fedora-apps`` on the 14 | Freenode IRC network. For questions about the Fedora Infrastructure machines 15 | on which Fedimg runs, or about related fedmsg, your best bet is to say 16 | something in ``#fedora-apps``. 17 | 18 | You can also send an email to the `infrastructure mailing list`_. 19 | 20 | Submitting changes 21 | ------------------ 22 | 23 | Fedimg uses `Git`_ and is hosted on `GitHub`. If you've managed to improve 24 | Fedimg in some way, make sure to open a `pull requests`_ at 25 | `github.com/fedora-infra/fedimg`_, or you may email a patch to the 26 | `infrastructure mailing list`_. 27 | 28 | .. _infrastructure mailing list: https://lists.fedoraproject.org/admin/lists/infrastructure.lists.fedoraproject.org/ 29 | .. _git: https://git-scm.com/ 30 | .. _pull requests: https://help.github.com/articles/creating-a-pull-request/ 31 | .. _github.com/fedora-infra/fedimg: https://github.com/fedora-infra/fedimg 32 | -------------------------------------------------------------------------------- /fedimg/services/ec2/ec2base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This file is part of fedimg. 3 | # Copyright (C) 2014-2018 Red Hat, Inc. 4 | # 5 | # fedimg is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU Affero General Public License as 7 | # published by the Free Software Foundation, either version 3 of the 8 | # License, or (at your option) any later version. 9 | # 10 | # fedimg is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | # Affero General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU Affero General Public 16 | # License along with fedimg; if not, see http://www.gnu.org/licenses, 17 | # or write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 19 | # 20 | # Authors: Sayan Chowdhury 21 | # 22 | 23 | from libcloud.compute.providers import get_driver 24 | from libcloud.compute.types import Provider 25 | 26 | 27 | class EC2Base(object): 28 | """ Comment goes here """ 29 | 30 | def _connect(self): 31 | cls = get_driver(Provider.EC2) 32 | driver = cls( 33 | self.access_key, 34 | self.secret_key, 35 | region=self.region 36 | ) 37 | 38 | return driver 39 | 40 | def set_region(self, region): 41 | """ Comment goes here """ 42 | self.region = region 43 | -------------------------------------------------------------------------------- /docs/development/installation.rst: -------------------------------------------------------------------------------- 1 | ================= 2 | Development Guide 3 | ================= 4 | 5 | A vagrant virtual machine is avalaible to make running the unit test easy. If you wish to use this virual machine as development environment 6 | you can check how to get started in the `testing documentation`_. 7 | 8 | 9 | Fedimg is application written in Python. It uses `libcloud`_ to connect with 10 | the external Cloud providers. 11 | 12 | #. Clone the repository from Github:: 13 | 14 | $ git clone git@github.com:fedora-infra/fedimg.git 15 | 16 | #. Installing the dependencies:: 17 | 18 | $ virtualenv fedimg_env 19 | $ source ./fedimg_env/bin/activate 20 | $ python setup.py develop 21 | 22 | #. Setting up the configuration:: 23 | 24 | $ mkdir -p /etc/fedimg/ 25 | $ cp fedimg-conf.toml.example /etc/fedimg/fedimg-conf.toml 26 | 27 | #. Setup AWS credentials 28 | 29 | Update the values of ``access_id`` and ``secret_key`` with your AWS tokens in 30 | ``/etc/fedimg/fedimg-conf.toml`` file. 31 | 32 | #. Setup the euca2ools configuration:: 33 | 34 | $ mkdir ~/.euca 35 | $ cp euca-fedimg.ini ~/.euca/config.ini 36 | 37 | #. Setup the euc2ools credentials 38 | 39 | Update the values of ``key-id`` and ``secret-key`` with your AWS tokens in the 40 | ``~/.euca/config.ini`` file. 41 | 42 | #. Run the fedmsg-hub:: 43 | 44 | $ fedmsg-hub 45 | 46 | Happy Hacking! 47 | 48 | .. _libcloud: https://libcloud.apache.org/ 49 | .. _testing documentation: https://github.com/fedora-infra/fedimg/blob/develop/docs/development/testing.rst -------------------------------------------------------------------------------- /docs/development/testing.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Testing 3 | ======= 4 | 5 | Fedimg provides a growing suite of unittests. The tests uses `pytest`_ Python 6 | package. 7 | 8 | Running the tests 9 | ----------------- 10 | 11 | Virtualenv 12 | ++++++++++ 13 | 14 | Refer to the documentation to setup your virtualenv. 15 | 16 | #. Activate the virtualenv:: 17 | 18 | $ cd /path/to/fedimg/git/source 19 | $ source ./fedimg_env/bin/activate 20 | 21 | #. Run the tests:: 22 | 23 | $ python setup.py test 24 | 25 | .. _pytest: https://pytest.org/ 26 | 27 | 28 | Vagrant box 29 | +++++++++++ 30 | 31 | Vagrant allows contributors to get quickly up and running with a fedimg development environment by automatically configuring a virtual machine. 32 | To get started, simply use these commands:: 33 | 34 | $ sudo dnf install ansible libvirt vagrant-libvirt vagrant-sshfs 35 | $ sudo systemctl enable libvirtd 36 | $ sudo systemctl start libvirtd 37 | $ git clone git@github.com:fedora-infra/fedimg.git 38 | $ cd fedimg 39 | $ cp devel/Vagrantfile.example Vagrantfile 40 | $ vagrant up 41 | 42 | You can ssh into your running Vagrant box like this:: 43 | 44 | # Make sure your fedimg checkout is your shell's cwd 45 | $ vagrant ssh 46 | 47 | The code from your development host will be mounted in ``/home/vagrant/fedimg`` 48 | in the guest. You can edit this code on the host, and the vagrant-sshfs plugin will cause the 49 | changes to automatically be reflected in the guest's ``/home/vagrant/fedimg`` folder. 50 | 51 | Run the tests:: 52 | 53 | $ vagrant ssh 54 | $ cd fedimg 55 | $ tox -------------------------------------------------------------------------------- /bin/trigger_upload.py: -------------------------------------------------------------------------------- 1 | #!/bin/env python 2 | # -*- coding: utf8 -*- 3 | """ Triggers an upload process with the specified raw.xz URL. """ 4 | 5 | import argparse 6 | import logging 7 | import logging.config 8 | import multiprocessing.pool 9 | 10 | import fedmsg.config 11 | import fedimg.uploader 12 | 13 | logging.config.dictConfig(fedmsg.config.load_config()['logging']) 14 | log = logging.getLogger('fedmsg') 15 | 16 | 17 | def trigger_upload(url, compose_id, push_notifications): 18 | upload_pool = multiprocessing.pool.ThreadPool(processes=4) 19 | fedimg.uploader.upload(upload_pool, [url], 20 | compose_id=compose_id, 21 | push_notifications=push_notifications) 22 | 23 | 24 | def get_args(): 25 | parser = argparse.ArgumentParser( 26 | description="Trigger a manual upload process with the " 27 | "specified raw.xz URL") 28 | parser.add_argument( 29 | "-u", "--url", type=str, help=".raw.xz URL", required=True) 30 | parser.add_argument( 31 | "-c", "--compose-id", type=str, help="compose id of the .raw.xz file", 32 | required=True) 33 | parser.add_argument( 34 | "-p", "--push-notifications", 35 | help="Bool to check if we need to push fedmsg notifications", 36 | action="store_true", required=False) 37 | args = parser.parse_args() 38 | 39 | return args.url, args.compose_id, args.push_notifications 40 | 41 | 42 | def main(): 43 | url, compose_id, push_notifications = get_args() 44 | trigger_upload(url, compose_id, push_notifications) 45 | 46 | if __name__ == '__main__': 47 | main() 48 | -------------------------------------------------------------------------------- /fedimg/messenger.py: -------------------------------------------------------------------------------- 1 | # This file is part of fedimg. 2 | # Copyright (C) 2014 Red Hat, Inc. 3 | # 4 | # fedimg is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as 6 | # published by the Free Software Foundation, either version 3 of the 7 | # License, or (at your option) any later version. 8 | # 9 | # fedimg is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | # Affero General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU Affero General Public 15 | # License along with fedimg; if not, see http://www.gnu.org/licenses, 16 | # or write to the Free Software Foundation, Inc., 17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 18 | # 19 | # Authors: David Gay 20 | # 21 | 22 | import fedmsg 23 | 24 | """ 25 | The latest Fedmsg meta code for Fedimg fedmsgs (what a mouthful!): 26 | https://github.com/fedora-infra/fedmsg_meta_fedora_infrastructure/blob/develop/fedmsg_meta_fedora_infrastructure/fedimg.py 27 | """ 28 | 29 | 30 | def notify(topic, msg): 31 | """ Takes a message topic, image name, an upload destination (ex. 32 | "EC2-eu-west-1"), and a status (ex. "failed"). Can also take an optional 33 | dictionary of addiitonal bits of information, such as an AMI ID for an 34 | image registered to AWS EC2. Emits a fedmsg appropriate 35 | for each image task (an upload or a test). """ 36 | 37 | fedmsg.publish(topic=topic, modname='fedimg', msg=msg) 38 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. Fedimg documentation master file, created by 2 | sphinx-quickstart on Sat May 5 00:26:21 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | ====== 7 | Fedimg 8 | ====== 9 | 10 | Fedimg is a `Python`_-powered service that is built 11 | to make Fedora Cloud and Atomic Host images available on popular cloud 12 | providers, such as `Amazon Web Services`_. 13 | 14 | Salient features: 15 | 16 | * Performs automatic uploads, triggered by activity on the `Fedmsg`_ bus. 17 | * Register images as Hardware Virtual Machine (HVM). 18 | * Emits fedmsg messages, providing the upload status and the Amazon Machine 19 | Image (AMI) info. 20 | * Packaged for EPEL7, Fedora as well as PyPI. 21 | 22 | License 23 | ======= 24 | Fedimg is licensed under the AGPL, version 3 or later. 25 | 26 | Contents: 27 | 28 | .. toctree:: 29 | :maxdepth: 2 30 | 31 | architecture/configuration 32 | architecture/consumer 33 | development/installation 34 | development/contribution 35 | development/testing 36 | 37 | .. toctree:: 38 | :maxdepth: 1 39 | 40 | release_notes 41 | 42 | Community 43 | ========= 44 | 45 | Fedimg is maintained by the Fedora Project and it's `source code`_ and `issue 46 | tracker`_ are on Github. Join the `mailing list`_ or/and the IRC 47 | channel on `FreeNode`_, ``#fedora-cloud`` for discussion on Fedimg. 48 | 49 | .. _Python: https://www.python.org/ 50 | .. _Amazon Web Services: https://aws.amazon.com/ 51 | .. _source code: https://github.com/fedora-infra/fedimg 52 | .. _mailing list: https://lists.fedoraproject.org/admin/lists/infrastructure.lists.fedoraproject.org/ 53 | .. _FreeNode: https://freenode.net/ 54 | .. _fedmsg: http://www.fedmsg.com 55 | .. _issue tracker: https://github.com/fedora-infra/fedimg/issues/ 56 | 57 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Fedimg 2 | 3 | Service to automatically upload built Fedora images to internal and external 4 | cloud providers. 5 | 6 | ## License 7 | 8 | Fedimg is licensed under AGPLv3 or newer. See the `LICENSE` file for more 9 | information. 10 | 11 | ## Documentation 12 | 13 | Official Fedimg documentation can be [found at 14 | RTFD](https://fedimg.readthedocs.org) or in `docs/`. 15 | 16 | If you are interested in contributing to fedimg, you can [read the developer documentation](https://github.com/fedora-infra/fedimg/tree/develop/docs/development). 17 | 18 | ## Triggering jobs 19 | 20 | Fedimg is designed to perform automatically when it sees the appropriate 21 | fedmsg. However, sometimes, it's useful to be able to quickly trigger 22 | a job manually. If Fedimg is installed properly, the `bin/trigger_upload.py` 23 | script can be used for this purpose: 24 | 25 | ``` 26 | usage: trigger_upload.py [-h] -u URL -c COMPOSE_ID [-p] 27 | 28 | Trigger a manual upload process with the specified raw.xz URL 29 | 30 | optional arguments: 31 | -h, --help show this help message and exit 32 | -u URL, --url URL .raw.xz URL 33 | -c COMPOSE_ID, --compose-id COMPOSE_ID 34 | compose id of the .raw.xz file 35 | -p, --push-notifications 36 | Bool to check if we need to push fedmsg notifications 37 | ``` 38 | 39 | This script simply skips the part where Fedimg listens for the fedmsg, and 40 | allows you to directly provide a URL to a raw.xz image file that you'd like 41 | uploaded. Otherwise, Fedimg performs the same as during automatic operation. 42 | 43 | ## Providers 44 | 45 | We hope to simultaneously upload our cloud images to a variety of internal and 46 | external spaces. Currently, the code supports Amazon EC2. Work has begun 47 | toward supporting Rackspace, GCE, and HP. We're currently waiting on some 48 | legal developments to determine what sort of account and access we'll have 49 | to these providers. 50 | 51 | ## Contributors 52 | 53 | * David Gay 54 | * Ralph Bean 55 | * Sayan Chowdhury 56 | -------------------------------------------------------------------------------- /fedimg/services/ec2/ec2copy.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This file is part of fedimg. 3 | # Copyright (C) 2014-2017 Red Hat, Inc. 4 | # 5 | # fedimg is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU Affero General Public License as 7 | # published by the Free Software Foundation, either version 3 of the 8 | # License, or (at your option) any later version. 9 | # 10 | # fedimg is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | # Affero General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU Affero General Public 16 | # License along with fedimg; if not, see http://www.gnu.org/licenses, 17 | # or write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 19 | # 20 | # Authors: Sayan Chowdhury 21 | # 22 | """ 23 | I don't know what to write here for documentation. 24 | """ 25 | import logging 26 | 27 | from fedimg.config import AWS_BASE_REGION 28 | from fedimg.services.ec2.ec2imgpublisher import EC2ImagePublisher 29 | 30 | LOG = logging.getLogger(__name__) 31 | 32 | 33 | def main(regions, access_id, secret_key, images=None, 34 | image_ids=None, push_notifications=False, compose_id=None): 35 | 36 | if images is None and image_ids is None: 37 | LOG.debug('Either images or image_ids is required') 38 | raise 39 | 40 | publisher = EC2ImagePublisher( 41 | compose_id=compose_id, 42 | access_key=access_id, 43 | secret_key=secret_key, 44 | push_notifications=True, 45 | ) 46 | 47 | if image_ids is None and images is not None: 48 | image_ids = [image.id for image in images] 49 | 50 | for image_id in image_ids: 51 | copied_images = publisher.copy_images_to_regions( 52 | image_id=image_id, 53 | regions=regions, 54 | base_region=AWS_BASE_REGION) 55 | copied_images = [elem.values() for elem in copied_images] 56 | publisher.publish_images(region_image_mapping=copied_images) 57 | -------------------------------------------------------------------------------- /bin/partial_upload.py: -------------------------------------------------------------------------------- 1 | #!/bin/env python 2 | # -*- coding: utf8 -*- 3 | """ Triggers a partial upload process with the specified raw.xz URL. """ 4 | 5 | import argparse 6 | 7 | from fedimg.config import AWS_ACCESS_ID 8 | from fedimg.config import AWS_SECRET_KEY 9 | from fedimg.config import AWS_BASE_REGION, AWS_REGIONS 10 | from fedimg.services.ec2.ec2copy import main as ec2copy 11 | from fedimg.services.ec2.ec2initiate import main as ec2main 12 | 13 | 14 | def get_args(): 15 | parser = argparse.ArgumentParser( 16 | description="Trigger a partial upload based on the arguments") 17 | parser.add_argument( 18 | "-u", "--url", type=str, help=".raw.xz URL", required=True) 19 | parser.add_argument( 20 | "-c", "--compose-id", type=str, help="compose id of the .raw.xz file", 21 | required=True) 22 | parser.add_argument( 23 | "-p", "--push-notifications", 24 | help="Bool to check if we need to push fedmsg notifications", 25 | action="store_true", required=False) 26 | parser.add_argument( 27 | "-v", "--volume", help="volume type for the image", required=False) 28 | 29 | args = parser.parse_args() 30 | 31 | return ( 32 | args.url, 33 | args.compose_id, 34 | args.push_notifications, 35 | args.volume 36 | ) 37 | 38 | 39 | def main(): 40 | url, compose_id, push_notifications, volume = get_args() 41 | 42 | if volume is not None: 43 | volume = [volume] 44 | 45 | images_metadata = ec2main( 46 | image_urls=url, 47 | access_id=AWS_ACCESS_ID, 48 | secret_key=AWS_SECRET_KEY, 49 | regions=None, 50 | volume_types=volume, 51 | push_notifications=push_notifications, 52 | compose_id=compose_id 53 | ) 54 | 55 | for image_metadata in images_metadata: 56 | image_id = image_metadata['image_id'] 57 | aws_regions = list(set(AWS_REGIONS) - set([AWS_BASE_REGION])) 58 | ec2copy( 59 | aws_regions, 60 | AWS_ACCESS_ID, 61 | AWS_SECRET_KEY, 62 | image_ids=[image_id], 63 | push_notifications=push_notifications, 64 | compose_id=compose_id 65 | ) 66 | 67 | 68 | if __name__ == '__main__': 69 | main() 70 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # This file is part of fedimg. 2 | # Copyright (C) 2014-2018 Red Hat, Inc. 3 | # 4 | # fedimg is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as 6 | # published by the Free Software Foundation, either version 3 of the 7 | # License, or (at your option) any later version. 8 | # 9 | # fedimg is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | # Affero General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU Affero General Public 15 | # License along with fedimg; if not, see http://www.gnu.org/licenses, 16 | # or write to the Free Software Foundation, Inc., 17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 18 | # 19 | # Authors: David Gay 20 | # Sayan Chowdhury 21 | # 22 | 23 | from setuptools import setup, find_packages 24 | 25 | setup( 26 | name='fedimg', 27 | version='2.4.0', 28 | description='Automatically upload Fedora Cloud images to cloud providers', 29 | classifiers=[ 30 | "Programming Language :: Python :: 2.7", 31 | "License :: OSI Approved :: GNU Affero General Public License" 32 | " v3 or later (AGPLv3+)", 33 | ], 34 | keywords='python Fedora cloud image uploader service', 35 | author='Fedora Infrastructure Team', 36 | author_email='infrastructure@lists.fedoraproject.org', 37 | url='https://github.com/fedora-infra/fedimg', 38 | license='AGPLv3+', 39 | include_package_data=True, 40 | zip_safe=False, 41 | setup_requires=[ 42 | 'pytest-runner', 43 | ], 44 | install_requires=[ 45 | 'fedmsg', 46 | 'fedfind', 47 | 'apache-libcloud', 48 | 'paramiko', 49 | 'toml' 50 | ], 51 | tests_require=[ 52 | 'flake8', 53 | 'pytest', 54 | 'pytest-cov', 55 | 'mock', 56 | 'vcrpy' 57 | ], 58 | packages=find_packages(), 59 | entry_points=""" 60 | [moksha.consumer] 61 | fedimgconsumer.prod = fedimg.consumers:FedimgConsumer 62 | fedimgconsumer.stg = fedimg.consumers:FedimgStagingConsumer 63 | fedimgconsumer.dev = fedimg.consumers:FedimgDevConsumer 64 | """, 65 | ) 66 | -------------------------------------------------------------------------------- /docs/architecture/configuration.rst: -------------------------------------------------------------------------------- 1 | Configuration 2 | ============= 3 | 4 | Fedimg pulls its configuration variables from the file located at 5 | `/etc/fedimg/fedimg-conf.toml`. A example file is included in the source of 6 | fedimg, and can be used as a starting point for writing your own configuration 7 | file. 8 | 9 | --------------- 10 | General options 11 | --------------- 12 | 13 | .. _conf-clean_up_on_failure: 14 | 15 | clean_up_on_failure 16 | ------------------- 17 | A boolean that, if ``True``, will destroy all the resources that was created, 18 | if there is an exception during the upload process. 19 | 20 | .. _conf-delete_image_on_failure: 21 | 22 | delete_image_on_failure 23 | ----------------------- 24 | 25 | A boolean that, if ``False``, will skip the the destruction of the 26 | uploaded image if there is an exception in the upload process. 27 | 28 | .. _conf-process_count: 29 | 30 | process_count 31 | ------------- 32 | 33 | An integer to specify the number of `worker process`_ that would be running. 34 | 35 | .. _conf-active_services: 36 | 37 | active_services 38 | --------------- 39 | 40 | A list of active third party services where Fedimg would upload the cloud 41 | images. Valid values within the list: `aws`. 42 | 43 | ----------- 44 | AWS options 45 | ----------- 46 | 47 | .. _conf-access_id: 48 | 49 | access_id 50 | --------- 51 | 52 | Access ID of the AWS account that will be used. 53 | 54 | .. _conf-secret-key: 55 | 56 | secret_key 57 | ---------- 58 | 59 | Secret Key of the AWS account that will be used. 60 | 61 | .. _conf-base-region: 62 | 63 | base_region 64 | ----------- 65 | 66 | The base region among the list of AWS regions where the upload process of the 67 | cloud images will be done. 68 | 69 | .. _conf-bucket-name: 70 | 71 | bucket_name 72 | ----------- 73 | 74 | A string that contains the name of the S3 bucket used in AWS. Fedimg does not 75 | create the S3 bucket so make sure that the bucket already exists before using. 76 | 77 | .. _conf-regions: 78 | 79 | regions 80 | ------- 81 | 82 | A list containing the list of regions where the AMIs would be copied from the 83 | ``base_region``. This regions should not contain the ``base_region`` to avoid 84 | duplicate images. 85 | 86 | .. _conf-volume-size: 87 | 88 | volume_size 89 | ----------- 90 | 91 | An integer, specifying the size of the volume to be created in GiB. 92 | 93 | .. _worker process: https://docs.python.org/2/library/multiprocessing.html#using-a-pool-of-workers 94 | -------------------------------------------------------------------------------- /fedimg/config.py: -------------------------------------------------------------------------------- 1 | # This file is part of fedimg. 2 | # Copyright (C) 2014-2018 Red Hat, Inc. 3 | # 4 | # fedimg is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as 6 | # published by the Free Software Foundation, either version 3 of the 7 | # License, or (at your option) any later version. 8 | # 9 | # fedimg is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | # Affero General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU Affero General Public 15 | # License along with fedimg; if not, see http://www.gnu.org/licenses, 16 | # or write to the Free Software Foundation, Inc., 17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 18 | # 19 | # Authors: David Gay 20 | # Sayan Chowdhury 21 | 22 | import os.path 23 | import toml 24 | 25 | config_file = '/etc/fedimg/fedimg-conf.toml' 26 | if not os.path.isfile(config_file): 27 | # Get the path of the configuration file 28 | project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 29 | config_file = os.path.join(project_root, 'fedimg-conf.toml.example') 30 | 31 | # Read in config file 32 | config = {} 33 | with open(config_file) as conffile: 34 | config = toml.loads(conffile.read()) 35 | 36 | # Fedimg Consumer configurations 37 | PROCESS_COUNT = config.get('general', {}).get('process_count', {}) 38 | STATUS_FILTER = ('FINISHED_INCOMPLETE', 'FINISHED') 39 | 40 | ACTIVE_SERVICES = config['general']['active_services'] 41 | CLEAN_UP_ON_FAILURE = config['general']['clean_up_on_failure'] 42 | DELETE_IMAGES_ON_FAILURE = config['general']['delete_images_on_failure'] 43 | 44 | AWS_ACCESS_ID = config.get('aws', {}).get('access_id') 45 | AWS_SECRET_KEY = config.get('aws', {}).get('secret_key') 46 | AWS_VOLUME_SIZE = config.get('aws', {}).get('volume_size') 47 | AWS_VOLUME_TYPES = config.get('aws', {}).get('volume_types') 48 | AWS_VOLUME_VIA_S3 = config.get('aws', {}).get('volume_via_s3') 49 | AWS_REGIONS = config.get('aws', {}).get('regions', {}) 50 | AWS_ROOT_VOLUME_SIZE = config.get('aws', {}).get('root_volume_size', {}) 51 | AWS_BASE_REGION = config.get('aws', {}).get('base_region', {}) 52 | AWS_DELETE_RESOURCES = config.get('aws', {}).get('delete_resources', True) 53 | AWS_S3_BUCKET_NAME = config.get('aws', {}).get('bucket_name', 54 | 'fedora-s3-bucket-fedimg') 55 | -------------------------------------------------------------------------------- /fedimg/uploader.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This file is part of fedimg. 3 | # Copyright (C) 2014-2018 Red Hat, Inc. 4 | # 5 | # fedimg is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU Affero General Public License as 7 | # published by the Free Software Foundation, either version 3 of the 8 | # License, or (at your option) any later version. 9 | # 10 | # fedimg is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | # Affero General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU Affero General Public 16 | # License along with fedimg; if not, see http://www.gnu.org/licenses, 17 | # or write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 19 | # 20 | # Authors: David Gay 21 | # Sayan Chowdhury 22 | """ 23 | This module checks for the active services (configured through the fedimg 24 | configuration file) and call the main method for the service. 25 | """ 26 | 27 | import logging 28 | 29 | from fedimg.config import ACTIVE_SERVICES 30 | from fedimg.config import AWS_ACCESS_ID, AWS_SECRET_KEY 31 | from fedimg.config import AWS_BASE_REGION, AWS_REGIONS 32 | from fedimg.services.ec2.ec2copy import main as ec2copy 33 | from fedimg.services.ec2.ec2initiate import main as ec2main 34 | 35 | _log = logging.getLogger(__name__) 36 | 37 | 38 | def upload(pool, urls, *args, **kwargs): 39 | """ 40 | Takes a list (urls) of one or more .raw.xz image files and 41 | sends them off to cloud services for registration. The upload 42 | jobs threadpool must be passed as `pool`. 43 | 44 | Args: 45 | pool (multithreading.pool.ThreadPool): The thread pool to parallelize 46 | the uploads. 47 | urls (list): List of cloud image urls. 48 | """ 49 | 50 | active_services = ACTIVE_SERVICES 51 | compose_id = kwargs.get('compose_id') 52 | push_notifications = kwargs.get('push_notifications') 53 | 54 | if 'aws' in active_services: 55 | _log.info('Starting to process AWS EC2Service.') 56 | images_metadata = ec2main( 57 | urls, 58 | AWS_ACCESS_ID, 59 | AWS_SECRET_KEY, 60 | [AWS_BASE_REGION], 61 | compose_id=compose_id, 62 | push_notifications=push_notifications, 63 | ) 64 | for image_metadata in images_metadata: 65 | image_id = image_metadata['image_id'] 66 | aws_regions = list(set(AWS_REGIONS) - set([AWS_BASE_REGION])) 67 | ec2copy( 68 | aws_regions, 69 | AWS_ACCESS_ID, 70 | AWS_SECRET_KEY, 71 | image_ids=[image_id], 72 | push_notifications=push_notifications, 73 | compose_id=compose_id 74 | ) 75 | _log.info('AWS EC2Service process is completed.') 76 | -------------------------------------------------------------------------------- /devel/Vagrantfile.example: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | # On your host: 5 | # git clone https://github.com/fedora-infra/fedimg.git 6 | # cd fedimg 7 | # cp devel/Vagrantfile.example Vagrantfile 8 | # vagrant up 9 | 10 | VAGRANTFILE_API_VERSION = "2" 11 | 12 | Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| 13 | config.vm.box_url = "https://download.fedoraproject.org/pub/fedora/linux/releases/28/Cloud/x86_64/images/Fedora-Cloud-Base-Vagrant-28-1.1.x86_64.vagrant-libvirt.box" 14 | config.vm.box = "f28-cloud-libvirt" 15 | config.vm.box_download_checksum = "d60f52c9cb04bfd4c5e950410611c746641f0b5f76830a53d44a0f1a43ab3fac" 16 | config.vm.box_download_checksum_type = "sha256" 17 | 18 | # This is an optional plugin that, if installed, updates the host's /etc/hosts 19 | # file with the hostname of the guest VM. In Fedora it is packaged as 20 | # ``vagrant-hostmanager`` 21 | if Vagrant.has_plugin?("vagrant-hostmanager") 22 | config.hostmanager.enabled = true 23 | config.hostmanager.manage_host = true 24 | end 25 | 26 | # Vagrant can share the source directory using rsync, NFS, or SSHFS (with the vagrant-sshfs 27 | # plugin). Consult the Vagrant documentation if you do not want to use SSHFS. 28 | config.vm.synced_folder ".", "/vagrant", disabled: true 29 | config.vm.synced_folder ".", "/home/vagrant/fedimg", type: "sshfs", sshfs_opts_append: "-o nonempty" 30 | 31 | # To cache update packages (which is helpful if frequently doing `vagrant destroy && vagrant up`) 32 | # you can create a local directory and share it to the guest's DNF cache. Uncomment the lines below 33 | # to create and use a dnf cache directory 34 | # 35 | # Dir.mkdir('.dnf-cache') unless File.exists?('.dnf-cache') 36 | # config.vm.synced_folder ".dnf-cache", "/var/cache/dnf", type: "sshfs", sshfs_opts_append: "-o nonempty" 37 | 38 | # Comment this line if you would like to disable the automatic update during provisioning 39 | config.vm.provision "shell", inline: "sudo dnf upgrade -y" 40 | 41 | # bootstrap and run with ansible 42 | config.vm.provision "shell", inline: "sudo dnf -y install python2-dnf libselinux-python python3-tox" 43 | config.vm.provision "shell", inline: "sudo dnf -y groupinstall 'C Development Tools and Libraries'" 44 | config.vm.provision "shell", inline: "sudo mkdir -p /etc/fedimg && sudo cp /home/vagrant/fedimg/fedimg-conf.toml.example /etc/fedimg/fedimg-conf.toml" 45 | 46 | 47 | 48 | # Create the fedimg dev box 49 | config.vm.define "fedimg" do |fedimg| 50 | fedimg.vm.host_name = "fedimg-dev.example.com" 51 | 52 | fedimg.vm.provider :libvirt do |domain| 53 | # Season to taste 54 | domain.cpus = 2 55 | domain.graphics_type = "spice" 56 | domain.memory = 1024 57 | domain.video_type = "qxl" 58 | 59 | # Uncomment the following line if you would like to enable libvirt's unsafe cache 60 | # mode. It is called unsafe for a reason, as it causes the virtual host to ignore all 61 | # fsync() calls from the guest. Only do this if you are comfortable with the possibility of 62 | # your development guest becoming corrupted (in which case you should only need to do a 63 | # vagrant destroy and vagrant up to get a new one). 64 | # 65 | # domain.volume_cache = "unsafe" 66 | end 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /docs/release_notes.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | Release Notes 3 | ============= 4 | 5 | 2.4.0 6 | ===== 7 | 8 | Developer Improvements 9 | ---------------------- 10 | 11 | * Initial patchset for AARCH64 uploading to EC2 12 | (`#156 `_). 13 | 14 | 2.3.0 15 | ===== 16 | 17 | Developer Improvements 18 | ---------------------- 19 | 20 | * tests: Fix the tests for the broken links 21 | (`#151 `_). 22 | * utils: Fix the tests for the travis test failures 23 | (`#152 `_). 24 | * setup: Update the AUTHORS & the AUTHORS EMAIL in setup.py 25 | (`#153 `_). 26 | 27 | Bug fixes 28 | --------- 29 | 30 | * config: Fix the config to pick local if needed 31 | (`#150 `_). 32 | 33 | Contributors 34 | ------------ 35 | 36 | The following developers contributed patches to Fedimg 2.2.0: 37 | 38 | - Sayan Chowdhury 39 | 40 | 2.2.0 41 | ===== 42 | 43 | Developer Improvements 44 | ---------------------- 45 | 46 | * utils: Cancel the conversion tasks if the are older then 24 hours 47 | (`#146 `_). 48 | 49 | Contributors 50 | ------------ 51 | 52 | The following developers contributed patches to Fedimg 2.2.0: 53 | 54 | - Sayan Chowdhury 55 | 56 | 2.1.0 57 | ===== 58 | 59 | Bug fixes 60 | --------- 61 | 62 | * Fix the version and release number in sphinx conf 63 | (`#133 `_). 64 | 65 | Developer Improvements 66 | ---------------------- 67 | 68 | * Support both type of resource ids (long & short) 69 | (`#141 `_). 70 | * Migrate fedimg to support to longer resource ids 71 | (`#137 `_). 72 | 73 | Contributors 74 | ------------ 75 | 76 | The following developers contributed patches to Fedimg 2.1.0: 77 | 78 | - Sayan Chowdhury 79 | 80 | 2.0.0 81 | ===== 82 | 83 | Deprecation 84 | ----------- 85 | 86 | * The fedmsg message ``fedimg.image.upload`` has been deprecated for all other 87 | regions other than the ``base_region`` 88 | (`#97 `_). 89 | 90 | 91 | Bug fixes 92 | --------- 93 | 94 | * Refactor the ``FedimgConsumer`` to handle F28 Two week Atomic 95 | (`#123 `_). 96 | 97 | Developer Improvements 98 | ---------------------- 99 | 100 | * Fix the tests for ``utils.py`` 101 | (`#124 `_). 102 | * Fix the FSF address in setup.py messed up during the version bump 103 | (`#127 `_). 104 | * Migrate from nose to pytest 105 | (`#128 `_). 106 | * Fix the filename type for CHANGELOG.rst 107 | (`#129 `_). 108 | * Raise exceptions when shell execution fails 109 | (`#130 `_). 110 | * Refactor the docs to use sphinx, and update the latest architecture 111 | (`#131 `_). 112 | 113 | Contributors 114 | ------------ 115 | 116 | The following developers contributed patches to Fedimg 2.0.0: 117 | 118 | - Sayan Chowdhury 119 | -------------------------------------------------------------------------------- /tests/test_uploader.py: -------------------------------------------------------------------------------- 1 | # This file is part of fedimg. 2 | # Copyright (C) 2014-2018 Red Hat, Inc. 3 | # 4 | # fedimg is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as 6 | # published by the Free Software Foundation, either version 3 of the 7 | # License, or (at your option) any later version. 8 | # 9 | # fedimg is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | # Affero General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU Affero General Public 15 | # License along with fedimg; if not, see http://www.gnu.org/licenses, 16 | # or write to the Free Software Foundation, Inc., 17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 18 | # 19 | # Authors: David Gay 20 | # Sayan Chowdhury 21 | 22 | import mock 23 | import multiprocessing.pool 24 | import unittest 25 | 26 | import fedimg.uploader 27 | 28 | 29 | class TestUploader(unittest.TestCase): 30 | 31 | @mock.patch('fedimg.uploader.ec2main', return_value=[]) 32 | @mock.patch('fedimg.uploader.ec2copy') 33 | @mock.patch('fedimg.uploader.ACTIVE_SERVICES', return_value=['hp']) 34 | def test_inactive_aws(self, active_services, ec2copy, ec2main): 35 | 36 | thread_pool = multiprocessing.pool.ThreadPool(processes=1) 37 | 38 | fedimg.uploader.upload( 39 | thread_pool, 40 | ['http://kojipkgs.fedoraproject.org/compose/Fedora-Cloud-27-20180317.0/compose/CloudImages/x86_64/images/Fedora-Cloud-Base-27-20180317.0.x86_64.raw.xz'], 41 | compose_id='Fedora-Cloud-27-20180317.0', 42 | push_notifications=True, 43 | ) 44 | 45 | self.assertIs(ec2main.called, False) 46 | self.assertIs(ec2copy.called, False) 47 | 48 | @mock.patch('fedimg.uploader.ec2main', return_value=[]) 49 | @mock.patch('fedimg.uploader.ec2copy') 50 | def test_active_aws_no_images(self, ec2copy, ec2main): 51 | thread_pool = multiprocessing.pool.ThreadPool(processes=1) 52 | 53 | fedimg.uploader.upload( 54 | thread_pool, 55 | ['http://kojipkgs.fedoraproject.org/compose/Fedora-Cloud-27-20180317.0/compose/CloudImages/x86_64/images/Fedora-Cloud-Base-27-20180317.0.x86_64.raw.xz'], 56 | compose_id='Fedora-Cloud-27-20180317.0', 57 | push_notifications=True, 58 | ) 59 | 60 | self.assertIs(ec2main.called, True) 61 | self.assertIs(ec2copy.called, False) 62 | 63 | @mock.patch('fedimg.uploader.ec2main') 64 | @mock.patch('fedimg.uploader.ec2copy') 65 | def test_active_aws_with_images(self, ec2copy, ec2main): 66 | thread_pool = multiprocessing.pool.ThreadPool(processes=1) 67 | ec2main.return_value = [{ 68 | 'image_id': 'i-abc1234', 69 | 'is_image_public': True, 70 | 'snapshot_id': 'snap-abc1234', 71 | 'is_snapshot_public': True, 72 | 'regions': 'us-east-1' 73 | }] 74 | 75 | fedimg.uploader.upload( 76 | thread_pool, 77 | ['http://kojipkgs.fedoraproject.org/compose/Fedora-Cloud-27-20180317.0/compose/CloudImages/x86_64/images/Fedora-Cloud-Base-27-20180317.0.x86_64.raw.xz'], 78 | compose_id='Fedora-Cloud-27-20180317.0', 79 | push_notifications=True, 80 | ) 81 | 82 | self.assertIs(ec2main.called, True) 83 | self.assertIs(ec2main.called, True) 84 | -------------------------------------------------------------------------------- /bin/list-the-amis.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | """ List the AMIs uploaded by fedimg in the last N days. 3 | 4 | Deps: $ sudo dnf install python-requests 5 | 6 | Author: Ralph Bean 7 | License: LGPLv2+ 8 | """ 9 | 10 | from __future__ import print_function 11 | 12 | import argparse 13 | import collections 14 | import datetime 15 | import functools 16 | import logging 17 | 18 | import requests 19 | 20 | log = logging.getLogger() 21 | 22 | base_url = 'https://apps.fedoraproject.org/datagrepper/raw' 23 | topic = "org.fedoraproject.prod.fedimg.image.upload" 24 | 25 | 26 | def get_page(page, pages, delta): 27 | """ Retrieve the JSON for a particular page of datagrepper results """ 28 | log.debug("Getting page %i of %s", page, pages) 29 | response = requests.get(base_url, params=dict( 30 | topic=topic, 31 | delta=delta, 32 | page=page, 33 | )) 34 | return response.json() 35 | 36 | 37 | def desirable(msg, args): 38 | if not msg['msg']['status'] == 'completed': 39 | return False 40 | if args.rawhide and 'rawhide' not in msg['msg']['image_name']: 41 | return False 42 | if not args.rawhide and 'rawhide' in msg['msg']['image_name']: 43 | return False 44 | return True 45 | 46 | 47 | def get_messages(args): 48 | """ Generator that yields messages from datagrepper """ 49 | 50 | delta = int(datetime.timedelta(days=args.days).total_seconds()) 51 | 52 | # Get the first page 53 | data = get_page(1, 'unknown', delta) 54 | for message in data['raw_messages']: 55 | if desirable(message, args): 56 | yield message 57 | 58 | more = functools.partial(get_page, pages=data['pages'], delta=delta) 59 | 60 | # Get all subsequent pages (if there are any...) 61 | for page in range(1, data['pages']): 62 | data = more(page + 1) 63 | 64 | for message in data['raw_messages']: 65 | if desirable(message, args): 66 | yield message 67 | 68 | 69 | def parse_args(): 70 | parser = argparse.ArgumentParser() 71 | parser.add_argument( 72 | '-d', '--days', default=3, type=int, 73 | help="Number of days of history to search. Default: 3") 74 | parser.add_argument( 75 | '-r', '--rawhide', action='store_true', 76 | help="Show rawhide instead of the branched (pre-)release") 77 | parser.add_argument( 78 | '-v', '--verbose', action='store_true', 79 | help="Produce lots of output") 80 | 81 | return parser.parse_args() 82 | 83 | 84 | if __name__ == '__main__': 85 | # 1 - Initialize 86 | args = parse_args() 87 | if args.verbose: 88 | logging.basicConfig(level=logging.DEBUG) 89 | else: 90 | logging.basicConfig(level=logging.WARNING) 91 | 92 | # 2 - Build a results dict 93 | results = collections.OrderedDict() 94 | messages = get_messages(args) 95 | for message in messages: 96 | key = message['msg']['image_name'] 97 | if not key in results: 98 | results[key] = [] 99 | results[key].append(message['msg']) 100 | 101 | # 3 - Print it out and format it 102 | for key, uploads in results.items(): 103 | for upload in uploads: 104 | extra = upload['extra'] 105 | print( 106 | key.ljust(45), 107 | upload.get('destination', '').ljust(20), 108 | extra.get('id', '').ljust(15), 109 | extra.get('virt_type', '').ljust(13), 110 | extra.get('vol_type', '').ljust(15), 111 | ) 112 | -------------------------------------------------------------------------------- /docs/services/ec2.md: -------------------------------------------------------------------------------- 1 | Fedimg is hooked into Amazon Web Services and registers images as public AMIs in all 2 | EC2 regions. This page explains the process, which takes place in 3 | `fedimg/services/ec2.py`. 4 | 5 | ## AMI types 6 | 7 | At the time of this writing, it seems that we'll eventually want to provide 8 | potentially six different AMIs for each Fedora Cloud image processed. 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 |
VirtualizationStorageResult
ParavirtualInstance-Store1 bundle *
ParavirtualEBS1 snapshot, 2 volume types
ParavirtualEBS (encrypted)1 snapshot, 2 volume types *
HVMInstance-Store1 bundle *
HVMEBS1 snapshot, 2 volume types
HVMEBS (encrypted)1 snapshot, 2 volume types *
19 | `*`: Not yet added to Fedimg 20 | 21 | *The "2 volume types" mentioned in this table are Standard and GP2. GP2 is a new 22 | SSD-based type Amazon is encouraging. It has different pricing and performance 23 | characteristics.* 24 | 25 | ## Terminology 26 | 27 | An **instance** is a virtual machine that exists on EC2. They are started from 28 | an AMI. To put it simply, an instance is a computer. 29 | 30 | An **AMI**, or **Amazon Machine Image**, is essentially a picture of a 31 | potential instance. The AMI defines everything about how the new instance will 32 | be, including the operating system and all files that the system will be born 33 | with. It is what is used to start a new instance on EC2. Fedimg registers 34 | Fedora images as public AMIs so they are freely available in all regions for 35 | testing and use. 36 | 37 | The **utility instance** is the instance which downloads the raw image file and 38 | writes it to a volume for snapshotting. It is the first instance that is 39 | launched during the EC2 service. 40 | 41 | The **test instance** is launched from an AMI that Fedimg has registered. 42 | Fedimg can be easily configured to perform tests on this instance to 43 | determine the health of the new AMI. It is the second instance that is created 44 | during the EC2 service. 45 | 46 | ## Process 47 | 48 | When the uploader calls on the EC2 service, the following happens: 49 | 50 | 1. The AWS AMI list in `/etc/fedimg.cfg` is read in. 51 | 52 | 2. A utility instance is deployed using the properties from the first item 53 | in the AMI list. 54 | 55 | 3. The utility instance uses `curl` to pull down the `.raw.xz` image file 56 | and writes it to a blank volume. This volume is then snapshotted and 57 | subsequently destroyed. 58 | 59 | 4. The volume snapshot is used to register the image as an AMI. Images 60 | are registered with both standard and GP2 volume types, as well as 61 | with both paravirtual and HVM virtualization. 62 | 63 | 5. The utility instance is shut down, and a test instance is started, 64 | using the AMI that was just registered. 65 | 66 | 6. The test script (configured in `/etc/fedimg.cfg`) is executed on the 67 | test node. If it exits with status code 0, the tests are considered 68 | to have passed. (In the future, [Tunir](http://tunir.readthedocs.org/en/latest/) 69 | will be used for testing.) 70 | 71 | 7. If the tests passed, the AMI is copied to all other EC2 regions. 72 | 73 | 8. All AMIs are made public. 74 | 75 | Fedmsgs are emitted throughout this process, notifying when an image upload 76 | or test is started, completed, or fails. 77 | 78 | ## Getting AMI info 79 | 80 | The EC2 service produces publicly-available AMIs in a variety of flavors. 81 | You can get the IDs and other information about these AMIs in a few different 82 | ways: 83 | 84 | 1. The preferred way at the moment is to just check Datagrepper. You can 85 | see the results of the latest image uploads by visiting a URL like 86 | [this](https://apps.fedoraproject.org/datagrepper/raw/?topic=org.fedoraproject.prod.fedimg.image.upload). 87 | Just click "Details" for any of the completed upload jobs, and you can 88 | see the AMI ID, as well as other info. 89 | 90 | 2. AMI info is displayed on the [releng dashboard](https://apps.fedoraproject.org/releng-dash/), 91 | though it's not quite complete yet. It only displays the very latest 92 | upload jobs, and only a few of them at a time. The releng dash is currently 93 | undergoing a rewrite, and this option for getting AMI info will be much 94 | more useful in the future. 95 | 96 | 3. If you have access to the machine that Fedimg is running on, or if you've 97 | triggered a manual upload job, Fedimg outputs AMI info to stdout as well 98 | as in the logs, which can be accessed with `journalctl`. Fedimg logging 99 | goes through fedmsg-hub, so you could check these logs with a command 100 | like `journalctl -u fedmsg-hub`. 101 | 102 | ## Custom exceptions 103 | 104 | The EC2 service defines three custom exceptions that can be raised by the service. 105 | 106 | - `EC2ServiceException`, a generic exception (inherits from `Exception`). 107 | - `EC2UtilityException`, which can arise when there is a problem with the 108 | utility instance (inherits from `EC2ServiceException`). 109 | - `EC2TestException`, which can arise when there is an issue with the test 110 | instance (inherits from `EC2ServiceException`). 111 | -------------------------------------------------------------------------------- /fedimg/consumers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This file is part of fedimg. 3 | # Copyright (C) 2014-2018 Red Hat, Inc. 4 | # 5 | # fedimg is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU Affero General Public License as 7 | # published by the Free Software Foundation, either version 3 of the 8 | # License, or (at your option) any later version. 9 | # 10 | # fedimg is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | # Affero General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU Affero General Public 16 | # License along with fedimg; if not, see http://www.gnu.org/licenses, 17 | # or write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 19 | # 20 | # Authors: David Gay 21 | # Sayan Chowdhury 22 | """ 23 | This is the `fedmsg consumer`_ that subscribes to the topic emitted after the 24 | completion of the nightly and production compose. The consumer on receving the 25 | message uploads the image using the API of the cloud providers. 26 | """ 27 | 28 | import logging 29 | import multiprocessing.pool 30 | 31 | import fedfind.exceptions 32 | import fedfind.release 33 | 34 | import fedmsg.consumers 35 | import fedmsg.encoding 36 | 37 | import fedimg.uploader 38 | from fedimg.config import PROCESS_COUNT, STATUS_FILTER 39 | from fedimg.utils import get_rawxz_urls, get_value_from_dict 40 | from fedimg.utils import cancel_stale_conversion_tasks 41 | 42 | _log = logging.getLogger(__name__) 43 | 44 | 45 | class FedimgConsumer(fedmsg.consumers.FedmsgConsumer): 46 | """ 47 | A `fedmsg consumer`_ that listens to the pungi compose topics and kicks 48 | of the process to upload the images to various cloud providers. 49 | 50 | Attributes: 51 | topic (str): The topics this consumer is subscribed to. Set to 52 | ``org.fedoraproject.prod.pungi.compose.status.change``. 53 | config_key (str): The key to set to ``True`` in the fedmsg config to 54 | enable this consumer. The key is ``fedimgconsumer.prod.enabled``. 55 | """ 56 | topic = ['org.fedoraproject.prod.pungi.compose.status.change'] 57 | config_key = "fedimgconsumer.prod.enabled" 58 | 59 | def __init__(self, *args, **kwargs): 60 | _log.info("FedimgConsumer initializing") 61 | super(FedimgConsumer, self).__init__(*args, **kwargs) 62 | 63 | # Threadpool for upload jobs 64 | _log.info("Creating thread pool of %s process", PROCESS_COUNT) 65 | self.upload_pool = multiprocessing.pool.ThreadPool( 66 | processes=PROCESS_COUNT 67 | ) 68 | _log.info("FedimgConsumer initialized") 69 | 70 | def consume(self, msg): 71 | """ 72 | This is called when we receive a message matching our topics. 73 | 74 | Args: 75 | msg (dict): The raw message from fedmsg. 76 | """ 77 | _log.info('Received %r %r', msg['topic'], msg['body']['msg_id']) 78 | 79 | msg_info = msg['body']['msg'] 80 | if msg_info['status'] not in STATUS_FILTER: 81 | _log.debug('%s is not valid status' % msg_info['status']) 82 | return 83 | 84 | cancel_stale_conversion_tasks() 85 | 86 | self.process_compose(msg_info['location'], msg_info['compose_id']) 87 | 88 | def process_compose(self, location, compose_id): 89 | try: 90 | compose_metadata = fedfind.release.get_release( 91 | cid=compose_id).metadata 92 | except fedfind.exceptions.UnsupportedComposeError: 93 | _log.debug("%r is unsupported compose" % compose_id) 94 | return 95 | 96 | images_meta = [] 97 | for arch in ['x86_64', 'aarch64']: 98 | cloud_meta = get_value_from_dict( 99 | compose_metadata, 'images', 'payload', 'images', 100 | 'Cloud', arch) 101 | atomic_meta = get_value_from_dict( 102 | compose_metadata, 'images', 'payload', 103 | 'images', 'AtomicHost', arch) 104 | 105 | if cloud_meta: 106 | images_meta.extend(cloud_meta) 107 | if atomic_meta: 108 | images_meta.extend(atomic_meta) 109 | 110 | if not images_meta: 111 | _log.debug('No compatible image found to process') 112 | return 113 | 114 | upload_urls = get_rawxz_urls(location, images_meta) 115 | if len(upload_urls) > 0: 116 | _log.info("Start processing compose id: %s", compose_id) 117 | fedimg.uploader.upload( 118 | pool=self.upload_pool, 119 | urls=upload_urls, 120 | compose_id=compose_id, 121 | push_notifications=True 122 | ) 123 | 124 | 125 | class FedimgStagingConsumer(FedimgConsumer): 126 | """ 127 | A `fedmsg consumer`_ that listens to the staging pungi compose topics and 128 | kicks of the process to upload the images to various cloud providers. 129 | 130 | Attributes: 131 | topic (str): The topics this consumer is subscribed to. Set to 132 | ``org.fedoraproject.stg.pungi.compose.status.change``. 133 | config_key (str): The key to set to ``True`` in the fedmsg config to 134 | enable this consumer. The key is ``fedimgconsumer.stg.enabled``. 135 | """ 136 | topic = ['org.fedoraproject.stg.pungi.compose.status.change'] 137 | config_key = "fedimgconsumer.stg.enabled" 138 | 139 | 140 | class FedimgDevConsumer(FedimgConsumer): 141 | """ 142 | A `fedmsg consumer`_ that listens to the dev pungi compose topics and 143 | kicks of the process to upload the images to various cloud providers. 144 | 145 | Attributes: 146 | topic (str): The topics this consumer is subscribed to. Set to 147 | ``org.fedoraproject.dev.pungi.compose.status.change``. 148 | config_key (str): The key to set to ``True`` in the fedmsg config to 149 | enable this consumer. The key is ``fedimgconsumer.dev.enabled``. 150 | """ 151 | topic = ['org.fedoraproject.dev.pungi.compose.status.change'] 152 | config_key = "fedimgconsumer.dev.enabled" 153 | -------------------------------------------------------------------------------- /tests/test_consumers.py: -------------------------------------------------------------------------------- 1 | # This file is part of fedimg. 2 | # Copyright (C) 2018 Red Hat, Inc. 3 | # 4 | # fedimg is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as 6 | # published by the Free Software Foundation, either version 3 of the 7 | # License, or (at your option) any later version. 8 | # 9 | # fedimg is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | # Affero General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU Affero General Public 15 | # License along with fedimg; if not, see http://www.gnu.org/licenses, 16 | # or write to the Free Software Foundation, Inc., 17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 18 | # 19 | # Authors: David Gay 20 | # Sayan Chowdhury 21 | # 22 | 23 | import os 24 | import mock 25 | import unittest 26 | 27 | import vcr 28 | 29 | import fedimg.consumers 30 | 31 | import utils 32 | 33 | 34 | cassette_dir = os.path.join(os.path.dirname(__file__), 'vcr-request-data') 35 | 36 | 37 | class TestFedimgConsumer(unittest.TestCase): 38 | 39 | def setUp(self): 40 | hub = utils.MockHub() 41 | self.consumer = fedimg.consumers.FedimgConsumer(hub) 42 | 43 | vcr_filename = os.path.join(cassette_dir, self.id()) 44 | self.vcr = vcr.use_cassette(vcr_filename, record_mode='new_episodes') 45 | self.vcr.__enter__() 46 | 47 | def tearDown(self): 48 | self.vcr.__exit__() 49 | 50 | @mock.patch('fedimg.consumers._log.debug') 51 | @mock.patch('fedimg.uploader.upload') 52 | def test_invalid_status(self, mock_upload, mock_log): 53 | msg = { 54 | "topic": "org.fedoraproject.prod.pungi.compose.status.change", 55 | "body": { 56 | "username": "root", 57 | "i": 1, 58 | "timestamp": 1521703054.0, 59 | "msg_id": "2018-6de5ec39-0e28-46aa-a359-99142682fdd9", 60 | "topic": "org.fedoraproject.prod.pungi.compose.status.change", 61 | "msg": { 62 | "status": "STARTED", 63 | "release_type": "ga", 64 | "compose_label": None, 65 | "compose_respin": 0, 66 | "compose_date": "20180321", 67 | "release_version": "28", 68 | "location": "http://kojipkgs.fedoraproject.org/compose/branched/Fedora-28-20180321.n.0/compose", 69 | "compose_type": "nightly", 70 | "release_is_layered": False, 71 | "release_name": "Fedora", 72 | "release_short": "Fedora", 73 | "compose_id": "Fedora-28-20180321.n.0" 74 | } 75 | } 76 | } 77 | self.consumer.consume(msg) 78 | mock_log.assert_called_with('STARTED is not valid status') 79 | 80 | @mock.patch('fedimg.consumers._log.debug') 81 | @mock.patch('fedimg.uploader.upload') 82 | def test_incompatible_images(self, mock_upload, mock_log): 83 | msg = { 84 | "topic": "org.fedoraproject.prod.pungi.compose.status.change", 85 | "body": { 86 | "username": "root", 87 | "i": 1, 88 | "timestamp": 1521648937.0, 89 | "msg_id": "2018-31162ce8-69ec-417e-bc7f-01653f4dcedf", 90 | "topic": "org.fedoraproject.prod.pungi.compose.status.change", 91 | "msg": { 92 | "status": "FINISHED_INCOMPLETE", 93 | "release_type": "ga", 94 | "compose_label": None, 95 | "compose_respin": 0, 96 | "compose_date": "20180426", 97 | "release_version": "Rawhide", 98 | "location": "http://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20180426.n.0/compose", 99 | "compose_type": "nightly", 100 | "release_is_layered": False, 101 | "release_name": "Fedora", 102 | "release_short": "Fedora", 103 | "compose_id": "Fedora-Rawhide-20180426.n.0" 104 | } 105 | } 106 | } 107 | self.consumer.consume(msg) 108 | mock_log.assert_called_with('No compatible image found to process') 109 | 110 | @mock.patch('fedimg.uploader.upload') 111 | def test_success_upload(self, upload): 112 | msg = { 113 | "topic": "org.fedoraproject.prod.pungi.compose.status.change", 114 | "body": { 115 | "username": "root", 116 | "i": 1, 117 | "timestamp": 1521271546.0, 118 | "msg_id": "2018-8c8c2c46-e5f8-4fe3-8f8b-80dd18b21947", 119 | "topic": "org.fedoraproject.prod.pungi.compose.status.change", 120 | "msg": { 121 | "status": "FINISHED", 122 | "release_type": "ga", 123 | "compose_label": "RC-20180425.0", 124 | "compose_respin": 0, 125 | "compose_date": "20180425", 126 | "release_version": "28", 127 | "location": "https://kojipkgs.fedoraproject.org/compose/28/latest-Fedora-28/compose", 128 | "compose_type": "production", 129 | "release_is_layered": False, 130 | "release_name": "Fedora-Cloud", 131 | "release_short": "Fedora-Cloud", 132 | "compose_id": "Fedora-28-20180425.0" 133 | } 134 | } 135 | } 136 | 137 | self.consumer.consume(msg) 138 | url = 'https://kojipkgs.fedoraproject.org/compose/28/latest-Fedora-28/compose/Cloud/x86_64/images/Fedora-Cloud-Base-28-1.1.x86_64.raw.xz' 139 | url1 = 'https://kojipkgs.fedoraproject.org/compose/28/latest-Fedora-28/compose/AtomicHost/x86_64/images/Fedora-AtomicHost-28-1.1.x86_64.raw.xz' 140 | upload.assert_called_with( 141 | pool=mock.ANY, 142 | urls=[url, url1], 143 | compose_id='Fedora-28-20180425.0', 144 | push_notifications=True 145 | ) 146 | -------------------------------------------------------------------------------- /fedimg/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This file is part of fedimg. 3 | # Copyright (C) 2014-2018 Red Hat, Inc. 4 | # 5 | # fedimg is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU Affero General Public License as 7 | # published by the Free Software Foundation, either version 3 of the 8 | # License, or (at your option) any later version. 9 | # 10 | # fedimg is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | # Affero General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU Affero General Public 16 | # License along with fedimg; if not, see http://www.gnu.org/licenses, 17 | # or write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 19 | # 20 | # Authors: David Gay 21 | # Ralph Bean 22 | # Sayan Chowdhury 23 | 24 | """ 25 | Utility functions for fedimg. 26 | """ 27 | 28 | import datetime 29 | import functools 30 | import logging 31 | import os 32 | import re 33 | import socket 34 | import subprocess 35 | import tempfile 36 | 37 | from libcloud.compute.providers import get_driver 38 | from libcloud.compute.types import Provider 39 | 40 | import paramiko 41 | 42 | from fedimg.exceptions import CommandRunFailed, SourceNotFound 43 | from fedimg.exceptions import UnCompressFailed 44 | 45 | _log = logging.getLogger(__name__) 46 | 47 | 48 | def get_file_arch(file_name): 49 | """ Takes a file name (probably of a .raw.xz image file) and returns 50 | the suspected architecture of the contained image. If it doesn't look 51 | like a 32-bit or 64-bit image, None is returned. """ 52 | if file_name.find('x86_64') != -1: 53 | return 'x86_64' 54 | elif file_name.find('aarch64') != -1: 55 | return 'arm64' 56 | else: 57 | return None 58 | 59 | 60 | def get_rawxz_urls(location, images): 61 | """ Iterates through all the images metadata and returns the url of .raw.xz 62 | files. 63 | """ 64 | rawxz_list = [f['path'] for f in images if f['path'].endswith('.raw.xz')] 65 | if not rawxz_list: 66 | return [] 67 | 68 | return map((lambda path: '{}/{}'.format(location, path)), rawxz_list) 69 | 70 | 71 | def get_virt_types_from_url(url): 72 | """ Takes a URL to a .raw.xz image file) and returns the suspected 73 | virtualization type that the image file should be registered as. """ 74 | return ['hvm'] 75 | 76 | 77 | def region_to_driver(region): 78 | """ Takes a region name (ex. 'eu-west-1') and returns 79 | the appropriate libcloud provider value. """ 80 | cls = get_driver(Provider.EC2) 81 | return functools.partial(cls, region=region) 82 | 83 | 84 | def ssh_connection_works(username, ip, keypath): 85 | """ Returns True if an SSH connection can me made to `username`@`ip`. """ 86 | ssh = paramiko.SSHClient() 87 | ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 88 | works = False 89 | try: 90 | ssh.connect(ip, username=username, 91 | key_filename=keypath) 92 | works = True 93 | except (paramiko.BadHostKeyException, 94 | paramiko.AuthenticationException, 95 | paramiko.SSHException, socket.error): 96 | pass 97 | ssh.close() 98 | return works 99 | 100 | 101 | def get_value_from_dict(_dict, *keys): 102 | for key in keys: 103 | try: 104 | _dict = _dict[key] 105 | except KeyError: 106 | return None 107 | return _dict 108 | 109 | 110 | def external_run_command(command): 111 | _log.info("Starting the command: %r" % command) 112 | ret = subprocess.Popen(' '.join(command), stdin=subprocess.PIPE, 113 | shell=True, stdout=subprocess.PIPE, 114 | stderr=subprocess.PIPE, close_fds=True) 115 | out, err = ret.communicate() 116 | retcode = ret.returncode 117 | _log.info("Finished executing the command: %r" % command) 118 | 119 | if retcode != 0: 120 | _log.error("Command failed during run") 121 | _log.error( 122 | "(output) %s, (error) %s, (retcode) %s" % (out, err, retcode)) 123 | else: 124 | _log.debug( 125 | "(output) %s, (error) %s, (retcode) %s" % (out, err, retcode)) 126 | 127 | return out, err, retcode 128 | 129 | 130 | def get_item_from_regex(output, regex): 131 | match = re.search(regex, output) 132 | if match is None: 133 | return '' 134 | else: 135 | return match.group(1) 136 | 137 | 138 | def get_file_name_image(image_url): 139 | return image_url.split('/')[-1] 140 | 141 | 142 | def unxz_source_file(file_path): 143 | output, error, retcode = external_run_command([ 144 | 'unxz', 145 | file_path 146 | ]) 147 | 148 | if retcode != 0: 149 | raise CommandRunFailed 150 | 151 | raw_file_path = file_path.rstrip('.xz') 152 | if not os.path.isfile(raw_file_path): 153 | raise UnCompressFailed 154 | 155 | return raw_file_path 156 | 157 | 158 | def get_source_from_image(image_url): 159 | tmpdir = tempfile.mkdtemp() 160 | file_name = get_file_name_image(image_url) 161 | file_path = os.path.join(tmpdir, file_name) 162 | 163 | _log.info("Preparing temporary directory for download: %r" % tmpdir) 164 | output, error, retcode = external_run_command([ 165 | 'wget', 166 | image_url, 167 | '-P', 168 | tmpdir 169 | ]) 170 | 171 | if retcode != 0: 172 | raise CommandRunFailed 173 | 174 | if not os.path.isfile(file_path): 175 | raise SourceNotFound 176 | 177 | return file_path 178 | 179 | 180 | def get_volume_type_from_image(image, region): 181 | return image.extra['block_device_mapping'][0]['ebs']['volume_type'] 182 | 183 | 184 | def get_virt_type_from_image(image): 185 | return 'hvm' 186 | 187 | 188 | def get_image_name_from_image(image_url, virt_type='', region='', respin='0', 189 | volume_type=''): 190 | 191 | file_name = get_file_name_image(image_url) 192 | build_name = file_name.replace('.raw.xz', '') 193 | 194 | return '-'.join( 195 | [x for x in [build_name, virt_type, region, volume_type, respin] if x]) 196 | 197 | 198 | def get_image_name_from_ami_name(image_name, region): 199 | name_vt_region, volume_type, respin = image_name.rsplit('-', 2) 200 | name_vt = name_vt_region.rsplit('-', 3)[:-3][0] 201 | 202 | return '-'.join( 203 | [x for x in [name_vt, region, volume_type, respin] if x]) 204 | 205 | 206 | def get_image_name_from_ami_name_for_fedmsg(image_name): 207 | name_vt_region, volume_type, respin = image_name.rsplit('-', 2) 208 | image_name = name_vt_region.rsplit('-', 4)[:-4][0] 209 | 210 | return image_name 211 | 212 | 213 | def cancel_stale_conversion_tasks(): 214 | _log.info("Preparing to cancel conversion tasks") 215 | output, error, retcode = external_run_command([ 216 | 'euca-describe-conversion-tasks --region us-east-1 | grep active', 217 | ]) 218 | 219 | if retcode != 0: 220 | return False 221 | 222 | tasks = output.split('\n')[1:-1] 223 | 224 | for task in tasks: 225 | expiration_datetime = datetime.datetime.strptime( 226 | task.split()[5], 227 | '%Y-%m-%dT%H:%M:%Sz' 228 | ) 229 | import_vol_task_id = task.split()[3] 230 | 231 | start_datetime = expiration_datetime - datetime.timedelta(days=7) 232 | now_datetime = datetime.datetime.now() 233 | 234 | delta_time = now_datetime - start_datetime 235 | 236 | if delta_time.total_seconds() >= 86400: 237 | _log.info( 238 | 'This is a stale task. Canceling %r' % import_vol_task_id) 239 | output, error, retcode = external_run_command([ 240 | 'euca-cancel-conversion-task --region us-east-1', 241 | import_vol_task_id 242 | ]) 243 | 244 | if retcode != 0: 245 | _log.info('Could not cancel the task %r' % import_vol_task_id) 246 | 247 | return True 248 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 16 | 17 | .PHONY: help 18 | help: 19 | @echo "Please use \`make ' where is one of" 20 | @echo " html to make standalone HTML files" 21 | @echo " dirhtml to make HTML files named index.html in directories" 22 | @echo " singlehtml to make a single large HTML file" 23 | @echo " pickle to make pickle files" 24 | @echo " json to make JSON files" 25 | @echo " htmlhelp to make HTML files and a HTML help project" 26 | @echo " qthelp to make HTML files and a qthelp project" 27 | @echo " applehelp to make an Apple Help Book" 28 | @echo " devhelp to make HTML files and a Devhelp project" 29 | @echo " epub to make an epub" 30 | @echo " epub3 to make an epub3" 31 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 32 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 33 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 34 | @echo " text to make text files" 35 | @echo " man to make manual pages" 36 | @echo " texinfo to make Texinfo files" 37 | @echo " info to make Texinfo files and run them through makeinfo" 38 | @echo " gettext to make PO message catalogs" 39 | @echo " changes to make an overview of all changed/added/deprecated items" 40 | @echo " xml to make Docutils-native XML files" 41 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 42 | @echo " linkcheck to check all external links for integrity" 43 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 44 | @echo " coverage to run coverage check of the documentation (if enabled)" 45 | @echo " dummy to check syntax errors of document sources" 46 | 47 | .PHONY: clean 48 | clean: 49 | rm -rf $(BUILDDIR)/* 50 | 51 | .PHONY: html 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | .PHONY: dirhtml 58 | dirhtml: 59 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 60 | @echo 61 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 62 | 63 | .PHONY: singlehtml 64 | singlehtml: 65 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 66 | @echo 67 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 68 | 69 | .PHONY: pickle 70 | pickle: 71 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 72 | @echo 73 | @echo "Build finished; now you can process the pickle files." 74 | 75 | .PHONY: json 76 | json: 77 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 78 | @echo 79 | @echo "Build finished; now you can process the JSON files." 80 | 81 | .PHONY: htmlhelp 82 | htmlhelp: 83 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 84 | @echo 85 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 86 | ".hhp project file in $(BUILDDIR)/htmlhelp." 87 | 88 | .PHONY: qthelp 89 | qthelp: 90 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 91 | @echo 92 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 93 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 94 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Fedimg.qhcp" 95 | @echo "To view the help file:" 96 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Fedimg.qhc" 97 | 98 | .PHONY: applehelp 99 | applehelp: 100 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 101 | @echo 102 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 103 | @echo "N.B. You won't be able to view it unless you put it in" \ 104 | "~/Library/Documentation/Help or install it in your application" \ 105 | "bundle." 106 | 107 | .PHONY: devhelp 108 | devhelp: 109 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 110 | @echo 111 | @echo "Build finished." 112 | @echo "To view the help file:" 113 | @echo "# mkdir -p $$HOME/.local/share/devhelp/Fedimg" 114 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Fedimg" 115 | @echo "# devhelp" 116 | 117 | .PHONY: epub 118 | epub: 119 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 120 | @echo 121 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 122 | 123 | .PHONY: epub3 124 | epub3: 125 | $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 126 | @echo 127 | @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." 128 | 129 | .PHONY: latex 130 | latex: 131 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 132 | @echo 133 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 134 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 135 | "(use \`make latexpdf' here to do that automatically)." 136 | 137 | .PHONY: latexpdf 138 | latexpdf: 139 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 140 | @echo "Running LaTeX files through pdflatex..." 141 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 142 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 143 | 144 | .PHONY: latexpdfja 145 | latexpdfja: 146 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 147 | @echo "Running LaTeX files through platex and dvipdfmx..." 148 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 149 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 150 | 151 | .PHONY: text 152 | text: 153 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 154 | @echo 155 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 156 | 157 | .PHONY: man 158 | man: 159 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 160 | @echo 161 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 162 | 163 | .PHONY: texinfo 164 | texinfo: 165 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 166 | @echo 167 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 168 | @echo "Run \`make' in that directory to run these through makeinfo" \ 169 | "(use \`make info' here to do that automatically)." 170 | 171 | .PHONY: info 172 | info: 173 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 174 | @echo "Running Texinfo files through makeinfo..." 175 | make -C $(BUILDDIR)/texinfo info 176 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 177 | 178 | .PHONY: gettext 179 | gettext: 180 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 181 | @echo 182 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 183 | 184 | .PHONY: changes 185 | changes: 186 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 187 | @echo 188 | @echo "The overview file is in $(BUILDDIR)/changes." 189 | 190 | .PHONY: linkcheck 191 | linkcheck: 192 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 193 | @echo 194 | @echo "Link check complete; look for any errors in the above output " \ 195 | "or in $(BUILDDIR)/linkcheck/output.txt." 196 | 197 | .PHONY: doctest 198 | doctest: 199 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 200 | @echo "Testing of doctests in the sources finished, look at the " \ 201 | "results in $(BUILDDIR)/doctest/output.txt." 202 | 203 | .PHONY: coverage 204 | coverage: 205 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 206 | @echo "Testing of coverage in the sources finished, look at the " \ 207 | "results in $(BUILDDIR)/coverage/python.txt." 208 | 209 | .PHONY: xml 210 | xml: 211 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 212 | @echo 213 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 214 | 215 | .PHONY: pseudoxml 216 | pseudoxml: 217 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 218 | @echo 219 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 220 | 221 | .PHONY: dummy 222 | dummy: 223 | $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy 224 | @echo 225 | @echo "Build finished. Dummy builder generates no files." 226 | -------------------------------------------------------------------------------- /fedimg/services/ec2/ec2initiate.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This file is part of fedimg. 3 | # Copyright (C) 2014-2018 Red Hat, Inc. 4 | # 5 | # fedimg is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU Affero General Public License as 7 | # published by the Free Software Foundation, either version 3 of the 8 | # License, or (at your option) any later version. 9 | # 10 | # fedimg is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | # Affero General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU Affero General Public 16 | # License along with fedimg; if not, see http://www.gnu.org/licenses, 17 | # or write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 19 | # 20 | # Authors: Sayan Chowdhury 21 | # 22 | """ 23 | I don't know what to write here for documentation 24 | """ 25 | 26 | import logging 27 | import os 28 | import shutil 29 | from itertools import product as itertools_product 30 | 31 | import fedimg.messenger 32 | from fedimg.config import AWS_BASE_REGION, AWS_VOLUME_TYPES 33 | from fedimg.config import AWS_ROOT_VOLUME_SIZE 34 | from fedimg.exceptions import CommandRunFailed, SourceNotFound 35 | from fedimg.exceptions import UnCompressFailed 36 | from fedimg.services.ec2.ec2imgpublisher import EC2ImagePublisher 37 | from fedimg.services.ec2.ec2imguploader import EC2ImageUploader 38 | from fedimg.utils import get_file_arch, get_image_name_from_image 39 | from fedimg.utils import get_image_name_from_ami_name_for_fedmsg 40 | from fedimg.utils import get_source_from_image, get_virt_types_from_url 41 | from fedimg.utils import unxz_source_file 42 | 43 | _log = logging.getLogger(__name__) 44 | 45 | 46 | def main(image_urls, access_id, secret_key, regions, volume_types=None, 47 | volume_via_s3=True, ex_virt_types=None, push_notifications=False, 48 | compose_id=None): 49 | """ 50 | The `ec2.ec2initiate.main` function iterates over the image urls and start 51 | uploading the image to the specified regions. The `image_urls`, 52 | `access_id`, and `regions` are the required params. `volume_types`, 53 | `ex_virt_types` are optional arguments, if not passed the values are picked 54 | up from the fedimg configuration. 55 | `volume_via_s3`, `push_notifications`, and `compose_id` are optional params 56 | with default values. 57 | 58 | Args: 59 | image_urls (list): List of the image urls to create AMIs. (reques 60 | access_id (str): AWS EC2 access id 61 | secret_key (str): AWS_EC2 secret key 62 | regions (list): List of AWS regions the AMI to be uploaded. 63 | volume_types (list): List of supported volumes for the AMIs to 64 | be created. 65 | volume_via_s3 (bool): If `True`, the images are uploaded via s3 method 66 | else using creating builder instances. 67 | ex_virt_types (list): List of the supported virts for the AMIs to 68 | be created. 69 | push_notifications (bool): If `True` the messages will be pushed to 70 | fedmsg, else skipped. 71 | compose_id: id of the current compose in process. 72 | """ 73 | 74 | root_volume_size = AWS_ROOT_VOLUME_SIZE 75 | published_images = [] 76 | 77 | if volume_types is None: 78 | volume_types = AWS_VOLUME_TYPES 79 | 80 | if regions is None: 81 | regions = [AWS_BASE_REGION] 82 | 83 | for image_url in image_urls: 84 | 85 | xz_file_path = None 86 | source = None 87 | upload_failed = False 88 | 89 | # If the virt types is not provided then select the supported virt 90 | # types from the image. 91 | if ex_virt_types is None: 92 | virt_types = get_virt_types_from_url(image_url) 93 | else: 94 | virt_types = ex_virt_types 95 | 96 | try: 97 | xz_file_path = get_source_from_image(image_url) 98 | source = unxz_source_file(xz_file_path) 99 | 100 | image_architecture = get_file_arch(image_url) 101 | 102 | uploader = EC2ImageUploader( 103 | compose_id=compose_id, 104 | access_key=access_id, 105 | secret_key=secret_key, 106 | root_volume_size=root_volume_size, 107 | image_architecture=image_architecture, 108 | volume_via_s3=volume_via_s3, 109 | push_notifications=push_notifications, 110 | image_url=image_url 111 | ) 112 | 113 | publisher = EC2ImagePublisher( 114 | compose_id=compose_id, 115 | access_key=access_id, 116 | secret_key=secret_key, 117 | push_notifications=push_notifications, 118 | image_url=image_url 119 | ) 120 | 121 | combinations = itertools_product( 122 | *[regions, virt_types, volume_types]) 123 | for region, virt_type, volume_type in combinations: 124 | uploader.set_region(region) 125 | _log.info('Region is set to: %r' % region) 126 | 127 | uploader.set_image_virt_type(virt_type) 128 | _log.info('Virtualization type is set to: %r' % virt_type) 129 | 130 | image_name = get_image_name_from_image( 131 | image_url=image_url, 132 | virt_type=virt_type, 133 | region=region, 134 | volume_type=volume_type 135 | ) 136 | uploader.set_image_name(image_name) 137 | _log.info('Image name is set to: %r' % image_name) 138 | 139 | uploader.set_image_volume_type(volume_type) 140 | _log.info('Volume type is set to: %r' % volume_type) 141 | 142 | uploader.set_availability_zone_for_region() 143 | 144 | if push_notifications: 145 | fedimg.messenger.notify( 146 | topic='image.upload', 147 | msg=dict( 148 | image_url=image_url, 149 | image_name=get_image_name_from_ami_name_for_fedmsg( 150 | image_name), 151 | destination=region, 152 | service='EC2', 153 | status='started', 154 | compose=compose_id, 155 | extra=dict( 156 | virt_type=virt_type, 157 | vol_type=volume_type 158 | ) 159 | ) 160 | ) 161 | image = uploader.create_image(source) 162 | 163 | published_images.extend(publisher.publish_images( 164 | region_image_mapping=[(region, image.id)] 165 | )) 166 | except CommandRunFailed: 167 | upload_failed = True 168 | _log.error("Could not execute the command for the requested " 169 | "image_url: %r" % image_url) 170 | except SourceNotFound: 171 | upload_failed = True 172 | _log.error("Could not find the source for the requested image_url" 173 | " : %r" % image_url) 174 | except UnCompressFailed: 175 | upload_failed = True 176 | _log.error("Could not uncompress the request image: %r" % 177 | image_url) 178 | except Exception as e: 179 | upload_failed = True 180 | _log.error(e.message) 181 | finally: 182 | if upload_failed and push_notifications: 183 | fedimg.messenger.notify( 184 | topic='image.upload', 185 | msg=dict( 186 | image_url=image_url, 187 | image_name=get_image_name_from_ami_name_for_fedmsg( 188 | image_name), 189 | destination=region, 190 | service='EC2', 191 | status='failed', 192 | compose=compose_id, 193 | extra=dict( 194 | virt_type=virt_type, 195 | vol_type=volume_type 196 | ) 197 | ) 198 | ) 199 | if xz_file_path: 200 | xz_file_dirname = os.path.dirname(xz_file_path) 201 | shutil.rmtree(xz_file_dirname) 202 | _log.info("Cleaned up %r" % xz_file_dirname) 203 | 204 | return published_images 205 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Fedimg documentation build configuration file, created by 4 | # sphinx-quickstart on Sat May 5 00:26:21 2018. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | # If extensions (or modules to document with autodoc) are in another directory, 16 | # add these directories to sys.path here. If the directory is relative to the 17 | # documentation root, use os.path.abspath to make it absolute, like shown here. 18 | # 19 | # import os 20 | # import sys 21 | # sys.path.insert(0, os.path.abspath('.')) 22 | 23 | # -- General configuration ------------------------------------------------ 24 | 25 | # If your documentation needs a minimal Sphinx version, state it here. 26 | # 27 | # needs_sphinx = '1.0' 28 | 29 | # Add any Sphinx extension module names here, as strings. They can be 30 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 31 | # ones. 32 | extensions = [ 33 | 'sphinx.ext.autodoc', 34 | 'sphinx.ext.coverage', 35 | 'sphinx.ext.viewcode', 36 | ] 37 | 38 | # Add any paths that contain templates here, relative to this directory. 39 | templates_path = ['_templates'] 40 | 41 | # The suffix(es) of source filenames. 42 | # You can specify multiple suffix as a list of string: 43 | # 44 | # source_suffix = ['.rst', '.md'] 45 | source_suffix = '.rst' 46 | 47 | # The encoding of source files. 48 | # 49 | # source_encoding = 'utf-8-sig' 50 | 51 | # The master toctree document. 52 | master_doc = 'index' 53 | 54 | # General information about the project. 55 | project = u'Fedimg' 56 | copyright = u'2018, Sayan Chowdhury, Ralph Bean, David Gay' 57 | author = u'Sayan Chowdhury, Ralph Bean, David Gay' 58 | 59 | # The version info for the project you're documenting, acts as replacement for 60 | # |version| and |release|, also used in various other places throughout the 61 | # built documents. 62 | # 63 | # The short X.Y version. 64 | version = u'2.0' 65 | # The full version, including alpha/beta/rc tags. 66 | release = u'2.0.0' 67 | 68 | # The language for content autogenerated by Sphinx. Refer to documentation 69 | # for a list of supported languages. 70 | # 71 | # This is also used if you do content translation via gettext catalogs. 72 | # Usually you set "language" from the command line for these cases. 73 | language = None 74 | 75 | # There are two options for replacing |today|: either, you set today to some 76 | # non-false value, then it is used: 77 | # 78 | # today = '' 79 | # 80 | # Else, today_fmt is used as the format for a strftime call. 81 | # 82 | # today_fmt = '%B %d, %Y' 83 | 84 | # List of patterns, relative to source directory, that match files and 85 | # directories to ignore when looking for source files. 86 | # This patterns also effect to html_static_path and html_extra_path 87 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 88 | 89 | # The reST default role (used for this markup: `text`) to use for all 90 | # documents. 91 | # 92 | # default_role = None 93 | 94 | # If true, '()' will be appended to :func: etc. cross-reference text. 95 | # 96 | # add_function_parentheses = True 97 | 98 | # If true, the current module name will be prepended to all description 99 | # unit titles (such as .. function::). 100 | # 101 | # add_module_names = True 102 | 103 | # If true, sectionauthor and moduleauthor directives will be shown in the 104 | # output. They are ignored by default. 105 | # 106 | # show_authors = False 107 | 108 | # The name of the Pygments (syntax highlighting) style to use. 109 | pygments_style = 'sphinx' 110 | 111 | # A list of ignored prefixes for module index sorting. 112 | # modindex_common_prefix = [] 113 | 114 | # If true, keep warnings as "system message" paragraphs in the built documents. 115 | # keep_warnings = False 116 | 117 | # If true, `todo` and `todoList` produce output, else they produce nothing. 118 | todo_include_todos = False 119 | 120 | 121 | # -- Options for HTML output ---------------------------------------------- 122 | 123 | # The theme to use for HTML and HTML Help pages. See the documentation for 124 | # a list of builtin themes. 125 | # 126 | html_theme = 'alabaster' 127 | 128 | # Theme options are theme-specific and customize the look and feel of a theme 129 | # further. For a list of options available for each theme, see the 130 | # documentation. 131 | # 132 | # html_theme_options = {} 133 | 134 | # Add any paths that contain custom themes here, relative to this directory. 135 | # html_theme_path = [] 136 | 137 | # The name for this set of Sphinx documents. 138 | # " v documentation" by default. 139 | # 140 | # html_title = u'Fedimg v1.3.0' 141 | 142 | # A shorter title for the navigation bar. Default is the same as html_title. 143 | # 144 | # html_short_title = None 145 | 146 | # The name of an image file (relative to this directory) to place at the top 147 | # of the sidebar. 148 | # 149 | # html_logo = None 150 | 151 | # The name of an image file (relative to this directory) to use as a favicon of 152 | # the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 153 | # pixels large. 154 | # 155 | # html_favicon = None 156 | 157 | # Add any paths that contain custom static files (such as style sheets) here, 158 | # relative to this directory. They are copied after the builtin static files, 159 | # so a file named "default.css" will overwrite the builtin "default.css". 160 | html_static_path = ['_static'] 161 | 162 | # Add any extra paths that contain custom files (such as robots.txt or 163 | # .htaccess) here, relative to this directory. These files are copied 164 | # directly to the root of the documentation. 165 | # 166 | # html_extra_path = [] 167 | 168 | # If not None, a 'Last updated on:' timestamp is inserted at every page 169 | # bottom, using the given strftime format. 170 | # The empty string is equivalent to '%b %d, %Y'. 171 | # 172 | # html_last_updated_fmt = None 173 | 174 | # If true, SmartyPants will be used to convert quotes and dashes to 175 | # typographically correct entities. 176 | # 177 | # html_use_smartypants = True 178 | 179 | # Custom sidebar templates, maps document names to template names. 180 | # 181 | # html_sidebars = {} 182 | 183 | # Additional templates that should be rendered to pages, maps page names to 184 | # template names. 185 | # 186 | # html_additional_pages = {} 187 | 188 | # If false, no module index is generated. 189 | # 190 | # html_domain_indices = True 191 | 192 | # If false, no index is generated. 193 | # 194 | # html_use_index = True 195 | 196 | # If true, the index is split into individual pages for each letter. 197 | # 198 | # html_split_index = False 199 | 200 | # If true, links to the reST sources are added to the pages. 201 | # 202 | # html_show_sourcelink = True 203 | 204 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 205 | # 206 | # html_show_sphinx = True 207 | 208 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 209 | # 210 | # html_show_copyright = True 211 | 212 | # If true, an OpenSearch description file will be output, and all pages will 213 | # contain a tag referring to it. The value of this option must be the 214 | # base URL from which the finished HTML is served. 215 | # 216 | # html_use_opensearch = '' 217 | 218 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 219 | # html_file_suffix = None 220 | 221 | # Language to be used for generating the HTML full-text search index. 222 | # Sphinx supports the following languages: 223 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' 224 | # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh' 225 | # 226 | # html_search_language = 'en' 227 | 228 | # A dictionary with options for the search language support, empty by default. 229 | # 'ja' uses this config value. 230 | # 'zh' user can custom change `jieba` dictionary path. 231 | # 232 | # html_search_options = {'type': 'default'} 233 | 234 | # The name of a javascript file (relative to the configuration directory) that 235 | # implements a search results scorer. If empty, the default will be used. 236 | # 237 | # html_search_scorer = 'scorer.js' 238 | 239 | # Output file base name for HTML help builder. 240 | htmlhelp_basename = 'Fedimgdoc' 241 | 242 | # -- Options for LaTeX output --------------------------------------------- 243 | 244 | latex_elements = { 245 | # The paper size ('letterpaper' or 'a4paper'). 246 | # 247 | # 'papersize': 'letterpaper', 248 | 249 | # The font size ('10pt', '11pt' or '12pt'). 250 | # 251 | # 'pointsize': '10pt', 252 | 253 | # Additional stuff for the LaTeX preamble. 254 | # 255 | # 'preamble': '', 256 | 257 | # Latex figure (float) alignment 258 | # 259 | # 'figure_align': 'htbp', 260 | } 261 | 262 | # Grouping the document tree into LaTeX files. List of tuples 263 | # (source start file, target name, title, 264 | # author, documentclass [howto, manual, or own class]). 265 | latex_documents = [ 266 | (master_doc, 'Fedimg.tex', u'Fedimg Documentation', 267 | u'Sayan Chowdhury, Ralph Bean, David Gay', 'manual'), 268 | ] 269 | 270 | # The name of an image file (relative to this directory) to place at the top of 271 | # the title page. 272 | # 273 | # latex_logo = None 274 | 275 | # For "manual" documents, if this is true, then toplevel headings are parts, 276 | # not chapters. 277 | # 278 | # latex_use_parts = False 279 | 280 | # If true, show page references after internal links. 281 | # 282 | # latex_show_pagerefs = False 283 | 284 | # If true, show URL addresses after external links. 285 | # 286 | # latex_show_urls = False 287 | 288 | # Documents to append as an appendix to all manuals. 289 | # 290 | # latex_appendices = [] 291 | 292 | # It false, will not define \strong, \code, itleref, \crossref ... but only 293 | # \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added 294 | # packages. 295 | # 296 | # latex_keep_old_macro_names = True 297 | 298 | # If false, no module index is generated. 299 | # 300 | # latex_domain_indices = True 301 | 302 | 303 | # -- Options for manual page output --------------------------------------- 304 | 305 | # One entry per manual page. List of tuples 306 | # (source start file, name, description, authors, manual section). 307 | man_pages = [ 308 | (master_doc, 'fedimg', u'Fedimg Documentation', 309 | [author], 1) 310 | ] 311 | 312 | # If true, show URL addresses after external links. 313 | # 314 | # man_show_urls = False 315 | 316 | 317 | # -- Options for Texinfo output ------------------------------------------- 318 | 319 | # Grouping the document tree into Texinfo files. List of tuples 320 | # (source start file, target name, title, author, 321 | # dir menu entry, description, category) 322 | texinfo_documents = [ 323 | (master_doc, 'Fedimg', u'Fedimg Documentation', 324 | author, 'Fedimg', 'One line description of project.', 325 | 'Miscellaneous'), 326 | ] 327 | 328 | # Documents to append as an appendix to all manuals. 329 | # 330 | # texinfo_appendices = [] 331 | 332 | # If false, no module index is generated. 333 | # 334 | # texinfo_domain_indices = True 335 | 336 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 337 | # 338 | # texinfo_show_urls = 'footnote' 339 | 340 | # If true, do not generate a @detailmenu in the "Top" node's menu. 341 | # 342 | # texinfo_no_detailmenu = False 343 | -------------------------------------------------------------------------------- /fedimg/services/ec2/ec2imgpublisher.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This file is part of fedimg. 3 | # Copyright (C) 2014-2018 Red Hat, Inc. 4 | # 5 | # fedimg is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU Affero General Public License as 7 | # published by the Free Software Foundation, either version 3 of the 8 | # License, or (at your option) any later version. 9 | # 10 | # fedimg is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | # Affero General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU Affero General Public 16 | # License along with fedimg; if not, see http://www.gnu.org/licenses, 17 | # or write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 19 | # 20 | # Authors: Sayan Chowdhury 21 | # 22 | 23 | import logging 24 | import re 25 | from time import sleep 26 | 27 | import fedimg.messenger 28 | from fedimg.services.ec2.ec2base import EC2Base 29 | from fedimg.utils import get_image_name_from_ami_name 30 | from fedimg.utils import get_image_name_from_ami_name_for_fedmsg 31 | 32 | _log = logging.getLogger(__name__) 33 | 34 | 35 | class EC2ImagePublisher(EC2Base): 36 | """ Comment goes here """ 37 | 38 | def __init__(self, **kwargs): 39 | defaults = { 40 | 'access_key': None, 41 | 'compose_id': None, 42 | 'image_id': None, 43 | 'image_name': 'Fedora-AMI', 44 | 'image_url': None, 45 | 'image_description': 'Fedora AMI Description', 46 | 'service': 'EC2', 47 | 'region': None, 48 | 'secret_key': None, 49 | 'visibility': 'all', 50 | 'push_notifications': False, 51 | } 52 | 53 | for (prop, default) in defaults.iteritems(): 54 | setattr(self, prop, kwargs.get(prop, default)) 55 | 56 | def _retry_till_image_is_public(self, image): 57 | """ Comment goes here """ 58 | 59 | driver = self._connect() 60 | 61 | is_image_public = False 62 | while True: 63 | try: 64 | is_image_public = driver.ex_modify_image_attribute( 65 | image, 66 | {'LaunchPermission.Add.1.Group': 'all'}) 67 | except Exception as e: 68 | if 'InvalidAMIID.Unavailable' in str(e): 69 | # The copy isn't completed yet, so wait for 20 seconds 70 | # more. 71 | sleep(20) 72 | continue 73 | break 74 | 75 | return is_image_public 76 | 77 | def _retry_till_snapshot_is_public(self, snapshot): 78 | 79 | driver = self._connect() 80 | 81 | while True: 82 | is_snapshot_public = driver.ex_modify_snapshot_attribute( 83 | snapshot, 84 | {'CreateVolumePermission.Add.1.Group': 'all'}) 85 | 86 | if is_snapshot_public: 87 | break 88 | 89 | return is_snapshot_public 90 | 91 | def _retry_till_image_is_available(self, image_id): 92 | driver = self._connect() 93 | 94 | while True: 95 | try: 96 | image = driver.get_image(image_id) 97 | image_name = image.name 98 | if image_name is None: 99 | continue 100 | return image 101 | except Exception as e: 102 | if 'InvalidAMIID.Unavailable' in str(e): 103 | _log.debug('Failed to find the AMI (%r). ' 104 | 'Sleeping for 20 seconds.' % image_id) 105 | # The copy isn't completed yet, so wait for 20 seconds 106 | # more. 107 | sleep(20) 108 | continue 109 | 110 | def _retry_till_snapshot_is_available(self, image): 111 | 112 | driver = self._connect() 113 | while True: 114 | image = driver.get_image(image.id) 115 | blk_mapping = image.extra['block_device_mapping'] 116 | snapshot_id = blk_mapping[0]['ebs']['snapshot_id'] 117 | 118 | if snapshot_id: 119 | break 120 | 121 | return snapshot_id 122 | 123 | def _generate_dummy_snapshot_object(self, snapshot_id): 124 | 125 | driver = self._connect() 126 | 127 | snapshot_obj = type('', (), {})() 128 | snapshot_obj.id = snapshot_id 129 | snapshot = driver.list_snapshots(snapshot=snapshot_obj) 130 | 131 | return snapshot 132 | 133 | def _retry_till_blk_mapping_is_available(self, image): 134 | 135 | while True: 136 | image = self._connect().get_image(image_id=image.id) 137 | blk_mapping = image.extra['block_device_mapping'] 138 | 139 | if blk_mapping: 140 | return blk_mapping 141 | 142 | def get_snapshot_from_image(self, image): 143 | """ Comment goes here """ 144 | if isinstance(image, str): 145 | image_id = image 146 | image = self._connect().get_image(image_id) 147 | 148 | blk_mapping = image.extra['block_device_mapping'] 149 | if not blk_mapping: 150 | blk_mapping = self._retry_till_blk_mapping_is_available(image) 151 | 152 | snapshot_id = blk_mapping[0]['ebs']['snapshot_id'] 153 | if snapshot_id is None: 154 | snapshot_id = self._retry_till_snapshot_is_available(image) 155 | 156 | snapshot = self._generate_dummy_snapshot_object(snapshot_id)[0] 157 | 158 | return snapshot 159 | 160 | def get_volume_type_from_image(self, image): 161 | if isinstance(image, str): 162 | image_id = image 163 | image = self._connect().get_image(image_id) 164 | 165 | blk_mapping = image.extra['block_device_mapping'] 166 | if not blk_mapping: 167 | blk_mapping = self._retry_till_blk_mapping_is_available(image) 168 | 169 | return blk_mapping[0]['ebs']['volume_type'] 170 | 171 | def get_arch_from_image(self, image): 172 | if isinstance(image, str): 173 | image_id = image 174 | image = self._connect().get_image(image_id) 175 | 176 | return image.extra['architecture'] 177 | 178 | def get_virt_type_from_image(self, image): 179 | return 'hvm' 180 | 181 | def publish_images(self, region_image_mapping=None): 182 | """ Comment goes here """ 183 | 184 | published_images = [] 185 | if region_image_mapping is None: 186 | return published_images 187 | 188 | for region, image_id in region_image_mapping: 189 | self.set_region(region) 190 | 191 | _log.info('Publish image (%s) in %s started' % (image_id, region)) 192 | image = self._connect().get_image(image_id=image_id) 193 | is_image_public = self._retry_till_image_is_public(image) 194 | _log.info('Publish image (%s) in %s completed', 195 | image_id, region) 196 | 197 | _log.info('Publish snaphsot for image (%s) in %s started', 198 | image_id, region) 199 | snapshot = self.get_snapshot_from_image(image) 200 | _log.info('Fetched snapshot for image (%s): %s', 201 | image_id, snapshot.id) 202 | is_snapshot_public = self._retry_till_snapshot_is_public(snapshot) 203 | _log.info('Publish snaphsot for image (%s) in %s completed', 204 | image_id, region) 205 | 206 | volume_type = self.get_volume_type_from_image(image) 207 | virt_type = self.get_virt_type_from_image(image) 208 | arch = self.get_arch_from_image(image) 209 | 210 | if self.push_notifications: 211 | fedimg.messenger.notify( 212 | topic='image.publish', 213 | msg=dict( 214 | image_name=get_image_name_from_ami_name_for_fedmsg( 215 | image.name), 216 | image_url=self.image_url, 217 | architecture=arch, 218 | destination=self.region, 219 | service=self.service, 220 | compose=self.compose_id, 221 | extra=dict( 222 | id=image.id, 223 | virt_type=virt_type, 224 | vol_type=volume_type 225 | ) 226 | ) 227 | ) 228 | 229 | published_images.append({ 230 | 'image_id': image.id, 231 | 'is_image_public': is_image_public, 232 | 'snapshot_id': snapshot.id, 233 | 'is_snapshot_public': is_snapshot_public, 234 | 'regions': self.region 235 | }) 236 | 237 | return published_images 238 | 239 | def copy_images_to_regions(self, image_id=None, base_region=None, 240 | regions=None): 241 | """ Comment goes here """ 242 | 243 | if (image_id is None) or (regions is None) or (base_region is None): 244 | return 245 | 246 | counter = 0 247 | copied_images = [] 248 | 249 | self.set_region(base_region) 250 | image = self._connect().get_image(image_id=image_id) 251 | if not image: 252 | return [] 253 | 254 | for region in regions: 255 | _log.info('Copy %s to %s started' % (image_id, region)) 256 | self.set_region(region) 257 | self.image_name = get_image_name_from_ami_name(image.name, region) 258 | 259 | while True: 260 | if counter > 0: 261 | self.image_name = re.sub( 262 | '\d(?!\d)', 263 | lambda x: str(int(x.group(0))+1), 264 | self.image_name 265 | ) 266 | try: 267 | copied_image = self._connect().copy_image( 268 | source_region=base_region, 269 | image=image, 270 | name=self.image_name, 271 | description=self.image_description) 272 | 273 | copied_image = self._retry_till_image_is_available( 274 | copied_image.id) 275 | 276 | virt_type = image.extra['virtualization_type'] 277 | 278 | blk_mapping = image.extra['block_device_mapping'] 279 | volume_type = blk_mapping[0]['ebs']['volume_type'] 280 | arch = self.get_arch_from_image(image) 281 | 282 | if self.push_notifications: 283 | fedimg.messenger.notify( 284 | topic='image.copy', 285 | msg=dict( 286 | image_name=( 287 | get_image_name_from_ami_name_for_fedmsg( 288 | copied_image.name)), 289 | destination=self.region, 290 | architecture=arch, 291 | service=self.service, 292 | compose_id=self.compose_id, 293 | extra=dict( 294 | id=copied_image.id, 295 | virt_type=virt_type, 296 | vol_type=volume_type, 297 | source_image_id=image.id 298 | ) 299 | ) 300 | ) 301 | 302 | _log.info('Copy %s to %s is completed.', image_id, region) 303 | copied_images.append({ 304 | 'region': region, 305 | 'copied_image_id': copied_image.id 306 | }) 307 | break 308 | 309 | except Exception as e: 310 | _log.info('Could not register with name: %r', 311 | self.image_name) 312 | if 'InvalidAMIName.Duplicate' in str(e): 313 | counter = counter + 1 314 | else: 315 | _log.info('Failed') 316 | break 317 | 318 | return copied_images 319 | 320 | def deprecate_images(self, image_ids=None, snapshot_perm='all'): 321 | raise NotImplementedError 322 | 323 | def delete_images(self, image_ids=None, snapshot_perm='all'): 324 | raise NotImplementedError 325 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | # This file is part of fedimg. 2 | # Copyright (C) 2014-2018 Red Hat, Inc. 3 | # 4 | # fedimg is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as 6 | # published by the Free Software Foundation, either version 3 of the 7 | # License, or (at your option) any later version. 8 | # 9 | # fedimg is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | # Affero General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU Affero General Public 15 | # License along with fedimg; if not, see http://www.gnu.org/licenses, 16 | # or write to the Free Software Foundation, Inc., 17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 18 | # 19 | # Authors: David Gay 20 | # Sayan Chowdhury 21 | 22 | import mock 23 | import unittest 24 | import subprocess 25 | import paramiko 26 | 27 | import pytest 28 | 29 | import fedimg 30 | import fedimg.utils 31 | 32 | from fedimg.exceptions import CommandRunFailed 33 | 34 | 35 | def mock_ssh_exception(*args, **kwargs): 36 | raise paramiko.SSHException("Could not connect") 37 | 38 | 39 | class TestFedimgUtils(unittest.TestCase): 40 | 41 | class MockPopen(object): 42 | def __init(self): 43 | pass 44 | 45 | def communicate(self, input=None): 46 | pass 47 | 48 | @property 49 | def returncode(self): 50 | pass 51 | 52 | def test_get_file_arch_x86_64(self): 53 | filename = 'Fedora-Atomic-26-1.5.x86_64.raw.xz' 54 | arch = fedimg.utils.get_file_arch(filename) 55 | assert arch == 'x86_64' 56 | 57 | def test_get_file_arch_aarch64(self): 58 | filename = 'Fedora-Cloud-Base-26-1.5.aarch64.raw.xz' 59 | arch = fedimg.utils.get_file_arch(filename) 60 | assert arch == None 61 | 62 | def test_get_rawxz_urls(self): 63 | images = [ 64 | { 65 | "arch": "x86_64", 66 | "format": "qcow2", 67 | "implant_md5": None, 68 | "mtime": 1499295621, 69 | "path": "CloudImages/x86_64/images/Fedora-Atomic-26-1.5.x86_64.qcow2", 70 | "size": 702779904, 71 | "subvariant": "Atomic", 72 | "type": "qcow2", 73 | "volume_id": None 74 | }, 75 | { 76 | "arch": "x86_64", 77 | "format": "raw.xz", 78 | "implant_md5": None, 79 | "mtime": 1499295718, 80 | "path": "CloudImages/x86_64/images/Fedora-Atomic-26-1.5.x86_64.raw.xz", 81 | "size": 529608216, 82 | "subvariant": "Atomic", 83 | "type": "raw-xz", 84 | "volume_id": None 85 | }, 86 | { 87 | "arch": "x86_64", 88 | "bootable": False, 89 | "disc_count": 1, 90 | "disc_number": 1, 91 | "format": "raw.xz", 92 | "implant_md5": None, 93 | "mtime": 1499291771, 94 | "path": "CloudImages/x86_64/images/Fedora-Cloud-Base-26-1.5.x86_64.raw.xz", 95 | "size": 154897200, 96 | "subvariant": "Cloud_Base", 97 | "type": "raw-xz", 98 | "volume_id": None 99 | }, 100 | { 101 | "arch": "x86_64", 102 | "format": "vagrant-libvirt.box", 103 | "implant_md5": None, 104 | "mtime": 1499291717, 105 | "path": "CloudImages/x86_64/images/Fedora-Cloud-Base-Vagrant-26-1.5.x86_64.vagrant-libvirt.box", 106 | "size": 229231805, 107 | "subvariant": "Cloud_Base", 108 | "type": "vagrant-libvirt", 109 | "volume_id": None 110 | }, 111 | ] 112 | 113 | urls = fedimg.utils.get_rawxz_urls('https://fedoraproject.org', images) 114 | atomic_url = 'https://fedoraproject.org/CloudImages/x86_64/images/Fedora-Atomic-26-1.5.x86_64.raw.xz' 115 | cloud_base_url = 'https://fedoraproject.org/CloudImages/x86_64/images/Fedora-Cloud-Base-26-1.5.x86_64.raw.xz' 116 | 117 | assert len(urls) == 2 118 | self.assertEqual(set(urls), {atomic_url, cloud_base_url}) 119 | 120 | def test_get_rawxz_urls_empty(self): 121 | images = [ 122 | { 123 | "arch": "x86_64", 124 | "format": "qcow2", 125 | "implant_md5": None, 126 | "mtime": 1499295621, 127 | "path": "CloudImages/x86_64/images/Fedora-Atomic-26-1.5.x86_64.qcow2", 128 | "size": 702779904, 129 | "subvariant": "Atomic", 130 | "type": "qcow2", 131 | "volume_id": None 132 | }, 133 | { 134 | "arch": "x86_64", 135 | "format": "vagrant-libvirt.box", 136 | "implant_md5": None, 137 | "mtime": 1499291717, 138 | "path": "CloudImages/x86_64/images/Fedora-Cloud-Base-Vagrant-26-1.5.x86_64.vagrant-libvirt.box", 139 | "size": 229231805, 140 | "subvariant": "Cloud_Base", 141 | "type": "vagrant-libvirt", 142 | "volume_id": None 143 | }, 144 | ] 145 | 146 | urls = fedimg.utils.get_rawxz_urls('https://somepage.org', images) 147 | self.assertEquals(urls, []) 148 | 149 | def test_get_virt_types_hvm_pv(self): 150 | url = 'https://somepage.org/Fedora-Cloud-Base-26-1.5.x86_64.raw.xz' 151 | vtypes = fedimg.utils.get_virt_types_from_url(url) 152 | assert vtypes == ['hvm'] 153 | 154 | def test_get_virt_types_only_hvm(self): 155 | url = 'https://somepage.org/Fedora-Atomic-26-1.5.x86_64.raw.xz' 156 | vtypes = fedimg.utils.get_virt_types_from_url(url) 157 | assert vtypes == ['hvm'] 158 | 159 | def test_get_value_from_dict(self): 160 | _dict = { 161 | "payload": { 162 | "compose": { 163 | "date": "20180326", 164 | "id": "Fedora-28-20180326.0", 165 | }, 166 | "images": { 167 | "AtomicHost": { 168 | "aarch64": [ 169 | { 170 | "arch": "aarch64", 171 | "format": "qcow2", 172 | "path": "AtomicHost/aarch64/images/Fedora-AtomicHost-28_Beta-1.1.aarch64.qcow2", 173 | }, 174 | { 175 | "arch": "aarch64", 176 | "format": "raw.xz", 177 | "path": "AtomicHost/aarch64/images/Fedora-AtomicHost-28_Beta-1.1.aarch64.raw.xz", 178 | }, 179 | ] 180 | } 181 | } 182 | } 183 | } 184 | 185 | valid_return = fedimg.utils.get_value_from_dict(_dict, 'payload', 'images', 'AtomicHost') 186 | invalid_return = fedimg.utils.get_value_from_dict(_dict, 'payload', 'images', 'Cloud') 187 | 188 | assert 'aarch64' in valid_return 189 | assert invalid_return == None 190 | 191 | @mock.patch('fedimg.utils._log.debug') 192 | def test_external_run_command(self, mock_log): 193 | mock_popen = TestFedimgUtils.MockPopen() 194 | mock_popen.communicate = mock.Mock(return_value=( 195 | "2018-03-27 00:00:00 (93 KB/s) 'Fedora-Atomic-26-1.5.x86_64.raw.xz' (1234569/123456789)", 196 | "" 197 | )) 198 | mock_returncode = mock.PropertyMock(return_value=1) 199 | type(mock_popen).returncode = mock_returncode 200 | setattr(subprocess, 'Popen', lambda *args, **kargs: mock_popen) 201 | 202 | fedimg.utils.external_run_command('wget https://somepage.org/Fedora-Atomic-26-1.5.x86_64.raw.xz -P /tmp/tmpABCDEF') 203 | 204 | mock_popen.communicate.assert_called_once_with() 205 | mock_returncode.assert_called_once_with() 206 | 207 | def test_get_item_from_regex(self): 208 | 209 | result = fedimg.utils.get_item_from_regex( 210 | 'Task completed: import-vol-abc12345', 211 | '\s(import-vol-\w{8})' 212 | ) 213 | 214 | assert result == 'import-vol-abc12345' 215 | 216 | negative_result = fedimg.utils.get_item_from_regex( 217 | 'Task completed: import-vol-1234', 218 | '\s(import-vol-\w{8})' 219 | ) 220 | 221 | assert negative_result == '' 222 | 223 | def test_get_file_name_image(self): 224 | 225 | result = fedimg.utils.get_file_name_image( 226 | 'https://somepage.org/Fedora-Atomic-26-1.5.x86_64.raw.xz' 227 | ) 228 | 229 | assert result == 'Fedora-Atomic-26-1.5.x86_64.raw.xz' 230 | 231 | @mock.patch('fedimg.utils.external_run_command') 232 | @mock.patch('os.path.isfile', return_value=True) 233 | def test_get_source_from_image(self, mock_isfile, mock_erc): 234 | mock_erc.return_value = ( 235 | "2018-03-27 00:00:00 (93 KB/s) 'Fedora-Atomic-26-1.5.x86_64.raw.xz' (1234569/123456789)", 236 | "", 237 | 0 238 | ) 239 | fedimg.utils.get_source_from_image('https://somepage.org/Fedora-Atomic-26-1.5.x86_64.raw.xz') 240 | 241 | mock_erc.assert_called_once_with([ 242 | 'wget', 243 | 'https://somepage.org/Fedora-Atomic-26-1.5.x86_64.raw.xz', 244 | '-P', 245 | mock.ANY 246 | ]) 247 | 248 | @mock.patch('fedimg.utils.external_run_command') 249 | @mock.patch('os.path.isfile', return_value=True) 250 | def test_unxz_source_file(self, mock_isfile, mock_erc): 251 | mock_erc.return_value = ( 252 | "/tmp/Fedora-Atomic-26-1.5.x86_64.raw", 253 | "", 254 | 0 255 | ) 256 | fedimg.utils.unxz_source_file('/tmp/Fedora-Atomic-26-1.5.x86_64.raw.xz') 257 | 258 | mock_erc.assert_called_once_with([ 259 | 'unxz', 260 | '/tmp/Fedora-Atomic-26-1.5.x86_64.raw.xz', 261 | ]) 262 | 263 | @mock.patch('fedimg.utils.external_run_command') 264 | @mock.patch('os.path.isfile', return_value=True) 265 | def test_get_source_from_image_retcode_1(self, mock_isfile, mock_erc): 266 | mock_erc.return_value = ( 267 | "2018-03-27 00:00:00 (93 KB/s) 'Fedora-Atomic-26-1.5.x86_64.raw.xz' (1234569/123456789)", 268 | "", 269 | 1 270 | ) 271 | 272 | with pytest.raises(CommandRunFailed): 273 | fedimg.utils.get_source_from_image('https://somepage.org/Fedora-Atomic-26-1.5.x86_64.raw.xz') 274 | 275 | def test_get_volume_type_from_image(self): 276 | mock_image = mock.Mock() 277 | mock_image.extra = { 278 | 'block_device_mapping': [{ 279 | 'ebs': { 280 | 'volume_type': 'gp2' 281 | } 282 | }] 283 | } 284 | region = 'us-east-1' 285 | 286 | volume_type = fedimg.utils.get_volume_type_from_image( 287 | mock_image, region) 288 | 289 | assert volume_type == 'gp2' 290 | 291 | def test_get_virt_type_from_image_hvm(self): 292 | mock_image = mock.Mock() 293 | mock_image.extra = { 294 | 'block_device_mapping': [{ 295 | 'device_name': '/dev/sda1' 296 | }] 297 | } 298 | 299 | virt_type = fedimg.utils.get_virt_type_from_image(mock_image) 300 | 301 | assert virt_type == 'hvm' 302 | 303 | def test_region_to_driver(self): 304 | driver_1 = fedimg.utils.region_to_driver('us-east-1') 305 | driver_2 = fedimg.utils.region_to_driver('eu-west-1') 306 | 307 | assert driver_1.func == driver_2.func 308 | 309 | @mock.patch('paramiko.SSHClient') 310 | def test_ssh_connection_works_true(self, mocksshclient): 311 | mocksshclient.connect.return_value = '' 312 | result = fedimg.utils.ssh_connection_works( 313 | 'testuser', 314 | '127.0.0.1', 315 | '/path/to/key' 316 | ) 317 | 318 | self.assertIs(result, True) 319 | 320 | @mock.patch('paramiko.SSHClient') 321 | def test_ssh_connection_works_false(self, mocksshclient): 322 | mockssh = mock.Mock() 323 | mocksshclient.return_value = mockssh 324 | mockssh.connect.side_effect = mock_ssh_exception 325 | result = fedimg.utils.ssh_connection_works( 326 | 'testuser', 327 | '127.0.0.1', 328 | '/path/to/key' 329 | ) 330 | 331 | self.assertIs(result, False) 332 | 333 | def test_get_image_name_from_image(self): 334 | image_name = fedimg.utils.get_image_name_from_image( 335 | 'https://somepage.org/Fedora-Atomic-26-1.5.x86_64.raw.xz', 336 | 'hvm', 337 | 'us-east-1', 338 | '0', 339 | 'gp2' 340 | ) 341 | 342 | assert image_name == 'Fedora-Atomic-26-1.5.x86_64-hvm-us-east-1-gp2-0' 343 | 344 | def test_get_image_name_from_image_without_vol_type(self): 345 | image_name = fedimg.utils.get_image_name_from_image( 346 | image_url='https://somepage.org/Fedora-Atomic-26-1.5.x86_64.raw.xz', 347 | virt_type='hvm', 348 | region='us-east-1', 349 | respin='0', 350 | ) 351 | 352 | assert image_name == 'Fedora-Atomic-26-1.5.x86_64-hvm-us-east-1-0' 353 | 354 | def test_get_image_name_from_ami_name(self): 355 | image_name = fedimg.utils.get_image_name_from_ami_name( 356 | 'Fedora-Cloud-Base-26-20180329.0.x86_64-paravirtual-us-east-1-gp2-0', 357 | 'eu-west-1' 358 | ) 359 | 360 | assert image_name == 'Fedora-Cloud-Base-26-20180329.0.x86_64-paravirtual-eu-west-1-gp2-0' 361 | 362 | def test_get_image_name_from_ami_name_for_fedmsg(self): 363 | image_name = fedimg.utils.get_image_name_from_ami_name_for_fedmsg( 364 | 'Fedora-Cloud-Base-26-20180329.0.x86_64-paravirtual-us-east-1-gp2-0', 365 | ) 366 | 367 | assert image_name == 'Fedora-Cloud-Base-26-20180329.0.x86_64' 368 | -------------------------------------------------------------------------------- /fedimg/services/ec2/ec2imguploader.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This file is part of fedimg. 3 | # Copyright (C) 2014-2018 Red Hat, Inc. 4 | # 5 | # fedimg is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU Affero General Public License as 7 | # published by the Free Software Foundation, either version 3 of the 8 | # License, or (at your option) any later version. 9 | # 10 | # fedimg is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | # Affero General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU Affero General Public 16 | # License along with fedimg; if not, see http://www.gnu.org/licenses, 17 | # or write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 19 | # 20 | # Authors: Sayan Chowdhury 21 | # 22 | import logging 23 | import re 24 | from time import sleep 25 | 26 | import fedimg.messenger 27 | from fedimg.config import AWS_DELETE_RESOURCES, AWS_S3_BUCKET_NAME 28 | from fedimg.services.ec2.ec2base import EC2Base 29 | from fedimg.utils import external_run_command, get_item_from_regex 30 | from fedimg.utils import get_image_name_from_ami_name_for_fedmsg 31 | 32 | _log = logging.getLogger(__name__) 33 | 34 | 35 | class EC2ImageUploader(EC2Base): 36 | """ 37 | The 'ec2.ec2imguploader.EC2ImageUploader` creates the Amazon Machine Image 38 | (AMI) from the source. 39 | 40 | Args: 41 | access_key (str): Access key for the account. 42 | availability_zone (str): Availability zone to use 43 | compose_id (str): ID of the Compose 44 | image_name (str): Name of the Image 45 | image_description (str): Description of the image 46 | image_virtualization_type (str): Virtulization Type of the Image 47 | image_architecture (str): Architecture of the Image 48 | image_url (str): URL of the the Image 49 | image_volume_type (str): Volume Type of the Image 50 | image_format (str): Format of the Image 51 | region (str): Region to upload 52 | service (str): Name of the Fedimg service. 53 | secret_key (str): Secret Key for the account 54 | s3_bucket_name (str): Name of the S3 bucket 55 | volume_via_s3 (bool): Bool to control AMI using S3-method 56 | root_volume_size (str): Size of the AWS volume 57 | push_notifications (bool): Push notifications to fedmsg 58 | """ 59 | 60 | def __init__(self, *args, **kwargs): 61 | defaults = { 62 | 'access_key': None, 63 | 'availability_zone': None, 64 | 'compose_id': None, 65 | 'image_name': 'Fedora-AMI', 66 | 'image_description': 'Fedora AMI Description', 67 | 'image_virtualization_type': 'hvm', 68 | 'image_architecture': 'x86_64', 69 | 'image_url': None, 70 | 'image_volume_type': 'gp2', 71 | 'image_format': 'raw', 72 | 'region': None, 73 | 'service': 'EC2', 74 | 'secret_key': None, 75 | 's3_bucket_name': AWS_S3_BUCKET_NAME, 76 | 'volume_via_s3': True, 77 | 'root_volume_size': 7, 78 | 'push_notifications': False, 79 | } 80 | 81 | for (prop, default) in defaults.iteritems(): 82 | setattr(self, prop, kwargs.get(prop, default)) 83 | 84 | def _determine_root_device_name(self): 85 | root_device_name = '/dev/sda' 86 | if self.image_virtualization_type == 'hvm': 87 | root_device_name = '/dev/sda1' 88 | 89 | _log.debug('Root device name is set to %r for %r' % ( 90 | root_device_name, self.image_virtualization_type)) 91 | 92 | return root_device_name 93 | 94 | def _create_block_device_map(self, snapshot): 95 | root_device_name = self._determine_root_device_name() 96 | block_device_map = { 97 | 'DeviceName': root_device_name, 98 | 'Ebs': { 99 | 'SnapshotId': snapshot.id, 100 | 'VolumeSize': self.root_volume_size, 101 | 'VolumeType': self.image_volume_type, 102 | 'DeleteOnTermination': True, 103 | } 104 | } 105 | 106 | _log.debug('Block device map created for %s' % snapshot.id) 107 | 108 | return [block_device_map] 109 | 110 | def _retry_and_get_volume_id(self, task_id): 111 | while True: 112 | output, err, retcode = external_run_command([ 113 | 'euca-describe-conversion-tasks', 114 | task_id, 115 | '--region', 116 | self.region 117 | ]) 118 | 119 | if 'completed' in output: 120 | _log.debug('Task %r complete. Fetching volume id...' % task_id) 121 | volume_id = get_item_from_regex(output, 122 | regex='\s(vol-\w{17})') 123 | if not volume_id: 124 | volume_id = get_item_from_regex(output, 125 | regex='\s(vol-\w{8})') 126 | 127 | if not volume_id: 128 | _log.error('Unable to fetch volume_id') 129 | raise Exception('Unable to fetch volume_id.') 130 | 131 | _log.debug('The id of the created volume: %r' % volume_id) 132 | 133 | return volume_id 134 | 135 | _log.debug('Failed to find complete. Task %r still running. ' 136 | 'Sleeping for 10 seconds.' % task_id) 137 | sleep(10) 138 | 139 | def _create_snapshot(self, volume): 140 | snapshot = self._connect().create_volume_snapshot( 141 | volume=volume, 142 | name=self.image_name 143 | ) 144 | snapshot_id = snapshot.id 145 | snapshot = self._retry_and_get_snapshot(snapshot_id) 146 | 147 | return snapshot 148 | 149 | def _retry_and_get_snapshot(self, snapshot_id): 150 | # FIXME: Rather that listing all snapshot. Add a patch to libcloud to 151 | # pull the snapshot using the snapshot id. 152 | snapshots = self._connect().list_snapshots() 153 | for snapshot in snapshots: 154 | if snapshot.id == snapshot_id: 155 | break 156 | 157 | while snapshot.extra['state'] != 'completed': 158 | snapshots = self._connect().list_snapshots() 159 | for snapshot in snapshots: 160 | if snapshot.id == snapshot_id: 161 | break 162 | 163 | return snapshot 164 | 165 | def _create_volume(self, source): 166 | if self.volume_via_s3: 167 | output, err, retcode = external_run_command([ 168 | 'euca-import-volume', 169 | source, 170 | '-f', 171 | self.image_format, 172 | '--region', 173 | self.region, 174 | '-b', 175 | self.s3_bucket_name, 176 | '-z', 177 | self.availability_zone 178 | ]) 179 | 180 | if retcode != 0: 181 | _log.error( 182 | 'Unable to import volume. Out: %s, err: %s, ret: %s', 183 | output, err, retcode) 184 | raise Exception('Creating the volume failed') 185 | 186 | _log.debug('Initiate task to upload the image via S3. ' 187 | 'Fetching task id...') 188 | 189 | task_id = get_item_from_regex(output, 190 | regex='\s(import-vol-\w{17})') 191 | 192 | # Support the shorter task ids 193 | if not task_id: 194 | task_id = get_item_from_regex(output, 195 | regex='\s(import-vol-\w{8})') 196 | 197 | if not task_id: 198 | _log.error('Unable to fetch task_id') 199 | raise Exception('Unable to fetch task_id.') 200 | 201 | _log.info('Fetched task_id: %r. Listening to the task.' % task_id) 202 | 203 | volume_id = self._retry_and_get_volume_id(task_id) 204 | 205 | volume = self.get_volume_from_volume_id(volume_id) 206 | _log.info('Finish fetching volume object using volume_id') 207 | 208 | return volume 209 | 210 | def _register_image(self, snapshot): 211 | counter = 0 212 | block_device_map = self._create_block_device_map(snapshot) 213 | root_device_name = self._determine_root_device_name() 214 | while True: 215 | if counter > 0: 216 | self.image_name = re.sub('\d(?!\d)$', 217 | lambda x: str(int(x.group(0))+1), 218 | self.image_name) 219 | try: 220 | _log.info('Registering the image in %r (snapshot id: %r) with ' 221 | 'name %r' % (self.region, snapshot.id, 222 | self.image_name)) 223 | image = self._connect().ex_register_image( 224 | name=self.image_name, 225 | description=self.image_description, 226 | virtualization_type=self.image_virtualization_type, 227 | architecture=self.image_architecture, 228 | block_device_mapping=block_device_map, 229 | root_device_name=root_device_name, 230 | ena_support=True 231 | ) 232 | 233 | if self.push_notifications: 234 | fedimg.messenger.notify( 235 | topic='image.upload', 236 | msg=dict( 237 | image_url=self.image_url, 238 | image_name=get_image_name_from_ami_name_for_fedmsg( 239 | self.image_name), 240 | destination=self.region, 241 | service=self.service, 242 | status='completed', 243 | compose=self.compose_id, 244 | extra=dict( 245 | id=image.id, 246 | virt_type=self.image_virtualization_type, 247 | vol_type=self.image_volume_type 248 | ) 249 | ) 250 | ) 251 | 252 | return image 253 | 254 | except Exception as e: 255 | _log.info('Could not register with name: %r' % self.image_name) 256 | if 'InvalidAMIName.Duplicate' in str(e): 257 | counter = counter + 1 258 | else: 259 | raise 260 | 261 | def _remove_volume(self, volume): 262 | _log.info('[CLEAN] Destroying volume: %r' % volume.id) 263 | self._connect().destroy_volume(volume) 264 | 265 | def clean_up(self, image_id, delete_snapshot=True, force=False): 266 | """ 267 | Clean the resources created during the upload process 268 | 269 | Args: 270 | image_id (str): ID of the AMI 271 | delete_snapshot (bool): Bool to control snapshot deletion along 272 | with AMI. 273 | force: Override AWS_DELETE_RESOURCES config value 274 | 275 | Returns: 276 | Boolean: True, if resources are deleted else False 277 | """ 278 | if not AWS_DELETE_RESOURCES and force: 279 | _log.info('Deleting resource is disabled by config.' 280 | 'Override by passing force=True.') 281 | return False 282 | 283 | if self.volume: 284 | self._connect().destroy_volume(self.volume) 285 | 286 | self._connect().deregister_image( 287 | image_id, 288 | delete_snapshot=delete_snapshot 289 | ) 290 | 291 | return True 292 | 293 | def set_image_virt_type(self, virt_type): 294 | """ 295 | Set the `image_virtualization_type` attribute of the `EC2ImageUploader` 296 | object. 297 | 298 | Args: 299 | virt_type (str): virtualization type to set for the object. 300 | """ 301 | self.image_virtualization_type = virt_type 302 | 303 | def set_image_url(self, image_url): 304 | """ 305 | Set the `image_url` attribute of the `EC2ImageUploader` object. 306 | 307 | Args: 308 | image_url (str): image_url to set for the object. 309 | """ 310 | self.image_url = image_url 311 | 312 | def set_image_name(self, image_name): 313 | """ 314 | Set the `image_name` attribute of the `EC2ImageUploader` object. 315 | 316 | Args: 317 | image_name (str): image_name to set for the object. 318 | """ 319 | self.image_name = image_name 320 | 321 | def set_image_volume_type(self, volume_type): 322 | """ 323 | Set the `volume_type` attribute of the `EC2ImageUploader` object. 324 | 325 | Args: 326 | volume_type (str): volume_type to set for the object. 327 | """ 328 | self.image_volume_type = volume_type 329 | 330 | def get_volume_from_volume_id(self, volume_id): 331 | """ Get the `` object from the volume_id. 332 | 333 | Args: 334 | volume_id (str): volume_id for the `EC2ImageUploader` object 335 | """ 336 | # FIXME: This is not a optimized way of get all the volumes. Rather 337 | # send a patch to libcloud to filter the volume based on the volume_id 338 | 339 | volumes = self._connect().list_volumes() 340 | 341 | for volume in volumes: 342 | if volume.id == volume_id: 343 | return volume 344 | 345 | def set_availability_zone_for_region(self): 346 | """ 347 | Returns a availability zone for the region 348 | """ 349 | self.availability_zone = self._connect().ex_list_availability_zones( 350 | only_available=True)[0].name 351 | 352 | def create_volume(self, source): 353 | """ 354 | Create a AWS volume out of the given source file 355 | 356 | Args: 357 | source (str): File path of the source file 358 | """ 359 | _log.info('Start creating the volume from source: %r' % source) 360 | return self._create_volume(source) 361 | 362 | def create_snapshot(self, source): 363 | """ 364 | Creates a AWS snapshot out of the given source file 365 | 366 | Args: 367 | source (str): File path of the source file 368 | 369 | Returns: 370 | snapshot: `VolumeSnapshot` object 371 | """ 372 | self.volume = self._create_volume(source) 373 | 374 | _log.info('Start creating snapshot from volume: %r' % self.volume.id) 375 | snapshot = self._create_snapshot(self.volume) 376 | 377 | self._remove_volume(self.volume) 378 | 379 | return snapshot 380 | 381 | def register_image(self, snapshot): 382 | """ 383 | Register the image for the given `` object, 384 | 385 | Args: 386 | snapshot: `VolumeSnapshot` object 387 | 388 | Returns: 389 | image: `NodeImage` object 390 | """ 391 | image = self._register_image(snapshot) 392 | 393 | return image 394 | 395 | def create_image(self, source): 396 | """ 397 | Create Amazon machin image out of the given source 398 | 399 | Args: 400 | source (str): path of the source cloud image file. 401 | 402 | Returns: 403 | image: returns the `NodeImage` object. 404 | """ 405 | 406 | snapshot = self.create_snapshot(source) 407 | _log.debug('Finished create snapshot: %r' % snapshot.id) 408 | 409 | _log.info('Start to register the image from the snapshot: %r', 410 | snapshot.id) 411 | image = self.register_image(snapshot) 412 | _log.debug('Finish registering the image with id: %r' % image.id) 413 | 414 | return image 415 | -------------------------------------------------------------------------------- /tests/vcr-request-data/test_consumers.TestFedimgConsumer.test_incompatible_images: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: null 4 | headers: 5 | Connection: [close] 6 | Host: [!!python/unicode 'kojipkgs.fedoraproject.org'] 7 | User-Agent: [Python-urllib/2.7] 8 | method: GET 9 | uri: https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20180426.n.0/compose 10 | response: 11 | body: {string: !!python/unicode ' 12 | 13 | 14 | 15 | 301 Moved Permanently 16 | 17 | 18 | 19 |

Moved Permanently

20 | 21 |

The document has moved here.

22 | 23 | 24 | 25 | '} 26 | headers: 27 | appserver: [proxy10.phx2.fedoraproject.org] 28 | apptime: [D=13359] 29 | connection: [close] 30 | content-length: ['295'] 31 | content-type: [text/html; charset=iso-8859-1] 32 | date: ['Fri, 27 Apr 2018 10:56:08 GMT'] 33 | location: ['https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20180426.n.0/compose/'] 34 | referrer-policy: [same-origin] 35 | server: [Apache/2.4.33 (Fedora)] 36 | strict-transport-security: [max-age=31536000; includeSubDomains; preload, max-age=31536000; 37 | includeSubDomains; preload] 38 | x-content-type-options: [nosniff] 39 | x-frame-options: [SAMEORIGIN] 40 | x-xss-protection: [1; mode=block] 41 | status: {code: 301, message: Moved Permanently} 42 | - request: 43 | body: null 44 | headers: 45 | Connection: [close] 46 | Host: [kojipkgs.fedoraproject.org] 47 | User-Agent: [Python-urllib/2.7] 48 | method: GET 49 | uri: https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20180426.n.0/compose/ 50 | response: 51 | body: {string: !!python/unicode "\n\n \n Index of /compose/rawhide/Fedora-Rawhide-20180426.n.0/compose\n\ 53 | \ \n \n

Index of /compose/rawhide/Fedora-Rawhide-20180426.n.0/compose

\n\ 54 | 
\"Icon Name\
 55 |         \                                          Last modified\
 56 |         \      Size  Description
 Parent Directory                                                   -\
 59 |         \   \n\"[DIR]\" AtomicHost/                                   2018-04-26 11:02    - \
 61 |         \  \n\"[DIR]\" AtomicWorkstation/                            2018-04-26 11:02    - \
 63 |         \  \n\"[DIR]\" Container/                                    2018-04-26 13:06    - \
 65 |         \  \n\"[DIR]\" Everything/                                   2018-04-26 10:03    - \
 67 |         \  \n\"[DIR]\" Labs/\
 68 |         \                                         2018-04-26 11:22    -   \n\"[DIR]\" Modular/     \
 70 |         \                                 2018-04-26 10:06    -   \n\"[DIR]\" Server/                         \
 72 |         \              2018-04-26 10:06    -   \n\"\ Spins/                                   \
 74 |         \     2018-04-26 11:19    -   \n\"[DIR]\"\ Workstation/                              \
 76 |         \    2018-04-26 10:06    -   \n\"[DIR]\"\ metadata/                                    \
 78 |         \ 2018-04-26 10:18    -   \n
\n\n"} 79 | headers: 80 | appserver: [proxy01.phx2.fedoraproject.org] 81 | apptime: [D=11408] 82 | connection: [close] 83 | content-length: ['2043'] 84 | content-type: [text/html;charset=ISO-8859-1] 85 | date: ['Fri, 27 Apr 2018 10:56:10 GMT'] 86 | referrer-policy: [same-origin] 87 | server: [Apache/2.4.33 (Fedora)] 88 | strict-transport-security: [max-age=31536000; includeSubDomains; preload, max-age=31536000; 89 | includeSubDomains; preload] 90 | x-content-type-options: [nosniff] 91 | x-frame-options: [SAMEORIGIN] 92 | x-xss-protection: [1; mode=block] 93 | status: {code: 200, message: OK} 94 | - request: 95 | body: null 96 | headers: 97 | Connection: [close] 98 | Host: [!!python/unicode 'kojipkgs.fedoraproject.org'] 99 | User-Agent: [Python-urllib/2.7] 100 | method: GET 101 | uri: https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20180426.n.0/STATUS 102 | response: 103 | body: {string: !!python/unicode 'DOOMED 104 | 105 | '} 106 | headers: 107 | accept-ranges: [bytes] 108 | appserver: [proxy10.phx2.fedoraproject.org] 109 | apptime: [D=4028] 110 | connection: [close] 111 | content-length: ['7'] 112 | date: ['Fri, 27 Apr 2018 10:56:11 GMT'] 113 | last-modified: ['Thu, 26 Apr 2018 13:45:47 GMT'] 114 | referrer-policy: [same-origin] 115 | server: [Apache/2.4.33 (Fedora)] 116 | strict-transport-security: [max-age=31536000; includeSubDomains; preload, max-age=31536000; 117 | includeSubDomains; preload] 118 | x-content-type-options: [nosniff] 119 | x-frame-options: [SAMEORIGIN] 120 | x-xss-protection: [1; mode=block] 121 | status: {code: 200, message: OK} 122 | - request: 123 | body: null 124 | headers: 125 | Connection: [close] 126 | Host: [!!python/unicode 'kojipkgs.fedoraproject.org'] 127 | User-Agent: [Python-urllib/2.7] 128 | method: GET 129 | uri: https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20180426.n.0/compose/metadata/composeinfo.json 130 | response: 131 | body: {string: !!python/unicode ' 132 | 133 | 134 | 135 | 404 Not Found 136 | 137 | 138 | 139 |

Not Found

140 | 141 |

The requested URL /compose/rawhide/Fedora-Rawhide-20180426.n.0/compose/metadata/composeinfo.json 142 | was not found on this server.

143 | 144 | 145 | 146 | '} 147 | headers: 148 | appserver: [proxy01.phx2.fedoraproject.org] 149 | apptime: [D=5382] 150 | connection: [close] 151 | content-length: ['275'] 152 | content-type: [text/html; charset=iso-8859-1] 153 | date: ['Fri, 27 Apr 2018 10:56:12 GMT'] 154 | referrer-policy: [same-origin] 155 | server: [Apache/2.4.33 (Fedora)] 156 | strict-transport-security: [max-age=31536000; includeSubDomains; preload, max-age=31536000; 157 | includeSubDomains; preload] 158 | x-content-type-options: [nosniff] 159 | x-frame-options: [SAMEORIGIN] 160 | x-xss-protection: [1; mode=block] 161 | status: {code: 404, message: Not Found} 162 | - request: 163 | body: null 164 | headers: 165 | Connection: [close] 166 | Host: [!!python/unicode 'kojipkgs.fedoraproject.org'] 167 | User-Agent: [Python-urllib/2.7] 168 | method: GET 169 | uri: https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20180426.n.0/compose/metadata/composeinfo.json 170 | response: 171 | body: {string: !!python/unicode ' 172 | 173 | 174 | 175 | 404 Not Found 176 | 177 | 178 | 179 |

Not Found

180 | 181 |

The requested URL /compose/rawhide/Fedora-Rawhide-20180426.n.0/compose/metadata/composeinfo.json 182 | was not found on this server.

183 | 184 | 185 | 186 | '} 187 | headers: 188 | appserver: [proxy10.phx2.fedoraproject.org] 189 | apptime: [D=2398] 190 | connection: [close] 191 | content-length: ['275'] 192 | content-type: [text/html; charset=iso-8859-1] 193 | date: ['Fri, 27 Apr 2018 10:56:14 GMT'] 194 | referrer-policy: [same-origin] 195 | server: [Apache/2.4.33 (Fedora)] 196 | strict-transport-security: [max-age=31536000; includeSubDomains; preload, max-age=31536000; 197 | includeSubDomains; preload] 198 | x-content-type-options: [nosniff] 199 | x-frame-options: [SAMEORIGIN] 200 | x-xss-protection: [1; mode=block] 201 | status: {code: 404, message: Not Found} 202 | - request: 203 | body: null 204 | headers: 205 | Connection: [close] 206 | Host: [!!python/unicode 'kojipkgs.fedoraproject.org'] 207 | User-Agent: [Python-urllib/2.7] 208 | method: GET 209 | uri: https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20180426.n.0/compose/metadata/composeinfo.json 210 | response: 211 | body: {string: !!python/unicode ' 212 | 213 | 214 | 215 | 404 Not Found 216 | 217 | 218 | 219 |

Not Found

220 | 221 |

The requested URL /compose/rawhide/Fedora-Rawhide-20180426.n.0/compose/metadata/composeinfo.json 222 | was not found on this server.

223 | 224 | 225 | 226 | '} 227 | headers: 228 | appserver: [proxy01.phx2.fedoraproject.org] 229 | apptime: [D=3168] 230 | connection: [close] 231 | content-length: ['275'] 232 | content-type: [text/html; charset=iso-8859-1] 233 | date: ['Fri, 27 Apr 2018 10:56:15 GMT'] 234 | referrer-policy: [same-origin] 235 | server: [Apache/2.4.33 (Fedora)] 236 | strict-transport-security: [max-age=31536000; includeSubDomains; preload, max-age=31536000; 237 | includeSubDomains; preload] 238 | x-content-type-options: [nosniff] 239 | x-frame-options: [SAMEORIGIN] 240 | x-xss-protection: [1; mode=block] 241 | status: {code: 404, message: Not Found} 242 | - request: 243 | body: null 244 | headers: 245 | Connection: [close] 246 | Host: [!!python/unicode 'kojipkgs.fedoraproject.org'] 247 | User-Agent: [Python-urllib/2.7] 248 | method: GET 249 | uri: https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20180426.n.0/compose/metadata/composeinfo.json 250 | response: 251 | body: {string: !!python/unicode ' 252 | 253 | 254 | 255 | 404 Not Found 256 | 257 | 258 | 259 |

Not Found

260 | 261 |

The requested URL /compose/rawhide/Fedora-Rawhide-20180426.n.0/compose/metadata/composeinfo.json 262 | was not found on this server.

263 | 264 | 265 | 266 | '} 267 | headers: 268 | appserver: [proxy10.phx2.fedoraproject.org] 269 | apptime: [D=3385] 270 | connection: [close] 271 | content-length: ['275'] 272 | content-type: [text/html; charset=iso-8859-1] 273 | date: ['Fri, 27 Apr 2018 10:56:16 GMT'] 274 | referrer-policy: [same-origin] 275 | server: [Apache/2.4.33 (Fedora)] 276 | strict-transport-security: [max-age=31536000; includeSubDomains; preload, max-age=31536000; 277 | includeSubDomains; preload] 278 | x-content-type-options: [nosniff] 279 | x-frame-options: [SAMEORIGIN] 280 | x-xss-protection: [1; mode=block] 281 | status: {code: 404, message: Not Found} 282 | - request: 283 | body: null 284 | headers: 285 | Connection: [close] 286 | Host: [!!python/unicode 'kojipkgs.fedoraproject.org'] 287 | User-Agent: [Python-urllib/2.7] 288 | method: GET 289 | uri: https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20180426.n.0/compose/metadata/composeinfo.json 290 | response: 291 | body: {string: !!python/unicode ' 292 | 293 | 294 | 295 | 404 Not Found 296 | 297 | 298 | 299 |

Not Found

300 | 301 |

The requested URL /compose/rawhide/Fedora-Rawhide-20180426.n.0/compose/metadata/composeinfo.json 302 | was not found on this server.

303 | 304 | 305 | 306 | '} 307 | headers: 308 | appserver: [proxy01.phx2.fedoraproject.org] 309 | apptime: [D=2412] 310 | connection: [close] 311 | content-length: ['275'] 312 | content-type: [text/html; charset=iso-8859-1] 313 | date: ['Fri, 27 Apr 2018 10:56:18 GMT'] 314 | referrer-policy: [same-origin] 315 | server: [Apache/2.4.33 (Fedora)] 316 | strict-transport-security: [max-age=31536000; includeSubDomains; preload, max-age=31536000; 317 | includeSubDomains; preload] 318 | x-content-type-options: [nosniff] 319 | x-frame-options: [SAMEORIGIN] 320 | x-xss-protection: [1; mode=block] 321 | status: {code: 404, message: Not Found} 322 | - request: 323 | body: null 324 | headers: 325 | Connection: [close] 326 | Host: [!!python/unicode 'kojipkgs.fedoraproject.org'] 327 | User-Agent: [Python-urllib/2.7] 328 | method: GET 329 | uri: https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20180426.n.0/COMPOSE_ID 330 | response: 331 | body: {string: !!python/unicode 'Fedora-Rawhide-20180426.n.0'} 332 | headers: 333 | accept-ranges: [bytes] 334 | appserver: [proxy01.phx2.fedoraproject.org] 335 | apptime: [D=4327] 336 | connection: [close] 337 | content-length: ['27'] 338 | date: ['Wed, 02 May 2018 13:11:45 GMT'] 339 | last-modified: ['Thu, 26 Apr 2018 05:17:37 GMT'] 340 | referrer-policy: [same-origin] 341 | server: [Apache/2.4.33 (Fedora)] 342 | strict-transport-security: [max-age=31536000; includeSubDomains; preload, max-age=31536000; 343 | includeSubDomains; preload] 344 | x-content-type-options: [nosniff] 345 | x-frame-options: [SAMEORIGIN] 346 | x-xss-protection: [1; mode=block] 347 | status: {code: 200, message: OK} 348 | - request: 349 | body: null 350 | headers: 351 | Connection: [close] 352 | Host: [!!python/unicode 'kojipkgs.fedoraproject.org'] 353 | User-Agent: [Python-urllib/2.7] 354 | method: GET 355 | uri: https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20180426.n.0/compose/metadata/composeinfo.json 356 | response: 357 | body: {string: !!python/unicode ' 358 | 359 | 360 | 361 | 404 Not Found 362 | 363 | 364 | 365 |

Not Found

366 | 367 |

The requested URL /compose/rawhide/Fedora-Rawhide-20180426.n.0/compose/metadata/composeinfo.json 368 | was not found on this server.

369 | 370 | 371 | 372 | '} 373 | headers: 374 | appserver: [proxy10.phx2.fedoraproject.org] 375 | apptime: [D=7125] 376 | connection: [close] 377 | content-length: ['275'] 378 | content-type: [text/html; charset=iso-8859-1] 379 | date: ['Wed, 02 May 2018 13:11:47 GMT'] 380 | referrer-policy: [same-origin] 381 | server: [Apache/2.4.33 (Fedora)] 382 | strict-transport-security: [max-age=31536000; includeSubDomains; preload, max-age=31536000; 383 | includeSubDomains; preload] 384 | x-content-type-options: [nosniff] 385 | x-frame-options: [SAMEORIGIN] 386 | x-xss-protection: [1; mode=block] 387 | status: {code: 404, message: Not Found} 388 | - request: 389 | body: null 390 | headers: 391 | Connection: [close] 392 | Host: [!!python/unicode 'kojipkgs.fedoraproject.org'] 393 | User-Agent: [Python-urllib/2.7] 394 | method: GET 395 | uri: https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20180426.n.0/compose/metadata/composeinfo.json 396 | response: 397 | body: {string: !!python/unicode ' 398 | 399 | 400 | 401 | 404 Not Found 402 | 403 | 404 | 405 |

Not Found

406 | 407 |

The requested URL /compose/rawhide/Fedora-Rawhide-20180426.n.0/compose/metadata/composeinfo.json 408 | was not found on this server.

409 | 410 | 411 | 412 | '} 413 | headers: 414 | appserver: [proxy01.phx2.fedoraproject.org] 415 | apptime: [D=6760] 416 | connection: [close] 417 | content-length: ['275'] 418 | content-type: [text/html; charset=iso-8859-1] 419 | date: ['Wed, 02 May 2018 13:11:48 GMT'] 420 | referrer-policy: [same-origin] 421 | server: [Apache/2.4.33 (Fedora)] 422 | strict-transport-security: [max-age=31536000; includeSubDomains; preload, max-age=31536000; 423 | includeSubDomains; preload] 424 | x-content-type-options: [nosniff] 425 | x-frame-options: [SAMEORIGIN] 426 | x-xss-protection: [1; mode=block] 427 | status: {code: 404, message: Not Found} 428 | - request: 429 | body: null 430 | headers: 431 | Connection: [close] 432 | Host: [!!python/unicode 'kojipkgs.fedoraproject.org'] 433 | User-Agent: [Python-urllib/2.7] 434 | method: GET 435 | uri: https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20180426.n.0/compose/metadata/composeinfo.json 436 | response: 437 | body: {string: !!python/unicode ' 438 | 439 | 440 | 441 | 404 Not Found 442 | 443 | 444 | 445 |

Not Found

446 | 447 |

The requested URL /compose/rawhide/Fedora-Rawhide-20180426.n.0/compose/metadata/composeinfo.json 448 | was not found on this server.

449 | 450 | 451 | 452 | '} 453 | headers: 454 | appserver: [proxy01.phx2.fedoraproject.org] 455 | apptime: [D=1903] 456 | connection: [close] 457 | content-length: ['275'] 458 | content-type: [text/html; charset=iso-8859-1] 459 | date: ['Wed, 02 May 2018 13:11:50 GMT'] 460 | referrer-policy: [same-origin] 461 | server: [Apache/2.4.33 (Fedora)] 462 | strict-transport-security: [max-age=31536000; includeSubDomains; preload, max-age=31536000; 463 | includeSubDomains; preload] 464 | x-content-type-options: [nosniff] 465 | x-frame-options: [SAMEORIGIN] 466 | x-xss-protection: [1; mode=block] 467 | status: {code: 404, message: Not Found} 468 | - request: 469 | body: null 470 | headers: 471 | Connection: [close] 472 | Host: [!!python/unicode 'kojipkgs.fedoraproject.org'] 473 | User-Agent: [Python-urllib/2.7] 474 | method: GET 475 | uri: https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20180426.n.0/compose/metadata/composeinfo.json 476 | response: 477 | body: {string: !!python/unicode ' 478 | 479 | 480 | 481 | 404 Not Found 482 | 483 | 484 | 485 |

Not Found

486 | 487 |

The requested URL /compose/rawhide/Fedora-Rawhide-20180426.n.0/compose/metadata/composeinfo.json 488 | was not found on this server.

489 | 490 | 491 | 492 | '} 493 | headers: 494 | appserver: [proxy10.phx2.fedoraproject.org] 495 | apptime: [D=14735] 496 | connection: [close] 497 | content-length: ['275'] 498 | content-type: [text/html; charset=iso-8859-1] 499 | date: ['Tue, 24 Jul 2018 20:44:01 GMT'] 500 | referrer-policy: [same-origin] 501 | server: [Apache/2.4.33 (Fedora)] 502 | strict-transport-security: [max-age=31536000; includeSubDomains; preload, max-age=31536000; 503 | includeSubDomains; preload] 504 | x-content-type-options: [nosniff] 505 | x-frame-options: [SAMEORIGIN] 506 | x-xss-protection: [1; mode=block] 507 | status: {code: 404, message: Not Found} 508 | - request: 509 | body: null 510 | headers: 511 | Connection: [close] 512 | Host: [!!python/unicode 'kojipkgs.fedoraproject.org'] 513 | User-Agent: [Python-urllib/2.7] 514 | method: GET 515 | uri: https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20180426.n.0/compose/metadata/composeinfo.json 516 | response: 517 | body: {string: !!python/unicode ' 518 | 519 | 520 | 521 | 404 Not Found 522 | 523 | 524 | 525 |

Not Found

526 | 527 |

The requested URL /compose/rawhide/Fedora-Rawhide-20180426.n.0/compose/metadata/composeinfo.json 528 | was not found on this server.

529 | 530 | 531 | 532 | '} 533 | headers: 534 | appserver: [proxy01.phx2.fedoraproject.org] 535 | apptime: [D=3433] 536 | connection: [close] 537 | content-length: ['275'] 538 | content-type: [text/html; charset=iso-8859-1] 539 | date: ['Tue, 24 Jul 2018 20:48:20 GMT'] 540 | referrer-policy: [same-origin] 541 | server: [Apache/2.4.33 (Fedora)] 542 | strict-transport-security: [max-age=31536000; includeSubDomains; preload, max-age=31536000; 543 | includeSubDomains; preload] 544 | x-content-type-options: [nosniff] 545 | x-frame-options: [SAMEORIGIN] 546 | x-xss-protection: [1; mode=block] 547 | status: {code: 404, message: Not Found} 548 | version: 1 549 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | 2 | 1.3.0 3 | ----- 4 | 5 | Pull Requests 6 | 7 | - (@sayanchowdhury) #103, consumers: Fix the metadata handling for Rawhide messages 8 | https://github.com/fedora-infra/fedimg/pull/103 9 | - (@dustymabe) #104, fix trace for incorrect log function call 10 | https://github.com/fedora-infra/fedimg/pull/104 11 | - (@sayanchowdhury) #100, services.ec2: Check if the AMI is complete before proceeding 12 | https://github.com/fedora-infra/fedimg/pull/100 13 | - (@sayanchowdhury) #117, services.ec2: Fix the ImportError in ec2imguploader 14 | https://github.com/fedora-infra/fedimg/pull/117 15 | - (@sayanchowdhury) #112, services.ec2: Delete the source for every image urls 16 | https://github.com/fedora-infra/fedimg/pull/112 17 | - (@sinnykumari) #116, Don't use extend() when object is NoneType 18 | https://github.com/fedora-infra/fedimg/pull/116 19 | - (@sinnykumari) #111, Include AMIs to missing AWS ec2 regions 20 | https://github.com/fedora-infra/fedimg/pull/111 21 | - (@sayanchowdhury) #99, utils: Extract the raw first before processing 22 | https://github.com/fedora-infra/fedimg/pull/99 23 | 24 | Commits 25 | 26 | - cf00cbedd consumers: Fix the metadata handling for Rawhide messages 27 | https://github.com/fedora-infra/fedimg/commit/cf00cbedd 28 | - b3441f12f fix trace for incorrect log function call 29 | https://github.com/fedora-infra/fedimg/commit/b3441f12f 30 | - 9170b0d60 services.ec2: Check if the AMI is complete before proceeding 31 | https://github.com/fedora-infra/fedimg/commit/9170b0d60 32 | - d82da63f1 Include AMIs to missing AWS ec2 regions 33 | https://github.com/fedora-infra/fedimg/commit/d82da63f1 34 | - 722151591 Don't use extend() when object is NoneType 35 | https://github.com/fedora-infra/fedimg/commit/722151591 36 | - b3dd56e51 services.ec2: Fix the ImportError in ec2imguploader 37 | https://github.com/fedora-infra/fedimg/commit/b3dd56e51 38 | - bf6450c7c services.ec2: Delete the source for every image urls 39 | https://github.com/fedora-infra/fedimg/commit/bf6450c7c 40 | - 6ad790b65 utils: Extract the raw first before processing 41 | https://github.com/fedora-infra/fedimg/commit/6ad790b65 42 | 43 | 1.2.0 44 | ----- 45 | 46 | Pull Requests 47 | 48 | - (@sayanchowdhury) #85, consumer: Support F28 Atomic & Cloud messages 49 | https://github.com/fedora-infra/fedimg/pull/85 50 | - (@sayanchowdhury) #86, services.ec2: Fix the volume_type attribute 51 | https://github.com/fedora-infra/fedimg/pull/86 52 | - (@sayanchowdhury) #87, services.ec2: Add the missing push_notifications args 53 | https://github.com/fedora-infra/fedimg/pull/87 54 | - (@sayanchowdhury) #89, scripts: Remove script to delete the ec2 nodes 55 | https://github.com/fedora-infra/fedimg/pull/89 56 | - (@sayanchowdhury) #88, services.ec2: Send a image.upload message when completed 57 | https://github.com/fedora-infra/fedimg/pull/88 58 | - (@sayanchowdhury) #90, consumers: Ignore the Unsupported releases 59 | https://github.com/fedora-infra/fedimg/pull/90 60 | - (@sayanchowdhury) #92, fedimg: Include the image_url in fedmsg messages 61 | https://github.com/fedora-infra/fedimg/pull/92 62 | - (@sayanchowdhury) #93, scripts: Fix the script to send proper format of compose id 63 | https://github.com/fedora-infra/fedimg/pull/93 64 | - (@sayanchowdhury) #91, Change to use of consistent logging variables 65 | https://github.com/fedora-infra/fedimg/pull/91 66 | - (@sayanchowdhury) #96, fedimg: Fix the script function argument order 67 | https://github.com/fedora-infra/fedimg/pull/96 68 | - (@sayanchowdhury) #94, utils: Don't change the name of the image, breaks the get_ami.py layout 69 | https://github.com/fedora-infra/fedimg/pull/94 70 | 71 | Commits 72 | 73 | - b252bc483 consumer: Support F28 Atomic & Cloud messages 74 | https://github.com/fedora-infra/fedimg/commit/b252bc483 75 | - 15e41e51f logging: Change the source to consistent logging 76 | https://github.com/fedora-infra/fedimg/commit/15e41e51f 77 | - 8e8f22dd9 tests: Fix the tests for the consistent logging change 78 | https://github.com/fedora-infra/fedimg/commit/8e8f22dd9 79 | - 307ce0bb0 scripts: Remove script to delete the ec2 nodes 80 | https://github.com/fedora-infra/fedimg/commit/307ce0bb0 81 | - 161a0e15c services.ec2: Fix the volume_type attribute 82 | https://github.com/fedora-infra/fedimg/commit/161a0e15c 83 | - 7355a5cb7 services.ec2: Add the missing push_notifications args 84 | https://github.com/fedora-infra/fedimg/commit/7355a5cb7 85 | - 6ac912293 services.ec2: Send a image.upload message when completed 86 | https://github.com/fedora-infra/fedimg/commit/6ac912293 87 | - 7fc9c32c4 consumers: Ignore the Unsupported releases 88 | https://github.com/fedora-infra/fedimg/commit/7fc9c32c4 89 | - e21f70dbc fedimg: Include the image_url in fedmsg messages 90 | https://github.com/fedora-infra/fedimg/commit/e21f70dbc 91 | - 3b9a86bdb scripts: Fix the script to send proper format of compose id 92 | https://github.com/fedora-infra/fedimg/commit/3b9a86bdb 93 | - de4af7935 fedimg: Fix the script function args 94 | https://github.com/fedora-infra/fedimg/commit/de4af7935 95 | - 3946ed1c3 utils: Don't change the name of the image, breaks the get_ami.py layout 96 | https://github.com/fedora-infra/fedimg/commit/3946ed1c3 97 | 98 | 1.1.0 99 | ----- 100 | 101 | Pull Requests 102 | 103 | - (@sayanchowdhury) #66, consumer: fedfind has deprecated `get_release_cid` method. 104 | https://github.com/fedora-infra/fedimg/pull/66 105 | - (@sayanchowdhury) #67, scripts: Fix the manual upload trigger script 106 | https://github.com/fedora-infra/fedimg/pull/67 107 | - (@sayanchowdhury) #68, fedimg: Remove redundant ec2.py file 108 | https://github.com/fedora-infra/fedimg/pull/68 109 | - (@sayanchowdhury) #76, scripts: Update the trigger_upload, and remove redundant code 110 | https://github.com/fedora-infra/fedimg/pull/76 111 | - (@sayanchowdhury) #75, Initial tests for fedimg 112 | https://github.com/fedora-infra/fedimg/pull/75 113 | - (@euank) #77, Fix numerous dependency issues, fix broken unit tests, add travis ci config 114 | https://github.com/fedora-infra/fedimg/pull/77 115 | - (@sayanchowdhury) #79, services.ec2: Change the bucket name to more related to fedimg 116 | https://github.com/fedora-infra/fedimg/pull/79 117 | - (@puiterwijk) #80, Error out if starting the task failed 118 | https://github.com/fedora-infra/fedimg/pull/80 119 | - (@sayanchowdhury) #70, fedimg.ec2: Add metadata to the `image.copy` fedmsg message 120 | https://github.com/fedora-infra/fedimg/pull/70 121 | - (@sayanchowdhury) #81, services.ec2: Deprecate the PV images 122 | https://github.com/fedora-infra/fedimg/pull/81 123 | - (@sayanchowdhury) #69, fedimg.ec2: Add the support for Elastic Network Adapter 124 | https://github.com/fedora-infra/fedimg/pull/69 125 | - (@sayanchowdhury) #82, uploader: Set push_notifications to True when automatic upload 126 | https://github.com/fedora-infra/fedimg/pull/82 127 | - (@sayanchowdhury) #84, Update the trigger_upload.py script to add push_notifications 128 | https://github.com/fedora-infra/fedimg/pull/84 129 | 130 | Commits 131 | 132 | - 84d0d69ef consumer: fedfind has deprecated `get_release_cid` method. 133 | https://github.com/fedora-infra/fedimg/commit/84d0d69ef 134 | - 7bdd06f56 scripts: Fix the manual upload trigger script 135 | https://github.com/fedora-infra/fedimg/commit/7bdd06f56 136 | - 33a86b79b fedimg: Remove redundant ec2.py file 137 | https://github.com/fedora-infra/fedimg/commit/33a86b79b 138 | - b0fa1c4f9 tests: Fix the test for consumers 139 | https://github.com/fedora-infra/fedimg/commit/b0fa1c4f9 140 | - e082ad464 tests: Add tests for the fedimg.uploader 141 | https://github.com/fedora-infra/fedimg/commit/e082ad464 142 | - 1788e9e6c tests: Add the tests for the fedimg.utils 143 | https://github.com/fedora-infra/fedimg/commit/1788e9e6c 144 | - ed5ddf5bf tests: Add a few more tests for utils. utils cov at 78% 145 | https://github.com/fedora-infra/fedimg/commit/ed5ddf5bf 146 | - 57d4414c0 tests: Add tests for fedimg.utils, coverage 100% 147 | https://github.com/fedora-infra/fedimg/commit/57d4414c0 148 | - 5ccc75652 scripts: Remove redundant imports in the trigger_upload script 149 | https://github.com/fedora-infra/fedimg/commit/5ccc75652 150 | - 84cf48443 docs: Update the README.md for the trigger upload script 151 | https://github.com/fedora-infra/fedimg/commit/84cf48443 152 | - 93e2358fc tests: Fix the copyright years in the test files 153 | https://github.com/fedora-infra/fedimg/commit/93e2358fc 154 | - 0bcb54661 tests: Use assertIs method to check for boolean 155 | https://github.com/fedora-infra/fedimg/commit/0bcb54661 156 | - b6b651f8a tests: Remove the redundant code 157 | https://github.com/fedora-infra/fedimg/commit/b6b651f8a 158 | - 57a2eec93 tests: Make a stronger assertion if the urls is made to atomic and cloud base 159 | https://github.com/fedora-infra/fedimg/commit/57a2eec93 160 | - 9b77f95f9 tests: Change the assertions to use self.assertIs 161 | https://github.com/fedora-infra/fedimg/commit/9b77f95f9 162 | - 0f3056847 tests: include 'vcr' dependency in setup.py 163 | https://github.com/fedora-infra/fedimg/commit/0f3056847 164 | - 1f29c9389 setup.py: specify test suite to use 165 | https://github.com/fedora-infra/fedimg/commit/1f29c9389 166 | - 9c29204c1 setup.py: use consistent quoting for dependencies 167 | https://github.com/fedora-infra/fedimg/commit/9c29204c1 168 | - 326ab9ef8 setup.py: add 'toml' dependency 169 | https://github.com/fedora-infra/fedimg/commit/326ab9ef8 170 | - b206d76da setup.py: add 'fedfind' dependency 171 | https://github.com/fedora-infra/fedimg/commit/b206d76da 172 | - 32533b038 tests: don't validate signatures for mockhub 173 | https://github.com/fedora-infra/fedimg/commit/32533b038 174 | - 4dc97ac3f tests: add travis.yml 175 | https://github.com/fedora-infra/fedimg/commit/4dc97ac3f 176 | - 5adef1a75 docs/devel: update test running instructions 177 | https://github.com/fedora-infra/fedimg/commit/5adef1a75 178 | - 87f470edb fedimg.ec2: Add metadata to the `image.copy` fedmsg message 179 | https://github.com/fedora-infra/fedimg/commit/87f470edb 180 | - 0ddfd41e9 fedimg.ec2: Add the support for Elastic Network Adapter 181 | https://github.com/fedora-infra/fedimg/commit/0ddfd41e9 182 | - 44ac7b8b3 services.ec2: Change the bucket name to more related to fedimg 183 | https://github.com/fedora-infra/fedimg/commit/44ac7b8b3 184 | - e5df4686f Error out if starting the task failed 185 | https://github.com/fedora-infra/fedimg/commit/e5df4686f 186 | - 744b729ce services.ec2: Deprecate the PV images 187 | https://github.com/fedora-infra/fedimg/commit/744b729ce 188 | - e1607cb5a uploader: Set push_notifications to True when automatic upload 189 | https://github.com/fedora-infra/fedimg/commit/e1607cb5a 190 | - 82b63d886 fedimg: Fix the trigger_upload script to include push_notifications arg 191 | https://github.com/fedora-infra/fedimg/commit/82b63d886 192 | - 0f8d6c08f readme: Update the trigger_upload usage in README file 193 | https://github.com/fedora-infra/fedimg/commit/0f8d6c08f 194 | - 0dc560f13 scripts: Make the -p arg optional 195 | https://github.com/fedora-infra/fedimg/commit/0dc560f13 196 | - 43103f59b scripts: Move logic inside the main function 197 | https://github.com/fedora-infra/fedimg/commit/43103f59b 198 | 199 | 1.0.1 200 | ----- 201 | 202 | Pull Requests 203 | 204 | - (@sayanchowdhury) #65, Fix the invalid syntax issue 205 | https://github.com/fedora-infra/fedimg/pull/65 206 | 207 | Commits 208 | 209 | - 8f0a92d4d utils: Fix the invalid syntax issue 210 | https://github.com/fedora-infra/fedimg/commit/8f0a92d4d 211 | 212 | 1.0.0 213 | ----- 214 | 215 | Pull Requests 216 | 217 | - (@sayanchowdhury) #61, fedimg: Migrate the CHANGELOG to rst from markdown 218 | https://github.com/fedora-infra/fedimg/pull/61 219 | - (@sayanchowdhury) #62, Break fedimg into multiple components so that it is easier to maintain the AMIs 220 | https://github.com/fedora-infra/fedimg/pull/62 221 | 222 | Commits 223 | 224 | - f1d54ee2f fedimg: Migrate the CHANGELOG to rst from md 225 | https://github.com/fedora-infra/fedimg/commit/f1d54ee2f 226 | - 2b7b49f8e ec2: Modularize the the structure of the ec2 227 | https://github.com/fedora-infra/fedimg/commit/2b7b49f8e 228 | - a52001442 ec2: Write a publisher that would make the images & the snapshot public 229 | https://github.com/fedora-infra/fedimg/commit/a52001442 230 | - ebda1518f ec2: Add a few methods to manage the AMI/Snapshots. 231 | https://github.com/fedora-infra/fedimg/commit/ebda1518f 232 | - 8a5fa90e3 ec2: Remove the code from the __init__.py file. 233 | https://github.com/fedora-infra/fedimg/commit/8a5fa90e3 234 | - 542ff1239 consumers: Add three different consumers for prod, stg & dev 235 | https://github.com/fedora-infra/fedimg/commit/542ff1239 236 | - c794d79fe config: Migrate to mange the configuration using toml 237 | https://github.com/fedora-infra/fedimg/commit/c794d79fe 238 | - 6852eb6c2 tests: Start fixing the tests using pytest 239 | https://github.com/fedora-infra/fedimg/commit/6852eb6c2 240 | - 897e55db3 tests: Fix the tests for fedimg.util 241 | https://github.com/fedora-infra/fedimg/commit/897e55db3 242 | - 952845601 fedimg: Modify setup according to dev, prod, and staging consumers 243 | https://github.com/fedora-infra/fedimg/commit/952845601 244 | - 13d13ee6a ec2: Create a script to initiate the complete process 245 | https://github.com/fedora-infra/fedimg/commit/13d13ee6a 246 | - b0b356966 ec2: Move util to utils, and fix the imports 247 | https://github.com/fedora-infra/fedimg/commit/b0b356966 248 | - 4a081a7b6 consumer: Implement the util methods (they raise NotImplementError now) 249 | https://github.com/fedora-infra/fedimg/commit/4a081a7b6 250 | - dbf341266 tests: Fix the test cases for the consumer 251 | https://github.com/fedora-infra/fedimg/commit/dbf341266 252 | - ea14ae1ef fedimg: Add logging statements to the source 253 | https://github.com/fedora-infra/fedimg/commit/ea14ae1ef 254 | - 829bfbc70 fedimg: Fix the config parsing for the general configurations 255 | https://github.com/fedora-infra/fedimg/commit/829bfbc70 256 | - ad2c85df6 utils: Implement the methods `external_run_command` & `get_source_from_image` 257 | https://github.com/fedora-infra/fedimg/commit/ad2c85df6 258 | - a54150605 services.ec2: Add methods to publish and copy the images to other regions 259 | https://github.com/fedora-infra/fedimg/commit/a54150605 260 | - b1edcbb84 services.ec2: Update ec2initiate with the publisher code 261 | https://github.com/fedora-infra/fedimg/commit/b1edcbb84 262 | - a1cf41b0d serices.ec2: Add a push_notifications flag to control msg bus push 263 | https://github.com/fedora-infra/fedimg/commit/a1cf41b0d 264 | - 7283f2e2c services.ec2: Update the publisher to send messages to fedmsg 265 | https://github.com/fedora-infra/fedimg/commit/7283f2e2c 266 | - f05c59c26 consumers: Add documentation to FedimgConsumer 267 | https://github.com/fedora-infra/fedimg/commit/f05c59c26 268 | - 6050d2cf4 uploader: Add documentation to the fedimg uploader 269 | https://github.com/fedora-infra/fedimg/commit/6050d2cf4 270 | - 9d885c461 services.ec2: Change the initiate to just handle upload of the images 271 | https://github.com/fedora-infra/fedimg/commit/9d885c461 272 | - 580759f29 services.ec2: Fix the sample config file 273 | https://github.com/fedora-infra/fedimg/commit/580759f29 274 | - 150a475c5 services.ec2: Create a utility method to create get the image_name 275 | https://github.com/fedora-infra/fedimg/commit/150a475c5 276 | - fff538fe6 uploader: Fix the BASE_REGION in the uploader method 277 | https://github.com/fedora-infra/fedimg/commit/fff538fe6 278 | - d5acfe690 services.ec2: Fix the downloading and uploading of the source 279 | https://github.com/fedora-infra/fedimg/commit/d5acfe690 280 | - 84826fa57 messenger: Change the name of the method that pushes fedmsg messages 281 | https://github.com/fedora-infra/fedimg/commit/84826fa57 282 | - c0cab6176 services.ec2: Fix the issues with the EC2ImgUploader 283 | https://github.com/fedora-infra/fedimg/commit/c0cab6176 284 | - 600cc7c7b services.ec2: Fix the issues in the EC2ImagePublisher 285 | https://github.com/fedora-infra/fedimg/commit/600cc7c7b 286 | - 9b37e0dd9 services.ec2: Attach EC2 copy to other regions into uploader 287 | https://github.com/fedora-infra/fedimg/commit/9b37e0dd9 288 | - 211787e6f services.ec2: Add documentation for EC2ImgUploader 289 | https://github.com/fedora-infra/fedimg/commit/211787e6f 290 | - dfd752a3c services.ec2: Change the return data from published images 291 | https://github.com/fedora-infra/fedimg/commit/dfd752a3c 292 | - 9f3eb7dfd services.ec2: Fix the ec2 image publisher 293 | https://github.com/fedora-infra/fedimg/commit/9f3eb7dfd 294 | - cd6a85fdb services.ec2: Change the bucket name according to Amazon S3 guidelines 295 | https://github.com/fedora-infra/fedimg/commit/cd6a85fdb 296 | - d56d74447 services.ec2: Delete the resources when failed or completed 297 | https://github.com/fedora-infra/fedimg/commit/d56d74447 298 | - 71cd44d05 services.ec2: Add the retry logic to fetch the snapshot details 299 | https://github.com/fedora-infra/fedimg/commit/71cd44d05 300 | - 0b826f42a config: Change the config to multiple lines 301 | https://github.com/fedora-infra/fedimg/commit/0b826f42a 302 | - 952500c8f fedimg: Replace the logger name to __name__ 303 | https://github.com/fedora-infra/fedimg/commit/952500c8f 304 | - 38f53f878 services.ec2: Fix the utility methods 305 | https://github.com/fedora-infra/fedimg/commit/38f53f878 306 | - e42754240 services.ec2: Return empty if the download fails 307 | https://github.com/fedora-infra/fedimg/commit/e42754240 308 | - 1df64ecc6 utils: @pypingou suggested to simplify the lambda statement 309 | https://github.com/fedora-infra/fedimg/commit/1df64ecc6 310 | - f39c64a23 utils: Add the shell=True params for the shell params 311 | https://github.com/fedora-infra/fedimg/commit/f39c64a23 312 | - db17bb599 config: Move the config in a single configuration file 313 | https://github.com/fedora-infra/fedimg/commit/db17bb599 314 | - 6855d49ae tests: Remove the code related to vcr 315 | https://github.com/fedora-infra/fedimg/commit/6855d49ae 316 | 317 | 0.7.5 318 | ----- 319 | 320 | Pull Requests 321 | 322 | - (@sayanchowdhury) #60, Snapshots in non us-east-1 don't get public 323 | https://github.com/fedora-infra/fedimg/pull/60 324 | 325 | Commits 326 | 327 | - d6f5457ff services.ec2: Make the snapshots in other regions public after run 328 | https://github.com/fedora-infra/fedimg/commit/d6f5457ff 329 | - c5d6d2820 services.ec2: Use the alternate driver to query the regions 330 | https://github.com/fedora-infra/fedimg/commit/c5d6d2820 331 | - 646a037a2 services.ec2: Add comment for the hack done for snapshots 332 | https://github.com/fedora-infra/fedimg/commit/646a037a2 333 | 334 | 0.7.4 335 | ----- 336 | 337 | Pull Requests 338 | 339 | - (@sayanchowdhury) #59, Retry till snapshot is public & Fix error handling. 340 | https://github.com/fedora-infra/fedimg/pull/59 341 | 342 | Commits 343 | 344 | - 0b3e6a0ca services.ec2: Fix the error handling in the EC2 Service 345 | https://github.com/fedora-infra/fedimg/commit/0b3e6a0ca 346 | - d1f2d873e services.ec2: Keep retrying for making the snapshot public 347 | https://github.com/fedora-infra/fedimg/commit/d1f2d873e 348 | 349 | 0.7.3 350 | ----- 351 | 352 | Pull Requests 353 | 354 | - (@sayanchowdhury) #58, services.ec2: Log if the image was successfully made public 355 | https://github.com/fedora-infra/fedimg/pull/58 356 | 357 | Commits 358 | 359 | - 1acc5904d services.ec2: Log if the image was successfully made public 360 | https://github.com/fedora-infra/fedimg/commit/1acc5904d 361 | 362 | 0.7.2 363 | ----- 364 | 365 | Pull Requests 366 | 367 | - (@sayanchowdhury) #57, cron: Update the cron according to the upgrade notes 368 | https://github.com/fedora-infra/fedimg/pull/57 369 | 370 | Commits 371 | 372 | - a0de6182f cron: Update the cron according to the upgrade notes 373 | https://github.com/fedora-infra/fedimg/commit/a0de6182f 374 | 375 | 0.7.1 376 | ----- 377 | 378 | Pull Requests 379 | 380 | - (@sayanchowdhury) #53, Drop the 'os' and 'ver' from the configuration file. Related to #46 381 | https://github.com/fedora-infra/fedimg/pull/53 382 | - (@sayanchowdhury) #55, Make the snapshots public so that AMIs can be copied to different accounts 383 | https://github.com/fedora-infra/fedimg/pull/55 384 | 385 | Commits 386 | 387 | - 985f9d8de Drop the 'os' and 'ver' from the configuration file. Related to #46 388 | https://github.com/fedora-infra/fedimg/commit/985f9d8de 389 | - b25cc4f14 Make the snapshots public so that AMIs can be copied to different accounts 390 | https://github.com/fedora-infra/fedimg/commit/b25cc4f14 391 | 392 | 0.7 393 | --- 394 | 395 | Pull Requests 396 | 397 | - (@ralphbean) #41, Setup logging for cronjob 398 | https://github.com/fedora-infra/fedimg/pull/41 399 | - (@coolsvap) #44, Update typos 400 | https://github.com/fedora-infra/fedimg/pull/44 401 | - (@nishant-mor) #46, Dropped 'os' and 'ver' from the AWS_AMIS config 402 | https://github.com/fedora-infra/fedimg/pull/46 403 | - (@ralphbean) #47, Pungi4 fixes. 404 | https://github.com/fedora-infra/fedimg/pull/47 405 | - (@ralphbean) #49, Add a nice log statement at the beginning stating what we're going to upload. 406 | https://github.com/fedora-infra/fedimg/pull/49 407 | - (@sayanchowdhury) #50, Fix to include nightly atomic uploads 408 | https://github.com/fedora-infra/fedimg/pull/50 409 | - (@sayanchowdhury) #51, Migrate fedimg to compose based 410 | https://github.com/fedora-infra/fedimg/pull/51 411 | - (@sayanchowdhury) #52, Send image raw_url to fedmsg instead of the build_name 412 | https://github.com/fedora-infra/fedimg/pull/52 413 | 414 | Commits 415 | 416 | - 60aa36b2a Setup logging for cronjob 417 | https://github.com/fedora-infra/fedimg/commit/60aa36b2a 418 | - 511497384 Update typo in GCE service 419 | https://github.com/fedora-infra/fedimg/commit/511497384 420 | - 6b9c3210d Update typo in rackspace service 421 | https://github.com/fedora-infra/fedimg/commit/6b9c3210d 422 | - f470cebef Update typo in hp service 423 | https://github.com/fedora-infra/fedimg/commit/f470cebef 424 | - 5a1c7ab51 Dropped 'os' and 'ver' from the AWS_AMIS config 425 | https://github.com/fedora-infra/fedimg/commit/5a1c7ab51 426 | - 05452ed71 ex2.py : Added new format of AWS_AMIS config 427 | https://github.com/fedora-infra/fedimg/commit/05452ed71 428 | - 20805fdd9 s/yum/dnf/ 429 | https://github.com/fedora-infra/fedimg/commit/20805fdd9 430 | - aec998075 Pungi4 fixes. 431 | https://github.com/fedora-infra/fedimg/commit/aec998075 432 | - 9d4873858 Add a nice log statement at the beginning stating what we're going to upload. 433 | https://github.com/fedora-infra/fedimg/commit/9d4873858 434 | - 156190880 Fix to include F24 nightly atomic uploads 435 | https://github.com/fedora-infra/fedimg/commit/156190880 436 | - 335d2236a Migrate fedimg from koji-based to compose-based 437 | https://github.com/fedora-infra/fedimg/commit/335d2236a 438 | - 7ae44d715 Minor fixes in the fedmsg consumer 439 | https://github.com/fedora-infra/fedimg/commit/7ae44d715 440 | - a3a2300ab Change KojiConsumer to FedimgConsumer 441 | https://github.com/fedora-infra/fedimg/commit/a3a2300ab 442 | - 1d0af12c1 Update the documenation to install fedfind while setting up 443 | https://github.com/fedora-infra/fedimg/commit/1d0af12c1 444 | - 0e199c95d An small indentation typo resulting into major issue 445 | https://github.com/fedora-infra/fedimg/commit/0e199c95d 446 | - 4f9e932f3 Send image raw_url to fedimg instead of the build_name 447 | https://github.com/fedora-infra/fedimg/commit/4f9e932f3 448 | 449 | 0.6.4 450 | ----- 451 | 452 | Commits 453 | 454 | - f94ade23f Typofix. 455 | https://github.com/fedora-infra/fedimg/commit/f94ade23f 456 | 457 | 0.6.3 458 | ----- 459 | 460 | Pull Requests 461 | 462 | - (@ralphbean) #33, Rearrange image.test fedmsg alerts. 463 | https://github.com/fedora-infra/fedimg/pull/33 464 | - (@ralphbean) #40, Use new-style of accessing ec2 drivers. 465 | https://github.com/fedora-infra/fedimg/pull/40 466 | 467 | Commits 468 | 469 | - b5daa8ea3 Ignore eggs dir. 470 | https://github.com/fedora-infra/fedimg/commit/b5daa8ea3 471 | - 99f51c92a Rearrange image.test fedmsg alerts. 472 | https://github.com/fedora-infra/fedimg/commit/99f51c92a 473 | - 677410c59 Add a script that lists the latest AMIs from datagrepper. 474 | https://github.com/fedora-infra/fedimg/commit/677410c59 475 | - 368816860 Closes #35, can kill any instance running more than 2 hours. 476 | https://github.com/fedora-infra/fedimg/commit/368816860 477 | - 05b540390 Fixes the typo in the command name. 478 | https://github.com/fedora-infra/fedimg/commit/05b540390 479 | - 9c230af02 Use new-style of accessing ec2 drivers. 480 | https://github.com/fedora-infra/fedimg/commit/9c230af02 481 | - f891dccc9 Remove CHANGELOG header. 482 | https://github.com/fedora-infra/fedimg/commit/f891dccc9 483 | - ddbb82523 Remove the spec file. We keep it in Fedora dist-git. 484 | https://github.com/fedora-infra/fedimg/commit/ddbb82523 485 | 486 | 487 | 0.6 488 | --- 489 | 490 | General 491 | 492 | - Use a single threadpool for all uploads to avoid leaking threads 493 | - Prevent major IndexError when checking Koji tasks that don't have raw.xz outputs 494 | - Increase number of fedmsg endpoints 495 | 496 | EC2Service 497 | 498 | - Use larger and more powerful instance types for utility and test instances 499 | - Typofix when naming PV images 500 | 501 | Docs 502 | 503 | - Add some basic contributor docs 504 | 505 | 506 | 0.5 507 | --- 508 | 509 | EC2Service 510 | 511 | - Use 7 GB volume size rather than 3 GB for now, since atomic images come out 512 | to be 6.1 GB 513 | - Implement gp2 volume type uploads 514 | - Image name now includes volume type 515 | - Simplify consumer filter code, eliminating 32 bit stuff for now 516 | - Add build name, virtualization type, and volume type to 'extra' 517 | dict in fedmsgs 518 | 519 | Tests 520 | 521 | - Fix up consumer test code 522 | - Add additional consumer tests to test build filter code 523 | 524 | Docs 525 | 526 | - Add info about volume size configuration 527 | - Tested on F21 528 | - Improve index page 529 | - Bring installation info up-to-date 530 | 531 | Misc 532 | - Commit atomic test script, to go with base test script 533 | - Reduce description in setup.py 534 | 535 | 536 | 0.4 537 | --- 538 | 539 | EC2Service 540 | 541 | - Fix alternate destinations not being set properly during image copy 542 | - Split util and test AMIs into dedicated lists 543 | - Allow for URL redirection while curling raw.xz image 544 | - Simplified registration AKI selection process 545 | - Major refactoring to allow for future expansion into many different types of AMIs 546 | - Uploads are now multithreaded 547 | - Volume size options added to config options 548 | - Better logging 549 | - Close a dangling SSH connection (thanks, threebean!) 550 | - Fix bug that caused only the first two AMIs to be made public 551 | 552 | Tests 553 | 554 | - Fix broken consumer test 555 | - Committed `uploadtest.py` for doing EC2Service test runs during development 556 | 557 | Docs 558 | 559 | - Update messaging docs 560 | - Add table of AMI types to EC2Service docs 561 | - Add AMI config format info 562 | 563 | Misc 564 | 565 | - Removed extraneous EC2Service-specific stuff from other service files 566 | - Better commenting 567 | 568 | 569 | 0.3.2 570 | ----- 571 | 572 | - Use fedmsg logging utilities 573 | - Convert old print statements to logging 574 | 575 | 576 | 0.3.1 577 | ----- 578 | 579 | - Cycle through and make copied AMIs public after uploads complete 580 | - Register AMI with description containing build name of source image file 581 | - Report AMI Ids when emitting related fedmsgs 582 | - Make sure all AMIs have a matching numerical extension across regions 583 | - Clean up a little EC2Service code 584 | - Typofixes, etc 585 | 586 | 587 | 0.3 588 | --- 589 | 590 | - Add utility function to get virtualization type for EC2 AMI registration 591 | - Make AMIs public after being tested and cpied 592 | - Tweaks to layout of config file 593 | - Only use 64 bit EBS utility instances 594 | - Remove hardcoded username 595 | - Rename some variables to be clearer 596 | - add clean_up_on_failure and delete_image_on_failure config options 597 | - Improve exception handling 598 | - Make sure snapshot is deleted on cleanup if no image has been registered 599 | - Add some starter tests 600 | - Move around some processes to more appropriate files 601 | - Don't attempt to provide an AKI when registering an image as HVM 602 | - Fix root device name for PV vs. HVM 603 | - Serious PEP 8 fixes 604 | - Fix up duplicate image name prevention code 605 | - Various typofixes and code cleanup 606 | 607 | 608 | 0.2.6 609 | ----- 610 | 611 | - Use proper buildroot macro in spec file 612 | - Preserve file timestamps when copying in spec file 613 | - Do not make library files executable, and don't give them shebangs 614 | - Add license headers to all Python files 615 | 616 | 617 | 0.2.5 618 | ----- 619 | 620 | - Remove coding from fedmsg.d/fedimg.py since it seems to make it executable 621 | - Make init file executable in spec install section, as well 622 | 623 | 624 | 0.2.4 625 | ----- 626 | 627 | - Shorten spec summary and remove trailing dot 628 | - Add shebang and coding to top of fedimg init file 629 | - Remove shebang from fedmsg.d/fedimg.py 630 | - Make all necessary fedimg files executable in spec install section 631 | 632 | 633 | 0.2.3 634 | ----- 635 | 636 | - Better IAM profile name example in the config 637 | - Addition to docs: section about setting up the config file 638 | - Fix strange saces and add missing comma to setup.py classifiers section 639 | 640 | 641 | 0.2.2 642 | ----- 643 | 644 | - Include .pyc and .pyo files for consumer in /etc/fedmsg.d/ 645 | - Add missing comma 646 | 647 | 648 | 0.2.1 649 | ----- 650 | 651 | - Fix `packages` argument in setup.py to take `find_packages()` 652 | 653 | 654 | 0.2.0 655 | ----- 656 | 657 | - Initial RPM release to Fedora 658 | - setup.py improvements 659 | - Config file is now read from /etc/fedimg.cfg 660 | - PEP 8 fixes 661 | 662 | 663 | 0.1.0 664 | ----- 665 | 666 | - Initial PyPI release 667 | 668 | --------------------------------------------------------------------------------