├── .coveragerc ├── .gitignore ├── .gitreview ├── .mailmap ├── .stestr.conf ├── .zuul.yaml ├── CONTRIBUTING.rst ├── LICENSE ├── MANIFEST.in ├── README.rst ├── doc └── source │ ├── conf.py │ ├── contributing.rst │ ├── index.rst │ ├── readme.rst │ ├── specs │ └── template.rst ├── requirements.txt ├── setup.cfg ├── setup.py ├── specs ├── .gitignore ├── backlog-template.rst ├── backlog │ ├── email-notification.rst │ ├── index.rst │ └── persistent-transport.rst ├── index.rst ├── juno │ └── index.rst ├── kilo │ ├── define-a-wire-protocol.rst │ ├── fifo-optional.rst │ ├── index.rst │ ├── notification-api.rst │ ├── persistent-transport.rst │ └── storage-capabilities.rst ├── liberty │ ├── finish-client-support-for-v1.1-features.rst │ ├── index.rst │ ├── policy_support.rst │ ├── pre-signed-url.rst │ └── tests-refactoring.rst ├── mitaka │ ├── index.rst │ ├── more-reserved-queue-attributes.rst │ └── websocket-binary-support.rst ├── new-storage-driver-template.rst ├── newton │ ├── index.rst │ ├── lazy-queues-in-subscriptions.rst │ ├── mistral-notifications.rst │ └── subscription-confirmation-support.rst ├── ocata │ ├── index.rst │ ├── osprofiler.rst │ ├── purge-queue.rst │ └── swift-storage.rst ├── pike │ ├── dead_letter_queue.rst │ ├── index.rst │ └── notification-delivery-policy.rst ├── queens │ ├── delayed-queues.rst │ ├── index.rst │ ├── remove-pool-group-from-zaqar.rst │ ├── support-md5-of-body.rst │ ├── support-more-backoff-functions.rst │ └── support-redis-as-mgmt-storage-backend.rst ├── rocky │ ├── index.rst │ ├── queue-filter-support.rst │ ├── remove-format-constraint-of-client-id.rst │ └── subscription-filtering-tags.rst ├── stein │ ├── delete-message-with-claim-id.rst │ ├── email-notification-by-internal-tool.rst │ ├── index.rst │ ├── introduce-topic-resource-for-notification.rst │ └── remove-pool-group-totally.rst ├── template.rst ├── ussuri │ ├── index.rst │ └── query-queues-with-count.rst ├── victoria │ ├── encrypted-messages-in-queue.rst │ └── index.rst ├── wallaby │ ├── index.rst │ └── support-extra-specs-to-subscription-confirming.rst ├── xena │ └── index.rst └── yoga │ ├── index.rst │ └── remove-v1-api-code-from-zaqar.rst ├── test-requirements.txt ├── tests ├── __init__.py └── test_title.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = zaqar-specs 4 | omit = zaqar-specs/tests/*,zaqar-specs/openstack/* 5 | 6 | [report] 7 | ignore_errors = True 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | 21 | # Installer logs 22 | pip-log.txt 23 | 24 | # Unit test / coverage reports 25 | .coverage 26 | .tox 27 | nosetests.xml 28 | .testrepository 29 | .stestr/ 30 | 31 | # Translations 32 | *.mo 33 | 34 | # Mr Developer 35 | .mr.developer.cfg 36 | .project 37 | .pydevproject 38 | 39 | # Complexity 40 | output/*.html 41 | output/*/index.html 42 | 43 | # Sphinx 44 | doc/build 45 | 46 | # pbr generates these 47 | AUTHORS 48 | ChangeLog 49 | 50 | # Editors 51 | *~ 52 | .*.swp 53 | -------------------------------------------------------------------------------- /.gitreview: -------------------------------------------------------------------------------- 1 | [gerrit] 2 | host=review.opendev.org 3 | port=29418 4 | project=openstack/zaqar-specs.git 5 | -------------------------------------------------------------------------------- /.mailmap: -------------------------------------------------------------------------------- 1 | # Format is: 2 | # 3 | # -------------------------------------------------------------------------------- /.stestr.conf: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | test_path=./tests 3 | top_dir=./ 4 | -------------------------------------------------------------------------------- /.zuul.yaml: -------------------------------------------------------------------------------- 1 | - project: 2 | templates: 3 | - openstack-specs-jobs 4 | check: 5 | jobs: 6 | - openstack-tox-py38 7 | gate: 8 | jobs: 9 | - openstack-tox-py38 10 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | ============================================= 2 | Contributing to: zaqar-specs 3 | ============================================= 4 | 5 | If you would like to contribute to the development of OpenStack, 6 | you must follow the steps in this page: 7 | 8 | https://docs.openstack.org/infra/manual/developers.html 9 | 10 | Once those steps have been completed, changes to OpenStack 11 | should be submitted for review via the Gerrit tool, following 12 | the workflow documented at: 13 | 14 | https://docs.openstack.org/infra/manual/developers.html#development-workflow 15 | 16 | Pull requests submitted through GitHub will be ignored. 17 | 18 | Bugs should be filed on Launchpad, not GitHub: 19 | 20 | https://bugs.launchpad.net/zaqar 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This work is licensed under a Creative Commons Attribution 3.0 Unported License. 2 | 3 | http://creativecommons.org/licenses/by/3.0/legalcode -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS 2 | include ChangeLog 3 | exclude .gitignore 4 | exclude .gitreview 5 | 6 | global-exclude *.pyc -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | Team and repository tags 3 | ======================== 4 | 5 | .. image:: https://governance.openstack.org/tc/badges/zaqar-specs.svg 6 | :target: https://governance.openstack.org/tc/reference/tags/index.html 7 | 8 | .. Change things from this point on 9 | 10 | =============================== 11 | zaqar-specs 12 | =============================== 13 | 14 | Queuing Service (Zaqar) Specifications 15 | 16 | * Free software: Apache license 17 | * Documentation: http://specs.openstack.org/openstack/zaqar-specs 18 | 19 | Features 20 | -------- 21 | 22 | * TODO 23 | -------------------------------------------------------------------------------- /doc/source/conf.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 10 | # implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | import datetime 15 | import os 16 | import sys 17 | 18 | sys.path.insert(0, os.path.abspath('../..')) 19 | # -- General configuration ---------------------------------------------------- 20 | 21 | # Add any Sphinx extension module names here, as strings. They can be 22 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 23 | extensions = [ 24 | 'sphinx.ext.autodoc', 25 | #'sphinx.ext.intersphinx', 26 | 'openstackdocstheme', 27 | 'yasfb', 28 | ] 29 | 30 | # Feed configuration for yasfb 31 | feed_base_url = 'https://specs.openstack.org/openstack/zaqar-specs' 32 | feed_author = 'OpenStack Zaqar Team' 33 | 34 | exclude_patterns = [ 35 | 'template.rst', 36 | '**/template.rst', 37 | ] 38 | 39 | # Optionally allow the use of sphinxcontrib.spelling to verify the 40 | # spelling of the documents. 41 | try: 42 | import sphinxcontrib.spelling 43 | extensions.append('sphinxcontrib.spelling') 44 | except ImportError: 45 | pass 46 | 47 | # autodoc generation is a bit aggressive and a nuisance when doing heavy 48 | # text edit cycles. 49 | # execute "export SPHINX_DEBUG=1" in your terminal to disable 50 | 51 | # The suffix of source filenames. 52 | source_suffix = '.rst' 53 | 54 | # The master toctree document. 55 | master_doc = 'index' 56 | 57 | # General information about the project. 58 | project = u'zaqar-specs' 59 | copyright = u'%s, OpenStack Foundation' % datetime.date.today().year 60 | 61 | # If true, '()' will be appended to :func: etc. cross-reference text. 62 | add_function_parentheses = True 63 | 64 | # If true, the current module name will be prepended to all description 65 | # unit titles (such as .. function::). 66 | add_module_names = True 67 | 68 | # The name of the Pygments (syntax highlighting) style to use. 69 | pygments_style = 'native' 70 | 71 | # openstackdocstheme options 72 | openstackdocs_repo_name = 'openstack/zaqar-specs' 73 | openstackdocs_auto_name = False 74 | openstackdocs_bug_project = 'zaqar' 75 | openstackdocs_bug_tag = '' 76 | 77 | # -- Options for HTML output -------------------------------------------------- 78 | 79 | # The theme to use for HTML and HTML Help pages. Major themes that come with 80 | # Sphinx are currently 'default' and 'sphinxdoc'. 81 | # html_theme_path = ["."] 82 | # html_theme = '_theme' 83 | # html_static_path = ['static'] 84 | html_theme = "openstackdocs" 85 | 86 | # Output file base name for HTML help builder. 87 | htmlhelp_basename = '%sdoc' % project 88 | 89 | # Grouping the document tree into LaTeX files. List of tuples 90 | # (source start file, target name, title, author, documentclass 91 | # [howto/manual]). 92 | latex_documents = [ 93 | ('index', 94 | '%s.tex' % project, 95 | u'%s Documentation' % project, 96 | u'OpenStack Foundation', 'manual'), 97 | ] 98 | 99 | # Example configuration for intersphinx: refer to the Python standard library. 100 | #intersphinx_mapping = {'http://docs.python.org/': None} 101 | -------------------------------------------------------------------------------- /doc/source/contributing.rst: -------------------------------------------------------------------------------- 1 | ============================================= 2 | Contributing to: zaqar-specs 3 | ============================================= 4 | 5 | If you would like to contribute to the development of OpenStack, 6 | you must follow the steps in this page: 7 | 8 | https://docs.openstack.org/infra/manual/developers.html 9 | 10 | Once those steps have been completed, changes to OpenStack 11 | should be submitted for review via the Gerrit tool, following 12 | the workflow documented at: 13 | 14 | https://docs.openstack.org/infra/manual/developers.html#development-workflow 15 | 16 | Pull requests submitted through GitHub will be ignored. 17 | 18 | Bugs should be filed on Launchpad, not GitHub: 19 | 20 | https://bugs.launchpad.net/zaqar 21 | -------------------------------------------------------------------------------- /doc/source/index.rst: -------------------------------------------------------------------------------- 1 | zaqar-specs Design Specifications 2 | ================================================== 3 | 4 | .. toctree:: 5 | :glob: 6 | :maxdepth: 2 7 | 8 | specs/* 9 | 10 | 11 | zaqar-specs Repository Information 12 | =================================================== 13 | 14 | .. toctree:: 15 | :maxdepth: 2 16 | 17 | README 18 | contributing 19 | 20 | 21 | Indices and tables 22 | ================== 23 | 24 | * :ref:`search` 25 | -------------------------------------------------------------------------------- /doc/source/readme.rst: -------------------------------------------------------------------------------- 1 | =============================== 2 | zaqar-specs 3 | =============================== 4 | 5 | Queuing Service (Zaqar) Specifications 6 | 7 | * Free software: Apache license 8 | * Documentation: http://specs.openstack.org/openstack/zaqar-specs 9 | 10 | Features 11 | -------- 12 | 13 | * TODO 14 | -------------------------------------------------------------------------------- /doc/source/specs: -------------------------------------------------------------------------------- 1 | ../../specs -------------------------------------------------------------------------------- /doc/source/template.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ============================= 14 | The title of your blueprint 15 | ============================= 16 | 17 | Include the URL of your launchpad blueprint: 18 | 19 | https://blueprints.launchpad.net/zaqar/+spec/example 20 | 21 | Introduction paragraph -- why are we doing anything? 22 | 23 | Problem description 24 | =================== 25 | 26 | A detailed description of the problem. 27 | 28 | Proposed change 29 | =============== 30 | 31 | Here is where you cover the change you propose to make in detail. How do you 32 | propose to solve this problem? 33 | 34 | If this is one part of a larger effort make it clear where this piece ends. In 35 | other words, what's the scope of this effort? 36 | 37 | Alternatives 38 | ------------ 39 | 40 | This is an optional section, where it does apply we'd just like a demonstration 41 | that some thought has been put into why the proposed approach is the best one. 42 | 43 | Implementation 44 | ============== 45 | 46 | Assignee(s) 47 | ----------- 48 | 49 | Who is leading the writing of the code? Or is this a blueprint where you're 50 | throwing it out there to see who picks it up? 51 | 52 | If more than one person is working on the implementation, please designate the 53 | primary author and contact. 54 | 55 | Primary assignee: 56 | 57 | 58 | Can list additional ids if they intend on doing substantial implementation work 59 | on this blueprint. 60 | 61 | Milestones 62 | ---------- 63 | 64 | Target Milestone for completion: 65 | Juno-2 66 | 67 | Work Items 68 | ---------- 69 | 70 | Work items or tasks -- break the feature up into the things that need to be 71 | done to implement it. Those parts might end up being done by different people, 72 | but we're mostly trying to understand the timeline for implementation. 73 | 74 | 75 | Dependencies 76 | ============ 77 | 78 | - Include specific references to specs and/or blueprints in zaqar, or in other 79 | projects, that this one either depends on or is related to. 80 | 81 | .. note:: 82 | 83 | This work is licensed under a Creative Commons Attribution 3.0 84 | Unported License. 85 | http://creativecommons.org/licenses/by/3.0/legalcode 86 | 87 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pbr!=2.1.0,>=2.0.0 # Apache-2.0 2 | openstackdocstheme>=2.2.1 # Apache-2.0 3 | sphinx>=2.0.0,!=2.1.0 # BSD 4 | yasfb>=0.8.0 5 | testrepository>=0.0.18 6 | testtools>=0.9.34 7 | stestr>=2.0.0 8 | flake8 9 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = zaqar-specs 3 | summary = Queuing Service (Zaqar) Specifications 4 | description-file = 5 | README.rst 6 | author = OpenStack 7 | author-email = openstack-dev@lists.openstack.org 8 | home-page = https://specs.openstack.org/openstack/zaqar-specs/ 9 | classifier = 10 | Environment :: OpenStack 11 | Intended Audience :: Developers 12 | License :: OSI Approved :: Apache Software License 13 | Operating System :: POSIX :: Linux 14 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | # implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT 18 | import setuptools 19 | 20 | setuptools.setup( 21 | setup_requires=['pbr'], 22 | pbr=True) 23 | -------------------------------------------------------------------------------- /specs/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/zaqar-specs/189334067a625b6957ad75ee7a6019120e93ae2b/specs/.gitignore -------------------------------------------------------------------------------- /specs/backlog-template.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ============================= 14 | The title of your blueprint 15 | ============================= 16 | 17 | Include the URL of your launchpad blueprint: 18 | 19 | https://blueprints.launchpad.net/zaqar/+spec/example 20 | 21 | Introduction paragraph -- why are we doing anything? 22 | 23 | Problem description 24 | =================== 25 | 26 | A detailed description of the problem. 27 | 28 | Proposed change 29 | =============== 30 | 31 | Here is where you cover the change you propose to make in detail. How do you 32 | propose to solve this problem? 33 | 34 | If this is one part of a larger effort make it clear where this piece ends. In 35 | other words, what's the scope of this effort? 36 | 37 | Drawbacks 38 | --------- 39 | 40 | Are there any drawbacks in this proposal? Does it change things in the 41 | API? Will it have an impact in critical areas? Will it make other 42 | areas more complex? 43 | 44 | Please, answer any of the questions above or elaborate on the 45 | drawbacks of this proposal if there're any. 46 | 47 | Alternatives 48 | ------------ 49 | 50 | This is an optional section, where it does apply we'd just like a demonstration 51 | that some thought has been put into why the proposed approach is the best one. 52 | 53 | Implementation 54 | ============== 55 | 56 | Assignee(s) 57 | ----------- 58 | 59 | Who is leading the writing of the code? Or is this a blueprint where you're 60 | throwing it out there to see who picks it up? 61 | 62 | If more than one person is working on the implementation, please designate the 63 | primary author and contact. 64 | 65 | Primary assignee: 66 | 67 | 68 | Can list additional ids if they intend on doing substantial implementation work 69 | on this blueprint. 70 | 71 | Work Items 72 | ---------- 73 | 74 | Work items or tasks -- break the feature up into the things that need to be 75 | done to implement it. Those parts might end up being done by different people, 76 | but we're mostly trying to understand the timeline for implementation. 77 | 78 | 79 | Dependencies 80 | ============ 81 | 82 | - Include specific references to specs and/or blueprints in zaqar, or in other 83 | projects, that this one either depends on or is related to. 84 | 85 | .. note:: 86 | 87 | This work is licensed under a Creative Commons Attribution 3.0 88 | Unported License. 89 | http://creativecommons.org/licenses/by/3.0/legalcode 90 | 91 | -------------------------------------------------------------------------------- /specs/backlog/email-notification.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ========================== 14 | Email Notification Support 15 | ========================== 16 | 17 | As a notification service, email is one of the importants subscribers we 18 | should support. Given email is the most way to get messages in the modern 19 | world . 20 | 21 | blueprint: https://blueprints.launchpad.net/zaqar/+spec/email-notification 22 | 23 | Problem description 24 | =================== 25 | 26 | Now Zaqar just introduces the notification support in Kilo, but only suppport 27 | webhook driver. The problem is users who do not have a webhook endpoint for 28 | Zaqar to hit, but do have access to email, cannot currently benefit from Zaqar 29 | notifications. 30 | 31 | 32 | Proposed change 33 | =============== 34 | 35 | Add a new task driver under /notification/task, which will leverage the 36 | built-in /usr/sbin/sendmail command to send messages. 37 | 38 | Proposed REST API request looks like:: 39 | 40 | POST /v2/queues/{queue_name}/subscriptions 41 | 42 | { 43 | 'subscriber': 'mailto:fake@gmail.com', 44 | 'ttl': 3600, 45 | 'options': {'subject': 'Alarm', 'cc': {}, 'bcc': {} } 46 | } 47 | 48 | 49 | The email notification driver will support the standard mailto protocol, that 50 | said, such as subject, cc and bcc could be a part of the 'subscriber' attribue. 51 | For example: 52 | 53 | mailto:someone@example.com?subject=This%20is%20the%20subject&cc= 54 | someone_else@example.com&body=This%20is%20the%20body 55 | 56 | Meanwhile, we can also support those email fields in the 'options' attribute. 57 | 58 | And to avoid spam, we can send a confirmation email firstly with a 59 | confirm_token, then by default the subcription is in 'inactive' status, until 60 | the email owner clicked the URL in the confirmation email. The URL will be like 61 | below:: 62 | 63 | /v2/queues/{queue_name}/subscriptions?subscriber=mailto:fake@gmail.com&confirm_token=22c4f150358e4ed287fa51e050d7f024 64 | 65 | 66 | Then Zaqar will update the subscription from 'inactive' to 'active'. 67 | 68 | 69 | Alternatives 70 | ------------ 71 | 72 | From the design/architecture perspective, if we don't have an email driver for 73 | notification, that means we may need a email-sending-as-a-service to achieve 74 | the same goal. It doesn't make much sense to do that given we can easily do 75 | that by leveraging the mailto in Zaqar's notification service. 76 | 77 | Data model impact 78 | ----------------- 79 | 80 | None. 81 | 82 | REST API impact 83 | --------------- 84 | 85 | None. 86 | 87 | Other end user impact 88 | --------------------- 89 | 90 | None. 91 | 92 | Deployer impact 93 | --------------- 94 | 95 | Deployers who deploy Zaqar will need to ensure that they have email systems 96 | configured if the email notification is enabled in 'subscriber_types'. 97 | 98 | Developer impact 99 | ---------------- 100 | 101 | None. 102 | 103 | Implementation 104 | ============== 105 | 106 | Assignee(s) 107 | ----------- 108 | 109 | Primary assignee: 110 | flwang (Feilong Wang) 111 | 112 | Milestones 113 | ---------- 114 | 115 | Target Milestone for completion: L-2 116 | 117 | Work Items 118 | ---------- 119 | 120 | * Add a task driver for email 121 | 122 | Dependencies 123 | ============ 124 | 125 | * Mail transfer agent needs to be configured on the host. 126 | 127 | Testing 128 | ======= 129 | 130 | * Unit tests 131 | * Functional test, like sending to $(whoami)@$(hostname) and then check with 132 | 'mail'. 133 | * Manual testing 134 | 135 | Documentation Impact 136 | ==================== 137 | 138 | * Feature need to be documented 139 | 140 | References 141 | ========== 142 | 143 | * https://docs.python.org/2/library/email-examples.html 144 | 145 | -------------------------------------------------------------------------------- /specs/backlog/index.rst: -------------------------------------------------------------------------------- 1 | ======================= 2 | Liberty Specifications 3 | ======================= 4 | 5 | .. toctree:: 6 | :glob: 7 | :maxdepth: 2 8 | 9 | * 10 | 11 | -------------------------------------------------------------------------------- /specs/backlog/persistent-transport.rst: -------------------------------------------------------------------------------- 1 | ==================== 2 | Persistent transport 3 | ==================== 4 | 5 | Persistent connections allow a single TCP connection to send and receive 6 | multiple requests/responses instead of opening a new TCP connection for 7 | each pair. When an application opens fewer TCP connections and keeps them 8 | open for a longer period, it causes less network traffic, use less time 9 | establishing new connections, and allows the TCP protocol to work more 10 | efficiently. 11 | 12 | Currently Zaqar has support for a non-persistent transport, leaving aside 13 | some use cases that would be covered in a more efficient and reliable way 14 | with a persistent transport solution. 15 | 16 | Some use cases that will be positively impacted by this change are: 17 | 18 | - Communication with browsers: Websocket will enhance the communication 19 | between browsers and Zaqar, a key factor in the integration of Zaqar with 20 | the OpenStack Dashboard (Horizon). 21 | 22 | - Notifications: this type of transport will constitute a better protocol 23 | for notifications, a feature planned for Kilo as well. 24 | 25 | https://blueprints.launchpad.net/zaqar/+spec/persistent-transport 26 | 27 | Problem description 28 | =================== 29 | 30 | Currently there is no way to establish persistent connections in Zaqar. 31 | This spec proposes adding this feature through a websocket implementation. 32 | 33 | During Kilo release we refactored the code to make the addition of this driver 34 | feasible, and we worked on adding the queues endpoint. 35 | 36 | To provide full support for the data plane, we should add the messages endpoint 37 | and the claims endpoint. 38 | 39 | The control plane operations do not require the performance that a websocket 40 | driver can provide, hence we are not going to add websocket support for the 41 | control plane. 42 | 43 | Proposed change 44 | =============== 45 | 46 | The proposed change, as stated in previous sections, is to add a persistent 47 | transport alternative to Zaqar. This can be done by using websocket. 48 | 49 | Websocket provide a full-duplex communication channel operating over a single 50 | socket that will remove overhead and significantly reduce complexity. 51 | 52 | Fallback 53 | -------- 54 | 55 | In case WebSocket is not available, the client will fallback to short-polling 56 | the REST API. 57 | 58 | Some of the reasons of WebSockets not being available are: 59 | 60 | - Firewalls could be configured to kill HTTP connections after a certain 61 | amount of time. 62 | 63 | - Concerns from a security standpoint will lead to killing any traffic 64 | over port 80 or 443 that doesn't look like HTTP. 65 | 66 | Messages serialization 67 | ---------------------- 68 | 69 | A serialization technology is needed in order to transport data. There are 70 | several alternatives, including JSON, MsgPack, Protobuf, Apache Avro and 71 | Cap'n Proto. 72 | 73 | In the previous cycle we noted that the whereas MsgPack was the best candidate 74 | -considering overall performance, easiness to use, languages support, 75 | data structures support-, it doesn't have good support for Javascript. Using 76 | MsgPack would impose a significative constraint to the Zaqar use cases, 77 | considering that many web applications nowaways are written with Javascript. 78 | For that reason, we decided to stick with JSON and maybe talk about a more 79 | performant serialization technology in a future change. 80 | 81 | Alternatives 82 | ------------ 83 | 84 | - Establish persistent connections enabling WSGI keep-alive 85 | - Long polling 86 | 87 | Implementation 88 | ============== 89 | 90 | Assignee(s) 91 | ----------- 92 | 93 | Primary assignee: vkmc 94 | 95 | Milestones 96 | ---------- 97 | 98 | Target Milestone for completion: Liberty-1 99 | 100 | Work Items 101 | ---------- 102 | 103 | * Implement messages controller 104 | * Implement claims controller 105 | * Write messages unit tests 106 | * Write claims unit tests 107 | * Would like to have: write functional tests 108 | * Perform benchmarks with the new driver 109 | 110 | WebSocket Libraries 111 | ------------------- 112 | 113 | After discussing several alternatives, including SockJS, SocketIO, Autobahn, 114 | ws4py and Tornado, it has been decided that Autobahn is going to be used to 115 | implement this feature. More details on this decision will be available in 116 | the `WebSockets wiki`_. 117 | 118 | .. _WebSockets wiki: https://wiki.openstack.org/wiki/Zaqar/specs/websockets 119 | 120 | Dependencies 121 | ============ 122 | 123 | * https://blueprints.launchpad.net/zaqar/+spec/cross-transport-api-spec 124 | 125 | .. note:: 126 | 127 | This work is licensed under a Creative Commons Attribution 3.0 128 | Unported License. 129 | http://creativecommons.org/licenses/by/3.0/legalcode 130 | 131 | 132 | -------------------------------------------------------------------------------- /specs/index.rst: -------------------------------------------------------------------------------- 1 | ================ 2 | Specifications 3 | ================ 4 | 5 | .. toctree:: 6 | :glob: 7 | :maxdepth: 1 8 | 9 | juno/index 10 | kilo/index 11 | liberty/index 12 | backlog/index 13 | mitaka/index 14 | newton/index 15 | ocata/index 16 | pike/index 17 | queens/index 18 | rocky/index 19 | stein/index 20 | ussuri/index 21 | victoria/index 22 | wallaby/index 23 | xena/index 24 | yoga/index 25 | -------------------------------------------------------------------------------- /specs/juno/index.rst: -------------------------------------------------------------------------------- 1 | ===================== 2 | Juno Specifications 3 | ===================== 4 | 5 | .. toctree:: 6 | :glob: 7 | :maxdepth: 2 8 | 9 | .. FIXME(flaper87): Uncomment after adding first real spec. 10 | .. * 11 | 12 | -------------------------------------------------------------------------------- /specs/kilo/define-a-wire-protocol.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ================================================== 14 | Design wire protocol for non RESTFul transports 15 | ================================================== 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/non-rest-wire-protocol 18 | 19 | This specs represents the work to be done to design a wire protocol that will 20 | respect the current API v1.1 and that will also be useful for non-RESTFul 21 | transmission protocols. 22 | 23 | Problem description 24 | =================== 25 | 26 | We currently have a well defined protocol that works well for HTTP. As a 27 | project, we're looking forward to welcome more transports that will work for 28 | different use-cases but we don't have a way to support them. This blueprint 29 | aims to define such protocol and the work needed to make it happen. 30 | 31 | Proposed change 32 | =============== 33 | 34 | Based on the existing protocol defined in this wiki page[0] and the rough spec 35 | proposed here[1], I'm proposing we use similar to the one below and complete 36 | the work started by the api-layer blueprint[2]. 37 | 38 | This blueprint, however, does not propose an API layer that should be used by 39 | *all* transports. Instead, it proposes an API layer that will be used by 40 | transports relying on lower-level transmission protocols like: websocket, raw 41 | tcp, etc. 42 | 43 | Ideally, one would expect Zaqar to support existing protocols like 44 | STOMP, AMQP, etc. While this may/may not be possible depending on the 45 | protocol, we should have a protocol that sticks to the service goals 46 | of being simple, straightforward and lightweight. Other protocols 47 | could be implemented as separate transports in the future. 48 | 49 | :: 50 | 51 | { 52 | "action": "post_message", 53 | "header": { 54 | "User-Agent": "python/2.7 killer-rabbit/1.2", 55 | "Date": "Wed, 2 8 Nov 2012 21:14:19 GMT", 56 | "Accept": "application/x-msgpack", 57 | "Client-ID": 30387f00-39a0-11e2-be4d-a8d15f34bae2, 58 | "X-Project-ID": 518b51ea133c4facadae42c328d6b77b, 59 | "X-Auth-Token": "7d2f63fd-4dcc-4752-8e9b-1d08f989cc00" 60 | }, 61 | "body": { 62 | ... 63 | } 64 | } 65 | 66 | [0] https://wiki.openstack.org/wiki/Zaqar/specs/api/v1.1 67 | [1] https://wiki.openstack.org/wiki/Zaqar/specs/zmq/api/v1 68 | [2] https://blueprints.launchpad.net/zaqar/+spec/cross-transport-api-spec 69 | 70 | Alternatives 71 | ------------ 72 | 73 | - Don't do anything and let each transport implement it's protocol. 74 | - Implement a transport on top of a different protocol 75 | 76 | Implementation 77 | ============== 78 | 79 | Assignee(s) 80 | ----------- 81 | 82 | Primary assignee: 83 | flaper87 84 | 85 | 86 | Milestones 87 | ---------- 88 | 89 | Target Milestone for completion: 90 | Kilo-1 91 | 92 | Work Items 93 | ---------- 94 | 95 | - Define the wire protocol 96 | - Create an API manager that process messages 97 | 98 | Dependencies 99 | ============ 100 | 101 | None 102 | 103 | .. note:: 104 | 105 | This work is licensed under a Creative Commons Attribution 3.0 106 | Unported License. 107 | http://creativecommons.org/licenses/by/3.0/legalcode 108 | 109 | -------------------------------------------------------------------------------- /specs/kilo/fifo-optional.rst: -------------------------------------------------------------------------------- 1 | .. This template should be in ReSTructured text. The filename in the 2 | git repository should match the launchpad URL, for example a URL of 3 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be 4 | named awesome-thing.rst. 5 | 6 | Please do not delete any of the sections in this template. If you 7 | have nothing to say for a whole section, just write: None 8 | 9 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html To test 10 | out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 11 | 12 | ============================ 13 | Make FIFO guarantee optional 14 | ============================ 15 | 16 | FIFO has been one of Zaqar's most wanted guarantees. Besides 17 | reliability, durability and other latency requirements, users have 18 | always requested FIFO to be there. This feature, however, has proven 19 | to be an issue for some scenarios that don't necessarily require it, 20 | hence this proposal. 21 | 22 | https://blueprints.launchpad.net/zaqar/+spec/make-fifo-optional 23 | 24 | Problem description 25 | =================== 26 | 27 | FIFO, despite being a great guarantee to have, brings in some scale 28 | and performance issues that Zaqar is not willing to accept as the 29 | default behavior. This spec proposes making FIFO optional and letting 30 | drivers capable of supporting such scenario to do so. 31 | 32 | Proposed change 33 | =============== 34 | 35 | The proposed change, as stated in previous sections, is to make FIFO 36 | optional. It is possible to do so through `flavors`. 37 | 38 | Not all store drivers are capable of supporting FIFO but those who 39 | are, will have FIFO listed in their supported capabilities and such 40 | capabilities will be exposed through the `flavors` ones. However, it's 41 | not as straightforward as it seems. See the `Work Items`_ section for 42 | a list of required changes that will make this possible. 43 | 44 | Drawbacks 45 | --------- 46 | 47 | As a side effect of this change, we'll have to relax the delivery 48 | guarantee for pub-sub. The reason being that walking through the queue 49 | won't prevent consumers to skip messages. 50 | 51 | Alternatives 52 | ------------ 53 | 54 | - Keep it as is 55 | - Remove FIFO completely 56 | 57 | Implementation 58 | ============== 59 | 60 | Assignee(s) 61 | ----------- 62 | 63 | Primary assignee: flaper87 64 | 65 | Milestones 66 | ---------- 67 | 68 | Target Milestone for completion: K-1 69 | 70 | Work Items 71 | ---------- 72 | 73 | * Allow driver to expose what features they support 74 | Each driver supporst a set of features and this set needs to be accessible 75 | from the upper layers 76 | * Standardize the *supported* capabilities 77 | Make a list of supported capabilities 78 | * Add a way to pass capabilities down to the driver 79 | We can do this when the driver is initialized since we know what the 80 | capabilities are at that time. 81 | * Support both, FIFO and non-FIFO, post methods. 82 | 83 | 84 | Dependencies 85 | ============ 86 | 87 | * https://review.opendev.org/#/c/126531/ 88 | 89 | .. note:: 90 | 91 | This work is licensed under a Creative Commons Attribution 3.0 92 | Unported License. 93 | http://creativecommons.org/licenses/by/3.0/legalcode 94 | -------------------------------------------------------------------------------- /specs/kilo/index.rst: -------------------------------------------------------------------------------- 1 | ==================== 2 | Kilo Specifications 3 | ==================== 4 | 5 | .. toctree:: 6 | :glob: 7 | :maxdepth: 2 8 | 9 | * 10 | -------------------------------------------------------------------------------- /specs/kilo/notification-api.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ================================ 14 | Zaqar's notification semantics 15 | ================================ 16 | 17 | One of Zaqar's goals, from an API semantics perspective, is to provide 18 | support for push notifications - or subscriptions, if you will. After 19 | having added support for all the queuing related semantcis to Zaqar's 20 | API, we can start adding support for other things that sit on top of 21 | the existing semantics like the ones mentioned in this spec. 22 | 23 | https://blueprints.launchpad.net/zaqar/+spec/notifications 24 | 25 | Other references: 26 | - https://etherpad.openstack.org/p/marconi-notifications-brainstorm 27 | - https://etherpad.openstack.org/p/juno-marconi-notifications-on-marconi 28 | - https://etherpad.openstack.org/p/IcehouseMarconiNotificationService 29 | - https://www.dropbox.com/s/d2q1viz4ejebg24/marconi-notifications.png 30 | 31 | 32 | Problem description 33 | =================== 34 | 35 | Zaqar's support for queuing semantics that allow users to publish and 36 | consume messages to and from the service. These semantics cover 37 | several use-cases but they leave out others like 'listening' for 38 | messages. There are, as well as for prod/cons scenarios, many 39 | use-cases for pub/sub that need to be supported. Some examples of such 40 | use-cases are: mobile push notifications, support for events 41 | triggering, meetering, etc. 42 | 43 | In order to cover the above scenarios, Zaqar needs to provide an API 44 | that will allow users for subscribing to specific queues and get 45 | messages from there without requiring to poll. This will allow for a 46 | better architecture on the client side, a more reliable interaction 47 | between Zaqar and the client in terms of connection management and it 48 | also opens the doors for supporting other scenarios. 49 | 50 | Proposed change 51 | =============== 52 | 53 | The proposal is to extend Zaqar's API with support for subscriptions 54 | without adding a separate service - please see the section below with 55 | regards to both services being merged. This proposal introduces the 56 | high-level changes required for this feature but the detailed API 57 | specification has yet to be written. 58 | 59 | Here's a, non-exahustive, list of resources required to support 60 | notifications: 61 | 62 | **Subscriptions**: Subscriptions define what the subscriber wants to 63 | listen on. It contains the data of the queues/topics, routing 64 | parameters, maximum retries and other configirations required to 65 | correctly filter messages and keep the high-level guarantees. 66 | 67 | A subscription is used by subscribers to listen on things. A non 68 | persistent subscriber - webhooks, sms, mobile push protocols - are 69 | configured in the subscription itself, whereas other subscribers using 70 | persistent connection can load a specific subscription they want to 71 | listen on. 72 | 73 | A subscription consists in a object containing the information of the 74 | source, destination, expiration and a custom field that contains 75 | information useful for the notifier. :: 76 | 77 | POST /v2/queues/{queue_name}/subscriptions 78 | 79 | { 80 | 'subscriber': 'http://trigger.me', 81 | 'ttl': 3600, 82 | 'options': {} 83 | } 84 | 85 | * **source**: The source queue for notifications 86 | * **subscriber**: The subscriber URI. This can be any of the supported 87 | subscriber URI and the implementation must be based on stevedore to 88 | allow external contributions. 89 | * **ttl**: time to live for this subscription. It's a positive number 90 | that expresses, in seconds, how long a subscription should exist. If 91 | omitted the subscription won't expire. 92 | * **options**: publisher specific options. 93 | 94 | **Publishers**: Publishers are users and/or third-party resources 95 | publishing messages to the notification endpoints. Users publishing 96 | messages to Zaqar don't need any specialized endpoint for 97 | this. Messages must go through the normal and already implemented 98 | message publishing API. However, third-party publishers require a 99 | specific configuration. Third-party publishers consume messages from 100 | other services and publish them in zaqar's queues. 101 | 102 | **Workers**: Publishing tasks need to be executed in a distributed - 103 | or local to the node - fashion, they need to be reliable, 104 | fault-tolerant and atomic. To do so, I've evaluated taskflow as a 105 | valid library for this job. Taskflow has support for 106 | local/distributed, blocking/concurrent task execution. It provides 107 | support for several execution models, persistence layers - including 108 | sqlite, mysql and zookeeper - and it has support for retries. The 109 | alternative to taskflow is writing our on implementation for this 110 | work, which doesn't seem to be worth it at this stage. 111 | 112 | Push Protocols 113 | -------------- 114 | 115 | There are several types of push technologies that could be supported, 116 | therefore the implementation must be generic enough to allow for new 117 | implementations to be done easily and also as external plugins. 118 | 119 | Nonetheless, this spec proposes starting with just the first of the 120 | plugins proposed below: 121 | 122 | - Webhooks 123 | - Email 124 | - Mobile push 125 | - SMS 126 | 127 | Why not 2 services 128 | ------------------ 129 | 130 | Zaqar has stated since its inception that it provides a single common 131 | API for things that other cloud services like AWS have in 2 separate 132 | services. Nonetheless, I think the real motivations for these 2 API's 133 | to be under the same service are: 134 | 135 | - Maintenance: It's easier to maintain and scale just 1 service type 136 | than a group of different services. 137 | - Consistency: It'll help reducing the code needed to implement the 138 | notification API since they both will be running under the same 139 | structure and users will know where to find the services easily. 140 | - Performance: Implementing the notification API in a separte service 141 | will require a communication between that service and the queuing 142 | service. At least, it'll require a constant polling from the backend 143 | (if it doesn't support push itself). By having everything under the 144 | same service, we can trigger notification for messages as they come 145 | in. 146 | 147 | Future Improvements 148 | ------------------- 149 | 150 | * Support for subscription's confirmation 151 | 152 | Alternatives 153 | ------------ 154 | 155 | - Split the above into 2 different services. As mentioned in the 156 | previous section, 2 services don't seem to be the ideal solution for 157 | this. 158 | 159 | Implementation 160 | ============== 161 | 162 | Assignee(s) 163 | ----------- 164 | 165 | Primary assignee: 166 | flwang 167 | 168 | Secondary assignees: 169 | flaper87 170 | 171 | Milestones 172 | ---------- 173 | 174 | Target Milestone for completion: 175 | Kilo-2 176 | 177 | Work Items 178 | ---------- 179 | 180 | * Work on the detailed API spec 181 | * Write storage code for notifications 182 | * Implement the API on top of the storage code. 183 | * Work on the publishers 184 | 185 | Dependencies 186 | ============ 187 | 188 | None 189 | 190 | .. note:: 191 | 192 | This work is licensed under a Creative Commons Attribution 3.0 193 | Unported License. 194 | http://creativecommons.org/licenses/by/3.0/legalcode 195 | 196 | -------------------------------------------------------------------------------- /specs/kilo/persistent-transport.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the 3 | git repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be 5 | named awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this template. If you 8 | have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html To test 11 | out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ==================== 14 | Persistent transport 15 | ==================== 16 | 17 | Persistent connections allow a single TCP connection to send and receive 18 | multiple requests/responses instead of opening a new TCP connection for 19 | each pair. When an application opens fewer TCP connections and keeps them 20 | open for a longer period, it causes less network traffic, use less time 21 | establishing new connections, and allows the TCP protocol to work more 22 | efficiently. 23 | 24 | Currently Zaqar has support for a non-persistent transport, leaving aside 25 | some use cases that would be covered in a more efficient and reliable way 26 | with a persistent transport solution. 27 | 28 | Some use cases that will be positively impacted by this change are: 29 | 30 | - Communication with browsers: Websockets will enhance the communication 31 | between browsers and Zaqar, a key factor in the integration of Zaqar with 32 | the OpenStack Dashboard (Horizon). 33 | 34 | - Notifications: this type of transport will constitute a better protocol 35 | for notifications, a feature planned for Kilo as well. 36 | 37 | https://blueprints.launchpad.net/zaqar/+spec/persistent-transport 38 | 39 | Problem description 40 | =================== 41 | 42 | Currently there is no way to establish persistent connections in Zaqar. 43 | This spec proposes adding this feature through a websockets implementation. 44 | 45 | Proposed change 46 | =============== 47 | 48 | The proposed change, as stated in previous sections, is to add a persistent 49 | transport alternative to Zaqar. This can be done by using websockets. 50 | 51 | Websockets provide a full-duplex communication channel operating over a single 52 | socket that will remove overhead and significantly reduce complexity. 53 | 54 | Currently the tranport layer in Zaqar is tightly bound to the WSGI transport, 55 | making this addition a complex task. 56 | 57 | Fallback 58 | -------- 59 | 60 | In case WebSockets are not available, the client will fallback to short-polling 61 | the REST API. 62 | 63 | Some of the reasons of WebSockets not being available are: 64 | 65 | - Firewalls could be configured to kill HTTP connections after a certain 66 | amount of time. 67 | 68 | - Concerns from a security standpoint will lead to killing any traffic 69 | over port 80 or 443 that doesn't look like HTTP. 70 | 71 | Messages serialization 72 | ---------------------- 73 | 74 | A serialization technology is needed in order to transport data. There are 75 | several alternatives, including MsgPack, Protobuf, Apache Avro and Cap'n Proto. 76 | 77 | Given that a high priority is being given to browsers compatibility, MsgPack 78 | is the best choice between those. Remaining alternatives doesn't have 79 | out-of-the-box Javascript support, making it harder to ensure compatibility 80 | with most web applications. 81 | 82 | Alternatives 83 | ------------ 84 | 85 | - Establish persistent connections enabling WSGI keep-alive 86 | - Long polling 87 | 88 | Implementation 89 | ============== 90 | 91 | Assignee(s) 92 | ----------- 93 | 94 | Primary assignee: vkmc 95 | 96 | Secondary assignee: cpallares 97 | 98 | Milestones 99 | ---------- 100 | 101 | Target Milestone for completion: < K-1 102 | 103 | Work Items 104 | ---------- 105 | 106 | * Implement the common API layer 107 | * Define the wire protocol for persistent transports 108 | * Implement the driver 109 | * Implement queues and messages controller 110 | * Implement claims controller 111 | * Implement pools, flavors and stats controllers 112 | 113 | WebSocket Libraries 114 | ------------------- 115 | 116 | After discussing several alternatives, including SockJS, SocketIO, Autobahn, 117 | ws4py and Tornado, it has been decided that ws4py is going to be used to 118 | implement this feature. More details on this decision will be available in 119 | the `WebSockets wiki`_. 120 | 121 | .. _WebSockets wiki: https://wiki.openstack.org/wiki/Zaqar/specs/websockets 122 | 123 | Dependencies 124 | ============ 125 | 126 | * https://review.opendev.org/#/c/122425/ 127 | * https://blueprints.launchpad.net/zaqar/+spec/cross-transport-api-spec 128 | 129 | .. note:: 130 | 131 | This work is licensed under a Creative Commons Attribution 3.0 132 | Unported License. 133 | http://creativecommons.org/licenses/by/3.0/legalcode 134 | 135 | 136 | -------------------------------------------------------------------------------- /specs/kilo/storage-capabilities.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ============================= 14 | Expose storage capabilities 15 | ============================= 16 | 17 | Flavors allow operators to expose some of the internal storage capabilities 18 | without giving the exact details of how the system is configured. As of now, 19 | the operators has total control over what capabilities are exposed to the user 20 | regardeless on whether these capabilities are indeed guaranteed or valid. A 21 | `capability` is a feature that is either specific to the storage or to 22 | Zaqar - durable, fifo, in-memory, for example - and they can be enabled/exposed 23 | in a per-flavor basis allowing end-users to choose the best flavor for a 24 | specific queue. 25 | 26 | https://blueprints.launchpad.net/zaqar/+spec/expose-storage-capabilities 27 | 28 | 29 | Problem description 30 | =================== 31 | 32 | Flavors are very powerful when it comes to give choices to end-users without 33 | taking control out from operators. However, there's no way for operators to 34 | know what capabilities each storage driver has, which makes this feature 35 | completely custom to whatever the operator thinks makes more sense. 36 | 37 | It's important to support operators on the choice of what features are exposed 38 | to the end-users through the API by making it clear which are supported by the 39 | drivers that have been deployed. Moreover, this is also important for doing a 40 | proper capacity planning based on the use-cases the service is trying to 41 | support. 42 | 43 | Proposed change 44 | =============== 45 | 46 | Capabilities 47 | ------------ 48 | 49 | This change request proposes defining a set of internal `capabilities` and a 50 | way for storage drivers to expose which capabilities are supported so that 51 | operators may know in advance what a flavor would support and what storage 52 | drivers they may need. 53 | 54 | The set of internal capabilities includes: 55 | 56 | - FIFO 57 | - AOD (At least once delivery) 58 | - high-throughput 59 | - claims 60 | 61 | Exposing Capabilities 62 | --------------------- 63 | 64 | This capabilities should be exposed by the storage driver iff supported. This 65 | spec proposes adding a new class method to the base class called `capabilities` 66 | that should return a set with a list of supported `capabilities`. The 67 | capabilities returned in this set cannot be custom, which means they must be 68 | supported internally.:: 69 | 70 | @six.add_metaclass(abc.ABCMeta) 71 | class DataDriverBase(DriverBase): 72 | ... 73 | 74 | @property 75 | def capabilities(self): 76 | return self._CAPABILITIES 77 | 78 | 79 | Drivers Load 80 | ------------ 81 | 82 | In addition to the above-mentioned changes, we also need a better way to load 83 | drivers based on the capabilities. That is, when drivers are loaded, it's 84 | necessary to pass down to the driver the capabilities that have been enabled 85 | per-flavor so that the driver itself can be specialized for some capabilities. 86 | In most of the cases, loading a driver will end up in always loading the same 87 | implementation. However, there are cases - FIFO, for example - where more 88 | specialized implementations may be needed depending on the storage. Therefore, 89 | instead of *always* loading the `DataDriver`, this spec proposes adding a new 90 | `get_driver` function that will let the storage itself load a driver according 91 | to the capabilities passed there.:: 92 | 93 | def get_driver(plane='data', capabilities=None): 94 | ... 95 | 96 | 97 | This means we won't need to register neither `DataDriver`s nor `ControlDriver`s 98 | as `entry_points` but just the new `get_driver` function. This will allow 99 | storage driver's to expose fewer things as public API's. A variant for the 100 | above `get_driver` proposal would be.:: 101 | 102 | def get_driver(base=base.BaseDataDriver, capabilities=None): 103 | ... 104 | 105 | This way we can avoid giving 'names' to this planes and just ask what we need 106 | in terms of 'signature'. However, it makes loading the actual driver a bit more 107 | complex and it'll also duplicate some logic across different storage drivers. 108 | 109 | Pools and Flavors 110 | ----------------- 111 | 112 | Flavors have support for capabilities. As of now, these capabilities 113 | are 100% custom. With the implementation of this spec, it'll be 114 | possible for flavors to know in advance what capabilities are 115 | supported based on the stores registered in the pool. 116 | 117 | Pools, however, shouldn't allow storage drivers with different 118 | capabilities to co-exist under the same pool. That is to say that 119 | pools will have to verify the storage capabilities when new nodes are 120 | added to the pool and ensure that the capabilities supported by the 121 | new node are consistent with the rest of the nodes in the pool. 122 | 123 | Alternatives 124 | ------------ 125 | 126 | Keep it custom and do nothing. 127 | 128 | Implementation 129 | ============== 130 | 131 | Assignee(s) 132 | ----------- 133 | 134 | Primary assignee: 135 | flaper87 136 | 137 | Secondary assignee: 138 | vkmc 139 | 140 | Milestones 141 | ---------- 142 | 143 | Target Milestone for completion: 144 | K-1 145 | 146 | Work Items 147 | ---------- 148 | 149 | * Define list of internal capabilities and document it 150 | * Add required abstractions to storage drivers to expose the supported 151 | capabilities 152 | * Let flavor's pass the capabilities down to the storage driver when it's being 153 | loaded. 154 | 155 | Dependencies 156 | ============ 157 | 158 | .. note:: 159 | 160 | This work is licensed under a Creative Commons Attribution 3.0 161 | Unported License. 162 | http://creativecommons.org/licenses/by/3.0/legalcode 163 | 164 | -------------------------------------------------------------------------------- /specs/liberty/finish-client-support-for-v1.1-features.rst: -------------------------------------------------------------------------------- 1 | ======================================== 2 | Finish client support for v1.1 features 3 | ======================================== 4 | 5 | https://blueprints.launchpad.net/zaqar/+spec/finish-client-support- 6 | for-v1.1-features 7 | 8 | Several endpoints have been added without being reflected on 9 | the client side. This endpoints include: 10 | 11 | - Pools 12 | - Flavors 13 | - Notifications 14 | 15 | Adding support for these features would make developing applications with Zaqar 16 | easier. 17 | 18 | Problem description 19 | =================== 20 | 21 | Python-zaqarclient currently has some missing features that need to be 22 | added. This spec describes which endpoints are missing and how those are going 23 | to be added to the client side. 24 | 25 | Proposed change 26 | =============== 27 | 28 | Based on the endpoints that are currently missing to the python-zaqarclient 29 | v1.1,the proposed change is to add support for the following features: 30 | 31 | 1. Pools: 32 | 33 | - pool_update 34 | - pool_list 35 | 36 | 2. flavors: 37 | 38 | - flavor_update 39 | - flavor_list 40 | 41 | 3. Notifications 42 | 43 | Alternatives 44 | ------------ 45 | 46 | Implementation 47 | ============== 48 | 49 | Assignee(s) 50 | ----------- 51 | 52 | Primary assignee: 53 | dynarro 54 | 55 | Milestones 56 | ---------- 57 | 58 | Target Milestone for completion: 59 | Liberty-2 60 | 61 | Work Items 62 | ---------- 63 | 64 | Add support to: 65 | 66 | - pools 67 | - flavors 68 | - notifications 69 | 70 | Add unit testing for: 71 | 72 | - pools 73 | - flavors 74 | - notifications 75 | 76 | Add developers documentation for: 77 | 78 | - pools 79 | - flavors 80 | - notifications 81 | 82 | Dependencies 83 | ============ 84 | 85 | None 86 | 87 | .. note:: 88 | 89 | This work is licensed under a Creative Commons Attribution 3.0 90 | Unported License. 91 | http://creativecommons.org/licenses/by/3.0/legalcode 92 | 93 | -------------------------------------------------------------------------------- /specs/liberty/index.rst: -------------------------------------------------------------------------------- 1 | ======================= 2 | Liberty Specifications 3 | ======================= 4 | 5 | .. toctree:: 6 | :glob: 7 | :maxdepth: 2 8 | 9 | * 10 | -------------------------------------------------------------------------------- /specs/liberty/policy_support.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ============================ 14 | Authorization Policy Support 15 | ============================ 16 | 17 | OpenStack components are supposed to check user privileges to perform any 18 | action. Generally these checks are role-based. See 19 | https://docs.openstack.org/keystone/latest/#approach-to-authorization-policy. 20 | Zaqar needs to support policies as well. 21 | 22 | Problem description 23 | =================== 24 | 25 | Presently Zaqar is missing fine-grained permissions for actions. For example, 26 | it's hard to allow one user only can get message from a queue but not be able 27 | to post message to the queue, or something like that. 28 | 29 | 30 | Proposed change 31 | =============== 32 | 33 | Add policy check for all Zaqar API endpoints. This could be done by following 34 | the same way as in other OpenStack components, by leveraging the oslo.policy 35 | module which will do all the underlying work. 36 | 37 | The implementation should be fairly simple, with oslo.policy's ``Enforcer`` 38 | class being instantiated with the policy file, then the ``enforce`` method used 39 | to check each API call. 40 | 41 | Proposed content of the policy file: 42 | 43 | .. sourcecode:: json 44 | 45 | { 46 | "context_is_admin": "role:admin", 47 | "admin_or_owner": "is_admin:True or project_id:%(project_id)s", 48 | "default": "rule:admin_or_owner", 49 | 50 | "queues:get_all": "", 51 | "queues:put": "", 52 | "queues:get": "", 53 | "queues:delete": "", 54 | 55 | "messages:post": "", 56 | "messages:get": "", 57 | "messages:bulk_get": "", 58 | "messages:bulk_delete": "", 59 | "messages:claim": "", 60 | 61 | "subscriptions:get_all": "", 62 | "subscriptions:create": "", 63 | "subscriptions:get": "", 64 | "subscriptions:delete": "", 65 | "subscriptions:update": "", 66 | } 67 | 68 | 69 | Alternatives 70 | ------------ 71 | 72 | None. 73 | 74 | Data model impact 75 | ----------------- 76 | 77 | None. 78 | 79 | REST API impact 80 | --------------- 81 | 82 | None. 83 | 84 | Other end user impact 85 | --------------------- 86 | 87 | None. 88 | 89 | Deployer impact 90 | --------------- 91 | 92 | An additional file, ``policy.json`` must be deployed. The deployer should 93 | verify the settings in that file are correct for their deployment, such that 94 | the correct users are allowed access. 95 | 96 | Developer impact 97 | ---------------- 98 | 99 | If there is any new API endpoint added for Zaqar, then policy rules in the 100 | json files should be updated accordingly. 101 | 102 | Implementation 103 | ============== 104 | 105 | Assignee(s) 106 | ----------- 107 | 108 | Primary assignee: 109 | kragniz (Louis Taylor) 110 | 111 | Milestones 112 | ---------- 113 | 114 | Target Milestone for completion: L-1 115 | 116 | Work Items 117 | ---------- 118 | 119 | * Add config options to point to control policy file and settings 120 | * Add policy check to all API calls 121 | * Add unit tests 122 | * Add documentation 123 | 124 | Dependencies 125 | ============ 126 | 127 | * oslo.policy 128 | * https://docs.openstack.org/keystone/latest/#approach-to-authorization-policy 129 | * https://docs.openstack.org/keystone/latest/configuration/config-options.html#policy 130 | * https://docs.openstack.org/keystone/latest/#keystone-api-protection-with-role-based-access-control-rbac 131 | -------------------------------------------------------------------------------- /specs/liberty/pre-signed-url.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ====================== 14 | Pre-Signed URLs 15 | ====================== 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/pre-signed-url 18 | 19 | There's a need to have pre-signed URLs - ala Swift's tempURL - that 20 | would grant temporary access, to non-authenticated users, to specific 21 | queues. 22 | 23 | Problem description 24 | =================== 25 | 26 | In cases where there's a need to allow "outsiders" of the system - i.e 27 | guest-agents - to interact with Zaqar, it's useful to have a way to 28 | grant them access so that they can do whatever is needed without 29 | giving these "outsiders" a username and password or any other kind of 30 | permission in the system. 31 | 32 | This access, however, needs to be temporary, revocable and 33 | granular. Outsiders shouldn't get access to more than 1 resource at a 34 | time and the permissions granted should identify what kind of 35 | operations they can execute. 36 | 37 | Proposed change 38 | =============== 39 | 40 | The proposed solution to the aforementioned problem is to use 41 | pre-signed URLs. Services like Swift use a similar approach - called 42 | temp URL there - to provide access to internal resources. The 43 | pre-signed URL consists of a URL that contains a `hash` with the 44 | encoded permissions. 45 | 46 | In order to make it easier to read/write this spec, let's split it in 47 | several parts. The first part will describe the parts composing the 48 | URL. The second part will describe how the URL will be generated and 49 | the third part how it'll be consumed. 50 | 51 | Pre-Signed URL 52 | -------------- 53 | 54 | A pre-signed URL ought to contain enough information for it to provide 55 | enough control over the resource being shared, without compromising 56 | security. Authorization is a must have and a huge part of this 57 | URL. Therefore, the information present in this URL has to be 58 | exhaustive in that perspective. 59 | 60 | As mentioned in a previous paragraph, the URL contains a hashed piece 61 | of information that serializes the required fields. Those fields are: 62 | 63 | * Project: The `keystone` tenant of the entity generating the URL 64 | * TTL: Expiration time (in seconds) for the URL 65 | * Queue: `Zaqar`'s queue name that this URL gives access to 66 | * HTTP Method: The `HTTP` method(s) this `URL` was created for. By 67 | selecting the HTTP method, it's possible to give either read or 68 | read/write access to a specific resource. 69 | 70 | The above fields will be part of the generated hash but they'll also 71 | be available as public information in the URL. The reason they're 72 | encoded is to have a way to verify that the URL has not been changed 73 | by the user whenever a request is made to `Zaqar`. 74 | 75 | The generated hash will be an HMAC-SHA1. To generate such signature, 76 | it is required to have a secret key. In swift, it's possible to have a 77 | key per-account. Unfortunately, that's not possible in Zaqar + 78 | Keystone, therefore this spec proposes adding a new configuration 79 | option that will contain the key to use.:: 80 | 81 | [signed_url] 82 | secret_key = some-very-strong-key 83 | 84 | 85 | URL Generation 86 | -------------- 87 | 88 | This spec proposes adding a new endpoint in the queue namespace that 89 | returns the generated signature and expiration time that'll grant 90 | access to this resource. The request and response for this operation: 91 | 92 | Request::: 93 | 94 | POST /v2/queues/shared_queue/share HTTP/1.1 95 | ... 96 | 97 | { 98 | 'methods': ['GET', 'POST'] 99 | } 100 | 101 | Response::: 102 | 103 | 104 | HTTP/1.0 201 OK 105 | ... 106 | 107 | { 108 | 'signature': '518b51ea133c4facadae42c328d6b77b', 109 | 'expires': 2015-05-31T19:00:17Z, 110 | 'project': '7d2f63fd4dcc47528e9b1d08f989cc00', 111 | 'url': '/v2/queues/shared_queue/messages', 112 | 'methods': ['GET', 'POST'] 113 | } 114 | 115 | This request sets a different expiration date for the URL. Note that 116 | the default method is `GET`. 117 | 118 | Request::: 119 | 120 | POST /v2/queues/shared_queue/share HTTP/1.1 121 | ... 122 | 123 | { 124 | 'expires': 2015-06-19T19:00:00Z 125 | } 126 | 127 | Response::: 128 | 129 | 130 | HTTP/1.0 201 OK 131 | ... 132 | 133 | { 134 | 'signature': '518b51ea133c4facadae42c328d6b77b', 135 | 'expires': 2015-06-19T19:00:00Z, 136 | 'project': '7d2f63fd4dcc47528e9b1d08f989cc00', 137 | 'url': '/v2/queues/shared_queue/messages' 138 | 'methods': ['GET'] 139 | } 140 | 141 | 142 | This request combines both parameters (methods and expires): 143 | 144 | Request::: 145 | 146 | POST /v2/queues/shared_queue/share HTTP/1.1 147 | ... 148 | 149 | { 150 | 'methods': ['GET', 'POST'], 151 | 'expires': 2015-06-19T19:00:00Z 152 | } 153 | 154 | Response::: 155 | 156 | 157 | HTTP/1.0 201 OK 158 | ... 159 | 160 | { 161 | 'signature': '518b51ea133c4facadae42c328d6b77b', 162 | 'expires': 2015-06-19T19:00:00Z, 163 | 'project': '7d2f63fd4dcc47528e9b1d08f989cc00', 164 | 'url': '/v2/queues/shared_queue/messages' 165 | 'methods': ['GET', 'POST'] 166 | } 167 | 168 | 169 | Consuming the URL 170 | ----------------- 171 | 172 | First and foremost, it's important to mention that **NONE** of the 173 | URL headers can/should be modified and/or omitted. As soon as one of 174 | them is, the signature verification will fail and therefore the 175 | request will respond 404. 176 | 177 | Requests for pre-signed URLs will be processed by a middleware that 178 | should be placed **before** keystone's middleware. This will allow us 179 | to authenticate the request in advance and skip keystone's 180 | authentication. A request using the signature generated in the 181 | previous section would look like: 182 | 183 | Request::: 184 | 185 | GET /v2/queues/shared_queue/messages HTTP/1.1 186 | Host: zaqar.example.com 187 | User-Agent: python/2.7 killer-rabbit/1.2 188 | Date: Wed, 28 Nov 2012 21:14:19 GMT 189 | Accept: application/json 190 | Accept-Encoding: gzip 191 | URL-Signature: 518b51ea133c4facadae42c328d6b77b 192 | URL-Expires: 2015-05-31T19:00:17Z 193 | X-Project-Id: 7d2f63fd4dcc47528e9b1d08f989cc00 194 | Client-ID: 30387f00-39a0-11e2-be4d-a8d15f34bae2 195 | 196 | Note that, in the above example, headers were chosen over query 197 | parameters. The main 2 reasons behind this choice are: 198 | 199 | 1. Consistency with other security related parameters - i.e 200 | X-Project-Id - that are sent in HTTP headers. 201 | 202 | 2. These new parameters don't belong in the `messages` request and 203 | won't affect `messages` navigation. 204 | 205 | Similarly, other requests like the one below can be done. 206 | 207 | Request::: 208 | 209 | GET /v2/queues/shared_queue/messages?marker=1355-237242-783&limit=10 HTTP/1.1 210 | Host: zaqar.example.com 211 | User-Agent: python/2.7 killer-rabbit/1.2 212 | Date: Wed, 28 Nov 2012 21:14:19 GMT 213 | Accept: application/json 214 | Accept-Encoding: gzip 215 | URL-Signature: 518b51ea133c4facadae42c328d6b77b 216 | URL-Expires: 2015-05-31T19:00:17Z 217 | X-Project-Id: 7d2f63fd4dcc47528e9b1d08f989cc00 218 | Client-ID: 30387f00-39a0-11e2-be4d-a8d15f34bae2 219 | 220 | Filtering and pagination are not part of the signature and fall into 221 | the `read` permissions that were granted on this. 222 | 223 | Posting messages will work the same way: 224 | 225 | Request::: 226 | 227 | POST /v2/queues/shared_queue/messages HTTP/1.1 228 | Host: zaqar.example.com 229 | User-Agent: python/2.7 killer-rabbit/1.2 230 | Date: Wed, 28 Nov 2012 21:14:19 GMT 231 | Accept: application/json 232 | Accept-Encoding: gzip 233 | URL-Signature: 518b51ea133c4facadae42c328d6b77b 234 | URL-Expires: 2015-05-31T19:00:17Z 235 | X-Project-Id: 7d2f63fd4dcc47528e9b1d08f989cc00 236 | Client-ID: 30387f00-39a0-11e2-be4d-a8d15f34bae2 237 | 238 | ... 239 | 240 | 241 | Other Notes 242 | ----------- 243 | 244 | 1. In the case of pre-signed URLs, the queue cannot be created 245 | lazily. This is to prevent cases where queues are deleted and 246 | users still have a valid URL. This is not a big issues in cases 247 | where there's just 1 pool. However, if there's a deployment using 248 | more than 1 type of pool, the lazily created queue may end up in an 249 | undesired pool and it'd be possible for an attacker to try a DoS on 250 | that pool. Therefore, whenever a pre-signed URL is created, if a 251 | pool doesn't exist, one will be created. 252 | 253 | 2. I'm not a fan of passing the `project-id` around but I can't think 254 | of another way to do this and still have the ability to preserve 255 | multi-tenancy without passing the project. 256 | 257 | 3. I don't like having the key set in the config file. In future 258 | versions, we could think of making this information part of the 259 | queue itself. The reason we can't do that right now is because we 260 | don't have private fields in the metadata. It should be easy enough 261 | to do it as an enhancement for this feature. 262 | 263 | 4. As a future enhancement, we could also use Barbican for key management. 264 | 265 | 266 | Drawbacks 267 | --------- 268 | 269 | Security issues may be added by this work. We ought to be extra 270 | careful on reviews and create a vulnerability team that is ready to 271 | address any issues that might come up. 272 | 273 | Alternatives 274 | ------------ 275 | 276 | None 277 | 278 | Implementation 279 | ============== 280 | 281 | Assignee(s) 282 | ----------- 283 | 284 | Primary assignee: 285 | flaper87 286 | 287 | 288 | Work Items 289 | ---------- 290 | 291 | 1. Write utilities to generate the signature with proper tests 292 | 2. Add the endpoint that generates the pre-signed URL 293 | 3. Create a middleware capable of processing these URL 294 | 295 | Dependencies 296 | ============ 297 | 298 | None 299 | 300 | .. note:: 301 | 302 | This work is licensed under a Creative Commons Attribution 3.0 303 | Unported License. 304 | http://creativecommons.org/licenses/by/3.0/legalcode 305 | 306 | -------------------------------------------------------------------------------- /specs/liberty/tests-refactoring.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | =================== 14 | Tests refactoring 15 | =================== 16 | 17 | Include the URL of your launchpad blueprint: 18 | 19 | https://blueprints.launchpad.net/zaqar/+spec/tests-refactoring 20 | 21 | Zaqar tests are currently split into 2 different places, we want to reunite 22 | them. 23 | 24 | Problem description 25 | =================== 26 | 27 | Tests currently exists in the tests top directory in the tree, but most of them 28 | import abstract base classes from the zaqar.tests python package and set 29 | attributes to be able to run them. There are also some tests definition in the 30 | tests directory. It's not obvious that tests in the zaqar.tests package are not 31 | run when using tox and in the gate, and it creates confusion on where the tests 32 | should properly live. 33 | 34 | Proposed change 35 | =============== 36 | 37 | Move all the tests in the zaqar.tests package, and delete the tests directory. 38 | 39 | To be able to have a reviewable stream, we will slowly add directories in 40 | zaqar/tests to the testrepository configuration, and move them in small chunks. 41 | 42 | Alternatives 43 | ------------ 44 | 45 | None. 46 | 47 | Implementation 48 | ============== 49 | 50 | Assignee(s) 51 | ----------- 52 | 53 | Primary assignee: 54 | therve 55 | 56 | Milestones 57 | ---------- 58 | 59 | Target Milestone for completion: 60 | Liberty-1 61 | 62 | Work Items 63 | ---------- 64 | 65 | 1. Add one tests directory to .testr.conf and move the tests over 66 | 2. Iterate over all the tests directories in zaqar/tests 67 | 3. Move remaining tests out of tests/ 68 | 69 | Dependencies 70 | ============ 71 | 72 | None. 73 | 74 | .. note:: 75 | 76 | This work is licensed under a Creative Commons Attribution 3.0 77 | Unported License. 78 | http://creativecommons.org/licenses/by/3.0/legalcode 79 | 80 | -------------------------------------------------------------------------------- /specs/mitaka/index.rst: -------------------------------------------------------------------------------- 1 | ======================= 2 | Mitaka Specifications 3 | ======================= 4 | 5 | .. toctree:: 6 | :glob: 7 | :maxdepth: 2 8 | 9 | * -------------------------------------------------------------------------------- /specs/mitaka/more-reserved-queue-attributes.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ============================== 14 | More reserved queue attributes 15 | ============================== 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/more-reserved-queue-attributes 18 | 19 | Currently Zaqar supports setting metadata/attributes when user creating queue. 20 | However, the metadata/attributes are not used very much by Zaqar itself. 21 | Now we only support '_flavor' but it would be nice if we can support 22 | more attributes to make the queue more flexible. 23 | 24 | Problem description 25 | =================== 26 | 27 | Now Zaqar supports max_messages_post_size, default_message_ttl, 28 | default_claim_ttl and default_claim_grace. All of them defined in Zaqar's 29 | configuration file. That means it's global setting, all queues have to share 30 | the same configurations. It's not really flexible from the end user 31 | perspective. So it would be nice if we can let end user set them and use 32 | the default value in configuration file if they are not defined. 33 | 34 | Proposed change 35 | =============== 36 | 37 | The changes are not complicated, which may change two actions: Post messages 38 | and create claim for a queue. 39 | 40 | 1. Post Message: 41 | At line stable/liberty/zaqar/transport/wsgi/v2_0/messages.py#L182, we need 42 | to get the queue's metadata so as to get the queue's max_messages_post_size 43 | and ttl defined by user when creating the queue. If those aren't exist, 44 | then just use the global configuration defined in configuration file. 45 | 46 | 2. Create Claim: 47 | Similar thing like above, we need to get the queue's metadata at line 48 | stable/liberty/zaqar/transport/wsgi/v2_0/claims.py#L81 and then use that 49 | to do the check. 50 | 51 | Drawbacks 52 | --------- 53 | 54 | It may introduce a tiny performance impact for message posting and claim 55 | creating. For example, instead of just checking if the queue is existing or 56 | not, Zaqar need to get the queue's metadata so that to get those attributes. 57 | But actually, we can use 'get_metadata' replace 'exists' to check if the queue 58 | exist or not, and they're calling the same collection.find_one() method, so 59 | technically, there is no performance impact. 60 | 61 | Alternatives 62 | ------------ 63 | 64 | Just keep current way, let all the queues share the same configurations. 65 | 66 | Implementation 67 | ============== 68 | 69 | Assignee(s) 70 | ----------- 71 | 72 | Primary assignee: 73 | flwang (flwang@catalyst.net.nz) 74 | 75 | Work Items 76 | ---------- 77 | 78 | 1. Support max_messages_size and ttl as queue attributes 79 | 2. Support claim_ttl and claim_grace as queue attributes 80 | 81 | 82 | Dependencies 83 | ============ 84 | 85 | N/A 86 | 87 | -------------------------------------------------------------------------------- /specs/mitaka/websocket-binary-support.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | =================================================== 14 | Support for binary data in the websocket transport 15 | =================================================== 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/websocket-binary-support 18 | 19 | Implement support for binary communications over websocket transport which will 20 | allow to reduce network traffic between Zaqar and it's clients and also to 21 | increase performance. 22 | 23 | Problem description 24 | =================== 25 | 26 | Currently websocket transport uses only text messages packed in JSON format to 27 | exchange data with clients. 28 | 29 | An attempt to send binary data to Zaqar's websocket transport will lead to 30 | receiving 'bad request' (HTTP error 400) response from Zaqar. 31 | 32 | But websocket protocol actually allows communication via binary messages. 33 | Data packed(serialized) in binary format can be smaller and also may take less 34 | CPU time to pack/unpack it. 35 | 36 | By using binary messages for communications it's possible to reduce network 37 | traffic between Zaqar and it's clients and also to increase Zaqar's 38 | performance. It can also increase client performance if the client is not 39 | written in JavaScript. 40 | 41 | Proposed change 42 | =============== 43 | 44 | We can serialize/deserialize data via MsgPack technology which was already 45 | proposed and approved in persistent-transport_ specification. 46 | 47 | .. _persistent-transport: https://github.com/openstack/zaqar-specs/blob/master/specs/kilo/approved/persistent-transport.rst 48 | 49 | Deserialization of incoming messages to Zaqar 50 | --------------------------------------------- 51 | 52 | As you can see in `protocol.py#L55`_, Zaqar already can distinguish binary and 53 | text messages. 54 | 55 | .. _protocol.py#L55: https://github.com/openstack/zaqar/blob/stable/liberty/zaqar/transport/websocket/protocol.py#L55 56 | 57 | So we will make Zaqar try to deserialize each received binary message via 58 | 'msgpack' library into a python object, just like Zaqar currently deserializes 59 | JSON messages via 'json' library. After successful deserialization the 60 | message can be processed as usual. 61 | 62 | Serialization of response and subscription messages from Zaqar 63 | -------------------------------------------------------------- 64 | 65 | The response from Zaqar will be serialized in the same format as the incoming 66 | request. 67 | 68 | The subscription messages from Zaqar will be serialized in binary or text 69 | format based on the format of subscription create request. 70 | 71 | Let's take a look at the case where the client suddenly changes it's messaging 72 | format while connection is open and subscription is still alive. There might 73 | be a problem when the client will not be able to deserialize the messages 74 | coming from the old subscription, because it has switched the format by 75 | changing some boolean variable (switch). But this situation is nearly 76 | impossible and, if not, the problem is solvable: 77 | 78 | #. It's hard to imagine a situation when it might be useful for the client to 79 | change it's serialization format. Such client will probably never exist. 80 | #. Even if such client will exist in the future, the client developer will 81 | take care of it by adding the code checking the format of each incoming 82 | message and deserializing the message accordingly inside "on message" 83 | method. Zaqar does exactly this, for example. All websocket implementations 84 | provide a convenient way to determine the format of each particular message. 85 | #. Even if the client developer will not take care of this potential problem 86 | (he is inexperienced, for example) and will make the client just expect 87 | all incoming messages in the new format after switching while the old 88 | subscription is still alive, the client will crash on the **first incoming 89 | message from this subscription** and the problem will not go unnoticed. 90 | 91 | 92 | Websocket html client example upgrade 93 | ------------------------------------- 94 | 95 | The websocket html `client example`_ will be changed to make it also being able 96 | to communicate with Zaqar via binary messages. 97 | 98 | .. _client example: https://github.com/openstack/zaqar/blob/stable/liberty/examples/websocket.html 99 | 100 | Drawbacks 101 | --------- 102 | 103 | The performance of web clients on JavaScript 104 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 105 | 106 | Seems like MsgPack serialization/deserialization works slower than JSON in 107 | JavaScript. Well, no surprise since JSON abbreviation expands to "JavaScript 108 | Object Notation". 109 | It's a drawback for Zaqar web clients as it decreases their performance. 110 | 111 | How the change addresses this drawback: 112 | 113 | #. The server is one and clients are many. The server processes much more 114 | messages than each particular client. So there are many cases when it's 115 | reasonable to use binary serialization to increase server performance by 116 | sacrificing performance of JavaScript clients. 117 | 118 | #. The additional delays on JavaScript clients are slightly compensated by 119 | faster responses from the server. 120 | 121 | #. JavaScript clients can always use good old text format for communications 122 | with the server, while other clients can use binary format with the same 123 | server simultaneously. 124 | 125 | Alternatives 126 | ------------ 127 | 128 | These alternatives completely exclude the nearly impossible case where the 129 | client may suddenly switch it's messaging format, while the connection is open 130 | and the old subscription is still alive, and while proper deserialization of 131 | incoming messages to the client is managed by some client's boolean variable 132 | (switch). 133 | 134 | Alternatives of serialization of response and subscription messages from Zaqar 135 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 136 | 137 | Optional parameter approach 138 | """"""""""""""""""""""""""" 139 | 140 | The response from Zaqar will be serialized in the same format as the incoming 141 | request. 142 | 143 | The subscription messages from Zaqar will be serialized in binary 144 | or text format based on the new optional parameter passed in the message 145 | body of subscription create request. We can add this parameter to websocket 146 | API, name it ``accept``, allow it to have only two possible values: ``json`` 147 | and ``msgpack`` and make ``json`` value as default if ``accept`` parameter was 148 | not passed. 149 | By making ``accept`` parameter optional, we are achieving backward 150 | compatibility. 151 | 152 | Subscription request message example before implementation:: 153 | 154 | { 155 | 'action': 'subscription_create', 156 | 'headers': {'Client-ID': 31209ff3-ba03-4cec-b4ca-655f4899f8aa, 157 | 'X-Project-ID': superproject}, 158 | 'body': {'queue_name': 'superqueue', 'ttl': 3600} 159 | } 160 | 161 | Subscription request message after implementation can be the same as before or 162 | it can also pass ``accept`` parameter explicitly:: 163 | 164 | { 165 | 'action': 'subscription_create', 166 | 'headers': {'Client-ID': 31209ff3-ba03-4cec-b4ca-655f4899f8aa, 167 | 'X-Project-ID': superproject}, 168 | 'body': {'queue_name': 'superqueue', 'ttl': 3600, 'accept': 'json'} 169 | } 170 | 171 | First message approach 172 | """""""""""""""""""""" 173 | 174 | The response and subscription messages from Zaqar will be serialized in binary 175 | or text format based on in which format the particular client has sent it's 176 | first successfully parsed message to Zaqar since connecting. 177 | 178 | Once the format of communications between Zaqar and the client was 179 | established by that way, the client will not be able to change it, unless the 180 | client will reconnect to Zaqar and send the first message in other format. 181 | 182 | Zaqar configuration approach 183 | """""""""""""""""""""""""""" 184 | 185 | The response and subscription messages from Zaqar will be serialized in binary 186 | or text format based on the boolean variable in Zaqar server configuration. 187 | 188 | Implementation 189 | ============== 190 | 191 | Assignee(s) 192 | ----------- 193 | 194 | Primary assignee: 195 | ubershy 196 | 197 | Milestones 198 | ---------- 199 | 200 | Target Milestone for completion: 201 | Mitaka-2 202 | 203 | Work Items 204 | ---------- 205 | 206 | None 207 | 208 | Dependencies 209 | ============ 210 | 211 | None 212 | 213 | .. note:: 214 | 215 | This work is licensed under a Creative Commons Attribution 3.0 216 | Unported License. 217 | http://creativecommons.org/licenses/by/3.0/legalcode 218 | 219 | -------------------------------------------------------------------------------- /specs/new-storage-driver-template.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ============================= 14 | The title of your blueprint 15 | ============================= 16 | 17 | Include the URL of your launchpad blueprint: 18 | 19 | https://blueprints.launchpad.net/zaqar/+spec/example 20 | 21 | Introduction paragraph -- why are we doing anything? 22 | 23 | Driver description 24 | ================== 25 | 26 | A detailed description of the driver. 27 | 28 | Proposed change 29 | =============== 30 | 31 | Here is where you cover the change you propose to make in detail. How do you 32 | propose to solve this problem? 33 | 34 | If this is one part of a larger effort make it clear where this piece ends. In 35 | other words, what's the scope of this effort? 36 | 37 | What are the driver's guarantees in terms of reliability? 38 | ========================================================= 39 | 40 | "Zaqar aims to be a fully-reliable service, therefore messages should never be 41 | lost under any circumstances except for when the message's expiration time 42 | (ttl) is reached..." 43 | 44 | What are the driver's guarantees in terms of scalability? 45 | ========================================================= 46 | 47 | Zaqar aims to be an easily scalable service. Zaqar provides an easy way to 48 | scale web-heads and to balance queue's load across several different storage 49 | clusters. However, the real scalability is provided by the storage itself. If 50 | the storage can't be unlimitedly scaled or it's hard to scale, it'll break this 51 | guarantee. Supported storage must be good for general use-cases and easy to 52 | scale unlimitedly. It's fine to have storage drivers for very specific 53 | use-cases but it's likely not going to be possible to maintain those drivers. 54 | 55 | What are the driver's guarantees in terms of interoperability? 56 | ============================================================== 57 | 58 | In order for applications to easily talk to different clouds supporting Zaqar, 59 | it is necessary for the service to guarantee interoperability as much as 60 | possible. Lots of the interoperability guarantees depend on how the service 61 | itself is deployed. However, the team strives to provide an interoperable 62 | service and guide operators on what the best way to deploy the service is. 63 | 64 | What are the driver's guarantees in terms of openness? 65 | ====================================================== 66 | 67 | Zaqar is an open-source software licensed under the Apache 2 license. It aims 68 | to be able to be installed in any deployments and circumstances. In order to do 69 | that, it tries to support technologies that will alow deployers to do so 70 | without any constraint. In order for the team to be able to maintain and 71 | support a storage driver, the driver has to be licensed under an open license 72 | and so has to be the technology the driver supports. The team won't be able to 73 | maintain drivers relying on closed technologies that cannot be tested/deployed. 74 | 75 | Implementation 76 | ============== 77 | 78 | Assignee(s) 79 | ----------- 80 | 81 | Who is leading the writing of the code? Or is this a blueprint where you're 82 | throwing it out there to see who picks it up? 83 | 84 | If more than one person is working on the implementation, please designate the 85 | primary author and contact. 86 | 87 | Primary assignee: 88 | 89 | 90 | Can list additional ids if they intend on doing substantial implementation work 91 | on this blueprint. 92 | 93 | Milestones 94 | ---------- 95 | 96 | Target Milestone for completion: 97 | Juno-2 98 | 99 | Work Items 100 | ---------- 101 | 102 | Work items or tasks -- break the feature up into the things that need to be 103 | done to implement it. Those parts might end up being done by different people, 104 | but we're mostly trying to understand the timeline for implementation. 105 | 106 | 107 | Dependencies 108 | ============ 109 | 110 | - Include specific references to specs and/or blueprints in zaqar, or in other 111 | projects, that this one either depends on or is related to. 112 | 113 | .. note:: 114 | 115 | This work is licensed under a Creative Commons Attribution 3.0 116 | Unported License. 117 | http://creativecommons.org/licenses/by/3.0/legalcode 118 | 119 | -------------------------------------------------------------------------------- /specs/newton/index.rst: -------------------------------------------------------------------------------- 1 | ======================= 2 | Newton Specifications 3 | ======================= 4 | 5 | .. toctree:: 6 | :glob: 7 | :maxdepth: 2 8 | 9 | * -------------------------------------------------------------------------------- /specs/newton/lazy-queues-in-subscriptions.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ============================ 14 | Lazy queues in subscriptions 15 | ============================ 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/lazy-queues-in-subscriptions 18 | 19 | Make queues lazy on operations with subscriptions, so the user will be able to 20 | subscribe to yet unexisting queue. 21 | 22 | Problem description 23 | =================== 24 | 25 | Queues are designed to be lazy resources in Zaqar's API v1.1 and API v2 in 26 | messaging. That means, for example, that the user can post messages to 27 | unexisting queue, and the queue will be automatically created. But now queues 28 | do not behave like lazy resources on operations with subscriptions. Before 29 | creating a subscription, the user must ensure the queue exists, or Zaqar will 30 | send error response with HTTP code 400. 31 | 32 | Proposed change 33 | =============== 34 | 35 | Make queues lazy on operations with subscriptions, so the user will be able 36 | to subscribe to yet unexisting queue. Make Zaqar do not send error response 37 | on queue creation operation in the case queue does not exists. 38 | 39 | Advantages: 40 | 41 | #. The behavior of API will be more consistent, because after the change 42 | queues will be lazy in all situations. 43 | #. It will be easier for the user to work with subscriptions. No need to do 44 | pre-check for queue existence, or handle 400 error response from Zaqar, or 45 | always pre-create queue. 46 | #. There will be no need to solve currently existing problem when the queue was 47 | deleted, but subscriptions to it are still existing. Problem solving will 48 | probably require storage driver code to delete also all subscriptions to 49 | a queue, when deleting the queue. It may be very undesirable for the user, 50 | because subscriptions will not guarantee to be existing till TTL expiration, 51 | and will need to be renewed also after queue delete and queue create 52 | operations. Proper solution to this problem will complicate behavior in many 53 | ways in both Zaqar and client's code. It will be probably backward 54 | incompatible change. 55 | 56 | The change is backward compatible. Client's code should work after the change. 57 | 58 | Drawbacks 59 | --------- 60 | 61 | None 62 | 63 | Alternatives 64 | ------------ 65 | 66 | Two alternatives: 67 | 68 | #. Keep the current behavior. 69 | #. There is opinion to make queues non-lazy resources in all situations like 70 | it was in API v1. But it will be backward incompatible change to our API 71 | v1.1 and API v2, so maybe make it in the distant future. 72 | 73 | Both of them will require solving the problem about still active subscriptions 74 | to a queue that was deleted. 75 | 76 | Implementation 77 | ============== 78 | 79 | Assignee(s) 80 | ----------- 81 | 82 | Primary assignee: 83 | ubershy 84 | 85 | Milestones 86 | ---------- 87 | 88 | Target Milestone for completion: 89 | Newton-1 90 | 91 | Work Items 92 | ---------- 93 | 94 | #. Remove checks for queue existence on operations with subscriptions from 95 | storage drivers. 96 | #. Remove the code that is catching ``QueueDoesNotExist`` exception on 97 | operations with subscriptions from transport drivers. 98 | #. Change functional and unit tests accordingly. 99 | 100 | Dependencies 101 | ============ 102 | 103 | None 104 | 105 | .. note:: 106 | 107 | This work is licensed under a Creative Commons Attribution 3.0 108 | Unported License. 109 | http://creativecommons.org/licenses/by/3.0/legalcode 110 | -------------------------------------------------------------------------------- /specs/newton/mistral-notifications.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ======================= 14 | Mistral Notifications 15 | ======================= 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/mistral-notifications 18 | 19 | Allow a message to a Zaqar queue to trigger a Mistral workflow via the Zaqar 20 | notification mechanism. 21 | 22 | Problem description 23 | =================== 24 | 25 | Developers of cloud applications expect to be able to build autonomous 26 | applications in the cloud. That is to say, applications that manage themselves 27 | by accessing the APIs of the cloud to manipulate their own infrastructure. 28 | (Examples of this include autoscaling and autorecovery.) This is one of the 29 | primary differences between a cloud platform and a simple virtualisation 30 | platform (the other being multi-tenancy). There are two parts to this that 31 | require integration, which is the purpose of this blueprint. 32 | 33 | The first is that the application must be able to receive information from the 34 | cloud. An example of this would be an Aodh alarm indicating that a server is 35 | overutilised. These notifications must be asynchronous, since the cloud is 36 | multitenant and cannot block waiting for any one user application to 37 | acknowledge it. They must also exhibit queueing semantics with at-least-once 38 | delivery and high durability, since the application may become unreliable if it 39 | misses notifications from the cloud. Not coincidentally, Zaqar offers exactly 40 | these semantics in a public, Keystone-authenticated API that is accessible to 41 | applications, and is therefore a natural choice. For this reason, a number of 42 | OpenStack projects have already started dispatching user notifications to Zaqar 43 | and more are expected in the near future. Already Aodh alarms support Zaqar as 44 | a target, and Heat can push stack and resources events as well as notifications 45 | about user hooks being triggered to Zaqar. 46 | 47 | The second is that the application must be able to perform arbitrary, and 48 | arbitrarily-complex actions. This is because in practice the Right Thing to do 49 | in cases like autoscaling and autorecovery is application-specific. There is 50 | also an entire universe of application-specific actions that a user might want 51 | to create. Of course an application can run these actions on a server 52 | provisioned with Nova, but this generally makes things more complex (and 53 | usually more expensive) than they need to be. For example, it is very hard to 54 | host autorecovery code on the servers that are being autorecovered themselves 55 | and still be reliable. Finally, OpenStack makes it difficult to provide 56 | appropriate Keystone credentials to servers provisioned with Nova. Mistral_ 57 | solves these problems by providing a lightweight, multi-tenant way of reliably 58 | running potentially long-running processes, with access to the OpenStack APIs 59 | as well as a number of other actions (some of which, like sending email and 60 | webhooks, are similar to Zaqar's notifications). 61 | 62 | The missing link to build fully autonomous applications is for messages 63 | (potentially, but not necessarily originating from the OpenStack cloud itself) 64 | on Zaqar queues to be able to trigger Mistral workflows (potentially, but not 65 | necessarily calling other OpenStack APIs). This would give developers of cloud 66 | applications an extremely flexible way of plugging together event-driven, 67 | application-specific, autonomous actions. 68 | 69 | .. _Mistral: https://wiki.openstack.org/wiki/Mistral 70 | 71 | Proposed change 72 | =============== 73 | 74 | Create a Zaqar notification sink plugin for Mistral. The effect of a 75 | notification to this sink would be to create a Mistral workflow Execution_ 76 | (i.e. to trigger a pre-existing Mistral workflow). 77 | 78 | The ``subscriber`` URI should be the URL of the Mistral executions endpoint, 79 | with the URI scheme ``trust+http`` or ``trust+https``. For example, 80 | ``trust+https://mistral.example.net/v2/executions``. This scheme indicates that 81 | Zaqar should create a Keystone trust that allows it to act on behalf of the 82 | user in making API calls to Mistral in the future. The trust ID will be 83 | inserted into the URL before it is stored in the form 84 | ``trust+http://trust_id@host/path``. This form is modelled after `the one used 85 | by Aodh`_. 86 | 87 | The trust lifetime should be slightly longer than the TTL of the subscription, 88 | or unlimited if there is no TTL for the subscription. Zaqar must delete the 89 | trust when deleting the subscription. 90 | 91 | When sending a notification, Zaqar will retrieve a trust token from Keystone 92 | using its own service user token and the trust ID stored in the URL. The trust 93 | token thus obtained should contain the correct tenant information to then make 94 | the request on behalf of the original user. 95 | 96 | Since in future Zaqar may want to make ``trust+http`` requests to other API 97 | endpoints, it should distinguish on more than just the URI scheme. When the 98 | subscription is created, Zaqar should need compare the URI with the Mistral 99 | executions endpoint URL obtained with the help of the Keystone catalog in order 100 | to distinguish between Mistral workflow triggers and ordinary webhooks. 101 | Fortunately, the URL is fixed for a given cloud, so the catalog would probably 102 | only need to be read once and it would be a straight string comparison from 103 | there. 104 | 105 | The ``options`` dict should contain the following keys: 106 | 107 | * ``workflow_id`` - The ID of the workflow to trigger 108 | * ``params`` - a dict of parameters that varies depending on the workflow type. 109 | e.g. a "reverse workflow" takes a ``task_name`` parameter to define the 110 | target task. 111 | * ``input`` - an arbitrary dict of keys and values to be passed as 112 | input to every workflow execution triggered by this notification. 113 | 114 | When creating the Mistral execution, the contents of the message and (later) 115 | the message ID will be passed in the environment (the ``env`` key in the 116 | ``params``). This allows the workflow to access the message data, but does not 117 | require it to declare a particular input for it (so the notification can be 118 | used to trigger *any* workflow). The message contents, interpreted as JSON, 119 | will be passed in a Mistral environment variable named ``notification``. When 120 | Zaqar supports passing the message id in a notification, it will be sent as the 121 | Mistral environment variable ``notification_id``. If these names conflict with 122 | the ``env`` passed by the user in ``params``, the user-provided data will be 123 | overwritten with that received in the message. Any other keys in the user's 124 | ``env`` will be preserved. If the user does not specify an ``env``, one will be 125 | created. The ``input`` dict, ``workflow_id`` and all other ``params`` will be 126 | passed through unmodified. 127 | 128 | While all the data is available to do a raw HTTP request, it is preferable if 129 | these calls are made through the python-mistralclient library. 130 | 131 | .. _Execution: https://docs.openstack.org/mistral/latest/#executions 132 | .. _the one used by Aodh: https://docs.openstack.org/aodh/latest/#trust-http 133 | 134 | Alternatives 135 | ------------ 136 | 137 | Instead of a push model, where Zaqar takes messages and notifies Mistral, it 138 | would also be possible to use a pull model where Mistral polls Zaqar topics for 139 | messages. However, while the Zaqar notification implementation already exists, 140 | there is no such existing component in Mistral that would be suitable for 141 | polling for triggers. It would need to poll large numbers of topics in 142 | different tenants. A similar design was considered and rejected for the 143 | notification feature of Zaqar; the same arguments apply here. 144 | 145 | An alternative authentication method might be to use pre-signed URLs, which are 146 | on the `Mistral roadmap`_. This might be quicker to implement, but in the 147 | longer term, Keystone trusts are probably preferable. 148 | 149 | Instead of whitelisting the Mistral executions URL, the ``trust+http`` scheme 150 | could be used to make requests to any OpenStack endpoint. However, in general 151 | the correct method of combining static information from the ``options`` dict 152 | with the contents of the message to obtain the call parameters will be 153 | different for every API. Since Mistral can already call most OpenStack APIs and 154 | supports a language (YAQL) for calculating the arguments using data from the 155 | notification and other input, the simplest way to achieve this is for the user 156 | to encapsulate any other OpenStack API call they wish to make in a Mistral 157 | workflow (which also allows them to define custom error handling). 158 | 159 | It would be nice if there were a way to identify an OpenStack resource with a 160 | URI without necessarily requiring a URL (containing redundant information about 161 | the location of the endpoint). AWS uses an `unofficial URN-like identifier`_ 162 | with an arn: (instead of urn:) scheme for this purpose. Something similar might 163 | be useful in other contexts in OpenStack too (for example, in Heat we would 164 | like to be able to distinguish between files in Swift containers or Glare links 165 | and ordinary HTTP URLs for the purposes of uploading user data, although there 166 | is some precedent for using ``swift+http`` as the scheme in the Swift case). 167 | However, this would require, at a minimum, wide cross-project agreement (and 168 | arguably IANA registration). There are no existing examples of anything like 169 | this in OpenStack. 170 | 171 | .. _Mistral roadmap: https://wiki.openstack.org/wiki/Mistral/Roadmap 172 | .. _unofficial URN-like identifier: http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html 173 | 174 | Implementation 175 | ============== 176 | 177 | Assignee(s) 178 | ----------- 179 | 180 | This is one of those blueprints where I'm throwing it out there to see who 181 | picks it up. 182 | 183 | Milestones 184 | ---------- 185 | 186 | Target Milestone for completion: 187 | Newton-3 188 | 189 | Work Items 190 | ---------- 191 | 192 | * Implement the Mistral notification plugin 193 | * Create a keystone trust and store its ID in the URI when setting up a 194 | ``trust+http(s)`` notification. Delete the trust again when the notification 195 | is deleted. 196 | * Add the ability to distinguish between Mistral URLs and other 197 | ``trust+http(s)`` URLs in the notification URI 198 | 199 | Dependencies 200 | ============ 201 | 202 | We won't be able to pass the message ID until 203 | https://review.opendev.org/#/c/276968/ or something equivalent merges. 204 | However, since it can be added to the Mistral environment later without 205 | rewriting any existing workflows (to declare a new input), this is in no way a 206 | blocker. 207 | 208 | .. note:: 209 | 210 | This work is licensed under a Creative Commons Attribution 3.0 211 | Unported License. 212 | http://creativecommons.org/licenses/by/3.0/legalcode 213 | 214 | -------------------------------------------------------------------------------- /specs/newton/subscription-confirmation-support.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | =================================== 14 | Subscription confirmation support 15 | =================================== 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/subscription-confirmation-support 18 | 19 | Support subscribers confirm that whether they want the notifications or not 20 | before Zaqar sends them. 21 | 22 | Problem description 23 | =================== 24 | 25 | Currently Zaqar can post notifications to the subscriber once the subscription 26 | is created. It doesn't need the subscriber to confirm the subscription. 27 | This will lead a problem that users could send junk information that whether 28 | subscriber wants or not. 29 | 30 | Proposed change 31 | =============== 32 | 33 | With `Amazon SNS`_ as reference, we will support to send HTTP POST request 34 | that contains subscription confirmation information(e.g. message type, 35 | pre-signed confirmation URL, signature information) to subscription endpoint. 36 | The endpoint needs to use the URL in notification to confirm subscription. 37 | When the subscription is deleted, Zaqar server will send a notification that 38 | the subscription with ID has been deleted. This mechanism tells endpoint no 39 | need to wait notifications any more. 40 | 41 | * The message types will be SubscriptionConfirmation, Notification 42 | and UnsubscribeConfirmation. 43 | * For backward compatibility, we also should add new option in Zaqar 44 | configuration file like this:: 45 | 46 | [notification] 47 | require_confirmation=true/false 48 | 49 | If the user omits this option in config, it will be 'false' by default. 50 | In further, we may change the default value to 'true' to let subscription 51 | more secure. 52 | 53 | So the workflow will be this in webhook: 54 | 55 | #. User creates subscription in Zaqar. 56 | #. Zaqar sends a HTTP POST request with subscription confirmation information. 57 | This request is like this: 58 | 59 | POST URL:: 60 | 61 | http://endpointUrl 62 | 63 | Request Body:: 64 | 65 | { 66 | 'MessageType': 'SubscriptionConfirmation', 67 | 'Message': 'You have chosen to subscribe to the queue: xxx.', 68 | 'URL-Signature': 'xxxx', 69 | 'URL-METHODS': 'PUT', 70 | 'URL-PATHS': '/v2/queues/{queue_name}/subscriptions/{subscriptions_id}/confirm', 71 | 'X-PROJECT-ID': 'xxxx', 72 | 'URL-EXPIRES': '3600-01-01T00:00:00', 73 | 'SubscribeURL': 'https://zaqar_server/v2/queues/{queue_name}/subscriptions/{subscriptions_id}/confirm', 74 | 'SubscribeBody': {'confirmed': True}, 75 | 'UnsubscribeBody': {'confirmed': False}, 76 | } 77 | #. Subscriber confirms this subscription by using those information 78 | in request body, and sends the pre-signed confirmation request to Zaqar. 79 | It's a new endpoint in Zaqar. 80 | This request is like this: 81 | 82 | PUT URL:: 83 | 84 | https://zaqar_server/v2/queues/{queue_name}/subscriptions/{subscriptions_id}/confirm 85 | 86 | Request Body:: 87 | 88 | { 89 | 'confirmed': True 90 | } 91 | 92 | Meanwhile, HTTP headers are also required by using pre-signed feature 93 | in Zaqar. If subscriber wants to unsubscribe this subscription, just needs 94 | to set 'confirmed' to False. 95 | If Zaqar processes this request successfully, Zaqar will response with: 96 | 97 | Response code:: 98 | 99 | 204 100 | 101 | If Zaqar has failed processsing this request because of some error, it will 102 | return error message in response body. 103 | #. Zaqar begins to send notifications to this endpoint after confirmation 104 | workflow is finished. 105 | 106 | .. note:: 107 | 108 | If Zaqar server didn't receive the response or subscriber didn't sent it 109 | since something wrong, subscription will keep 'confirmed' as false, and 110 | Zaqar will support to resend confirmation request by calling subscription 111 | creation API again. 112 | 113 | So the workflow will be this in email: 114 | 115 | #. User creates subscription in Zaqar. 116 | #. Zaqar sends an email with subscription confirmation page link. 117 | Email context is like this:: 118 | "You have chosen to subscribe to the queue: {queue name}." 119 | "This queue belongs to project: {project id}." 120 | "To confirm this subscription, click or visit this linke below:" 121 | "{confirmation link}" 122 | #. When subscriber visits this link, he gets a page which calls 123 | Zaqar by ajax request to confirm the subscription as same as 124 | webhook way. 125 | #. The page shows information about was the subscription successfully 126 | confirmed or not. 127 | #. Zaqar begins to send notifications to this endpoint. 128 | 129 | .. note:: 130 | 131 | About the confirmation page, Zaqar will host a sample page in Zaqar 132 | tree, but it's not a part of Zaqar service. Cloud administrator should 133 | provide hosting for this page manually. After this page is hosted 134 | successfully, cloud administrator should specify this page location 135 | in Zaqar config "external_confirmation_url". 136 | 137 | Drawbacks 138 | --------- 139 | None 140 | 141 | Alternatives 142 | ------------ 143 | Another option is when subscriber creates a subscription, he also need to 144 | call Zaqar's confirmation API to confirm it before confirmation is expired. 145 | In this option, Zaqar will not send any confirmation notification by itself. 146 | 147 | Data model impact 148 | ----------------- 149 | Add new status "confirmed" into subscription model. 150 | 151 | Comparison AWS SNS API and Zaqar API 152 | ------------------------------------ 153 | For developer to better understand this feature, this spec compares SNS API 154 | and Zaqar API what difference between them. 155 | 156 | Subscription Confirmation Request 157 | SNS is using GET request like this:: 158 | 159 | GET https://sns.us-west-2.amazonaws.com/?Action=ConfirmSubscription&TopicArn={Topic}&Token={Token} 160 | 161 | Zaqar is using PUT request like this:: 162 | 163 | PUT https://zaqar_server/v2/queues/{queue_name}/subscriptions/{subscriptions_id}/confirm 164 | 165 | Request Body:: 166 | 167 | { 168 | 'confirmed': True 169 | } 170 | 171 | 172 | Implementation 173 | ============== 174 | 175 | Assignee(s) 176 | ----------- 177 | 178 | Primary assignee: 179 | wangxiyuan 180 | 181 | Secondary assignee: 182 | wanghao 183 | 184 | Milestones 185 | ---------- 186 | 187 | Target Milestone for completion: 188 | Newton-3 189 | 190 | Work Items 191 | ---------- 192 | 193 | #. Implement subscriber confirmation API in wsgi. 194 | #. Update subscription resource to add new property "confirmed" 195 | on MongoDB driver. 196 | #. Add new config options. 197 | #. Update notification to send confirmation message in way email. 198 | #. Implement subscriber confirmation API in websocket. 199 | #. Update subscription resource on other drivers. 200 | #. Update notification to send confirmation message in way webhook. 201 | #. Add message type. 202 | 203 | Dependencies 204 | ============ 205 | 206 | _`Amazon SNS`: http://docs.aws.amazon.com/sns/latest/dg/SendMessageToHttp.html 207 | -------------------------------------------------------------------------------- /specs/ocata/index.rst: -------------------------------------------------------------------------------- 1 | ======================= 2 | Ocata Specifications 3 | ======================= 4 | 5 | .. toctree:: 6 | :glob: 7 | :maxdepth: 2 8 | 9 | * 10 | -------------------------------------------------------------------------------- /specs/ocata/osprofiler.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the 3 | git repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be 5 | named awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this template. If you 8 | have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html To test 11 | out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ========== 14 | OSProfiler 15 | ========== 16 | 17 | OpenStack consists of multiple projects, composed of multiple services, 18 | working together to make different services available for the users. 19 | In case something is not working as expected, its a really complex task 20 | to determine what is going wrong and to locate the module causing the 21 | bottleneck. 22 | 23 | The OSProfiler tool aims to solve this issue by making possible to generate 24 | one trace per request affecting all involved services and build a tree of 25 | calls. 26 | 27 | More details on how OSProfiler works can be find in the 28 | `OSProfiler Readme `_ 29 | 30 | This is even more important for a project like Zaqar in which performance 31 | in communications is a key feature. 32 | 33 | https://blueprints.launchpad.net/zaqar/+spec/osprofiler 34 | 35 | Problem description 36 | =================== 37 | 38 | Currently there is no way to easily detect Zaqar's bottlenecks. 39 | 40 | Now that some projects have successfully integrated with OSProfiler, 41 | it is important to enable OSProfiler for Zaqar as well and contribute 42 | by providing a continuous trace information view for a request traversing 43 | the different services. 44 | 45 | This spec proposes adding the OSProfiler tool to Zaqar. 46 | 47 | Proposed change 48 | =============== 49 | 50 | The proposed change is to integrate the OSProfiler tool to Zaqar. 51 | According to the requirements for this addition detailed in 52 | `OSProfiler Readme `_, 53 | this shouldn't be a complex task. The steps required to do this are detailed 54 | in the `Work Items`_ section. 55 | 56 | Main functional change for Zaqar server: 57 | https://review.opendev.org/#/c/141356/ 58 | 59 | Alternatives 60 | ------------ 61 | 62 | - Not to add the OSProfiler tool 63 | 64 | Implementation 65 | ============== 66 | 67 | Assignee(s) 68 | ----------- 69 | 70 | Primary assignee: zhiyan 71 | 72 | Other assignee: wangxiyuan(wxy) 73 | 74 | Milestones 75 | ---------- 76 | 77 | Target Milestone for completion: O-2 78 | 79 | Work Items 80 | ---------- 81 | 82 | * Add initialization of OSProfiler when Zaqar server starts 83 | 84 | * Add tracing support for WSGI 85 | 86 | * Add tracing support for MongoDB 87 | 88 | * Add tracing support for Redis 89 | 90 | * Add tracing support for SQLAlchemy 91 | 92 | * Add tracing support for Pooling 93 | 94 | * Add profiler CONF group 95 | 96 | * Enable or disable profiling fully 97 | 98 | * Enable or disable MongoDB tracing 99 | 100 | * Enable or disable Redis tracing 101 | 102 | * Enable or disable SQLAlchemy tracing 103 | 104 | * Enable or disable Pooling tracing 105 | 106 | * Configure HMAC key(s) to protect trace data 107 | 108 | * Add profile support for client 109 | 110 | * Test the "resting" overhead when profiling is disabled and without feature 111 | 112 | * Update document(s) to make sure new options are well documented. 113 | 114 | .. note:: 115 | 116 | Usually DB requests create a lot of trace info, so its important to 117 | be able to enable/disable those on demand 118 | 119 | Dependencies 120 | ============ 121 | 122 | None 123 | 124 | .. note:: 125 | 126 | This work is licensed under a Creative Commons Attribution 3.0 127 | Unported License. 128 | http://creativecommons.org/licenses/by/3.0/legalcode 129 | -------------------------------------------------------------------------------- /specs/ocata/purge-queue.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | =========== 14 | Purge queue 15 | =========== 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/purge-queue 18 | 19 | Though user can delete messages, claims or subscriptions from a queue, but 20 | there is no way to delete many of them at once. 21 | 22 | Problem description 23 | =================== 24 | 25 | Now Zaqar is missing a fast way to delete many resources at once for a given 26 | queue. This could be handy for user to clean the queue and keep all the 27 | metadata of the queue. 28 | 29 | Proposed change 30 | =============== 31 | 32 | New endpoint for action 'purge' for queue: 33 | 34 | /v2/queues/myqueue/purge 35 | 36 | POST body: 37 | 38 | .. code:: json 39 | 40 | {"resource_types": ["messages", "subscriptions"]} 41 | 42 | So the idea is if there is no POST body, by default, all the resources under 43 | the queue will be delete. Otherwise, if the key 'resource_types' in the POST 44 | body, then Zaqar will delete resources based on the given resource types. It's 45 | a list and it could be one of the combinations of the two types: 'messages', 46 | and 'subscriptions'. 47 | 48 | Currently, it's hard to list "claims" under a queue, so this feature won't 49 | support clean "claims". 50 | 51 | Drawbacks 52 | --------- 53 | 54 | N/A 55 | 56 | Alternatives 57 | ------------ 58 | 59 | User has to delete messages and subscriptions of the queue manually. 60 | 61 | Implementation 62 | ============== 63 | 64 | Assignee(s) 65 | ----------- 66 | 67 | Primary assignee: 68 | flwang (flwang@catalyst.net.nz) 69 | 70 | Work Items 71 | ---------- 72 | 73 | 1. Add a new method 'purge' for storage queue controller which will delete all 74 | messages and subscriptions under the queue. 75 | 76 | 77 | Dependencies 78 | ============ 79 | 80 | N/A 81 | 82 | -------------------------------------------------------------------------------- /specs/ocata/swift-storage.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ===================== 14 | Swift storage backend 15 | ===================== 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/swift-storage 18 | 19 | Swift is a widely deployed object storage solution for OpenStack. While it's 20 | not particularly designed for high throughput of lots of small objects, which 21 | is what Zaqar mostly does, it scales really well, and is highly fault tolerant. 22 | It would also provide an alternative to people that want to avoid to deploy 23 | and manage MongoDB. 24 | 25 | Driver description 26 | ================== 27 | 28 | The driver only implements the data part, SQL being available for the 29 | control driver. Containers are used per entity and per project, to provide 30 | multitenancy. 31 | 32 | Proposed change 33 | =============== 34 | 35 | The driver uses one global service user, and stores all messages in one 36 | project. It uses suffix on container names to scope them. It will use mostly 37 | one container per entity type, plus other for additional indexes. We may need 38 | to increase or decrease the number of containers used, to find the appropriate 39 | performance balance. 40 | 41 | It may be worth marking the driver experimental first, so that we can change 42 | the storage model without caring about migrating data. 43 | 44 | What are the driver's guarantees in terms of reliability? 45 | ========================================================= 46 | 47 | We rely on Swift being configured appropriately to provide message reliability. 48 | By default objects are replicated 3 times, which offers good guarantees. 49 | 50 | What are the driver's guarantees in terms of scalability? 51 | ========================================================= 52 | 53 | Again, we rely on Swift scalability guarantees here. 54 | 55 | What are the driver's guarantees in terms of interoperability? 56 | ============================================================== 57 | 58 | We use publicly available Swift APIs, so the data is easily auditable and 59 | recoverable behind Zaqar. 60 | 61 | What are the driver's guarantees in terms of openness? 62 | ====================================================== 63 | 64 | We use python-swiftclient to access Swift, which uses the Apache 2.0 license. 65 | 66 | Implementation 67 | ============== 68 | 69 | Assignee(s) 70 | ----------- 71 | 72 | Primary assignee: 73 | therve 74 | 75 | 76 | Milestones 77 | ---------- 78 | 79 | Target Milestone for completion: 80 | Ocata-3 81 | 82 | Work Items 83 | ---------- 84 | 85 | * Implement storage driver, and make unit tests pass. 86 | * Add a voting gate against Swift. 87 | * Possibly make some performance testing to tweak how objects are stored. 88 | 89 | 90 | Dependencies 91 | ============ 92 | 93 | - Depend on Swift fix for if-none-match: https://review.opendev.org/395582 94 | 95 | .. note:: 96 | 97 | This work is licensed under a Creative Commons Attribution 3.0 98 | Unported License. 99 | http://creativecommons.org/licenses/by/3.0/legalcode 100 | 101 | -------------------------------------------------------------------------------- /specs/pike/dead_letter_queue.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ================= 14 | Dead Letter Queue 15 | ================= 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/dead-letter-queue 18 | 19 | 20 | Problem description 21 | =================== 22 | 23 | Currently, Zaqar can't do anything if the message has been claimed many times 24 | and still can't be processed successfully. That could be nice if Zaqar can 25 | provide the dead letter queue support so that user can provide an existing 26 | queue A for queue B's dead letter queue. Dead letter queue storing these 27 | messages allows developers to look for common patterns and potential software 28 | problems. 29 | 30 | Proposed change 31 | =============== 32 | 33 | 1. Add three new reserved attributes for queue's metadata: ``_max_claim_count`` 34 | and ``_dead_letter_queue``. 35 | 36 | ``_max_claim_count`` is the max number the message can be claimed. Generally, 37 | it means the message cannot be processed successfully. There is no default 38 | value for this attribute. If it's not set, then that means this feature won't 39 | be enabled for current queue. 40 | 41 | ``_dead_letter_queue`` is the target the message will be moved to when the 42 | message can't processed successfully after meet the max claim count. It's not 43 | supported to add queue C as the dead letter queue for queue B where queue B has 44 | been set as a dead letter queue for queue A. It's not the case we want to 45 | address in this spec. Technically, even though user can set a dead letter queue 46 | C for queue B(when queue B is a dead letter queue of queue A), because the 47 | moving is triggered by claim, so that means if user won't explicitly claim 48 | messages in queue B, then nothing happened. 49 | 50 | ``_dead_letter_queue_messages_ttl`` is the new TTL setting for messages when 51 | moved to DLQ. If it's not set, current TTL will be kept. 52 | 53 | 2. Make sure the DLQ and current queue are in the same pool when doing 54 | validation to get better performance. 55 | 56 | 3. Add a new filed for message in database which can be used to save the claim 57 | count. 58 | 59 | 4. Changes for claim create method 60 | 61 | 4.1. get current claim count from the message, update the claim count by 62 | plus one. 63 | 64 | 4.2. check if the claim count has exceeded the max claim count defined in 65 | queue's metadata. If it doesn't exceed the ``_max_claim_count``, just do 66 | normal claim. 67 | 68 | 4.3. If the claim count does exceed the ``_max_claim_count``, move the 69 | message to the dead letter queue directly. Given the goal of DLQ, we'd like 70 | to keep the claim information of the message. So depends on the storage 71 | backend, the "move" could introduce a limitation. For example, for MongoDB 72 | backend, we may have to move the message from the source queue to DLQ by 73 | changing the queue name directly. Which means the two have to be created on 74 | same storage pool. 75 | 76 | After moved the message, Zaqar will do nothing against those 77 | messages in dead letter queue. User could/will use these message to debug 78 | why those messages can't be processed successfully. In other words, allows 79 | developers to look for common patterns and potential software problems from 80 | those messages. 81 | 82 | 4.4. messages moved to DLQ will be updated their TTL based on the setting 83 | ``_dead_letter_queue_messages_ttl``. If it's not set, currentl TTL of 84 | messages will be kept. 85 | 86 | 87 | Drawbacks 88 | --------- 89 | 90 | Based on current design, we may have a limitation because of the technical 91 | trade off. When move the message from source queue to dead letter queue, it'd 92 | be nice to keep the claim info, but if we want to do that, we have to make sure 93 | the two queues are on same storage pool. The implementation could be different 94 | on different back ends. 95 | 96 | Alternatives 97 | ------------ 98 | 99 | None 100 | 101 | Implementation 102 | ============== 103 | 104 | Assignee(s) 105 | ----------- 106 | 107 | Primary assignee: 108 | flwang 109 | 110 | Milestones 111 | ---------- 112 | 113 | Target Milestone for completion: 114 | Pike-3 115 | 116 | Work Items 117 | ---------- 118 | 119 | #. Add _max_claim_count and _dead_letter_queue as reserved attributes in queue 120 | metadata 121 | #. Add validation for the new reserved attributes 122 | #. Add a new filed for message to count the claim, this needs to be done 123 | against maongoDB, swift and redis. 124 | #. Add change in claim create which will add the claim count for messages and 125 | check if the count has exceed the max claim count defined in queue's 126 | metadata. If it has exceeded, move the message to the dead letter queue. 127 | #. Add release note for this feature 128 | #. Update API reference 129 | #. Add user/developer document for this feature 130 | #. Change unit, functional and tempest tests accordingly. 131 | 132 | Dependencies 133 | ============ 134 | 135 | None 136 | 137 | .. note:: 138 | 139 | This work is licensed under a Creative Commons Attribution 3.0 140 | Unported License. 141 | http://creativecommons.org/licenses/by/3.0/legalcode 142 | -------------------------------------------------------------------------------- /specs/pike/index.rst: -------------------------------------------------------------------------------- 1 | ===================== 2 | Pike Specifications 3 | ===================== 4 | 5 | .. toctree:: 6 | :glob: 7 | :maxdepth: 2 8 | 9 | * -------------------------------------------------------------------------------- /specs/pike/notification-delivery-policy.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ============================ 14 | Notification Delivery Policy 15 | ============================ 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/notification-delivery-policy 18 | 19 | Sometimes, because of network/internet connection issue or the load of the 20 | target services/applications, the notification sent from Zaqar to the 21 | subscriber may fail, so it would be nice if we can support a retry and 22 | user can define the retry policy in the options of subscription. 23 | 24 | This feature is only for HTTP/HTTPS subscription way, not including the Email 25 | notification. 26 | 27 | Problem description 28 | =================== 29 | 30 | Now Zaqar didn't have notification delivery policy, that means there is no 31 | retries if endpoint has some problem that can't receive the notification. 32 | For users or applications, it's missing important message without notification. 33 | 34 | AWS SNS service has supported the policy, that would be a good reference for 35 | Zaqar[1]. 36 | 37 | Proposed change 38 | =============== 39 | 40 | For clarification what we want to do, there is some questions to be answered: 41 | 42 | 1. When do the notification delivery policy work? 43 | 44 | Zaqar only attempts a retry after a failed delivery attempt. 45 | Those situations as a failed delivery attempt: 46 | 47 | * HTTP status in the range 500-599. 48 | * HTTP status outside the range 200-599. 49 | * A request timeout. 50 | * Any connection error such as connection timeout, endpoint unreachable, etc. 51 | 52 | 2. How do the notification delivery policy work? 53 | 54 | You can use delivery policies to control not only the total number of retries, 55 | but also the time delay between each retry. 56 | 57 | There are total four discrete phases that user can use to set policy of retry: 58 | 59 | * Immediate Retry Phase: Also called the no delay phase, this phase occurs 60 | immediately after the initial delivery attempt. The value user set for 61 | 'Retries with no delay' determines the number of retries immediately after 62 | the initial delivery attempt. There is no delay between retries in this 63 | phase. 64 | 65 | * Pre-Backoff Phase: The pre-backoff phase follows the immediate retry phase. 66 | Use this phase to create a set of retries that occur before a backoff 67 | function applies to the retries. Use the 'Minimum delay retries' setting to 68 | specify the number of retries in the Pre-Backoff Phase. User can control the 69 | time delay between retries in this phase by using the 'Minimum delay' 70 | setting. 71 | 72 | * Backoff Phase: This phase is called the backoff phase because user can 73 | control the delay between retries in this phase using the retry backoff 74 | function. Set the 'Minimum delay' and the 'Maximum delay', and then select a 75 | Retry backoff function to define how quickly the delay increases from the 76 | minimum delay to the maximum delay. 77 | 78 | * Post-Backoff Phase: The post-backoff phase follows the backoff phase. 79 | Use the 'Maximum delay retries' setting to specify the number of retries in 80 | the post-backoff phase. You can control the time delay between retries in 81 | this phase by using the 'Maximum delay' setting. 82 | 83 | User can create and apply the retry policies to queues and subscriptions when 84 | creating them. So when Zaqar send the notification to those subscribers, 85 | it will use those policies to make retried after initial delivery attempt 86 | failed. 87 | 88 | ..note:: 89 | 90 | By default, if there are retry policies on both queue and subscription, 91 | Zaqar will use the subscription's policy. If user don't want the override 92 | he can set the ignore_subscription_override=True in _retry_policy metadata 93 | of the queue. 94 | 95 | 3. What changes will bring into Zaqar? 96 | 97 | * First, user can create retry policies in metadata which's in request body of 98 | queue creation. 99 | 100 | * Second, user also can create retry policies in options which's in request 101 | body of subscription creation. If retry policy are set to both queue and 102 | subscription, Zaqar will use subscription policy instead of queue policy 103 | by default, but if ignore_subscription_override is True in queue's metadata 104 | then Zaqar will still use the retry policy in queue. 105 | 106 | * Third, change the logic of notification, when fail is existing when sending 107 | notification to endpoint, then use the policy which has binded to this 108 | subscription to make retries, if subscription doesn't have retry policy, then 109 | Zaqar will use the policy in queue's metadata. 110 | 111 | Since 'metadata' in queue and 'options' in subscription are dict, the data 112 | model of retry policy should be like this:: 113 | 114 | { 115 | '_retry_policy': { 116 | 'retries_with_no_delay': , 117 | 'minimum_delay_retries': , 118 | 'minimum_delay': , 119 | 'maximum_delay': , 120 | 'maximum_delay_retries': , 121 | 'retry_backoff_function': , 122 | 'ignore_subscription_override': 123 | } 124 | } 125 | 126 | 'minimum_delay' and 'maximum_delay' mean delay time in seconds. 127 | 'retry_backoff_function' mean name of retry backoff function. 128 | There will be a enum in Zaqar that contain all valid values. At first step, 129 | Zaqar only supports one function: 'linear'. 130 | If value of retry_policy is empty dict, that Zaqar will use default 131 | value to those keys: 132 | * retries_with_no_delay=3 133 | * minimum_delay_retries=3 134 | * minimum_delay=5 135 | * maximum_delay=30 136 | * maximum_delay_retries=3 137 | * retry_backoff_function=linear 138 | * ignore_subscription_override=False 139 | Those default values will be configurable in Queens. 140 | 141 | ..note:: 142 | 143 | Retry policy is only applied for webhook(http/https) subscriber. 144 | Zaqar will support linear retry backoff function only in Pike release, 145 | it will increase the delay time linearly within 10 times from minimum delay 146 | to maximum delay. We will support more retry backoff functions in future. 147 | For avoiding the users receive notification message repeatedly 148 | if connection is reconnected during retry process, once Zaqar sends the 149 | notification successfully when doing retry, it will stop the whole retry 150 | process. 151 | 152 | Example 153 | ------- 154 | 155 | For better to know how does this work, there is an example case. 156 | Assume user use the default value of retry policy in a queue A, 157 | and create a subscription with HTTP subscriber: http://192.168.1.100:8080. 158 | When Zaqar send the notifications to this subscriber, it get a response with 159 | HTTP code 500, then the retry policy will go to work: 160 | 161 | Phase 1: Immediate Retry. Zaqar will call the subscriber with no delay by 3 162 | times. If there is no one successful, then go to Phase 2. If one of 163 | retries is successful, the retry process will be end. 164 | 165 | Phase 2: Pre-Backoff. According to the minimum_delay_retries and minimum_delay. 166 | Zaqar will make a total of 3 retries with a 5 second delay between 167 | each retry. If there is no one successful, then go to Phase 3. 168 | If one of retries is successful, the retry process will be end. 169 | 170 | Phase 3: Backoff. By using the linear function, Zaqar will make the delay 171 | between retries to increase at a constant rate over the course of the 172 | backoff phase. So as the constant rate is 5, Zaqar will call 173 | subscriber 12 times with delay time: 174 | [5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60]. 175 | If there is on one successful, then go to Phase 4. 176 | If one of retries is successful, the retry process will be end. 177 | 178 | Phase 4: Post-Backoff. According to the maximum_delay and 179 | maximum_delay_retries, Zaqar will make a total of 3 retries with a 180 | 60 second delay between each retry. After this phase, no matter the 181 | call is successful or not, the retry process will be end. 182 | 183 | Drawbacks 184 | --------- 185 | 186 | N/A 187 | 188 | Alternatives 189 | ------------ 190 | 191 | N/A 192 | 193 | Implementation 194 | ============== 195 | 196 | Assignee(s) 197 | ----------- 198 | 199 | Primary assignee: 200 | wanghao (sxmatch1986@gmail.com) 201 | 202 | Milestones 203 | ---------- 204 | 205 | P-3 206 | 207 | Work Items 208 | ---------- 209 | 210 | * Add verification for retry policy when creating queue and subscription. 211 | * Change the notification process for applying the policy. 212 | * UTs for this feature. 213 | 214 | 215 | Dependencies 216 | ============ 217 | 218 | [1]: http://docs.aws.amazon.com/sns/latest/dg/DeliveryPolicies.html 219 | -------------------------------------------------------------------------------- /specs/queens/delayed-queues.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ====================================== 14 | Delayed Queues 15 | ====================================== 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/delayed-queues 18 | 19 | Delay queues can postpone the delivery of new messages in a queue for a 20 | specific number of seconds. Which is useful for an auditing observer process 21 | the auditor would be guaranteed to see messages before they were claimed, 22 | and could even edit messages if needed. 23 | 24 | Problem description 25 | =================== 26 | 27 | Now one of the big function gaps of Zaqar is the delayed queue. Currently, all 28 | the message posted to the queue will be visible immediately. That's enough for 29 | most of the user cases. However, for some user case, user want the message to 30 | be unavailable to end users for a specific period of time. 31 | 32 | Proposed change 33 | =============== 34 | 35 | Generally, a new attribute of queue named ``_default_message_delay``. 36 | If the attribute value greater than 0, then queue is a delayed queue, 37 | otherwise, it's a normal queue. 38 | 39 | All messages sent to the delayed queue can not be immediately claimed and 40 | must wait for a ``delay``. The ``delay`` can be set or update as a metadata 41 | when the queue is created. 42 | 43 | Then, when user get or claim messages on a delayed queue, Zaqar will take 44 | ``delay`` as a query condition, which is as below: 45 | 46 | Message Create + Delay < Current Time 47 | 48 | Basically, there are 3 scenarios: 49 | 50 | 1. Normal message sent to delayed queue 51 | 52 | If a normal message without ``delay`` attribute sent to a delayed queue, 53 | then the message will use the queue's default delay 54 | ``_default_message_delay``. 55 | 56 | 2. Message with attribute ``delay`` sent to delayed queue 57 | 58 | If message with ``delay`` attribute sent to a delayed queue, then Zaqar will 59 | use the message's delay seconds value instead of the delayed queue's delay 60 | seconds value. 61 | 62 | 3. Message sent to normal queue 63 | 64 | A normal queue is a queue with the ``_default_message_delay`` value of 0. 65 | Whether the message is a normal message or a message with a ``delay`` 66 | attribute, it will not have a delay feature when it is sent to the normal 67 | queue. Zaqar does not perform conditional filtering on the ``delay`` 68 | attribute when get/claim messages. This means the delay feature does not 69 | affect the performance of normal queue. 70 | 71 | Specific implementation steps: 72 | 73 | 1. Add a new attribute ``_default_message_delay`` for all queue's metadata. 74 | 75 | The attribute ``_default_message_delay`` default value is 0 means 76 | it is a normal queue. Its ranges from 0 to ``max_message_delay`` 77 | seconds, the ``max_message_delay`` is configurable and the default 78 | value is 900 seconds (15 mins). 79 | 80 | 2. Add a new attribute just for message's request body: ``delay``. 81 | 82 | When post messages to queue, user can add ``delay`` like ``ttl``. 83 | The message's delay time has a higher priority than queues. If 84 | message ``ttl < delay``, the message will never be claimed, because 85 | it is in a delay period and the message's ``ttl`` has expired, so the 86 | message will be deleted by backend. 87 | 88 | 3. Add a new field ``d`` in message backend. 89 | 90 | The field ``d`` is a delay expires timestamp. It can be saved 91 | to the backend storage. When the timestamp is less than or equal to 92 | the current time, the delay expires. 93 | 94 | ``d`` = ``delay`` + ``MESSAGE_CREATED_TIMESTAMP`` 95 | 96 | 4. Make sure the message can not be claimed until the delay is expired. 97 | 98 | 5. If the queue is a normal queue means the 'delayed queues' feature wasn't 99 | being used, it is necessary to ensure that it is close to zero overhead. 100 | 101 | 6. Support for mongo, redis and swift. 102 | 103 | Drawbacks 104 | --------- 105 | 106 | This may introduce a little bit performance impact, because this needs another 107 | new condition for current messages query. But it will not affect the normal 108 | queue too much. 109 | 110 | Alternatives 111 | ------------ 112 | 113 | There is no good option for this feature. User have to wait enough seconds to 114 | get / claim messages against Zaqar. 115 | 116 | 117 | Implementation 118 | ============== 119 | 120 | Assignee(s) 121 | ----------- 122 | 123 | Primary assignee: 124 | cdyangzhenyu 125 | 126 | Milestones 127 | ---------- 128 | 129 | Target Milestone for completion: 130 | Queens Q-2 131 | 132 | Work Items 133 | ---------- 134 | 135 | #. Add mongodb support. 136 | #. Add redis support. 137 | #. Add swift support. 138 | #. Add release note for this feature. 139 | #. Update API reference. 140 | #. Add user/developer document for this feature. 141 | #. Change unit, functional and tempest tests accordingly. 142 | 143 | Dependencies 144 | ============ 145 | 146 | None 147 | -------------------------------------------------------------------------------- /specs/queens/index.rst: -------------------------------------------------------------------------------- 1 | ======================= 2 | Queens Specifications 3 | ======================= 4 | 5 | .. toctree:: 6 | :glob: 7 | :maxdepth: 2 8 | 9 | support-redis-as-mgmt-storage-backend 10 | support-more-backoff-functions 11 | delayed-queues 12 | remove-pool-group-from-zaqar 13 | support-md5-of-body 14 | -------------------------------------------------------------------------------- /specs/queens/support-md5-of-body.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ================================ 14 | Support Checksum Of Message Body 15 | ================================ 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/support-md5-of-body 18 | 19 | Now, Zaqar will add a non-URL-encoded message body checksum function. 20 | This is essential for the security of the message body, it can prevent the 21 | message from being tampered. 22 | 23 | Problem description 24 | =================== 25 | 26 | Currently, Zaqar has no checksum of the non-URL-encoded message body. It is 27 | a good feature for the security of the message body. 28 | 29 | AWS SQS service has supported the function, that would be a good reference for 30 | Zaqar[1]. 31 | 32 | Proposed change 33 | =============== 34 | 35 | When user claim or get the messages, the checksum will be returned with message 36 | body together, So user can use it to verify that the message body is correct. 37 | Currently zaqar only support the checksum of the non-URL-encoded message body 38 | based on MD5 digests. We may support other algorithms in future versions, 39 | including SHA1, sha256, sha512, and so on. So, It is necessary to change 40 | the following: 41 | 42 | 1. Add new property named ``checksum`` for message. This property is 43 | a string type. 44 | 45 | When you send a message to queue, Zaqar will use the default checksum 46 | algorithm MD5 to calculate the ``checksum`` value of the non-URL-encoded 47 | message body. Finally Zaqar will store the value on the backend. 48 | Now the backend Mongodb, Redis and Swift can be supported. 49 | 50 | 2. Return the property ``checksum`` for message when user claims or gets 51 | the message. 52 | 53 | When user gets or claims the message, The API will return the k-v pair of 54 | ``checksum`` in the body of the message. The user can then use this value to 55 | verify that the body of the newly retrieved message is correct or not. 56 | 57 | 3. This feature is backward compatible. 58 | 59 | For old messages that do not have the checksum attribute, the checksum will 60 | return ``None`` for both ``claim`` and ``get`` operation. This feature will 61 | also transit smoothly as the old messages gradually expire. 62 | 63 | 4. Add new data model for message body and queue on different backend. 64 | 65 | MESSAGE:: 66 | 67 | +----------------------+---------+ 68 | | Name | Field | 69 | +======================+=========+ 70 | | checksum | cs | 71 | +----------------------+---------+ 72 | | body | b | 73 | +----------------------+---------+ 74 | | ... | ... | 75 | +----------------------+---------+ 76 | 77 | Drawbacks 78 | --------- 79 | 80 | None 81 | 82 | Alternatives 83 | ------------ 84 | 85 | None 86 | 87 | Implementation 88 | ============== 89 | 90 | Assignee(s) 91 | ----------- 92 | 93 | Primary assignee: 94 | cdyangzhenyu 95 | 96 | Milestones 97 | ---------- 98 | 99 | Target Milestone for completion: 100 | Queens RC1 101 | 102 | Work Items 103 | ---------- 104 | 105 | #. Add message body checksum support for Mongodb, Redis and Swift. 106 | #. Add release note for this feature. 107 | #. Update API reference. 108 | #. Add user/developer document for this feature. 109 | #. Change unit, functional and tempest tests accordingly. 110 | 111 | Dependencies 112 | ============ 113 | 114 | None 115 | 116 | References 117 | ========== 118 | 119 | [1]:https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_Message.html 120 | -------------------------------------------------------------------------------- /specs/queens/support-more-backoff-functions.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | =============================== 14 | Support More Backoff Functions 15 | =============================== 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/support-more-backoff-functions 18 | 19 | Sometimes, because of network/internet connection issue or the load of the 20 | target services/applications, the notification sent from Zaqar to the 21 | subscriber may fail. That means there is no retries if endpoint has some 22 | problem that can't receive the notification. For users or applications, 23 | it's missing important message without notification. To solve this problem, 24 | Zaqar has supported notification delivery policy in pike release [1]. 25 | 26 | Problem description 27 | ==================== 28 | 29 | Now that Zaqar has supported only linear retry backoff function in Pike 30 | release,it increases the delay time linearly from minimum delay to maximum 31 | delay. We will support more retry backoff functions in Zaqar now. 32 | 33 | Fortunately, AWS SNS service has supported the policy, that would be a good 34 | reference for Zaqar[2]. 35 | 36 | Proposed change 37 | ================ 38 | 39 | For clarification what we want to do, there are some questions to be answered: 40 | 41 | 1. How many retry backoff functions Will Zaqar support? 42 | 43 | You can choose from four retry backoff functions as follows: 44 | 45 | * Linear. 46 | * Arithmetic. 47 | * Geometric. 48 | * Exponential. 49 | 50 | 51 | 2. What difference between these four retry backoff functions? 52 | 53 | As we can see the picture in chapter Backoff Phase in [2], The screen shot 54 | shows how each retry backoff function affects the delay associated with 55 | messages during the backoff period. The vertical axis represents the delay 56 | in seconds associated with each of the 10 retries. The horizontal axis 57 | represents the retry number. The minimum delay is 5 seconds, and the 58 | maximum delay is 260 seconds. 59 | 60 | 61 | 3. The formula of these four retry backoff functions 62 | 63 | Generally, we assume that the minimum delay is MIN, the maximum delay is 64 | MAX, and the total retry number is NUM. From three element above, we can 65 | conclude the formula of these four retry backoff functions. 66 | 67 | * Linear. 68 | 69 | LINEAR_INTERVAL = ( MAX - MIN ) / NUM 70 | time1 = MIN 71 | time2 = MIN + LINEAR_INTERVAL*1 72 | time3 = MIN + LINEAR_INTERVAL*2 73 | timen = MIN + LINEAR_INTERVAL*(n-1) 74 | timeN = MAX 75 | 76 | The Linear retry backoff functions has already been implemented in Pike 77 | release, we only list here for reference and comparing. 78 | 79 | * Geometric 80 | 81 | we assume the common ratio of a geometric series is K: 82 | time1 = MIN 83 | time2 = K * MIN 84 | time3 = K^2 * MIN 85 | timen = K^(n-1) * MIN 86 | timeN = K^(NUM-1) * MIN = MAX 87 | 88 | so : 89 | 90 | K^(NUM-1) = MAX/MIN 91 | lg[ K^(NUM-1) ] = lg( MAX/MIN ) 92 | (NUM-1) * lg(K) = lg( MAX/MIN ) 93 | lg(K) = lg( MAX/MIN ) / (NUM-1) 94 | K = 10^[ lg( MAX/MIN ) / (NUM-1) ] 95 | 96 | Finally: 97 | 98 | timen = {10^[ lg( MAX/MIN ) / (NUM-1) ]}^(n-1) * MIN 99 | 100 | * Exponential. 101 | 102 | we assume the power exponent of a exponential series is k, coefficient is p: 103 | 104 | the general function of power exponent of a exponential series is: 105 | y = p * k^x 106 | 107 | time1 = MIN = p*k^1 108 | timeN = MAX = p*k^NUM 109 | MAX/MIN = k^(NUM-1) 110 | lg(MAX/MIN) = (NUM-1)*lg(k) 111 | lg(k) = lg(MAX/MIN)/ (NUM-1) 112 | k = 10^[lg(MAX/MIN)/ (NUM-1)] 113 | p = MIN/{10^[lg(MAX/MIN)/ (NUM-1)]} 114 | 115 | Finally: 116 | 117 | timen = p*k^n 118 | timen = MIN/{10^[lg(MAX/MIN)/ (NUM-1)]} * {10^[lg(MAX/MIN)/ (NUM-1)]} ^n 119 | 120 | * Arithmetic. 121 | 122 | we assume the common difference of a arithmetic series is d: 123 | 124 | time1 = MIN 125 | time2 = time1 + d*1 126 | time3 = time2 + d*2 127 | time4 = time3 + d*3 128 | 129 | timen = timen-1 + d*(n-1) 130 | 131 | timeN = timeN-1 + d*(NUM-1) = MAX 132 | 133 | So: 134 | timeN - time1 = NUM*(NUM-1)/2*d 135 | 136 | d = 2*(MAX-MIN)/[NUM*(NUM-1)] 137 | 138 | timen - time1 = n*(n-1)/2 * { 2*(MAX-MIN)/[NUM*(NUM-1)] } 139 | 140 | Finally: 141 | 142 | timen = n*(n-1)/2 * { 2*(MAX-MIN)/[NUM*(NUM-1)] } + MIN 143 | 144 | 4. What changes will bring into Zaqar? 145 | 146 | * First, we will define three functions according to formula of these retry 147 | backoff functions. 148 | 149 | * Second, change the logic of notification, when fail is existing when sending 150 | notification to endpoint, then use the policy which has binded to this 151 | subscription to make retries. In Backoff Phase, we will select the retry 152 | backoff functions according to user choice to resend notification. 153 | 154 | 155 | Drawbacks 156 | --------- 157 | 158 | N/A 159 | 160 | Alternatives 161 | ------------ 162 | 163 | N/A 164 | 165 | Implementation 166 | ============== 167 | 168 | Assignee(s) 169 | ----------- 170 | 171 | Primary assignee: 172 | gecong (ge.cong@zte.com.cn) 173 | 174 | Milestones 175 | ---------- 176 | 177 | Q-2 178 | 179 | Work Items 180 | ---------- 181 | 182 | * Add three functions according to formula of these retry 183 | backoff functions. 184 | * Change the Backoff Phase of notification process for applying the retry 185 | backoff functions. 186 | * UTs for this feature. 187 | 188 | 189 | Dependencies 190 | ============= 191 | 192 | 193 | [1]:https://specs.openstack.org/openstack/zaqar-specs/specs/pike/notification-delivery-policy.html 194 | [2]:http://docs.aws.amazon.com/sns/latest/dg/DeliveryPolicies.html 195 | -------------------------------------------------------------------------------- /specs/queens/support-redis-as-mgmt-storage-backend.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ====================================== 14 | Support redis as mgmt storage backend 15 | ====================================== 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/support-redis-as-management-storage-backend 18 | 19 | 20 | Redis is different than other database solutions in many ways, it support 21 | two modes: memory storage and disk storage, We can select one of the two modes 22 | by modifying redis.conf. If we add save options in redis.conf, redis can 23 | cyclically save the data into disk: 24 | save 900 1 25 | save 300 10 26 | save 60 10000 27 | 28 | Redis includes a fairly high performance implementation of database. 29 | Currently, zaqar supports redis, mongodb and swift as the storage backend. 30 | After performance testing, we find that the performance of redis 31 | is the highest among these storage-backend. Using Redis, we may provide a 32 | system which is very practical to use, easy to administer and scale, while 33 | providing excellent performances. 34 | 35 | 36 | Problem description 37 | =================== 38 | 39 | In order to enhance the performance of zaqar, it is necessary to support 40 | redis as a management database backend. 41 | 42 | At present, Zaqar has supported sqlalchemy and mongodb as management storage 43 | backend, that would be a good reference for redis. 44 | 45 | 46 | 47 | Proposed change 48 | =============== 49 | 50 | 1. Add CatalogueController in redis storage: 51 | 52 | Add such functions: insert, update, delete, list, get, drop_all etc. 53 | 54 | 55 | 2. Add FlavorsController in redis storage: 56 | 57 | Add such functions: insert, update, delete, list, get, drop_all etc. 58 | 59 | 3. Add PoolsController in redis storage: 60 | 61 | Add such functions: insert, update, delete, list, get, drop_all etc. 62 | 63 | 64 | Drawbacks 65 | --------- 66 | None 67 | 68 | Alternatives 69 | ------------ 70 | MongoDB SQL 71 | 72 | Data model impact 73 | ----------------- 74 | 75 | We need to add three table models, as follows: 76 | 77 | 1. Redis Data Structures for table catalogue: 78 | 79 | * All Project (Redis sorted set): 80 | 81 | Set of all queue_ids, ordered by name. Used to delete the all 82 | records of table catalogue. 83 | 84 | Key: catalogue 85 | 86 | +--------+-----------------------------+ 87 | | Id | Value | 88 | +========+=============================+ 89 | | name | . | 90 | +--------+-----------------------------+ 91 | 92 | * Project Index (Redis sorted set): 93 | 94 | Set of all queue_ids for the given project, ordered by name. 95 | 96 | Key: .catalogue 97 | 98 | +--------+-----------------------------+ 99 | | Id | Value | 100 | +========+=============================+ 101 | | name | . | 102 | +--------+-----------------------------+ 103 | 104 | * Queue and pool Information (Redis hash): 105 | 106 | Key: ..catalogue 107 | 108 | +----------------------+---------+ 109 | | Name | Field | 110 | +======================+=========+ 111 | | Project | p | 112 | +----------------------+---------+ 113 | | Queue | p_q | 114 | +----------------------+---------+ 115 | | Pool | p_p | 116 | +----------------------+---------+ 117 | 118 | 2. Redis Data Structures for table flavor: 119 | 120 | * All flavor_ids (Redis sorted set): 121 | 122 | Set of all flavor_ids, ordered by name. Used to delete the all 123 | records of table flavors. 124 | 125 | Key: flavors 126 | 127 | +--------+-----------------------------+ 128 | | Id | Value | 129 | +========+=============================+ 130 | | name | | 131 | +--------+-----------------------------+ 132 | 133 | * Project Index (Redis sorted set): 134 | 135 | Set of all flavors for the given project, ordered by name. 136 | 137 | Key: .flavors 138 | 139 | +--------+-----------------------------+ 140 | | Id | Value | 141 | +========+=============================+ 142 | | name | | 143 | +--------+-----------------------------+ 144 | 145 | * Flavor Information (Redis hash): 146 | 147 | Key: .flavors 148 | 149 | +----------------------+---------+ 150 | | Name | Field | 151 | +======================+=========+ 152 | | flavor | f | 153 | +----------------------+---------+ 154 | | project | p | 155 | +----------------------+---------+ 156 | | capabilities | c | 157 | +----------------------+---------+ 158 | 159 | 3. Redis Data Structures for table pools: 160 | 161 | * All pool (Redis sorted set): 162 | 163 | Set of all pool_ids, ordered by name. Used to delete the all 164 | records of table pools. 165 | 166 | Key: pools 167 | 168 | +--------+-----------------------------+ 169 | | Id | Value | 170 | +========+=============================+ 171 | | name | | 172 | +--------+-----------------------------+ 173 | 174 | * Flavor Index (Redis sorted set): 175 | 176 | Set of all pool_ids for the given flavor, ordered by name. 177 | 178 | Key: .pools 179 | 180 | +--------+-----------------------------+ 181 | | Id | Value | 182 | +========+=============================+ 183 | | name | | 184 | +--------+-----------------------------+ 185 | 186 | * Pools Information (Redis hash): 187 | 188 | Key: .pools 189 | 190 | +----------------------+---------+ 191 | | Name | Field | 192 | +======================+=========+ 193 | | pool | p | 194 | +----------------------+---------+ 195 | | uri | u | 196 | +----------------------+---------+ 197 | | weight | w | 198 | +----------------------+---------+ 199 | | options | o | 200 | +----------------------+---------+ 201 | 202 | 203 | Implementation 204 | ============== 205 | 206 | Assignee(s) 207 | ----------- 208 | 209 | Primary assignee: 210 | gengchc2 211 | 212 | Secondary assignee: 213 | gecong 214 | 215 | Milestones 216 | ---------- 217 | 218 | Target Milestone for completion: 219 | Queens-Q2 220 | 221 | Work Items 222 | ---------- 223 | 224 | #. Add CatalogueController in redis storage. 225 | #. Add FlavorsController in redis storage. 226 | #. Add PoolsController in redis storage. 227 | 228 | Dependencies 229 | ============ 230 | 231 | [1]:https://review.opendev.org/#/c/345133/ 232 | 233 | Testing 234 | ======= 235 | 236 | Both unit and Tempest tests need to be created to cover the code change. 237 | 238 | 239 | Documentation Impact 240 | ==================== 241 | 242 | #. Add docs about the configuration of redis and HA. 243 | 244 | References 245 | ========== 246 | None 247 | 248 | [2]:http://paste.openstack.org/show/621298/ 249 | 250 | -------------------------------------------------------------------------------- /specs/rocky/index.rst: -------------------------------------------------------------------------------- 1 | ======================= 2 | Rocky Specifications 3 | ======================= 4 | 5 | .. toctree:: 6 | :glob: 7 | :maxdepth: 2 8 | 9 | queue-filter-support 10 | remove-format-constraint-of-client-id 11 | subscription-filtering-tags 12 | -------------------------------------------------------------------------------- /specs/rocky/queue-filter-support.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://sphinx-doc.org/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ===================== 14 | Queue filter support 15 | ===================== 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/queue-filter-support 18 | 19 | Metadata is a group of key-value pairs that belong to a queue. When a queue is 20 | created, the user who creates the queue can add the metadata {"keyx": "valuex"} 21 | to distinguish it. It is necessary for zaqar to support the function of queue 22 | filter so that Users can select the queue by the specified key-value of 23 | metadata. In this blueprints, we will also add queue filter by name. 24 | 25 | 26 | Problem description 27 | =================== 28 | 29 | Zaqar doesn't support queue filter when queue listing, 30 | we may add this feature now. 31 | 32 | 33 | Proposed change 34 | =============== 35 | When queue listing, we add filter option in query string parameters 36 | like this :: 37 | 38 | GET /v2/queues?key1=value1&key2=value2&name=value3 39 | 40 | If metadata of queue and name is consistent with the filter, the queues will be 41 | listed to the user, otherwise the queue will be filtered. 42 | If filter option is enabled, the metadata of the queue will be returned 43 | to user. 44 | 45 | API Impact 46 | ----------- 47 | Lists queues:: 48 | 49 | GET: /v2/queues?key1=value1&key2=value2 50 | 51 | RESPONSE CODE: 200 52 | 53 | Drawbacks 54 | --------- 55 | 56 | N/A 57 | 58 | Alternatives 59 | ------------ 60 | 61 | N/A 62 | 63 | 64 | Implementation 65 | ============== 66 | 67 | Assignee(s) 68 | ----------- 69 | 70 | gecong 71 | 72 | Milestones 73 | ---------- 74 | 75 | Rocky-1 76 | 77 | Work Items 78 | ---------- 79 | 80 | #* Add filter parameter for queues listing REST API. 81 | #* Update API reference. 82 | #* Add release note for this feature. 83 | #* UTs for this feature. 84 | 85 | Dependencies 86 | ============ 87 | 88 | N/A 89 | 90 | -------------------------------------------------------------------------------- /specs/rocky/remove-format-constraint-of-client-id.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://sphinx-doc.org/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ===================================== 14 | Remove format constraint of client id 15 | ===================================== 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/remove-format-constraint-of-client-id 18 | 19 | Since some clients use different format of client id not only uuid, 20 | so Zaqar will support this function. This also requires user to ensure the 21 | client id is immutable. 22 | 23 | 24 | Problem description 25 | =================== 26 | 27 | Now Zaqar has format constraint to client id, must be UUID format. But this 28 | constraint is not very reasonable under some cases, like user want to use 29 | user id in LDAP as client id, but Zaqar doesn't allow this now. 30 | 31 | 32 | Proposed change 33 | =============== 34 | * Add Three config options: 35 | 36 | #. 'client_id_uuid_safe': Defines the format of client id, the value could be 37 | "strict" or "off". "strict" means the format of client id must be uuid, 38 | "off" means the restriction be removed. The default value is 'strict'. 39 | #. 'min_length_client_id': Defines the minimum length of client id if remove 40 | the uuid restriction. Default value is 10. 41 | #. 'max_length_client_id': Defines the maximum length of client id if 42 | remove the uuid restriction. Default value is 36. 43 | 44 | * Will change the method 'require_client_id' in wsgi/helpers.py to support 45 | validating the different format of client id. 46 | 47 | API Impact 48 | ----------- 49 | For WSGI: Will impact all APIs in v1.1 and v2 version. 50 | 51 | For WebSocket: Will impact message list and message post in v2 version. And we 52 | will support others APIs in following patch. 53 | 54 | Drawbacks 55 | --------- 56 | There may be a tiny performance impact when user uses very long client id since 57 | Zaqar need to store the client id in storage layer. 58 | 59 | 60 | Alternatives 61 | ------------ 62 | 63 | N/A 64 | 65 | 66 | Implementation 67 | ============== 68 | 69 | Assignee(s) 70 | ----------- 71 | 72 | wanghao 73 | 74 | Milestones 75 | ---------- 76 | 77 | Rocky-3 78 | 79 | Work Items 80 | ---------- 81 | 82 | #* Change the Zaqar server code. 83 | #* Update API reference. 84 | #* Add release note for this feature. 85 | #* UTs for this feature. 86 | 87 | Dependencies 88 | ============ 89 | 90 | N/A 91 | -------------------------------------------------------------------------------- /specs/rocky/subscription-filtering-tags.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://sphinx-doc.org/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | =============================== 14 | Subscription filtering policies 15 | =============================== 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/subscription-filtering-tags 18 | 19 | By default, a subscriber of zaqar topic receives every message published 20 | to the topic. To receive only a subset of the messages, a subscriber assigns 21 | a filter policy to the topic subscription. So in this case, zaqar need to 22 | support message filtering. This function is similar to Amazon SNS message 23 | filtering[1]. 24 | 25 | This feature supports all subscription endpoint, including mail, webhook. 26 | 27 | Problem description 28 | =================== 29 | 30 | The current subscription lacks flexibility due to the fact that it does 31 | not support message filtering. When a subscriber subscribes to a topic, 32 | all messages which sent to the topic will be sent to this subscriber. 33 | So, this feature is intended to solve the problem, which allows the user 34 | to specify the filter policies when creating a subscription and publish 35 | messages, and ultimately, the filter policies matching decides which subscriber 36 | will receive the message. 37 | 38 | Proposed change 39 | =============== 40 | 41 | The filter policy is a simple JSON object. The policy contains attributes that 42 | define which messages the subscriber receives. When you publish a message to a 43 | topic, Zaqar compares the message attributes to the attributes in the filter 44 | policy for each of the topic's subscriptions. If there is a match between the 45 | attributes, Zaqar sends the message to the subscriber. Otherwise, Zaqar skips 46 | the subscriber without sending the message to it. 47 | 48 | With filter policies, you can simplify your message filtering criteria into 49 | your topic subscriptions. With this consolidation, you can offload the message 50 | filtering logic from subscribers and the message routing logic from publishers. 51 | Therefore, you do not need to filter messages by creating a separate topic for 52 | each filtering condition. Instead, you can use a single topic, and you can 53 | differentiate your messages with attributes. Each subscriber receives and 54 | processes only those messages accepted by its filter policy. 55 | 56 | For example, you could use a single topic to publish all messages generated by 57 | transactions from your online retail site. To each message, you could assign an 58 | attribute that indicates the type of transaction, such as order_placed, 59 | order_cancelled, or order_declined. With filter policies, you can route each 60 | message to the different subscriber that is meant to process the message's 61 | transaction type. 62 | 63 | To do this, a new attribute named ``filter_policies`` need to be added to both 64 | the message and subscription. Note that this property is optional and does not 65 | affect existing functionality. 66 | 67 | When you create a subscription, you can specify the filter policies, 68 | just like this:: 69 | 70 | ``{'subscriber': 'http://example.com/order_placed', 71 | 'options':{'filter_policies': ['order_placed']}}`` 72 | 73 | Then when you send a message to the topic, you can choose to add the 74 | attribute, just like this:: 75 | 76 | ``{'body': 'test', 'filter_policies': ['order_placed']}`` 77 | 78 | Finally, zaqar decides which subscribers the message should be sent to 79 | according to the following rules: 80 | 81 | * If the subscriber does not have any filter_policies, all messages will be 82 | sent to the subscriber. 83 | 84 | * If the message does not contain any filter_policies, it will not be 85 | sent to the subscriber which has filter_policies. 86 | 87 | * If the message and the subscriber's filter_policies have intersecting 88 | collections, the message will be sent to the subscriber. 89 | 90 | * If the message and the subscriber both have filter_policies, but 91 | there is no intersection set, no message will be sent to the 92 | subscriber. 93 | 94 | * The relationship of the content in `order_placed` is *or*, it means 95 | that the message will be sent to the subscriptions that match anyone 96 | filters in `order_placed`. 97 | 98 | Drawbacks 99 | --------- 100 | 101 | N/A 102 | 103 | Alternatives 104 | ------------ 105 | 106 | N/A 107 | 108 | Implementation 109 | ============== 110 | 111 | Assignee(s) 112 | ----------- 113 | 114 | Primary assignee: 115 | cdyangzhenyu 116 | 117 | Milestones 118 | ---------- 119 | 120 | Target Milestone for completion: 121 | Rocky-3 122 | 123 | Work Items 124 | ---------- 125 | 126 | #. Change the message subscription process for applying this feature. 127 | #. Add release note for this feature. 128 | #. Update API reference. 129 | #. Add user/developer document for this feature. 130 | #. Change unit, functional and tempest tests accordingly. 131 | 132 | Dependencies 133 | ============ 134 | 135 | None 136 | 137 | References 138 | ========== 139 | 140 | [1] https://docs.aws.amazon.com/sns/latest/dg/message-filtering.html 141 | -------------------------------------------------------------------------------- /specs/stein/delete-message-with-claim-id.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://sphinx-doc.org/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ============================= 14 | Delete Message With Claim ID 15 | ============================= 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/delete-message-with-claim-id 18 | 19 | Delete Message with claim id means that when a user deletes a message, the 20 | message must be claimed. If you want to delete a message, you will have to use 21 | both message id and claim id. This can improve the security of the message. 22 | 23 | Problem description 24 | =================== 25 | 26 | Currently, any client who knows the message ID can delete the message if it not 27 | be claimed. It could cause some unexpected problems. A better way to delete a 28 | message is make sure the message is deleted by the client who is claiming the 29 | message. Amazon SQS use receipt handler to delete a message[1]. Zaqar can use 30 | claim id and message id to delete messages. 31 | 32 | Proposed change 33 | =============== 34 | 35 | Add a new configuration item named ``message_delete_with_claim_id``, default 36 | value is ``False``, means it is backwards compatible. You can modify this 37 | configuration item to decide whether to turn on the switch. If you change it to 38 | ``True``, you need to forcibly carry the claim id when delete messages. If the 39 | claim ID is invalid, the message can not be deleted. You must re-claim the 40 | messages, and then delete it. 41 | 42 | ..note:: 43 | 44 | No matter "message_delete_with_claim_id" is True of False, admin can 45 | always delete a message without claim_id. 46 | 47 | API Impact 48 | ----------- 49 | Delete single message 50 | DELETE: /v2/queues/test_queue/messages/{message_id}?claim_id={claim_id} 51 | 52 | RESPONSE CODE: 204 53 | 54 | Delete messages 55 | DELETE: /v2/queues/test_queue/messages?ids={messages_ids}&claim_ids={claim_ids} 56 | 57 | RESPONSE CODE: 204 58 | 59 | 60 | Drawbacks 61 | --------- 62 | 63 | None 64 | 65 | Alternatives 66 | ------------ 67 | 68 | None 69 | 70 | Implementation 71 | ============== 72 | 73 | Assignee(s) 74 | ----------- 75 | 76 | Primary assignee: 77 | cdyangzhenyu 78 | 79 | Secondary assignee: 80 | gecong 81 | wanghao 82 | 83 | Milestones 84 | ---------- 85 | 86 | Target Milestone for completion: 87 | stein RC3 88 | 89 | Work Items 90 | ---------- 91 | 92 | #. Modify message delete code. 93 | #. Add release note for this feature. 94 | #. Update API reference. 95 | #. Add user/developer document for this feature. 96 | #. Change unit, functional and tempest tests accordingly. 97 | 98 | Dependencies 99 | ============ 100 | 101 | None 102 | 103 | References 104 | ========== 105 | 106 | [1]:https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-queue-message-identifiers.html -------------------------------------------------------------------------------- /specs/stein/email-notification-by-internal-tool.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This work is aimed to implement zaqar email delivery 3 | through smtp service,and Chinese translation support. 4 | 5 | Support zaqar_sendmail 6 | ================================== 7 | 8 | https://blueprints.launchpad.net/zaqar/+spec/zaqar-email-delivery 9 | 10 | Problem description 11 | =================== 12 | 13 | Currently the email subscription in Zaqar relay on the third part tools, such 14 | as "sendmail". It means that deployer should install it out of Zaqar. If he 15 | forgets, Zaqar will raise internal error. This work let Zaqar support email 16 | subscription by itself using the ``smtp`` python library. 17 | 18 | Use Cases 19 | --------- 20 | 21 | The modification enables users to send email subscriptions without the third 22 | part tools. 23 | 24 | Proposed change 25 | =============== 26 | 27 | A new config option ``smtp_mode`` will be added, it has two value choices, 28 | ["third_part", "self_local"]. ``third_part`` means Zaqar will use the tools 29 | from which ``smtp_command`` config option provides to send email subscription. 30 | ``self_local`` means Zaqar will use the ``smtp`` python library instead. 31 | 32 | A email template file will be added as well. Some parameters in the file should 33 | be configured firstly before using. Those parameters include something like 34 | smtp service account/password, smtp service address and so on. These value 35 | should come from config options as well. 36 | 37 | The email body in the template file should be configured as well. Currently, 38 | it can only be managed by operators out of Zaqar. In the future, we can use 39 | jinja template way to let it be configurable by Zaqar. 40 | 41 | Once ``self_local`` is used, Zaqar will fill up the template file first, then 42 | use ``stmp`` lib to send the email subscription. 43 | 44 | Alternatives 45 | ------------ 46 | 47 | None 48 | 49 | 50 | REST API impact 51 | --------------- 52 | 53 | None 54 | 55 | Security impact 56 | --------------- 57 | 58 | None 59 | 60 | Notifications impact 61 | -------------------- 62 | 63 | None 64 | 65 | 66 | Other end user impact 67 | --------------------- 68 | 69 | None 70 | 71 | Performance Impact 72 | ------------------ 73 | 74 | None 75 | 76 | Other deployer impact 77 | --------------------- 78 | 79 | A new config option called ``smtp_mode`` is added. To keep backward 80 | compatibility the default behavior is keeping use third part tools to send 81 | email subscription. If you don't want to install the third part email tools 82 | anymore, please change its value to ``self_local``. 83 | 84 | Developer impact 85 | ---------------- 86 | 87 | None 88 | 89 | 90 | Implementation 91 | ============== 92 | 93 | Assignee(s) 94 | ----------- 95 | 96 | changyufei 97 | 98 | Work Items 99 | ---------- 100 | 101 | 1. Add a new config option ``smtp_mode`` 102 | 2. Add the email template 103 | 3. Add the logic to use ``smtp`` python lib 104 | 4. The test code should be added. 105 | 5. Document. 106 | 107 | Milestones 108 | ---------- 109 | 110 | R-3 111 | 112 | 113 | Dependencies 114 | ============ 115 | 116 | None 117 | 118 | Testing 119 | ======= 120 | 121 | For functional test, here is no good way to test it in Upstream since 122 | OpenStack CI/CD environment doesn't has a email service. Or we may ask the 123 | community for help to give Zaqar team a test account within @openstack.org 124 | email system. Or any other third part company can provide one for us. Otherwise 125 | we could only test it locally to make sure it works well. 126 | 127 | 128 | Documentation Impact 129 | ==================== 130 | 131 | A new send email way will be added. It should be documented. 132 | 133 | References 134 | ========== 135 | 136 | None 137 | -------------------------------------------------------------------------------- /specs/stein/index.rst: -------------------------------------------------------------------------------- 1 | ======================= 2 | Stein Specifications 3 | ======================= 4 | 5 | .. toctree:: 6 | :glob: 7 | :maxdepth: 2 8 | 9 | email-notification-by-internal-tool 10 | delete-message-with-claim-id 11 | introduce-topic-resource-for-notification 12 | remove-pool-group-totally 13 | -------------------------------------------------------------------------------- /specs/stein/introduce-topic-resource-for-notification.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://sphinx-doc.org/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ========================================= 14 | Introduce Topic Resource For Notification 15 | ========================================= 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/introduce-topic-resource-for-notification 18 | 19 | We want to introduce a new resource called Topic into Zaqar. 20 | Topic is a concept from AWS Simple Notification Service (SNS), it will has 21 | relevance with subscriptions. User can send message to a topic, 22 | and then the subscribers will get the message according to different protocols, 23 | like http, email, sms, etc. 24 | 25 | Problem description 26 | =================== 27 | 28 | Now basically Zaqar has integrated two types of services, 29 | Messaging Queue Service and Notification Service. We create subscriptions 30 | based on queue, and also introduce retry policy for subscription and queue. 31 | It's not very clear from the service view, especially for the users who have 32 | been used to the SNS and SQS in AWS 33 | 34 | So in Rocky, we want to introduce the Topic resource into Zaqar, and split 35 | Messaging Queue Service and Notification Service clearly. 36 | 37 | AWS SNS service has the resource named Topic, that would be a good reference 38 | for Zaqar[1]. 39 | 40 | Proposed change 41 | =============== 42 | 43 | To implement the Topic resource, one idea is to define the Topic as a special 44 | queue with some special feature that is different from common queue. 45 | 46 | 1. Could set the users or projects who can send topic message to this Topic 47 | 48 | 2. Could set the users or projects who can subscribe this Topic. 49 | 50 | 3. The messages will be stored in Topic but those messages could NOT be claimed 51 | since they are just used for sending to subscribers. When sending 52 | successfully, Zaqar will delete this message from topic, if not, will retry 53 | the sending process. 54 | 55 | Besides those features, user also can set the retry policy to Topic and 56 | Subscription both. 57 | 58 | After introducing Topic to Zaqar, we will keep the subscription of common 59 | queue for a while, but we also want to change the users' behavior to use Topic 60 | if they want to subscribe something in Zaqar. So maybe we remove the 61 | subscription of common queue in future. 62 | 63 | API Impact 64 | ---------- 65 | 66 | 1. Topic Creation:: 67 | 68 | PUT: /v2/topics/{topic_name} 69 | BODY: 70 | { 71 | "_allowed_sending_users": , 72 | "_allowed_sending_projects": , 73 | "_allowed_subscribe_users": , 74 | "_allowed_subscribe_projects": , 75 | "_allowed_subscribe_protocols": , 76 | "_retry_policy": 77 | } 78 | RESPONSE CODES: 201, 204 79 | ERROR RESPONSE CODES: 80 | * BadRequest (400) 81 | * Unauthorized (401) 82 | * ServiceUnavailable (503) 83 | 84 | .. note:: 85 | 86 | "_allowed_sending_users": define which users can send message to topic. 87 | "_allowed_sending_projects": define which projects can send message to topic. 88 | Of course you can set the project owned this topic here too. 89 | "_allowed_subscribe_users": define which users can subscribe topic. 90 | "_allowed_subscribe_projects": define which projects can subscribe topic. 91 | The default behavior is allowing all the users under the project who create 92 | this topic send message, and all projects could subscribe this topic. 93 | Default subscribe protocols are email, webhook, and trust. 94 | The retry policy is same as Queue's retry policy. 95 | 96 | 2. Topic Update:: 97 | 98 | PATCH: /v2/topics/{topic_name} 99 | BODY: 100 | [ 101 | { 102 | "op": "replace", 103 | "path": "/metadata/allowed_sending_projects", 104 | "value": [pro_id1, pro_id2, ... ,pro_idN] 105 | } 106 | ] 107 | RESPONSE CODE: 200 108 | ERROR RESPONSE CODES: 109 | * BadRequest (400) 110 | * Unauthorized (401) 111 | * Not Found (404) 112 | * Conflict (409) 113 | * ServiceUnavailable (503) 114 | 115 | 3. Topic Query List:: 116 | 117 | GET: /v2/topics 118 | RESPONSE BODY: 119 | { 120 | "topics": [ 121 | { 122 | "topic_name": xxx, 123 | "href": xxx 124 | } 125 | ], 126 | "links": [ 127 | { 128 | "href": '/v2/topic?marker=wellington', 129 | "rel": "next" 130 | } 131 | ] 132 | } 133 | RESPONSE CODE: 200 134 | ERROR RESPONSE CODES: 135 | * BadRequest (400) 136 | * Unauthorized (401) 137 | * ServiceUnavailable (503) 138 | 139 | 4. Topic Query Detail:: 140 | 141 | GET: /v2/topics/{topic_name} 142 | RESPONSE BODY: 143 | { 144 | "_allowed_sending_users": [user_id1, ... ,user_idN], 145 | "_allowed_sending_projects": [pro_id1, ... ,pro_idN], 146 | "_allowed_subscribe_users": [user_id1, ... ,user_idN], 147 | "_allowed_subscribe_projects": [pro_id1, ... ,pro_idN], 148 | "_allowed_subscribe_protocols": [email, webhook, trust], 149 | "_retry_policy": {} 150 | } 151 | RESPONSE CODE: 200 152 | ERROR RESPONSE CODES: 153 | * BadRequest (400) 154 | * Unauthorized (401) 155 | * ServiceUnavailable (503) 156 | 157 | 5. Topic Delete:: 158 | 159 | DELETE: /v2/topics/{topic_name} 160 | RESPONSE CODE: 204 161 | ERROR RESPONSE CODES: 162 | * BadRequest (400) 163 | * Unauthorized (401) 164 | * ServiceUnavailable (503) 165 | 166 | Drawbacks 167 | --------- 168 | 169 | N/A 170 | 171 | Alternatives 172 | ------------ 173 | 174 | N/A 175 | 176 | Implementation 177 | ============== 178 | 179 | Assignee(s) 180 | ----------- 181 | 182 | Primary assignee: 183 | wanghao (sxmatch1986@gmail.com) 184 | 185 | Milestones 186 | ---------- 187 | 188 | R-3 189 | 190 | Work Items 191 | ---------- 192 | 193 | * Implement the Topic API. 194 | * Implement the process of CRUD. 195 | * UTs for this feature. 196 | * DOC support. 197 | 198 | Dependencies 199 | ============ 200 | 201 | [1]: http://docs.aws.amazon.com/sns/latest/dg/welcome.html 202 | -------------------------------------------------------------------------------- /specs/stein/remove-pool-group-totally.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://sphinx-doc.org/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | =========================== 14 | remove-pool-group-totally 15 | =========================== 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/remove-pool-group-totally 18 | 19 | remove-pool-group-totally. 20 | 21 | Problem description 22 | =================== 23 | 24 | Currently pool group is used in pool and flavor resource, but the pool group 25 | only supports a 1:1 mapping with flavor. So it's not necessary to keep it 26 | since we can map 1 flavor : N pool directly. Another issue is currently there 27 | is no API to handle the pool group resource, which is very annoying to 28 | maintain by operators. 29 | 30 | For making a clarification to user, this bp proposes to remove useless 31 | pool group from Zaqar, just keeps the pool resource. 32 | 33 | For backward compatibility, we have split this work into two steps: 34 | 35 | 1. In Queens, we have supported the old way to use pool_group and the new way 36 | without it in Flavor both. 37 | 38 | 2. In Stein, we will remove the pool_group totally and only keep the new way 39 | in Flavor and Pool. 40 | 41 | This bp will implement the second step and totally remove the pool group from 42 | zaqar. 43 | 44 | 45 | Proposed change 46 | =============== 47 | 48 | 1. Modify pool and flavor operation API: 49 | 50 | * Remove group from flavor operation API: like creat, update, show, list. 51 | * Remove group from pool operation API: like creat, update, show, list. 52 | 53 | 2. Remove the related logic codes about group in the zaqar server: 54 | 55 | 3. The Data model of flavor and pool resources have been changed in Queens, 56 | There is no need to discard pool_group field and Drop table PoolGroup 57 | we will just keep it in table for degrade convenience. 58 | 59 | 4. Modify the related logic codes about group in the zaqar client: 60 | 61 | * openstack messaging flavor create 62 | * openstack messaging flavor update 63 | * openstack messaging pool create 64 | * openstack messaging pool update 65 | 66 | 67 | Drawbacks 68 | --------- 69 | None 70 | 71 | Alternatives 72 | ------------ 73 | None 74 | 75 | Data model impact 76 | ----------------- 77 | None 78 | 79 | REST API impact 80 | --------------- 81 | 82 | 1. Create flavor API 83 | The Request JSON when create flavor:: 84 | 85 | PUT: /v2/flavors/{flavor_name} 86 | 87 | { 88 | "pool_list": [pool1, pool2] 89 | } 90 | 91 | The response JSON when Create flavor:: 92 | 93 | Normal response codes: 201 94 | 95 | Error response codes: 96 | •BadRequest (400) 97 | •Unauthorized (401) 98 | •Forbidden (403) 99 | 100 | 2. Update flavor API 101 | The Request JSON when update flavor:: 102 | 103 | PATCH: /v2/flavors/{flavor_name} 104 | 105 | { 106 | "pool_list": [pool1, pool2, pool3] 107 | } 108 | 109 | The response JSON when update flavor:: 110 | 111 | 112 | { 113 | "href": "/v2/flavors/testflavor", 114 | "name": "testflavor", 115 | "capabilities": [ 116 | "FIFO", 117 | "CLAIMS", 118 | "DURABILITY", 119 | "AOD", 120 | "HIGH_THROUGHPUT" 121 | ], 122 | "pool_list": [pool1, pool2, pool3] 123 | } 124 | Normal response codes: 200 125 | 126 | Error response codes: 127 | •BadRequest (400) 128 | •Unauthorized (401) 129 | •Forbidden (403) 130 | •Not Found (404) 131 | •ServiceUnavailable (503) 132 | 133 | 3. Shows details for a flavor API 134 | The response JSON when show details flavor:: 135 | 136 | GET: /v2/flavors/{flavor_name} 137 | 138 | { 139 | "href": "/v2/flavors/testflavor", 140 | "capabilities": [ 141 | "FIFO", 142 | "CLAIMS", 143 | "DURABILITY", 144 | "AOD", 145 | "HIGH_THROUGHPUT" 146 | ], 147 | "pool_list": [pool1, pool2], 148 | "name": "testflavor" 149 | } 150 | 151 | The response JSON when show details flavor:: 152 | 153 | Normal response codes: 200 154 | 155 | Error response codes: 156 | •BadRequest (400) 157 | •Unauthorized (401) 158 | •Forbidden (403) 159 | •Not Found (404) 160 | •ServiceUnavailable (503) 161 | 162 | 4. List flavor API 163 | The response JSON when list flavors:: 164 | 165 | GET: /v2/flavors 166 | 167 | { 168 | "flavors": [ 169 | { 170 | "href": "/v2/flavors/test_flavor1", 171 | "name": "test_flavor1", 172 | "pool_list": [pool1, pool2] 173 | }, 174 | { 175 | "href": "/v2/flavors/test_flavor2", 176 | "name": "test_flavor2", 177 | "pool_list": [pool3, pool4] 178 | } 179 | ], 180 | "links": [ 181 | { 182 | "href": "/v2/flavors?marker=test_flavor2", 183 | "rel": "next" 184 | } 185 | ] 186 | } 187 | 188 | The response JSON when list flavors:: 189 | 190 | Normal response codes: 200 191 | 192 | Error response codes: 193 | •Unauthorized (401) 194 | •Forbidden (403) 195 | 196 | 197 | 5. Create pools API 198 | The Request JSON when create pools:: 199 | 200 | PUT: /v2/pools/{pool_name} 201 | 202 | { 203 | "weight": 100, 204 | "uri": "mongodb://127.0.0.1:27017", 205 | "options":{ 206 | "max_retry_sleep": 1 207 | }, 208 | "flavor": "testflavor" 209 | } 210 | 211 | 212 | The response JSON when Create pools:: 213 | 214 | Normal response codes: 201 215 | 216 | Error response codes: 217 | •BadRequest (400) 218 | •Unauthorized (401) 219 | ••Conflict (409) 220 | 221 | 6. Update pools API 222 | The Request JSON when update pools:: 223 | 224 | PATCH: /v2/pools/{pool_name} 225 | 226 | { 227 | "weight": 60, 228 | "uri": "mongodb://127.0.0.1:27017", 229 | "options":{ 230 | "max_retry_sleep": 1 231 | }, 232 | "flavor": "testflavor1" 233 | } 234 | 235 | The response JSON when update pools:: 236 | 237 | 238 | { 239 | "href": "/v2/pools/test_pool", 240 | "name": "test_pool", 241 | "weight": 60, 242 | "uri": "mongodb://127.0.0.1:27017", 243 | "flavor": "testflavor1" 244 | } 245 | Normal response codes: 200 246 | 247 | Error response codes: 248 | •BadRequest (400) 249 | •Unauthorized (401) 250 | •Not Found (404) 251 | •ServiceUnavailable (503) 252 | 253 | 7. Shows details for a pool API 254 | The response JSON when show details pool:: 255 | 256 | GET: /v2/pools/{pool_name} 257 | 258 | { 259 | "href": "/v2/pools/test_pool", 260 | "flavor": "flavor1", 261 | "name": "test_pool", 262 | "weight": 100, 263 | "uri": "mongodb://127.0.0.1:27017" 264 | } 265 | 266 | The response JSON when show details pool:: 267 | 268 | Normal response codes: 200 269 | 270 | Error response codes: 271 | •BadRequest (400) 272 | •Unauthorized (401) 273 | •ServiceUnavailable (503) 274 | 275 | 8. List pools API 276 | The response JSON when list pools:: 277 | 278 | GET: /v2/pools 279 | 280 | { 281 | "pools": [ 282 | { 283 | "href": "/v2/pools/test_pool1", 284 | "flavor": "flavor1", 285 | "name": "test_pool1", 286 | "weight": 60, 287 | "uri": "mongodb://192.168.1.10:27017" 288 | }, 289 | { 290 | "href": "/v2/pools/test_pool2", 291 | "flavor": "flavor1", 292 | "name": "test_pool2", 293 | "weight": 40, 294 | "uri": "mongodb://192.168.1.20:27017" 295 | } 296 | ], 297 | "links": [ 298 | { 299 | "href": "/v2/pools?marker=test_pool2", 300 | "rel": "next" 301 | } 302 | ] 303 | } 304 | 305 | The response JSON when list pools:: 306 | 307 | Normal response codes: 200 308 | 309 | Error response codes: 310 | •Unauthorized (401) 311 | •Not Found (404) 312 | 313 | 314 | We use the v2 interface, just add pool_list in the flavor API and add the 315 | flavor in the pool API. we will remove the old way in this bp. 316 | 317 | #. Old Way: 318 | 319 | * configure group in pool API and in flavor API; 320 | 321 | #. New Way: 322 | 323 | * configure pool_list in flavor API 324 | * add a pool to flavor: 325 | 326 | #. method one: update pool_list in flavor API 327 | #. method two: config a pool with the flavor in pool API. 328 | 329 | 330 | Implementation 331 | ============== 332 | 333 | Assignee(s) 334 | ----------- 335 | 336 | Primary assignee: 337 | gecong 338 | 339 | Milestones 340 | ---------- 341 | 342 | Target Milestone for completion: 343 | Stein 344 | 345 | Work Items 346 | ---------- 347 | 348 | #. Modify pool and flavor operation APIs. 349 | #. Remove the related logic codes about group in the zaqar server. 350 | 351 | Dependencies 352 | ============ 353 | 354 | None 355 | 356 | Testing 357 | ======= 358 | 359 | Both unit and Tempest tests need to be created to cover the code change. 360 | 361 | 362 | Documentation Impact 363 | ==================== 364 | 365 | The Zaqar API documentation will need to be updated to reflect the REST 366 | API changes. 367 | 368 | References 369 | ========== 370 | None 371 | 372 | -------------------------------------------------------------------------------- /specs/template.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://www.sphinx-doc.org/en/stable/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ============================= 14 | The title of your blueprint 15 | ============================= 16 | 17 | Include the URL of your launchpad blueprint: 18 | 19 | https://blueprints.launchpad.net/zaqar/+spec/example 20 | 21 | Introduction paragraph -- why are we doing anything? 22 | 23 | Problem description 24 | =================== 25 | 26 | A detailed description of the problem. 27 | 28 | Proposed change 29 | =============== 30 | 31 | Here is where you cover the change you propose to make in detail. How do you 32 | propose to solve this problem? 33 | 34 | If this is one part of a larger effort make it clear where this piece ends. In 35 | other words, what's the scope of this effort? 36 | 37 | Drawbacks 38 | --------- 39 | 40 | Are there any drawbacks in this proposal? Does it change things in the 41 | API? Will it have an impact in critical areas? Will it make other 42 | areas more complex? 43 | 44 | Please, answer any of the questions above or elaborate on the 45 | drawbacks of this proposal if there're any. 46 | 47 | Alternatives 48 | ------------ 49 | 50 | This is an optional section, where it does apply we'd just like a demonstration 51 | that some thought has been put into why the proposed approach is the best one. 52 | 53 | Implementation 54 | ============== 55 | 56 | Assignee(s) 57 | ----------- 58 | 59 | Who is leading the writing of the code? Or is this a blueprint where you're 60 | throwing it out there to see who picks it up? 61 | 62 | If more than one person is working on the implementation, please designate the 63 | primary author and contact. 64 | 65 | Primary assignee: 66 | 67 | 68 | Can list additional ids if they intend on doing substantial implementation work 69 | on this blueprint. 70 | 71 | Milestones 72 | ---------- 73 | 74 | Target Milestone for completion: 75 | Juno-2 76 | 77 | Work Items 78 | ---------- 79 | 80 | Work items or tasks -- break the feature up into the things that need to be 81 | done to implement it. Those parts might end up being done by different people, 82 | but we're mostly trying to understand the timeline for implementation. 83 | 84 | 85 | Dependencies 86 | ============ 87 | 88 | - Include specific references to specs and/or blueprints in zaqar, or in other 89 | projects, that this one either depends on or is related to. 90 | 91 | .. note:: 92 | 93 | This work is licensed under a Creative Commons Attribution 3.0 94 | Unported License. 95 | http://creativecommons.org/licenses/by/3.0/legalcode 96 | 97 | -------------------------------------------------------------------------------- /specs/ussuri/index.rst: -------------------------------------------------------------------------------- 1 | ======================= 2 | Ussuri Specifications 3 | ======================= 4 | 5 | .. toctree:: 6 | :glob: 7 | :maxdepth: 2 8 | 9 | query-queues-with-count.rst 10 | -------------------------------------------------------------------------------- /specs/ussuri/query-queues-with-count.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://sphinx-doc.org/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ======================= 14 | Query Queues With Count 15 | ======================= 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/query-queues-with-count 18 | 19 | This will support query queues with 'with_count=true' filter, Zaqar will return 20 | the amount of queues in backend storage. This feature will be very convenient 21 | to users to know how many resources they own. 22 | 23 | Problem description 24 | =================== 25 | 26 | Currently, Zaqar can't return the amount of queues when querying the queue. 27 | It depends on users themselves to calculate the number one by one. For other 28 | clients or applications also need to do it after invoking Zaqar's API. Its 29 | quite inconvenient for users or developers. 30 | 31 | Proposed change 32 | =============== 33 | 34 | Add a new query filter item named ``with_count``, default value is ``False``. 35 | When querying queues with "with_count=true" in url, Zaqar will add the 36 | function to calculate total number of queus in backend storage and 37 | return the amount of queues in response body like "count=100". 38 | 39 | API Impact 40 | ----------- 41 | Query queue list: 42 | 43 | .. code-block:: 44 | 45 | GET: /v2/queues?with_count=true 46 | 47 | RESPONSE CODE: 200 48 | RESPONSE BODY: 49 | { 50 | "count": 100, 51 | "queues": [...] 52 | } 53 | 54 | Drawbacks 55 | --------- 56 | 57 | None 58 | 59 | Alternatives 60 | ------------ 61 | 62 | None 63 | 64 | Implementation 65 | ============== 66 | 67 | Assignee(s) 68 | ----------- 69 | 70 | Primary assignee: 71 | wanghao 72 | 73 | Secondary assignee: 74 | None 75 | 76 | Milestones 77 | ---------- 78 | 79 | Target Milestone for completion: 80 | ussuri RC2 81 | 82 | Work Items 83 | ---------- 84 | 85 | #. Modify transport code. 86 | #. Add release note for this feature. 87 | #. Update API reference. 88 | #. Change unit, functional and tempest tests accordingly. 89 | #. Add client support. 90 | 91 | Dependencies 92 | ============ 93 | 94 | None 95 | 96 | References 97 | ========== 98 | 99 | None 100 | -------------------------------------------------------------------------------- /specs/victoria/encrypted-messages-in-queue.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://sphinx-doc.org/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | =========================== 14 | Encrypted Messages in Queue 15 | =========================== 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/encrypted-messages-in-queue 18 | 19 | The queue in Zaqar will support to encrypt messages before storing them into 20 | storage backends, also could support to decrypt (or not) messages when those 21 | are claimed by consumer. This feature will enhance the security of messaging 22 | service. 23 | 24 | Problem description 25 | =================== 26 | 27 | Currently, Zaqar can't encrypt any messages and just store those messages into 28 | storage backends. That'll bring some security issues like information leakage 29 | or hacker attack. 30 | 31 | Proposed change 32 | =============== 33 | 34 | 1. Add one new metadata in queue object that will indicate how to 35 | encrypt messages: 36 | 37 | #. "_enable_encrypt_messages=true/false" : this will tell Zaqar whether encrypt 38 | messages before storing them into backends or not. 39 | 40 | 2. Support to use algorithms to encrypt messages in transport layer before 41 | storing them into backends. In V cycle, Zaqar may just support the global 42 | key for encryption and decryption, and it will be held by Zaqar service. 43 | Users don't need to encrypt/decypt message by themselves. 44 | 45 | 3. Go a step further, Zaqar also could support to let user upload their public 46 | key and hold the private key by themselves. In this case, Zaqar just need to 47 | encrypt the message by using public key and return the encrypted messages to 48 | user. 49 | 50 | 4. About the algorithms, in V cycle, Zaqar will introduce the AES-256 51 | encryption at first. In next cycles, Zaqar can suppot asymmetric 52 | encryption to let user upload public key and keep the private key 53 | by their own. 54 | 55 | .. note:: 56 | 57 | About the option of encryption algorithms and keys, Zaqar would support 58 | specify them throught more metadatas of queues, but it will be done in next 59 | serveral cycles. In Victoria, we will choose one algorithm (like AES256) to 60 | support and support to storage the keys by Zaqar itself or other 61 | service like Barbican. 62 | 63 | API Impact 64 | ----------- 65 | 66 | Create queue list: 67 | 68 | .. code-block:: 69 | 70 | POST: /v2/queues/queue_name 71 | 72 | RESPONSE CODE: 200 73 | REQUEST BODY: 74 | { 75 | "_enable_encrypt_messages": true 76 | } 77 | 78 | Drawbacks 79 | --------- 80 | 81 | The ecryption algorithms will impact the performance of storing 82 | messages into backends 83 | and getting the messages from the queue. 84 | 85 | This depends on which kind of encryption algorithms we choose and support. 86 | 87 | Alternatives 88 | ------------ 89 | 90 | None 91 | 92 | Implementation 93 | ============== 94 | 95 | Assignee(s) 96 | ----------- 97 | 98 | Primary assignee: 99 | wanghao 100 | 101 | Secondary assignee: 102 | None 103 | 104 | Milestones 105 | ---------- 106 | 107 | Target Milestone for completion: 108 | victoria M-2 109 | 110 | Work Items 111 | ---------- 112 | 113 | #. Modify api and transport code. 114 | #. Add support for encryption algorithm and key management. 115 | #. Add release note for this feature. 116 | #. Update API reference. 117 | #. Change unit, functional and tempest tests accordingly. 118 | #. Add client support. 119 | 120 | Dependencies 121 | ============ 122 | 123 | None 124 | 125 | References 126 | ========== 127 | 128 | None 129 | -------------------------------------------------------------------------------- /specs/victoria/index.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | Victoria Specifications 3 | ======================== 4 | 5 | .. toctree:: 6 | :glob: 7 | :maxdepth: 2 8 | 9 | encrypted-messages-in-queue 10 | -------------------------------------------------------------------------------- /specs/wallaby/index.rst: -------------------------------------------------------------------------------- 1 | ======================= 2 | Wallaby Specifications 3 | ======================= 4 | 5 | .. toctree:: 6 | :glob: 7 | :maxdepth: 2 8 | 9 | support-extra-specs-to-subscription-confirming 10 | -------------------------------------------------------------------------------- /specs/wallaby/support-extra-specs-to-subscription-confirming.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://sphinx-doc.org/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ============================================== 14 | Support Extra Specs to Subscription-confirming 15 | ============================================== 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/support-extra-specs-to-subscription-confirming 18 | 19 | This requirement came from a true scenariowhen people use the subscription 20 | function of Zaqar, for example, text messages, they need to return some extra 21 | information like message authentication code in confirming process. 22 | Now Zaqar still cant support it, that will impact the usage of subscription. 23 | 24 | Problem description 25 | =================== 26 | 27 | Currently, Zaqar can't support extra information in subscription 28 | confirming process. This will block user to input information that 29 | is needed when subcription is confirming. There is a true case that 30 | came from Zaqar's user. They want to input the message authentication 31 | code into Zaqar when the subscription is confirming, the code will be 32 | used to identify the subscriber. So Zaqar should support this kind of 33 | mechanism. 34 | 35 | Proposed change 36 | =============== 37 | 38 | 1. Introduce a key-value called "extra_spec" in confirming request body. 39 | 40 | 2. Introduce a driver mechanism to let vendors to implement what they 41 | want to do with extra_spec information. 42 | 43 | API Impact 44 | ----------- 45 | 46 | Subscription confirming request: 47 | 48 | .. code-block:: 49 | 50 | PUT: /v2/subscriptions/subscription_id/confirm 51 | 52 | RESPONSE CODE: 204 53 | REQUEST BODY: 54 | { 55 | "confirmed": true, 56 | "extra_spec": {"message_authentication_code": "xxxxxx"} 57 | } 58 | 59 | Drawbacks 60 | --------- 61 | 62 | None 63 | 64 | Alternatives 65 | ------------ 66 | 67 | None 68 | 69 | Implementation 70 | ============== 71 | 72 | Assignee(s) 73 | ----------- 74 | 75 | Primary assignee: 76 | wanghao 77 | 78 | Secondary assignee: 79 | None 80 | 81 | Milestones 82 | ---------- 83 | 84 | Target Milestone for completion: 85 | wallaby M-2 86 | 87 | Work Items 88 | ---------- 89 | 90 | #. Modify api and transport code. 91 | #. Add driver mechanism to handler the extra spec. 92 | #. Add release note for this feature. 93 | #. Update API reference. 94 | #. Change unit, functional and tempest tests accordingly. 95 | #. Add client support. 96 | 97 | Dependencies 98 | ============ 99 | 100 | None 101 | 102 | References 103 | ========== 104 | 105 | None 106 | -------------------------------------------------------------------------------- /specs/xena/index.rst: -------------------------------------------------------------------------------- 1 | ===================== 2 | Xena Specifications 3 | ===================== 4 | 5 | .. toctree:: 6 | :glob: 7 | :maxdepth: 2 8 | -------------------------------------------------------------------------------- /specs/yoga/index.rst: -------------------------------------------------------------------------------- 1 | ===================== 2 | Yoga Specifications 3 | ===================== 4 | 5 | .. toctree:: 6 | :glob: 7 | :maxdepth: 2 8 | 9 | remove-v1-api-code-from-zaqar 10 | -------------------------------------------------------------------------------- /specs/yoga/remove-v1-api-code-from-zaqar.rst: -------------------------------------------------------------------------------- 1 | .. 2 | This template should be in ReSTructured text. The filename in the git 3 | repository should match the launchpad URL, for example a URL of 4 | https://blueprints.launchpad.net/zaqar/+spec/awesome-thing should be named 5 | awesome-thing.rst. 6 | 7 | Please do not delete any of the sections in this 8 | template. If you have nothing to say for a whole section, just write: None 9 | 10 | For help with syntax, see http://sphinx-doc.org/rest.html 11 | To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html 12 | 13 | ============================= 14 | Remove V1 API Code From Zaqar 15 | ============================= 16 | 17 | https://blueprints.launchpad.net/zaqar/+spec/remove-v1-api-code 18 | 19 | Now there is no need for V1 APIs, its old and could be completely replaced by V2 APIs. 20 | So it's time to remove it from the code tree. This action is first one of steps to optimize 21 | Zaqar's codes and functions. 22 | 23 | Problem description 24 | =================== 25 | 26 | Currently, Zaqar has tree versions of APIs, V1, V1_1 and V2. The V1 is the oldest apis that 27 | there is no need for them right now. All new features are implemented under V2 APIs. So 28 | its time to remove V1 APIs and optimize the Zaqar's codes. 29 | 30 | Proposed change 31 | =============== 32 | 33 | 1. Remove all V1 APIs codes and change the basic APIs to V1_1. 34 | 35 | 2. Clean the unit tests and tempest tests about V1 APIs. 36 | 37 | 3. Modify the API doc according the changes. 38 | 39 | API Impact 40 | ----------- 41 | 42 | The whole V1 APIs code will be removed from Zaqar's service. 43 | 44 | Drawbacks 45 | --------- 46 | 47 | If someone are still using V1 APIs, they must change the API version to 48 | V1_1 or V2(best to V2) after Yoga release. 49 | 50 | Alternatives 51 | ------------ 52 | 53 | None 54 | 55 | Implementation 56 | ============== 57 | 58 | Assignee(s) 59 | ----------- 60 | 61 | Primary assignee: 62 | wanghao 63 | 64 | Secondary assignee: 65 | None 66 | 67 | Milestones 68 | ---------- 69 | 70 | Target Milestone for completion: 71 | Yoga M-1 72 | 73 | Work Items 74 | ---------- 75 | 76 | #. Remove V1 api and transport code. 77 | #. Remove unit and functional tests for V1. 78 | #. Add release note for this feature. 79 | #. Update API reference according those changes. 80 | #. Change tempest tests accordingly. 81 | #. remove client support. 82 | 83 | Dependencies 84 | ============ 85 | 86 | None 87 | 88 | References 89 | ========== 90 | 91 | None 92 | -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/zaqar-specs/189334067a625b6957ad75ee7a6019120e93ae2b/test-requirements.txt -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openstack/zaqar-specs/189334067a625b6957ad75ee7a6019120e93ae2b/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_title.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 2 | # not use this file except in compliance with the License. You may obtain 3 | # a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | # License for the specific language governing permissions and limitations 11 | # under the License. 12 | 13 | import glob 14 | import os 15 | import re 16 | 17 | import docutils.core 18 | import testtools 19 | 20 | 21 | class TestTitles(testtools.TestCase): 22 | def _get_title(self, section_tree): 23 | section = { 24 | 'subtitles': [], 25 | } 26 | for node in section_tree: 27 | if node.tagname == 'title': 28 | section['name'] = node.rawsource 29 | elif node.tagname == 'section': 30 | subsection = self._get_title(node) 31 | section['subtitles'].append(subsection['name']) 32 | return section 33 | 34 | def _get_titles(self, spec): 35 | titles = {} 36 | for node in spec: 37 | if node.tagname == 'section': 38 | section = self._get_title(node) 39 | titles[section['name']] = section['subtitles'] 40 | return titles 41 | 42 | def _check_titles(self, spec, titles): 43 | self.assertTrue(len(titles) >= 4, 44 | "Titles count in '%s' doesn't match expected" % spec) 45 | problem = 'Problem description' 46 | driver = 'Driver description' 47 | self.assertTrue(problem in titles or driver in titles) 48 | 49 | proposed = 'Proposed change' 50 | self.assertIn(proposed, titles) 51 | if problem in titles: 52 | self.assertIn('Alternatives', titles[proposed], spec) 53 | 54 | impl = 'Implementation' 55 | self.assertIn(impl, titles) 56 | self.assertIn('Assignee(s)', titles[impl]) 57 | self.assertIn('Work Items', titles[impl]) 58 | 59 | deps = 'Dependencies' 60 | self.assertIn(deps, titles) 61 | 62 | def _check_lines_wrapping(self, tpl, raw): 63 | for i, line in enumerate(raw.split("\n")): 64 | if "http://" in line or "https://" in line or "/queues" in line: 65 | continue 66 | self.assertTrue( 67 | len(line) < 80, 68 | msg="%s:%d: Line limited to a maximum of 79 characters." % 69 | (tpl, i + 1)) 70 | 71 | def _check_no_cr(self, tpl, raw): 72 | matches = re.findall('\r', raw) 73 | self.assertEqual( 74 | len(matches), 0, 75 | "Found %s literal carriage returns in file %s" % 76 | (len(matches), tpl)) 77 | 78 | def _check_trailing_spaces(self, tpl, raw): 79 | for i, line in enumerate(raw.split("\n")): 80 | trailing_spaces = re.findall(" +$", line) 81 | msg = "Found trailing spaces on line %s of %s" % (i + 1, tpl) 82 | self.assertEqual(len(trailing_spaces), 0, msg) 83 | 84 | def test_template(self): 85 | # NOTE (e0ne): adding 'template.rst' to ignore dirs to exclude it from 86 | # os.listdir output 87 | ignored_dirs = {'template.rst', 'api'} 88 | 89 | files = ['specs/template.rst'] 90 | 91 | # NOTE (e0ne): We don't check specs in 'api' directory because 92 | # they don't match template.rts. Uncomment code below it you want 93 | # to test them. 94 | # files.extend(glob.glob('specs/api/*/*')) 95 | 96 | releases = set(os.listdir('specs')) - ignored_dirs 97 | for release in releases: 98 | specs = glob.glob('specs/%s/*' % release) 99 | files.extend(specs) 100 | 101 | for filename in files: 102 | self.assertTrue(filename.endswith(".rst"), 103 | "spec's file must use 'rst' extension.") 104 | if filename.split('/')[-1] != 'index.rst': 105 | with open(filename) as f: 106 | data = f.read() 107 | 108 | spec = docutils.core.publish_doctree(data) 109 | titles = self._get_titles(spec) 110 | self._check_titles(filename, titles) 111 | self._check_lines_wrapping(filename, data) 112 | self._check_no_cr(filename, data) 113 | self._check_trailing_spaces(filename, data) 114 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | minversion = 3.2.0 3 | envlist = docs,pep8 4 | skipsdist = True 5 | ignore_basepython_conflict = True 6 | 7 | 8 | [testenv] 9 | basepython = python3 10 | usedevelop = True 11 | setenv = 12 | VIRTUAL_ENV={envdir} 13 | deps = -r{toxinidir}/requirements.txt 14 | -r{toxinidir}/test-requirements.txt 15 | commands = stestr run --slowest {posargs} 16 | 17 | [testenv:venv] 18 | commands = {posargs} 19 | 20 | [testenv:docs] 21 | commands = sphinx-build -a -W -b html doc/source doc/build/html 22 | 23 | [testenv:spelling] 24 | deps = 25 | -r{toxinidir}/requirements.txt 26 | sphinxcontrib-spelling 27 | PyEnchant 28 | commands = sphinx-build -b spelling doc/source doc/build/spelling 29 | 30 | [testenv:cover] 31 | setenv = 32 | PYTHON=coverage run --source specs --parallel-mode 33 | commands = 34 | stestr run {posargs} 35 | coverage combine 36 | coverage html -d cover 37 | coverage xml -o cover/coverage.xml 38 | 39 | [testenv:pep8] 40 | commands = flake8 41 | 42 | [flake8] 43 | # E123, E125 skipped as they are invalid PEP-8. 44 | 45 | show-source = True 46 | ignore = E123,E125 47 | exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build 48 | --------------------------------------------------------------------------------