├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── NOTICE ├── README.md ├── _metadata_sherlock.py ├── _metadata_watson.py ├── at.tf ├── at.tfvars ├── atindown.sh ├── atinup.sh ├── atrun.sh ├── deploy ├── datawire-sherlock │ ├── centos-7-x64 │ │ └── etc │ │ │ └── systemd │ │ │ └── system │ │ │ └── sherlock.service │ ├── common │ │ └── etc │ │ │ └── datawire │ │ │ └── sherlock.conf │ └── ubuntu-14.04-x64 │ │ └── etc │ │ └── init │ │ └── sherlock.conf └── datawire-watson │ ├── centos-7-x64 │ └── etc │ │ └── systemd │ │ └── system │ │ └── watson.service │ ├── common │ └── etc │ │ └── datawire │ │ └── watson.conf.proto │ └── ubuntu-14.04-x64 │ └── etc │ └── init │ └── watson.conf ├── docs ├── README.md └── source │ ├── _static │ └── datawire.css │ ├── _templates │ └── layout.html │ ├── architecture.rst │ ├── bakerstreet.png │ ├── conf.py │ ├── deployment.rst │ ├── extract_tags.py │ ├── index.rst │ ├── overview.rst │ ├── parameters.py │ ├── quickstart.rst │ └── reference.rst ├── pkg-sherlock ├── pkg-watson ├── resources └── infrastructure-setup │ ├── bin │ ├── test_service.py │ └── watson_controller.py │ ├── config │ └── watson_controller.yml │ └── templates │ ├── sherlock.conf.tpl │ ├── watson-test_service-nopath.conf.tpl │ └── watson-test_service-path.conf.tpl ├── setup_sherlock.py ├── setup_watson.py ├── sherlock ├── tests ├── conftest.py └── test_bakerstreet.py └── watson /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.*~ 3 | *~ 4 | *.class 5 | build 6 | tags 7 | dist 8 | bakerstreet.egg-info 9 | 10 | # --- Intellij IDEA --- 11 | *.iml 12 | .idea 13 | 14 | # --- Build and Test Automation --- 15 | tmp 16 | *.tfstate 17 | *.tfstate.backup 18 | test_*.xml 19 | 20 | # --- KDE --- 21 | .directory -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | All notable modifications to the Baker Street project (bakerstreet.io) will be documented in this file. 3 | 4 | ## [0.5] - 2015-09-25 5 | ### Changed 6 | - Renamed "liveness_url" to "health_check_url" in Watson config file. 7 | - Improved Watson and Sherlock config file comments. 8 | 9 | ### Added 10 | - Added a new Watson configuration property "service_name" that replaces setting a service_name in Watson's service URL. This allows Baker Street to route traffic to microservices that are exposed on a specific URL path. (issue: datawire/bakerstreet#4) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright 2015 The Baker Street Authors. All rights reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Baker Street 2 | 3 | Baker Street is a service discovery and routing system designed for microservice architectures. 4 | 5 | Baker Street simplifies scaling, testing, and upgrading microservices by: 6 | 7 | * automatically splitting traffic among all healthy services sharing the same name in the system 8 | * making load balancing more efficient and robust by using local load balancers 9 | * removing problematic instances from the rotation more quickly by using local health checkers 10 | * enabling canary testing for staged testing and deployment of service upgrades 11 | 12 | Baker Street consists of three components: 13 | 14 | * Sherlock - an HAProxy-based routing system with local instances corresponding to each instance of your application to determine where connections from that instance should go 15 | * Watson - a health checker with local instances corresponding to each instance of your application 16 | * Datawire Directory - a global service discovery mechanism that receives availability information from each Watson instance and pushes changes in availability to local Sherlock instances as needed 17 | 18 | ## Baker Street System Requirements 19 | 20 | Baker Street works on any flavor of Enterprise Linux 7 or on Ubuntu 14.04 LTS. Since Baker Street must be co-located with your service, your service must also run on one of these platforms if you wish to use it with Baker Street. Baker Street has no other requirements; you are free to use the language, framework, and tools of your choice while integrating with it. 21 | 22 | That said, if you wish to test your installation using a simple sample service as outlined in the Baker Street documentation, you will also need the following: 23 | 24 | * JDK 1.8 25 | * maven 3 or higher 26 | 27 | ## Installing Baker Street 28 | 29 | We expect it to take approximately 15 minutes to install a working local development environment with all three components. 30 | 31 | Directions for installing Baker Street locally can be found [here](http://bakerstreet.io/docs/quickstart.html#setup). 32 | 33 | ## Next Steps 34 | 35 | Additional information about Baker Street's design and architecture can be found [here](http://bakerstreet.io/docs/architecture.html). 36 | 37 | Baker Street components all support a variety of options available via configuration files. For example, each component supports a range of logging levels that can be independently toggled within these configuration files. Information on how to configure each component can be found [here](http://bakerstreet.io/docs/reference.html). 38 | 39 | ## Additional Information 40 | 41 | For additional information, visit the Baker Street website at [http://www.bakerstreet.io](http://www.bakerstreet.io). 42 | 43 | Please post any questions about Baker Street on [Stack Overflow](http://www.stackoverflow.com) using the tag bakerstreet. 44 | -------------------------------------------------------------------------------- /_metadata_sherlock.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 datawire. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __all__ = [ 16 | "__title__", "__summary__", "__uri__", "__version__", "__author__", 17 | "__email__", "__license__", "__copyright__" 18 | ] 19 | 20 | __title__ = "bakerstreet" 21 | __summary__ = "Client-side load balancing for microservices" 22 | __uri__ = "http://bakerstreet.io/" 23 | 24 | __version__ = "0.5" 25 | 26 | __author__ = "datawire.io" 27 | __email__ = "hello@datawire.io" 28 | 29 | __license__ = "Apache License, Version 2.0" 30 | __copyright__ = "2015 %s" % __author__ 31 | -------------------------------------------------------------------------------- /_metadata_watson.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 datawire. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __all__ = [ 16 | "__title__", "__summary__", "__uri__", "__version__", "__author__", 17 | "__email__", "__license__", "__copyright__" 18 | ] 19 | 20 | __title__ = "bakerstreet" 21 | __summary__ = "Client-side load balancing for microservices" 22 | __uri__ = "http://bakerstreet.io/" 23 | 24 | __version__ = "0.5" 25 | 26 | __author__ = "datawire.io" 27 | __email__ = "hello@datawire.io" 28 | 29 | __license__ = "Apache License, Version 2.0" 30 | __copyright__ = "2015 %s" % __author__ 31 | -------------------------------------------------------------------------------- /at.tf: -------------------------------------------------------------------------------- 1 | variable "aws_region" { 2 | default = "us-east-1" 3 | description = "the region to use" 4 | } 5 | 6 | variable "aws_access_key" { description = "the API access key" } 7 | variable "aws_secret_key" { description = "the API secret key" } 8 | 9 | variable "resources_owner" { description = "the owner of the created resources" } 10 | 11 | variable "ec2_ami" { 12 | default { 13 | "us-east-1" = "ami-96a818fe" 14 | "us-west-2" = "ami-c7d092f7" 15 | } 16 | description = "the AMI to launch the EC2 instance with" 17 | } 18 | 19 | variable "deploy_id" { description = "a unique ID for the deployment" } 20 | variable "remote_user" { description = "the name of the user to login to the system with for provisioning" } 21 | 22 | variable "package_repository" { description = "Owner and repository name (fmt: (/)" } 23 | 24 | variable "proton_rpm" { description = "the path to the Proton RPM file" } 25 | variable "datawire_rpm" { description = "the path to the Datawire Common RPM file" } 26 | variable "directory_rpm" { description = "the path to the Datawire Directory RPM file" } 27 | 28 | variable "watson_rpm" { description = "the path to the Datawire Watson RPM file" } 29 | variable "sherlock_rpm" { description = "the path to the Datawire Sherlock RPM file" } 30 | 31 | provider "aws" { 32 | region = "${var.aws_region}" 33 | 34 | access_key = "${var.aws_access_key}" 35 | secret_key = "${var.aws_secret_key}" 36 | } 37 | 38 | resource "aws_key_pair" "baker_street" { 39 | key_name = "baker_street-${var.deploy_id}" 40 | public_key = "${file("${path.module}/tmp/temporary_key.pub")}" 41 | } 42 | 43 | resource "aws_vpc" "baker_street" { 44 | cidr_block = "10.0.0.0/16" 45 | enable_dns_hostnames = false 46 | enable_dns_support = true 47 | instance_tenancy = "default" 48 | tags { 49 | Environment = "test" 50 | Name = "baker_street" 51 | Owner = "${var.resources_owner}" 52 | Role = "baker_street" 53 | } 54 | } 55 | 56 | resource "aws_internet_gateway" "baker_street_igw" { 57 | tags { 58 | Environment = "test" 59 | Name = "baker_street" 60 | Owner = "${var.resources_owner}" 61 | Role = "baker_street" 62 | } 63 | vpc_id = "${aws_vpc.baker_street.id}" 64 | } 65 | 66 | resource "aws_subnet" "baker_street" { 67 | cidr_block = "10.0.1.0/24" 68 | map_public_ip_on_launch = true 69 | tags { 70 | Environment = "test" 71 | Name = "baker_street" 72 | Owner = "${var.resources_owner}" 73 | Role = "baker_street" 74 | } 75 | vpc_id = "${aws_vpc.baker_street.id}" 76 | } 77 | 78 | resource "aws_route_table" "baker_street" { 79 | route = { 80 | cidr_block = "0.0.0.0/0" 81 | gateway_id = "${aws_internet_gateway.baker_street_igw.id}" 82 | } 83 | tags { 84 | Environment = "test" 85 | Name = "baker_street" 86 | Owner = "${var.resources_owner}" 87 | Role = "baker_street" 88 | } 89 | vpc_id = "${aws_vpc.baker_street.id}" 90 | } 91 | 92 | resource "aws_route_table_association" "baker_street" { 93 | route_table_id = "${aws_route_table.baker_street.id}" 94 | subnet_id = "${aws_subnet.baker_street.id}" 95 | } 96 | 97 | resource "aws_security_group" "baker_street" { 98 | egress { 99 | cidr_blocks = ["0.0.0.0/0"] 100 | from_port = 0 101 | protocol = "-1" 102 | to_port = 0 103 | } 104 | ingress { 105 | cidr_blocks = ["0.0.0.0/0"] 106 | from_port = 22 107 | protocol = "tcp" 108 | to_port = 22 109 | } 110 | ingress { 111 | cidr_blocks = ["0.0.0.0/0"] 112 | from_port = 5672 113 | protocol = "tcp" 114 | to_port = 5672 115 | } 116 | ingress { 117 | cidr_blocks = ["0.0.0.0/0"] 118 | from_port = 5001 119 | protocol = "tcp" 120 | to_port = 5002 121 | } 122 | ingress { 123 | cidr_blocks = ["0.0.0.0/0"] 124 | from_port = 9001 125 | protocol = "tcp" 126 | to_port = 9001 127 | } 128 | ingress { 129 | cidr_blocks = ["0.0.0.0/0"] 130 | from_port = -1 131 | protocol = "icmp" 132 | to_port = -1 133 | } 134 | name = "baker_street" 135 | tags { 136 | Environment = "test" 137 | Name = "baker_street" 138 | Owner = "${var.resources_owner}" 139 | Role = "baker_street" 140 | } 141 | vpc_id = "${aws_vpc.baker_street.id}" 142 | } 143 | 144 | resource "aws_instance" "directory" { 145 | ami = "${lookup(var.ec2_ami, var.aws_region)}" 146 | associate_public_ip_address = true 147 | depends_on = ["aws_internet_gateway.baker_street_igw", "aws_key_pair.baker_street"] 148 | instance_type = "t2.micro" 149 | key_name = "${aws_key_pair.baker_street.key_name}" 150 | monitoring = false 151 | subnet_id = "${aws_subnet.baker_street.id}" 152 | tags { 153 | Environment = "test" 154 | Name = "directory-test-${var.resources_owner}" 155 | Owner = "${var.resources_owner}" 156 | Role = "directory" 157 | } 158 | vpc_security_group_ids = [ 159 | "${aws_security_group.baker_street.id}" 160 | ] 161 | 162 | connection { 163 | user = "${var.remote_user}" 164 | key_file = "${path.module}/tmp/temporary_key" 165 | } 166 | 167 | provisioner "file" { 168 | source = "${var.proton_rpm}" 169 | destination = "/home/centos/datawire-proton.rpm" 170 | } 171 | 172 | provisioner "file" { 173 | source = "${var.datawire_rpm}" 174 | destination = "/home/centos/datawire.rpm" 175 | } 176 | 177 | provisioner "file" { 178 | source = "${var.directory_rpm}" 179 | destination = "/home/centos/datawire-directory.rpm" 180 | } 181 | 182 | provisioner "remote-exec" { 183 | inline = [ 184 | "sudo yum -y install datawire-proton.rpm datawire.rpm datawire-directory.rpm", 185 | "printf '[Datawire]\ndirectory_host=${self.private_ip}' | sudo tee -a /etc/datawire/directory.conf", 186 | "sudo systemctl start directory.service" 187 | ] 188 | } 189 | } 190 | 191 | resource "template_file" "watson_config_nopath" { 192 | filename = "${path.module}/resources/infrastructure-setup/templates/watson-test_service-nopath.conf.tpl" 193 | vars { 194 | directory_host = "${aws_instance.directory.private_ip}" 195 | } 196 | } 197 | 198 | resource "template_file" "watson_config_path" { 199 | filename = "${path.module}/resources/infrastructure-setup/templates/watson-test_service-path.conf.tpl" 200 | vars { 201 | directory_host = "${aws_instance.directory.private_ip}" 202 | } 203 | } 204 | 205 | resource "template_file" "sherlock_config" { 206 | filename = "${path.module}/resources/infrastructure-setup/templates/sherlock.conf.tpl" 207 | vars { 208 | directory_host = "${aws_instance.directory.private_ip}" 209 | } 210 | } 211 | 212 | resource "aws_instance" "test_service" { 213 | ami = "${lookup(var.ec2_ami, var.aws_region)}" 214 | associate_public_ip_address = true 215 | depends_on = [ 216 | "aws_internet_gateway.baker_street_igw", 217 | "aws_key_pair.baker_street", 218 | "aws_instance.directory" 219 | ] 220 | instance_type = "t2.micro" 221 | key_name = "${aws_key_pair.baker_street.key_name}" 222 | monitoring = false 223 | subnet_id = "${aws_subnet.baker_street.id}" 224 | tags { 225 | Environment = "test" 226 | Name = "test_service-test-${var.resources_owner}" 227 | Owner = "${var.resources_owner}" 228 | Role = "foobar_service" 229 | } 230 | vpc_security_group_ids = [ 231 | "${aws_security_group.baker_street.id}" 232 | ] 233 | 234 | connection { 235 | user = "${var.remote_user}" 236 | key_file = "${path.module}/tmp/temporary_key" 237 | } 238 | 239 | provisioner "file" { 240 | source = "${var.proton_rpm}" 241 | destination = "/home/centos/datawire-proton.rpm" 242 | } 243 | 244 | provisioner "file" { 245 | source = "${var.datawire_rpm}" 246 | destination = "/home/centos/datawire.rpm" 247 | } 248 | 249 | provisioner "file" { 250 | source = "${var.watson_rpm}" 251 | destination = "/home/centos/datawire-watson.rpm" 252 | } 253 | 254 | provisioner "file" { 255 | source = "${path.module}/resources/infrastructure-setup/bin/test_service.py" 256 | destination = "/home/centos/test_service.py" 257 | } 258 | 259 | provisioner "file" { 260 | source = "${path.module}/resources/infrastructure-setup/bin/watson_controller.py" 261 | destination = "/home/centos/watson_controller.py" 262 | } 263 | 264 | provisioner "file" { 265 | source = "${path.module}/resources/infrastructure-setup/config/watson_controller.yml" 266 | destination = "/home/centos/watson_controller.yml" 267 | } 268 | 269 | provisioner "remote-exec" { 270 | inline = [ 271 | "sudo rpm -iUvh http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-5.noarch.rpm", 272 | "sudo yum -y install python-pip", 273 | "yes | sudo pip install flask", 274 | 275 | "sudo yum -y install datawire-proton.rpm datawire.rpm datawire-watson.rpm", 276 | 277 | "chmod +x /home/centos/test_service.py", 278 | "chmod +x /home/centos/watson_controller.py", 279 | 280 | "sudo cat > /tmp/watson-test_service-nopath.conf << EOF", 281 | "${template_file.watson_config_nopath.rendered}", 282 | "EOF", 283 | 284 | "printf '\nservice_url: http://${self.private_ip}:5001' | sudo tee -a /tmp/watson-test_service-nopath.conf", 285 | 286 | "sudo cat > /tmp/watson-test_service-path.conf << EOF", 287 | "${template_file.watson_config_path.rendered}", 288 | "EOF", 289 | 290 | "printf '\nservice_url: http://${self.private_ip}:5001/hello' | sudo tee -a /tmp/watson-test_service-path.conf", 291 | 292 | "sudo mv /tmp/watson-test_service-nopath.conf /etc/datawire/watson-test_service-nopath.conf", 293 | "sudo mv /tmp/watson-test_service-path.conf /etc/datawire/watson-test_service-path.conf", 294 | 295 | "nohup /home/centos/test_service.py &> test_service.out&", 296 | "nohup /home/centos/watson_controller.py -c /home/centos/watson_controller.yml &> watson_controller.out&", 297 | 298 | "sleep 5", 299 | ] 300 | } 301 | } 302 | 303 | resource "aws_instance" "test_runner" { 304 | ami = "${lookup(var.ec2_ami, var.aws_region)}" 305 | associate_public_ip_address = true 306 | depends_on = [ 307 | "aws_internet_gateway.baker_street_igw", 308 | "aws_key_pair.baker_street", 309 | "aws_instance.directory" 310 | ] 311 | instance_type = "t2.micro" 312 | key_name = "${aws_key_pair.baker_street.key_name}" 313 | monitoring = false 314 | subnet_id = "${aws_subnet.baker_street.id}" 315 | tags { 316 | Environment = "test" 317 | Name = "test_runner-test-${var.resources_owner}" 318 | Owner = "${var.resources_owner}" 319 | Role = "foobar_service_tests" 320 | } 321 | vpc_security_group_ids = [ 322 | "${aws_security_group.baker_street.id}" 323 | ] 324 | 325 | connection { 326 | user = "${var.remote_user}" 327 | key_file = "${path.module}/tmp/temporary_key" 328 | } 329 | 330 | provisioner "file" { 331 | source = "${var.proton_rpm}" 332 | destination = "/home/centos/datawire-proton.rpm" 333 | } 334 | 335 | provisioner "file" { 336 | source = "${var.datawire_rpm}" 337 | destination = "/home/centos/datawire.rpm" 338 | } 339 | 340 | provisioner "file" { 341 | source = "${var.sherlock_rpm}" 342 | destination = "/home/centos/datawire-sherlock.rpm" 343 | } 344 | 345 | provisioner "file" { 346 | source = "${path.module}/tests/conftest.py" 347 | destination = "/home/centos/conftest.py" 348 | } 349 | 350 | provisioner "file" { 351 | source = "${path.module}/tests/test_bakerstreet.py" 352 | destination = "/home/centos/test_bakerstreet.py" 353 | } 354 | 355 | provisioner "remote-exec" { 356 | inline = [ 357 | "sudo rpm -iUvh http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-5.noarch.rpm", 358 | "sudo yum -y install python-pip", 359 | "sudo yum -y install datawire-proton.rpm datawire.rpm datawire-sherlock.rpm", 360 | 361 | "yes | sudo pip install pytest flask", 362 | 363 | "sudo cat > /tmp/sherlock.conf << EOF", 364 | "${template_file.sherlock_config.rendered}", 365 | "EOF", 366 | "sudo mv /tmp/sherlock.conf /etc/datawire/sherlock.conf", 367 | 368 | "sudo systemctl start sherlock.service", 369 | 370 | "sleep 5" 371 | ] 372 | } 373 | } 374 | 375 | output "test_runner_public_ip" { 376 | value = "${aws_instance.test_runner.public_ip}" 377 | } 378 | 379 | output "test_service_private_ip" { 380 | value = "${aws_instance.test_service.private_ip}" 381 | } -------------------------------------------------------------------------------- /at.tfvars: -------------------------------------------------------------------------------- 1 | aws_region = "us-west-2" 2 | 3 | remote_user = "centos" 4 | resources_owner = "at" -------------------------------------------------------------------------------- /atindown.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eux 2 | 3 | WORK_DIR=$(pwd) 4 | TEMP_DIR="${PWD}/tmp" 5 | DEPLOY_ID=$(cat $TEMP_DIR/deploy_id) 6 | 7 | for i in "$@"; do 8 | case $i in 9 | -w=*|--work-directory=*) 10 | WORK_DIR="${i#*=}" 11 | shift 12 | ;; 13 | -i=*|--deploy-id=*) 14 | DEPLOY_ID="${i#*=}" 15 | shift 16 | ;; 17 | *) 18 | echo "unknown option (option: $i)" 19 | exit 1 20 | ;; 21 | esac 22 | done 23 | 24 | terraform destroy --force\ 25 | -var-file=at.tfvars\ 26 | -var "package_repository=datawire/stable"\ 27 | -var "aws_access_key=${AWS_ACCESS_KEY_ID}"\ 28 | -var "aws_secret_key=${AWS_SECRET_ACCESS_KEY}"\ 29 | -var "deploy_id=${DEPLOY_ID}"\ 30 | -var "proton_rpm=${TEMP_DIR}/datawire-proton.rpm"\ 31 | -var "datawire_rpm=${TEMP_DIR}/datawire.rpm"\ 32 | -var "directory_rpm=${TEMP_DIR}/datawire-directory.rpm"\ 33 | -var "watson_rpm=${TEMP_DIR}/datawire-watson.rpm"\ 34 | -var "sherlock_rpm=${TEMP_DIR}/datawire-sherlock.rpm"\ 35 | "${WORK_DIR}" -------------------------------------------------------------------------------- /atinup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eux 2 | 3 | WORK_DIR=$(pwd) 4 | DEPLOY_ID=$(echo "obase=16; $(date +%s%3N)" | bc | tr '[:upper:]' '[:lower:]') 5 | 6 | for i in "$@"; do 7 | case $i in 8 | -w=*|--work-directory=*) 9 | WORK_DIR="${i#*=}" 10 | shift 11 | ;; 12 | -i=*|--deploy-id=*) 13 | DEPLOY_ID="${i#*=}" 14 | shift 15 | ;; 16 | *) 17 | echo "unknown option (option: $i)" 18 | exit 1 19 | ;; 20 | esac 21 | done 22 | 23 | TEMP_DIR="${PWD}/tmp" 24 | mkdir -p "${TEMP_DIR}" 25 | if [ ! -f "${TEMP_DIR}/deploy_id" ]; then 26 | echo "${DEPLOY_ID}" > "${TEMP_DIR}/deploy_id" 27 | else 28 | DEPLOY_ID=$(cat "${TEMP_DIR}/deploy_id") 29 | fi 30 | 31 | # Generate a temporary SSH key pair 32 | TEMP_PRIVATE_KEY_NAME="temporary_key" 33 | TEMP_PUBLIC_KEY_NAME="${TEMP_PRIVATE_KEY_NAME}.pub" 34 | 35 | if [ ! -f "${TEMP_DIR}/${TEMP_PRIVATE_KEY_NAME}" -a ! -f "${TEMP_DIR}/${TEMP_PUBLIC_KEY_NAME}" ]; then 36 | echo "creating temporary SSH public-private key pair (name: ${TEMP_PRIVATE_KEY_NAME})" 37 | ssh-keygen -q -b 2048 -t rsa -f "${TEMP_DIR}/${TEMP_PRIVATE_KEY_NAME}" -N "" 38 | fi 39 | 40 | # Start the provisioning process 41 | terraform apply\ 42 | -var-file=at.tfvars\ 43 | -var "package_repository=datawire/stable"\ 44 | -var "aws_access_key=${AWS_ACCESS_KEY_ID}"\ 45 | -var "aws_secret_key=${AWS_SECRET_ACCESS_KEY}"\ 46 | -var "deploy_id=${DEPLOY_ID}"\ 47 | -var "proton_rpm=${TEMP_DIR}/datawire-proton.rpm"\ 48 | -var "datawire_rpm=${TEMP_DIR}/datawire.rpm"\ 49 | -var "directory_rpm=${TEMP_DIR}/datawire-directory.rpm"\ 50 | -var "watson_rpm=${TEMP_DIR}/datawire-watson.rpm"\ 51 | -var "sherlock_rpm=${TEMP_DIR}/datawire-sherlock.rpm"\ 52 | "${WORK_DIR}" -------------------------------------------------------------------------------- /atrun.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | TEST_RUNNER_PUBLIC_IP=$(terraform output test_runner_public_ip) 4 | TEST_SERVICE_PRIVATE_IP=$(terraform output test_service_private_ip) 5 | 6 | ssh -o StrictHostKeyChecking=no -i tmp/temporary_key centos@${TEST_RUNNER_PUBLIC_IP} "py.test /home/centos/test_bakerstreet.py --wc-host=${TEST_SERVICE_PRIVATE_IP}:5002 --junitxml=/home/centos/test_bakerstreet.xml" 7 | scp -o StrictHostKeyChecking=no -i tmp/temporary_key centos@${TEST_RUNNER_PUBLIC_IP}:/home/centos/test_bakerstreet.xml . 8 | -------------------------------------------------------------------------------- /deploy/datawire-sherlock/centos-7-x64/etc/systemd/system/sherlock.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Datawire Baker Sherlock 3 | ConditionPathExists=/etc/datawire/sherlock.conf 4 | After=network.target 5 | 6 | [Service] 7 | ExecStart=/usr/bin/sherlock -c /etc/datawire/sherlock.conf 8 | Restart=always 9 | RestartPreventExitStatus=78 10 | 11 | [Install] 12 | WantedBy=default.target 13 | -------------------------------------------------------------------------------- /deploy/datawire-sherlock/common/etc/datawire/sherlock.conf: -------------------------------------------------------------------------------- 1 | [Sherlock] 2 | ; The path to the HAProxy executable 3 | proxy: /usr/sbin/haproxy 4 | 5 | ; The directory where HAProxy runs from and reads its configuration. 6 | rundir: /opt/datawire/run 7 | 8 | ; The debounce period in seconds. The debounce period is designed to prevent HAProxy from constantly restarting due 9 | ; to changes. 10 | debounce: 2 11 | dir_debounce: 2 12 | 13 | ; logging level (default in datawire.conf). Valid options are: DEBUG, INFO, WARNING, ERROR, or CRITICAL. 14 | ;logging: WARNING 15 | -------------------------------------------------------------------------------- /deploy/datawire-sherlock/ubuntu-14.04-x64/etc/init/sherlock.conf: -------------------------------------------------------------------------------- 1 | description "Datawire Baker Sherlock" 2 | 3 | start on runlevel [2345] 4 | stop on runlevel [!2345] 5 | 6 | pre-start script 7 | [ -s /etc/datawire/sherlock.conf ] || stop && exit 0 8 | end script 9 | 10 | exec /usr/bin/sherlock -c /etc/datawire/sherlock.conf 11 | respawn 12 | normal exit 0 78 13 | -------------------------------------------------------------------------------- /deploy/datawire-watson/centos-7-x64/etc/systemd/system/watson.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Datawire Baker Watson 3 | ConditionPathExists=/etc/datawire/watson.conf 4 | After=network.target 5 | 6 | [Service] 7 | ExecStart=/usr/bin/watson -c /etc/datawire/watson.conf 8 | Restart=always 9 | RestartPreventExitStatus=78 10 | 11 | [Install] 12 | WantedBy=default.target 13 | -------------------------------------------------------------------------------- /deploy/datawire-watson/common/etc/datawire/watson.conf.proto: -------------------------------------------------------------------------------- 1 | [Watson] 2 | ; The hostname (or IP address) and port number of the service. Optionally a path may be specified by appending it after 3 | ; the host portion of the URI. 4 | ; 5 | ; Examples: http://localhost:9000 or http://localhost:9000/foo 6 | 7 | service_url: http://hostname:port 8 | 9 | ; The name of the service. This must be unique within the Datawire directory. The name must also satisfy the following 10 | ; constraints: 11 | ; 12 | ; * minimum length: 1 character 13 | ; * maximum length: 100 characters 14 | ; * only lower case letters, numerical digits, underscores, and hyphens allowed 15 | ; * must start with a letter or underscore 16 | 17 | service_name: foobar 18 | 19 | ; The service health check URL. The URL must respond to HTTP GET requests. The HTTP status code 200 indicates the service is healthy, any other status code in the response indicates that it is not. 20 | ; 21 | ; Warning: Be careful using property reference syntax to blindly populate health_check_url here (e.g. %(service_url)) because 22 | ; defining an additional path will cause problems. For example, if service_url is http://localhost:9000/foo and 23 | ; you use $(service_url)/health then Watson will health check http://localhost:9000/foo/health which is probably not the intent. 24 | ; 25 | ; Examples: http://localhost:9000/health 26 | 27 | health_check_url: http://hostname:port/health 28 | 29 | ; The number of seconds between health checks. 30 | 31 | period: 3 32 | 33 | ; logging level (default in datawire.conf). Valid options are: DEBUG, INFO, WARNING, ERROR, or CRITICAL. 34 | 35 | ;logging: WARNING 36 | -------------------------------------------------------------------------------- /deploy/datawire-watson/ubuntu-14.04-x64/etc/init/watson.conf: -------------------------------------------------------------------------------- 1 | description "Datawire Baker Watson" 2 | 3 | start on runlevel [2345] 4 | stop on runlevel [!2345] 5 | 6 | pre-start script 7 | [ -s /etc/datawire/watson.conf ] || stop && exit 0 8 | end script 9 | 10 | exec /usr/bin/watson -c /etc/datawire/watson.conf 11 | respawn 12 | normal exit 0 78 13 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | Quickstart 2 | ========== 3 | 4 | You will need sphinx to build the documentation. You can install it via pip: 5 | 6 | pip install sphinx 7 | 8 | The documentation looks better with the following package installed, 9 | however it is not requried: 10 | 11 | pip install sphinx-better-theme 12 | 13 | Once you have sphinx installed, you can build the documentation by 14 | running the following command from this directory: 15 | 16 | sphinx-build source html 17 | 18 | The second argument of the above command is the output directory. In 19 | other words this will generate the documentation into the "html" 20 | directory. If you wish to put it somewhere else you may supply a 21 | different path. Once the build completes you can point your web 22 | browser at the output directory you supplied and inspect the result. 23 | 24 | Details 25 | ======= 26 | 27 | Documentation is written using Sphinx. You must have [Sphinx 28 | installed](http://sphinx-doc.org/latest/install.html) to build the 29 | documentation. 30 | 31 | Example Tags 32 | ============ 33 | 34 | You can put python files in source/examples which can be 35 | referenced as examples from within the documentation/tutorials. 36 | 37 | To make a named tag within an example python file, use a line of the 38 | form `# ` to begin the tag (`qwhere `tag_name` is replaced 39 | with the name of the tag) and a line of the form `# ` to 40 | end the tag. Tags *do not* have to be properly nested. 41 | 42 | Upon build, tags will be preprocessed into new source files in the 43 | source/tags directory. If the example file was called 44 | source/examples/example1.py and contained the tags `hello`, `world`, 45 | and `python`, the files in the tags directory after preprocessing 46 | would be source/tags/example1.py, source/tags/example1.py.hello.py, 47 | source/tags/example1.py.world.py, and 48 | source/tags/example1.py.python.py 49 | 50 | **NOTE**: *Files in source/tags are automatically generated and should 51 | never be editted directly*. 52 | 53 | To show the source of one of the tags files in a documentation file (.rst), use the syntax: 54 | 55 | ``` 56 | .. literalinclude:: ../tags/example1.py.hello.py 57 | ``` 58 | 59 | (with the path replaced appropriately). 60 | 61 | Example: 62 | 63 | source/examples/test.py: 64 | 65 | ```python 66 | # hello 67 | # 68 | # 69 | # A comment 70 | print "Hello world!" 71 | # 72 | # 73 | # Another comment 74 | print "hello world2!" 75 | # 76 | # 77 | ``` 78 | -------------------------------------------------------------------------------- /docs/source/_static/datawire.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-family: "Lato"; 3 | } 4 | 5 | p { 6 | text-align: left; 7 | } -------------------------------------------------------------------------------- /docs/source/_templates/layout.html: -------------------------------------------------------------------------------- 1 | {% extends "!layout.html" %} 2 | {% block extrahead %} 3 | 4 | 5 | {% endblock %} 6 | 7 | {% set css_files = css_files + ['_static/datawire.css'] %} 8 | 9 | {% block footer %} 10 | {{ super() }} 11 | 12 | 22 | {% endblock %} 23 | -------------------------------------------------------------------------------- /docs/source/architecture.rst: -------------------------------------------------------------------------------- 1 | .. _architecture: 2 | 3 | Design and Architecture 4 | ======================= 5 | 6 | Baker is patterned after `AirBnb's SmartStack 7 | `_ which 8 | is an excellent piece of software and design. The aforereferenced 9 | blog post gives a terrific overview of the different approaches to 10 | service discovery and routing, which we generally agree with (hence, 11 | our adoption of the overall approach). 12 | 13 | Baker does make several different design decisions than SmartStack. 14 | 15 | #. Baker isolates its use of HAProxy from the user. We did this 16 | because HAProxy is commonly deployed in architectures where it is 17 | both a bottleneck and a single point of failure and that usage 18 | significantly influences its design and engineering with respect to 19 | robustness and performance. 20 | 21 | In this architecture HAProxy is paired with each service client and 22 | therefore neither a single point of failure nor a bottleneck. While 23 | HAProxy is an excellent conservative choice to start with, we 24 | expect it may ultimately make sense to replace HAProxy with a 25 | component that makes a different set of tradeoffs that better fit 26 | its role. 27 | 28 | For example HAProxy supports HTTP and TCP, but does not natively 29 | support other protocols. In particular, HAProxy does not support 30 | any async messaging protocols, which are important for many use 31 | cases in microservices. Also, HAProxy's load balancing algorithms 32 | tend to assume that it is the only thing dispatching load to a 33 | given backend, however that is not the case in this architecture. 34 | 35 | #. Baker uses the Datawire directory service instead of Zookeeper. 36 | Zookeeper provides a strongly consistent model; the directory 37 | service focuses on availability. This also simplifies Baker 38 | deployment. 39 | 40 | Deployment Model 41 | ---------------- 42 | 43 | A Baker deployment involves three different types of nodes. A 44 | directory node, a service node, and a client node. Although a node can 45 | take on all three roles and in practice many nodes will be both 46 | service and client nodes, it is conceptually easier to consider the 47 | three distinct node types individually. 48 | 49 | Directory Node 50 | ~~~~~~~~~~~~~~ 51 | 52 | The Directory node functions as a rendezvous point between services and 53 | clients. Each service node broadcasts its presence and location to the 54 | directory, and the directory notifies any interested client nodes as 55 | service nodes come and go. 56 | 57 | There is exactly one Directory node per Baker deployment. Every 58 | service and client must be configured with the location of this node. 59 | We expect to support multiple Directory nodes in future versions of 60 | Baker. 61 | 62 | Service Nodes 63 | ~~~~~~~~~~~~~ 64 | 65 | A service node is responsible for keeping the directory accurately 66 | informed of its presence and location. Baker relies on the ``watson`` 67 | process to check the health of the service and communicate its 68 | presence and location to the directory. ``watson`` should always be 69 | deployed on the same server/container as the service it is 70 | monitoring. This deployment model minimizes issues related to node 71 | failure or network partitions, since ``watson`` can monitor the host 72 | service directly over localhost. 73 | 74 | Client Nodes 75 | ~~~~~~~~~~~~ 76 | 77 | A client node maintains a table of the location of all relevant 78 | service nodes and updates this whenever the directory notifies the 79 | client of changes in the state of services. Baker relies on the 80 | ``sherlock`` process to gather information from the directory about 81 | all available services and dynamically proxy connections from its 82 | co-located client processes accordingly. ``sherlock`` should always be 83 | deployed on the same server/container as the client processes it is 84 | supporting. 85 | 86 | Discovery Protocol 87 | ------------------ 88 | 89 | Service Nodes 90 | ~~~~~~~~~~~~~ 91 | 92 | A service node contains the following state: 93 | 94 | * The address of the directory. 95 | * The virtual address of the service. 96 | * The physical address of the service. 97 | * A heartbeat interval. 98 | 99 | A service node is responsible for initiating contact with the 100 | directory and informing the directory of its presence at least once 101 | per heartbeat interval. If the directory node is unavailable for any 102 | reason, the service node will continue to retry indefinitely. 103 | 104 | Client Nodes 105 | ~~~~~~~~~~~~ 106 | 107 | A client node contains the following state: 108 | 109 | * The address of the directory. 110 | * An initially empty table of routes mapping virtual to physical 111 | addresses. 112 | 113 | A client node is responsible for initiating contact with the directory 114 | and registering its interest in receiving updates. If the directory 115 | node is unavailable for any reason, the client node will continue to 116 | retry indefinitely. 117 | 118 | Directory Node 119 | ~~~~~~~~~~~~~~ 120 | 121 | The directory node contains the following state: 122 | 123 | * An initially empty table of routes mapping virtual to physical 124 | addresses. 125 | * An initially empty list of client nodes interested in receiving 126 | updates. 127 | 128 | The directory node is responsible for adding, updating, and removing 129 | entries to/from the table of routes based on the presence/absence of 130 | heartbeats from service nodes. The directory will also update 131 | interested clients when an entry is added, updated, or removed. When a 132 | client initially registers interest in receiving updates, the new 133 | client will be caught up by receiving the full table of routes prior 134 | to being informed of any subsequent updates. 135 | 136 | Startup Order and Node Failures 137 | ------------------------------- 138 | 139 | Several key properties of the discovery protocol allow it to be 140 | resilient to both arbitrary startup order and node failures. 141 | 142 | #. The service and client nodes are guaranteed to retry if the 143 | directory node is not currently available. 144 | 145 | #. The directory contains no state that is not dynamically constructed 146 | from the set of active service and client nodes. 147 | 148 | #. The directory catches up client nodes as they join. 149 | 150 | #. The client nodes replicate the routes published by the directory. 151 | 152 | Given the above, any configuration of nodes is guaranteed to reach the 153 | same (or equivalent) end state regardless of startup order. 154 | Additionally, given the final property, even if the directory node 155 | fails, all existing service and client nodes will continue to function 156 | normally. The only impact on the overall system will be the inability 157 | to dynamically add new service or client nodes. 158 | 159 | These properties make for an extremely resilient design even with a 160 | single directory node. Given that adding and removing service and 161 | client nodes is not a high volume operation, a deployment can easily 162 | tolerate temporary failures of the directory node, as well as short 163 | periods of downtime if needed for maintenance. 164 | 165 | Network Partitions 166 | ------------------ 167 | 168 | In the event of a network partition, all client nodes reachable from 169 | the directory will be dynamically updated to use only reachable 170 | service nodes. Client nodes that are not reachable from the directory 171 | will continue to attempt to access all service nodes that were 172 | available prior to the network partition. 173 | 174 | Future Work 175 | ----------- 176 | 177 | We expect to extend the system in a future release to support multiple 178 | directory nodes. This will provide the following benefits: 179 | 180 | #. Scalability and availability of directory services for deployments 181 | where adding and removing service and/or client nodes *is* expected 182 | to be a high volume operation. 183 | 184 | #. The ability to provide better introspection for nodes that are not 185 | reachable from the directory in the event of a network partition. 186 | -------------------------------------------------------------------------------- /docs/source/bakerstreet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datawire/bakerstreet/0be67aa67d6cc82d63b644a99ce13460e01e040c/docs/source/bakerstreet.png -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Baker documentation build configuration file, created by 4 | # sphinx-quickstart on Fri Jan 30 10:39:21 2015. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | import sys 16 | import os 17 | 18 | try: 19 | from better import better_theme_path 20 | better_theme = True 21 | except ImportError: 22 | sys.stderr.write("Could not import bootstrap theme. Is it installed?%s" % os.linesep) 23 | better_theme = False 24 | 25 | # If extensions (or modules to document with autodoc) are in another directory, 26 | # add these directories to sys.path here. If the directory is relative to the 27 | # documentation root, use os.path.abspath to make it absolute, like shown here. 28 | #sys.path.insert(0, os.path.abspath('.')) 29 | 30 | sys.path.insert(0, os.path.abspath('.')) 31 | import parameters 32 | rst_epilog = ".. |Repository| replace:: Datawire repository on PackageCloud" 33 | rst_epilog += "\n.. _Repository: %s\n" % parameters.install 34 | rst_epilog += "\n".join([".. |%s| replace:: %s" % (name, getattr(parameters, name)) 35 | for name in dir(parameters) if name[0] != "_" and name != "version"]) 36 | 37 | # -- General configuration ------------------------------------------------ 38 | 39 | # If your documentation needs a minimal Sphinx version, state it here. 40 | #needs_sphinx = '1.0' 41 | 42 | # Add any Sphinx extension module names here, as strings. They can be 43 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 44 | # ones. 45 | extensions = [ 46 | 'sphinx.ext.autodoc', 47 | 'sphinx.ext.doctest', 48 | 'sphinx.ext.todo', 49 | 'sphinx.ext.coverage', 50 | 'sphinx.ext.ifconfig', 51 | ] 52 | 53 | # Add any paths that contain templates here, relative to this directory. 54 | templates_path = ['_templates'] 55 | 56 | # The suffix of source filenames. 57 | source_suffix = '.rst' 58 | 59 | # The encoding of source files. 60 | #source_encoding = 'utf-8-sig' 61 | 62 | # The master toctree document. 63 | master_doc = 'index' 64 | 65 | # General information about the project. 66 | project = u'Baker' 67 | copyright = u'2015,2016 Datawire' 68 | 69 | # The version info for the project you're documenting, acts as replacement for 70 | # |version| and |release|, also used in various other places throughout the 71 | # built documents. 72 | # 73 | # The short X.Y version. 74 | version = parameters.version 75 | # The full version, including alpha/beta/rc tags. 76 | release = parameters.version 77 | 78 | # The language for content autogenerated by Sphinx. Refer to documentation 79 | # for a list of supported languages. 80 | #language = None 81 | 82 | # There are two options for replacing |today|: either, you set today to some 83 | # non-false value, then it is used: 84 | #today = '' 85 | # Else, today_fmt is used as the format for a strftime call. 86 | #today_fmt = '%B %d, %Y' 87 | 88 | # List of patterns, relative to source directory, that match files and 89 | # directories to ignore when looking for source files. 90 | exclude_patterns = [] 91 | 92 | # The reST default role (used for this markup: `text`) to use for all 93 | # documents. 94 | #default_role = None 95 | 96 | # If true, '()' will be appended to :func: etc. cross-reference text. 97 | #add_function_parentheses = True 98 | 99 | # If true, the current module name will be prepended to all description 100 | # unit titles (such as .. function::). 101 | #add_module_names = True 102 | 103 | # If true, sectionauthor and moduleauthor directives will be shown in the 104 | # output. They are ignored by default. 105 | #show_authors = False 106 | 107 | # The name of the Pygments (syntax highlighting) style to use. 108 | pygments_style = 'sphinx' 109 | 110 | # A list of ignored prefixes for module index sorting. 111 | #modindex_common_prefix = [] 112 | 113 | # If true, keep warnings as "system message" paragraphs in the built documents. 114 | #keep_warnings = False 115 | 116 | 117 | # -- Options for HTML output ---------------------------------------------- 118 | 119 | # The theme to use for HTML and HTML Help pages. See the documentation for 120 | # a list of builtin themes. 121 | if better_theme: 122 | html_theme_path = [better_theme_path] 123 | html_theme = 'better' 124 | html_theme_options = { 125 | 'linktotheme': False, 126 | } 127 | else: 128 | html_theme = 'haiku' 129 | 130 | # Theme options are theme-specific and customize the look and feel of a theme 131 | # further. For a list of options available for each theme, see the 132 | # documentation. 133 | #html_theme_options = {} 134 | 135 | # Add any paths that contain custom themes here, relative to this directory. 136 | #html_theme_path = [] 137 | 138 | # The name for this set of Sphinx documents. If None, it defaults to 139 | # " v documentation". 140 | #html_title = None 141 | 142 | # A shorter title for the navigation bar. Default is the same as html_title. 143 | #html_short_title = None 144 | 145 | # The name of an image file (relative to this directory) to place at the top 146 | # of the sidebar. 147 | #html_logo = None 148 | 149 | # The name of an image file (within the static path) to use as favicon of the 150 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 151 | # pixels large. 152 | #html_favicon = None 153 | 154 | # Add any paths that contain custom static files (such as style sheets) here, 155 | # relative to this directory. They are copied after the builtin static files, 156 | # so a file named "default.css" will overwrite the builtin "default.css". 157 | html_static_path = ['_static'] 158 | 159 | # Add any extra paths that contain custom files (such as robots.txt or 160 | # .htaccess) here, relative to this directory. These files are copied 161 | # directly to the root of the documentation. 162 | #html_extra_path = [] 163 | 164 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 165 | # using the given strftime format. 166 | #html_last_updated_fmt = '%b %d, %Y' 167 | 168 | # If true, SmartyPants will be used to convert quotes and dashes to 169 | # typographically correct entities. 170 | #html_use_smartypants = True 171 | 172 | # Custom sidebar templates, maps document names to template names. 173 | html_sidebars = { 174 | '**': ['localtoc.html', 'sourcelink.html', 'searchbox.html'], 175 | } 176 | 177 | # Additional templates that should be rendered to pages, maps page names to 178 | # template names. 179 | #html_additional_pages = {} 180 | 181 | # If false, no module index is generated. 182 | #html_domain_indices = True 183 | 184 | # If false, no index is generated. 185 | #html_use_index = True 186 | 187 | # If true, the index is split into individual pages for each letter. 188 | #html_split_index = False 189 | 190 | # If true, links to the reST sources are added to the pages. 191 | html_show_sourcelink = False 192 | 193 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 194 | html_show_sphinx = False 195 | 196 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 197 | #html_show_copyright = True 198 | 199 | # If true, an OpenSearch description file will be output, and all pages will 200 | # contain a tag referring to it. The value of this option must be the 201 | # base URL from which the finished HTML is served. 202 | #html_use_opensearch = '' 203 | 204 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 205 | #html_file_suffix = None 206 | 207 | # Output file base name for HTML help builder. 208 | htmlhelp_basename = 'Bakerdoc' 209 | 210 | 211 | # -- Options for LaTeX output --------------------------------------------- 212 | 213 | latex_elements = { 214 | # The paper size ('letterpaper' or 'a4paper'). 215 | #'papersize': 'letterpaper', 216 | 217 | # The font size ('10pt', '11pt' or '12pt'). 218 | #'pointsize': '10pt', 219 | 220 | # Additional stuff for the LaTeX preamble. 221 | #'preamble': '', 222 | } 223 | 224 | # Grouping the document tree into LaTeX files. List of tuples 225 | # (source start file, target name, title, 226 | # author, documentclass [howto, manual, or own class]). 227 | latex_documents = [ 228 | ('index', 'Baker.tex', u'Baker Street Documentation', 229 | u'Baker', 'manual'), 230 | ] 231 | 232 | # The name of an image file (relative to this directory) to place at the top of 233 | # the title page. 234 | #latex_logo = None 235 | 236 | # For "manual" documents, if this is true, then toplevel headings are parts, 237 | # not chapters. 238 | #latex_use_parts = False 239 | 240 | # If true, show page references after internal links. 241 | #latex_show_pagerefs = False 242 | 243 | # If true, show URL addresses after external links. 244 | #latex_show_urls = False 245 | 246 | # Documents to append as an appendix to all manuals. 247 | #latex_appendices = [] 248 | 249 | # If false, no module index is generated. 250 | #latex_domain_indices = True 251 | 252 | 253 | # -- Options for manual page output --------------------------------------- 254 | 255 | # One entry per manual page. List of tuples 256 | # (source start file, name, description, authors, manual section). 257 | man_pages = [ 258 | ('index', 'baker', u'Baker Street Documentation', 259 | [u'Baker'], 1) 260 | ] 261 | 262 | # If true, show URL addresses after external links. 263 | #man_show_urls = False 264 | 265 | 266 | # -- Options for Texinfo output ------------------------------------------- 267 | 268 | # Grouping the document tree into Texinfo files. List of tuples 269 | # (source start file, target name, title, author, 270 | # dir menu entry, description, category) 271 | texinfo_documents = [ 272 | ('index', 'Baker', u'Baker Street Documentation', 273 | u'Baker', 'Baker', 'One line description of project.', 274 | 'Miscellaneous'), 275 | ] 276 | 277 | # Documents to append as an appendix to all manuals. 278 | #texinfo_appendices = [] 279 | 280 | # If false, no module index is generated. 281 | #texinfo_domain_indices = True 282 | 283 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 284 | #texinfo_show_urls = 'footnote' 285 | 286 | # If true, do not generate a @detailmenu in the "Top" node's menu. 287 | #texinfo_no_detailmenu = False 288 | -------------------------------------------------------------------------------- /docs/source/deployment.rst: -------------------------------------------------------------------------------- 1 | .. _deployment: 2 | 3 | Deployment 4 | ========== 5 | 6 | The following instructions explain how to set up and utilize Baker in 7 | a production environment. The instructions below assume that you have 8 | set up the |Repository|_ on all relevant machines/VMs. 9 | 10 | On Enterprise Linux 7 11 | 12 | .. parsed-literal:: 13 | 14 | $ curl -s |script_rpm| | sudo bash 15 | 16 | On Ubuntu 14.04 LTS 17 | 18 | .. parsed-literal:: 19 | 20 | $ curl -s |script_deb| | sudo bash 21 | 22 | 23 | 24 | For simplicity, the examples assume a set of named servers, VMs, or 25 | containers set up under the domain ``example.com``. Some pieces of the 26 | system may want to run on stable resources; these are presented as 27 | named machines, e.g., ``database.example.com``. Other pieces will 28 | likely be deployed, upgraded, removed, etc. on an ongoing basis; these 29 | would run on elastically-deployed resources named something like 30 | ``vm123.example.com``. 31 | 32 | The example services themselves use HTTP to communicate, typically 33 | running in an application server like Tomcat. An example service 34 | called ``greeting`` running on the host ``vm678.example.com`` could be 35 | accessed at the URL ``http://vm678.example.com/greeting``. Any HTTP 36 | client would suffice. The command line examples will use ``curl`` but 37 | each of the following are roughly equivalent:: 38 | 39 | curl http://vm678.example.com/greeting 40 | lynx -dump http://vm678.example.com/greeting 41 | w3m -dump http://vm678.example.com/greeting 42 | wget -O - http://vm678.example.com/greeting 43 | 44 | Directory 45 | --------- 46 | 47 | The Datawire Directory is at the core of Baker. It should run on a 48 | stable, reliable system that experiences relatively few interruptions. 49 | In practice, the Directory is able to recover from server restarts 50 | quickly and efficiently. The other components in Baker are designed to 51 | handle a brief interruption of Directory availability without any 52 | trouble. (For more details on this see the Baker :ref:`architecture`.) 53 | 54 | The hostname of the Directory will appear in service URLs and so it 55 | must have a well known stable hostname that is accessible to all 56 | client and service nodes participating in your deployment. 57 | 58 | Install the directory using the package tool appropriate for your 59 | system:: 60 | 61 | $ ssh directory.example.com 62 | 63 | directory $ sudo yum install datawire-directory 64 | (or) 65 | directory $ sudo apt-get install datawire-directory 66 | 67 | Configure the well known and stable hostname of the directory as 68 | follows:: 69 | 70 | directory $ cd /etc/datawire 71 | directory $ sudo nano datawire.conf 72 | directory $ cat datawire.conf 73 | [DEFAULT] 74 | ; logging level may be DEBUG, INFO, WARNING, ERROR, or CRITICAL 75 | logging: WARNING 76 | 77 | [Datawire] 78 | ; Change this to the well known and stable hostname of the directory for your deployment. 79 | directory_host: directory.example.com 80 | 81 | Once the directory hostname is configured, use the appropriate tool to 82 | start (or restart) the directory. On Enterprise Linux 7:: 83 | 84 | directory $ sudo systemctl start directory.service 85 | 86 | On Ubuntu 14.04 LTS:: 87 | 88 | directory $ sudo service directory start 89 | 90 | Sherlock 91 | -------- 92 | 93 | Software that needs to use a service will use Sherlock to find and 94 | access an instance of that service transparently. Such software might 95 | be as simple as a command line HTTP tool like ``curl``, or it might be 96 | a large, complicated system that needs access to dozens of services to 97 | perform the core operations of a business. 98 | 99 | Installing Sherlock:: 100 | 101 | $ ssh vm123.example.com 102 | 103 | vm123 $ sudo yum install datawire-sherlock 104 | (or) 105 | vm123 $ sudo apt-get install datawire-sherlock 106 | 107 | Once sherlock is installed, edit the datawire.conf to reference the 108 | well known directory set up in the previous section:: 109 | 110 | vm123 $ cd /etc/datawire 111 | vm123 $ sudo nano datawire.conf 112 | vm123 $ cat datawire.conf 113 | [DEFAULT] 114 | ; logging level may be DEBUG, INFO, WARNING, ERROR, or CRITICAL 115 | logging: WARNING 116 | 117 | [Datawire] 118 | ; Change this to the well known and stable hostname of the directory for your deployment. 119 | directory_host: directory.example.com 120 | 121 | You can also tweak the sherlock preferences in sherlock.conf, however 122 | the defaults will generally work well:: 123 | 124 | vm123 $ cd /etc/datawire 125 | vm123 $ sudo nano sherlock.conf 126 | vm123 $ cat sherlock.conf 127 | [Sherlock] 128 | proxy: /usr/sbin/haproxy 129 | rundir: /opt/datawire/run 130 | debounce: 2 ; seconds 131 | dir_debounce: 2 ; seconds 132 | ; logging level (default in datawire.conf) may be DEBUG, INFO, WARNING, ERROR, or CRITICAL 133 | ;logging: WARNING 134 | 135 | Now any process on your VM can access services by name without needing 136 | to know where instances of the service are running:: 137 | 138 | vm123 $ curl http://localhost:8000/ 139 | 140 | By going through HAProxy, each live instance of a service is accessed 141 | in round-robin fashion. If an instance drops out, e.g., for 142 | maintenance, Watson notifies the directory, which allows Sherlock to 143 | update the HAProxy configuration and keep requests flowing through the 144 | remaining instances. When that instance comes back, Sherlock again 145 | makes the appropriate adjustments to HAProxy. New instances get added 146 | to the pool automatically in much the same way. 147 | 148 | Watson 149 | ------ 150 | 151 | Service instances that want to be dynamically accessible will use 152 | watson to advertise their presence to the datawire directory. Watson 153 | must run co-located on the same machine/VM as the service instance. 154 | Watson will periodically check the health of the service instance and 155 | register its location and status with the directory. 156 | 157 | Installing Watson:: 158 | 159 | $ ssh vm101.example.com 160 | 161 | vm101 $ sudo yum install datawire-watson 162 | (or) 163 | vm101 $ sudo apt-get install datawire-watson 164 | 165 | Once Watson is installed, edit the datawire.conf to reference the well 166 | known directory set up in the first section:: 167 | 168 | vm101 $ cd /etc/datawire 169 | vm101 $ sudo nano datawire.conf 170 | vm101 $ cat datawire.conf 171 | [DEFAULT] 172 | ; logging level may be DEBUG, INFO, WARNING, ERROR, or CRITICAL 173 | logging: WARNING 174 | 175 | [Datawire] 176 | ; Change this to the well known and stable hostname of the directory for your deployment. 177 | directory_host: directory.example.com 178 | 179 | Now copy the example watson configuration found in 180 | /etc/datawire/watson.conf.proto and configure it for your service: 181 | 182 | #. Provide the base url for your service. 183 | #. Provide the url for health checks on your service. 184 | 185 | :: 186 | 187 | vm101 $ cd /etc/datawire 188 | vm101 $ sudo cp watson.conf.proto watson.conf 189 | vm101 $ sudo nano watson.conf 190 | vm101 $ cat watson.conf 191 | [Watson] 192 | ; service_name must uniquely identify your service 193 | service_url: http://vm101.example.com:8080/example-service 194 | liveness_url: %(service_url)s/liveness_check 195 | period: 3 ; seconds between liveness checks 196 | ; logging level (default in datawire.conf) may be DEBUG, INFO, WARNING, ERROR, or CRITICAL 197 | ;logging: WARNING 198 | 199 | More Services 200 | ------------- 201 | 202 | As your system grows in complexity, your network of microservices will 203 | grow as well. Some services will only offer access to a resource but 204 | not utilize other services in the system. However, many services will 205 | benefit from using other services too. It is common to end up with a 206 | network of communicating services. Baker makes it easy for 207 | microservices to communicate with each other, and other Datawire 208 | components help to organize, manage, and understand the complicated 209 | topologies that may arise. 210 | -------------------------------------------------------------------------------- /docs/source/extract_tags.py: -------------------------------------------------------------------------------- 1 | #!/bin/env python 2 | 3 | import sys 4 | import os 5 | import re 6 | 7 | EXAMPLE_DIR = "examples" 8 | TAG_DIR = "tags" 9 | 10 | FILE_PATH = os.path.dirname(os.path.realpath(__file__)) 11 | 12 | def clear_folder(folder): 13 | # Subroutine derived from http://stackoverflow.com/questions/185936/delete-folder-contents-in-python 14 | for the_file in os.listdir(folder): 15 | file_path = os.path.join(folder, the_file) 16 | try: 17 | if os.path.isfile(file_path): 18 | os.unlink(file_path) 19 | except Exception: 20 | sys.stderr.write("Could not delete file %s.%s" % (file_path, os.linesep)) 21 | 22 | tag_pattern = re.compile(r"^\s*#+\s*<(/?)([^/>][^>]*)>\s*$") 23 | 24 | class TagMatch: 25 | is_match = False 26 | is_start = False 27 | tag = None 28 | def __init__(self, line): 29 | match = tag_pattern.search(line) 30 | if not match: 31 | return 32 | self.is_match = True 33 | self.is_start = len(match.group(1)) == 0 34 | self.tag = match.group(2) 35 | 36 | class Tag: 37 | def __init__(self, name, start): 38 | self.name = name 39 | self.start = start 40 | self.end = None 41 | self.line_gen = () 42 | 43 | def ended(self, ended_at): 44 | if self.end is not None: 45 | raise ValueError("Tag ended for second time at %i" % ended_at) 46 | self.end = ended_at 47 | 48 | def get_tags(read_file): 49 | f = open(read_file, 'r') 50 | non_tag_lines = [] 51 | tags = {} 52 | for line in f: 53 | tag_match = TagMatch(line) 54 | if tag_match.is_match: 55 | # Process tag 56 | if tag_match.is_start: 57 | if tag_match.tag in tags: 58 | raise ValueError("Duplicate tag %s" % tag_match.tag) 59 | tag = Tag(name=tag_match.tag, start=len(non_tag_lines)) 60 | tags[tag.name] = tag 61 | else: 62 | if tag_match.tag not in tags: 63 | raise ValueError("End tag before start tag for %s" % tag_match.tag) 64 | tag = tags[tag_match.tag] 65 | tag.ended(len(non_tag_lines)) 66 | tag.line_gen = (non_tag_lines[x] for x in xrange(tag.start, tag.end)) 67 | else: 68 | non_tag_lines.append(line) 69 | f.close() 70 | return (tags, non_tag_lines) 71 | 72 | def process_tags(example_file_name): 73 | read_path = os.path.join(FILE_PATH, EXAMPLE_DIR, example_file_name) 74 | tags, non_tag_lines = get_tags(read_path) 75 | for tag_name in tags: 76 | tag = tags[tag_name] 77 | if not tag.end: 78 | raise ValueError("Tag started but not ended: %s." % tag) 79 | filename = "%s.%s.py" % (example_file_name, tag_name) 80 | write_path = os.path.join(FILE_PATH, TAG_DIR, filename) 81 | f = open(write_path, 'w') 82 | for line in tag.line_gen: 83 | f.write(line) 84 | f.close() 85 | write_path = os.path.join(FILE_PATH, TAG_DIR, example_file_name) 86 | f = open(write_path, 'w') 87 | for line in non_tag_lines: 88 | f.write(line) 89 | f.close() 90 | 91 | full_tags_dir = os.path.join(FILE_PATH, TAG_DIR) 92 | if not os.path.exists(full_tags_dir): 93 | os.makedirs(full_tags_dir) 94 | clear_folder(full_tags_dir) 95 | example_files = os.listdir(os.path.join(FILE_PATH, EXAMPLE_DIR)) 96 | for example_file in example_files: 97 | process_tags(example_file) 98 | 99 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | Baker Street 2 | ============ 3 | 4 | .. Baker documentation master file, created by 5 | sphinx-quickstart on Tue Jan 27 12:04:31 2015. 6 | You can adapt this file completely to your liking, but it should at least 7 | contain the root `toctree` directive. 8 | 9 | .. image:: bakerstreet.png 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | overview 15 | quickstart 16 | deployment 17 | architecture 18 | reference 19 | -------------------------------------------------------------------------------- /docs/source/overview.rst: -------------------------------------------------------------------------------- 1 | Baker Street 2 | ============ 3 | 4 | Baker Street (hereafter shortened as Baker) is a service discovery and 5 | routing system designed for microservice architectures. Baker 6 | simplifies scaling, testing, and upgrading microservices. 7 | 8 | Why use Baker? 9 | -------------- 10 | 11 | Suppose you have two microservices, A and B, that communicate with 12 | each other. For scalability and availability, both microservices are 13 | deployed on multiple servers (or containers). 14 | 15 | Baker solves several problems: 16 | 17 | #. Service discovery. Requests from A to B automatically locate an 18 | available instance of the B microservice. 19 | 20 | #. Load balancing. Requests from A to B are automatically distributed 21 | between all available instances of B, without requiring a central 22 | load balancer in front of B. 23 | 24 | #. Upgrades and testing. New versions of a microservice can be 25 | easily introduced by adding a new server instance to the service 26 | pool. Load will be distributed to the new version, and old 27 | instances can be turned off as new ones are introduced to the pool. 28 | 29 | 30 | Deploying Baker 31 | --------------- 32 | 33 | Baker runs on any modern flavor of Linux. Baker works with any 34 | application or microservice, regardless of programming 35 | language. Usually, you just need a small configuration change in your 36 | microservice to start using Baker. 37 | 38 | How Baker Works 39 | --------------- 40 | 41 | Baker consists of three main components: 42 | 43 | * Sherlock, which is responsible for routing a microservice's 44 | connections to the right destination 45 | * Watson, which provides real-time liveness detection of a given 46 | microservice 47 | * the Datawire Directory, which keeps track of all microservices and 48 | their associated routes 49 | 50 | Sherlock and Watson are deployed on every microservice container or 51 | server. Each microservice is then configured to proxy its connections 52 | through Sherlock (behind the scenes, we use the super-reliable, 53 | super-fast HAProxy). Sherlock uses information from the Directory to 54 | route connections to the appropriate destination. 55 | -------------------------------------------------------------------------------- /docs/source/parameters.py: -------------------------------------------------------------------------------- 1 | 2 | def _getvar(var, path, default=None): 3 | with open(path) as f: 4 | for line in f: 5 | if var in line and "=" in line and "__all__" not in line: 6 | g = {} 7 | l = {} 8 | exec line in g, l 9 | return l[var] 10 | return default 11 | 12 | def _version(): 13 | import os 14 | return _getvar("__version__", os.path.join(os.path.dirname(__file__), 15 | "../../_metadata_watson.py"), 16 | "X.X") 17 | 18 | def _repo(): 19 | return "stable" 20 | 21 | version = _version() 22 | repo = _repo() 23 | install = "https://packagecloud.io/datawire/%s/install" % repo 24 | script_rpm = "https://packagecloud.io/install/repositories/datawire/%s/script.rpm.sh" % repo 25 | script_deb = "https://packagecloud.io/install/repositories/datawire/%s/script.deb.sh" % repo 26 | -------------------------------------------------------------------------------- /docs/source/quickstart.rst: -------------------------------------------------------------------------------- 1 | .. _quick_start: 2 | 3 | Quick Start 4 | =========== 5 | 6 | Ready to dive right in? This page will take you through deploying Baker in a minimal environment and then using it for load balancing and upgrading a simple service. 7 | 8 | You will need an Enterprise Linux 7 (RHEL 7, CentOS 7, etc.) or Ubuntu 14.04 LTS machine, JDK 1.8 and Maven 3+ for the simple service, and about 15 minutes of your time. Note that Baker itself does not require Java or Maven; they are only needed if you wish to run the example outlined on this page. 9 | 10 | Setup 11 | ----- 12 | 13 | The sample deployment shown here will run everything on ``localhost``. The :ref:`deployment` section of this manual covers how to set things up in a production environment. 14 | 15 | The simple service in this example is the Greeting service from `Spring's RESTful Web Service Guide `_. Set up that service:: 16 | 17 | $ curl https://github.com/spring-guides/gs-rest-service/archive/master.zip -LO 18 | $ unzip -q master.zip 19 | $ cd gs-rest-service-master/complete/ 20 | $ mvn -q package 21 | $ env SERVER_PORT=9001 java -jar target/gs-rest-service-0.1.0.jar > /dev/null 2>&1 & 22 | 23 | **Note**: Running Maven without the ``-q`` option generates a lot of output, which may be helpful if there are any problems with the build process. 24 | 25 | Verify access to the Greeting service using a web browser or a command line tool like ``curl``:: 26 | 27 | $ curl http://localhost:9001/greeting 28 | {"id":1,"content":"Hello, World!"} 29 | 30 | The ``id`` field in the output is a counter. If you hit the service repeatedly, that field will increment. Once set up, Watson will monitor this service by accessing it periodically, causing the counter to increment in the background. 31 | 32 | Install 33 | ------- 34 | 35 | On Enterprise Linux 7, add access to the |Repository|_ and use ``yum`` to perform the installation 36 | 37 | .. parsed-literal:: 38 | 39 | $ curl -s |script_rpm| | sudo bash 40 | [...] 41 | $ sudo yum install datawire-directory datawire-sherlock datawire-watson 42 | 43 | On Ubuntu 14.04 LTS, add access to the |Repository|_ and use ``apt-get`` to perform the installation 44 | 45 | .. parsed-literal:: 46 | 47 | $ curl -s |script_deb| | sudo bash 48 | [...] 49 | $ sudo apt-get install datawire-directory datawire-sherlock datawire-watson 50 | 51 | Configure 52 | --------- 53 | 54 | Baker looks for its configuration files in ``/etc/datawire``. A sample 55 | configuration file for Watson is installed there. Copy it and edit it 56 | to reference your service:: 57 | 58 | $ cd /etc/datawire 59 | $ sudo cp watson.conf.proto watson.conf 60 | $ sudo nano watson.conf 61 | [...] 62 | 63 | $ cat watson.conf 64 | [Watson] 65 | ; The hostname (or IP address) and port number of the service. Optionally a path may be specified by appending it after 66 | ; the host portion of the URI. 67 | ; 68 | ; Examples: http://localhost:9000 or http://localhost:9000/foo 69 | 70 | service_url: http://localhost:9001/greeting 71 | 72 | ; The name of the service. This must be unique within the Datawire directory. The name must also satisfy the following 73 | ; constraints: 74 | ; 75 | ; * minimum length: 1 character 76 | ; * maximum length: 100 characters 77 | ; * only lower case letters, numerical digits, underscores, and hyphens allowed 78 | ; * must start with a letter or underscore 79 | 80 | service_name: greeting 81 | 82 | ; The service health check URL. The URL must respond to HTTP GET requests. The HTTP status code 200 indicates the service is healthy, any other status code in the response indicates that it is not. 83 | ; 84 | ; Warning: Be careful using property reference syntax to blindly populate health_check_url here (e.g. %(service_url)) because 85 | ; defining an additional path will cause problems. For example, if service_url is http://localhost:9000/foo and 86 | ; you use $(service_url)/health then Watson will health check http://localhost:9000/foo/health which is probably not the intent. 87 | ; 88 | ; Examples: http://localhost:9000/health 89 | 90 | health_check_url: http://localhost:9001/greeting 91 | 92 | ; The number of seconds between health checks. 93 | 94 | period: 3 95 | 96 | ; logging level (default in datawire.conf). Valid options are: DEBUG, INFO, WARNING, ERROR, or CRITICAL. 97 | 98 | ;logging: WARNING 99 | 100 | Launch 101 | ------ 102 | 103 | Once you have configured Watson, launching the Baker components is easy using your operating system's standard controls. On Enterprise Linux 7:: 104 | 105 | $ sudo systemctl start directory.service 106 | $ sudo systemctl start sherlock.service 107 | $ sudo systemctl start watson.service 108 | 109 | On Ubuntu 14.04 LTS:: 110 | 111 | $ sudo service directory start 112 | $ sudo service sherlock start 113 | $ sudo service watson start 114 | 115 | Access your service through Baker to verify things are working okay:: 116 | 117 | $ curl http://localhost:8000/greeting 118 | {"id":17,"content":"Hello, World!"} 119 | 120 | Watson notifies the Directory that the Greeting microservice on ``http://localhost:9001/`` is running. Sherlock sets up HAProxy to route ``greeting`` requests to that microservice. Your ``curl`` above gets proxied to the right place. Note that your ``id`` field will likely be a different value, depending on how long Watson has run and how many times you accessed the service manually. 121 | 122 | Load Balancing 123 | -------------- 124 | 125 | Let's add more Greeting microservice instances for load balancing:: 126 | 127 | $ cd /path/to/gs-rest-service-master/complete/ 128 | $ env SERVER_PORT=9002 java -jar target/gs-rest-service-0.1.0.jar > /dev/null 2>&1 & 129 | $ env SERVER_PORT=9003 java -jar target/gs-rest-service-0.1.0.jar > /dev/null 2>&1 & 130 | 131 | We will need to add a Watson instance for each one. Normally, you would run one microservice per server, VM, or container; see the :ref:`deployment` section for more detail. For this quick start, we have run them all on the same host, so we must run corresponding Watson instances manually:: 132 | 133 | $ cp /etc/datawire/watson.conf watson-9002.conf 134 | $ cp /etc/datawire/watson.conf watson-9003.conf 135 | $ nano watson-9002.conf watson-9003.conf 136 | [...] 137 | 138 | $ cat watson-9002.conf 139 | [Watson] 140 | ; The hostname (or IP address) and port number of the service. Optionally a path may be specified by appending it after 141 | ; the host portion of the URI. 142 | ; 143 | ; Examples: http://localhost:9000 or http://localhost:9000/foo 144 | 145 | service_url: http://localhost:9002/greeting 146 | 147 | ; The name of the service. This must be unique within the Datawire directory. The name must also satisfy the following 148 | ; constraints: 149 | ; 150 | ; * minimum length: 1 character 151 | ; * maximum length: 100 characters 152 | ; * only lower case letters, numerical digits, underscores, and hyphens allowed 153 | ; * must start with a letter or underscore 154 | 155 | service_name: greeting 156 | 157 | ; The service health check URL. The URL must respond to HTTP GET requests. The HTTP status code 200 indicates the service is healthy, any other status code in the response indicates that it is not. 158 | ; 159 | ; Warning: Be careful using property reference syntax to blindly populate health_check_url here (e.g. %(service_url)) because 160 | ; defining an additional path will cause problems. For example, if service_url is http://localhost:9000/foo and 161 | ; you use $(service_url)/health then Watson will health check http://localhost:9000/foo/health which is probably not the intent. 162 | ; 163 | ; Examples: http://localhost:9000/health 164 | 165 | health_check_url: http://localhost:9002/greeting 166 | 167 | ; The number of seconds between health checks. 168 | 169 | period: 3 170 | 171 | ; logging level (default in datawire.conf). Valid options are: DEBUG, INFO, WARNING, ERROR, or CRITICAL. 172 | 173 | ;logging: WARNING 174 | 175 | $ cat watson-9003.conf 176 | [Watson] 177 | ; The hostname (or IP address) and port number of the service. Optionally a path may be specified by appending it after 178 | ; the host portion of the URI. 179 | ; 180 | ; Examples: http://localhost:9000 or http://localhost:9000/foo 181 | 182 | service_url: http://localhost:9003/greeting 183 | 184 | ; The name of the service. This must be unique within the Datawire directory. The name must also satisfy the following 185 | ; constraints: 186 | ; 187 | ; * minimum length: 1 character 188 | ; * maximum length: 100 characters 189 | ; * only lower case letters, numerical digits, underscores, and hyphens allowed 190 | ; * must start with a letter or underscore 191 | 192 | service_name: greeting 193 | 194 | ; The service health check URL. The URL must respond to HTTP GET requests. The HTTP status code 200 indicates the service is healthy, any other status code in the response indicates that it is not. 195 | ; 196 | ; Warning: Be careful using property reference syntax to blindly populate health_check_url here (e.g. %(service_url)) because 197 | ; defining an additional path will cause problems. For example, if service_url is http://localhost:9000/foo and 198 | ; you use $(service_url)/health then Watson will health check http://localhost:9000/foo/health which is probably not the intent. 199 | ; 200 | ; Examples: http://localhost:9000/health 201 | 202 | health_check_url: http://localhost:9003/greeting 203 | 204 | ; The number of seconds between health checks. 205 | 206 | period: 3 207 | 208 | ; logging level (default in datawire.conf). Valid options are: DEBUG, INFO, WARNING, ERROR, or CRITICAL. 209 | 210 | ;logging: WARNING 211 | 212 | $ watson -c watson-9002.conf & 213 | $ watson -c watson-9003.conf & 214 | 215 | Sherlock and HAProxy will automatically and transparently load balance over these three microservice instances because they all have the same service name ``http://localhost:8000/greeting``. The ``curl`` command above will access each of them in turn:: 216 | 217 | $ for i in 1 2 3 4 5 ; do curl http://localhost:8000/greeting ; echo ; done 218 | {"id":18,"content":"Hello, World!"} 219 | {"id":16,"content":"Hello, World!"} 220 | {"id":54,"content":"Hello, World!"} 221 | {"id":19,"content":"Hello, World!"} 222 | {"id":17,"content":"Hello, World!"} 223 | 224 | Upgrade 225 | ------- 226 | 227 | Let's upgrade the Greeting service. Duplicate the Greeting service tree and edit line 11 in ``GreetingController.java``:: 228 | 229 | $ cd ../.. 230 | $ mkdir v2 231 | $ cd v2 232 | $ unzip -q ../master.zip 233 | $ cd gs-rest-service-master/complete/ 234 | $ nano src/main/java/hello/GreetingController.java 235 | $ grep -n Hello src/main/java/hello/GreetingController.java 236 | 11: private static final String template = "Hello 2.0, %s!"; 237 | $ mvn -q package 238 | 239 | Instead of upgrading all of Greeting to the new version, let's perform a *canary test*. Roll out one new instance of Greeting 2.0 and its associated Watson:: 240 | 241 | $ env SERVER_PORT=9004 java -jar target/gs-rest-service-0.1.0.jar > /dev/null 2>&1 & 242 | $ cp /etc/datawire/watson.conf watson-9004.conf 243 | $ nano watson-9004.conf 244 | [...] 245 | 246 | $ cat watson-9004.conf 247 | [Watson] 248 | ; The hostname (or IP address) and port number of the service. Optionally a path may be specified by appending it after 249 | ; the host portion of the URI. 250 | ; 251 | ; Examples: http://localhost:9000 or http://localhost:9000/foo 252 | 253 | service_url: http://localhost:9004/greeting 254 | 255 | ; The name of the service. This must be unique within the Datawire directory. The name must also satisfy the following 256 | ; constraints: 257 | ; 258 | ; * minimum length: 1 character 259 | ; * maximum length: 100 characters 260 | ; * only lower case letters, numerical digits, underscores, and hyphens allowed 261 | ; * must start with a letter or underscore 262 | 263 | service_name: greeting 264 | 265 | ; The service health check URL. The URL must respond to HTTP GET requests. The HTTP status code 200 indicates the service is healthy, any other status code in the response indicates that it is not. 266 | ; 267 | ; Warning: Be careful using property reference syntax to blindly populate health_check_url here (e.g. %(service_url)) because 268 | ; defining an additional path will cause problems. For example, if service_url is http://localhost:9000/foo and 269 | ; you use $(service_url)/health then Watson will health check http://localhost:9000/foo/health which is probably not the intent. 270 | ; 271 | ; Examples: http://localhost:9000/health 272 | 273 | health_check_url: http://localhost:9004/greeting 274 | 275 | ; The number of seconds between health checks. 276 | 277 | period: 3 278 | 279 | ; logging level (default in datawire.conf). Valid options are: DEBUG, INFO, WARNING, ERROR, or CRITICAL. 280 | 281 | ;logging: WARNING 282 | 283 | $ watson -c watson-9004.conf & 284 | 285 | Baker will direct a subset of all traffic to that new instance automatically:: 286 | 287 | $ for i in 1 2 3 4 5 ; do curl http://localhost:8000/greeting ; echo ; done 288 | {"id":112,"content":"Hello, World!"} 289 | {"id":77,"content":"Hello, World!"} 290 | {"id":75,"content":"Hello, World!"} 291 | {"id":6,"content":"Hello 2.0, World!"} 292 | {"id":113,"content":"Hello, World!"} 293 | 294 | Let your upgraded Greeting service soak test as long as is desired. Problems? Just kill Greeting 2.0; Baker will keep the requests flowing. Everything going smoothly? Upgrade the remaining instances one at a time without any interruption of service. 295 | 296 | Summary 297 | ------- 298 | 299 | Congratulations on making your way through the Baker quick start! 300 | You've seen that Baker can be deployed quickly and easily, in many 301 | cases with no changes to your service. You've used Baker to perform 302 | load balancing and a safe upgrade with no interruption of 303 | service. You've been able to do all these without deploying and 304 | configuring a central load balancer for each of your microservices, a 305 | scenario which introduces a single point of failure and adds 306 | additional management overhead. 307 | 308 | Next Steps 309 | ---------- 310 | 311 | #. Read about :ref:`deployment`, which shows how you would deploy Baker over your network of microservices. 312 | #. Learn more about Baker's :ref:`architecture`. 313 | -------------------------------------------------------------------------------- /docs/source/reference.rst: -------------------------------------------------------------------------------- 1 | .. _reference: 2 | 3 | Reference 4 | ========= 5 | 6 | The Datawire Baker components rely on configuration files to manage setup. The individual components look in ``/etc/datawire`` and ``~/.datawire`` for standard files, with the contents of the latter overriding the contents of the former. The ``--config`` command line option allows an additional file to be specified, with its contents overriding the prior files. 7 | 8 | Every component starts by loading ``datawire.conf``:: 9 | 10 | [DEFAULT] 11 | ; logging level may be DEBUG, INFO, WARNING, ERROR, or CRITICAL 12 | logging: WARNING 13 | 14 | [Datawire] 15 | ; Change this to the well known and stable hostname of the directory for your deployment. 16 | directory_host: localhost 17 | 18 | This configuration file allows the specification of the default logging level for every Datawire component, but individual component configurations may override this by specifying a different ``logging`` directive. Component log output goes to stdout, with the operating system's service control mechanism (systemd, Upstart, etc.) handling output redirection (to the system journal, to ``/var/log/upstart/*.log``, etc.). 19 | 20 | This configuration file also contains the central designation for the host running the Datawire Directory. As one directory can serve an entire site installation, it is convenient to have this hostname specified in one place. 21 | 22 | Directory 23 | --------- 24 | 25 | A single Datawire Directory allows all other Baker components to communicate. Unlike the other components, the directory allows significant configuration by command line options. 26 | 27 | At the command line:: 28 | 29 | usage: directory [-h] [-c FILE] [-n HOST] [-p PORT] [-a ADDRESS] [-l LEVEL] 30 | [-V] 31 | 32 | optional arguments: 33 | -h, --help show this help message and exit 34 | -c FILE, --config FILE 35 | read from additional config file 36 | -n HOST, --host HOST network host (defaults to localhost) 37 | -p PORT, --port PORT network port (defaults to 5672) 38 | -a ADDRESS, --address ADDRESS 39 | amqp address, defaults to //[:') 47 | def get_watson_info(name): 48 | resp = Response(status=404) 49 | 50 | if name in WATSON_INSTANCES: 51 | resp = Response(response=json.dumps(dict(watson_pid=WATSON_INSTANCES.get(name))), status=200, mimetype="application/json") 52 | 53 | return resp 54 | 55 | @app.route('/watsons/', methods=['DELETE']) 56 | def kill_watson(name): 57 | pid = WATSON_INSTANCES.get(name, None) 58 | if pid is not None: 59 | kill_subprocess(pid) 60 | WATSON_INSTANCES.pop(name, None) 61 | 62 | return Response(status=200) 63 | 64 | @app.route('/watsons', methods=['GET']) 65 | def show_watsons(): 66 | return Response(response=json.dumps(dict(WATSON_INSTANCES)), status=200, mimetype='application/json') 67 | 68 | @app.route('/watsons', methods=['POST']) 69 | def start_watson(): 70 | config_name = request.args.get('config_name') 71 | if config_name not in get_known_watson_configs(): 72 | print get_known_watson_configs() 73 | print "no config (name: %s)" % config_name 74 | return Response(status=400) 75 | 76 | if config_name in WATSON_INSTANCES: 77 | print "already loaded" 78 | return Response(status=400) 79 | 80 | try: 81 | proc = subprocess.Popen(["%s" % WATSON_EXECUTABLE_PATH, 82 | "-c", "/etc/datawire/%s" % config_name]) 83 | 84 | pid = int(proc.pid) 85 | print "loaded (pid: %s)" % pid 86 | WATSON_INSTANCES[config_name] = int(pid) 87 | return Response(status=200) 88 | except Exception as e: 89 | app.logger.error(e) 90 | return Response(status=500) 91 | 92 | 93 | @app.route('/info') 94 | def info(): 95 | return Response(response=json.dumps(dict(datawire_config_root=DATAWIRE_CONFIG_ROOT, 96 | watson_executable_path=WATSON_EXECUTABLE_PATH)), 97 | status=200, 98 | mimetype='application/json') 99 | 100 | 101 | def main(): 102 | parser = ArgumentParser() 103 | parser.add_argument("-c", "--config", help="read from additional config file", metavar="FILE") 104 | args = parser.parse_args() 105 | 106 | config = None 107 | with open(args.config, 'r') as stream: 108 | config = yaml.load(stream) 109 | 110 | global DATAWIRE_CONFIG_ROOT 111 | DATAWIRE_CONFIG_ROOT = config['datawire_config_root'] 112 | 113 | global WATSON_EXECUTABLE_PATH 114 | WATSON_EXECUTABLE_PATH = config['watson_executable_path'] 115 | 116 | app.run(debug=True, 117 | host=config['watson_controller_listen_address'], 118 | port=int(config['watson_controller_port'])) 119 | 120 | if __name__ == "__main__": 121 | main() 122 | 123 | -------------------------------------------------------------------------------- /resources/infrastructure-setup/config/watson_controller.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # file: watson_controller.yml 3 | # 4 | # Copy to this to the location of watson_controller.py on the remote machine 5 | 6 | datawire_config_root: /etc/datawire 7 | 8 | watson_controller_port : 5002 9 | watson_controller_listen_address: 0.0.0.0 10 | watson_executable_path: /bin/watson -------------------------------------------------------------------------------- /resources/infrastructure-setup/templates/sherlock.conf.tpl: -------------------------------------------------------------------------------- 1 | [Datawire] 2 | directory_host = ${directory_host} 3 | 4 | [Sherlock] 5 | ; The path to the HAProxy executable 6 | proxy: /usr/sbin/haproxy 7 | 8 | ; The directory where HAProxy runs from and reads its configuration. 9 | rundir: /opt/datawire/run 10 | 11 | ; The debounce period in seconds. The debounce period is designed to prevent HAProxy from constantly restarting due 12 | ; to changes. 13 | debounce: 2 14 | dir_debounce: 2 15 | 16 | ; logging level (default in datawire.conf). Valid options are: DEBUG, INFO, WARNING, ERROR, or CRITICAL. 17 | logging: DEBUG 18 | 19 | -------------------------------------------------------------------------------- /resources/infrastructure-setup/templates/watson-test_service-nopath.conf.tpl: -------------------------------------------------------------------------------- 1 | [Datawire] 2 | directory_host = ${directory_host} 3 | 4 | [Watson] 5 | ; The name of the service. This must be unique within the Datawire directory. The name must also satisfy the following 6 | ; constraints. 7 | ; 8 | ; Constraints 9 | ; ----------- 10 | ; length: 1..100 characters 11 | ; case: lower-case only 12 | ; allowed characters: alphanumeric, underscore and hyphen. 13 | ; misc: must start with a letter or underscore 14 | 15 | service_name: bar 16 | 17 | ; The service health check URL. The URL must respond to HTTP GET requests. 18 | ; 19 | ; Warning: Be careful using property reference syntax to blindly populate service_url here (e.g. %(service_url)s because 20 | ; defining an additional path will cause problems. For example, if service_url is http://localhost:9000/foo and 21 | ; you use $(service_url)/health then Watson will health check http://localhost:9000/foo/health which is most 22 | ; likely what you do not want to do. 23 | ; 24 | ; Examples: http://localhost:9000/health 25 | 26 | health_check_url: http://localhost:5001/health 27 | 28 | ; The number of seconds between health checks. 29 | 30 | period: 3 31 | 32 | ; logging level (default in datawire.conf). Valid options are: DEBUG, INFO, WARNING, ERROR, or CRITICAL. 33 | 34 | logging: DEBUG 35 | 36 | ; The hostname (or IP address) and port number of the service. Optionally a path may be specified by appending it after 37 | ; the host portion of the URI. 38 | ; 39 | ; Examples: http://localhost:9000 or http://localhost:9000/foo -------------------------------------------------------------------------------- /resources/infrastructure-setup/templates/watson-test_service-path.conf.tpl: -------------------------------------------------------------------------------- 1 | [Datawire] 2 | directory_host = ${directory_host} 3 | 4 | [Watson] 5 | ; The name of the service. This must be unique within the Datawire directory. The name must also satisfy the following 6 | ; constraints. 7 | ; 8 | ; Constraints 9 | ; ----------- 10 | ; length: 1..100 characters 11 | ; case: lower-case only 12 | ; allowed characters: alphanumeric, underscore and hyphen. 13 | ; misc: must start with a letter or underscore 14 | 15 | service_name: foo 16 | 17 | ; The service health check URL. The URL must respond to HTTP GET requests. 18 | ; 19 | ; Warning: Be careful using property reference syntax to blindly populate service_url here (e.g. %(service_url)s because 20 | ; defining an additional path will cause problems. For example, if service_url is http://localhost:9000/foo and 21 | ; you use $(service_url)/health then Watson will health check http://localhost:9000/foo/health which is most 22 | ; likely what you do not want to do. 23 | ; 24 | ; Examples: http://localhost:9000/health 25 | 26 | health_check_url: http://localhost:5001/health 27 | 28 | ; The number of seconds between health checks. 29 | 30 | period: 3 31 | 32 | ; logging level (default in datawire.conf). Valid options are: DEBUG, INFO, WARNING, ERROR, or CRITICAL. 33 | 34 | logging: DEBUG 35 | 36 | ; The hostname (or IP address) and port number of the service. Optionally a path may be specified by appending it after 37 | ; the host portion of the URI. 38 | ; 39 | ; Examples: http://localhost:9000 or http://localhost:9000/foobar -------------------------------------------------------------------------------- /setup_sherlock.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | from setuptools.dist import Distribution 3 | 4 | class PurePythonDistribution(Distribution): 5 | def is_pure(self): 6 | return True 7 | 8 | 9 | metadata = {} 10 | with open("_metadata_sherlock.py") as fp: 11 | exec(fp.read(), metadata) 12 | 13 | setup(name='bakerstreet', 14 | version=metadata["__version__"], 15 | description=metadata["__summary__"], 16 | author=metadata["__author__"], 17 | author_email=metadata["__email__"], 18 | url=metadata["__uri__"], 19 | license=metadata["__license__"], 20 | install_requires=['datawire-common'], 21 | scripts=['sherlock', 'watson'], 22 | distclass=PurePythonDistribution) 23 | -------------------------------------------------------------------------------- /setup_watson.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | from setuptools.dist import Distribution 3 | 4 | class PurePythonDistribution(Distribution): 5 | def is_pure(self): 6 | return True 7 | 8 | 9 | metadata = {} 10 | with open("_metadata_watson.py") as fp: 11 | exec(fp.read(), metadata) 12 | 13 | setup(name='bakerstreet', 14 | version=metadata["__version__"], 15 | description=metadata["__summary__"], 16 | author=metadata["__author__"], 17 | author_email=metadata["__email__"], 18 | url=metadata["__uri__"], 19 | license=metadata["__license__"], 20 | install_requires=['datawire-common'], 21 | scripts=['sherlock', 'watson'], 22 | distclass=PurePythonDistribution) 23 | -------------------------------------------------------------------------------- /sherlock: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2015 The Baker Street Authors. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """ 18 | Sherlock 19 | 20 | - Subscribe to the directory 21 | - Track routes that target HTTP 22 | - Keep an HAProxy config file updated with tracked routes 23 | """ 24 | 25 | import os 26 | import logging 27 | from argparse import ArgumentParser 28 | from urlparse import urlparse 29 | from time import time, ctime 30 | from subprocess import Popen 31 | 32 | from proton.reactor import Reactor 33 | from datawire import Configuration, Processor, Receiver 34 | 35 | from _metadata_sherlock import __version__ 36 | 37 | logging.basicConfig(datefmt="%Y-%m-%d %H:%M:%S", 38 | format="%(asctime)s sherlock %(name)s %(levelname)s %(message)s") 39 | log = logging.getLogger() 40 | 41 | confBase = """ 42 | global 43 | daemon 44 | maxconn 256 45 | 46 | defaults 47 | mode http 48 | timeout connect 5000ms 49 | timeout client 50000ms 50 | timeout server 50000ms 51 | 52 | frontend http-in 53 | bind *:8000""".split("\n") 54 | 55 | 56 | class Sherlock(object): 57 | 58 | def __init__(self, args): 59 | self.receiver = Receiver(args.directory, Processor(self)) 60 | self.route_map = {} # address -> [ url, url, ... ], policy 61 | 62 | self.debounce_interval = args.debounce 63 | self.directory_debounce_interval = args.dir_debounce 64 | 65 | self.updated = True 66 | self.last_modification_time = 0 67 | self.current_debounce = self.debounce_interval 68 | self.previous_config = None 69 | 70 | self.haproxy_config_path = os.path.join(args.rundir, "haproxy.conf") 71 | self.haproxy_pid_path = os.path.join(args.rundir, "haproxy.pid") 72 | self.haproxy_command = "%s -f %s -p %s" % (args.proxy, self.haproxy_config_path, self.haproxy_pid_path) 73 | 74 | def on_reactor_init(self, event): 75 | self.receiver.start(event.reactor) 76 | event.reactor.schedule(self.current_debounce, self) 77 | 78 | def on_link_remote_open(self, event): 79 | log.info("Detected new connection to the directory at %s", ctime()) 80 | self.current_debounce = self.directory_debounce_interval 81 | 82 | def on_message(self, event): 83 | if event.message.subject != "routes": 84 | return 85 | 86 | msg = event.message 87 | address = msg.body[0] 88 | routes = msg.body[1] 89 | policy = msg.properties["policy"] 90 | self.route_map[address] = ([target 91 | for (host, port, target), owner in routes 92 | if target and target.upper().startswith("HTTP")], 93 | policy) 94 | 95 | self.updated = True 96 | self.last_modification_time = time() 97 | event.reactor.schedule(self.current_debounce, self) 98 | 99 | def on_timer_task(self, event): 100 | if not self.updated: 101 | return 102 | 103 | elapsed = time() - self.last_modification_time 104 | if elapsed < self.current_debounce: 105 | event.reactor.schedule(self.current_debounce - elapsed, self) 106 | return 107 | 108 | self.updated = False 109 | self.current_debounce = self.debounce_interval 110 | self.update_haproxy() 111 | 112 | def render(self): 113 | 114 | """Generates the HAProxy configuration file contents""" 115 | 116 | head, frontends, backends = [], [], [] 117 | for address, (routes, policy) in sorted(self.route_map.items()): 118 | if len(routes) > 0: 119 | # there is always going to be at least one route so grab one and parse the path out of it since that is 120 | # our rewrite content 121 | route_url = urlparse(routes[0]) 122 | route_path = route_url.path 123 | 124 | internal_url = urlparse(address) 125 | service_name = internal_url.path[1:] 126 | backend = "BE" + "_" + service_name 127 | acl_name = "IS" + "_" + service_name 128 | frontends.append("\n acl %s path_beg %s" % (acl_name, "/%s" % service_name)) 129 | frontends.append(" use_backend %s if %s" % (backend, acl_name)) 130 | backends.append("\nbackend %s" % backend) 131 | 132 | # Rewrites the incoming URL so that the service name is removed and replaced with the path component of 133 | # the service_url as defined in watson's configuration. Afterwards the rest of the request line is added 134 | # back onto the rewritten path (e.g. query parameters, fragments) 135 | backends.append(" reqrep ^([^\ :]*)\ /%s(.*) \\1\ %s\\2" % (service_name, 136 | (route_path if route_path else "/"))) 137 | for url in sorted(routes): 138 | internal_url = urlparse(url) 139 | host = internal_url.hostname 140 | port = internal_url.port or 80 141 | name = "%s_%s" % (host, port) 142 | backends.append(" server %s %s:%s maxconn 32" % (name, host, port)) 143 | 144 | return "\n".join(head + confBase + frontends + backends) 145 | 146 | def update_haproxy(self): 147 | 148 | """Updates HAProxy configuration and then restarts the HAProxy process to read the configuration changes""" 149 | 150 | haproxy_config_content = self.render() 151 | if haproxy_config_content != self.previous_config: 152 | self.previous_config = haproxy_config_content 153 | with open(self.haproxy_config_path, "wb") as outf: 154 | outf.write("# Last update %s\n" % ctime()) 155 | outf.write(haproxy_config_content) 156 | outf.write("\n") 157 | log.info("Wrote new configuration file to %s at %s", self.haproxy_config_path, ctime()) 158 | command = self.haproxy_command 159 | try: 160 | command += " -sf %s" % open(self.haproxy_pid_path).read() 161 | except IOError: 162 | pass 163 | try: 164 | proc = Popen(command.split(), close_fds=True) 165 | proc.wait() 166 | 167 | log.info("Launched %s", command) 168 | except OSError as exc: 169 | log.error("Failed to launch %r", command) 170 | log.error(" (%s)", exc) 171 | else: 172 | log.info("Duplicate output suppressed at %s", ctime()) 173 | 174 | default_config = """ 175 | [DEFAULT] 176 | logging: WARNING 177 | 178 | [Datawire] 179 | directory_host: 180 | 181 | [Sherlock] 182 | proxy: /usr/sbin/haproxy 183 | rundir: . 184 | debounce: 2 ; seconds 185 | dir_debounce: 2 ; seconds 186 | """ 187 | 188 | 189 | def main(): 190 | parser = ArgumentParser() 191 | parser.add_argument("-c", "--config", help="read from additional config file", metavar="FILE") 192 | parser.add_argument("-V", "--version", action="version", version="%(prog)s " + __version__) 193 | args = parser.parse_args() 194 | 195 | loader = Configuration(default_config) 196 | loader.add_file_relative("sherlock.conf") 197 | if args.config: 198 | loader.add_file_absolute(args.config) 199 | 200 | try: 201 | config = loader.parse() 202 | args.directory_host = config.get("Datawire", "directory_host") 203 | args.proxy = config.get("Sherlock", "proxy") 204 | args.rundir = config.get("Sherlock", "rundir") 205 | args.debounce = config.getint("Sherlock", "debounce") 206 | args.dir_debounce = config.getint("Sherlock", "dir_debounce") 207 | args.logging = config.get("Sherlock", "logging") 208 | except Exception: 209 | log.exception("Failed to load configuration") 210 | loader.exit_with_config_error("Failed to load configuration") 211 | 212 | log.setLevel(getattr(logging, args.logging.upper())) 213 | if not loader.parsed_filenames: 214 | log.warning("No configuration files found. Falling back to defaults.") 215 | if not args.directory_host: 216 | log.warning("No directory_host configured. Falling back to localhost.") 217 | args.directory_host = "localhost" 218 | 219 | args.directory = "//%s/directory" % args.directory_host 220 | 221 | Reactor(Sherlock(args)).run() 222 | 223 | 224 | if __name__ == "__main__": 225 | main() 226 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | def pytest_addoption(parser): 4 | parser.addoption("--wc-host", action="store", help="Host address of the Watson controller service") 5 | 6 | 7 | @pytest.fixture 8 | def wc_host(request): 9 | return request.config.getoption("--wc-host") 10 | -------------------------------------------------------------------------------- /tests/test_bakerstreet.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import time 3 | 4 | def test_resolve_service_with_path(wc_host): 5 | """Test that Baker Street will route traffic from the name to the proper path while also preserving URL query 6 | parameters""" 7 | 8 | load_watson_with_config(wc_host, "watson-test_service-path.conf") 9 | time.sleep(5) 10 | assert requests.get("http://localhost:8000/foo", params=dict(name="Homer")).text == "Hi, Homer!" 11 | 12 | def test_resolve_service_without_path(wc_host): 13 | """Test that Baker Street will route traffic properly""" 14 | 15 | load_watson_with_config(wc_host, "watson-test_service-nopath.conf") 16 | time.sleep(5) 17 | assert requests.get("http://localhost:8000/bar").text == "Hi, everybody!" 18 | 19 | def load_watson_with_config(wc_host, config_name): 20 | resp = requests.post("http://%s/watsons" % wc_host, params=dict(config_name=config_name)) 21 | -------------------------------------------------------------------------------- /watson: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2015 The Baker Street Authors. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """ 18 | Watson 19 | 20 | - Periodically GET a URL 21 | - Tether to the directory if GET was successful 22 | """ 23 | 24 | import urllib2 25 | import logging 26 | import re 27 | from argparse import ArgumentParser 28 | 29 | from proton.reactor import Reactor 30 | 31 | from datawire import Configuration, Tether 32 | 33 | from _metadata_watson import __version__ 34 | 35 | logging.basicConfig(datefmt="%Y-%m-%d %H:%M:%S", 36 | format="%(asctime)s watson %(name)s %(levelname)s %(message)s") 37 | log = logging.getLogger() 38 | 39 | 40 | class LivenessByHTTPGet(object): 41 | 42 | def __init__(self, url): 43 | self.url = url 44 | self.okay = set([200]) 45 | 46 | def __call__(self): 47 | try: 48 | res = urllib2.urlopen(self.url) 49 | if res.getcode() in self.okay: 50 | return True 51 | except urllib2.URLError: 52 | pass 53 | return False 54 | 55 | 56 | class Watson(object): 57 | 58 | def __init__(self, args, testLiveness): 59 | self.tetherArgs = args.directory, args.address, args.service_url 60 | self.tether = None 61 | self.testingPeriod = args.period 62 | self.testLiveness = testLiveness 63 | self.justStarted = True 64 | self.url = args.service_url 65 | 66 | def on_reactor_init(self, event): 67 | event.reactor.schedule(0, self) 68 | 69 | def on_timer_task(self, event): 70 | if self.testLiveness(): 71 | # Alive 72 | if self.tether is None: 73 | # Just came to life 74 | log.info("DEAD -> LIVE (%s)", self.url) 75 | self.tether = Tether(*self.tetherArgs) 76 | self.tether.start(event.reactor) 77 | else: 78 | # Dead 79 | if self.tether is not None: 80 | # Just died 81 | log.info("LIVE -> DEAD (%s)", self.url) 82 | self.tether.stop(event.reactor) 83 | self.tether = None 84 | log.debug(" liveness check at %s for service %s", self.testLiveness.url, self.url) 85 | elif self.justStarted: 86 | log.info("START -> DEAD (%s)", self.url) 87 | self.justStarted = False 88 | log.debug("liveness check at %s for service %s", self.testLiveness.url, self.url) 89 | event.reactor.schedule(self.testingPeriod, self) 90 | 91 | default_config = """ 92 | [DEFAULT] 93 | logging: WARNING 94 | 95 | [Datawire] 96 | directory_host: 97 | 98 | [Watson] 99 | period: 3 100 | """ 101 | 102 | def create_config_fail_message(reason=None): 103 | return "Failed to load configuration%s" % (": %s." % reason if reason else ".") 104 | 105 | 106 | def validate_service_name(config_loader, service_name): 107 | 108 | """ 109 | Checks that a service name is not empty or all whitespace as well as whether the service name fits certain 110 | requirements such as beginning with a a letter or underscore and that the overall service name is no longer than 111 | one hundred characters. 112 | 113 | :param config_loader: the configuration loader 114 | :param service_name: the name of the service to validate 115 | :return: None 116 | """ 117 | 118 | pattern = re.compile("^([a-z_])([a-z0-9_-]*)$") 119 | if not service_name or service_name.isspace() or not pattern.match(service_name) or len(service_name) > 100: 120 | log.exception(create_config_fail_message("invalid service name")) 121 | config_loader.exit_with_config_error("Configuration error: invalid service name; not defined or did not match " 122 | "pattern (pattern: %s)" % pattern.pattern) 123 | 124 | 125 | def main(): 126 | parser = ArgumentParser() 127 | parser.add_argument("-c", "--config", help="read from additional config file", metavar="FILE") 128 | parser.add_argument("-V", "--version", action="version", version="%(prog)s " + __version__) 129 | args = parser.parse_args() 130 | 131 | loader = Configuration(default_config) 132 | loader.add_file_relative("watson.conf") 133 | if args.config: 134 | loader.add_file_absolute(args.config) 135 | 136 | config = loader.parse() 137 | if not loader.parsed_filenames: 138 | loader.exit_with_config_error(create_config_fail_message("configuration not found")) 139 | 140 | try: 141 | args.directory_host = config.get("Datawire", "directory_host") 142 | args.period = config.getint("Watson", "period") 143 | args.logging = config.get("Watson", "logging") 144 | except Exception: 145 | log.exception("Failed to load configuration") 146 | loader.exit_with_config_error(create_config_fail_message()) 147 | 148 | if args.period < 1: 149 | args.period = 1 150 | log.warning("Setting service check period to minimum value of one second.") 151 | 152 | try: 153 | args.service_url = config.get("Watson", "service_url") 154 | args.service_name = config.get("Watson", "service_name") 155 | args.health_check_url = config.get("Watson", "health_check_url") 156 | except Exception: 157 | loader.exit_with_config_error( 158 | create_config_fail_message("ensure service_url, service_name and health_check_url are defined")) 159 | 160 | validate_service_name(loader, args.service_name) 161 | 162 | log.setLevel(getattr(logging, args.logging.upper())) 163 | if not args.directory_host: 164 | log.warning("No directory_host configured. Falling back to localhost.") 165 | args.directory_host = "localhost" 166 | 167 | args.directory = "//%s/directory" % args.directory_host 168 | args.address = "//%s/%s" % (args.directory_host, args.service_name) 169 | 170 | checker = LivenessByHTTPGet(args.health_check_url) 171 | 172 | log.info("Starting Watson... " 173 | + "(directory: %s, service_name: %s, service_url: %s, health: %s)" % (args.directory, 174 | args.service_name, 175 | args.service_url, 176 | args.health_check_url)) 177 | 178 | Reactor(Watson(args, checker)).run() 179 | 180 | if __name__ == "__main__": 181 | main() 182 | --------------------------------------------------------------------------------