├── .github ├── CONTRIBUTING.md ├── ISSUE_TEMPLATE │ ├── config.yml │ ├── issue-bug.yml │ └── issue-enhance.yml ├── PULL_REQUEST_TEMPLATE.md └── TEMPLATE-README.md ├── .gitignore ├── CODEOWNERS ├── ISSUE_TEMPLATE.md ├── LICENSE ├── NOTICE ├── README.md ├── bin └── test.bash ├── commandrunner ├── commandrunner_suite_test.go ├── fakes │ └── fake_runner.go ├── runner.go └── runner_test.go ├── config ├── assets │ ├── ports-negative.yml │ └── ports-too-high.yml ├── config.go ├── config_suite_test.go └── config_test.go ├── docs └── 01-usage.md ├── example_config ├── example.json └── route.yml ├── healthchecker ├── fakes │ └── fake_health_checker.go ├── healthchecker_suite_test.go ├── script_checker.go └── script_checker_test.go ├── integration ├── init_test.go ├── main_test.go └── tcp_route_registration_test.go ├── main.go ├── messagebus ├── messagebus.go ├── messagebus_suite_test.go ├── messagebus_test.go └── messagebusfakes │ └── fake_message_bus.go ├── registrar ├── periodic_health_check_close_chans.go ├── registrar.go ├── registrar_suite_test.go ├── registrar_test.go ├── routes_config_watcher.go └── routes_config_watcher_test.go └── routingapi ├── api.go ├── api_test.go ├── init_test.go ├── routingapi_test.go └── routingapifakes └── fake_uaa_client.go /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributor License Agreement 2 | --------------- 3 | 4 | Follow these steps to make a contribution to any of our open source repositories: 5 | 6 | 1. Ensure that you have completed our CLA Agreement for [individuals](https://www.cloudfoundry.org/wp-content/uploads/2015/07/CFF_Individual_CLA.pdf) or [corporations](https://www.cloudfoundry.org/wp-content/uploads/2015/07/CFF_Corporate_CLA.pdf). 7 | 8 | 1. Set your name and email (these should match the information on your submitted CLA) 9 | ``` 10 | git config --global user.name "Firstname Lastname" 11 | git config --global user.email "your_email@example.com" 12 | ``` 13 | 14 | 1. All contributions must be sent using GitHub pull requests as they create a nice audit trail and structured approach. 15 | 16 | The originating github user has to either have a github id on-file with the list of approved users that have signed 17 | the CLA or they can be a public "member" of a GitHub organization for a group that has signed the corporate CLA. 18 | This enables the corporations to manage their users themselves instead of having to tell us when someone joins/leaves an organization. By removing a user from an organization's GitHub account, their new contributions are no longer approved because they are no longer covered under a CLA. 19 | 20 | If a contribution is deemed to be covered by an existing CLA, then it is analyzed for engineering quality and product 21 | fit before merging it. 22 | 23 | If a contribution is not covered by the CLA, then the automated CLA system notifies the submitter politely that we 24 | cannot identify their CLA and ask them to sign either an individual or corporate CLA. This happens automatically as a 25 | comment on pull requests. 26 | 27 | When the project receives a new CLA, it is recorded in the project records, the CLA is added to the database for the 28 | automated system uses, then we manually make the Pull Request as having a CLA on-file. 29 | 30 | 31 | Initial Setup 32 | --------------- 33 | - Install docker 34 | 35 | - Add required directories 36 | 37 | ```bash 38 | # create parent directory 39 | mkdir -p ~/workspace 40 | cd ~/workspace 41 | 42 | # clone ci 43 | git clone https://github.com/cloudfoundry/wg-app-platform-runtime-ci.git 44 | 45 | # clone repo 46 | git clone https://github.com/cloudfoundry/routing-release.git --recursive 47 | cd routing-release 48 | ``` 49 | 50 | Running Tests 51 | --------------- 52 | 53 | > [!TIP] 54 | > Running tests for this repo requires a DB flavor. The following scripts will default to mysql DB. Set DB environment variable for alternate DBs. Valid Options: mysql-8.0(or mysql),mysql-5.7,postgres 55 | 56 | - `./scripts/create-docker-container.bash`: This will create a docker container with appropriate mounts. This 57 | script can be used for interactive development with a long running container. 58 | - `./scripts/test-in-docker.bash`: Create docker container and run all tests and setup in a single script. 59 | - `./scripts/test-in-docker.bash `: For running tests under a specific package and/or sub-package 60 | 61 | When inside docker container: 62 | 63 | - `/repo/scripts/docker/build-binaries.bash`: (REQUIRED) This will build required binaries for running tests. 64 | - `/repo/scripts/docker/test.bash`: This will run all tests in this repo. 65 | - `/repo/scripts/docker/test.bash `: This will only run a package's tests 66 | - `/repo/scripts/docker/test.bash `: This will only run sub-package tests for package 67 | - `/repo/scripts/docker/tests-template.bash`: This will test bosh-spec templates. 68 | - `/repo/scripts/docker/lint.bash`: This will run required linters. 69 | 70 | > [!IMPORTANT] 71 | > If you are about to submit a PR, please make sure to run `./scripts/test-in-docker.bash` for MySQL and Postgres to ensure everything is tested in clean container. If you are developing, you can create create a docker container first, then the only required script to run before testing your specific component is `build-binaries.bash`. 72 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | contact_links: 3 | - name: CloudFoundry slack 4 | url: https://cloudfoundry.slack.com 5 | about: For help or questions about this component, you can reach the maintainers on Slack 6 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/issue-bug.yml: -------------------------------------------------------------------------------- 1 | name: Bug 2 | description: Report a defect, such as a bug or regression. 3 | title: "Start the title with a verb (e.g. Change header styles). Use the imperative mood in the title (e.g. Fix, not Fixed or Fixes header styles)" 4 | labels: 5 | - bug 6 | body: 7 | - type: textarea 8 | id: current 9 | attributes: 10 | label: Current behavior 11 | validations: 12 | required: true 13 | - type: markdown 14 | id: current_md 15 | attributes: 16 | value: | 17 | - Explain, in detail, what the current state of the world is 18 | - Include code snippets, log output, and analysis as necessary to explain the whole problem 19 | - Include links to logs, GitHub issues, slack conversations, etc.. to tell us where the problem came from 20 | - Steps to reproduce 21 | - type: textarea 22 | id: desired 23 | attributes: 24 | label: Desired behavior 25 | validations: 26 | required: true 27 | - type: markdown 28 | id: desired_md 29 | attributes: 30 | value: | 31 | - Describe how the problem should be fixed 32 | - Does this require a new bosh release? 33 | - Does it require configuration changes in cf-deployment? 34 | - Do we need to have a special release note? 35 | - Do we need to update repo documentation? 36 | - type: input 37 | id: version 38 | attributes: 39 | label: Affected Version 40 | description: Please enter the version 41 | validations: 42 | required: true 43 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/issue-enhance.yml: -------------------------------------------------------------------------------- 1 | name: Enhance 2 | description: Propose an enhancement or new feature. 3 | title: "Start the title with a verb (e.g. Change header styles). Use the imperative mood in the title (e.g. Fix, not Fixed or Fixes header styles)" 4 | labels: 5 | - enhancement 6 | body: 7 | - type: textarea 8 | id: change 9 | attributes: 10 | label: Proposed Change 11 | validations: 12 | required: true 13 | - type: markdown 14 | id: change_md 15 | attributes: 16 | value: | 17 | Briefly explain why this feature is necessary in the following format 18 | 19 | **As a** *developer/operator/whatever* 20 | **I want** *this ability to do X* 21 | **So that** *I can do Y* 22 | 23 | - Provide details of where this request is coming from including links, GitHub Issues, etc.. 24 | - Provide details of prior work (if applicable) including links to commits, github issues, etc... 25 | - type: textarea 26 | id: acceptance 27 | attributes: 28 | label: Acceptance criteria 29 | validations: 30 | required: true 31 | - type: markdown 32 | id: acceptance_md 33 | attributes: 34 | value: | 35 | Detail the exact work that is required to accept this story in the following format 36 | 37 | **Scenario:** *describe scenario* 38 | **Given** *I have some sort of configuration* 39 | **When** *I do X* 40 | **And** *do Y* 41 | **Then** *I see the desired behavior* 42 | 43 | - type: textarea 44 | id: related 45 | attributes: 46 | label: Related links 47 | description: Please list related links for this issue 48 | placeholder: | 49 | - [ ] code.cloudfoundry.org/bbs for links 50 | - [x] cloudfoundry/rep#123 for issues/prs 51 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | - [ ] Read the [Contributing document](../blob/-/.github/CONTRIBUTING.md). 2 | 3 | Summary 4 | --------------- 5 | 10 | 11 | 12 | Backward Compatibility 13 | --------------- 14 | Breaking Change? **Yes/No** 15 | 22 | -------------------------------------------------------------------------------- /.github/TEMPLATE-README.md: -------------------------------------------------------------------------------- 1 | 2 | > [!IMPORTANT] 3 | > Content in this directory is managed by the CI task `sync-dot-github-dir`. 4 | 5 | Changing templates 6 | --------------- 7 | These templates are synced from [these shared templates](https://github.com/cloudfoundry/wg-app-platform-runtime-ci/tree/main/shared/github). 8 | Each pipeline will contain a `sync-dot-github-dir-*` job for updating the content of these files. 9 | If you would like to modify these, please change them in the shared group. 10 | It's also possible to override the templates on pipeline's parent directory by introducing a custom 11 | template in `$PARENT_TEMPLATE_DIR/github/FILENAME` or `$PARENT_TEMPLATE_DIR/github/REPO_NAME/FILENAME` in CI repo 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | *.test 6 | 7 | # Folders 8 | _obj 9 | _test 10 | 11 | # Architecture specific extensions/prefixes 12 | *.[568vq] 13 | [568vq].out 14 | 15 | *.cgo1.go 16 | *.cgo2.c 17 | _cgo_defun.c 18 | _cgo_gotypes.go 19 | _cgo_export.* 20 | 21 | _testmain.go 22 | 23 | *.exe 24 | 25 | .idea 26 | 27 | route-registrar 28 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @cloudfoundry/wg-app-runtime-platform-networking-approvers 2 | -------------------------------------------------------------------------------- /ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Please report all issues and feature requests in [cloudfoundry/routing-release](https://github.com/cloudfoundry/routing-release) instead of here. Thanks! 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014-Present CloudFoundry.org Foundation, Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # route-registrar 2 | 3 | > [!CAUTION] 4 | > This repository has been in-lined (using git-subtree) into routing-release. Please make any 5 | > future contributions directly to routing-release. 6 | 7 | [![Go Report 8 | Card](https://goreportcard.com/badge/code.cloudfoundry.org/route-registrar)](https://goreportcard.com/report/code.cloudfoundry.org/route-registrar) 9 | [![Go 10 | Reference](https://pkg.go.dev/badge/code.cloudfoundry.org/route-registrar.svg)](https://pkg.go.dev/code.cloudfoundry.org/route-registrar) 11 | 12 | A standalone executable written in golang that continuously broadcasts a 13 | routes to the [gorouter](https://github.com/cloudfoundry/gorouter). This 14 | is designed to be a general purpose solution, packaged as a BOSH job to 15 | be colocated with components that need to broadcast their routes to the 16 | gorouter, so that those components don't need to maintain logic for 17 | route registration. 18 | 19 | > \[!NOTE\] 20 | > 21 | > This repository should be imported as 22 | > `code.cloudfoundry.org/route-registrar`. 23 | 24 | # Docs 25 | 26 | - [Usage](./docs/01-usage.md) 27 | 28 | # Contributing 29 | 30 | See the [Contributing.md](./.github/CONTRIBUTING.md) for more 31 | information on how to contribute. 32 | 33 | # Working Group Charter 34 | 35 | This repository is maintained by [App Runtime 36 | Platform](https://github.com/cloudfoundry/community/blob/main/toc/working-groups/app-runtime-platform.md) 37 | under `Networking` area. 38 | 39 | > \[!IMPORTANT\] 40 | > 41 | > Content in this file is managed by the [CI task 42 | > `sync-readme`](https://github.com/cloudfoundry/wg-app-platform-runtime-ci/blob/main/shared/tasks/sync-readme/metadata.yml) 43 | > and is generated by CI following a convention. 44 | -------------------------------------------------------------------------------- /bin/test.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu 4 | set -o pipefail 5 | 6 | # shellcheck disable=SC2068 7 | # Double-quoting array expansion here causes ginkgo to fail 8 | go run github.com/onsi/ginkgo/v2/ginkgo ${@} 9 | -------------------------------------------------------------------------------- /commandrunner/commandrunner_suite_test.go: -------------------------------------------------------------------------------- 1 | package commandrunner_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo/v2" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestCommandrunner(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Commandrunner Suite") 13 | } 14 | -------------------------------------------------------------------------------- /commandrunner/fakes/fake_runner.go: -------------------------------------------------------------------------------- 1 | // This file was generated by counterfeiter 2 | package fakes 3 | 4 | import ( 5 | "bytes" 6 | "sync" 7 | 8 | "code.cloudfoundry.org/route-registrar/commandrunner" 9 | ) 10 | 11 | type FakeRunner struct { 12 | RunStub func(outbuf, errbuff *bytes.Buffer) error 13 | runMutex sync.RWMutex 14 | runArgsForCall []struct { 15 | outbuf *bytes.Buffer 16 | errbuff *bytes.Buffer 17 | } 18 | runReturns struct { 19 | result1 error 20 | } 21 | WaitStub func() error 22 | waitMutex sync.RWMutex 23 | waitArgsForCall []struct{} 24 | waitReturns struct { 25 | result1 error 26 | } 27 | KillStub func() error 28 | killMutex sync.RWMutex 29 | killArgsForCall []struct{} 30 | killReturns struct { 31 | result1 error 32 | } 33 | } 34 | 35 | func (fake *FakeRunner) Run(outbuf *bytes.Buffer, errbuff *bytes.Buffer) error { 36 | fake.runMutex.Lock() 37 | fake.runArgsForCall = append(fake.runArgsForCall, struct { 38 | outbuf *bytes.Buffer 39 | errbuff *bytes.Buffer 40 | }{outbuf, errbuff}) 41 | fake.runMutex.Unlock() 42 | if fake.RunStub != nil { 43 | return fake.RunStub(outbuf, errbuff) 44 | } else { 45 | return fake.runReturns.result1 46 | } 47 | } 48 | 49 | func (fake *FakeRunner) RunCallCount() int { 50 | fake.runMutex.RLock() 51 | defer fake.runMutex.RUnlock() 52 | return len(fake.runArgsForCall) 53 | } 54 | 55 | func (fake *FakeRunner) RunArgsForCall(i int) (*bytes.Buffer, *bytes.Buffer) { 56 | fake.runMutex.RLock() 57 | defer fake.runMutex.RUnlock() 58 | return fake.runArgsForCall[i].outbuf, fake.runArgsForCall[i].errbuff 59 | } 60 | 61 | func (fake *FakeRunner) RunReturns(result1 error) { 62 | fake.RunStub = nil 63 | fake.runReturns = struct { 64 | result1 error 65 | }{result1} 66 | } 67 | 68 | func (fake *FakeRunner) Wait() error { 69 | fake.waitMutex.Lock() 70 | fake.waitArgsForCall = append(fake.waitArgsForCall, struct{}{}) 71 | fake.waitMutex.Unlock() 72 | if fake.WaitStub != nil { 73 | return fake.WaitStub() 74 | } else { 75 | return fake.waitReturns.result1 76 | } 77 | } 78 | 79 | func (fake *FakeRunner) WaitCallCount() int { 80 | fake.waitMutex.RLock() 81 | defer fake.waitMutex.RUnlock() 82 | return len(fake.waitArgsForCall) 83 | } 84 | 85 | func (fake *FakeRunner) WaitReturns(result1 error) { 86 | fake.WaitStub = nil 87 | fake.waitReturns = struct { 88 | result1 error 89 | }{result1} 90 | } 91 | 92 | func (fake *FakeRunner) Kill() error { 93 | fake.killMutex.Lock() 94 | fake.killArgsForCall = append(fake.killArgsForCall, struct{}{}) 95 | fake.killMutex.Unlock() 96 | if fake.KillStub != nil { 97 | return fake.KillStub() 98 | } else { 99 | return fake.killReturns.result1 100 | } 101 | } 102 | 103 | func (fake *FakeRunner) KillCallCount() int { 104 | fake.killMutex.RLock() 105 | defer fake.killMutex.RUnlock() 106 | return len(fake.killArgsForCall) 107 | } 108 | 109 | func (fake *FakeRunner) KillReturns(result1 error) { 110 | fake.KillStub = nil 111 | fake.killReturns = struct { 112 | result1 error 113 | }{result1} 114 | } 115 | 116 | var _ commandrunner.Runner = new(FakeRunner) 117 | -------------------------------------------------------------------------------- /commandrunner/runner.go: -------------------------------------------------------------------------------- 1 | package commandrunner 2 | 3 | import ( 4 | "bytes" 5 | "os/exec" 6 | ) 7 | 8 | //go:generate counterfeiter . Runner 9 | 10 | type Runner interface { 11 | Run(outbuf, errbuff *bytes.Buffer) error 12 | Wait() error 13 | Kill() error 14 | } 15 | 16 | type runner struct { 17 | scriptPath string 18 | cmdErrChan chan error 19 | cmd *exec.Cmd 20 | } 21 | 22 | func NewRunner(scriptPath string) Runner { 23 | return &runner{ 24 | scriptPath: scriptPath, 25 | cmdErrChan: make(chan error, 1), 26 | } 27 | } 28 | 29 | // Wait blocks on the result of the command. It should be called after Run(). 30 | func (r *runner) Wait() error { 31 | return <-r.cmdErrChan 32 | } 33 | 34 | // Run is non-blocking. Users should call Wait to get the result. 35 | func (r *runner) Run(outbuf, errbuf *bytes.Buffer) error { 36 | r.cmd = exec.Command("/bin/sh", "-c", r.scriptPath) 37 | 38 | r.cmd.Stdout = outbuf 39 | r.cmd.Stderr = errbuf 40 | 41 | err := r.cmd.Start() 42 | // Untested because we can't force sh to fail in test 43 | if err != nil { 44 | return err 45 | } 46 | 47 | go func() { 48 | r.cmdErrChan <- r.cmd.Wait() 49 | }() 50 | 51 | return nil 52 | } 53 | 54 | func (r *runner) Kill() error { 55 | return r.cmd.Process.Kill() 56 | } 57 | -------------------------------------------------------------------------------- /commandrunner/runner_test.go: -------------------------------------------------------------------------------- 1 | package commandrunner_test 2 | 3 | import ( 4 | "bytes" 5 | "os/exec" 6 | 7 | "code.cloudfoundry.org/route-registrar/commandrunner" 8 | . "github.com/onsi/ginkgo/v2" 9 | . "github.com/onsi/gomega" 10 | "github.com/onsi/gomega/gexec" 11 | 12 | "os" 13 | "path/filepath" 14 | ) 15 | 16 | const ( 17 | golangExecutable = ` 18 | package main 19 | 20 | import "fmt" 21 | 22 | func main() { 23 | fmt.Println("Hello from a binary") 24 | }` 25 | ) 26 | 27 | var _ = Describe("CommandRunner", func() { 28 | var ( 29 | executable string 30 | tmpDir string 31 | tmpGoPkgPath string 32 | outbuf bytes.Buffer 33 | errbuf bytes.Buffer 34 | r commandrunner.Runner 35 | ) 36 | 37 | BeforeEach(func() { 38 | var err error 39 | tmpDir, err = os.MkdirTemp("", "route-registrar-commandrunner-test") 40 | Expect(err).NotTo(HaveOccurred()) 41 | 42 | executable = filepath.Join(tmpDir, "healthchecker.sh") 43 | scriptText := "echo 'my-stdout'; >&2 echo 'my-stderr'; exit 0\n" 44 | 45 | err = os.WriteFile(executable, []byte(scriptText), os.ModePerm) 46 | Expect(err).NotTo(HaveOccurred()) 47 | 48 | cwd, err := os.Getwd() 49 | Expect(err).NotTo(HaveOccurred()) 50 | 51 | err = os.Chdir(tmpDir) 52 | Expect(err).NotTo(HaveOccurred()) 53 | 54 | _, err = exec.Command("go", "mod", "init", "foo").CombinedOutput() 55 | Expect(err).NotTo(HaveOccurred()) 56 | 57 | err = os.Chdir(cwd) 58 | Expect(err).NotTo(HaveOccurred()) 59 | 60 | tmpGoPkgPath, err = os.MkdirTemp(tmpDir, "tmp-foo") 61 | Expect(err).NotTo(HaveOccurred()) 62 | 63 | outbuf = bytes.Buffer{} 64 | errbuf = bytes.Buffer{} 65 | }) 66 | 67 | AfterEach(func() { 68 | err := os.RemoveAll(tmpDir) 69 | Expect(err).NotTo(HaveOccurred()) 70 | }) 71 | 72 | Describe("Run", func() { 73 | JustBeforeEach(func() { 74 | r = commandrunner.NewRunner(executable) 75 | }) 76 | 77 | It("captures stdout and stderr", func() { 78 | err := r.Run(&outbuf, &errbuf) 79 | Expect(err).NotTo(HaveOccurred()) 80 | err = r.Wait() 81 | Expect(err).NotTo(HaveOccurred()) 82 | 83 | Expect(outbuf.String()).Should(ContainSubstring("my-stdout")) 84 | Expect(errbuf.String()).Should(ContainSubstring("my-stderr")) 85 | }) 86 | 87 | It("runs the command in the background", func() { 88 | err := r.Run(&outbuf, &errbuf) 89 | Expect(err).NotTo(HaveOccurred()) 90 | 91 | err = r.Wait() 92 | Expect(err).NotTo(HaveOccurred()) 93 | }) 94 | 95 | Context("when the script exits with a non-zero code", func() { 96 | BeforeEach(func() { 97 | scriptText := "exit 1\n" 98 | os.WriteFile(executable, []byte(scriptText), os.ModePerm) 99 | }) 100 | 101 | It("places the error on the error chan", func() { 102 | err := r.Run(&outbuf, &errbuf) 103 | Expect(err).NotTo(HaveOccurred()) 104 | 105 | err = r.Wait() 106 | Expect(err).To(HaveOccurred()) 107 | }) 108 | }) 109 | 110 | Describe("running a binary", func() { 111 | BeforeEach(func() { 112 | executableFilepath := filepath.Join(tmpGoPkgPath, "main.go") 113 | err := os.WriteFile(executableFilepath, []byte(golangExecutable), os.ModePerm) 114 | Expect(err).NotTo(HaveOccurred()) 115 | 116 | executable, err = gexec.Build(executableFilepath) 117 | Expect(err).ShouldNot(HaveOccurred()) 118 | }) 119 | 120 | It("runs a binary without error", func() { 121 | err := r.Run(&outbuf, &errbuf) 122 | Expect(err).NotTo(HaveOccurred()) 123 | 124 | err = r.Wait() 125 | Expect(err).NotTo(HaveOccurred()) 126 | 127 | Expect(outbuf.String()).To(Equal("Hello from a binary\n")) 128 | }) 129 | }) 130 | 131 | Describe("running a script with a shebang", func() { 132 | BeforeEach(func() { 133 | executable = filepath.Join(tmpDir, "healthchecker.sh") 134 | scriptText := "#!/bin/sh\necho 'my-stdout'; >&2 echo 'my-stderr'; exit 0\n" 135 | 136 | err := os.WriteFile(executable, []byte(scriptText), os.ModePerm) 137 | Expect(err).NotTo(HaveOccurred()) 138 | }) 139 | 140 | It("runs the script without error", func() { 141 | err := r.Run(&outbuf, &errbuf) 142 | Expect(err).NotTo(HaveOccurred()) 143 | 144 | err = r.Wait() 145 | Expect(err).NotTo(HaveOccurred()) 146 | 147 | Expect(outbuf.String()).To(Equal("my-stdout\n")) 148 | }) 149 | }) 150 | }) 151 | 152 | Describe("Kill", func() { 153 | BeforeEach(func() { 154 | r = commandrunner.NewRunner(executable) 155 | }) 156 | Context("when the kill succeeds", func() { 157 | BeforeEach(func() { 158 | scriptText := "sleep 10; exit 0\n" 159 | os.WriteFile(executable, []byte(scriptText), os.ModePerm) 160 | 161 | var outbuf, errbuf bytes.Buffer 162 | r.Run(&outbuf, &errbuf) 163 | }) 164 | 165 | It("returns no error", func() { 166 | err := r.Kill() 167 | Expect(err).NotTo(HaveOccurred()) 168 | }) 169 | }) 170 | 171 | Context("when the kill does not succeed", func() { 172 | BeforeEach(func() { 173 | scriptText := "exit 0\n" 174 | os.WriteFile(executable, []byte(scriptText), os.ModePerm) 175 | 176 | var outbuf, errbuf bytes.Buffer 177 | r.Run(&outbuf, &errbuf) 178 | 179 | err := r.Wait() 180 | Expect(err).NotTo(HaveOccurred()) 181 | }) 182 | 183 | It("places an error on the errChan", func() { 184 | err := r.Kill() 185 | Expect(err).To(HaveOccurred()) 186 | }) 187 | }) 188 | }) 189 | }) 190 | -------------------------------------------------------------------------------- /config/assets/ports-negative.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: some-route-name 3 | host: some-host 4 | type: tcp 5 | port: 8080 6 | tls_port: -1 7 | external_port: 61445 8 | tags: 9 | optional_tag_field: some_tag_value 10 | another_tag_field: some_other_value 11 | uris: 12 | - some_uri1 13 | - some_uri2 14 | server_cert_domain_san: some.service.internal 15 | route_service_url: https://route-service.example.com 16 | router_group: some-router-group 17 | protocol: http1 18 | registration_interval: 10s 19 | health_check: 20 | name: health-check-name 21 | script_path: /path/to/check/executable 22 | timeout: 5s 23 | options: 24 | loadbalancing: least-connection 25 | 26 | -------------------------------------------------------------------------------- /config/assets/ports-too-high.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: some-route-name 3 | host: some-host 4 | type: tcp 5 | port: 65536 6 | tls_port: 8443 7 | external_port: 61445 8 | tags: 9 | optional_tag_field: some_tag_value 10 | another_tag_field: some_other_value 11 | uris: 12 | - some_uri1 13 | - some_uri2 14 | server_cert_domain_san: some.service.internal 15 | route_service_url: https://route-service.example.com 16 | router_group: some-router-group 17 | protocol: http1 18 | registration_interval: 10s 19 | health_check: 20 | name: health-check-name 21 | script_path: /path/to/check/executable 22 | timeout: 5s 23 | options: 24 | loadbalancing: least-connection 25 | 26 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/url" 7 | "os" 8 | "strconv" 9 | "time" 10 | 11 | "code.cloudfoundry.org/multierror" 12 | ) 13 | 14 | type MessageBusServerSchema struct { 15 | Host string `json:"host"` 16 | User string `json:"user"` 17 | Password string `json:"password"` 18 | } 19 | 20 | type RoutingAPISchema struct { 21 | APIURL string `json:"api_url"` 22 | OAuthURL string `json:"oauth_url"` 23 | ClientID string `json:"client_id"` 24 | ClientSecret string `json:"client_secret"` 25 | CACerts string `json:"ca_certs"` 26 | SkipSSLValidation bool `json:"skip_ssl_validation"` 27 | 28 | ClientCertificatePath string `json:"client_cert_path"` 29 | ClientPrivateKeyPath string `json:"client_private_key_path"` 30 | ServerCACertificatePath string `json:"server_ca_cert_path"` 31 | MaxTTL string `json:"max_ttl,omitempty"` 32 | } 33 | 34 | type HealthCheckSchema struct { 35 | Name string `json:"name" yaml:"name"` 36 | ScriptPath string `json:"script_path" yaml:"script_path"` 37 | Timeout string `json:"timeout" yaml:"timeout"` 38 | } 39 | 40 | type ConfigSchema struct { 41 | MessageBusServers []MessageBusServerSchema `json:"message_bus_servers"` 42 | RoutingAPI RoutingAPISchema `json:"routing_api"` 43 | Routes []RouteSchema `json:"routes"` 44 | DynamicConfigGlobs []string `json:"dynamic_config_globs"` 45 | NATSmTLSConfig ClientTLSConfigSchema `json:"nats_mtls_config"` 46 | Host string `json:"host"` 47 | AvailabilityZone string `json:"availability_zone"` 48 | UnregistrationMessageLimit *int `json:"unregistration_message_limit,omitempty"` 49 | } 50 | 51 | type RouteSchema struct { 52 | Type string `json:"type" yaml:"type"` 53 | Name string `json:"name" yaml:"name"` 54 | Host string `json:"host" yaml:"host"` 55 | Port *uint16 `json:"port" yaml:"port"` 56 | Protocol string `json:"protocol" yaml:"protocol"` 57 | SniPort *uint16 `json:"sni_port" yaml:"sni_port"` 58 | TLSPort *uint16 `json:"tls_port" yaml:"tls_port"` 59 | Tags map[string]string `json:"tags" yaml:"tags"` 60 | URIs []string `json:"uris" yaml:"uris"` 61 | RouterGroup string `json:"router_group" yaml:"router_group"` 62 | ExternalPort *uint16 `json:"external_port,omitempty" yaml:"external_port,omitempty"` 63 | RouteServiceUrl string `json:"route_service_url" yaml:"route_service_url"` 64 | RegistrationInterval string `json:"registration_interval,omitempty" yaml:"registration_interval,omitempty"` 65 | HealthCheck *HealthCheckSchema `json:"health_check,omitempty" yaml:"health_check,omitempty"` 66 | ServerCertDomainSAN string `json:"server_cert_domain_san,omitempty" yaml:"server_cert_domain_san,omitempty"` 67 | SniRoutableSan string `json:"sni_routable_san,omitempty" yaml:"sni_routable_san,omitempty"` 68 | Options *Options `json:"options,omitempty" yaml:"options,omitempty"` 69 | } 70 | 71 | type Options struct { 72 | LoadBalancingAlgorithm LoadBalancingAlgorithm `json:"loadbalancing,omitempty" yaml:"loadbalancing,omitempty"` 73 | } 74 | 75 | type LoadBalancingAlgorithm string 76 | 77 | var supportedLoadBalancingAlgorithms = []LoadBalancingAlgorithm{RoundRobin, LeastConns} 78 | 79 | const ( 80 | RoundRobin LoadBalancingAlgorithm = "round-robin" 81 | LeastConns LoadBalancingAlgorithm = "least-connection" 82 | ) 83 | 84 | type ClientTLSConfigSchema struct { 85 | Enabled bool `json:"enabled"` 86 | CertPath string `json:"cert_path"` 87 | KeyPath string `json:"key_path"` 88 | CAPath string `json:"ca_path"` 89 | } 90 | 91 | type MessageBusServer struct { 92 | Host string 93 | User string 94 | Password string 95 | } 96 | 97 | type RoutingAPI struct { 98 | APIURL string 99 | OAuthURL string 100 | ClientID string 101 | ClientSecret string 102 | CACerts string 103 | SkipSSLValidation bool 104 | 105 | ClientCertificatePath string 106 | ClientPrivateKeyPath string 107 | ServerCACertificatePath string 108 | 109 | MaxTTL time.Duration 110 | } 111 | 112 | type HealthCheck struct { 113 | Name string 114 | ScriptPath string 115 | Timeout time.Duration 116 | } 117 | 118 | type Config struct { 119 | MessageBusServers []MessageBusServer 120 | RoutingAPI RoutingAPI 121 | Routes []Route 122 | DynamicConfigGlobs []string 123 | NATSmTLSConfig ClientTLSConfig 124 | Host string 125 | AvailabilityZone string `json:"availability_zone"` 126 | UnregistrationMessageLimit int 127 | } 128 | 129 | type ClientTLSConfig struct { 130 | Enabled bool 131 | CertPath string 132 | KeyPath string 133 | CAPath string 134 | } 135 | 136 | type Route struct { 137 | Type string 138 | Name string 139 | Port *uint16 140 | Protocol string 141 | TLSPort *uint16 142 | Tags map[string]string 143 | URIs []string 144 | RouterGroup string 145 | Host string 146 | ExternalPort *uint16 147 | RouteServiceUrl string 148 | RegistrationInterval time.Duration 149 | HealthCheck *HealthCheck 150 | ServerCertDomainSAN string 151 | Options *Options 152 | } 153 | 154 | func NewConfigSchemaFromFile(configFile string) (ConfigSchema, error) { 155 | var config ConfigSchema 156 | 157 | c, err := os.ReadFile(configFile) 158 | if err != nil { 159 | return ConfigSchema{}, err 160 | } 161 | 162 | err = json.Unmarshal(c, &config) 163 | if err != nil { 164 | return ConfigSchema{}, err 165 | } 166 | 167 | return config, nil 168 | } 169 | 170 | func (c ConfigSchema) ParseSchemaAndSetDefaultsToConfig() (*Config, error) { 171 | errors := multierror.NewMultiError("config") 172 | 173 | if c.UnregistrationMessageLimit == nil { 174 | defaultUnregistrationLimit := 5 175 | c.UnregistrationMessageLimit = &defaultUnregistrationLimit 176 | } 177 | 178 | if *c.UnregistrationMessageLimit <= 0 { 179 | errors.Add(fmt.Errorf("unregistration_message_limit must be a positive integer")) 180 | } 181 | 182 | tcp_routes := 0 183 | 184 | routes := []Route{} 185 | for index, r := range c.Routes { 186 | route, err := RouteFromSchema(r, index, c.Host) 187 | if err != nil { 188 | errors.Add(err) 189 | continue 190 | } 191 | 192 | if route.Type == "tcp" { 193 | tcp_routes++ 194 | } 195 | 196 | routes = append(routes, *route) 197 | } 198 | 199 | messageBusServers, err := messageBusServersFromSchema(c.MessageBusServers) 200 | if err != nil && (len(routes)-tcp_routes > 0) { 201 | errors.Add(err) 202 | } 203 | 204 | routingAPI, err := routingAPIFromSchema(c.RoutingAPI) 205 | if err != nil && tcp_routes > 0 { 206 | errors.Add(err) 207 | } 208 | 209 | if errors.Length() > 0 { 210 | return nil, errors 211 | } 212 | 213 | natsTLSConfig := clientTLSConfigFromSchema(c.NATSmTLSConfig) 214 | 215 | config := Config{ 216 | Host: c.Host, 217 | AvailabilityZone: c.AvailabilityZone, 218 | UnregistrationMessageLimit: *c.UnregistrationMessageLimit, 219 | MessageBusServers: messageBusServers, 220 | Routes: routes, 221 | DynamicConfigGlobs: c.DynamicConfigGlobs, 222 | NATSmTLSConfig: natsTLSConfig, 223 | } 224 | if routingAPI != nil { 225 | config.RoutingAPI = *routingAPI 226 | } 227 | 228 | return &config, nil 229 | } 230 | 231 | func nameOrIndex(r RouteSchema, index int) string { 232 | if r.Name != "" { 233 | return fmt.Sprintf(`"%s"`, r.Name) 234 | } 235 | 236 | return strconv.Itoa(index) 237 | } 238 | 239 | func parseRegistrationInterval(registrationInterval string) (time.Duration, error) { 240 | var duration time.Duration 241 | 242 | if registrationInterval == "" { 243 | return duration, fmt.Errorf("no registration_interval") 244 | } 245 | 246 | var err error 247 | duration, err = time.ParseDuration(registrationInterval) 248 | if err != nil { 249 | return duration, fmt.Errorf("invalid registration_interval: %s", err.Error()) 250 | } 251 | 252 | if duration <= 0 { 253 | return duration, fmt.Errorf("invalid registration_interval: interval must be greater than 0") 254 | } 255 | 256 | return duration, nil 257 | } 258 | 259 | func RouteFromSchema(r RouteSchema, index int, host string) (*Route, error) { 260 | errors := multierror.NewMultiError(fmt.Sprintf("route %s", nameOrIndex(r, index))) 261 | 262 | if r.Type != "tcp" && r.Type != "sni" && r.Name == "" { 263 | errors.Add(fmt.Errorf("no name")) 264 | } 265 | 266 | if r.Host == "" { 267 | if host == "" { 268 | errors.Add(fmt.Errorf("no host")) 269 | } else { 270 | r.Host = host 271 | } 272 | } 273 | 274 | if r.Port == nil && r.TLSPort == nil && r.SniPort == nil { 275 | errors.Add(fmt.Errorf("no port")) 276 | } 277 | if r.Port != nil && *r.Port <= 0 { 278 | errors.Add(fmt.Errorf("invalid port: %d", *r.Port)) 279 | } 280 | if r.TLSPort != nil && *r.TLSPort <= 0 { 281 | errors.Add(fmt.Errorf("invalid tls_port: %d", *r.TLSPort)) 282 | } 283 | 284 | if r.Type != "tcp" && r.Type != "sni" { 285 | if len(r.URIs) == 0 { 286 | errors.Add(fmt.Errorf("no URIs")) 287 | } 288 | 289 | for _, u := range r.URIs { 290 | if u == "" { 291 | errors.Add(fmt.Errorf("empty URIs")) 292 | break 293 | } 294 | } 295 | 296 | _, err := url.Parse(r.RouteServiceUrl) 297 | if err != nil { 298 | errors.Add(err) 299 | } 300 | } else { 301 | if r.RouterGroup == "" { 302 | errors.Add(fmt.Errorf("missing router_group")) 303 | } 304 | if r.ExternalPort != nil && *r.ExternalPort <= 0 { 305 | errors.Add(fmt.Errorf("invalid port: %d", *r.ExternalPort)) 306 | } 307 | } 308 | 309 | if r.Protocol != "" && r.Protocol != "http1" && r.Protocol != "http2" { 310 | errors.Add(fmt.Errorf("unknown protocol: %s", r.Protocol)) 311 | } 312 | 313 | if r.Options != nil { 314 | if r.Options.LoadBalancingAlgorithm != "" { 315 | err := validatePerRouteLoadBalancingAlgorithm(r.Options.LoadBalancingAlgorithm) 316 | if err != nil { 317 | errors.Add(err) 318 | } 319 | } 320 | } 321 | 322 | registrationInterval, err := parseRegistrationInterval(r.RegistrationInterval) 323 | if err != nil { 324 | errors.Add(err) 325 | } 326 | 327 | var healthCheck *HealthCheck 328 | if r.HealthCheck != nil { 329 | healthCheck, err = healthCheckFromSchema(r.HealthCheck, registrationInterval) 330 | if err != nil { 331 | errors.Add(err) 332 | } 333 | } 334 | 335 | if errors.Length() > 0 { 336 | return nil, errors 337 | } 338 | 339 | route := Route{ 340 | Type: r.Type, 341 | Name: r.Name, 342 | Host: r.Host, 343 | Port: r.Port, 344 | Protocol: r.Protocol, 345 | TLSPort: r.TLSPort, 346 | Tags: r.Tags, 347 | URIs: r.URIs, 348 | RouterGroup: r.RouterGroup, 349 | ExternalPort: r.ExternalPort, 350 | RouteServiceUrl: r.RouteServiceUrl, 351 | ServerCertDomainSAN: r.ServerCertDomainSAN, 352 | RegistrationInterval: registrationInterval, 353 | HealthCheck: healthCheck, 354 | Options: r.Options, 355 | } 356 | 357 | if r.Type == "sni" { 358 | route.Port = r.SniPort 359 | route.ServerCertDomainSAN = r.SniRoutableSan 360 | route.Type = "tcp" 361 | } 362 | return &route, nil 363 | } 364 | 365 | func validatePerRouteLoadBalancingAlgorithm(loadBalancingAlgo LoadBalancingAlgorithm) error { 366 | for _, lbAlgo := range supportedLoadBalancingAlgorithms { 367 | if loadBalancingAlgo == lbAlgo { 368 | return nil 369 | } 370 | } 371 | return fmt.Errorf("unknown load balancing algorithm: %s. Allowed values: %s", loadBalancingAlgo, supportedLoadBalancingAlgorithms) 372 | } 373 | 374 | func healthCheckFromSchema( 375 | healthCheckSchema *HealthCheckSchema, 376 | registrationInterval time.Duration, 377 | ) (*HealthCheck, error) { 378 | errors := multierror.NewMultiError("healthcheck") 379 | 380 | healthCheck := &HealthCheck{ 381 | Name: healthCheckSchema.Name, 382 | ScriptPath: healthCheckSchema.ScriptPath, 383 | } 384 | 385 | if healthCheck.Name == "" { 386 | errors.Add(fmt.Errorf("no name")) 387 | } 388 | 389 | if healthCheck.ScriptPath == "" { 390 | errors.Add(fmt.Errorf("no script_path")) 391 | } 392 | 393 | if healthCheckSchema.Timeout == "" && registrationInterval > 0 { 394 | if errors.Length() > 0 { 395 | return nil, errors 396 | } 397 | 398 | healthCheck.Timeout = registrationInterval / 2 399 | return healthCheck, nil 400 | } 401 | 402 | var err error 403 | healthCheck.Timeout, err = time.ParseDuration(healthCheckSchema.Timeout) 404 | if err != nil { 405 | errors.Add(fmt.Errorf("invalid healthcheck timeout: %s", err.Error())) 406 | return nil, errors 407 | } 408 | 409 | if healthCheck.Timeout <= 0 { 410 | errors.Add(fmt.Errorf("invalid healthcheck timeout: %s", healthCheck.Timeout)) 411 | return nil, errors 412 | } 413 | 414 | if healthCheck.Timeout >= registrationInterval && registrationInterval > 0 { 415 | errors.Add(fmt.Errorf( 416 | "invalid healthcheck timeout: %v must be less than the registration interval: %v", 417 | healthCheck.Timeout, 418 | registrationInterval, 419 | )) 420 | return nil, errors 421 | } 422 | 423 | if errors.Length() > 0 { 424 | return nil, errors 425 | } 426 | 427 | return healthCheck, nil 428 | } 429 | 430 | func messageBusServersFromSchema(servers []MessageBusServerSchema) ([]MessageBusServer, error) { 431 | messageBusServers := []MessageBusServer{} 432 | if len(servers) < 1 { 433 | return nil, fmt.Errorf("message_bus_servers must have at least one entry") 434 | } 435 | 436 | for _, m := range servers { 437 | messageBusServers = append( 438 | messageBusServers, 439 | MessageBusServer(m), 440 | ) 441 | } 442 | 443 | return messageBusServers, nil 444 | } 445 | 446 | func parseMaxTTL(max_ttl string) time.Duration { 447 | ttl, _ := time.ParseDuration(max_ttl) 448 | if ttl <= 0 { 449 | ttl = 2 * time.Minute 450 | } 451 | 452 | return ttl 453 | } 454 | 455 | func routingAPIFromSchema(api RoutingAPISchema) (*RoutingAPI, error) { 456 | if api.APIURL == "" { 457 | return nil, fmt.Errorf("routing_api must have an api_url") 458 | } 459 | 460 | apiURL, err := url.Parse(api.APIURL) 461 | 462 | if err != nil { 463 | return nil, fmt.Errorf("routing_api must a vaid URL") 464 | } 465 | 466 | if api.OAuthURL == "" { 467 | return nil, fmt.Errorf("routing_api must have an oauth_url") 468 | } 469 | if api.ClientID == "" { 470 | return nil, fmt.Errorf("routing_api must have a client_id") 471 | } 472 | if api.ClientSecret == "" { 473 | return nil, fmt.Errorf("routing_api must have a client_secret") 474 | } 475 | 476 | if apiURL.Scheme == "https" { 477 | if api.ClientCertificatePath == "" { 478 | return nil, fmt.Errorf("routing_api must have a client_certificate_path") 479 | } 480 | if api.ClientPrivateKeyPath == "" { 481 | return nil, fmt.Errorf("routing_api must have a client_private_key_path") 482 | } 483 | if api.ServerCACertificatePath == "" { 484 | return nil, fmt.Errorf("routing_api must have a server_ca_cert_path") 485 | } 486 | } 487 | 488 | maxTTL := parseMaxTTL(api.MaxTTL) 489 | 490 | return &RoutingAPI{ 491 | APIURL: api.APIURL, 492 | OAuthURL: api.OAuthURL, 493 | ClientID: api.ClientID, 494 | ClientSecret: api.ClientSecret, 495 | CACerts: api.CACerts, 496 | SkipSSLValidation: api.SkipSSLValidation, 497 | ClientCertificatePath: api.ClientCertificatePath, 498 | ClientPrivateKeyPath: api.ClientPrivateKeyPath, 499 | ServerCACertificatePath: api.ServerCACertificatePath, 500 | MaxTTL: maxTTL, 501 | }, nil 502 | } 503 | 504 | func clientTLSConfigFromSchema(clientTLSConfigSchema ClientTLSConfigSchema) ClientTLSConfig { 505 | return ClientTLSConfig(clientTLSConfigSchema) 506 | } 507 | -------------------------------------------------------------------------------- /config/config_suite_test.go: -------------------------------------------------------------------------------- 1 | package config_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo/v2" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestConfig(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Config Suite") 13 | } 14 | -------------------------------------------------------------------------------- /docs/01-usage.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Usage 3 | expires_at: never 4 | tags: [routing-release,route-registrar] 5 | --- 6 | 7 | 8 | 9 | * [Usage](#usage) 10 | * [Configuration](#configuration) 11 | * [SNI Routing](#sni-routing) 12 | * [Health check](#health-check) 13 | * [Options](#options) 14 | 15 | 16 | 17 | # Usage 18 | 19 | ## Configuration 20 | 21 | The route-registrar expects a configuration json file like the one below: 22 | ```json 23 | { 24 | "message_bus_servers": [ 25 | { 26 | "host": "NATS_SERVER_HOST:PORT", 27 | "user": "NATS_SERVER_USERNAME", 28 | "password": "NATS_SERVER_PASSWORD" 29 | } 30 | ], 31 | "host": "HOSTNAME_OR_IP_OF_ROUTE_DESTINATION", 32 | "routes": [ 33 | { 34 | "name": "SOME_ROUTE_NAME", 35 | "tls_port": "TLS_PORT_OF_ROUTE_DESTINATION", 36 | "tags": { 37 | "optional_tag_field": "some_tag_value", 38 | "another_tag_field": "some_other_value" 39 | }, 40 | "uris": [ 41 | "some_source_uri_for_the_router_to_map_to_the_destination", 42 | "some_other_source_uri_for_the_router_to_map_to_the_destination" 43 | ], 44 | "server_cert_domain_san": "some.service.internal", 45 | "route_service_url": "https://route-service.example.com", 46 | "registration_interval": "REGISTRATION_INTERVAL", 47 | "health_check": { 48 | "name": "HEALTH_CHECK_NAME", 49 | "script_path": "/path/to/check/executable", 50 | "timeout": "HEALTH_CHECK_TIMEOUT" 51 | }, 52 | "options": { 53 | "loadbalancing": "least-connection" 54 | } 55 | } 56 | ] 57 | } 58 | ``` 59 | 60 | - `message_bus_servers` is an array of data with location and credentials for 61 | the NATS servers; route-registrar currently registers and deregisters routes 62 | via NATS messages. `message_bus_servers.host` must include both hostname and 63 | port; e.g. `host: 10.0.32.11:4222` 64 | - `host` is the destination hostname or IP for the routes being registered. To 65 | Gorouter, these are backends. 66 | - `routes` is required and is an array of hashes. For each route collection: 67 | - `name` must be provided and be a string 68 | - `port` or `tls_port` are for the destination host (backend). At least one 69 | must be provided and must be a positive integer > 1. 70 | - `server_cert_domain_san` is the SAN on the destination host's TLS 71 | certificate. Required when `tls_port` is provided. 72 | - `uris` are the routes being registered for the destination `host`. Must be 73 | provided and be a non empty array of strings. All URIs in a given route 74 | collection will be mapped to the same host and port. 75 | - `registration_interval` is the interval for which routes are registered 76 | with NATS. Must be provided and be a string with units (e.g. "20s"). It 77 | must parse to a positive time duration e.g. "-5s" is not permitted. 78 | - `route_service_url` is optional. When provided, Gorouter will proxy 79 | requests received for the `uris` above to this address. 80 | - `health_check` is optional and explained in more detail below. 81 | - `options` is optional and explained in more detail below. 82 | 83 | Run route-registrar binaries using the following command 84 | 85 | ```bash 86 | route-registrar -configPath FILE_PATH_TO_CONFIG_JSON -pidfile PATH_TO_PIDFILE 87 | ``` 88 | 89 | ## SNI Routing 90 | The route registrar can be used to setup SNI routing. This is an example route json: 91 | ``` 92 | { 93 | "routes": [ 94 | { 95 | "type": "sni", 96 | "external_port": "TLS_PORT_OF_ROUTE_SOURCE", 97 | "name": "SOME_ROUTE_NAME", 98 | "sni_port": "TLS_PORT_OF_ROUTE_DESTINATION", 99 | } 100 | ] 101 | } 102 | ``` 103 | 104 | ## Health check 105 | 106 | If the `health_check` is not configured for a route collection, the routes are continually registered according to the `registration_interval`. 107 | 108 | If the `health_check` is configured, then, at the `registration_interval`, 109 | the executable provided at `health_check.script_path` is invoked. 110 | The following applies: 111 | - if the executable exits with success, the routes are registered. 112 | - if the executable exits with error, the routes are deregistered. 113 | - if `health_check.timeout` is configured, it must parse to a positive time 114 | duration (similar to `registration_interval`), and the executable must exit 115 | within the timeout. If the executable does not terminate within the timeout, 116 | it is forcibly terminated (with `SIGKILL`) and the routes are deregistered. 117 | - if `health_check.timeout` is not configured, the executable must exit within 118 | half the `registration_interval`. If the executable does not terminate within 119 | the timeout, it is forcibly terminated (with `SIGKILL`) and the routes are 120 | deregistered. 121 | 122 | ## Options 123 | Custom per-route options can be defined and applied to specific routes exclusively. 124 | - `loadbalancing` enables the selection of a load balancing algorithm for routing incoming requests to the backend. It is possible to choose between `round-robin` and `least-connection`. In cases where this option is not specified, the algorithm [defined by the platform operator](https://github.com/cloudfoundry/routing-release/blob/develop/jobs/gorouter/spec#L101) is applied. 125 | 126 | -------------------------------------------------------------------------------- /example_config/example.json: -------------------------------------------------------------------------------- 1 | { 2 | "host": "127.0.0.1", 3 | "routes": [ 4 | { 5 | "name": "route-0", 6 | "port": 3000, 7 | "uris": [ 8 | "my-app.my-domain.com" 9 | ], 10 | "registration_interval": "20s" 11 | }, 12 | { 13 | "name": "route-1", 14 | "tls_port": 3001, 15 | "protocol": "http1", 16 | "uris": [ 17 | "my-other-app.my-domain.com" 18 | ], 19 | "options": { 20 | "loadbalancing": "least-connection" 21 | }, 22 | "registration_interval": "10s", 23 | "server_cert_domain_san": "my.internal.cert" 24 | }, 25 | { 26 | "name": "route-2", 27 | "host": "128.0.0.1", 28 | "port": 3000, 29 | "tls_port": 3001, 30 | "protocol": "http2", 31 | "uris": [ 32 | "my-other-app.my-domain.com" 33 | ], 34 | "registration_interval": "10s", 35 | "server_cert_domain_san": "my.internal.cert" 36 | }, 37 | { 38 | "type": "tcp", 39 | "port": 15000, 40 | "host": "127.0.1.1", 41 | "external_port": 5000, 42 | "router_group": "some-router-group", 43 | "registration_interval": "10s" 44 | }, 45 | { 46 | "type": "sni", 47 | "sni_port": 17000, 48 | "external_port": 16000, 49 | "sni_routable_san": "sni.internal", 50 | "router_group": "some-router-group", 51 | "registration_interval": "10s" 52 | } 53 | ], 54 | "message_bus_servers": [ 55 | { 56 | "host": "some-host", 57 | "user": "some-user", 58 | "password": "some-password" 59 | }, 60 | { 61 | "host": "another-host", 62 | "user": "another-user", 63 | "password": "another-password" 64 | } 65 | ], 66 | "routing_api": { 67 | "api_url": "http://api.example.com", 68 | "oauth_url": "https://uaa.somewhere", 69 | "client_id": "clientid", 70 | "client_secret": "secret", 71 | "max_ttl": "30s" 72 | }, 73 | "nats_mtls_config": { 74 | "enabled": true, 75 | "cert_path": "cert-path", 76 | "key_path": "key-path", 77 | "ca_path": "ca-path" 78 | }, 79 | "dynamic_config_globs": ["/some/config/*/path1", "/some/config/*/path2"], 80 | "availability_zone": "some-zone", 81 | "unregistration_message_limit": 5 82 | } 83 | -------------------------------------------------------------------------------- /example_config/route.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: some-route-name 3 | host: some-host 4 | type: tcp 5 | port: 8080 6 | tls_port: 8443 7 | external_port: 61445 8 | tags: 9 | optional_tag_field: some_tag_value 10 | another_tag_field: some_other_value 11 | uris: 12 | - some_uri1 13 | - some_uri2 14 | server_cert_domain_san: some.service.internal 15 | route_service_url: https://route-service.example.com 16 | router_group: some-router-group 17 | protocol: http1 18 | registration_interval: 10s 19 | health_check: 20 | name: health-check-name 21 | script_path: /path/to/check/executable 22 | timeout: 5s 23 | options: 24 | loadbalancing: least-connection 25 | 26 | -------------------------------------------------------------------------------- /healthchecker/fakes/fake_health_checker.go: -------------------------------------------------------------------------------- 1 | // This file was generated by counterfeiter 2 | package fakes 3 | 4 | import ( 5 | "sync" 6 | "time" 7 | 8 | "code.cloudfoundry.org/route-registrar/commandrunner" 9 | "code.cloudfoundry.org/route-registrar/healthchecker" 10 | ) 11 | 12 | type FakeHealthChecker struct { 13 | CheckStub func(runner commandrunner.Runner, scriptPath string, timeout time.Duration) (bool, error) 14 | checkMutex sync.RWMutex 15 | checkArgsForCall []struct { 16 | runner commandrunner.Runner 17 | scriptPath string 18 | timeout time.Duration 19 | } 20 | checkReturns struct { 21 | result1 bool 22 | result2 error 23 | } 24 | } 25 | 26 | func (fake *FakeHealthChecker) Check(runner commandrunner.Runner, scriptPath string, timeout time.Duration) (bool, error) { 27 | fake.checkMutex.Lock() 28 | fake.checkArgsForCall = append(fake.checkArgsForCall, struct { 29 | runner commandrunner.Runner 30 | scriptPath string 31 | timeout time.Duration 32 | }{runner, scriptPath, timeout}) 33 | fake.checkMutex.Unlock() 34 | if fake.CheckStub != nil { 35 | return fake.CheckStub(runner, scriptPath, timeout) 36 | } else { 37 | return fake.checkReturns.result1, fake.checkReturns.result2 38 | } 39 | } 40 | 41 | func (fake *FakeHealthChecker) CheckCallCount() int { 42 | fake.checkMutex.RLock() 43 | defer fake.checkMutex.RUnlock() 44 | return len(fake.checkArgsForCall) 45 | } 46 | 47 | func (fake *FakeHealthChecker) CheckArgsForCall(i int) (commandrunner.Runner, string, time.Duration) { 48 | fake.checkMutex.RLock() 49 | defer fake.checkMutex.RUnlock() 50 | return fake.checkArgsForCall[i].runner, fake.checkArgsForCall[i].scriptPath, fake.checkArgsForCall[i].timeout 51 | } 52 | 53 | func (fake *FakeHealthChecker) CheckReturns(result1 bool, result2 error) { 54 | fake.CheckStub = nil 55 | fake.checkReturns = struct { 56 | result1 bool 57 | result2 error 58 | }{result1, result2} 59 | } 60 | 61 | var _ healthchecker.HealthChecker = new(FakeHealthChecker) 62 | -------------------------------------------------------------------------------- /healthchecker/healthchecker_suite_test.go: -------------------------------------------------------------------------------- 1 | package healthchecker_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestRoute_register(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | 13 | RunSpecs(t, "Health Checker Suite") 14 | } 15 | -------------------------------------------------------------------------------- /healthchecker/script_checker.go: -------------------------------------------------------------------------------- 1 | package healthchecker 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "os/exec" 7 | "time" 8 | 9 | "code.cloudfoundry.org/lager/v3" 10 | "code.cloudfoundry.org/route-registrar/commandrunner" 11 | ) 12 | 13 | //go:generate counterfeiter . HealthChecker 14 | 15 | type HealthChecker interface { 16 | Check(runner commandrunner.Runner, scriptPath string, timeout time.Duration) (bool, error) 17 | } 18 | 19 | type healthChecker struct { 20 | logger lager.Logger 21 | } 22 | 23 | func NewHealthChecker(logger lager.Logger) HealthChecker { 24 | return &healthChecker{ 25 | logger: logger, 26 | } 27 | } 28 | 29 | func (h healthChecker) Check(runner commandrunner.Runner, scriptPath string, timeout time.Duration) (bool, error) { 30 | h.logger.Info( 31 | "Executing script", 32 | lager.Data{"scriptPath": scriptPath}, 33 | ) 34 | 35 | var outbuf, errbuf bytes.Buffer 36 | err := runner.Run(&outbuf, &errbuf) 37 | if err != nil { 38 | h.logger.Info( 39 | "Error starting script", 40 | lager.Data{ 41 | "script": scriptPath, 42 | "error": err.Error(), 43 | "stdout": outbuf.String(), 44 | "stderr": errbuf.String(), 45 | }, 46 | ) 47 | return false, err 48 | } 49 | 50 | if timeout <= 0 { 51 | err := runner.Wait() 52 | return h.handleOutput(scriptPath, err, outbuf, errbuf) 53 | } 54 | 55 | commandErrChan := make(chan error) 56 | go func() { 57 | commandErrChan <- runner.Wait() 58 | }() 59 | 60 | select { 61 | case <-time.After(timeout): 62 | h.logger.Info( 63 | "Script failed to exit within timeout", 64 | lager.Data{ 65 | "script": scriptPath, 66 | "stdout": outbuf.String(), 67 | "stderr": errbuf.String(), 68 | "timeout": timeout, 69 | }, 70 | ) 71 | err := runner.Kill() 72 | if err != nil { 73 | h.logger.Error("Failed killing script", 74 | err, 75 | lager.Data{ 76 | "script": scriptPath, 77 | }, 78 | ) 79 | } 80 | return false, fmt.Errorf("Script failed to exit within %v", timeout) 81 | 82 | case err := <-commandErrChan: 83 | return h.handleOutput(scriptPath, err, outbuf, errbuf) 84 | } 85 | } 86 | 87 | func (h healthChecker) handleOutput(scriptPath string, err error, outbuf, errbuf bytes.Buffer) (bool, error) { 88 | if err != nil { 89 | h.logger.Info( 90 | "Script exited with error", 91 | lager.Data{ 92 | "script": scriptPath, 93 | "error": err.Error(), 94 | "stdout": outbuf.String(), 95 | "stderr": errbuf.String(), 96 | }, 97 | ) 98 | 99 | // If the script exited non-zero then we do not consider that an error 100 | _, ok := err.(*exec.ExitError) 101 | if ok { 102 | return false, nil 103 | } 104 | 105 | // Untested due to difficulty of reproducing this case under test 106 | // E.g. this path would be encountered for I/O errors between the script 107 | // and the golang parent process which we cannot force in a test. 108 | return false, err 109 | } 110 | 111 | h.logger.Info( 112 | "Script exited without error", 113 | lager.Data{ 114 | "script": scriptPath, 115 | "stdout": outbuf.String(), 116 | "stderr": errbuf.String(), 117 | }, 118 | ) 119 | return true, nil 120 | } 121 | -------------------------------------------------------------------------------- /healthchecker/script_checker_test.go: -------------------------------------------------------------------------------- 1 | package healthchecker_test 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "os" 7 | "os/exec" 8 | "path/filepath" 9 | "time" 10 | 11 | "code.cloudfoundry.org/route-registrar/commandrunner/fakes" 12 | "code.cloudfoundry.org/route-registrar/healthchecker" 13 | . "github.com/onsi/ginkgo/v2" 14 | . "github.com/onsi/gomega" 15 | "github.com/onsi/gomega/gbytes" 16 | 17 | "code.cloudfoundry.org/lager/v3" 18 | "code.cloudfoundry.org/lager/v3/lagertest" 19 | ) 20 | 21 | var _ = Describe("ScriptHealthChecker", func() { 22 | var ( 23 | logger lager.Logger 24 | runner *fakes.FakeRunner 25 | tmpDir string 26 | scriptPath string 27 | timeout time.Duration 28 | 29 | h healthchecker.HealthChecker 30 | ) 31 | 32 | BeforeEach(func() { 33 | var err error 34 | tmpDir, err = os.MkdirTemp(os.TempDir(), "healthchecker-test") 35 | Expect(err).ToNot(HaveOccurred()) 36 | 37 | scriptPath = filepath.Join(tmpDir, "healthchecker.sh") 38 | 39 | logger = lagertest.NewTestLogger("Script healthchecker test") 40 | 41 | runner = new(fakes.FakeRunner) //commandrunner.NewRunner(scriptPath) 42 | runner.RunStub = func(outbuf, errbuf *bytes.Buffer) error { 43 | outbuf.WriteString("my-stdout") 44 | errbuf.WriteString("my-stderr") 45 | return nil 46 | } 47 | 48 | runner.WaitStub = func() error { 49 | return nil 50 | } 51 | 52 | h = healthchecker.NewHealthChecker(logger) 53 | }) 54 | 55 | It("logs stdout and stderr from the runner", func() { 56 | _, _ = h.Check(runner, scriptPath, timeout) 57 | 58 | Expect(logger).Should(gbytes.Say("stderr\":\"my-stderr")) 59 | Expect(logger).Should(gbytes.Say("stdout\":\"my-stdout")) 60 | }) 61 | 62 | Context("When the runner returns no errors", func() { 63 | It("returns true without error", func() { 64 | result, err := h.Check(runner, scriptPath, timeout) 65 | Expect(err).ShouldNot(HaveOccurred()) 66 | Expect(result).To(BeTrue()) 67 | }) 68 | }) 69 | 70 | Context("When the runner returns an error on the execution channel", func() { 71 | BeforeEach(func() { 72 | runner.WaitStub = func() error { 73 | return &exec.ExitError{} 74 | } 75 | }) 76 | 77 | It("returns false without error", func() { 78 | result, err := h.Check(runner, scriptPath, timeout) 79 | Expect(err).ShouldNot(HaveOccurred()) 80 | Expect(result).To(BeFalse()) 81 | }) 82 | }) 83 | 84 | Context("when the runner returns an immediate error", func() { 85 | BeforeEach(func() { 86 | runner.RunStub = func(outbuf, errbuf *bytes.Buffer) error { 87 | return errors.New("BOO") 88 | } 89 | }) 90 | 91 | It("returns error", func() { 92 | _, err := h.Check(runner, scriptPath, timeout) 93 | Expect(err).Should(HaveOccurred()) 94 | }) 95 | }) 96 | 97 | Context("when the timeout is positive", func() { 98 | BeforeEach(func() { 99 | timeout = 50 * time.Millisecond 100 | }) 101 | 102 | Context("when the runner exits within timeout", func() { 103 | BeforeEach(func() { 104 | runner.WaitStub = func() error { 105 | time.Sleep(20 * time.Millisecond) 106 | return nil 107 | } 108 | }) 109 | 110 | It("returns true without error", func() { 111 | result, err := h.Check(runner, scriptPath, timeout) 112 | Expect(err).ShouldNot(HaveOccurred()) 113 | Expect(result).To(BeTrue()) 114 | }) 115 | }) 116 | 117 | Context("when the runner does not exit within the timeout", func() { 118 | BeforeEach(func() { 119 | runner.WaitStub = func() error { 120 | time.Sleep(5 * time.Second) 121 | return nil 122 | } 123 | }) 124 | 125 | It("returns error", func() { 126 | _, err := h.Check(runner, scriptPath, timeout) 127 | Expect(err).Should(HaveOccurred()) 128 | }) 129 | 130 | It("kills the healthcheck process", func() { 131 | h.Check(runner, scriptPath, timeout) 132 | Expect(runner.KillCallCount()).To(Equal(1)) 133 | }) 134 | }) 135 | }) 136 | }) 137 | -------------------------------------------------------------------------------- /integration/init_test.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | 7 | "code.cloudfoundry.org/routing-api/test_helpers" 8 | 9 | . "github.com/onsi/ginkgo/v2" 10 | . "github.com/onsi/gomega" 11 | "github.com/onsi/gomega/gexec" 12 | 13 | "testing" 14 | ) 15 | 16 | const ( 17 | routeRegistrarPackage = "code.cloudfoundry.org/route-registrar/" 18 | ) 19 | 20 | var ( 21 | routeRegistrarBinPath string 22 | pidFile string 23 | configFile string 24 | natsPort uint16 25 | 26 | tempDir string 27 | ) 28 | 29 | func TestIntegration(t *testing.T) { 30 | RegisterFailHandler(Fail) 31 | RunSpecs(t, "Integration Suite") 32 | } 33 | 34 | var _ = SynchronizedBeforeSuite(func() []byte { 35 | path, err := gexec.Build(routeRegistrarPackage, "-race") 36 | Expect(err).ShouldNot(HaveOccurred()) 37 | 38 | return []byte(path) 39 | }, func(data []byte) { 40 | routeRegistrarBinPath = string(data) 41 | 42 | tempDir, err := os.MkdirTemp(os.TempDir(), "route-registrar") 43 | Expect(err).ToNot(HaveOccurred()) 44 | 45 | pidFile = filepath.Join(tempDir, "route-registrar.pid") 46 | 47 | natsPort = test_helpers.NextAvailPort() 48 | 49 | configFile = filepath.Join(tempDir, "registrar_settings.json") 50 | }) 51 | 52 | var _ = SynchronizedAfterSuite(func() { 53 | }, func() { 54 | err := os.RemoveAll(tempDir) 55 | Expect(err).ShouldNot(HaveOccurred()) 56 | 57 | gexec.CleanupBuildArtifacts() 58 | }) 59 | -------------------------------------------------------------------------------- /integration/main_test.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net" 7 | "os" 8 | "os/exec" 9 | "time" 10 | 11 | tls_helpers "code.cloudfoundry.org/cf-routing-test-helpers/tls" 12 | "code.cloudfoundry.org/route-registrar/config" 13 | "code.cloudfoundry.org/route-registrar/messagebus" 14 | "code.cloudfoundry.org/routing-api/test_helpers" 15 | "code.cloudfoundry.org/tlsconfig" 16 | "github.com/nats-io/nats.go" 17 | "github.com/onsi/gomega/gbytes" 18 | "github.com/onsi/gomega/gexec" 19 | 20 | . "github.com/onsi/ginkgo/v2" 21 | . "github.com/onsi/gomega" 22 | ) 23 | 24 | var _ = Describe("Main", func() { 25 | var ( 26 | natsCmd *exec.Cmd 27 | testSpyClient *nats.Conn 28 | ) 29 | 30 | BeforeEach(func() { 31 | natsUsername := "nats" 32 | natsPassword := "nats" 33 | natsHost := "127.0.0.1" 34 | 35 | rootConfig := initConfig() 36 | writeConfig(rootConfig) 37 | 38 | natsServer, exists := os.LookupEnv("NATS_SERVER_BINARY") 39 | if !exists { 40 | fmt.Println("You need nats-server installed and set NATS_SERVER_BINARY env variable") 41 | os.Exit(1) 42 | } 43 | 44 | natsCmd = exec.Command( 45 | natsServer, 46 | "-p", fmt.Sprintf("%d", natsPort), 47 | "--user", natsUsername, 48 | "--pass", natsPassword, 49 | ) 50 | 51 | err := natsCmd.Start() 52 | Expect(err).NotTo(HaveOccurred()) 53 | 54 | natsAddress := fmt.Sprintf("127.0.0.1:%d", natsPort) 55 | 56 | Eventually(func() error { 57 | _, err := net.Dial("tcp", natsAddress) 58 | return err 59 | }).Should(Succeed()) 60 | 61 | servers := []string{ 62 | fmt.Sprintf( 63 | "nats://%s:%s@%s:%d", 64 | natsUsername, 65 | natsPassword, 66 | natsHost, 67 | natsPort, 68 | ), 69 | } 70 | 71 | opts := nats.GetDefaultOptions() 72 | opts.Servers = servers 73 | 74 | Eventually(func() error { 75 | testSpyClient, err = opts.Connect() 76 | return err 77 | }).ShouldNot(HaveOccurred()) 78 | 79 | }) 80 | 81 | AfterEach(func() { 82 | Expect(natsCmd.Process.Kill()).To(Succeed()) 83 | natsAddress := fmt.Sprintf("127.0.0.1:%d", natsPort) 84 | Eventually(func() error { 85 | _, err := net.Dial("tcp", natsAddress) 86 | return err 87 | }).ShouldNot(Succeed()) 88 | }) 89 | 90 | It("Writes pid to the provided pidfile", func() { 91 | command := exec.Command( 92 | routeRegistrarBinPath, 93 | fmt.Sprintf("-pidfile=%s", pidFile), 94 | fmt.Sprintf("-configPath=%s", configFile), 95 | ) 96 | session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) 97 | Expect(err).ShouldNot(HaveOccurred()) 98 | 99 | Eventually(session.Out).Should(gbytes.Say("Initializing")) 100 | Eventually(session.Out).Should(gbytes.Say("Writing pid")) 101 | Eventually(session.Out).Should(gbytes.Say("Running")) 102 | 103 | session.Kill().Wait() 104 | Eventually(session).Should(gexec.Exit()) 105 | 106 | pidFileContents, err := os.ReadFile(pidFile) 107 | Expect(err).ShouldNot(HaveOccurred()) 108 | 109 | Expect(len(pidFileContents)).To(BeNumerically(">", 0)) 110 | }) 111 | 112 | It("registers routes via NATS", func() { 113 | const ( 114 | topic = "router.register" 115 | ) 116 | 117 | registered := make(chan string) 118 | testSpyClient.Subscribe(topic, func(msg *nats.Msg) { 119 | registered <- string(msg.Data) 120 | }) 121 | 122 | command := exec.Command( 123 | routeRegistrarBinPath, 124 | fmt.Sprintf("-configPath=%s", configFile), 125 | ) 126 | session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) 127 | Expect(err).ShouldNot(HaveOccurred()) 128 | 129 | Eventually(session.Out).Should(gbytes.Say("Initializing")) 130 | Eventually(session.Out).Should(gbytes.Say("Running")) 131 | Eventually(session.Out, 10*time.Second).Should(gbytes.Say("Registering")) 132 | 133 | var receivedMessage string 134 | Eventually(registered, 10*time.Second).Should(Receive(&receivedMessage)) 135 | 136 | i12345 := uint16(12345) 137 | expectedRegistryMessage := messagebus.Message{ 138 | URIs: []string{"uri-1", "uri-2"}, 139 | Host: "127.0.0.1", 140 | Port: &i12345, 141 | Tags: map[string]string{"tag1": "val1", "tag2": "val2"}, 142 | } 143 | 144 | var registryMessage messagebus.Message 145 | err = json.Unmarshal([]byte(receivedMessage), ®istryMessage) 146 | Expect(err).ShouldNot(HaveOccurred()) 147 | 148 | Expect(registryMessage.URIs).To(Equal(expectedRegistryMessage.URIs)) 149 | Expect(registryMessage.Port).To(Equal(expectedRegistryMessage.Port)) 150 | Expect(registryMessage.Tags).To(Equal(expectedRegistryMessage.Tags)) 151 | 152 | session.Kill().Wait() 153 | Eventually(session).Should(gexec.Exit()) 154 | }) 155 | 156 | It("Starts correctly and shuts down on SIGINT", func() { 157 | command := exec.Command( 158 | routeRegistrarBinPath, 159 | fmt.Sprintf("-configPath=%s", configFile), 160 | ) 161 | session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) 162 | Expect(err).ShouldNot(HaveOccurred()) 163 | 164 | Eventually(session.Out).Should(gbytes.Say("Initializing")) 165 | Eventually(session.Out).Should(gbytes.Say("Running")) 166 | Eventually(session.Out, 10*time.Second).Should(gbytes.Say("Registering")) 167 | 168 | session.Interrupt().Wait(10 * time.Second) 169 | Eventually(session.Out).Should(gbytes.Say("Caught signal")) 170 | Eventually(session.Out).Should(gbytes.Say("Unregistering")) 171 | Eventually(session).Should(gexec.Exit()) 172 | Expect(session.ExitCode()).To(BeZero()) 173 | }) 174 | 175 | It("Starts correctly and shuts down on SIGTERM", func() { 176 | command := exec.Command( 177 | routeRegistrarBinPath, 178 | fmt.Sprintf("-configPath=%s", configFile), 179 | ) 180 | session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) 181 | Expect(err).ShouldNot(HaveOccurred()) 182 | 183 | Eventually(session.Out).Should(gbytes.Say("Initializing")) 184 | Eventually(session.Out).Should(gbytes.Say("Running")) 185 | Eventually(session.Out, 10*time.Second).Should(gbytes.Say("Registering")) 186 | 187 | session.Terminate().Wait(10 * time.Second) 188 | Eventually(session.Out).Should(gbytes.Say("Caught signal")) 189 | Eventually(session.Out).Should(gbytes.Say("Unregistering")) 190 | Eventually(session).Should(gexec.Exit()) 191 | Expect(session.ExitCode()).To(BeZero()) 192 | }) 193 | 194 | Context("When the config validation fails", func() { 195 | BeforeEach(func() { 196 | rootConfig := initConfig() 197 | 198 | rootConfig.Routes[0].RegistrationInterval = "asdf" 199 | writeConfig(rootConfig) 200 | }) 201 | 202 | It("exits with error", func() { 203 | command := exec.Command( 204 | routeRegistrarBinPath, 205 | fmt.Sprintf("-configPath=%s", configFile), 206 | ) 207 | session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) 208 | Expect(err).ShouldNot(HaveOccurred()) 209 | 210 | Eventually(session.Out).Should(gbytes.Say("Initializing")) 211 | Eventually(session.Err).Should(gbytes.Say(`1 error with 'route "My route"'`)) 212 | Eventually(session.Err).Should(gbytes.Say("registration_interval: time: invalid duration \"asdf\"")) 213 | 214 | Eventually(session).Should(gexec.Exit()) 215 | Expect(session.ExitCode()).ToNot(BeZero()) 216 | }) 217 | }) 218 | 219 | Context("When route registrar is configured to use mTLS to connect to NATS", func() { 220 | var ( 221 | natsCAPath string 222 | mtlsNATSCertPath, mtlsNATSKeyPath string 223 | tlsTestSpyClient *nats.Conn 224 | tlsNATSCmd *exec.Cmd 225 | ) 226 | 227 | BeforeEach(func() { 228 | natsHost := "127.0.0.1" 229 | natsTLSPort := test_helpers.NextAvailPort() 230 | 231 | // The server cert and client cert are the same 232 | natsCAPath, mtlsNATSCertPath, mtlsNATSKeyPath, _ = tls_helpers.GenerateCaAndMutualTlsCerts() 233 | 234 | tlsNATSCmd = startNatsTLS(natsHost, natsTLSPort, natsCAPath, mtlsNATSCertPath, mtlsNATSKeyPath) 235 | 236 | tlsServers := []string{ 237 | fmt.Sprintf( 238 | "nats://%s:%d", 239 | natsHost, 240 | natsTLSPort, 241 | ), 242 | } 243 | 244 | tlsOpts := nats.GetDefaultOptions() 245 | tlsOpts.Servers = tlsServers 246 | 247 | spyClientTLSConfig, err := tlsconfig.Build( 248 | tlsconfig.WithInternalServiceDefaults(), 249 | tlsconfig.WithIdentityFromFile(mtlsNATSCertPath, mtlsNATSKeyPath), 250 | ).Client( 251 | tlsconfig.WithAuthorityFromFile(natsCAPath), 252 | ) 253 | Expect(err).NotTo(HaveOccurred()) 254 | 255 | tlsOpts.TLSConfig = spyClientTLSConfig 256 | 257 | tlsTestSpyClient, err = tlsOpts.Connect() 258 | Expect(err).ToNot(HaveOccurred()) 259 | 260 | Expect(err).ShouldNot(HaveOccurred()) 261 | 262 | // Ensure nats server is listening before tests 263 | Eventually(func() string { 264 | connStatus := tlsTestSpyClient.Status() 265 | return fmt.Sprintf("%v", connStatus) 266 | }, 5*time.Second).Should(Equal("CONNECTED")) 267 | 268 | Expect(err).ShouldNot(HaveOccurred()) 269 | 270 | rootConfig := initConfig() 271 | rootConfig.MessageBusServers = []config.MessageBusServerSchema{ 272 | { 273 | Host: fmt.Sprintf("%s:%d", natsHost, natsTLSPort), 274 | }, 275 | } 276 | rootConfig.NATSmTLSConfig = config.ClientTLSConfigSchema{ 277 | Enabled: true, 278 | CertPath: mtlsNATSCertPath, 279 | KeyPath: mtlsNATSKeyPath, 280 | CAPath: natsCAPath, 281 | } 282 | writeConfig(rootConfig) 283 | }) 284 | 285 | AfterEach(func() { 286 | tlsTestSpyClient.Close() 287 | Expect(os.Remove(mtlsNATSCertPath)).To(Succeed()) 288 | Expect(os.Remove(mtlsNATSKeyPath)).To(Succeed()) 289 | 290 | Expect(tlsNATSCmd.Process.Kill()).To(Succeed()) 291 | }) 292 | 293 | It("registers routes via NATS", func() { 294 | const ( 295 | topic = "router.register" 296 | ) 297 | 298 | registered := make(chan string) 299 | tlsTestSpyClient.Subscribe(topic, func(msg *nats.Msg) { 300 | registered <- string(msg.Data) 301 | }) 302 | 303 | command := exec.Command( 304 | routeRegistrarBinPath, 305 | fmt.Sprintf("-configPath=%s", configFile), 306 | ) 307 | session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) 308 | Expect(err).ShouldNot(HaveOccurred()) 309 | 310 | Eventually(session.Out).Should(gbytes.Say("Initializing")) 311 | Eventually(session.Out).Should(gbytes.Say("Running")) 312 | Eventually(session.Out, 10*time.Second).Should(gbytes.Say("Registering")) 313 | 314 | var receivedMessage string 315 | Eventually(registered, 10*time.Second).Should(Receive(&receivedMessage)) 316 | 317 | i12345 := uint16(12345) 318 | expectedRegistryMessage := messagebus.Message{ 319 | URIs: []string{"uri-1", "uri-2"}, 320 | Host: "127.0.0.1", 321 | Port: &i12345, 322 | Tags: map[string]string{"tag1": "val1", "tag2": "val2"}, 323 | } 324 | 325 | var registryMessage messagebus.Message 326 | err = json.Unmarshal([]byte(receivedMessage), ®istryMessage) 327 | Expect(err).ShouldNot(HaveOccurred()) 328 | 329 | Expect(registryMessage.URIs).To(Equal(expectedRegistryMessage.URIs)) 330 | Expect(registryMessage.Port).To(Equal(expectedRegistryMessage.Port)) 331 | Expect(registryMessage.Tags).To(Equal(expectedRegistryMessage.Tags)) 332 | 333 | session.Kill().Wait() 334 | Eventually(session).Should(gexec.Exit()) 335 | }) 336 | }) 337 | }) 338 | 339 | func initConfig() config.ConfigSchema { 340 | aPort := uint16(12345) 341 | 342 | registrationInterval := "1s" 343 | 344 | messageBusServers := []config.MessageBusServerSchema{ 345 | { 346 | Host: fmt.Sprintf("127.0.0.1:%d", natsPort), 347 | User: "nats", 348 | Password: "nats", 349 | }, 350 | } 351 | 352 | routes := []config.RouteSchema{ 353 | { 354 | Name: "My route", 355 | Port: &aPort, 356 | URIs: []string{"uri-1", "uri-2"}, 357 | Tags: map[string]string{"tag1": "val1", "tag2": "val2"}, 358 | RegistrationInterval: registrationInterval, 359 | }, 360 | } 361 | 362 | return config.ConfigSchema{ 363 | MessageBusServers: messageBusServers, 364 | Host: "127.0.0.1", 365 | Routes: routes, 366 | } 367 | } 368 | 369 | func writeConfig(config config.ConfigSchema) { 370 | fileToWrite, err := os.Create(configFile) 371 | Expect(err).ShouldNot(HaveOccurred()) 372 | 373 | data, err := json.Marshal(config) 374 | Expect(err).ShouldNot(HaveOccurred()) 375 | 376 | _, err = fileToWrite.Write(data) 377 | Expect(err).ShouldNot(HaveOccurred()) 378 | } 379 | 380 | func startNatsTLS(host string, port uint16, caFile, certFile, keyFile string) *exec.Cmd { 381 | fmt.Fprintf(GinkgoWriter, "Starting nats-server on port %d\n", port) 382 | natsServer, exists := os.LookupEnv("NATS_SERVER_BINARY") 383 | if !exists { 384 | fmt.Println("You need nats-server installed and set NATS_SERVER_BINARY env variable") 385 | os.Exit(1) 386 | } 387 | 388 | cmd := exec.Command( 389 | natsServer, 390 | "-p", fmt.Sprintf("%d", port), 391 | "--tlsverify", 392 | "--tlscacert", caFile, 393 | "--tlscert", certFile, 394 | "--tlskey", keyFile, 395 | ) 396 | 397 | err := cmd.Start() 398 | if err != nil { 399 | fmt.Printf("nats-server failed to start: %v\n", err) 400 | } 401 | 402 | natsTimeout := 10 * time.Second 403 | natsPollingInterval := 20 * time.Millisecond 404 | Eventually(func() error { 405 | _, err := net.Dial("tcp", fmt.Sprintf("%s:%d", host, port)) 406 | return err 407 | }, natsTimeout, natsPollingInterval).Should(Succeed()) 408 | 409 | fmt.Fprintf(GinkgoWriter, "nats-server running on port %d\n", port) 410 | return cmd 411 | } 412 | -------------------------------------------------------------------------------- /integration/tcp_route_registration_test.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "crypto/tls" 5 | "fmt" 6 | "net" 7 | "net/http" 8 | "os" 9 | "os/exec" 10 | "time" 11 | 12 | tls_helpers "code.cloudfoundry.org/cf-routing-test-helpers/tls" 13 | "code.cloudfoundry.org/route-registrar/config" 14 | 15 | "github.com/onsi/gomega/gbytes" 16 | "github.com/onsi/gomega/gexec" 17 | "github.com/onsi/gomega/ghttp" 18 | 19 | . "github.com/onsi/ginkgo/v2" 20 | . "github.com/onsi/gomega" 21 | ) 22 | 23 | var _ = Describe("TCP Route Registration", func() { 24 | var ( 25 | oauthServer *ghttp.Server 26 | routingAPIServer *ghttp.Server 27 | natsCmd *exec.Cmd 28 | rootConfig config.ConfigSchema 29 | oauthHandlers []http.HandlerFunc 30 | ) 31 | 32 | BeforeEach(func() { 33 | routingAPICAFileName, routingAPICAPrivateKey := tls_helpers.GenerateCa() 34 | _, _, serverTLSConfig := tls_helpers.GenerateCertAndKey(routingAPICAFileName, routingAPICAPrivateKey) 35 | routingAPIClientCertPath, routingAPIClientPrivateKeyPath, _ := tls_helpers.GenerateCertAndKey(routingAPICAFileName, routingAPICAPrivateKey) 36 | 37 | routingAPIServer = ghttp.NewUnstartedServer() 38 | routingAPIServer.HTTPTestServer.TLS = &tls.Config{} 39 | routingAPIServer.HTTPTestServer.TLS.RootCAs = tls_helpers.CertPool(routingAPICAFileName) 40 | routingAPIServer.HTTPTestServer.TLS.ClientCAs = tls_helpers.CertPool(routingAPICAFileName) 41 | routingAPIServer.HTTPTestServer.TLS.ClientAuth = tls.RequireAndVerifyClientCert 42 | routingAPIServer.HTTPTestServer.TLS.CipherSuites = []uint16{tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256} 43 | routingAPIServer.HTTPTestServer.TLS.Certificates = []tls.Certificate{serverTLSConfig} 44 | 45 | routingAPIResponses := []http.HandlerFunc{ 46 | ghttp.CombineHandlers( 47 | ghttp.VerifyRequest("GET", "/routing/v1/router_groups"), 48 | ghttp.RespondWith(200, `[{ 49 | "guid": "router-group-guid", 50 | "name": "my-router-group", 51 | "type": "tcp", 52 | "reservable_ports": "1024-1025" 53 | }]`), 54 | ), 55 | ghttp.CombineHandlers( 56 | ghttp.VerifyRequest("POST", "/routing/v1/tcp_routes/create"), 57 | ghttp.VerifyJSON(`[{ 58 | "router_group_guid":"router-group-guid", 59 | "backend_port":1234, 60 | "backend_tls_port":-1, 61 | "instance_id": "", 62 | "backend_ip":"127.0.0.1", 63 | "port":5678, 64 | "modification_tag":{ 65 | "guid":"", 66 | "index":0 67 | }, 68 | "ttl": 1, 69 | "isolation_segment":"" 70 | }]`), 71 | ghttp.RespondWith(200, ""), 72 | ), 73 | } 74 | routingAPIServer.AppendHandlers(routingAPIResponses...) 75 | routingAPIServer.SetAllowUnhandledRequests(true) // sometimes multiple creates happen 76 | 77 | oauthServer = ghttp.NewUnstartedServer() 78 | oauthHandlers = []http.HandlerFunc{ 79 | ghttp.CombineHandlers( 80 | ghttp.VerifyRequest("POST", "/oauth/token"), 81 | ghttp.RespondWith(200, `{ 82 | "access_token": "some-access-token", 83 | "token_type": "bearer", 84 | "expires_in": 3600 85 | }`, 86 | http.Header{"Content-Type": []string{"application/json"}}, 87 | ), 88 | ), 89 | ghttp.CombineHandlers( 90 | ghttp.VerifyRequest("POST", "/oauth/token"), 91 | ghttp.RespondWith(200, `{ 92 | "access_token": "some-access-token", 93 | "token_type": "bearer", 94 | "expires_in": 3600 95 | }`, 96 | http.Header{"Content-Type": []string{"application/json"}}, 97 | ), 98 | ), 99 | } 100 | 101 | rootConfig = initConfig() 102 | rootConfig.RoutingAPI.ClientID = "my-client" 103 | rootConfig.RoutingAPI.ClientSecret = "my-secret" 104 | rootConfig.RoutingAPI.ClientCertificatePath = routingAPIClientCertPath 105 | rootConfig.RoutingAPI.ClientPrivateKeyPath = routingAPIClientPrivateKeyPath 106 | rootConfig.RoutingAPI.ServerCACertificatePath = routingAPICAFileName 107 | 108 | port := uint16(1234) 109 | externalPort := uint16(5678) 110 | routes := []config.RouteSchema{{ 111 | Name: "my-route", 112 | Type: "tcp", 113 | Port: &port, 114 | ExternalPort: &externalPort, 115 | URIs: []string{"my-host"}, 116 | RouterGroup: "my-router-group", 117 | RegistrationInterval: "100ns", 118 | }} 119 | rootConfig.Routes = routes 120 | natsCmd = startNats() 121 | }) 122 | 123 | JustBeforeEach(func() { 124 | oauthServer.AppendHandlers(oauthHandlers...) 125 | oauthServer.Start() 126 | rootConfig.RoutingAPI.OAuthURL = oauthServer.URL() 127 | }) 128 | 129 | AfterEach(func() { 130 | Expect(natsCmd.Process.Kill()).To(Succeed()) 131 | routingAPIServer.Close() 132 | oauthServer.Close() 133 | }) 134 | 135 | Context("when provided a tcp route", func() { 136 | JustBeforeEach(func() { 137 | routingAPIServer.HTTPTestServer.Start() 138 | rootConfig.RoutingAPI.APIURL = routingAPIServer.URL() 139 | writeConfig(rootConfig) 140 | }) 141 | 142 | var session *gexec.Session 143 | 144 | BeforeEach(func() { 145 | var err error 146 | session, err = registerRoute() 147 | Expect(err).ShouldNot(HaveOccurred()) 148 | }) 149 | 150 | AfterEach(func() { 151 | session.Kill() 152 | }) 153 | 154 | It("registers it with the routing API", func() { 155 | Eventually(session.Out).Should(gbytes.Say("Initializing")) 156 | Eventually(session.Out).Should(gbytes.Say("creating routing API connection")) 157 | Eventually(session.Out).Should(gbytes.Say("Writing pid")) 158 | Eventually(session.Out).Should(gbytes.Say("Running")) 159 | Eventually(session.Out).Should(gbytes.Say("Mapped new router group")) 160 | Eventually(session.Out).Should(gbytes.Say("Upserted route")) 161 | }) 162 | Context("when UAA errors intermittently occur", func() { 163 | BeforeEach(func() { 164 | oauthHandlers = []http.HandlerFunc{ 165 | ghttp.CombineHandlers( 166 | ghttp.VerifyRequest("POST", "/oauth/token"), 167 | ghttp.RespondWith(500, `{}`, http.Header{"Content-Type": []string{"application/json"}}), 168 | ), 169 | ghttp.CombineHandlers( 170 | ghttp.VerifyRequest("POST", "/oauth/token"), 171 | ghttp.RespondWith(200, `{ 172 | "access_token": "some-access-token", 173 | "token_type": "bearer", 174 | "expires_in": 3600 175 | }`, 176 | http.Header{"Content-Type": []string{"application/json"}}, 177 | ), 178 | ), 179 | } 180 | }) 181 | It("Retries UAA token refreshes if problems were encountered", func() { 182 | Eventually(session.Out).Should(gbytes.Say("error-fetching-token")) 183 | Consistently(session.Out, 5*time.Second).ShouldNot(gbytes.Say("token-error")) 184 | }) 185 | }) 186 | 187 | Context("when UAA errors consistently ooccur", func() { 188 | BeforeEach(func() { 189 | oauthHandlers = []http.HandlerFunc{ 190 | ghttp.CombineHandlers( 191 | ghttp.VerifyRequest("POST", "/oauth/token"), 192 | ghttp.RespondWith(500, `{}`, http.Header{"Content-Type": []string{"application/json"}}), 193 | ), 194 | ghttp.CombineHandlers( 195 | ghttp.VerifyRequest("POST", "/oauth/token"), 196 | ghttp.RespondWith(500, `{}`, http.Header{"Content-Type": []string{"application/json"}}), 197 | ), 198 | ghttp.CombineHandlers( 199 | ghttp.VerifyRequest("POST", "/oauth/token"), 200 | ghttp.RespondWith(500, `{}`, http.Header{"Content-Type": []string{"application/json"}}), 201 | ), 202 | ghttp.CombineHandlers( 203 | ghttp.VerifyRequest("POST", "/oauth/token"), 204 | ghttp.RespondWith(500, `{}`, http.Header{"Content-Type": []string{"application/json"}}), 205 | ), 206 | } 207 | }) 208 | It("Gives up and returns a token error", func() { 209 | Eventually(session.Out, 5*time.Second).Should(gbytes.Say("token-error")) 210 | Eventually(session.Out, 5*time.Second).Should(gbytes.Say("error\":\"oauth2: cannot fetch token:")) 211 | }) 212 | }) 213 | }) 214 | 215 | Context("when routing API uses TLS", func() { 216 | Context("when provided a tcp route", func() { 217 | JustBeforeEach(func() { 218 | routingAPIServer.HTTPTestServer.StartTLS() 219 | rootConfig.RoutingAPI.APIURL = routingAPIServer.URL() 220 | writeConfig(rootConfig) 221 | }) 222 | 223 | var session *gexec.Session 224 | 225 | BeforeEach(func() { 226 | var err error 227 | session, err = registerRoute() 228 | Expect(err).ShouldNot(HaveOccurred()) 229 | }) 230 | 231 | AfterEach(func() { 232 | session.Kill() 233 | }) 234 | 235 | It("registers it with the routing API", func() { 236 | Eventually(session.Out).Should(gbytes.Say("Initializing")) 237 | Eventually(session.Out).Should(gbytes.Say("creating routing API connection")) 238 | Eventually(session.Out).Should(gbytes.Say("Writing pid")) 239 | Eventually(session.Out).Should(gbytes.Say("Running")) 240 | Eventually(session.Out).Should(gbytes.Say("Mapped new router group")) 241 | Eventually(session.Out).Should(gbytes.Say("Upserted route")) 242 | // Upserted Route content verified with expected body in the ghttp server setup 243 | }) 244 | }) 245 | }) 246 | }) 247 | 248 | func registerRoute() (*gexec.Session, error) { 249 | command := exec.Command( 250 | routeRegistrarBinPath, 251 | "-logLevel=debug", 252 | fmt.Sprintf("-pidfile=%s", pidFile), 253 | fmt.Sprintf("-configPath=%s", configFile), 254 | ) 255 | 256 | return gexec.Start(command, GinkgoWriter, GinkgoWriter) 257 | } 258 | 259 | func startNats() *exec.Cmd { 260 | natsUsername := "nats" 261 | natsPassword := "nats" 262 | 263 | natsServer, exists := os.LookupEnv("NATS_SERVER_BINARY") 264 | if !exists { 265 | fmt.Println("You need nats-server installed and set NATS_SERVER_BINARY env variable") 266 | os.Exit(1) 267 | } 268 | natsCmd := exec.Command( 269 | natsServer, 270 | "-p", fmt.Sprintf("%d", natsPort), 271 | "--user", natsUsername, 272 | "--pass", natsPassword, 273 | ) 274 | 275 | err := natsCmd.Start() 276 | Expect(err).NotTo(HaveOccurred()) 277 | 278 | natsAddress := fmt.Sprintf("127.0.0.1:%d", natsPort) 279 | 280 | Eventually(func() error { 281 | _, err := net.Dial("tcp", natsAddress) 282 | return err 283 | }).Should(Succeed()) 284 | 285 | return natsCmd 286 | } 287 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "errors" 7 | "flag" 8 | "log" 9 | "net/http" 10 | "net/url" 11 | "os" 12 | "os/signal" 13 | "strconv" 14 | "syscall" 15 | "time" 16 | 17 | "code.cloudfoundry.org/clock" 18 | "code.cloudfoundry.org/lager/v3" 19 | "code.cloudfoundry.org/lager/v3/lagerflags" 20 | "code.cloudfoundry.org/route-registrar/config" 21 | "code.cloudfoundry.org/route-registrar/healthchecker" 22 | "code.cloudfoundry.org/route-registrar/messagebus" 23 | "code.cloudfoundry.org/route-registrar/registrar" 24 | "code.cloudfoundry.org/route-registrar/routingapi" 25 | routing_api "code.cloudfoundry.org/routing-api" 26 | "code.cloudfoundry.org/routing-api/uaaclient" 27 | "code.cloudfoundry.org/tlsconfig" 28 | 29 | "github.com/tedsuo/ifrit" 30 | ) 31 | 32 | func main() { 33 | var configPath string 34 | flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 35 | 36 | pidfile := flags.String("pidfile", "", "Path to pid file") 37 | lagerflags.AddFlags(flags) 38 | 39 | flags.StringVar(&configPath, "configPath", "", "path to configuration file with json encoded content") 40 | err := flags.Set("configPath", "registrar_settings.yml") 41 | if err != nil { 42 | log.Fatalf("Failed to set up configPath flag") 43 | } 44 | // #nosec G104 - setting flags.ExitOnError means this function will never return an error 45 | flags.Parse(os.Args[1:]) 46 | 47 | logger, _ := lagerflags.New("Route Registrar") 48 | 49 | logger.Info("Initializing") 50 | 51 | configSchema, err := config.NewConfigSchemaFromFile(configPath) 52 | if err != nil { 53 | logger.Fatal("error parsing file: %s\n", err) 54 | } 55 | 56 | c, err := configSchema.ParseSchemaAndSetDefaultsToConfig() 57 | if err != nil { 58 | log.Fatalln(err) 59 | } 60 | 61 | hc := healthchecker.NewHealthChecker(logger) 62 | 63 | logger.Info("creating nats connection") 64 | messageBus := messagebus.NewMessageBus(logger, c.AvailabilityZone) 65 | 66 | var routingAPI *routingapi.RoutingAPI 67 | if c.RoutingAPI.APIURL != "" { 68 | logger.Info("creating routing API connection") 69 | 70 | tlsConfig := &tls.Config{InsecureSkipVerify: c.RoutingAPI.SkipSSLValidation} 71 | if c.RoutingAPI.CACerts != "" { 72 | certBytes, err := os.ReadFile(c.RoutingAPI.CACerts) 73 | if err != nil { 74 | log.Fatalf("Failed to read ca cert file: %s", err.Error()) 75 | } 76 | 77 | caCertPool := x509.NewCertPool() 78 | if ok := caCertPool.AppendCertsFromPEM(certBytes); !ok { 79 | log.Fatal(errors.New("Unable to load caCert")) 80 | } 81 | tlsConfig.RootCAs = caCertPool 82 | } 83 | 84 | tr := &http.Transport{ 85 | TLSClientConfig: tlsConfig, 86 | } 87 | 88 | httpClient := &http.Client{Transport: tr} 89 | httpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { 90 | return http.ErrUseLastResponse 91 | } 92 | 93 | oauthUrl, err := url.Parse(c.RoutingAPI.OAuthURL) 94 | if err != nil { 95 | log.Fatalf("Could not parse RoutingAPI OAuth URL: %s", err) 96 | } 97 | port, err := strconv.ParseUint(oauthUrl.Port(), 10, 16) 98 | if err != nil { 99 | log.Fatalf("RoutingAPI OAuth port (%s) not an integer: %s", oauthUrl.Port(), err) 100 | } 101 | 102 | uaaConfig := uaaclient.Config{ 103 | Port: uint16(port), 104 | Protocol: oauthUrl.Scheme, 105 | SkipSSLValidation: c.RoutingAPI.SkipSSLValidation, 106 | ClientName: c.RoutingAPI.ClientID, 107 | ClientSecret: c.RoutingAPI.ClientSecret, 108 | CACerts: c.RoutingAPI.CACerts, 109 | TokenEndpoint: oauthUrl.Hostname(), 110 | } 111 | clk := clock.NewClock() 112 | uaaClient, err := uaaclient.NewTokenFetcher(false, uaaConfig, clk, 3, 500*time.Millisecond, 30, logger) 113 | if err != nil { 114 | log.Fatalln(err) 115 | } 116 | 117 | apiClient, err := newAPIClient(c) 118 | 119 | if err != nil { 120 | logger.Fatal("failed-to-create-tls-config", err) 121 | } 122 | 123 | routingAPI = routingapi.NewRoutingAPI(logger, uaaClient, apiClient, c.RoutingAPI.MaxTTL) 124 | } 125 | 126 | r := registrar.NewRegistrar(*c, hc, logger, messageBus, routingAPI, 10*time.Second) 127 | 128 | if *pidfile != "" { 129 | pid := strconv.Itoa(os.Getpid()) 130 | err := os.WriteFile(*pidfile, []byte(pid), 0644) 131 | logger.Info("Writing pid", lager.Data{"pid": pid, "file": *pidfile}) 132 | if err != nil { 133 | logger.Fatal( 134 | "error writing pid to pidfile", 135 | err, 136 | lager.Data{ 137 | "pid": pid, 138 | "pidfile": *pidfile, 139 | }, 140 | ) 141 | } 142 | } 143 | 144 | sigChan := make(chan os.Signal, 1) 145 | signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) 146 | 147 | logger.Info("Running") 148 | 149 | process := ifrit.Invoke(r) 150 | for { 151 | select { 152 | case s := <-sigChan: 153 | logger.Info("Caught signal", lager.Data{"signal": s}) 154 | process.Signal(s) 155 | case err := <-process.Wait(): 156 | if err != nil { 157 | logger.Fatal("Exiting with error", err) 158 | } 159 | logger.Info("Exiting without error") 160 | os.Exit(0) 161 | } 162 | } 163 | } 164 | 165 | func newAPIClient(c *config.Config) (routing_api.Client, error) { 166 | apiURL, err := url.Parse(c.RoutingAPI.APIURL) 167 | if err != nil { 168 | return nil, err 169 | } 170 | 171 | var client routing_api.Client 172 | 173 | if apiURL.Scheme == "https" { 174 | routingAPITLSConfig, err := tlsconfig.Build( 175 | tlsconfig.WithInternalServiceDefaults(), 176 | tlsconfig.WithIdentityFromFile(c.RoutingAPI.ClientCertificatePath, c.RoutingAPI.ClientPrivateKeyPath), 177 | ).Client( 178 | tlsconfig.WithAuthorityFromFile(c.RoutingAPI.ServerCACertificatePath), 179 | ) 180 | if err != nil { 181 | return nil, err 182 | } 183 | 184 | client = routing_api.NewClientWithTLSConfig(c.RoutingAPI.APIURL, routingAPITLSConfig) 185 | } else { 186 | client = routing_api.NewClient(c.RoutingAPI.APIURL, c.RoutingAPI.SkipSSLValidation) 187 | } 188 | 189 | return client, nil 190 | } 191 | -------------------------------------------------------------------------------- /messagebus/messagebus.go: -------------------------------------------------------------------------------- 1 | package messagebus 2 | 3 | import ( 4 | "crypto/tls" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "net/url" 9 | "sync/atomic" 10 | "time" 11 | 12 | "code.cloudfoundry.org/lager/v3" 13 | "code.cloudfoundry.org/route-registrar/config" 14 | "github.com/nats-io/nats.go" 15 | ) 16 | 17 | //go:generate counterfeiter . MessageBus 18 | 19 | type MessageBus interface { 20 | Connect(servers []config.MessageBusServer, tlsConfig *tls.Config) error 21 | SendMessage(subject string, route config.Route, privateInstanceId string) error 22 | Close() 23 | } 24 | 25 | type msgBus struct { 26 | natsHost *atomic.Value 27 | natsConn *nats.Conn 28 | availabilityZone string 29 | logger lager.Logger 30 | } 31 | 32 | type Message struct { 33 | URIs []string `json:"uris"` 34 | Host string `json:"host"` 35 | Protocol string `json:"protocol,omitempty"` 36 | Port *uint16 `json:"port,omitempty"` 37 | TLSPort *uint16 `json:"tls_port,omitempty"` 38 | Tags map[string]string `json:"tags"` 39 | RouteServiceUrl string `json:"route_service_url,omitempty"` 40 | PrivateInstanceId string `json:"private_instance_id"` 41 | ServerCertDomainSAN string `json:"server_cert_domain_san,omitempty"` 42 | AvailabilityZone string `json:"availability_zone,omitempty"` 43 | Options map[string]string `json:"options,omitempty"` 44 | } 45 | 46 | const LoadBalancingAlgorithm string = "loadbalancing" 47 | 48 | func NewMessageBus(logger lager.Logger, availabilityZone string) MessageBus { 49 | return &msgBus{ 50 | logger: logger, 51 | natsHost: &atomic.Value{}, 52 | availabilityZone: availabilityZone, 53 | } 54 | } 55 | 56 | func (m *msgBus) Connect(servers []config.MessageBusServer, tlsConfig *tls.Config) error { 57 | 58 | var natsServers []string 59 | var natsHosts []string 60 | for _, server := range servers { 61 | natsServers = append( 62 | natsServers, 63 | fmt.Sprintf("nats://%s:%s@%s", server.User, server.Password, server.Host), 64 | ) 65 | natsHosts = append(natsHosts, server.Host) 66 | } 67 | 68 | opts := nats.GetDefaultOptions() 69 | opts.Servers = natsServers 70 | opts.TLSConfig = tlsConfig 71 | opts.PingInterval = 20 * time.Second 72 | 73 | opts.ClosedCB = func(conn *nats.Conn) { 74 | m.logger.Error("nats-connection-closed", errors.New("unexpected nats conn closed"), lager.Data{"nats-host": m.natsHost.Load()}) 75 | } 76 | 77 | opts.DisconnectedCB = func(conn *nats.Conn) { 78 | m.logger.Info("nats-connection-disconnected", lager.Data{"nats-host": m.natsHost.Load()}) 79 | } 80 | 81 | opts.ReconnectedCB = func(conn *nats.Conn) { 82 | natsHost, err := parseNatsUrl(conn.ConnectedUrl()) 83 | if err != nil { 84 | m.logger.Error("nats-url-parse-failed", err, lager.Data{"nats-host": natsHost}) 85 | } 86 | m.natsHost.Store(natsHost) 87 | m.logger.Info("nats-connection-reconnected", lager.Data{"nats-host": m.natsHost.Load()}) 88 | } 89 | 90 | natsConn, err := opts.Connect() 91 | if err != nil { 92 | m.logger.Error("nats-connection-failed", err, lager.Data{"nats-hosts": natsHosts}) 93 | return err 94 | } 95 | 96 | natsHost, err := parseNatsUrl(natsConn.ConnectedUrl()) 97 | if err != nil { 98 | m.logger.Error("nats-url-parse-failed", err, lager.Data{"nats-host": natsHost}) 99 | } 100 | 101 | m.natsHost.Store(natsHost) 102 | m.logger.Info("nats-connection-successful", lager.Data{"nats-host": m.natsHost.Load()}) 103 | m.natsConn = natsConn 104 | 105 | return nil 106 | } 107 | 108 | func (m msgBus) SendMessage(subject string, route config.Route, privateInstanceId string) error { 109 | m.logger.Debug("creating-message", lager.Data{"subject": subject, "route": route, "privateInstanceId": privateInstanceId}) 110 | 111 | routeOptions := m.mapRouteOptions(route) 112 | 113 | msg := &Message{ 114 | URIs: route.URIs, 115 | Host: route.Host, 116 | Port: route.Port, 117 | Protocol: route.Protocol, 118 | TLSPort: route.TLSPort, 119 | Tags: route.Tags, 120 | RouteServiceUrl: route.RouteServiceUrl, 121 | ServerCertDomainSAN: route.ServerCertDomainSAN, 122 | PrivateInstanceId: privateInstanceId, 123 | AvailabilityZone: m.availabilityZone, 124 | Options: routeOptions, 125 | } 126 | 127 | json, err := json.Marshal(msg) 128 | if err != nil { 129 | // Untested as we cannot force json.Marshal to return error. 130 | return err 131 | } 132 | 133 | m.logger.Debug("publishing-message", lager.Data{"msg": string(json)}) 134 | 135 | return m.natsConn.Publish(subject, json) 136 | } 137 | 138 | func (m msgBus) mapRouteOptions(route config.Route) map[string]string { 139 | if route.Options != nil { 140 | routeOptions := make(map[string]string) 141 | if route.Options.LoadBalancingAlgorithm != "" { 142 | routeOptions[LoadBalancingAlgorithm] = string(route.Options.LoadBalancingAlgorithm) 143 | } 144 | return routeOptions 145 | } 146 | return nil 147 | } 148 | 149 | func (m msgBus) Close() { 150 | m.natsConn.Close() 151 | } 152 | 153 | func parseNatsUrl(natsUrl string) (string, error) { 154 | natsURL, err := url.Parse(natsUrl) 155 | natsHostStr := "" 156 | if err != nil { 157 | return "", err 158 | } else { 159 | natsHostStr = natsURL.Host 160 | } 161 | 162 | return natsHostStr, nil 163 | } 164 | -------------------------------------------------------------------------------- /messagebus/messagebus_suite_test.go: -------------------------------------------------------------------------------- 1 | package messagebus_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | var ( 11 | natsPort int 12 | ) 13 | 14 | func TestMessagebus(t *testing.T) { 15 | RegisterFailHandler(Fail) 16 | 17 | RunSpecs(t, "Messagebus Suite") 18 | } 19 | 20 | var _ = BeforeSuite(func() { 21 | natsPort = 20000 + GinkgoParallelProcess() 22 | }) 23 | -------------------------------------------------------------------------------- /messagebus/messagebus_test.go: -------------------------------------------------------------------------------- 1 | package messagebus_test 2 | 3 | import ( 4 | "crypto/tls" 5 | "encoding/json" 6 | "fmt" 7 | "net" 8 | "os" 9 | "os/exec" 10 | "strconv" 11 | "time" 12 | 13 | tls_helpers "code.cloudfoundry.org/cf-routing-test-helpers/tls" 14 | "code.cloudfoundry.org/lager/v3" 15 | "code.cloudfoundry.org/lager/v3/lagertest" 16 | "code.cloudfoundry.org/route-registrar/config" 17 | "code.cloudfoundry.org/route-registrar/messagebus" 18 | "code.cloudfoundry.org/tlsconfig" 19 | "github.com/nats-io/nats.go" 20 | 21 | . "github.com/onsi/ginkgo/v2" 22 | . "github.com/onsi/gomega" 23 | "github.com/onsi/gomega/gbytes" 24 | ) 25 | 26 | var _ = Describe("Messagebus test Suite", func() { 27 | var ( 28 | natsCmd *exec.Cmd 29 | natsHost string 30 | natsUsername string 31 | natsPassword string 32 | 33 | testSpyClient *nats.Conn 34 | 35 | logger lager.Logger 36 | messageBusServers []config.MessageBusServer 37 | messageBus messagebus.MessageBus 38 | ) 39 | 40 | BeforeEach(func() { 41 | natsUsername = "nats-user" 42 | natsPassword = "nats-pw" 43 | natsHost = "127.0.0.1" 44 | 45 | natsCmd = startNats(natsHost, natsPort, natsUsername, natsPassword) 46 | 47 | logger = lagertest.NewTestLogger("nats-test") 48 | var err error 49 | servers := []string{ 50 | fmt.Sprintf( 51 | "nats://%s:%s@%s:%d", 52 | natsUsername, 53 | natsPassword, 54 | natsHost, 55 | natsPort, 56 | ), 57 | } 58 | 59 | opts := nats.GetDefaultOptions() 60 | opts.Servers = servers 61 | 62 | testSpyClient, err = opts.Connect() 63 | Expect(err).ToNot(HaveOccurred()) 64 | 65 | // Ensure nats server is listening before tests 66 | Eventually(func() string { 67 | connStatus := testSpyClient.Status() 68 | return fmt.Sprintf("%v", connStatus) 69 | }, 5*time.Second).Should(Equal("CONNECTED")) 70 | 71 | Expect(err).ShouldNot(HaveOccurred()) 72 | 73 | messageBusServer := config.MessageBusServer{ 74 | Host: fmt.Sprintf("%s:%d", natsHost, natsPort), 75 | User: natsUsername, 76 | Password: natsPassword, 77 | } 78 | 79 | messageBusServers = []config.MessageBusServer{messageBusServer} 80 | 81 | messageBus = messagebus.NewMessageBus(logger, "some-az") 82 | }) 83 | 84 | AfterEach(func() { 85 | testSpyClient.Close() 86 | 87 | err := natsCmd.Process.Kill() 88 | Expect(err).NotTo(HaveOccurred()) 89 | _, err = natsCmd.Process.Wait() 90 | Expect(err).NotTo(HaveOccurred()) 91 | }) 92 | 93 | Describe("Connect", func() { 94 | It("connects without error", func() { 95 | err := messageBus.Connect(messageBusServers, nil) 96 | Expect(err).ShouldNot(HaveOccurred()) 97 | }) 98 | 99 | Context("when tls config is provided", func() { 100 | var ( 101 | natsTlsHost string 102 | natsTlsPort int 103 | natsTlsCmd *exec.Cmd 104 | tlsMessageBusServers []config.MessageBusServer 105 | natsCAPath string 106 | mtlsNATSServerCertPath string 107 | mtlsNATSServerKeyPath string 108 | mtlsNATSClientCert tls.Certificate 109 | ) 110 | BeforeEach(func() { 111 | natsTlsHost = "127.0.0.1" 112 | natsTlsPort = natsPort + 1000 113 | natsCAPath, mtlsNATSServerCertPath, mtlsNATSServerKeyPath, mtlsNATSClientCert = tls_helpers.GenerateCaAndMutualTlsCerts() 114 | 115 | natsTlsCmd = startNatsTls(natsTlsHost, natsTlsPort, natsCAPath, mtlsNATSServerCertPath, mtlsNATSServerKeyPath, "testuser", "testpw") 116 | 117 | tlsServers := []string{ 118 | fmt.Sprintf( 119 | "nats://%s:%d", 120 | natsTlsHost, 121 | natsTlsPort, 122 | ), 123 | } 124 | 125 | tlsOpts := nats.GetDefaultOptions() 126 | tlsOpts.Servers = tlsServers 127 | tlsOpts.User = "testuser" 128 | tlsOpts.Password = "testpw" 129 | 130 | spyClientTlsConfig, err := tlsconfig.Build( 131 | tlsconfig.WithInternalServiceDefaults(), 132 | tlsconfig.WithIdentity(mtlsNATSClientCert), 133 | ).Client( 134 | tlsconfig.WithAuthorityFromFile(natsCAPath), 135 | ) 136 | Expect(err).NotTo(HaveOccurred()) 137 | 138 | tlsOpts.TLSConfig = spyClientTlsConfig 139 | 140 | tlsTestSpyClient, err := tlsOpts.Connect() 141 | Expect(err).ToNot(HaveOccurred()) 142 | 143 | // Ensure nats server is listening before tests 144 | Eventually(func() string { 145 | connStatus := tlsTestSpyClient.Status() 146 | return fmt.Sprintf("%v", connStatus) 147 | }, 5*time.Second).Should(Equal("CONNECTED")) 148 | 149 | Expect(err).ShouldNot(HaveOccurred()) 150 | 151 | tlsMessageBusServer := config.MessageBusServer{ 152 | Host: fmt.Sprintf("%s:%d", natsTlsHost, natsTlsPort), 153 | User: "testuser", 154 | Password: "testpw", 155 | } 156 | 157 | tlsMessageBusServers = []config.MessageBusServer{tlsMessageBusServer} 158 | 159 | tlsTestSpyClient.Close() 160 | 161 | messageBusServers = []config.MessageBusServer{} 162 | }) 163 | AfterEach(func() { 164 | 165 | err := natsTlsCmd.Process.Kill() 166 | Expect(err).NotTo(HaveOccurred()) 167 | _, err = natsTlsCmd.Process.Wait() 168 | Expect(err).NotTo(HaveOccurred()) 169 | }) 170 | 171 | It("connects without error", func() { 172 | var ( 173 | err error 174 | clientTlsConfig *tls.Config 175 | ) 176 | 177 | clientTlsConfig, err = tlsconfig.Build( 178 | tlsconfig.WithInternalServiceDefaults(), 179 | tlsconfig.WithIdentity(mtlsNATSClientCert), 180 | ).Client( 181 | tlsconfig.WithAuthorityFromFile(natsCAPath), 182 | ) 183 | Expect(err).NotTo(HaveOccurred()) 184 | 185 | err = messageBus.Connect(tlsMessageBusServers, clientTlsConfig) 186 | Expect(err).NotTo(HaveOccurred()) 187 | }) 188 | }) 189 | 190 | Context("when no servers are provided", func() { 191 | BeforeEach(func() { 192 | messageBusServers = []config.MessageBusServer{} 193 | }) 194 | 195 | It("returns error", func() { 196 | err := messageBus.Connect(messageBusServers, nil) 197 | Expect(err).Should(HaveOccurred()) 198 | }) 199 | }) 200 | 201 | Context("when nats connection is successful", func() { 202 | BeforeEach(func() { 203 | err := messageBus.Connect(messageBusServers, nil) 204 | Expect(err).ShouldNot(HaveOccurred()) 205 | }) 206 | It("logs a message", func() { 207 | Eventually(logger).Should(gbytes.Say(`nats-connection-successful`)) 208 | Eventually(logger).Should(gbytes.Say(natsHost)) 209 | }) 210 | }) 211 | 212 | Context("when nats connection closes", func() { 213 | BeforeEach(func() { 214 | err := messageBus.Connect(messageBusServers, nil) 215 | Expect(err).ShouldNot(HaveOccurred()) 216 | messageBus.Close() 217 | }) 218 | 219 | It("logs a message", func() { 220 | Eventually(logger).Should(gbytes.Say(`nats-connection-disconnected`)) 221 | Eventually(logger).Should(gbytes.Say(natsHost)) 222 | Eventually(logger).Should(gbytes.Say(`nats-connection-closed`)) 223 | Eventually(logger).Should(gbytes.Say(natsHost)) 224 | }) 225 | }) 226 | }) 227 | 228 | Describe("SendMessage", func() { 229 | const ( 230 | topic = "router.registrar" 231 | privateInstanceId = "some_id" 232 | ) 233 | 234 | var ( 235 | route config.Route 236 | ) 237 | 238 | BeforeEach(func() { 239 | err := messageBus.Connect(messageBusServers, nil) 240 | Expect(err).ShouldNot(HaveOccurred()) 241 | 242 | port := uint16(12345) 243 | 244 | route = config.Route{ 245 | Name: "some_name", 246 | Host: "some_host", 247 | Port: &port, 248 | TLSPort: &port, 249 | URIs: []string{"uri1", "uri2"}, 250 | RouteServiceUrl: "https://rs.example.com", 251 | Tags: map[string]string{"tag1": "val1", "tag2": "val2"}, 252 | ServerCertDomainSAN: "cf.cert.internal", 253 | } 254 | }) 255 | 256 | It("send messages", func() { 257 | registered := make(chan string) 258 | testSpyClient.Subscribe(topic, func(msg *nats.Msg) { 259 | registered <- string(msg.Data) 260 | }) 261 | 262 | // Wait for the nats library to register our callback. 263 | // We use a sleep because there's no way to know that the callback was 264 | // registered successfully (e.g. they don't provide a channel) 265 | time.Sleep(20 * time.Millisecond) 266 | 267 | err := messageBus.SendMessage(topic, route, privateInstanceId) 268 | Expect(err).ShouldNot(HaveOccurred()) 269 | 270 | // Assert that we got the right message 271 | var receivedMessage string 272 | Eventually(registered, 2).Should(Receive(&receivedMessage)) 273 | 274 | expectedRegistryMessage := messagebus.Message{ 275 | URIs: route.URIs, 276 | Host: route.Host, 277 | Port: route.Port, 278 | TLSPort: route.TLSPort, 279 | RouteServiceUrl: route.RouteServiceUrl, 280 | Tags: route.Tags, 281 | ServerCertDomainSAN: "cf.cert.internal", 282 | AvailabilityZone: "some-az", 283 | } 284 | 285 | var registryMessage messagebus.Message 286 | err = json.Unmarshal([]byte(receivedMessage), ®istryMessage) 287 | Expect(err).ShouldNot(HaveOccurred()) 288 | 289 | Expect(registryMessage.URIs).To(Equal(expectedRegistryMessage.URIs)) 290 | Expect(registryMessage.Port).To(Equal(expectedRegistryMessage.Port)) 291 | Expect(registryMessage.Host).To(Equal(expectedRegistryMessage.Host)) 292 | Expect(registryMessage.Protocol).To(BeEmpty()) 293 | Expect(registryMessage.RouteServiceUrl).To(Equal(expectedRegistryMessage.RouteServiceUrl)) 294 | Expect(registryMessage.Tags).To(Equal(expectedRegistryMessage.Tags)) 295 | Expect(registryMessage.AvailabilityZone).To(Equal(expectedRegistryMessage.AvailabilityZone)) 296 | }) 297 | 298 | Context("when the connection is already closed", func() { 299 | BeforeEach(func() { 300 | err := messageBus.Connect(messageBusServers, nil) 301 | Expect(err).ShouldNot(HaveOccurred()) 302 | 303 | messageBus.Close() 304 | }) 305 | 306 | It("returns error", func() { 307 | err := messageBus.SendMessage(topic, route, privateInstanceId) 308 | Expect(err).Should(HaveOccurred()) 309 | }) 310 | }) 311 | }) 312 | 313 | Describe("SendMessage for h2 route", func() { 314 | const ( 315 | topic = "router.registrar" 316 | privateInstanceId = "some_id" 317 | ) 318 | 319 | var ( 320 | route config.Route 321 | ) 322 | 323 | BeforeEach(func() { 324 | err := messageBus.Connect(messageBusServers, nil) 325 | Expect(err).ShouldNot(HaveOccurred()) 326 | 327 | port := uint16(12345) 328 | 329 | route = config.Route{ 330 | Name: "some_name", 331 | Port: &port, 332 | Host: "some_host", 333 | TLSPort: &port, 334 | Protocol: "http2", 335 | URIs: []string{"uri1", "uri2"}, 336 | RouteServiceUrl: "https://rs.example.com", 337 | Tags: map[string]string{"tag1": "val1", "tag2": "val2"}, 338 | ServerCertDomainSAN: "cf.cert.internal", 339 | } 340 | }) 341 | 342 | It("send messages", func() { 343 | registered := make(chan string) 344 | testSpyClient.Subscribe(topic, func(msg *nats.Msg) { 345 | registered <- string(msg.Data) 346 | }) 347 | 348 | // Wait for the nats library to register our callback. 349 | // We use a sleep because there's no way to know that the callback was 350 | // registered successfully (e.g. they don't provide a channel) 351 | time.Sleep(20 * time.Millisecond) 352 | 353 | err := messageBus.SendMessage(topic, route, privateInstanceId) 354 | Expect(err).ShouldNot(HaveOccurred()) 355 | 356 | // Assert that we got the right message 357 | var receivedMessage string 358 | Eventually(registered, 2).Should(Receive(&receivedMessage)) 359 | 360 | expectedRegistryMessage := messagebus.Message{ 361 | URIs: route.URIs, 362 | Host: route.Host, 363 | Port: route.Port, 364 | Protocol: route.Protocol, 365 | TLSPort: route.TLSPort, 366 | RouteServiceUrl: route.RouteServiceUrl, 367 | Tags: route.Tags, 368 | ServerCertDomainSAN: "cf.cert.internal", 369 | AvailabilityZone: "some-az", 370 | } 371 | 372 | var registryMessage messagebus.Message 373 | err = json.Unmarshal([]byte(receivedMessage), ®istryMessage) 374 | Expect(err).ShouldNot(HaveOccurred()) 375 | 376 | Expect(registryMessage.URIs).To(Equal(expectedRegistryMessage.URIs)) 377 | Expect(registryMessage.Port).To(Equal(expectedRegistryMessage.Port)) 378 | Expect(registryMessage.Host).To(Equal(expectedRegistryMessage.Host)) 379 | Expect(registryMessage.Protocol).To(Equal(expectedRegistryMessage.Protocol)) 380 | Expect(registryMessage.RouteServiceUrl).To(Equal(expectedRegistryMessage.RouteServiceUrl)) 381 | Expect(registryMessage.Tags).To(Equal(expectedRegistryMessage.Tags)) 382 | }) 383 | }) 384 | Describe("SendMessage with per-route options", func() { 385 | const ( 386 | topic = "router.registrar" 387 | privateInstanceId = "some_id" 388 | ) 389 | 390 | var ( 391 | route config.Route 392 | ) 393 | 394 | BeforeEach(func() { 395 | err := messageBus.Connect(messageBusServers, nil) 396 | Expect(err).ShouldNot(HaveOccurred()) 397 | 398 | port := uint16(12345) 399 | 400 | route = config.Route{ 401 | Name: "some_name", 402 | TLSPort: &port, 403 | Host: "some_host", 404 | URIs: []string{"uri1", "uri2"}, 405 | ServerCertDomainSAN: "cf.cert.internal", 406 | Options: &config.Options{LoadBalancingAlgorithm: config.LeastConns}, 407 | } 408 | }) 409 | 410 | It("sends messages", func() { 411 | registered := make(chan string) 412 | testSpyClient.Subscribe(topic, func(msg *nats.Msg) { 413 | registered <- string(msg.Data) 414 | }) 415 | 416 | // Wait for the nats library to register our callback. 417 | // We use a sleep because there's no way to know that the callback was 418 | // registered successfully (e.g. they don't provide a channel) 419 | time.Sleep(20 * time.Millisecond) 420 | 421 | err := messageBus.SendMessage(topic, route, privateInstanceId) 422 | Expect(err).ShouldNot(HaveOccurred()) 423 | 424 | // Assert that we got the right message 425 | var receivedMessage string 426 | Eventually(registered, 2).Should(Receive(&receivedMessage)) 427 | 428 | expectedRegistryMessage := messagebus.Message{ 429 | URIs: route.URIs, 430 | Host: route.Host, 431 | TLSPort: route.TLSPort, 432 | ServerCertDomainSAN: "cf.cert.internal", 433 | AvailabilityZone: "some-az", 434 | Options: map[string]string{"loadbalancing": string(route.Options.LoadBalancingAlgorithm)}, 435 | } 436 | 437 | var registryMessage messagebus.Message 438 | err = json.Unmarshal([]byte(receivedMessage), ®istryMessage) 439 | Expect(err).ShouldNot(HaveOccurred()) 440 | 441 | Expect(registryMessage.URIs).To(Equal(expectedRegistryMessage.URIs)) 442 | Expect(registryMessage.Protocol).To(BeEmpty()) 443 | Expect(registryMessage.AvailabilityZone).To(Equal(expectedRegistryMessage.AvailabilityZone)) 444 | 445 | Expect(registryMessage.Options).To(Equal(expectedRegistryMessage.Options)) 446 | }) 447 | 448 | Context("when the connection is already closed", func() { 449 | BeforeEach(func() { 450 | err := messageBus.Connect(messageBusServers, nil) 451 | Expect(err).ShouldNot(HaveOccurred()) 452 | 453 | messageBus.Close() 454 | }) 455 | 456 | It("returns error", func() { 457 | err := messageBus.SendMessage(topic, route, privateInstanceId) 458 | Expect(err).Should(HaveOccurred()) 459 | }) 460 | }) 461 | }) 462 | }) 463 | 464 | func startNats(host string, port int, username, password string) *exec.Cmd { 465 | fmt.Fprintf(GinkgoWriter, "Starting nats-server on port %d\n", port) 466 | 467 | natsServer, exists := os.LookupEnv("NATS_SERVER_BINARY") 468 | if !exists { 469 | fmt.Println("You need nats-server installed and set NATS_SERVER_BINARY env variable") 470 | os.Exit(1) 471 | } 472 | 473 | cmd := exec.Command( 474 | natsServer, 475 | "-p", strconv.Itoa(port), 476 | "--user", username, 477 | "--pass", password) 478 | 479 | err := cmd.Start() 480 | if err != nil { 481 | fmt.Printf("nats-server failed to start: %v\n", err) 482 | } 483 | 484 | natsTimeout := 10 * time.Second 485 | natsPollingInterval := 20 * time.Millisecond 486 | Eventually(func() error { 487 | _, err := net.Dial("tcp", fmt.Sprintf("%s:%d", host, port)) 488 | return err 489 | }, natsTimeout, natsPollingInterval).Should(Succeed()) 490 | 491 | fmt.Fprintf(GinkgoWriter, "nats-server running on port %d\n", port) 492 | return cmd 493 | } 494 | 495 | func startNatsTls(host string, port int, caFile, certFile, keyFile, username, password string) *exec.Cmd { 496 | fmt.Fprintf(GinkgoWriter, "Starting TLS nats-server on port %d\n", port) 497 | 498 | natsServer, exists := os.LookupEnv("NATS_SERVER_BINARY") 499 | if !exists { 500 | fmt.Println("You need nats-server installed and set NATS_SERVER_BINARY env variable") 501 | os.Exit(1) 502 | } 503 | cmd := exec.Command( 504 | natsServer, 505 | "-p", strconv.Itoa(port), 506 | "--tlsverify", 507 | "--tlscacert", caFile, 508 | "--tlscert", certFile, 509 | "--tlskey", keyFile, 510 | "--user", username, 511 | "--pass", password, 512 | ) 513 | 514 | err := cmd.Start() 515 | if err != nil { 516 | fmt.Printf("TLS nats-server failed to start: %v\n", err) 517 | } 518 | 519 | natsTimeout := 10 * time.Second 520 | natsPollingInterval := 20 * time.Millisecond 521 | Eventually(func() error { 522 | _, err := net.Dial("tcp", fmt.Sprintf("%s:%d", host, port)) 523 | return err 524 | }, natsTimeout, natsPollingInterval).Should(Succeed()) 525 | 526 | fmt.Fprintf(GinkgoWriter, "TLS nats-server running on port %d\n", port) 527 | return cmd 528 | } 529 | -------------------------------------------------------------------------------- /messagebus/messagebusfakes/fake_message_bus.go: -------------------------------------------------------------------------------- 1 | // Code generated by counterfeiter. DO NOT EDIT. 2 | package messagebusfakes 3 | 4 | import ( 5 | "crypto/tls" 6 | "sync" 7 | 8 | "code.cloudfoundry.org/route-registrar/config" 9 | "code.cloudfoundry.org/route-registrar/messagebus" 10 | ) 11 | 12 | type FakeMessageBus struct { 13 | CloseStub func() 14 | closeMutex sync.RWMutex 15 | closeArgsForCall []struct { 16 | } 17 | ConnectStub func([]config.MessageBusServer, *tls.Config) error 18 | connectMutex sync.RWMutex 19 | connectArgsForCall []struct { 20 | arg1 []config.MessageBusServer 21 | arg2 *tls.Config 22 | } 23 | connectReturns struct { 24 | result1 error 25 | } 26 | connectReturnsOnCall map[int]struct { 27 | result1 error 28 | } 29 | SendMessageStub func(string, config.Route, string) error 30 | sendMessageMutex sync.RWMutex 31 | sendMessageArgsForCall []struct { 32 | arg1 string 33 | arg2 config.Route 34 | arg3 string 35 | } 36 | sendMessageReturns struct { 37 | result1 error 38 | } 39 | sendMessageReturnsOnCall map[int]struct { 40 | result1 error 41 | } 42 | invocations map[string][][]interface{} 43 | invocationsMutex sync.RWMutex 44 | } 45 | 46 | func (fake *FakeMessageBus) Close() { 47 | fake.closeMutex.Lock() 48 | fake.closeArgsForCall = append(fake.closeArgsForCall, struct { 49 | }{}) 50 | stub := fake.CloseStub 51 | fake.recordInvocation("Close", []interface{}{}) 52 | fake.closeMutex.Unlock() 53 | if stub != nil { 54 | fake.CloseStub() 55 | } 56 | } 57 | 58 | func (fake *FakeMessageBus) CloseCallCount() int { 59 | fake.closeMutex.RLock() 60 | defer fake.closeMutex.RUnlock() 61 | return len(fake.closeArgsForCall) 62 | } 63 | 64 | func (fake *FakeMessageBus) CloseCalls(stub func()) { 65 | fake.closeMutex.Lock() 66 | defer fake.closeMutex.Unlock() 67 | fake.CloseStub = stub 68 | } 69 | 70 | func (fake *FakeMessageBus) Connect(arg1 []config.MessageBusServer, arg2 *tls.Config) error { 71 | var arg1Copy []config.MessageBusServer 72 | if arg1 != nil { 73 | arg1Copy = make([]config.MessageBusServer, len(arg1)) 74 | copy(arg1Copy, arg1) 75 | } 76 | fake.connectMutex.Lock() 77 | ret, specificReturn := fake.connectReturnsOnCall[len(fake.connectArgsForCall)] 78 | fake.connectArgsForCall = append(fake.connectArgsForCall, struct { 79 | arg1 []config.MessageBusServer 80 | arg2 *tls.Config 81 | }{arg1Copy, arg2}) 82 | stub := fake.ConnectStub 83 | fakeReturns := fake.connectReturns 84 | fake.recordInvocation("Connect", []interface{}{arg1Copy, arg2}) 85 | fake.connectMutex.Unlock() 86 | if stub != nil { 87 | return stub(arg1, arg2) 88 | } 89 | if specificReturn { 90 | return ret.result1 91 | } 92 | return fakeReturns.result1 93 | } 94 | 95 | func (fake *FakeMessageBus) ConnectCallCount() int { 96 | fake.connectMutex.RLock() 97 | defer fake.connectMutex.RUnlock() 98 | return len(fake.connectArgsForCall) 99 | } 100 | 101 | func (fake *FakeMessageBus) ConnectCalls(stub func([]config.MessageBusServer, *tls.Config) error) { 102 | fake.connectMutex.Lock() 103 | defer fake.connectMutex.Unlock() 104 | fake.ConnectStub = stub 105 | } 106 | 107 | func (fake *FakeMessageBus) ConnectArgsForCall(i int) ([]config.MessageBusServer, *tls.Config) { 108 | fake.connectMutex.RLock() 109 | defer fake.connectMutex.RUnlock() 110 | argsForCall := fake.connectArgsForCall[i] 111 | return argsForCall.arg1, argsForCall.arg2 112 | } 113 | 114 | func (fake *FakeMessageBus) ConnectReturns(result1 error) { 115 | fake.connectMutex.Lock() 116 | defer fake.connectMutex.Unlock() 117 | fake.ConnectStub = nil 118 | fake.connectReturns = struct { 119 | result1 error 120 | }{result1} 121 | } 122 | 123 | func (fake *FakeMessageBus) ConnectReturnsOnCall(i int, result1 error) { 124 | fake.connectMutex.Lock() 125 | defer fake.connectMutex.Unlock() 126 | fake.ConnectStub = nil 127 | if fake.connectReturnsOnCall == nil { 128 | fake.connectReturnsOnCall = make(map[int]struct { 129 | result1 error 130 | }) 131 | } 132 | fake.connectReturnsOnCall[i] = struct { 133 | result1 error 134 | }{result1} 135 | } 136 | 137 | func (fake *FakeMessageBus) SendMessage(arg1 string, arg2 config.Route, arg3 string) error { 138 | fake.sendMessageMutex.Lock() 139 | ret, specificReturn := fake.sendMessageReturnsOnCall[len(fake.sendMessageArgsForCall)] 140 | fake.sendMessageArgsForCall = append(fake.sendMessageArgsForCall, struct { 141 | arg1 string 142 | arg2 config.Route 143 | arg3 string 144 | }{arg1, arg2, arg3}) 145 | stub := fake.SendMessageStub 146 | fakeReturns := fake.sendMessageReturns 147 | fake.recordInvocation("SendMessage", []interface{}{arg1, arg2, arg3}) 148 | fake.sendMessageMutex.Unlock() 149 | if stub != nil { 150 | return stub(arg1, arg2, arg3) 151 | } 152 | if specificReturn { 153 | return ret.result1 154 | } 155 | return fakeReturns.result1 156 | } 157 | 158 | func (fake *FakeMessageBus) SendMessageCallCount() int { 159 | fake.sendMessageMutex.RLock() 160 | defer fake.sendMessageMutex.RUnlock() 161 | return len(fake.sendMessageArgsForCall) 162 | } 163 | 164 | func (fake *FakeMessageBus) SendMessageCalls(stub func(string, config.Route, string) error) { 165 | fake.sendMessageMutex.Lock() 166 | defer fake.sendMessageMutex.Unlock() 167 | fake.SendMessageStub = stub 168 | } 169 | 170 | func (fake *FakeMessageBus) SendMessageArgsForCall(i int) (string, config.Route, string) { 171 | fake.sendMessageMutex.RLock() 172 | defer fake.sendMessageMutex.RUnlock() 173 | argsForCall := fake.sendMessageArgsForCall[i] 174 | return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 175 | } 176 | 177 | func (fake *FakeMessageBus) SendMessageReturns(result1 error) { 178 | fake.sendMessageMutex.Lock() 179 | defer fake.sendMessageMutex.Unlock() 180 | fake.SendMessageStub = nil 181 | fake.sendMessageReturns = struct { 182 | result1 error 183 | }{result1} 184 | } 185 | 186 | func (fake *FakeMessageBus) SendMessageReturnsOnCall(i int, result1 error) { 187 | fake.sendMessageMutex.Lock() 188 | defer fake.sendMessageMutex.Unlock() 189 | fake.SendMessageStub = nil 190 | if fake.sendMessageReturnsOnCall == nil { 191 | fake.sendMessageReturnsOnCall = make(map[int]struct { 192 | result1 error 193 | }) 194 | } 195 | fake.sendMessageReturnsOnCall[i] = struct { 196 | result1 error 197 | }{result1} 198 | } 199 | 200 | func (fake *FakeMessageBus) Invocations() map[string][][]interface{} { 201 | fake.invocationsMutex.RLock() 202 | defer fake.invocationsMutex.RUnlock() 203 | fake.closeMutex.RLock() 204 | defer fake.closeMutex.RUnlock() 205 | fake.connectMutex.RLock() 206 | defer fake.connectMutex.RUnlock() 207 | fake.sendMessageMutex.RLock() 208 | defer fake.sendMessageMutex.RUnlock() 209 | copiedInvocations := map[string][][]interface{}{} 210 | for key, value := range fake.invocations { 211 | copiedInvocations[key] = value 212 | } 213 | return copiedInvocations 214 | } 215 | 216 | func (fake *FakeMessageBus) recordInvocation(key string, args []interface{}) { 217 | fake.invocationsMutex.Lock() 218 | defer fake.invocationsMutex.Unlock() 219 | if fake.invocations == nil { 220 | fake.invocations = map[string][][]interface{}{} 221 | } 222 | if fake.invocations[key] == nil { 223 | fake.invocations[key] = [][]interface{}{} 224 | } 225 | fake.invocations[key] = append(fake.invocations[key], args) 226 | } 227 | 228 | var _ messagebus.MessageBus = new(FakeMessageBus) 229 | -------------------------------------------------------------------------------- /registrar/periodic_health_check_close_chans.go: -------------------------------------------------------------------------------- 1 | package registrar 2 | 3 | import ( 4 | "reflect" 5 | 6 | "code.cloudfoundry.org/route-registrar/config" 7 | ) 8 | 9 | type PeriodicHealthcheckCloseChans struct { 10 | chans []PeriodicHealthcheckCloseChan 11 | } 12 | 13 | type PeriodicHealthcheckCloseChan struct { 14 | route config.Route 15 | closeChan chan struct{} 16 | } 17 | 18 | func (p *PeriodicHealthcheckCloseChans) Add(route config.Route) chan struct{} { 19 | closeChan := make(chan struct{}) 20 | p.chans = append(p.chans, PeriodicHealthcheckCloseChan{ 21 | route: route, 22 | closeChan: closeChan, 23 | }) 24 | return closeChan 25 | } 26 | 27 | func (p *PeriodicHealthcheckCloseChans) CloseForRoute(route config.Route) { 28 | for i, c := range p.chans { 29 | if reflect.DeepEqual(c.route, route) { 30 | close(c.closeChan) 31 | p.chans = append(p.chans[:i], p.chans[i+1:]...) 32 | return 33 | } 34 | } 35 | } 36 | 37 | func (p *PeriodicHealthcheckCloseChans) CloseAll() { 38 | for _, c := range p.chans { 39 | close(c.closeChan) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /registrar/registrar.go: -------------------------------------------------------------------------------- 1 | package registrar 2 | 3 | import ( 4 | "crypto/tls" 5 | "fmt" 6 | "os" 7 | "time" 8 | 9 | "code.cloudfoundry.org/tlsconfig" 10 | uuid "github.com/nu7hatch/gouuid" 11 | "github.com/tedsuo/ifrit" 12 | 13 | "code.cloudfoundry.org/route-registrar/commandrunner" 14 | "code.cloudfoundry.org/route-registrar/config" 15 | "code.cloudfoundry.org/route-registrar/healthchecker" 16 | "code.cloudfoundry.org/route-registrar/messagebus" 17 | 18 | "code.cloudfoundry.org/lager/v3" 19 | ) 20 | 21 | type Registrar interface { 22 | Run(signals <-chan os.Signal, ready chan<- struct{}) error 23 | } 24 | 25 | type api interface { 26 | RegisterRoute(route config.Route) error 27 | UnregisterRoute(route config.Route) error 28 | } 29 | 30 | type registrar struct { 31 | logger lager.Logger 32 | config config.Config 33 | healthChecker healthchecker.HealthChecker 34 | messageBus messagebus.MessageBus 35 | routingAPI api 36 | privateInstanceId string 37 | dynamicConfigDiscoveryInterval time.Duration 38 | } 39 | 40 | func NewRegistrar( 41 | clientConfig config.Config, 42 | healthChecker healthchecker.HealthChecker, 43 | logger lager.Logger, 44 | messageBus messagebus.MessageBus, 45 | routingAPI api, 46 | dynamicConfigDiscoveryInterval time.Duration, 47 | ) Registrar { 48 | aUUID, err := uuid.NewV4() 49 | if err != nil { 50 | panic(err) 51 | } 52 | return ®istrar{ 53 | config: clientConfig, 54 | logger: logger, 55 | privateInstanceId: aUUID.String(), 56 | healthChecker: healthChecker, 57 | messageBus: messageBus, 58 | routingAPI: routingAPI, 59 | dynamicConfigDiscoveryInterval: dynamicConfigDiscoveryInterval, 60 | } 61 | } 62 | 63 | func (r *registrar) Run(signals <-chan os.Signal, ready chan<- struct{}) error { 64 | var err error 65 | var tlsConfig *tls.Config 66 | 67 | if r.config.NATSmTLSConfig.Enabled { 68 | tlsConfig, err = tlsconfig.Build( 69 | tlsconfig.WithInternalServiceDefaults(), 70 | tlsconfig.WithIdentityFromFile(r.config.NATSmTLSConfig.CertPath, r.config.NATSmTLSConfig.KeyPath), 71 | ).Client( 72 | tlsconfig.WithAuthorityFromFile(r.config.NATSmTLSConfig.CAPath), 73 | ) 74 | 75 | if err != nil { 76 | return fmt.Errorf("failed building NATS mTLS config: %s", err) 77 | } 78 | } 79 | 80 | if len(r.config.MessageBusServers) > 0 { 81 | err = r.messageBus.Connect(r.config.MessageBusServers, tlsConfig) 82 | if err != nil { 83 | return err 84 | } 85 | defer r.messageBus.Close() 86 | } 87 | close(ready) 88 | 89 | nohealthcheckChan := make(chan config.Route, len(r.config.Routes)) 90 | errChan := make(chan config.Route, len(r.config.Routes)) 91 | healthyChan := make(chan config.Route, len(r.config.Routes)) 92 | unhealthyChan := make(chan config.Route, len(r.config.Routes)) 93 | 94 | periodicHealthcheckCloseChans := &PeriodicHealthcheckCloseChans{} 95 | 96 | for _, route := range r.config.Routes { 97 | closeChan := periodicHealthcheckCloseChans.Add(route) 98 | 99 | go r.periodicallyDetermineHealth( 100 | route, 101 | nohealthcheckChan, 102 | errChan, 103 | healthyChan, 104 | unhealthyChan, 105 | closeChan, 106 | ) 107 | } 108 | 109 | routeDiscovered := make(chan config.Route) 110 | routeRemoved := make(chan config.Route) 111 | 112 | var routesConfigWatcher ifrit.Runner 113 | if len(r.config.DynamicConfigGlobs) > 0 { 114 | routesConfigWatcher = NewRoutesConfigWatcher(r.logger, r.dynamicConfigDiscoveryInterval, r.config.DynamicConfigGlobs, r.config.Host, routeDiscovered, routeRemoved) 115 | } else { 116 | routesConfigWatcher = NewNoopRoutesConfigWatcher() 117 | } 118 | 119 | routesConfigWatcherProcess := ifrit.Background(routesConfigWatcher) 120 | 121 | unregistrationCount := map[string]int{} 122 | 123 | routesConfigWatcherChannel := routesConfigWatcherProcess.Wait() 124 | 125 | for { 126 | select { 127 | case route := <-nohealthcheckChan: 128 | r.logger.Info("no healthchecker found for route", lager.Data{"route": route}) 129 | 130 | err := r.registerRoutes(route) 131 | if err != nil { 132 | return err 133 | } 134 | case route := <-errChan: 135 | r.logger.Info("healthchecker errored for route", lager.Data{"route": route}) 136 | 137 | routeKey := generateRouteKey(route) 138 | if unregistrationCount[routeKey] < r.config.UnregistrationMessageLimit { 139 | err := r.unregisterRoutes(route) 140 | if err != nil { 141 | return err 142 | } 143 | 144 | unregistrationCount[routeKey]++ 145 | } 146 | case route := <-healthyChan: 147 | r.logger.Info("healthchecker returned healthy for route", lager.Data{"route": route}) 148 | 149 | routeKey := generateRouteKey(route) 150 | 151 | err := r.registerRoutes(route) 152 | if err != nil { 153 | return err 154 | } 155 | 156 | unregistrationCount[routeKey] = 0 157 | case route := <-unhealthyChan: 158 | r.logger.Info("healthchecker returned unhealthy for route", lager.Data{"route": route}) 159 | 160 | routeKey := generateRouteKey(route) 161 | if unregistrationCount[routeKey] < r.config.UnregistrationMessageLimit { 162 | err := r.unregisterRoutes(route) 163 | if err != nil { 164 | return err 165 | } 166 | 167 | unregistrationCount[routeKey]++ 168 | } 169 | case route := <-routeDiscovered: 170 | r.logger.Info("discovered route", lager.Data{"route": route}) 171 | 172 | closeChan := periodicHealthcheckCloseChans.Add(route) 173 | 174 | go r.periodicallyDetermineHealth( 175 | route, 176 | nohealthcheckChan, 177 | errChan, 178 | healthyChan, 179 | unhealthyChan, 180 | closeChan, 181 | ) 182 | 183 | case route := <-routeRemoved: 184 | r.logger.Info("route removed", lager.Data{"route": route}) 185 | periodicHealthcheckCloseChans.CloseForRoute(route) 186 | 187 | routeKey := generateRouteKey(route) 188 | if unregistrationCount[routeKey] < r.config.UnregistrationMessageLimit { 189 | err := r.unregisterRoutes(route) 190 | if err != nil { 191 | return err 192 | } 193 | 194 | unregistrationCount[routeKey]++ 195 | } 196 | 197 | case err := <-routesConfigWatcherChannel: 198 | if err != nil { 199 | r.logger.Error("config watcher failed", err) 200 | return err 201 | } 202 | 203 | case s := <-signals: 204 | r.logger.Info("Received signal; shutting down") 205 | 206 | routesConfigWatcherProcess.Signal(s) 207 | 208 | periodicHealthcheckCloseChans.CloseAll() 209 | 210 | for _, route := range r.config.Routes { 211 | err := r.unregisterRoutes(route) 212 | if err != nil { 213 | return err 214 | } 215 | } 216 | return nil 217 | } 218 | } 219 | } 220 | 221 | func (r registrar) periodicallyDetermineHealth( 222 | route config.Route, 223 | nohealthcheckChan chan<- config.Route, 224 | errChan chan<- config.Route, 225 | healthyChan chan<- config.Route, 226 | unhealthyChan chan<- config.Route, 227 | closeChan chan struct{}, 228 | ) { 229 | ticker := time.NewTicker(route.RegistrationInterval) 230 | defer ticker.Stop() 231 | 232 | // fire ticker on process startup 233 | r.determineHealth(route, nohealthcheckChan, errChan, healthyChan, unhealthyChan) 234 | for { 235 | select { 236 | case <-ticker.C: 237 | r.determineHealth(route, nohealthcheckChan, errChan, healthyChan, unhealthyChan) 238 | case <-closeChan: 239 | return 240 | } 241 | } 242 | } 243 | 244 | func (r registrar) determineHealth(route config.Route, nohealthcheckChan chan<- config.Route, errChan chan<- config.Route, healthyChan chan<- config.Route, unhealthyChan chan<- config.Route) { 245 | if route.HealthCheck == nil || route.HealthCheck.ScriptPath == "" { 246 | nohealthcheckChan <- route 247 | } else { 248 | runner := commandrunner.NewRunner(route.HealthCheck.ScriptPath) 249 | healthy, err := r.healthChecker.Check(runner, route.HealthCheck.ScriptPath, route.HealthCheck.Timeout) 250 | if err != nil { 251 | errChan <- route 252 | } else if healthy { 253 | healthyChan <- route 254 | } else { 255 | unhealthyChan <- route 256 | } 257 | } 258 | } 259 | 260 | func (r registrar) registerRoutes(route config.Route) error { 261 | r.logger.Info("Registering route", lager.Data{"route": route}) 262 | 263 | var err error 264 | if route.Type == "tcp" { 265 | err = r.routingAPI.RegisterRoute(route) 266 | } else { 267 | err = r.messageBus.SendMessage("router.register", route, r.privateInstanceId) 268 | } 269 | if err != nil { 270 | return err 271 | } 272 | 273 | r.logger.Info("Registered routes successfully") 274 | 275 | return nil 276 | } 277 | 278 | func (r registrar) unregisterRoutes(route config.Route) error { 279 | r.logger.Info("Unregistering route", lager.Data{"route": route}) 280 | 281 | var err error 282 | if route.Type == "tcp" { 283 | err = r.routingAPI.UnregisterRoute(route) 284 | } else { 285 | err = r.messageBus.SendMessage("router.unregister", route, r.privateInstanceId) 286 | } 287 | if err != nil { 288 | return err 289 | } 290 | 291 | r.logger.Info("Unregistered routes successfully") 292 | 293 | return nil 294 | } 295 | 296 | func generateRouteKey(route config.Route) string { 297 | routeKey := fmt.Sprintf("%v", route) 298 | return routeKey 299 | } 300 | -------------------------------------------------------------------------------- /registrar/registrar_suite_test.go: -------------------------------------------------------------------------------- 1 | package registrar_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | var ( 11 | natsPort int 12 | ) 13 | 14 | func TestRoute_register(t *testing.T) { 15 | RegisterFailHandler(Fail) 16 | 17 | RunSpecs(t, "Registrar Suite") 18 | } 19 | 20 | var _ = BeforeSuite(func() { 21 | natsPort = 40000 + GinkgoParallelProcess() 22 | }) 23 | -------------------------------------------------------------------------------- /registrar/registrar_test.go: -------------------------------------------------------------------------------- 1 | package registrar_test 2 | 3 | import ( 4 | "crypto/tls" 5 | "errors" 6 | "fmt" 7 | "os" 8 | "path/filepath" 9 | "time" 10 | 11 | "github.com/nats-io/nats.go" 12 | . "github.com/onsi/ginkgo/v2" 13 | . "github.com/onsi/gomega" 14 | "gopkg.in/yaml.v3" 15 | 16 | tls_helpers "code.cloudfoundry.org/cf-routing-test-helpers/tls" 17 | "code.cloudfoundry.org/lager/v3" 18 | "code.cloudfoundry.org/lager/v3/lagertest" 19 | "code.cloudfoundry.org/route-registrar/commandrunner" 20 | "code.cloudfoundry.org/route-registrar/config" 21 | healthchecker_fakes "code.cloudfoundry.org/route-registrar/healthchecker/fakes" 22 | messagebus_fakes "code.cloudfoundry.org/route-registrar/messagebus/messagebusfakes" 23 | "code.cloudfoundry.org/route-registrar/registrar" 24 | ) 25 | 26 | var _ = Describe("Registrar.RegisterRoutes", func() { 27 | var ( 28 | fakeMessageBus *messagebus_fakes.FakeMessageBus 29 | 30 | natsHost string 31 | natsUsername string 32 | natsPassword string 33 | 34 | rrConfig config.Config 35 | 36 | logger lager.Logger 37 | 38 | signals chan os.Signal 39 | ready chan struct{} 40 | 41 | r registrar.Registrar 42 | 43 | fakeHealthChecker *healthchecker_fakes.FakeHealthChecker 44 | ) 45 | 46 | BeforeEach(func() { 47 | natsUsername = "nats-user" 48 | natsPassword = "nats-pw" 49 | natsHost = "127.0.0.1" 50 | 51 | logger = lagertest.NewTestLogger("Registrar test") 52 | servers := []string{ 53 | fmt.Sprintf( 54 | "nats://%s:%s@%s:%d", 55 | natsUsername, 56 | natsPassword, 57 | natsHost, 58 | natsPort, 59 | ), 60 | } 61 | 62 | opts := nats.GetDefaultOptions() 63 | opts.Servers = servers 64 | 65 | messageBusServer := config.MessageBusServer{ 66 | Host: fmt.Sprintf("%s:%d", natsHost, natsPort), 67 | User: natsUsername, 68 | Password: natsPassword, 69 | } 70 | 71 | rrConfig = config.Config{ 72 | // doesn't matter if these are the same, just want to send a slice 73 | MessageBusServers: []config.MessageBusServer{messageBusServer, messageBusServer}, 74 | Host: "my host", 75 | NATSmTLSConfig: config.ClientTLSConfig{ 76 | Enabled: false, 77 | CertPath: "should-not-be-used", 78 | KeyPath: "should-not-be-used", 79 | CAPath: "should-not-be-used", 80 | }, 81 | UnregistrationMessageLimit: 5, 82 | } 83 | 84 | signals = make(chan os.Signal, 1) 85 | ready = make(chan struct{}, 1) 86 | port := uint16(8080) 87 | port2 := uint16(8081) 88 | 89 | registrationInterval := 100 * time.Millisecond 90 | rrConfig.Routes = []config.Route{ 91 | { 92 | Name: "my route 1", 93 | Host: "route 1 host", 94 | Port: &port, 95 | URIs: []string{ 96 | "my uri 1.1", 97 | "my uri 1.2", 98 | }, 99 | Tags: map[string]string{ 100 | "tag1.1": "value1.1", 101 | "tag1.2": "value1.2", 102 | }, 103 | RegistrationInterval: registrationInterval, 104 | }, 105 | { 106 | Name: "my route 2", 107 | Host: "route 2 host", 108 | TLSPort: &port2, 109 | URIs: []string{ 110 | "my uri 2.1", 111 | "my uri 2.2", 112 | }, 113 | Tags: map[string]string{ 114 | "tag2.1": "value2.1", 115 | "tag2.2": "value2.2", 116 | }, 117 | RegistrationInterval: registrationInterval, 118 | ServerCertDomainSAN: "my.internal.cert", 119 | }, 120 | } 121 | 122 | fakeHealthChecker = new(healthchecker_fakes.FakeHealthChecker) 123 | fakeMessageBus = new(messagebus_fakes.FakeMessageBus) 124 | 125 | r = registrar.NewRegistrar(rrConfig, fakeHealthChecker, logger, fakeMessageBus, nil, time.Minute) 126 | }) 127 | 128 | It("connects to messagebus", func() { 129 | runStatus := make(chan error) 130 | go func() { 131 | runStatus <- r.Run(signals, ready) 132 | }() 133 | <-ready 134 | 135 | Expect(fakeMessageBus.ConnectCallCount()).To(Equal(1)) 136 | _, passedTLSConfig := fakeMessageBus.ConnectArgsForCall(0) 137 | Expect(passedTLSConfig).To(BeNil()) 138 | }) 139 | 140 | Context("when the client TLS config is enabled", func() { 141 | BeforeEach(func() { 142 | rrConfig.NATSmTLSConfig.Enabled = true 143 | natsCAPath, mtlsNATSClientCertPath, mtlsNATClientKeyPath, _ := tls_helpers.GenerateCaAndMutualTlsCerts() 144 | rrConfig.NATSmTLSConfig.CAPath = natsCAPath 145 | rrConfig.NATSmTLSConfig.CertPath = mtlsNATSClientCertPath 146 | rrConfig.NATSmTLSConfig.KeyPath = mtlsNATClientKeyPath 147 | }) 148 | 149 | JustBeforeEach(func() { 150 | r = registrar.NewRegistrar(rrConfig, fakeHealthChecker, logger, fakeMessageBus, nil, time.Minute) 151 | }) 152 | 153 | It("connects to the message bus with a TLS config", func() { 154 | runStatus := make(chan error) 155 | go func() { 156 | runStatus <- r.Run(signals, ready) 157 | }() 158 | Eventually(ready).Should(BeClosed()) 159 | 160 | Expect(fakeMessageBus.ConnectCallCount()).To(Equal(1)) 161 | _, passedTLSConfig := fakeMessageBus.ConnectArgsForCall(0) 162 | Expect(passedTLSConfig).NotTo(BeNil()) 163 | }) 164 | 165 | Context("when the client TLS config is invalid", func() { 166 | BeforeEach(func() { 167 | rrConfig.NATSmTLSConfig.CertPath = "invalid" 168 | }) 169 | 170 | It("forwards the error parsing the TLS config", func() { 171 | runStatus := make(chan error) 172 | go func() { 173 | runStatus <- r.Run(signals, ready) 174 | }() 175 | 176 | var returned error 177 | Eventually(runStatus, 3).Should(Receive(&returned)) 178 | 179 | Expect(returned).To(MatchError(ContainSubstring("failed building NATS mTLS config"))) 180 | }) 181 | }) 182 | }) 183 | 184 | Context("when connecting to messagebus errors", func() { 185 | var err error 186 | 187 | BeforeEach(func() { 188 | err = errors.New("Failed to connect") 189 | 190 | fakeMessageBus.ConnectStub = func([]config.MessageBusServer, *tls.Config) error { 191 | return err 192 | } 193 | }) 194 | 195 | It("forwards the error", func() { 196 | runStatus := make(chan error) 197 | go func() { 198 | runStatus <- r.Run(signals, ready) 199 | }() 200 | 201 | returned := <-runStatus 202 | 203 | Expect(returned).To(Equal(err)) 204 | }) 205 | }) 206 | 207 | It("unregisters on shutdown", func() { 208 | runStatus := make(chan error) 209 | go func() { 210 | runStatus <- r.Run(signals, ready) 211 | }() 212 | <-ready 213 | 214 | // wait for the initial events to be sent upon calling Run(), before shutting it off 215 | Eventually(fakeMessageBus.SendMessageCallCount, 100*time.Millisecond).Should(BeNumerically(">", 1)) 216 | close(signals) 217 | err := <-runStatus 218 | Expect(err).ShouldNot(HaveOccurred()) 219 | 220 | Eventually(fakeMessageBus.SendMessageCallCount, 3).Should(BeNumerically(">", 3)) 221 | 222 | subject, route, privateInstanceId := fakeMessageBus.SendMessageArgsForCall(2) 223 | Expect(subject).To(Equal("router.unregister")) 224 | Expect(route.Name).To(Equal(rrConfig.Routes[0].Name)) 225 | Expect(route.URIs).To(Equal(rrConfig.Routes[0].URIs)) 226 | Expect(route.Port).To(Equal(rrConfig.Routes[0].Port)) 227 | Expect(route.Host).To(Equal(rrConfig.Routes[0].Host)) 228 | Expect(route.Tags).To(Equal(rrConfig.Routes[0].Tags)) 229 | Expect(privateInstanceId).NotTo(Equal("")) 230 | 231 | subject, route, privateInstanceId = fakeMessageBus.SendMessageArgsForCall(3) 232 | Expect(subject).To(Equal("router.unregister")) 233 | Expect(route.Name).To(Equal(rrConfig.Routes[1].Name)) 234 | Expect(route.URIs).To(Equal(rrConfig.Routes[1].URIs)) 235 | Expect(route.Port).To(Equal(rrConfig.Routes[1].Port)) 236 | Expect(route.Host).To(Equal(rrConfig.Routes[1].Host)) 237 | Expect(route.Tags).To(Equal(rrConfig.Routes[1].Tags)) 238 | Expect(privateInstanceId).NotTo(Equal("")) 239 | }) 240 | 241 | Context("when unregistering routes errors", func() { 242 | var err error 243 | 244 | BeforeEach(func() { 245 | err = errors.New("Failed to register") 246 | 247 | fakeMessageBus.SendMessageStub = func(string, config.Route, string) error { 248 | return err 249 | } 250 | }) 251 | 252 | It("forwards the error", func() { 253 | runStatus := make(chan error) 254 | go func() { 255 | runStatus <- r.Run(signals, ready) 256 | }() 257 | 258 | <-ready 259 | close(signals) 260 | returned := <-runStatus 261 | 262 | Expect(returned).To(Equal(err)) 263 | }) 264 | }) 265 | 266 | Context("on startup", func() { 267 | BeforeEach(func() { 268 | port := uint16(8080) 269 | rrConfig.Routes = []config.Route{ 270 | { 271 | Name: "my route 1", 272 | Port: &port, 273 | URIs: []string{ 274 | "my uri 1.1", 275 | "my uri 1.2", 276 | }, 277 | Tags: map[string]string{ 278 | "tag1.1": "value1.1", 279 | "tag1.2": "value1.2", 280 | }, 281 | // RegistrationInterval is > the wait period in our Eventually() to ensure we've triggered on initial Run() 282 | RegistrationInterval: 10 * time.Second, 283 | }, 284 | } 285 | r = registrar.NewRegistrar(rrConfig, fakeHealthChecker, logger, fakeMessageBus, nil, time.Minute) 286 | }) 287 | It("immediately registers all URIs", func() { 288 | runStatus := make(chan error) 289 | go func() { 290 | runStatus <- r.Run(signals, ready) 291 | }() 292 | <-ready 293 | 294 | Eventually(fakeMessageBus.SendMessageCallCount, 1).Should(Equal(1)) 295 | 296 | subject, route, privateInstanceId := fakeMessageBus.SendMessageArgsForCall(0) 297 | Expect(subject).To(Equal("router.register")) 298 | 299 | Expect(len(rrConfig.Routes)).To(Equal(1)) 300 | firstRoute := rrConfig.Routes[0] 301 | 302 | Expect(route.Name).To(Equal(firstRoute.Name)) 303 | Expect(route.URIs).To(Equal(firstRoute.URIs)) 304 | Expect(route.Host).To(Equal(firstRoute.Host)) 305 | Expect(route.Port).To(Equal(firstRoute.Port)) 306 | Expect(privateInstanceId).NotTo(Equal("")) 307 | }) 308 | }) 309 | 310 | Context("when configured with dynamic config blobs", func() { 311 | var ( 312 | dynamicConfigDir string 313 | port uint16 314 | ) 315 | 316 | BeforeEach(func() { 317 | port = 8080 318 | rrConfig.Routes = []config.Route{ 319 | { 320 | Name: "my route 1", 321 | Port: &port, 322 | URIs: []string{ 323 | "my uri 1.1", 324 | "my uri 1.2", 325 | }, 326 | Tags: map[string]string{ 327 | "tag1.1": "value1.1", 328 | "tag1.2": "value1.2", 329 | }, 330 | // RegistrationInterval is > the wait period in our Eventually() to ensure we've triggered on initial Run() 331 | RegistrationInterval: 1 * time.Minute, 332 | }, 333 | } 334 | 335 | var err error 336 | dynamicConfigDir, err = os.MkdirTemp(os.TempDir(), "config-") 337 | Expect(err).NotTo(HaveOccurred()) 338 | 339 | rrConfig.DynamicConfigGlobs = []string{fmt.Sprintf("%s/config.yml", dynamicConfigDir)} 340 | 341 | r = registrar.NewRegistrar(rrConfig, fakeHealthChecker, logger, fakeMessageBus, nil, 100*time.Millisecond) 342 | }) 343 | 344 | AfterEach(func() { 345 | Expect(os.RemoveAll(dynamicConfigDir)).To(Succeed()) 346 | }) 347 | 348 | It("starts health checking discovered routes", func() { 349 | runStatus := make(chan error) 350 | go func() { 351 | runStatus <- r.Run(signals, ready) 352 | }() 353 | <-ready 354 | Eventually(fakeMessageBus.SendMessageCallCount, 1).Should(Equal(1)) 355 | 356 | routesBytes, err := yaml.Marshal(registrar.RoutesConfigSchema{Routes: []config.RouteSchema{ 357 | { 358 | Name: "some-dynamic-route", 359 | Port: &port, 360 | RegistrationInterval: "1s", 361 | URIs: []string{"some-dynamic-route.apps.com"}, 362 | }, 363 | }}) 364 | Expect(err).NotTo(HaveOccurred()) 365 | err = os.WriteFile(filepath.Join(dynamicConfigDir, "config.yml"), routesBytes, 0644) 366 | Expect(err).NotTo(HaveOccurred()) 367 | 368 | Eventually(fakeMessageBus.SendMessageCallCount, 2).Should(Equal(2)) 369 | 370 | subject, route, privateInstanceId := fakeMessageBus.SendMessageArgsForCall(1) 371 | Expect(subject).To(Equal("router.register")) 372 | 373 | Expect(len(rrConfig.Routes)).To(Equal(1)) 374 | 375 | Expect(route.Name).To(Equal("some-dynamic-route")) 376 | Expect(route.URIs).To(Equal([]string{"some-dynamic-route.apps.com"})) 377 | Expect(route.Port).To(Equal(&port)) 378 | Expect(route.Host).To(Equal(rrConfig.Host)) 379 | Expect(privateInstanceId).NotTo(Equal("")) 380 | 381 | routesBytes, err = yaml.Marshal(registrar.RoutesConfigSchema{Routes: []config.RouteSchema{}}) 382 | Expect(err).NotTo(HaveOccurred()) 383 | err = os.WriteFile(filepath.Join(dynamicConfigDir, "config.yml"), routesBytes, 0644) 384 | Expect(err).NotTo(HaveOccurred()) 385 | 386 | Eventually(fakeMessageBus.SendMessageCallCount, 2).Should(Equal(3)) 387 | subject, route, privateInstanceId = fakeMessageBus.SendMessageArgsForCall(2) 388 | Expect(subject).To(Equal("router.unregister")) 389 | 390 | Expect(len(rrConfig.Routes)).To(Equal(1)) 391 | 392 | Expect(route.Name).To(Equal("some-dynamic-route")) 393 | Expect(route.URIs).To(Equal([]string{"some-dynamic-route.apps.com"})) 394 | Expect(route.Host).To(Equal(rrConfig.Host)) 395 | Expect(route.Port).To(Equal(&port)) 396 | Expect(privateInstanceId).NotTo(Equal("")) 397 | }) 398 | }) 399 | 400 | It("periodically registers all URIs for all routes", func() { 401 | runStatus := make(chan error) 402 | go func() { 403 | runStatus <- r.Run(signals, ready) 404 | }() 405 | <-ready 406 | 407 | Eventually(fakeMessageBus.SendMessageCallCount, 3).Should(BeNumerically(">", 1)) 408 | 409 | subject, route, privateInstanceId := fakeMessageBus.SendMessageArgsForCall(0) 410 | Expect(subject).To(Equal("router.register")) 411 | 412 | var firstRoute, secondRoute config.Route 413 | if route.Name == rrConfig.Routes[0].Name { 414 | firstRoute = rrConfig.Routes[0] 415 | secondRoute = rrConfig.Routes[1] 416 | } else { 417 | firstRoute = rrConfig.Routes[1] 418 | secondRoute = rrConfig.Routes[0] 419 | } 420 | 421 | Expect(route.Name).To(Equal(firstRoute.Name)) 422 | Expect(route.URIs).To(Equal(firstRoute.URIs)) 423 | Expect(route.Host).To(Equal(firstRoute.Host)) 424 | Expect(route.Port).To(Equal(firstRoute.Port)) 425 | Expect(privateInstanceId).NotTo(Equal("")) 426 | 427 | subject, route, privateInstanceId = fakeMessageBus.SendMessageArgsForCall(1) 428 | Expect(subject).To(Equal("router.register")) 429 | 430 | Expect(route.Name).To(Equal(secondRoute.Name)) 431 | Expect(route.URIs).To(Equal(secondRoute.URIs)) 432 | Expect(route.Host).To(Equal(secondRoute.Host)) 433 | Expect(route.Port).To(Equal(secondRoute.Port)) 434 | Expect(privateInstanceId).NotTo(Equal("")) 435 | }) 436 | 437 | Context("when registering routes errors", func() { 438 | var err error 439 | 440 | BeforeEach(func() { 441 | err = errors.New("Failed to register") 442 | 443 | fakeMessageBus.SendMessageStub = func(string, config.Route, string) error { 444 | return err 445 | } 446 | }) 447 | 448 | It("forwards the error", func() { 449 | runStatus := make(chan error) 450 | go func() { 451 | runStatus <- r.Run(signals, ready) 452 | }() 453 | 454 | <-ready 455 | returned := <-runStatus 456 | 457 | Expect(returned).To(Equal(err)) 458 | }) 459 | }) 460 | 461 | Context("given a healthcheck", func() { 462 | var scriptPath string 463 | 464 | BeforeEach(func() { 465 | scriptPath = "/path/to/some/script/" 466 | 467 | timeout := 100 * time.Millisecond 468 | rrConfig.Routes[0].HealthCheck = &config.HealthCheck{ 469 | Name: "My Healthcheck process", 470 | ScriptPath: scriptPath, 471 | Timeout: timeout, 472 | } 473 | rrConfig.Routes[1].HealthCheck = &config.HealthCheck{ 474 | Name: "My Healthcheck process 2", 475 | ScriptPath: scriptPath, 476 | Timeout: timeout, 477 | } 478 | 479 | r = registrar.NewRegistrar(rrConfig, fakeHealthChecker, logger, fakeMessageBus, nil, time.Minute) 480 | }) 481 | 482 | Context("and the healthcheck succeeds", func() { 483 | BeforeEach(func() { 484 | fakeHealthChecker.CheckReturns(true, nil) 485 | 486 | r = registrar.NewRegistrar(rrConfig, fakeHealthChecker, logger, fakeMessageBus, nil, time.Minute) 487 | }) 488 | 489 | It("registers routes", func() { 490 | runStatus := make(chan error) 491 | go func() { 492 | runStatus <- r.Run(signals, ready) 493 | }() 494 | <-ready 495 | 496 | Eventually(fakeMessageBus.SendMessageCallCount, 3).Should(BeNumerically(">", 1)) 497 | 498 | subject, route, privateInstanceId := fakeMessageBus.SendMessageArgsForCall(0) 499 | Expect(subject).To(Equal("router.register")) 500 | 501 | var firstRoute, secondRoute config.Route 502 | if route.Name == rrConfig.Routes[0].Name { 503 | firstRoute = rrConfig.Routes[0] 504 | secondRoute = rrConfig.Routes[1] 505 | } else { 506 | firstRoute = rrConfig.Routes[1] 507 | secondRoute = rrConfig.Routes[0] 508 | } 509 | 510 | Expect(route.Name).To(Equal(firstRoute.Name)) 511 | Expect(route.URIs).To(Equal(firstRoute.URIs)) 512 | Expect(route.Host).To(Equal(firstRoute.Host)) 513 | Expect(route.Port).To(Equal(firstRoute.Port)) 514 | Expect(privateInstanceId).NotTo(Equal("")) 515 | 516 | subject, route, privateInstanceId = fakeMessageBus.SendMessageArgsForCall(1) 517 | Expect(subject).To(Equal("router.register")) 518 | 519 | Expect(route.Name).To(Equal(secondRoute.Name)) 520 | Expect(route.URIs).To(Equal(secondRoute.URIs)) 521 | Expect(route.Host).To(Equal(secondRoute.Host)) 522 | Expect(route.Port).To(Equal(secondRoute.Port)) 523 | Expect(privateInstanceId).NotTo(Equal("")) 524 | }) 525 | 526 | Context("when registering routes errors", func() { 527 | var err error 528 | 529 | BeforeEach(func() { 530 | err = errors.New("Failed to register") 531 | 532 | fakeMessageBus.SendMessageStub = func(string, config.Route, string) error { 533 | return err 534 | } 535 | }) 536 | 537 | It("forwards the error", func() { 538 | runStatus := make(chan error) 539 | go func() { 540 | runStatus <- r.Run(signals, ready) 541 | }() 542 | 543 | <-ready 544 | returned := <-runStatus 545 | 546 | Expect(returned).To(Equal(err)) 547 | }) 548 | }) 549 | }) 550 | 551 | Context("when the healthcheck fails", func() { 552 | BeforeEach(func() { 553 | fakeHealthChecker.CheckReturns(false, nil) 554 | 555 | r = registrar.NewRegistrar(rrConfig, fakeHealthChecker, logger, fakeMessageBus, nil, time.Minute) 556 | }) 557 | 558 | It("unregisters routes", func() { 559 | runStatus := make(chan error) 560 | go func() { 561 | runStatus <- r.Run(signals, ready) 562 | }() 563 | <-ready 564 | 565 | Eventually(fakeMessageBus.SendMessageCallCount, 3).Should(BeNumerically(">", 1)) 566 | 567 | subject, route, privateInstanceId := fakeMessageBus.SendMessageArgsForCall(0) 568 | Expect(subject).To(Equal("router.unregister")) 569 | 570 | var firstRoute, secondRoute config.Route 571 | if route.Name == rrConfig.Routes[0].Name { 572 | firstRoute = rrConfig.Routes[0] 573 | secondRoute = rrConfig.Routes[1] 574 | } else { 575 | firstRoute = rrConfig.Routes[1] 576 | secondRoute = rrConfig.Routes[0] 577 | } 578 | 579 | Expect(route.Name).To(Equal(firstRoute.Name)) 580 | Expect(route.URIs).To(Equal(firstRoute.URIs)) 581 | Expect(route.Port).To(Equal(firstRoute.Port)) 582 | Expect(route.Host).To(Equal(firstRoute.Host)) 583 | Expect(privateInstanceId).NotTo(Equal("")) 584 | 585 | subject, route, privateInstanceId = fakeMessageBus.SendMessageArgsForCall(1) 586 | Expect(subject).To(Equal("router.unregister")) 587 | 588 | Expect(route.Name).To(Equal(secondRoute.Name)) 589 | Expect(route.URIs).To(Equal(secondRoute.URIs)) 590 | Expect(route.Port).To(Equal(secondRoute.Port)) 591 | Expect(route.Host).To(Equal(secondRoute.Host)) 592 | Expect(privateInstanceId).NotTo(Equal("")) 593 | }) 594 | 595 | Context("when unregistering routes errors", func() { 596 | var err error 597 | 598 | BeforeEach(func() { 599 | err = errors.New("Failed to unregister") 600 | 601 | fakeMessageBus.SendMessageStub = func(string, config.Route, string) error { 602 | return err 603 | } 604 | }) 605 | 606 | It("forwards the error", func() { 607 | runStatus := make(chan error) 608 | go func() { 609 | runStatus <- r.Run(signals, ready) 610 | }() 611 | 612 | <-ready 613 | returned := <-runStatus 614 | 615 | Expect(returned).To(Equal(err)) 616 | }) 617 | }) 618 | }) 619 | 620 | Context("when the healthcheck keeps failing", func() { 621 | Context("when there is one route with a failing endpoint", func() { 622 | BeforeEach(func() { 623 | timeout := 100 * time.Millisecond 624 | registrationInterval := 100 * time.Millisecond 625 | port := uint16(8080) 626 | rrConfig.Routes = []config.Route{ 627 | { 628 | Name: "my route 1", 629 | Port: &port, 630 | URIs: []string{ 631 | "my uri 1.1", 632 | }, 633 | Tags: map[string]string{ 634 | "tag1.1": "value1.1", 635 | "tag1.2": "value1.2", 636 | }, 637 | RegistrationInterval: registrationInterval, 638 | HealthCheck: &config.HealthCheck{ 639 | Name: "My Healthcheck process", 640 | ScriptPath: "pass", 641 | Timeout: timeout, 642 | }, 643 | }, 644 | } 645 | 646 | fakeHealthChecker.CheckReturns(false, nil) 647 | r = registrar.NewRegistrar(rrConfig, fakeHealthChecker, logger, fakeMessageBus, nil, time.Minute) 648 | }) 649 | 650 | It("only sends five unregistration messages per route", func() { 651 | runStatus := make(chan error) 652 | go func() { 653 | runStatus <- r.Run(signals, ready) 654 | }() 655 | <-ready 656 | 657 | Eventually(fakeMessageBus.SendMessageCallCount, 3).Should(Equal(5)) 658 | 659 | for i := 0; i < 5; i++ { 660 | subject, route, privateInstanceId := fakeMessageBus.SendMessageArgsForCall(i) 661 | Expect(subject).To(Equal("router.unregister")) 662 | Expect(route.Name).To(Equal(rrConfig.Routes[0].Name)) 663 | Expect(route.URIs).To(Equal(rrConfig.Routes[0].URIs)) 664 | Expect(route.Port).To(Equal(rrConfig.Routes[0].Port)) 665 | Expect(route.Host).To(Equal(rrConfig.Routes[0].Host)) 666 | Expect(privateInstanceId).NotTo(Equal("")) 667 | } 668 | 669 | Consistently(fakeMessageBus.SendMessageCallCount, 3).Should(Equal(5)) 670 | }) 671 | }) 672 | 673 | Context("when there are multiple routes with failing endpoints", func() { 674 | var ( 675 | route1Name string 676 | route2Name string 677 | ) 678 | 679 | BeforeEach(func() { 680 | timeout := 100 * time.Millisecond 681 | registrationInterval := 100 * time.Millisecond 682 | port := uint16(8080) 683 | route1Name = "my route 1" 684 | route2Name = "my route 2" 685 | rrConfig.Routes = []config.Route{ 686 | { 687 | Name: route1Name, 688 | Port: &port, 689 | URIs: []string{ 690 | "my uri 1.1", 691 | }, 692 | Tags: map[string]string{ 693 | "tag1.1": "value1.1", 694 | "tag1.2": "value1.2", 695 | }, 696 | RegistrationInterval: registrationInterval, 697 | HealthCheck: &config.HealthCheck{ 698 | Name: "My Healthcheck process", 699 | ScriptPath: "pass", 700 | Timeout: timeout, 701 | }, 702 | }, 703 | { 704 | Name: route2Name, 705 | Port: &port, 706 | URIs: []string{ 707 | "my uri 1.1", 708 | }, 709 | Tags: map[string]string{ 710 | "tag1.1": "value1.1", 711 | "tag1.2": "value1.2", 712 | }, 713 | RegistrationInterval: registrationInterval, 714 | HealthCheck: &config.HealthCheck{ 715 | Name: "My Healthcheck process", 716 | ScriptPath: "fail", 717 | Timeout: timeout, 718 | }, 719 | }, 720 | } 721 | 722 | fakeHealthChecker.CheckReturns(false, nil) 723 | r = registrar.NewRegistrar(rrConfig, fakeHealthChecker, logger, fakeMessageBus, nil, time.Minute) 724 | }) 725 | 726 | It("sends five registration messages for each route", func() { 727 | runStatus := make(chan error) 728 | go func() { 729 | runStatus <- r.Run(signals, ready) 730 | }() 731 | <-ready 732 | 733 | Eventually(fakeMessageBus.SendMessageCallCount, 3).Should(Equal(10)) 734 | 735 | route1Counter := 0 736 | route2Counter := 0 737 | 738 | for i := 0; i < 10; i++ { 739 | subject, route, _ := fakeMessageBus.SendMessageArgsForCall(i) 740 | Expect(subject).To(Equal("router.unregister")) 741 | 742 | if route.Name == route1Name { 743 | route1Counter++ 744 | } 745 | 746 | if route.Name == route2Name { 747 | route2Counter++ 748 | } 749 | } 750 | 751 | Expect(route1Counter).To(Equal(5)) 752 | Expect(route2Counter).To(Equal(5)) 753 | Consistently(fakeMessageBus.SendMessageCallCount, 3).Should(Equal(10)) 754 | }) 755 | }) 756 | Context("when one route has a failing healthcheck and another route has as passing healthcheck", func() { 757 | var ( 758 | route1Name string 759 | route2Name string 760 | ) 761 | 762 | BeforeEach(func() { 763 | timeout := 100 * time.Millisecond 764 | registrationInterval := 100 * time.Millisecond 765 | port := uint16(8080) 766 | route1Name = "my route 1" 767 | route2Name = "my route 2" 768 | rrConfig.Routes = []config.Route{ 769 | { 770 | Name: route1Name, 771 | Port: &port, 772 | Host: "my host 1", 773 | URIs: []string{ 774 | "my uri 1.1", 775 | }, 776 | Tags: map[string]string{ 777 | "tag1.1": "value1.1", 778 | "tag1.2": "value1.2", 779 | }, 780 | RegistrationInterval: registrationInterval, 781 | HealthCheck: &config.HealthCheck{ 782 | Name: "My Healthcheck process", 783 | ScriptPath: "pass", 784 | Timeout: timeout, 785 | }, 786 | }, 787 | { 788 | Name: route2Name, 789 | Port: &port, 790 | Host: "my host 2", 791 | URIs: []string{ 792 | "my uri 1.1", 793 | }, 794 | Tags: map[string]string{ 795 | "tag1.1": "value1.1", 796 | "tag1.2": "value1.2", 797 | }, 798 | RegistrationInterval: registrationInterval, 799 | HealthCheck: &config.HealthCheck{ 800 | Name: "My Healthcheck process", 801 | ScriptPath: "fail", 802 | Timeout: timeout, 803 | }, 804 | }, 805 | } 806 | 807 | fakeHealthChecker.CheckStub = func(cr commandrunner.Runner, path string, timeout time.Duration) (bool, error) { 808 | if path == "pass" { 809 | return true, nil 810 | } 811 | return false, errors.New("oh no I failed") 812 | } 813 | 814 | r = registrar.NewRegistrar(rrConfig, fakeHealthChecker, logger, fakeMessageBus, nil, time.Minute) 815 | }) 816 | 817 | It("only sends five unregistration messages for the failing app", func() { 818 | runStatus := make(chan error) 819 | go func() { 820 | runStatus <- r.Run(signals, ready) 821 | }() 822 | <-ready 823 | 824 | Eventually(fakeMessageBus.SendMessageCallCount, 3).Should(BeNumerically(">", 15)) 825 | 826 | route2Counter := 0 // failing app 827 | 828 | for i := 0; i < 15; i++ { 829 | subject, route, _ := fakeMessageBus.SendMessageArgsForCall(i) 830 | 831 | if route.Name == route1Name { 832 | Expect(subject).To(Equal("router.register")) 833 | } 834 | 835 | if route.Name == route2Name { 836 | Expect(subject).To(Equal("router.unregister")) 837 | route2Counter++ 838 | } 839 | } 840 | 841 | Expect(route2Counter).To(Equal(5)) 842 | }) 843 | }) 844 | }) 845 | 846 | Context("when a route is healthy, then becomes unhealthy, then healthy, and then unhealthy again", func() { 847 | var ( 848 | routeName string 849 | runCounter int 850 | ) 851 | 852 | BeforeEach(func() { 853 | timeout := 100 * time.Millisecond 854 | registrationInterval := 100 * time.Millisecond 855 | port := uint16(8080) 856 | routeName = "my route 1" 857 | rrConfig.Routes = []config.Route{ 858 | { 859 | Name: routeName, 860 | Host: "my host 1", 861 | Port: &port, 862 | URIs: []string{ 863 | "my uri 1.1", 864 | }, 865 | Tags: map[string]string{ 866 | "tag1.1": "value1.1", 867 | "tag1.2": "value1.2", 868 | }, 869 | RegistrationInterval: registrationInterval, 870 | HealthCheck: &config.HealthCheck{ 871 | Name: "My Healthcheck process", 872 | ScriptPath: "fail->pass->fail", 873 | Timeout: timeout, 874 | }, 875 | }, 876 | } 877 | 878 | runCounter = 0 879 | 880 | fakeHealthChecker.CheckStub = func( 881 | runner commandrunner.Runner, 882 | path string, 883 | timeout time.Duration, 884 | ) (bool, error) { 885 | runCounter++ 886 | if runCounter <= 5 { 887 | return true, nil 888 | } 889 | 890 | if runCounter <= 10 { 891 | return false, errors.New("some failure") 892 | } 893 | 894 | if runCounter <= 15 { 895 | return true, nil 896 | } 897 | 898 | return false, errors.New("some failure") 899 | } 900 | 901 | r = registrar.NewRegistrar(rrConfig, fakeHealthChecker, logger, fakeMessageBus, nil, time.Minute) 902 | }) 903 | 904 | It("registers and unregisters properly as the route's health changes", func() { 905 | runStatus := make(chan error) 906 | go func() { 907 | runStatus <- r.Run(signals, ready) 908 | }() 909 | <-ready 910 | 911 | Eventually(fakeMessageBus.SendMessageCallCount, 3).Should(Equal(20)) 912 | 913 | for i := 0; i < 20; i++ { 914 | subject, route, _ := fakeMessageBus.SendMessageArgsForCall(i) 915 | Expect(route.Name).To(Equal(routeName)) 916 | if i < 5 { 917 | Expect(subject).To(Equal("router.register")) 918 | continue 919 | } 920 | if i < 10 { 921 | Expect(subject).To(Equal("router.unregister")) 922 | continue 923 | } 924 | if i < 15 { 925 | Expect(subject).To(Equal("router.register")) 926 | continue 927 | } 928 | if i < 20 { 929 | Expect(subject).To(Equal("router.unregister")) 930 | } 931 | } 932 | }) 933 | }) 934 | 935 | Context("when the healthcheck errors", func() { 936 | var healthcheckErr error 937 | 938 | BeforeEach(func() { 939 | healthcheckErr = fmt.Errorf("boom") 940 | fakeHealthChecker.CheckReturns(true, healthcheckErr) 941 | 942 | r = registrar.NewRegistrar(rrConfig, fakeHealthChecker, logger, fakeMessageBus, nil, time.Minute) 943 | }) 944 | 945 | It("unregisters routes", func() { 946 | runStatus := make(chan error) 947 | go func() { 948 | runStatus <- r.Run(signals, ready) 949 | }() 950 | <-ready 951 | 952 | Eventually(fakeMessageBus.SendMessageCallCount, 3).Should(BeNumerically(">", 1)) 953 | 954 | subject, route, privateInstanceId := fakeMessageBus.SendMessageArgsForCall(0) 955 | Expect(subject).To(Equal("router.unregister")) 956 | 957 | var firstRoute, secondRoute config.Route 958 | if route.Name == rrConfig.Routes[0].Name { 959 | firstRoute = rrConfig.Routes[0] 960 | secondRoute = rrConfig.Routes[1] 961 | } else { 962 | firstRoute = rrConfig.Routes[1] 963 | secondRoute = rrConfig.Routes[0] 964 | } 965 | 966 | Expect(route.Name).To(Equal(firstRoute.Name)) 967 | Expect(route.URIs).To(Equal(firstRoute.URIs)) 968 | Expect(route.Host).To(Equal(firstRoute.Host)) 969 | Expect(route.Port).To(Equal(firstRoute.Port)) 970 | Expect(privateInstanceId).NotTo(Equal("")) 971 | 972 | subject, route, privateInstanceId = fakeMessageBus.SendMessageArgsForCall(1) 973 | Expect(subject).To(Equal("router.unregister")) 974 | 975 | Expect(route.Name).To(Equal(secondRoute.Name)) 976 | Expect(route.URIs).To(Equal(secondRoute.URIs)) 977 | Expect(route.Host).To(Equal(secondRoute.Host)) 978 | Expect(route.Port).To(Equal(secondRoute.Port)) 979 | Expect(privateInstanceId).NotTo(Equal("")) 980 | }) 981 | 982 | Context("when unregistering routes errors", func() { 983 | var err error 984 | 985 | BeforeEach(func() { 986 | err = errors.New("Failed to unregister") 987 | 988 | fakeMessageBus.SendMessageStub = func(string, config.Route, string) error { 989 | return err 990 | } 991 | }) 992 | 993 | It("forwards the error", func() { 994 | runStatus := make(chan error) 995 | go func() { 996 | runStatus <- r.Run(signals, ready) 997 | }() 998 | 999 | <-ready 1000 | returned := <-runStatus 1001 | 1002 | Expect(returned).To(Equal(err)) 1003 | }) 1004 | }) 1005 | }) 1006 | 1007 | Context("when the healthcheck is in progress", func() { 1008 | BeforeEach(func() { 1009 | fakeHealthChecker.CheckStub = func(commandrunner.Runner, string, time.Duration) (bool, error) { 1010 | time.Sleep(10 * time.Second) 1011 | return true, nil 1012 | } 1013 | 1014 | r = registrar.NewRegistrar(rrConfig, fakeHealthChecker, logger, fakeMessageBus, nil, time.Minute) 1015 | }) 1016 | 1017 | It("returns instantly upon interrupt", func() { 1018 | runStatus := make(chan error) 1019 | go func() { 1020 | runStatus <- r.Run(signals, ready) 1021 | }() 1022 | <-ready 1023 | 1024 | // Must be greater than the registration interval to ensure the loop runs 1025 | // at least once 1026 | time.Sleep(1500 * time.Millisecond) 1027 | 1028 | close(signals) 1029 | Eventually(runStatus, 100*time.Millisecond).Should(Receive(BeNil())) 1030 | }) 1031 | }) 1032 | }) 1033 | }) 1034 | -------------------------------------------------------------------------------- /registrar/routes_config_watcher.go: -------------------------------------------------------------------------------- 1 | package registrar 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "reflect" 7 | "time" 8 | 9 | "code.cloudfoundry.org/lager/v3" 10 | "code.cloudfoundry.org/route-registrar/config" 11 | "gopkg.in/yaml.v3" 12 | ) 13 | 14 | type RoutesConfigSchema struct { 15 | Routes []config.RouteSchema `json:"routes"` 16 | } 17 | 18 | type routesConfigWatcher struct { 19 | globs []string 20 | host string 21 | logger lager.Logger 22 | watchInterval time.Duration 23 | discoveredRoutes map[string][]config.Route 24 | routeDiscoveredChan chan config.Route 25 | routeRemovedChan chan config.Route 26 | } 27 | 28 | func NewRoutesConfigWatcher(logger lager.Logger, watchInterval time.Duration, globs []string, host string, routeDiscoveredChan chan config.Route, routeRemovedChan chan config.Route) *routesConfigWatcher { 29 | return &routesConfigWatcher{ 30 | globs: globs, 31 | host: host, 32 | logger: logger.Session("routes-config-watcher"), 33 | watchInterval: watchInterval, 34 | routeDiscoveredChan: routeDiscoveredChan, 35 | routeRemovedChan: routeRemovedChan, 36 | discoveredRoutes: map[string][]config.Route{}, 37 | } 38 | } 39 | 40 | func (r *routesConfigWatcher) Run(signals <-chan os.Signal, ready chan<- struct{}) error { 41 | close(ready) 42 | 43 | timer := time.NewTicker(r.watchInterval) 44 | for { 45 | select { 46 | case <-timer.C: 47 | err := r.discoverRoutesFromConfigFiles() 48 | if err != nil { 49 | r.logger.Error("failed-to-discover-config-files", err) 50 | return err 51 | } 52 | 53 | case s := <-signals: 54 | r.logger.Info("caught-signal", lager.Data{"signal": s}) 55 | return nil 56 | } 57 | } 58 | } 59 | 60 | func (r *routesConfigWatcher) discoverRoutesFromConfigFiles() error { 61 | allFiles := map[string]bool{} 62 | 63 | for _, glob := range r.globs { 64 | files, err := filepath.Glob(glob) 65 | if err != nil { 66 | r.logger.Error("failed-to-glob-config-files", err, lager.Data{"glob": glob}) 67 | return err 68 | } 69 | 70 | for _, f := range files { 71 | allFiles[f] = true 72 | 73 | r.registerNewRoutesFromConfigFile(f) 74 | } 75 | } 76 | 77 | for f := range r.discoveredRoutes { 78 | if _, ok := allFiles[f]; !ok { 79 | r.logger.Info("removing-routes-from-config-file", lager.Data{"file": f}) 80 | for _, route := range r.discoveredRoutes[f] { 81 | r.routeRemovedChan <- route 82 | } 83 | delete(r.discoveredRoutes, f) 84 | } 85 | } 86 | 87 | return nil 88 | } 89 | 90 | func (r *routesConfigWatcher) registerNewRoutesFromConfigFile(configFile string) { 91 | b, err := os.ReadFile(configFile) 92 | if err != nil { 93 | r.logger.Error("failed-to-read-macthed-file", err) 94 | return 95 | } 96 | var routesConfig RoutesConfigSchema 97 | err = yaml.Unmarshal(b, &routesConfig) 98 | if err != nil { 99 | r.logger.Error("failed-to-parse-file", err) 100 | return 101 | } 102 | 103 | if _, ok := r.discoveredRoutes[configFile]; !ok { 104 | r.discoveredRoutes[configFile] = []config.Route{} 105 | } 106 | 107 | configRoutes := []config.Route{} 108 | 109 | for i, routeSchema := range routesConfig.Routes { 110 | route, err := config.RouteFromSchema(routeSchema, i, r.host) 111 | if err != nil { 112 | r.logger.Error("failed-to-parse-route", err) 113 | continue 114 | } 115 | 116 | if route != nil { 117 | configRoutes = append(configRoutes, *route) 118 | 119 | if !containsRoute(r.discoveredRoutes[configFile], *route) { 120 | r.discoveredRoutes[configFile] = append(r.discoveredRoutes[configFile], *route) 121 | r.routeDiscoveredChan <- *route 122 | } 123 | } 124 | } 125 | 126 | for i, route := range r.discoveredRoutes[configFile] { 127 | if !containsRoute(configRoutes, route) { 128 | r.discoveredRoutes[configFile] = append(r.discoveredRoutes[configFile][:i], r.discoveredRoutes[configFile][i+1:]...) 129 | r.routeRemovedChan <- route 130 | } 131 | } 132 | } 133 | 134 | func containsRoute(routes []config.Route, route config.Route) bool { 135 | for _, r := range routes { 136 | if reflect.DeepEqual(r, route) { 137 | return true 138 | } 139 | } 140 | 141 | return false 142 | } 143 | 144 | type noopRoutesConfigWatcher struct{} 145 | 146 | func NewNoopRoutesConfigWatcher() *noopRoutesConfigWatcher { 147 | return &noopRoutesConfigWatcher{} 148 | } 149 | 150 | func (r *noopRoutesConfigWatcher) Run(signals <-chan os.Signal, ready chan<- struct{}) error { 151 | close(ready) 152 | 153 | <-signals 154 | return nil 155 | } 156 | -------------------------------------------------------------------------------- /registrar/routes_config_watcher_test.go: -------------------------------------------------------------------------------- 1 | package registrar_test 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | 8 | . "github.com/onsi/ginkgo/v2" 9 | . "github.com/onsi/gomega" 10 | "github.com/onsi/gomega/gbytes" 11 | "github.com/tedsuo/ifrit" 12 | "gopkg.in/yaml.v3" 13 | 14 | "code.cloudfoundry.org/lager/v3" 15 | "code.cloudfoundry.org/lager/v3/lagertest" 16 | "code.cloudfoundry.org/route-registrar/config" 17 | "code.cloudfoundry.org/route-registrar/registrar" 18 | ) 19 | 20 | var _ = Describe("RoutesConfigWatcher", func() { 21 | var ( 22 | routesConfigWatcher ifrit.Runner 23 | logger lager.Logger 24 | 25 | process ifrit.Process 26 | 27 | route1Schema, route2Schema, route3Schema, route4Schema config.RouteSchema 28 | route1, route2, route3, route4 config.Route 29 | 30 | routesDiscovered, routesRemoved chan config.Route 31 | cfgDir string 32 | glob, host string 33 | ) 34 | 35 | BeforeEach(func() { 36 | logger = lagertest.NewTestLogger("Registrar test") 37 | var err error 38 | cfgDir, err = os.MkdirTemp(os.TempDir(), "config-") 39 | Expect(err).NotTo(HaveOccurred()) 40 | 41 | glob = fmt.Sprintf("%s/config-*.yml*", cfgDir) 42 | host = "127.0.0.1" 43 | routesDiscovered = make(chan config.Route) 44 | routesRemoved = make(chan config.Route) 45 | 46 | routesConfigWatcher = registrar.NewRoutesConfigWatcher(logger, time.Second, []string{glob}, host, routesDiscovered, routesRemoved) 47 | 48 | port := uint16(8080) 49 | route1 = config.Route{ 50 | Name: "tcp-without-host", 51 | Type: "tcp", 52 | Host: "127.0.0.1", 53 | Port: &port, 54 | RouterGroup: "some-router-group", 55 | RegistrationInterval: time.Second, 56 | URIs: []string{"some-route-1.apps.com"}, 57 | Tags: map[string]string{}, 58 | } 59 | route1Schema = config.RouteSchema{ 60 | Name: "tcp-without-host", 61 | Type: "tcp", 62 | Port: &port, 63 | RouterGroup: "some-router-group", 64 | RegistrationInterval: "1s", 65 | URIs: []string{"some-route-1.apps.com"}, 66 | } 67 | 68 | route2 = config.Route{ 69 | Name: "tcp-with-host", 70 | Type: "tcp", 71 | Host: "168.0.0.1", 72 | Port: &port, 73 | RouterGroup: "some-router-group", 74 | RegistrationInterval: 2 * time.Second, 75 | URIs: []string{"some-route-2.apps.com"}, 76 | Tags: map[string]string{}, 77 | } 78 | route2Schema = config.RouteSchema{ 79 | Name: "tcp-with-host", 80 | Type: "tcp", 81 | Host: "168.0.0.1", 82 | Port: &port, 83 | RouterGroup: "some-router-group", 84 | RegistrationInterval: "2s", 85 | URIs: []string{"some-route-2.apps.com"}, 86 | } 87 | 88 | route3 = config.Route{ 89 | Name: "some-route-3", 90 | Host: "127.0.0.1", 91 | Port: &port, 92 | RegistrationInterval: 3 * time.Second, 93 | URIs: []string{"some-route-3.apps.com"}, 94 | Tags: map[string]string{}, 95 | } 96 | route3Schema = config.RouteSchema{ 97 | Name: "some-route-3", 98 | Port: &port, 99 | RegistrationInterval: "3s", 100 | URIs: []string{"some-route-3.apps.com"}, 101 | } 102 | 103 | route4 = config.Route{ 104 | Name: "some-route-4", 105 | Host: "127.0.0.1", 106 | Port: &port, 107 | RegistrationInterval: 3 * time.Second, 108 | URIs: []string{"some-route-4.apps.com"}, 109 | Tags: map[string]string{}, 110 | } 111 | route4Schema = config.RouteSchema{ 112 | Name: "some-route-4", 113 | Port: &port, 114 | RegistrationInterval: "3s", 115 | URIs: []string{"some-route-4.apps.com"}, 116 | } 117 | }) 118 | 119 | JustBeforeEach(func() { 120 | process = ifrit.Background(routesConfigWatcher) 121 | }) 122 | 123 | AfterEach(func() { 124 | process.Signal(os.Interrupt) 125 | Eventually(process.Wait(), 5*time.Second).Should(Receive()) 126 | os.RemoveAll(cfgDir) 127 | }) 128 | 129 | Context("when directory has no files", func() { 130 | It("does not discover any routes", func() { 131 | Consistently(routesDiscovered).ShouldNot(Receive()) 132 | }) 133 | 134 | Context("when config file is created", func() { 135 | It("loads all routes from the config", func() { 136 | Consistently(routesDiscovered).ShouldNot(Receive()) 137 | 138 | routesBytes1, err := yaml.Marshal(registrar.RoutesConfigSchema{Routes: []config.RouteSchema{route1Schema, route2Schema}}) 139 | Expect(err).NotTo(HaveOccurred()) 140 | cfgFile1, err := os.CreateTemp(cfgDir, "config-1.yml") 141 | Expect(err).NotTo(HaveOccurred()) 142 | _, err = cfgFile1.Write(routesBytes1) 143 | Expect(err).NotTo(HaveOccurred()) 144 | 145 | routesBytes2, err := yaml.Marshal(registrar.RoutesConfigSchema{Routes: []config.RouteSchema{route3Schema}}) 146 | Expect(err).NotTo(HaveOccurred()) 147 | cfgFile2, err := os.CreateTemp(cfgDir, "config-2.yml") 148 | Expect(err).NotTo(HaveOccurred()) 149 | _, err = cfgFile2.Write(routesBytes2) 150 | Expect(err).NotTo(HaveOccurred()) 151 | 152 | var receivedRoute config.Route 153 | Eventually(routesDiscovered, 2).Should(Receive(&receivedRoute)) 154 | Expect(receivedRoute).To(Equal(route1)) 155 | Eventually(routesDiscovered, 2).Should(Receive(&receivedRoute)) 156 | Expect(receivedRoute).To(Equal(route2)) 157 | Eventually(routesDiscovered, 2).Should(Receive(&receivedRoute)) 158 | Expect(receivedRoute).To(Equal(route3)) 159 | }) 160 | }) 161 | }) 162 | 163 | Context("when directory has config files already", func() { 164 | var ( 165 | cfgFile1 *os.File 166 | cfgFile2 *os.File 167 | ) 168 | 169 | BeforeEach(func() { 170 | var err error 171 | cfgFile1, err = os.CreateTemp(cfgDir, "config-1.yml") 172 | Expect(err).NotTo(HaveOccurred()) 173 | 174 | cfgFile2, err = os.CreateTemp(cfgDir, "config-2.yml") 175 | Expect(err).NotTo(HaveOccurred()) 176 | }) 177 | 178 | AfterEach(func() { 179 | os.Remove(cfgFile1.Name()) 180 | os.Remove(cfgFile2.Name()) 181 | }) 182 | 183 | Context("when config files have routes", func() { 184 | BeforeEach(func() { 185 | routesBytes1, err := yaml.Marshal(registrar.RoutesConfigSchema{Routes: []config.RouteSchema{route1Schema, route2Schema}}) 186 | Expect(err).NotTo(HaveOccurred()) 187 | _, err = cfgFile1.Write(routesBytes1) 188 | Expect(err).NotTo(HaveOccurred()) 189 | 190 | routesBytes2, err := yaml.Marshal(registrar.RoutesConfigSchema{Routes: []config.RouteSchema{route3Schema}}) 191 | Expect(err).NotTo(HaveOccurred()) 192 | _, err = cfgFile2.Write(routesBytes2) 193 | Expect(err).NotTo(HaveOccurred()) 194 | }) 195 | 196 | It("loads all routes from the config", func() { 197 | var receivedRoute config.Route 198 | Eventually(routesDiscovered, 2).Should(Receive(&receivedRoute)) 199 | Expect(receivedRoute).To(Equal(route1)) 200 | Eventually(routesDiscovered, 2).Should(Receive(&receivedRoute)) 201 | Expect(receivedRoute).To(Equal(route2)) 202 | Eventually(routesDiscovered, 2).Should(Receive(&receivedRoute)) 203 | Expect(receivedRoute).To(Equal(route3)) 204 | }) 205 | 206 | Context("when config file is updated and new route is added", func() { 207 | It("notifies that route is discovered", func() { 208 | var receivedRoute config.Route 209 | Eventually(routesDiscovered, 2).Should(Receive(&receivedRoute)) 210 | Expect(receivedRoute).To(Equal(route1)) 211 | Eventually(routesDiscovered, 2).Should(Receive(&receivedRoute)) 212 | Expect(receivedRoute).To(Equal(route2)) 213 | Eventually(routesDiscovered, 2).Should(Receive(&receivedRoute)) 214 | Expect(receivedRoute).To(Equal(route3)) 215 | 216 | routesBytes2, err := yaml.Marshal(registrar.RoutesConfigSchema{Routes: []config.RouteSchema{route3Schema, route4Schema}}) 217 | Expect(err).NotTo(HaveOccurred()) 218 | err = cfgFile2.Truncate(0) 219 | Expect(err).NotTo(HaveOccurred()) 220 | _, err = cfgFile2.Seek(0, 0) 221 | Expect(err).NotTo(HaveOccurred()) 222 | _, err = cfgFile2.Write(routesBytes2) 223 | Expect(err).NotTo(HaveOccurred()) 224 | 225 | Eventually(routesDiscovered, 2).Should(Receive(&receivedRoute)) 226 | Expect(receivedRoute).To(Equal(route4)) 227 | }) 228 | }) 229 | 230 | Context("when config file is updated and route is removed", func() { 231 | It("notifies that route is removed", func() { 232 | var receivedRoute config.Route 233 | Eventually(routesDiscovered, 2).Should(Receive(&receivedRoute)) 234 | Expect(receivedRoute).To(Equal(route1)) 235 | Eventually(routesDiscovered, 2).Should(Receive(&receivedRoute)) 236 | Expect(receivedRoute).To(Equal(route2)) 237 | Eventually(routesDiscovered, 2).Should(Receive(&receivedRoute)) 238 | Expect(receivedRoute).To(Equal(route3)) 239 | 240 | routesBytes1, err := yaml.Marshal(registrar.RoutesConfigSchema{Routes: []config.RouteSchema{route1Schema}}) 241 | Expect(err).NotTo(HaveOccurred()) 242 | err = cfgFile1.Truncate(0) 243 | Expect(err).NotTo(HaveOccurred()) 244 | _, err = cfgFile1.Seek(0, 0) 245 | Expect(err).NotTo(HaveOccurred()) 246 | _, err = cfgFile1.Write(routesBytes1) 247 | Expect(err).NotTo(HaveOccurred()) 248 | 249 | Eventually(routesRemoved, 2).Should(Receive(&receivedRoute)) 250 | Expect(receivedRoute).To(Equal(route2)) 251 | 252 | Consistently(routesRemoved).ShouldNot(Receive()) 253 | }) 254 | }) 255 | 256 | Context("when config file is removed", func() { 257 | It("removes routes from that config file", func() { 258 | var receivedRoute config.Route 259 | Eventually(routesDiscovered, 2).Should(Receive(&receivedRoute)) 260 | Expect(receivedRoute).To(Equal(route1)) 261 | Eventually(routesDiscovered, 2).Should(Receive(&receivedRoute)) 262 | Expect(receivedRoute).To(Equal(route2)) 263 | Eventually(routesDiscovered, 2).Should(Receive(&receivedRoute)) 264 | Expect(receivedRoute).To(Equal(route3)) 265 | 266 | err := os.Remove(cfgFile1.Name()) 267 | Expect(err).NotTo(HaveOccurred()) 268 | 269 | Eventually(routesRemoved, 2).Should(Receive(&receivedRoute)) 270 | Expect(receivedRoute).To(Equal(route1)) 271 | 272 | Eventually(routesRemoved, 2).Should(Receive(&receivedRoute)) 273 | Expect(receivedRoute).To(Equal(route2)) 274 | }) 275 | }) 276 | }) 277 | 278 | Context("when config file is in wrong format", func() { 279 | BeforeEach(func() { 280 | _, err := cfgFile1.Write([]byte(`invalid`)) 281 | Expect(err).NotTo(HaveOccurred()) 282 | }) 283 | 284 | It("logs an error and continues scaning", func() { 285 | Eventually(logger, 2).Should(gbytes.Say("failed-to-parse-file")) 286 | }) 287 | }) 288 | 289 | Context("when host is not set globally and in config file", func() { 290 | BeforeEach(func() { 291 | routesConfigWatcher = registrar.NewRoutesConfigWatcher(logger, time.Second, []string{glob}, "", routesDiscovered, routesRemoved) 292 | port := uint16(8080) 293 | routesBytes1, err := yaml.Marshal(registrar.RoutesConfigSchema{Routes: []config.RouteSchema{ 294 | { 295 | Name: "tcp-without-host", 296 | Type: "tcp", 297 | Port: &port, 298 | RouterGroup: "some-router-group", 299 | RegistrationInterval: "1s", 300 | URIs: []string{"some-route-1.apps.com"}, 301 | }, 302 | }}) 303 | Expect(err).NotTo(HaveOccurred()) 304 | _, err = cfgFile1.Write(routesBytes1) 305 | Expect(err).NotTo(HaveOccurred()) 306 | }) 307 | 308 | It("logs an error and continues scaning", func() { 309 | Eventually(logger, 2).Should(gbytes.Say("failed-to-parse-route")) 310 | Eventually(logger).Should(gbytes.Say("no host")) 311 | }) 312 | }) 313 | }) 314 | }) 315 | -------------------------------------------------------------------------------- /routingapi/api.go: -------------------------------------------------------------------------------- 1 | package routingapi 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "code.cloudfoundry.org/route-registrar/config" 9 | "golang.org/x/oauth2" 10 | 11 | "code.cloudfoundry.org/routing-api/models" 12 | 13 | "code.cloudfoundry.org/lager/v3" 14 | routing_api "code.cloudfoundry.org/routing-api" 15 | ) 16 | 17 | type RoutingAPI struct { 18 | logger lager.Logger 19 | uaaClient uaaClient 20 | apiClient routing_api.Client 21 | routerGroupGUID map[string]string 22 | 23 | routingAPIMaxTTL time.Duration 24 | } 25 | 26 | //go:generate counterfeiter . uaaClient 27 | type uaaClient interface { 28 | FetchToken(context.Context, bool) (*oauth2.Token, error) 29 | } 30 | 31 | func NewRoutingAPI(logger lager.Logger, uaaClient uaaClient, apiClient routing_api.Client, routingAPIMaxTTL time.Duration) *RoutingAPI { 32 | return &RoutingAPI{ 33 | uaaClient: uaaClient, 34 | apiClient: apiClient, 35 | logger: logger, 36 | routerGroupGUID: make(map[string]string), 37 | 38 | routingAPIMaxTTL: routingAPIMaxTTL, 39 | } 40 | } 41 | 42 | func (r *RoutingAPI) refreshToken() error { 43 | r.logger.Info("refresh-token") 44 | token, err := r.uaaClient.FetchToken(context.Background(), false) 45 | if err != nil { 46 | r.logger.Error("token-error", err) 47 | return err 48 | } 49 | 50 | r.logger.Debug("set-token", lager.Data{"token": token}) 51 | r.apiClient.SetToken(token.AccessToken) 52 | return nil 53 | } 54 | 55 | func (r *RoutingAPI) getRouterGroupGUID(name string) (string, error) { 56 | guid, exists := r.routerGroupGUID[name] 57 | if exists { 58 | return guid, nil 59 | } 60 | 61 | routerGroup, err := r.apiClient.RouterGroupWithName(name) 62 | if err != nil { 63 | return "", err 64 | } 65 | if routerGroup.Guid == "" { 66 | return "", fmt.Errorf("Router group '%s' not found", name) 67 | } 68 | 69 | r.logger.Info("Mapped new router group", lager.Data{ 70 | "router_group": name, 71 | "guid": routerGroup.Guid}) 72 | 73 | r.routerGroupGUID[name] = routerGroup.Guid 74 | return routerGroup.Guid, nil 75 | } 76 | 77 | func (r *RoutingAPI) makeTcpRouteMapping(route config.Route) (models.TcpRouteMapping, error) { 78 | routerGroupGUID, err := r.getRouterGroupGUID(route.RouterGroup) 79 | if err != nil { 80 | return models.TcpRouteMapping{}, err 81 | } 82 | 83 | r.logger.Info("Creating mapping", lager.Data{}) 84 | 85 | return models.NewTcpRouteMapping( 86 | routerGroupGUID, 87 | *route.ExternalPort, 88 | route.Host, 89 | *route.Port, 90 | -1, 91 | "", 92 | nilIfEmpty(&route.ServerCertDomainSAN), 93 | calculateTTL(route.RegistrationInterval, r.routingAPIMaxTTL), 94 | models.ModificationTag{}, 95 | ), nil 96 | } 97 | 98 | const TTL_BUFFER float64 = 2.1 99 | 100 | // add a buffer to the registration interval so that it is not the same as the 101 | // TTL 102 | func calculateTTL(requestedTTL, maxTTL time.Duration) int { 103 | ttl := time.Duration(float64(requestedTTL) * TTL_BUFFER) 104 | if ttl > maxTTL { 105 | return int(maxTTL.Seconds()) 106 | } 107 | // ensure a bare minimum of TTL in case registration interval is <1s 108 | if int(ttl.Seconds()) < 1 { 109 | return 1 110 | } 111 | return int(ttl.Seconds()) 112 | } 113 | 114 | func nilIfEmpty(str *string) *string { 115 | if str == nil || *str == "" { 116 | return nil 117 | } 118 | return str 119 | } 120 | 121 | func (r *RoutingAPI) RegisterRoute(route config.Route) error { 122 | err := r.refreshToken() 123 | if err != nil { 124 | r.logger.Error("Failed to refresh UAA token", err) 125 | return err 126 | } 127 | 128 | routeMapping, err := r.makeTcpRouteMapping(route) 129 | if err != nil { 130 | r.logger.Error("Failed to make route mapping", err, lager.Data{"route": route}) 131 | return err 132 | } 133 | 134 | err = r.apiClient.UpsertTcpRouteMappings([]models.TcpRouteMapping{ 135 | routeMapping}) 136 | if err != nil { 137 | r.logger.Error("Failed to upsert route mapping", err, lager.Data{"route-mapping": routeMapping}) 138 | return err 139 | } 140 | 141 | r.logger.Info("Upserted route", lager.Data{"route-mapping": routeMapping}) 142 | return nil 143 | } 144 | 145 | func (r *RoutingAPI) UnregisterRoute(route config.Route) error { 146 | err := r.refreshToken() 147 | if err != nil { 148 | r.logger.Error("Failed to refresh UAA token", err) 149 | return err 150 | } 151 | 152 | routeMapping, err := r.makeTcpRouteMapping(route) 153 | if err != nil { 154 | r.logger.Error("Failed to make route mapping", err, lager.Data{"route": route}) 155 | return err 156 | } 157 | 158 | err = r.apiClient.DeleteTcpRouteMappings([]models.TcpRouteMapping{routeMapping}) 159 | if err != nil { 160 | r.logger.Error("Failed to delete route mapping", err, lager.Data{"route-mapping": routeMapping}) 161 | return err 162 | } 163 | r.logger.Info("Deleted route", lager.Data{"route-mapping": routeMapping}) 164 | return nil 165 | } 166 | -------------------------------------------------------------------------------- /routingapi/api_test.go: -------------------------------------------------------------------------------- 1 | package routingapi 2 | 3 | import ( 4 | "time" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | "golang.org/x/oauth2" 9 | 10 | fakeuaa "code.cloudfoundry.org/route-registrar/routingapi/routingapifakes" 11 | 12 | "code.cloudfoundry.org/lager/v3" 13 | "code.cloudfoundry.org/lager/v3/lagertest" 14 | "code.cloudfoundry.org/route-registrar/config" 15 | "code.cloudfoundry.org/routing-api/fake_routing_api" 16 | "code.cloudfoundry.org/routing-api/models" 17 | ) 18 | 19 | var _ = Describe("Routing API", func() { 20 | var ( 21 | client *fake_routing_api.FakeClient 22 | uaaClient *fakeuaa.FakeUaaClient 23 | 24 | api *RoutingAPI 25 | logger lager.Logger 26 | 27 | port uint16 28 | externalPort uint16 29 | ) 30 | 31 | BeforeEach(func() { 32 | logger = lagertest.NewTestLogger("routing api test") 33 | uaaClient = &fakeuaa.FakeUaaClient{} 34 | uaaClient.FetchTokenReturns(&oauth2.Token{AccessToken: "my-token"}, nil) 35 | client = &fake_routing_api.FakeClient{} 36 | client.RouterGroupWithNameReturns(models.RouterGroup{Guid: "router-group-guid"}, nil) 37 | api = NewRoutingAPI(logger, uaaClient, client, 2*time.Minute) 38 | 39 | port = 1234 40 | externalPort = 5678 41 | }) 42 | 43 | It("Sets SNI hostname if ServerCertDomainSAN is present.", func() { 44 | tcpRouteMapping, err := api.makeTcpRouteMapping(config.Route{ 45 | Port: &port, 46 | ExternalPort: &externalPort, 47 | RouterGroup: "my-router-group", 48 | ServerCertDomainSAN: "sniHostname", 49 | }) 50 | Expect(err).NotTo(HaveOccurred()) 51 | Expect(tcpRouteMapping.SniHostname).ToNot(BeNil()) 52 | Expect(*tcpRouteMapping.SniHostname).To(Equal("sniHostname")) 53 | }) 54 | 55 | It("SNI hostname nil if ServerCertDomainSAN is not present.", func() { 56 | tcpRouteMapping, err := api.makeTcpRouteMapping(config.Route{ 57 | Port: &port, 58 | ExternalPort: &externalPort, 59 | RouterGroup: "my-router-group", 60 | }) 61 | Expect(err).NotTo(HaveOccurred()) 62 | Expect(tcpRouteMapping.SniHostname).To(BeNil()) 63 | }) 64 | }) 65 | -------------------------------------------------------------------------------- /routingapi/init_test.go: -------------------------------------------------------------------------------- 1 | package routingapi_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestRoutingapi(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Routing API Suite") 13 | } 14 | -------------------------------------------------------------------------------- /routingapi/routingapi_test.go: -------------------------------------------------------------------------------- 1 | package routingapi_test 2 | 3 | import ( 4 | "errors" 5 | "time" 6 | 7 | . "github.com/onsi/ginkgo/v2" 8 | . "github.com/onsi/gomega" 9 | "golang.org/x/oauth2" 10 | 11 | "code.cloudfoundry.org/lager/v3" 12 | "code.cloudfoundry.org/route-registrar/config" 13 | "code.cloudfoundry.org/route-registrar/routingapi" 14 | "code.cloudfoundry.org/routing-api/fake_routing_api" 15 | "code.cloudfoundry.org/routing-api/models" 16 | 17 | "code.cloudfoundry.org/lager/v3/lagertest" 18 | fakeuaa "code.cloudfoundry.org/route-registrar/routingapi/routingapifakes" 19 | ) 20 | 21 | var _ = Describe("Routing API", func() { 22 | var ( 23 | client *fake_routing_api.FakeClient 24 | uaaClient *fakeuaa.FakeUaaClient 25 | 26 | api *routingapi.RoutingAPI 27 | logger lager.Logger 28 | 29 | port uint16 30 | externalPort uint16 31 | registrationInterval time.Duration 32 | 33 | maxTTL time.Duration 34 | ) 35 | 36 | BeforeEach(func() { 37 | maxTTL = 2 * time.Minute 38 | 39 | logger = lagertest.NewTestLogger("routing api test") 40 | uaaClient = &fakeuaa.FakeUaaClient{} 41 | uaaClient.FetchTokenReturns(&oauth2.Token{AccessToken: "my-token"}, nil) 42 | client = &fake_routing_api.FakeClient{} 43 | api = routingapi.NewRoutingAPI(logger, uaaClient, client, maxTTL) 44 | 45 | port = 1234 46 | externalPort = 5678 47 | registrationInterval = 20 * time.Second 48 | }) 49 | 50 | Describe("RegisterRoute", func() { 51 | BeforeEach(func() { 52 | client.RouterGroupWithNameReturns(models.RouterGroup{Guid: "router-group-guid"}, nil) 53 | }) 54 | 55 | Context("when given a valid route", func() { 56 | It("registers the route using TTL that is larger than the registration interval", func() { 57 | err := api.RegisterRoute(config.Route{ 58 | Name: "test-route", 59 | Port: &port, 60 | ExternalPort: &externalPort, 61 | Host: "myhost", 62 | RegistrationInterval: registrationInterval, 63 | RouterGroup: "my-router-group", 64 | }) 65 | Expect(err).NotTo(HaveOccurred()) 66 | Expect(uaaClient.FetchTokenCallCount()).To(Equal(1)) 67 | 68 | Expect(client.SetTokenCallCount()).To(Equal(1)) 69 | Expect(client.SetTokenArgsForCall(0)).To(Equal("my-token")) 70 | 71 | Expect(client.RouterGroupWithNameCallCount()).To(Equal(1)) 72 | Expect(client.RouterGroupWithNameArgsForCall(0)).To(Equal("my-router-group")) 73 | 74 | expectedTTL := 42 75 | expectedRouteMapping := models.TcpRouteMapping{TcpMappingEntity: models.TcpMappingEntity{ 76 | RouterGroupGuid: "router-group-guid", 77 | HostPort: 1234, 78 | ExternalPort: 5678, 79 | HostIP: "myhost", 80 | HostTLSPort: -1, 81 | SniHostname: nil, 82 | InstanceId: "", 83 | TTL: &expectedTTL, 84 | }} 85 | Expect(client.UpsertTcpRouteMappingsCallCount()).To(Equal(1)) 86 | Expect(client.UpsertTcpRouteMappingsArgsForCall(0)).To(Equal([]models.TcpRouteMapping{expectedRouteMapping})) 87 | }) 88 | 89 | Context("when the registration interval results in a TTL > maxTTL", func() { 90 | It("Caps the maxTTL", func() { 91 | err := api.RegisterRoute(config.Route{ 92 | Name: "test-route", 93 | Port: &port, 94 | ExternalPort: &externalPort, 95 | Host: "myhost", 96 | RegistrationInterval: time.Duration(100 * time.Second), 97 | RouterGroup: "my-router-group", 98 | }) 99 | Expect(err).NotTo(HaveOccurred()) 100 | 101 | expectedTTL := 120 102 | expectedRouteMapping := models.TcpRouteMapping{TcpMappingEntity: models.TcpMappingEntity{ 103 | RouterGroupGuid: "router-group-guid", 104 | HostPort: 1234, 105 | ExternalPort: 5678, 106 | HostIP: "myhost", 107 | HostTLSPort: -1, 108 | SniHostname: nil, 109 | InstanceId: "", 110 | TTL: &expectedTTL, 111 | }} 112 | Expect(client.UpsertTcpRouteMappingsCallCount()).To(Equal(1)) 113 | Expect(client.UpsertTcpRouteMappingsArgsForCall(0)).To(Equal([]models.TcpRouteMapping{expectedRouteMapping})) 114 | }) 115 | }) 116 | }) 117 | 118 | Context("when the registration interval is equal to the max_ttl for routing api", func() { 119 | BeforeEach(func() { 120 | registrationInterval = maxTTL 121 | }) 122 | 123 | It("does not add a buffer and caps TTL at max_ttl", func() { 124 | expectedRegistrationInterval := int(maxTTL.Seconds()) 125 | 126 | err := api.RegisterRoute(config.Route{ 127 | Name: "test-route", 128 | Port: &port, 129 | ExternalPort: &externalPort, 130 | Host: "myhost", 131 | RegistrationInterval: registrationInterval, 132 | RouterGroup: "my-router-group", 133 | }) 134 | 135 | Expect(err).ToNot(HaveOccurred()) 136 | Expect(client.UpsertTcpRouteMappingsCallCount()).To(Equal(1)) 137 | Expect(client.UpsertTcpRouteMappingsArgsForCall(0)[0].TcpMappingEntity.TTL).To(Equal(&expectedRegistrationInterval)) 138 | }) 139 | }) 140 | 141 | Context("when the registration interval is greater than the max_ttl for routing api", func() { 142 | BeforeEach(func() { 143 | registrationInterval = maxTTL + 10 144 | }) 145 | 146 | It("caps TTL at max_ttl", func() { 147 | expectedRegistrationInterval := int(maxTTL.Seconds()) 148 | 149 | err := api.RegisterRoute(config.Route{ 150 | Name: "test-route", 151 | Port: &port, 152 | ExternalPort: &externalPort, 153 | Host: "myhost", 154 | RegistrationInterval: registrationInterval, 155 | RouterGroup: "my-router-group", 156 | }) 157 | 158 | Expect(err).ToNot(HaveOccurred()) 159 | Expect(client.UpsertTcpRouteMappingsCallCount()).To(Equal(1)) 160 | Expect(client.UpsertTcpRouteMappingsArgsForCall(0)[0].TcpMappingEntity.TTL).To(Equal(&expectedRegistrationInterval)) 161 | }) 162 | }) 163 | 164 | Context("when the route mapping fails to register", func() { 165 | BeforeEach(func() { 166 | client.UpsertTcpRouteMappingsReturns(errors.New("registration error")) 167 | }) 168 | 169 | It("returns an error", func() { 170 | err := api.RegisterRoute(config.Route{ 171 | Name: "test-route", 172 | Port: &port, 173 | ExternalPort: &externalPort, 174 | Host: "myhost", 175 | RegistrationInterval: time.Duration(registrationInterval) * time.Second, 176 | RouterGroup: "my-router-group", 177 | }) 178 | 179 | Expect(err).To(HaveOccurred()) 180 | Expect(err).To(MatchError("registration error")) 181 | }) 182 | }) 183 | }) 184 | 185 | Describe("UnregisterRoute", func() { 186 | BeforeEach(func() { 187 | client.RouterGroupWithNameReturns(models.RouterGroup{Guid: "router-group-guid"}, nil) 188 | }) 189 | 190 | Context("when given a valid route", func() { 191 | It("unregisters the route", func() { 192 | err := api.UnregisterRoute(config.Route{ 193 | Name: "test-route", 194 | Port: &port, 195 | ExternalPort: &externalPort, 196 | Host: "myhost", 197 | RegistrationInterval: registrationInterval, 198 | RouterGroup: "my-router-group", 199 | }) 200 | Expect(err).NotTo(HaveOccurred()) 201 | Expect(uaaClient.FetchTokenCallCount()).To(Equal(1)) 202 | 203 | Expect(client.SetTokenCallCount()).To(Equal(1)) 204 | Expect(client.SetTokenArgsForCall(0)).To(Equal("my-token")) 205 | 206 | Expect(client.RouterGroupWithNameCallCount()).To(Equal(1)) 207 | Expect(client.RouterGroupWithNameArgsForCall(0)).To(Equal("my-router-group")) 208 | 209 | expectedTTL := 42 210 | routeMapping := models.TcpRouteMapping{TcpMappingEntity: models.TcpMappingEntity{ 211 | RouterGroupGuid: "router-group-guid", 212 | HostPort: 1234, 213 | ExternalPort: 5678, 214 | HostIP: "myhost", 215 | TTL: &expectedTTL, 216 | HostTLSPort: -1, 217 | SniHostname: nil, 218 | InstanceId: "", 219 | }} 220 | 221 | Expect(client.DeleteTcpRouteMappingsCallCount()).To(Equal(1)) 222 | Expect(client.DeleteTcpRouteMappingsArgsForCall(0)).To(Equal([]models.TcpRouteMapping{routeMapping})) 223 | }) 224 | }) 225 | 226 | Context("when the route mapping fails to unregister", func() { 227 | BeforeEach(func() { 228 | client.DeleteTcpRouteMappingsReturns(errors.New("unregistration error")) 229 | }) 230 | It("returns an error", func() { 231 | err := api.UnregisterRoute(config.Route{ 232 | Name: "test-route", 233 | Port: &port, 234 | ExternalPort: &externalPort, 235 | Host: "myhost", 236 | RegistrationInterval: time.Duration(registrationInterval) * time.Second, 237 | RouterGroup: "my-router-group", 238 | }) 239 | 240 | Expect(err).To(HaveOccurred()) 241 | Expect(err).To(MatchError("unregistration error")) 242 | }) 243 | }) 244 | }) 245 | 246 | Context("when an error occurs", func() { 247 | Context("when a UAA token cannot be fetched", func() { 248 | BeforeEach(func() { 249 | uaaClient.FetchTokenReturns(&oauth2.Token{}, errors.New("my fetch error")) 250 | }) 251 | 252 | It("returns an error", func() { 253 | err := api.RegisterRoute(config.Route{}) 254 | Expect(uaaClient.FetchTokenCallCount()).To(Equal(1)) 255 | Expect(err).To(HaveOccurred()) 256 | Expect(err).To(MatchError("my fetch error")) 257 | }) 258 | }) 259 | 260 | Context("when the router group name fails to return", func() { 261 | BeforeEach(func() { 262 | client.RouterGroupWithNameReturns(models.RouterGroup{}, errors.New("my router group failed")) 263 | }) 264 | 265 | It("returns an error", func() { 266 | err := api.RegisterRoute(config.Route{ 267 | Name: "test-route", 268 | Port: &port, 269 | ExternalPort: &externalPort, 270 | Host: "myhost", 271 | RegistrationInterval: time.Duration(registrationInterval) * time.Second, 272 | RouterGroup: "my-router-group", 273 | }) 274 | Expect(err).To(HaveOccurred()) 275 | Expect(err).To(MatchError("my router group failed")) 276 | }) 277 | }) 278 | }) 279 | }) 280 | -------------------------------------------------------------------------------- /routingapi/routingapifakes/fake_uaa_client.go: -------------------------------------------------------------------------------- 1 | // Code generated by counterfeiter. DO NOT EDIT. 2 | package routingapifakes 3 | 4 | import ( 5 | "context" 6 | "sync" 7 | 8 | "golang.org/x/oauth2" 9 | ) 10 | 11 | type FakeUaaClient struct { 12 | FetchTokenStub func(context.Context, bool) (*oauth2.Token, error) 13 | fetchTokenMutex sync.RWMutex 14 | fetchTokenArgsForCall []struct { 15 | arg1 context.Context 16 | arg2 bool 17 | } 18 | fetchTokenReturns struct { 19 | result1 *oauth2.Token 20 | result2 error 21 | } 22 | fetchTokenReturnsOnCall map[int]struct { 23 | result1 *oauth2.Token 24 | result2 error 25 | } 26 | invocations map[string][][]interface{} 27 | invocationsMutex sync.RWMutex 28 | } 29 | 30 | func (fake *FakeUaaClient) FetchToken(arg1 context.Context, arg2 bool) (*oauth2.Token, error) { 31 | fake.fetchTokenMutex.Lock() 32 | ret, specificReturn := fake.fetchTokenReturnsOnCall[len(fake.fetchTokenArgsForCall)] 33 | fake.fetchTokenArgsForCall = append(fake.fetchTokenArgsForCall, struct { 34 | arg1 context.Context 35 | arg2 bool 36 | }{arg1, arg2}) 37 | stub := fake.FetchTokenStub 38 | fakeReturns := fake.fetchTokenReturns 39 | fake.recordInvocation("FetchToken", []interface{}{arg1, arg2}) 40 | fake.fetchTokenMutex.Unlock() 41 | if stub != nil { 42 | return stub(arg1, arg2) 43 | } 44 | if specificReturn { 45 | return ret.result1, ret.result2 46 | } 47 | return fakeReturns.result1, fakeReturns.result2 48 | } 49 | 50 | func (fake *FakeUaaClient) FetchTokenCallCount() int { 51 | fake.fetchTokenMutex.RLock() 52 | defer fake.fetchTokenMutex.RUnlock() 53 | return len(fake.fetchTokenArgsForCall) 54 | } 55 | 56 | func (fake *FakeUaaClient) FetchTokenCalls(stub func(context.Context, bool) (*oauth2.Token, error)) { 57 | fake.fetchTokenMutex.Lock() 58 | defer fake.fetchTokenMutex.Unlock() 59 | fake.FetchTokenStub = stub 60 | } 61 | 62 | func (fake *FakeUaaClient) FetchTokenArgsForCall(i int) (context.Context, bool) { 63 | fake.fetchTokenMutex.RLock() 64 | defer fake.fetchTokenMutex.RUnlock() 65 | argsForCall := fake.fetchTokenArgsForCall[i] 66 | return argsForCall.arg1, argsForCall.arg2 67 | } 68 | 69 | func (fake *FakeUaaClient) FetchTokenReturns(result1 *oauth2.Token, result2 error) { 70 | fake.fetchTokenMutex.Lock() 71 | defer fake.fetchTokenMutex.Unlock() 72 | fake.FetchTokenStub = nil 73 | fake.fetchTokenReturns = struct { 74 | result1 *oauth2.Token 75 | result2 error 76 | }{result1, result2} 77 | } 78 | 79 | func (fake *FakeUaaClient) FetchTokenReturnsOnCall(i int, result1 *oauth2.Token, result2 error) { 80 | fake.fetchTokenMutex.Lock() 81 | defer fake.fetchTokenMutex.Unlock() 82 | fake.FetchTokenStub = nil 83 | if fake.fetchTokenReturnsOnCall == nil { 84 | fake.fetchTokenReturnsOnCall = make(map[int]struct { 85 | result1 *oauth2.Token 86 | result2 error 87 | }) 88 | } 89 | fake.fetchTokenReturnsOnCall[i] = struct { 90 | result1 *oauth2.Token 91 | result2 error 92 | }{result1, result2} 93 | } 94 | 95 | func (fake *FakeUaaClient) Invocations() map[string][][]interface{} { 96 | fake.invocationsMutex.RLock() 97 | defer fake.invocationsMutex.RUnlock() 98 | fake.fetchTokenMutex.RLock() 99 | defer fake.fetchTokenMutex.RUnlock() 100 | copiedInvocations := map[string][][]interface{}{} 101 | for key, value := range fake.invocations { 102 | copiedInvocations[key] = value 103 | } 104 | return copiedInvocations 105 | } 106 | 107 | func (fake *FakeUaaClient) recordInvocation(key string, args []interface{}) { 108 | fake.invocationsMutex.Lock() 109 | defer fake.invocationsMutex.Unlock() 110 | if fake.invocations == nil { 111 | fake.invocations = map[string][][]interface{}{} 112 | } 113 | if fake.invocations[key] == nil { 114 | fake.invocations[key] = [][]interface{}{} 115 | } 116 | fake.invocations[key] = append(fake.invocations[key], args) 117 | } 118 | --------------------------------------------------------------------------------