├── .github ├── PULL_REQUEST_TEMPLATE.md └── workflows │ └── tests.yml ├── .gitignore ├── .travis.yml ├── ChangeLog.md ├── LICENSE ├── LICENSE-APACHE2 ├── LICENSE-MPL-RabbitMQ ├── Makefile ├── README.md ├── erlang.mk ├── rebar.config ├── rebar.lock ├── src ├── aten.app.src ├── aten.erl ├── aten_app.erl ├── aten_detect.erl ├── aten_detector.erl ├── aten_emitter.erl ├── aten_sink.erl └── aten_sup.erl └── test └── aten_SUITE.erl /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Proposed Changes 2 | 3 | Please describe the big picture of your changes here to communicate to the 4 | RabbitMQ team why we should accept this pull request. If it fixes a bug or 5 | resolves a feature request, be sure to link to that issue. 6 | 7 | A pull request that doesn't explain **why** the change was made has a much 8 | lower chance of being accepted. 9 | 10 | If English isn't your first language, don't worry about it and try to 11 | communicate the problem you are trying to solve to the best of your abilities. 12 | As long as we can understand the intent, it's all good. 13 | 14 | ## Types of Changes 15 | 16 | What types of changes does your code introduce to this project? 17 | _Put an `x` in the boxes that apply_ 18 | 19 | - [ ] Bug fix (non-breaking change which fixes issue #NNNN) 20 | - [ ] New feature (non-breaking change which adds functionality) 21 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) 22 | - [ ] Documentation (correction or otherwise) 23 | - [ ] Cosmetics (whitespace, appearance) 24 | 25 | ## Checklist 26 | 27 | _Put an `x` in the boxes that apply. You can also fill these out after creating 28 | the PR. If you're unsure about any of them, don't hesitate to ask on the 29 | mailing list. We're here to help! This is simply a reminder of what we are 30 | going to look for before merging your code._ 31 | 32 | - [ ] I have read the `CONTRIBUTING.md` document 33 | - [ ] I have signed the CA (see https://cla.pivotal.io/sign/rabbitmq) 34 | - [ ] All tests pass locally with my changes 35 | - [ ] I have added tests that prove my fix is effective or that my feature works 36 | - [ ] I have added necessary documentation (if appropriate) 37 | - [ ] Any dependent changes have been merged and published in related repositories 38 | 39 | ## Further Comments 40 | 41 | If this is a relatively large or complex change, kick off the discussion by 42 | explaining why you chose the solution you did and what alternatives you 43 | considered, etc. 44 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: push 3 | jobs: 4 | build: 5 | runs-on: ubuntu-latest 6 | strategy: 7 | fail-fast: false 8 | matrix: 9 | erlang_version: 10 | - "26" 11 | - "27" 12 | steps: 13 | - name: CHECKOUT 14 | uses: actions/checkout@v2 15 | - name: CONFIGURE ERLANG 16 | uses: erlef/setup-beam@v1 17 | with: 18 | otp-version: ${{ matrix.erlang_version }} 19 | - name: Deps 20 | run: make test-deps 21 | - name: Dialyze 22 | run: make dialyze 23 | - name: Build tests 24 | run: make test-build 25 | - name: EUnit 26 | run: make eunit 27 | - name: aten_SUITE 28 | run: make ct-aten 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | erl_crash.dump 2 | .sw? 3 | .*.sw? 4 | *.beam 5 | /.erlang.mk/ 6 | /cover/ 7 | /deps/ 8 | /doc/ 9 | /ebin/ 10 | /logs/ 11 | /plugins/ 12 | /xrefr 13 | elvis 14 | callgrind* 15 | ct.coverdata 16 | test/ct.cover.spec 17 | _build 18 | 19 | aten.d 20 | *.plt 21 | *.d 22 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # vim:sw=2:et: 2 | 3 | os: linux 4 | dist: bionic 5 | sudo: false 6 | 7 | language: erlang 8 | notifications: 9 | email: 10 | on_success: never 11 | on_failure: always 12 | 13 | otp_release: 14 | otp_release: 15 | - "23.0.2" 16 | - "22.3.4" 17 | 18 | script: 19 | - make xref 20 | - make check 21 | - make dialyze 22 | -------------------------------------------------------------------------------- /ChangeLog.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | ## Changes Between 0.5.4 and 0.6.0 (in development) 4 | 5 | No changes yet. 6 | 7 | ## Changes Between 0.5.3 and 0.5.4 (July 20th, 2020) 8 | 9 | ### License Change 10 | 11 | The library is now double-licensed under the Apache Software License 2.0 12 | and Mozilla Public License 2.0 (previously: under the ASL2 and Mozilla Public License 1.1). 13 | 14 | ### Minimum Supported Erlang Version Bump 15 | 16 | The library now requires OTP 21.3 or a later version. 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This package, Aten, is dual-licensed under 2 | the Apache License v2 and the Mozilla Public License v2.0. 3 | 4 | For the Apache License, please see the file LICENSE-APACHE2. 5 | 6 | For the Mozilla Public License, please see the file LICENSE-MPL-RabbitMQ. 7 | 8 | For attribution of copyright and other details of provenance, please 9 | refer to the source code. 10 | 11 | If you have any questions regarding licensing, please contact us at 12 | info@rabbitmq.com. 13 | -------------------------------------------------------------------------------- /LICENSE-APACHE2: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | https://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2017 Pivotal Software Inc. 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | https://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /LICENSE-MPL-RabbitMQ: -------------------------------------------------------------------------------- 1 | Mozilla Public License Version 2.0 2 | ================================== 3 | 4 | 1. Definitions 5 | -------------- 6 | 7 | 1.1. "Contributor" 8 | means each individual or legal entity that creates, contributes to 9 | the creation of, or owns Covered Software. 10 | 11 | 1.2. "Contributor Version" 12 | means the combination of the Contributions of others (if any) used 13 | by a Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | means Covered Software of a particular Contributor. 17 | 18 | 1.4. "Covered Software" 19 | means Source Code Form to which the initial Contributor has attached 20 | the notice in Exhibit A, the Executable Form of such Source Code 21 | Form, and Modifications of such Source Code Form, in each case 22 | including portions thereof. 23 | 24 | 1.5. "Incompatible With Secondary Licenses" 25 | means 26 | 27 | (a) that the initial Contributor has attached the notice described 28 | in Exhibit B to the Covered Software; or 29 | 30 | (b) that the Covered Software was made available under the terms of 31 | version 1.1 or earlier of the License, but not also under the 32 | terms of a Secondary License. 33 | 34 | 1.6. "Executable Form" 35 | means any form of the work other than Source Code Form. 36 | 37 | 1.7. "Larger Work" 38 | means a work that combines Covered Software with other material, in 39 | a separate file or files, that is not Covered Software. 40 | 41 | 1.8. "License" 42 | means this document. 43 | 44 | 1.9. "Licensable" 45 | means having the right to grant, to the maximum extent possible, 46 | whether at the time of the initial grant or subsequently, any and 47 | all of the rights conveyed by this License. 48 | 49 | 1.10. "Modifications" 50 | means any of the following: 51 | 52 | (a) any file in Source Code Form that results from an addition to, 53 | deletion from, or modification of the contents of Covered 54 | Software; or 55 | 56 | (b) any new file in Source Code Form that contains any Covered 57 | Software. 58 | 59 | 1.11. "Patent Claims" of a Contributor 60 | means any patent claim(s), including without limitation, method, 61 | process, and apparatus claims, in any patent Licensable by such 62 | Contributor that would be infringed, but for the grant of the 63 | License, by the making, using, selling, offering for sale, having 64 | made, import, or transfer of either its Contributions or its 65 | Contributor Version. 66 | 67 | 1.12. "Secondary License" 68 | means either the GNU General Public License, Version 2.0, the GNU 69 | Lesser General Public License, Version 2.1, the GNU Affero General 70 | Public License, Version 3.0, or any later versions of those 71 | licenses. 72 | 73 | 1.13. "Source Code Form" 74 | means the form of the work preferred for making modifications. 75 | 76 | 1.14. "You" (or "Your") 77 | means an individual or a legal entity exercising rights under this 78 | License. For legal entities, "You" includes any entity that 79 | controls, is controlled by, or is under common control with You. For 80 | purposes of this definition, "control" means (a) the power, direct 81 | or indirect, to cause the direction or management of such entity, 82 | whether by contract or otherwise, or (b) ownership of more than 83 | fifty percent (50%) of the outstanding shares or beneficial 84 | ownership of such entity. 85 | 86 | 2. License Grants and Conditions 87 | -------------------------------- 88 | 89 | 2.1. Grants 90 | 91 | Each Contributor hereby grants You a world-wide, royalty-free, 92 | non-exclusive license: 93 | 94 | (a) under intellectual property rights (other than patent or trademark) 95 | Licensable by such Contributor to use, reproduce, make available, 96 | modify, display, perform, distribute, and otherwise exploit its 97 | Contributions, either on an unmodified basis, with Modifications, or 98 | as part of a Larger Work; and 99 | 100 | (b) under Patent Claims of such Contributor to make, use, sell, offer 101 | for sale, have made, import, and otherwise transfer either its 102 | Contributions or its Contributor Version. 103 | 104 | 2.2. Effective Date 105 | 106 | The licenses granted in Section 2.1 with respect to any Contribution 107 | become effective for each Contribution on the date the Contributor first 108 | distributes such Contribution. 109 | 110 | 2.3. Limitations on Grant Scope 111 | 112 | The licenses granted in this Section 2 are the only rights granted under 113 | this License. No additional rights or licenses will be implied from the 114 | distribution or licensing of Covered Software under this License. 115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 116 | Contributor: 117 | 118 | (a) for any code that a Contributor has removed from Covered Software; 119 | or 120 | 121 | (b) for infringements caused by: (i) Your and any other third party's 122 | modifications of Covered Software, or (ii) the combination of its 123 | Contributions with other software (except as part of its Contributor 124 | Version); or 125 | 126 | (c) under Patent Claims infringed by Covered Software in the absence of 127 | its Contributions. 128 | 129 | This License does not grant any rights in the trademarks, service marks, 130 | or logos of any Contributor (except as may be necessary to comply with 131 | the notice requirements in Section 3.4). 132 | 133 | 2.4. Subsequent Licenses 134 | 135 | No Contributor makes additional grants as a result of Your choice to 136 | distribute the Covered Software under a subsequent version of this 137 | License (see Section 10.2) or under the terms of a Secondary License (if 138 | permitted under the terms of Section 3.3). 139 | 140 | 2.5. Representation 141 | 142 | Each Contributor represents that the Contributor believes its 143 | Contributions are its original creation(s) or it has sufficient rights 144 | to grant the rights to its Contributions conveyed by this License. 145 | 146 | 2.6. Fair Use 147 | 148 | This License is not intended to limit any rights You have under 149 | applicable copyright doctrines of fair use, fair dealing, or other 150 | equivalents. 151 | 152 | 2.7. Conditions 153 | 154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 155 | in Section 2.1. 156 | 157 | 3. Responsibilities 158 | ------------------- 159 | 160 | 3.1. Distribution of Source Form 161 | 162 | All distribution of Covered Software in Source Code Form, including any 163 | Modifications that You create or to which You contribute, must be under 164 | the terms of this License. You must inform recipients that the Source 165 | Code Form of the Covered Software is governed by the terms of this 166 | License, and how they can obtain a copy of this License. You may not 167 | attempt to alter or restrict the recipients' rights in the Source Code 168 | Form. 169 | 170 | 3.2. Distribution of Executable Form 171 | 172 | If You distribute Covered Software in Executable Form then: 173 | 174 | (a) such Covered Software must also be made available in Source Code 175 | Form, as described in Section 3.1, and You must inform recipients of 176 | the Executable Form how they can obtain a copy of such Source Code 177 | Form by reasonable means in a timely manner, at a charge no more 178 | than the cost of distribution to the recipient; and 179 | 180 | (b) You may distribute such Executable Form under the terms of this 181 | License, or sublicense it under different terms, provided that the 182 | license for the Executable Form does not attempt to limit or alter 183 | the recipients' rights in the Source Code Form under this License. 184 | 185 | 3.3. Distribution of a Larger Work 186 | 187 | You may create and distribute a Larger Work under terms of Your choice, 188 | provided that You also comply with the requirements of this License for 189 | the Covered Software. If the Larger Work is a combination of Covered 190 | Software with a work governed by one or more Secondary Licenses, and the 191 | Covered Software is not Incompatible With Secondary Licenses, this 192 | License permits You to additionally distribute such Covered Software 193 | under the terms of such Secondary License(s), so that the recipient of 194 | the Larger Work may, at their option, further distribute the Covered 195 | Software under the terms of either this License or such Secondary 196 | License(s). 197 | 198 | 3.4. Notices 199 | 200 | You may not remove or alter the substance of any license notices 201 | (including copyright notices, patent notices, disclaimers of warranty, 202 | or limitations of liability) contained within the Source Code Form of 203 | the Covered Software, except that You may alter any license notices to 204 | the extent required to remedy known factual inaccuracies. 205 | 206 | 3.5. Application of Additional Terms 207 | 208 | You may choose to offer, and to charge a fee for, warranty, support, 209 | indemnity or liability obligations to one or more recipients of Covered 210 | Software. However, You may do so only on Your own behalf, and not on 211 | behalf of any Contributor. You must make it absolutely clear that any 212 | such warranty, support, indemnity, or liability obligation is offered by 213 | You alone, and You hereby agree to indemnify every Contributor for any 214 | liability incurred by such Contributor as a result of warranty, support, 215 | indemnity or liability terms You offer. You may include additional 216 | disclaimers of warranty and limitations of liability specific to any 217 | jurisdiction. 218 | 219 | 4. Inability to Comply Due to Statute or Regulation 220 | --------------------------------------------------- 221 | 222 | If it is impossible for You to comply with any of the terms of this 223 | License with respect to some or all of the Covered Software due to 224 | statute, judicial order, or regulation then You must: (a) comply with 225 | the terms of this License to the maximum extent possible; and (b) 226 | describe the limitations and the code they affect. Such description must 227 | be placed in a text file included with all distributions of the Covered 228 | Software under this License. Except to the extent prohibited by statute 229 | or regulation, such description must be sufficiently detailed for a 230 | recipient of ordinary skill to be able to understand it. 231 | 232 | 5. Termination 233 | -------------- 234 | 235 | 5.1. The rights granted under this License will terminate automatically 236 | if You fail to comply with any of its terms. However, if You become 237 | compliant, then the rights granted under this License from a particular 238 | Contributor are reinstated (a) provisionally, unless and until such 239 | Contributor explicitly and finally terminates Your grants, and (b) on an 240 | ongoing basis, if such Contributor fails to notify You of the 241 | non-compliance by some reasonable means prior to 60 days after You have 242 | come back into compliance. Moreover, Your grants from a particular 243 | Contributor are reinstated on an ongoing basis if such Contributor 244 | notifies You of the non-compliance by some reasonable means, this is the 245 | first time You have received notice of non-compliance with this License 246 | from such Contributor, and You become compliant prior to 30 days after 247 | Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, 251 | counter-claims, and cross-claims) alleging that a Contributor Version 252 | directly or indirectly infringes any patent, then the rights granted to 253 | You by any and all Contributors for the Covered Software under Section 254 | 2.1 of this License shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 257 | end user license agreements (excluding distributors and resellers) which 258 | have been validly granted by You or Your distributors under this License 259 | prior to termination shall survive termination. 260 | 261 | ************************************************************************ 262 | * * 263 | * 6. Disclaimer of Warranty * 264 | * ------------------------- * 265 | * * 266 | * Covered Software is provided under this License on an "as is" * 267 | * basis, without warranty of any kind, either expressed, implied, or * 268 | * statutory, including, without limitation, warranties that the * 269 | * Covered Software is free of defects, merchantable, fit for a * 270 | * particular purpose or non-infringing. The entire risk as to the * 271 | * quality and performance of the Covered Software is with You. * 272 | * Should any Covered Software prove defective in any respect, You * 273 | * (not any Contributor) assume the cost of any necessary servicing, * 274 | * repair, or correction. This disclaimer of warranty constitutes an * 275 | * essential part of this License. No use of any Covered Software is * 276 | * authorized under this License except under this disclaimer. * 277 | * * 278 | ************************************************************************ 279 | 280 | ************************************************************************ 281 | * * 282 | * 7. Limitation of Liability * 283 | * -------------------------- * 284 | * * 285 | * Under no circumstances and under no legal theory, whether tort * 286 | * (including negligence), contract, or otherwise, shall any * 287 | * Contributor, or anyone who distributes Covered Software as * 288 | * permitted above, be liable to You for any direct, indirect, * 289 | * special, incidental, or consequential damages of any character * 290 | * including, without limitation, damages for lost profits, loss of * 291 | * goodwill, work stoppage, computer failure or malfunction, or any * 292 | * and all other commercial damages or losses, even if such party * 293 | * shall have been informed of the possibility of such damages. This * 294 | * limitation of liability shall not apply to liability for death or * 295 | * personal injury resulting from such party's negligence to the * 296 | * extent applicable law prohibits such limitation. Some * 297 | * jurisdictions do not allow the exclusion or limitation of * 298 | * incidental or consequential damages, so this exclusion and * 299 | * limitation may not apply to You. * 300 | * * 301 | ************************************************************************ 302 | 303 | 8. Litigation 304 | ------------- 305 | 306 | Any litigation relating to this License may be brought only in the 307 | courts of a jurisdiction where the defendant maintains its principal 308 | place of business and such litigation shall be governed by laws of that 309 | jurisdiction, without reference to its conflict-of-law provisions. 310 | Nothing in this Section shall prevent a party's ability to bring 311 | cross-claims or counter-claims. 312 | 313 | 9. Miscellaneous 314 | ---------------- 315 | 316 | This License represents the complete agreement concerning the subject 317 | matter hereof. If any provision of this License is held to be 318 | unenforceable, such provision shall be reformed only to the extent 319 | necessary to make it enforceable. Any law or regulation which provides 320 | that the language of a contract shall be construed against the drafter 321 | shall not be used to construe this License against a Contributor. 322 | 323 | 10. Versions of the License 324 | --------------------------- 325 | 326 | 10.1. New Versions 327 | 328 | Mozilla Foundation is the license steward. Except as provided in Section 329 | 10.3, no one other than the license steward has the right to modify or 330 | publish new versions of this License. Each version will be given a 331 | distinguishing version number. 332 | 333 | 10.2. Effect of New Versions 334 | 335 | You may distribute the Covered Software under the terms of the version 336 | of the License under which You originally received the Covered Software, 337 | or under the terms of any subsequent version published by the license 338 | steward. 339 | 340 | 10.3. Modified Versions 341 | 342 | If you create software not governed by this License, and you want to 343 | create a new license for such software, you may create and use a 344 | modified version of this License if you rename the license and remove 345 | any references to the name of the license steward (except to note that 346 | such modified license differs from this License). 347 | 348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 349 | Licenses 350 | 351 | If You choose to distribute Source Code Form that is Incompatible With 352 | Secondary Licenses under the terms of this version of the License, the 353 | notice described in Exhibit B of this License must be attached. 354 | 355 | Exhibit A - Source Code Form License Notice 356 | ------------------------------------------- 357 | 358 | This Source Code Form is subject to the terms of the Mozilla Public 359 | License, v. 2.0. If a copy of the MPL was not distributed with this 360 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 361 | 362 | If it is not possible or desirable to put the notice in a particular 363 | file, then You may include the notice in a location (such as a LICENSE 364 | file in a relevant directory) where a recipient would be likely to look 365 | for such a notice. 366 | 367 | You may add additional accurate notices of copyright ownership. 368 | 369 | Exhibit B - "Incompatible With Secondary Licenses" Notice 370 | --------------------------------------------------------- 371 | 372 | This Source Code Form is "Incompatible With Secondary Licenses", as 373 | defined by the Mozilla Public License, v. 2.0. 374 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJECT = aten 2 | PROJECT_DESCRIPTION = A New Adaptive Accrual Failure Detector 3 | PROJECT_VERSION = 0.6.0 4 | 5 | dep_meck = git https://github.com/eproxus/meck.git 1.0.0 6 | dep_eunit_formatters = git https://github.com/seancribbs/eunit_formatters.git v0.5.0 7 | 8 | define PROJECT_ENV 9 | [ 10 | ] 11 | endef 12 | 13 | TEST_DEPS = proper meck eunit_formatters 14 | 15 | LOCAL_DEPS = sasl crypto 16 | 17 | PLT_APPS += eunit meck proper syntax_tools erts kernel stdlib common_test inets 18 | 19 | DIALYZER_OPTS += --src -r test 20 | EUNIT_OPTS = no_tty, {report, {eunit_progress, [colored, profile]}} 21 | 22 | include erlang.mk 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Aten 2 | 3 | This is an implementation of [A New Adaptive Accrual Failure Detector for Dependable Distributed Systems](https://dl.acm.org/citation.cfm?id=1244129) 4 | for distributed Erlang. 5 | 6 | Aten is an essential dependency of [Ra](https://github.com/rabbitmq/ra). 7 | 8 | ## Project Maturity 9 | 10 | `aten` is a reasonably mature project that has been used in production 11 | for a few years. Breaking public API changes are unlikely. 12 | 13 | ## Build Status 14 | 15 | ![Actions](https://github.com/rabbitmq/aten/actions/workflows/tests.yml/badge.svg) 16 | 17 | ## Use 18 | 19 | ``` erl 20 | %% start the aten application if not part of a release 21 | {ok, _Apps} = aten:start(), 22 | %% register interest in status changes for a node 23 | ok = aten:register(node1@host), 24 | %% receive status change events 25 | receive 26 | {node_event, node1@host, down} -> ok; 27 | {node_event, node1@host, up} -> ok 28 | end 29 | 30 | %% deregister interest 31 | ok = aten:deregister(node1@host), 32 | ``` 33 | 34 | ## Build 35 | 36 | ``` shell 37 | gmake 38 | ``` 39 | 40 | Or with Rebar 3 41 | 42 | ``` shell 43 | 44 | rebar3 compile 45 | ``` 46 | 47 | ## License 48 | 49 | `aten` is [dual-licensed](./LICENSE) under the Mozilla Public License 2.0 50 | and the Apache Public License 2.0. 51 | 52 | (c) 2017-2024 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. 53 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {minimum_otp_vsn, "22.3"}. 2 | {erl_opts, [debug_info]}. 3 | {deps, [ 4 | {eqwalizer_support, 5 | {git_subdir, 6 | "https://github.com/whatsapp/eqwalizer.git", 7 | {branch, "main"}, 8 | "eqwalizer_support"}} 9 | ]}. 10 | {project_plugins, [rebar3_hex]}. 11 | {profiles, 12 | [{test, [{deps, [meck, proper]}]}] 13 | }. 14 | {dist_node, [ 15 | {sname, 'aten'} 16 | ]}. 17 | {dialyzer, [{warnings, 18 | [error_handling, 19 | unmatched_returns]}]}. 20 | {xref_extra_paths, ["test"]}. 21 | {xref_checks,[undefined_function_calls, 22 | undefined_functions, 23 | locals_not_used, 24 | % exports_not_used, 25 | deprecated_function_calls, 26 | deprecated_functions]}. 27 | -------------------------------------------------------------------------------- /rebar.lock: -------------------------------------------------------------------------------- 1 | [{<<"eqwalizer_support">>, 2 | {git_subdir,"https://github.com/whatsapp/eqwalizer.git", 3 | {ref,"4531d5207dba6bdd82c3fd4adf68a61f7d569874"}, 4 | "eqwalizer_support"}, 5 | 0}]. 6 | -------------------------------------------------------------------------------- /src/aten.app.src: -------------------------------------------------------------------------------- 1 | {application,aten, 2 | [{id,"git"}, 3 | {description,"Erlang node failure detector"}, 4 | {vsn,"0.6.0"}, 5 | {licenses,["Apache","MPL"]}, 6 | {links,[{"github","https://github.com/rabbitmq/aten"}]}, 7 | {modules,[aten,aten_app,aten_detect,aten_detector,aten_emitter, 8 | aten_sink,aten_sup]}, 9 | {registered,[aten_sup]}, 10 | {applications,[kernel,stdlib,sasl,crypto]}, 11 | {mod,{aten_app,[]}}, 12 | {env,[{poll_interval,1000}, 13 | {scaling_factor,1.5}, 14 | {heartbeat_interval,100}, 15 | {detection_threshold,0.99}]}]}. 16 | -------------------------------------------------------------------------------- /src/aten.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2018-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | -module(aten). 8 | 9 | -export([ 10 | start/0, 11 | register/1, 12 | unregister/1 13 | ]). 14 | 15 | start() -> 16 | application:ensure_all_started(aten). 17 | 18 | -spec register(node()) -> ok | ignore. 19 | register(Node) when Node == node() -> 20 | ignore; 21 | register(Node) -> 22 | aten_detector:register(Node). 23 | 24 | -spec unregister(node()) -> ok | ignore. 25 | unregister(Node) when Node == node() -> 26 | ignore; 27 | unregister(Node) -> 28 | aten_detector:unregister(Node). 29 | -------------------------------------------------------------------------------- /src/aten_app.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2018-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | -module(aten_app). 8 | -behaviour(application). 9 | 10 | -export([start/2]). 11 | -export([stop/1]). 12 | 13 | start(_Type, _Args) -> 14 | aten_sup:start_link(). 15 | 16 | stop(_State) -> 17 | ok. 18 | -------------------------------------------------------------------------------- /src/aten_detect.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2018-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | -module(aten_detect). 8 | 9 | -export([ 10 | init/1, 11 | sample_now/1, 12 | get_failure_probability/1 13 | ]). 14 | 15 | -type sample() :: integer(). 16 | 17 | -define(WINDOW_SIZE, 1000). 18 | 19 | -record(state, {freshness :: undefined | non_neg_integer(), 20 | samples = array:new(?WINDOW_SIZE) :: array:array(sample()), 21 | next_index = 0 :: non_neg_integer(), 22 | max_size = 1000 :: non_neg_integer(), 23 | factor = 1.5 :: number()}). 24 | 25 | 26 | -opaque state() :: #state{}. 27 | 28 | -export_type([state/0]). 29 | 30 | -spec init(number()) -> state(). 31 | init(Factor) -> 32 | #state{factor = Factor}. 33 | 34 | -spec sample_now(state()) -> state(). 35 | sample_now(State) -> 36 | append(ts(), State). 37 | 38 | -spec get_failure_probability(state()) -> float(). 39 | get_failure_probability(State) -> 40 | failure_prob_at(ts(), State). 41 | 42 | %% Internal 43 | 44 | append(Ts, #state{freshness = undefined} = State) -> 45 | State#state{freshness = Ts}; 46 | append(Ts0, #state{freshness = F, 47 | samples = Samples, 48 | next_index = Next} = State) when is_number(F) -> 49 | Ts = Ts0 - F, 50 | State#state{samples = array:set(Next, Ts, Samples), 51 | next_index = (Next + 1) rem ?WINDOW_SIZE, 52 | freshness = Ts0}. 53 | 54 | failure_prob_at(_At, #state{freshness = undefined}) -> 55 | 0.0; 56 | failure_prob_at(At, #state{freshness = F, 57 | factor = A, 58 | samples = Samples}) -> 59 | T = At - F, 60 | {TotNum, SmallNum} = array:foldl( 61 | fun(_, undefined, Acc) -> 62 | Acc; 63 | (_, S, {Tot, Smaller}) when S * A =< T -> 64 | {Tot+1, Smaller+1}; 65 | (_, _S, {Tot, Smaller}) -> 66 | {Tot+1, Smaller} 67 | end, {0, 0}, Samples), 68 | SmallNum / max(1, TotNum). % avoid div/0 69 | 70 | ts() -> 71 | erlang:monotonic_time(microsecond). 72 | 73 | -ifdef(TEST). 74 | -include_lib("eunit/include/eunit.hrl"). 75 | 76 | detect_test() -> 77 | S0 = #state{}, 78 | ?assert(failure_prob_at(10, S0) =:= 0.0), 79 | S = lists:foldl(fun append/2, S0, [1, 5, 4, 10, 13, 20, 25]), 80 | ?assert(failure_prob_at(28, S) < 0.5), 81 | ?assert(failure_prob_at(40, S) == 1.0), 82 | S1 = append(10, S0), 83 | ?assertEqual(0.0, failure_prob_at(10, S1)), 84 | % we cannot detect failures with only a single sample 85 | % ?assert(failure_prob_at(100, S1) > 0.0), 86 | ok. 87 | 88 | -endif. 89 | -------------------------------------------------------------------------------- /src/aten_detector.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2018-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | -module(aten_detector). 8 | 9 | -behaviour(gen_server). 10 | 11 | %% API functions 12 | -export([start_link/0, 13 | register/1, 14 | unregister/1 15 | ]). 16 | 17 | %% gen_server callbacks 18 | -export([init/1, 19 | handle_call/3, 20 | handle_cast/2, 21 | handle_info/2, 22 | terminate/2, 23 | code_change/3]). 24 | 25 | -define(POLL_INTERVAL_MS, 1000). 26 | -define(DEFAULT_THRESHOLD, 0.99). 27 | 28 | -record(state, {tref :: reference() | undefined, 29 | interval = ?POLL_INTERVAL_MS :: non_neg_integer(), 30 | threshold = ?DEFAULT_THRESHOLD :: float(), 31 | node_states = #{} :: #{node() => float()}, % last threshold 32 | watchers = #{} :: #{node() => #{pid() => reference()}}}). 33 | 34 | %%%=================================================================== 35 | %%% API functions 36 | %%%=================================================================== 37 | 38 | start_link() -> 39 | gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). 40 | 41 | register(Node) -> 42 | Me = self(), 43 | gen_server:cast(?MODULE, {register, Node, Me}). 44 | 45 | unregister(Node) -> 46 | Me = self(), 47 | gen_server:cast(?MODULE, {unregister, Node, Me}). 48 | 49 | %%%=================================================================== 50 | %%% gen_server callbacks 51 | %%%=================================================================== 52 | 53 | init([]) -> 54 | Thresh = application:get_env(aten, detection_threshold, 55 | ?DEFAULT_THRESHOLD), 56 | Interval = application:get_env(aten, poll_interval, 57 | ?POLL_INTERVAL_MS), 58 | {ok, set_timer(#state{threshold = Thresh, 59 | interval = Interval})}. 60 | 61 | handle_call(_Request, _From, State) -> 62 | Reply = ok, 63 | {reply, Reply, State}. 64 | 65 | handle_cast({register, Node, Pid}, 66 | #state{watchers = Watchers0, 67 | node_states = Curr, 68 | threshold = Thresh} = State) -> 69 | Pids0 = maps:get(Node, Watchers0, #{}), 70 | Pids = case Pids0 of 71 | #{Pid := _Mon} -> 72 | Pids0; 73 | _ -> 74 | %% this is a new registration, emit the current state 75 | case Curr of 76 | #{Node := Last} when Last < Thresh -> 77 | %% the node is known and active 78 | Pid ! {node_event, Node, up}, 79 | ok; 80 | _ -> 81 | %% otherwise it must be down 82 | Pid ! {node_event, Node, down}, 83 | ok 84 | end, 85 | Pids0#{Pid => erlang:monitor(process, Pid)} 86 | end, 87 | Watchers = maps:put(Node, Pids, Watchers0), 88 | {noreply, State#state{watchers = Watchers}}; 89 | handle_cast({unregister, Node, Pid}, 90 | #state{watchers = Watchers0} = State) -> 91 | Watchers = case Watchers0 of 92 | #{Node := Pids0} -> 93 | case maps:take(Pid, Pids0) of 94 | error -> 95 | Watchers0; 96 | {Ref, Pids} -> 97 | erlang:demonitor(Ref), 98 | maps:update(Node, Pids, Watchers0) 99 | end; 100 | _ -> 101 | Watchers0 102 | end, 103 | {noreply, State#state{watchers = Watchers}}. 104 | 105 | handle_info(poll, #state{threshold = Th, 106 | node_states = Prev, 107 | watchers = Watchers} = State0) -> 108 | State = set_timer(State0), 109 | Probs = aten_sink:get_failure_probabilities(), 110 | {Up, Down} = analyse(Probs, Prev, Th), 111 | ok = notify(Watchers, Down, down), 112 | ok = notify(Watchers, Up, up), 113 | {noreply, State#state{node_states = Probs}}; 114 | handle_info({'DOWN', _Mon, process, Pid, _R}, 115 | #state{watchers = Watchers0} = State) -> 116 | Watchers = maps:map(fun(_Node, Pids0) -> 117 | % Note: this returns the new map as required by 118 | % maps:map/2 119 | maps:remove(Pid, Pids0) 120 | end, 121 | Watchers0), 122 | {noreply, State#state{watchers = Watchers}}. 123 | 124 | terminate(_Reason, _State) -> 125 | ok. 126 | 127 | code_change(_OldVsn, State, _Extra) -> 128 | {ok, State}. 129 | 130 | %%%=================================================================== 131 | %%% Internal functions 132 | %%%=================================================================== 133 | 134 | notify(_Watchers, [], _Evt) -> 135 | ok; 136 | notify(Watchers, [Node | Nodes], Evt) -> 137 | case Watchers of 138 | #{Node := Pids} -> 139 | _ = maps:map(fun(Pid, _) -> 140 | Pid ! {node_event, Node, Evt} 141 | end, Pids), 142 | ok; 143 | _ -> 144 | ok 145 | end, 146 | notify(Watchers, Nodes, Evt). 147 | 148 | analyse_one(_Curr, undefined, _Thresh) -> 149 | up; 150 | analyse_one(Curr, Prev, Thresh) 151 | when Prev < Thresh andalso Curr >= Thresh -> 152 | down; 153 | analyse_one(Curr, Prev, Thresh) 154 | when Prev >= Thresh andalso Curr < Thresh -> 155 | up; 156 | analyse_one(_Curr, _Prev, _Thresh) -> 157 | no_change. 158 | 159 | analyse(Curr, Prev, Thresh) -> 160 | Down0 = maps:fold(fun (N, Sample, Acc) -> 161 | case maps:get(N, Curr, undefined) of 162 | undefined -> 163 | case Sample >= Thresh of 164 | true -> 165 | %% already down 166 | %% this should already have been 167 | %% been notified 168 | Acc; 169 | _ -> 170 | [N | Acc] 171 | end; 172 | _ -> 173 | Acc 174 | end 175 | end, [], Prev), 176 | lists:foldl(fun ({Node, Sample}, {Up, Down} = Acc) -> 177 | Last = maps:get(Node, Prev, undefined), 178 | case analyse_one(Sample, Last, Thresh) of 179 | up -> 180 | {[Node | Up], Down}; 181 | down -> 182 | {Up, [Node | Down]}; 183 | no_change -> 184 | Acc 185 | end 186 | end, {[], Down0}, maps:to_list(Curr)). 187 | 188 | set_timer(State) -> 189 | TRef = erlang:send_after(State#state.interval, self(), poll), 190 | State#state{tref = TRef}. 191 | 192 | 193 | -ifdef(TEST). 194 | -include_lib("eunit/include/eunit.hrl"). 195 | 196 | analyse_test() -> 197 | Curr = #{n1 => 1.0, % down 198 | n2 => 0.1, % up 199 | n3 => 0.5 % no change 200 | }, 201 | Prev = #{n1 => 0.2, % down 202 | n2 => 1.0, % up 203 | n3 => 0.4, 204 | n4 => 0.3 % down (not in Curr) 205 | }, 206 | 207 | Want = {[n2], [n1,n4]}, 208 | Got = analyse(Curr, Prev, 0.98), 209 | ?assertEqual(Want, Got), 210 | ok. 211 | 212 | analyse_one_test() -> 213 | Th = 0.99, 214 | up = analyse_one(0.0, undefined, Th), 215 | down = analyse_one(1.0, 0.0, Th), 216 | up = analyse_one(0.5, 1.0, Th), 217 | no_change = analyse_one(0.4, 0.5, Th), 218 | ok. 219 | 220 | -endif. 221 | -------------------------------------------------------------------------------- /src/aten_emitter.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2018-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | -module(aten_emitter). 8 | 9 | -behaviour(gen_server). 10 | 11 | %% API functions 12 | -export([start_link/0]). 13 | 14 | %% gen_server callbacks 15 | -export([init/1, 16 | handle_call/3, 17 | handle_cast/2, 18 | handle_info/2, 19 | terminate/2, 20 | code_change/3]). 21 | 22 | -define(INTERVAL, 100). 23 | 24 | -record(state, {tref :: reference() | undefined, 25 | counters :: counters:counters_ref(), 26 | interval = ?INTERVAL :: non_neg_integer(), 27 | %% nodes for which a hearbeat is currently being sent in a separate 28 | %% process without nosuspend 29 | blocked = [] :: [node()]}). 30 | 31 | -type state() :: #state{}. 32 | 33 | 34 | 35 | %%% aten_emitter - emits heartbeats to all connected nodes periodically 36 | 37 | %%%=================================================================== 38 | %%% API functions 39 | %%%=================================================================== 40 | 41 | -spec start_link() -> {ok, pid()} | ignore | {error, term()}. 42 | start_link() -> 43 | gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). 44 | 45 | %%%=================================================================== 46 | %%% gen_server callbacks 47 | %%%=================================================================== 48 | 49 | -spec init(term()) -> {ok, state()}. 50 | init([]) -> 51 | Interval = application:get_env(aten, heartbeat_interval, ?INTERVAL), 52 | {ok, update_state(#state{interval = Interval, 53 | counters = counters:new(1, [])}, 54 | [])}. 55 | 56 | handle_call(_Request, _From, State) -> 57 | Reply = counters:get(State#state.counters, 1), 58 | {reply, Reply, State}. 59 | 60 | handle_cast(_Msg, State) -> 61 | {noreply, State}. 62 | 63 | handle_info(emit_heartbeats, 64 | #state{counters = Cnt, 65 | blocked = Blocked0} = State) -> 66 | Blocked = lists:foldl( 67 | fun (Node, Acc) -> 68 | case aten_sink:beat(Node) of 69 | nosuspend -> 70 | counters:add(Cnt, 1, 1), 71 | Self = self(), 72 | spawn(fun () -> 73 | ok = aten_sink:beat_blocking(Node), 74 | Self ! {unblock, Node}, 75 | ok 76 | end), 77 | [Node | Acc]; 78 | _ -> 79 | Acc 80 | end 81 | end, Blocked0, nodes() -- Blocked0), 82 | {noreply, update_state(State, Blocked)}; 83 | handle_info({unblock, Node}, #state{blocked = Blocked} = State) -> 84 | {noreply, State#state{blocked = lists:delete(Node, Blocked)}}. 85 | 86 | terminate(_Reason, _State) -> 87 | ok. 88 | 89 | code_change(_OldVsn, State, _Extra) -> 90 | {ok, State}. 91 | 92 | %%%=================================================================== 93 | %%% Internal functions 94 | %%%=================================================================== 95 | update_state(State, Blocked) -> 96 | TRef = erlang:send_after(State#state.interval, self(), emit_heartbeats), 97 | State#state{tref = TRef, 98 | blocked = Blocked}. 99 | -------------------------------------------------------------------------------- /src/aten_sink.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2018-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | -module(aten_sink). 8 | 9 | -behaviour(gen_server). 10 | 11 | %% API functions 12 | -export([start_link/0, 13 | get_failure_probabilities/0, 14 | beat/1, 15 | beat_blocking/1]). 16 | 17 | %% gen_server callbacks 18 | -export([init/1, 19 | handle_call/3, 20 | handle_cast/2, 21 | handle_info/2, 22 | terminate/2, 23 | code_change/3]). 24 | 25 | -record(state, {data = #{} :: #{node() => aten_detect:state()}, 26 | monitors = #{} :: #{node() => boolean()}, 27 | factor = 1.5 :: float()}). 28 | -type state() :: #state{}. 29 | 30 | %%% aten_sink 31 | 32 | %%%=================================================================== 33 | %%% API functions 34 | %%%=================================================================== 35 | 36 | -spec start_link() -> {ok, pid()} | ignore | {error, term()}. 37 | start_link() -> 38 | gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). 39 | 40 | get_failure_probabilities() -> 41 | Data = gen_server:call(?MODULE, get_data), 42 | get_probabilities(Data). 43 | 44 | -spec beat(node()) -> ok | noconnect | nosuspend. 45 | beat(DestNode) -> 46 | Dest = {?MODULE, DestNode}, 47 | Msg = {hb, node()}, 48 | erlang:send(Dest, {'$gen_cast', Msg}, [noconnect, nosuspend]). 49 | 50 | -spec beat_blocking(node()) -> ok. 51 | beat_blocking(DestNode) -> 52 | Dest = {?MODULE, DestNode}, 53 | Msg = {hb, node()}, 54 | _ = gen_server:cast(Dest, Msg), 55 | ok. 56 | 57 | %%%=================================================================== 58 | %%% gen_server callbacks 59 | %%%=================================================================== 60 | 61 | -spec init(term()) -> {ok, state()}. 62 | init([]) -> 63 | F = application:get_env(aten, scaling_factor, 1.5), 64 | {ok, #state{factor = F}}. 65 | 66 | handle_call(get_data, _From, State) -> 67 | {reply, State#state.data, State}. 68 | 69 | handle_cast({hb, Node}, #state{data = Data0, 70 | monitors = Monitors0, 71 | factor = Factor} = State) -> 72 | Monitors = maybe_monitor_node(Node, Monitors0), 73 | Data = case Data0 of 74 | #{Node := S} -> 75 | Data0#{Node => aten_detect:sample_now(S)}; 76 | _ -> 77 | Data0#{Node => aten_detect:init(Factor)} 78 | end, 79 | {noreply, State#state{data = Data, monitors = Monitors}}. 80 | 81 | handle_info({nodedown, Node}, #state{data = Data0, monitors = Monitors0} = State) -> 82 | % Note: do NOT unregister the node monitor here - it is unnecessary 83 | % and will actually end up trying to re-connect to the node 84 | % resulting in an infinite number of nodedown messages 85 | Data = maps:remove(Node, Data0), 86 | Monitors = maps:remove(Node, Monitors0), 87 | {noreply, State#state{data = Data, monitors = Monitors}}; 88 | handle_info(_Msg, State) -> 89 | {noreply, State}. 90 | 91 | terminate(_Reason, _State) -> 92 | ok. 93 | 94 | code_change(_OldVsn, State, _Extra) -> 95 | {ok, State}. 96 | 97 | %%%=================================================================== 98 | %%% Internal functions 99 | %%%=================================================================== 100 | 101 | maybe_monitor_node(Node, Monitors) -> 102 | IsMonitored = maps:get(Node, Monitors, false), 103 | case IsMonitored of 104 | true -> Monitors; 105 | false -> 106 | Monitors#{Node => erlang:monitor_node(Node, true)} 107 | end. 108 | 109 | get_probabilities(Data) -> 110 | maps:map(fun (_Key, Value) -> 111 | aten_detect:get_failure_probability(Value) 112 | end, Data). 113 | -------------------------------------------------------------------------------- /src/aten_sup.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2018-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. 6 | %% 7 | -module(aten_sup). 8 | -behaviour(supervisor). 9 | 10 | -export([start_link/0]). 11 | -export([init/1]). 12 | 13 | start_link() -> 14 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 15 | 16 | init([]) -> 17 | AtenSink = #{id => aten_sink, 18 | start => {aten_sink, start_link, []}}, 19 | AtenEmitter = #{id => aten_emitter, 20 | start => {aten_emitter, start_link, []}}, 21 | AtenDetector = #{id => aten_detector, 22 | start => {aten_detector, start_link, []}}, 23 | Procs = [AtenSink, AtenEmitter, AtenDetector], 24 | {ok, {{one_for_one, 1, 5}, Procs}}. 25 | -------------------------------------------------------------------------------- /test/aten_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(aten_SUITE). 2 | 3 | -export([ 4 | all/0, 5 | groups/0, 6 | init_per_group/2, end_per_group/2, 7 | init_per_testcase/2, end_per_testcase/2 8 | ]). 9 | 10 | -export([ 11 | distribution_flood/1, 12 | detect_node_partition/1, 13 | detect_node_stop_start/1, 14 | unregister_does_not_detect/1, 15 | register_unknown_emits_down/1, 16 | register_detects_down/1, 17 | watchers_cleanup/1 18 | ]). 19 | 20 | -include_lib("common_test/include/ct.hrl"). 21 | -include_lib("eunit/include/eunit.hrl"). 22 | 23 | -define(POLLINT, 1000). 24 | -define(HBINT, 250). 25 | -define(SCALE, 1.5). 26 | 27 | all() -> 28 | [ 29 | {group, tests} 30 | ]. 31 | 32 | all_tests() -> 33 | [ 34 | distribution_flood, 35 | detect_node_partition, 36 | detect_node_stop_start, 37 | unregister_does_not_detect, 38 | register_unknown_emits_down, 39 | register_detects_down, 40 | watchers_cleanup 41 | ]. 42 | 43 | groups() -> 44 | [ 45 | {tests, [], all_tests()} 46 | ]. 47 | 48 | init_per_group(_, Config) -> 49 | _ = application:load(aten), 50 | ok = application:set_env(aten, poll_interval, ?POLLINT), 51 | ok = application:set_env(aten, heartbeat_interval, ?HBINT), 52 | ok = application:set_env(aten, scaling_factor, ?SCALE), 53 | application:ensure_all_started(aten), 54 | Config. 55 | 56 | end_per_group(_, Config) -> 57 | _ = application:stop(ra), 58 | Config. 59 | 60 | init_per_testcase(_TestCase, Config) -> 61 | meck:new(aten_sink, [passthrough]), 62 | application:stop(aten), 63 | application:start(aten), 64 | Config. 65 | 66 | end_per_testcase(_Case, _Config) -> 67 | meck:unload(), 68 | ok. 69 | 70 | echo_proc() -> 71 | receive 72 | {P, Data} -> 73 | P ! Data, 74 | echo_proc(); 75 | stop -> 76 | ok 77 | end. 78 | 79 | 80 | sink_proc() -> 81 | receive 82 | _ -> 83 | sink_proc() 84 | end. 85 | 86 | load_proc(EPid, SPid, Data) -> 87 | EPid ! {SPid, Data}, 88 | load_proc(EPid, SPid, Data). 89 | 90 | distribution_flood(_Config) -> 91 | S1 = make_node_name(?FUNCTION_NAME), 92 | ok = aten:register(S1), 93 | receive 94 | {node_event, S1, down} -> ok 95 | after 5000 -> 96 | exit(node_event_timeout) 97 | end, 98 | {ok, P1, S1} = start_peer(?FUNCTION_NAME), 99 | ct:pal("Node ~w Nodes ~w", [node(), nodes()]), 100 | receive 101 | {node_event, S1, up} -> ok 102 | after 5000 -> 103 | exit(node_event_timeout) 104 | end, 105 | timer:sleep(30000), 106 | ct:pal("PRE nosuspends ~b", 107 | [gen_server:call({aten_emitter, S1}, dummy)]), 108 | 109 | %% generate some load on the distribution channel 110 | ct:pal("flooding..."), 111 | EPid = spawn(S1, fun echo_proc/0), 112 | SPid = spawn(fun sink_proc/0), 113 | Data = crypto:strong_rand_bytes(8024 * 2), 114 | LPid = spawn(fun () -> load_proc(EPid, SPid, Data) end), 115 | receive 116 | {node_event, S1, down} -> 117 | ct:pal("DOWN!! nosuspends ~b", 118 | [gen_server:call({aten_emitter, S1}, dummy)]), 119 | % %% check if it changes 120 | receive 121 | {node_event, S1, up} -> 122 | ct:pal("UP again!"), 123 | ok 124 | after ?POLLINT + 20 -> 125 | flush(), 126 | peer:stop(P1), 127 | exit(unexpected_down) 128 | end 129 | after 60000 -> 130 | ct:pal("NO DOWN nosuspends ~b", [gen_server:call({aten_emitter, S1}, dummy)]), 131 | ok 132 | end, 133 | 134 | exit(LPid, normal), 135 | exit(SPid, normal), 136 | exit(EPid, normal), 137 | peer:stop(P1), 138 | ok. 139 | 140 | 141 | detect_node_partition(_Config) -> 142 | S1 = make_node_name(?FUNCTION_NAME), 143 | ok = aten:register(S1), 144 | receive 145 | {node_event, S1, down} -> ok 146 | after 5000 -> 147 | exit(node_event_timeout) 148 | end, 149 | {ok, P1, S1} = start_peer(?FUNCTION_NAME), 150 | ct:pal("Node ~w Nodes ~w", [node(), nodes()]), 151 | receive 152 | {node_event, S1, up} -> ok 153 | after 5000 -> 154 | exit(node_event_timeout) 155 | end, 156 | %% give it enough time to generate more than one sample 157 | timer:sleep(2000), 158 | simulate_partition(S1), 159 | 160 | receive 161 | {node_event, S1, down} -> ok 162 | after 5000 -> 163 | flush(), 164 | exit(node_event_timeout) 165 | end, 166 | meck:unload(aten_sink), 167 | 168 | receive 169 | {node_event, S1, up} -> ok 170 | after 5000 -> 171 | flush(), 172 | exit(node_event_timeout) 173 | end, 174 | ok = peer:stop(P1), 175 | ok = aten:unregister(S1), 176 | ok. 177 | 178 | detect_node_stop_start(_Config) -> 179 | S1 = make_node_name(?FUNCTION_NAME), 180 | ok = aten:register(S1), 181 | {ok, P1, S1} = start_peer(?FUNCTION_NAME), 182 | ct:pal("Node ~w Nodes ~w", [node(), nodes()]), 183 | receive 184 | {node_event, S1, up} -> ok 185 | after 5000 -> 186 | exit(node_event_timeout) 187 | end, 188 | 189 | %% give it enough time to generate more than one sample 190 | timer:sleep(1000), 191 | 192 | ok = peer:stop(P1), 193 | receive 194 | {node_event, S1, down} -> ok 195 | after 5000 -> 196 | exit(node_event_timeout) 197 | end, 198 | 199 | {ok, P2, S1} = start_peer(?FUNCTION_NAME), 200 | receive 201 | {node_event, S1, up} -> ok 202 | after 5000 -> 203 | exit(node_event_timeout) 204 | end, 205 | ok = peer:stop(P2), 206 | ok = aten:unregister(S1), 207 | ok. 208 | 209 | unregister_does_not_detect(_Config) -> 210 | S1 = make_node_name(?FUNCTION_NAME), 211 | S2 = make_node_name(unregister_does_not_detect_2), 212 | ok = aten:register(S1), 213 | ok = aten:register(S2), 214 | wait_for({node_event, S1, down}), 215 | wait_for({node_event, S2, down}), 216 | {ok, P1, S1} = start_peer(?FUNCTION_NAME), 217 | {ok, P2, S2} = start_peer(unregister_does_not_detect_2), 218 | ct:pal("Node ~w Nodes ~w", [node(), nodes()]), 219 | wait_for({node_event, S1, up}), 220 | wait_for({node_event, S2, up}), 221 | {monitored_by, MonByPids} = erlang:process_info(self(), monitored_by), 222 | DetectorPid = whereis(aten_detector), 223 | %% one monitor for the process by each node 224 | ?assertEqual(2, length([P || P <- MonByPids, P == DetectorPid])), 225 | ok = aten:unregister(S1), 226 | %% aten:unmregister is a cast so we need to call in before asserting 227 | gen_server:call(aten_detector, any), 228 | {monitored_by, MonByPidsAfter} = erlang:process_info(self(), monitored_by), 229 | ?assertEqual(1, length([P || P <- MonByPidsAfter, P == DetectorPid])), 230 | peer:stop(P1), 231 | receive 232 | {node_event, S1, Evt} -> 233 | exit({unexpected_node_event, S1, Evt}) 234 | after 1000 -> 235 | ok 236 | end, 237 | peer:stop(P2), 238 | wait_for({node_event, S2, down}), 239 | ok. 240 | 241 | register_unknown_emits_down(_Config) -> 242 | S1 = make_node_name(?FUNCTION_NAME), 243 | ok = aten:register(S1), 244 | receive 245 | {node_event, S1, down} -> ok 246 | after 5000 -> 247 | exit(node_event_timeout) 248 | end, 249 | ok = aten:unregister(S1), 250 | ok. 251 | 252 | register_detects_down(_Config) -> 253 | S1 = make_node_name(?FUNCTION_NAME), 254 | ok = aten:register(S1), 255 | receive 256 | {node_event, S1, down} -> ok 257 | after 5000 -> 258 | exit(node_event_timeout) 259 | end, 260 | {ok, P1, S1} = start_peer(?FUNCTION_NAME), 261 | receive 262 | {node_event, S1, up} -> ok 263 | after 5000 -> 264 | flush(), 265 | exit(node_event_timeout_2) 266 | end, 267 | simulate_partition(S1), 268 | receive 269 | {node_event, S1, down} -> ok 270 | after 5000 -> 271 | flush(), 272 | exit(node_event_timeout_3) 273 | end, 274 | ok = aten:unregister(S1), 275 | %% re-register should detect down 276 | ok = aten:register(S1), 277 | receive 278 | {node_event, S1, down} -> ok 279 | after 5000 -> 280 | exit(node_event_timeout_4) 281 | end, 282 | ok = aten:unregister(S1), 283 | 284 | peer:stop(P1), 285 | ok. 286 | 287 | watchers_cleanup(_Config) -> 288 | Node = make_node_name(?FUNCTION_NAME), 289 | Self = self(), 290 | Watcher = spawn_watcher(Node, Self), 291 | ok = aten:register(Node), 292 | %% first clear out all the initial notifications 293 | receive 294 | {watcher_node_down, Node} -> ok 295 | after 5000 -> 296 | exit(node_event_timeout) 297 | end, 298 | receive 299 | {node_event, Node, down} -> ok 300 | after 5000 -> 301 | exit(node_event_timeout) 302 | end, 303 | {ok, Peer, Node} = start_peer(?FUNCTION_NAME), 304 | ct:pal("Node ~w Nodes ~w", [node(), nodes()]), 305 | receive 306 | {watcher_node_up, Node} -> ok 307 | after 5000 -> 308 | exit(node_event_timeout) 309 | end, 310 | receive 311 | {node_event, Node, up} -> ok 312 | after 5000 -> 313 | exit(node_event_timeout) 314 | end, 315 | 316 | State0 = sys:get_state(aten_detector), 317 | Watchers0 = element(6, State0), 318 | #{Node := #{Watcher := _}} = Watchers0, 319 | #{Node := #{Self := _}} = Watchers0, 320 | 321 | Watcher ! stop, 322 | 323 | timer:sleep(200), 324 | ok = peer:stop(Peer), 325 | 326 | receive 327 | {watcher_node_down, Node} -> 328 | exit(stopped_watcher_receive_message) 329 | after 50 -> 330 | ok 331 | end, 332 | receive 333 | {node_event, Node, down} -> ok 334 | after 5000 -> 335 | exit(node_event_timeout) 336 | end, 337 | 338 | State1 = sys:get_state(aten_detector), 339 | Watchers1 = element(6, State1), 340 | #{Node := Pids} = Watchers1, 341 | #{Node := #{Self := _}} = Watchers1, 342 | none = maps:get(Watcher, Pids, none), 343 | 344 | State2 = sys:get_state(aten_sink), 345 | NodeMap = element(2, State2), 346 | none = maps:get(Node, NodeMap, none), 347 | 348 | ok = aten:unregister(Node). 349 | 350 | spawn_watcher(Node, Pid) -> 351 | spawn(fun Fun() -> 352 | ok = aten:register(Node), 353 | receive 354 | {node_event, Node, up} -> 355 | Pid ! {watcher_node_up, Node}, 356 | Fun(); 357 | {node_event, Node, down} -> 358 | Pid ! {watcher_node_down, Node}, 359 | Fun(); 360 | stop -> ok 361 | end 362 | end). 363 | 364 | 365 | %% simulates a partition from a remote node by dropping messages 366 | %% received from some specific node 367 | simulate_partition(Node) -> 368 | meck:expect(aten_sink, handle_cast, 369 | fun ({hb, N}, State) when N =:= Node -> 370 | %% drop this message 371 | ct:pal("Dropping hb from ~w", [Node]), 372 | {noreply, State}; 373 | (Msg, State) -> 374 | aten_sink:handle_cast(Msg, State) 375 | end). 376 | 377 | get_current_host() -> 378 | N = atom_to_list(node()), 379 | {ok, list_to_atom(after_char($@, N))}. 380 | 381 | make_node_name(N) -> 382 | {ok, Host} = get_current_host(), 383 | list_to_atom(lists:flatten(io_lib:format("~s@~s", [N, Host]))). 384 | 385 | start_peer(N) -> 386 | Pa = filename:dirname(code:which(aten)), 387 | ct:pal("starting node ~w with ~s", [N, Pa]), 388 | {ok, P, S} = ?CT_PEER(#{name => N, args => ["-pa", Pa]}), 389 | ok = rpc:call(S, application, load, [aten]), 390 | ok = rpc:call(S, application, set_env, [aten, poll_interval, ?POLLINT]), 391 | ok = rpc:call(S, application, set_env, [aten, heartbeat_interval, ?HBINT]), 392 | ok = rpc:call(S, application, set_env, [aten, scaling_factor, ?SCALE]), 393 | {ok, _} = rpc:call(S, application, ensure_all_started, [aten]), 394 | {ok, P, S}. 395 | 396 | after_char(_, []) -> []; 397 | after_char(Char, [Char|Rest]) -> Rest; 398 | after_char(Char, [_|Rest]) -> after_char(Char, Rest). 399 | 400 | 401 | flush() -> 402 | receive M -> 403 | ct:pal("flushed ~w", [M]), 404 | flush() 405 | after ?POLLINT -> 406 | ok 407 | end. 408 | 409 | wait_for(Evt) -> 410 | receive 411 | Evt -> ok 412 | after 5000 -> 413 | exit({wait_for_timeout, Evt}) 414 | end. 415 | --------------------------------------------------------------------------------