├── .github ├── ISSUE_TEMPLATE.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── .pluginsync.yml ├── .sync.yml ├── .travis.yml ├── CONTRIBUTING.md ├── Godeps ├── Godeps.json └── Readme ├── LICENSE ├── METRICS.md ├── Makefile ├── NOTICE ├── README.md ├── VAGRANT.md ├── Vagrantfile ├── assets ├── deploy-distributed.png ├── deploy-distributed.svg └── grafana-dashboard.png ├── examples ├── configs │ └── snap-config-example.json ├── grafana │ └── mesos.json └── tasks │ ├── mesos-agent-influxdb.json │ ├── mesos-all-file.json │ └── mesos-master-influxdb.json ├── main.go ├── main_test.go ├── mesos ├── agent │ ├── agent.go │ └── agent_test.go ├── client │ ├── client.go │ └── client_test.go ├── master │ ├── master.go │ └── master_test.go ├── mesos.go ├── mesos_integration_test.go ├── mesos_pb2 │ └── mesos_pb2.go └── mesos_test.go └── scripts ├── build.sh ├── common.sh ├── deps.sh ├── large.sh ├── medium.sh ├── pre_deploy.sh ├── provision-travis.sh ├── provision-vagrant.sh ├── test.sh └── test ├── large_spec.rb └── spec_helper.rb /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 10 | 11 | **Snap daemon version** (use `snapteld -v`): 12 | 13 | **Environment**: 14 | - **Cloud provider or hardware configuration**: 15 | - **OS** (e.g. from /etc/os-release): 16 | - **Kernel** (e.g. `uname -a`): 17 | - **Relevant tools** (e.g. plugins used with Snap): 18 | - **Others** (e.g. deploying with Ansible): 19 | 20 | 21 | **What happened**: 22 | 23 | 24 | **What you expected to happen**: 25 | 26 | 27 | **Steps to reproduce it** (as minimally and precisely as possible): 28 | 29 | 1. 30 | 2. 31 | 3. 32 | 33 | 34 | **Anything else do we need to know** (e.g. issue happens only occasionally): 35 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 7 | Fixes # 8 | 9 | Summary of changes: 10 | - 11 | - 12 | - 13 | 14 | How to verify it: 15 | - 16 | 17 | Testing done: 18 | - 19 | 20 | A picture of a snapping turtle (not required but encouraged): 21 | - 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # File managed by pluginsync 2 | # 3 | # NOTE: please commit OS/Editor specific settings in your .gitignore_global 4 | # .idea 5 | # .DS_Store 6 | # 7 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 8 | *.o 9 | *.a 10 | *.so 11 | 12 | # Folders 13 | _obj 14 | _test 15 | 16 | # Architecture specific extensions/prefixes 17 | *.[568vq] 18 | [568vq].out 19 | 20 | *.cgo1.go 21 | *.cgo2.c 22 | _cgo_defun.c 23 | _cgo_gotypes.go 24 | _cgo_export.* 25 | 26 | _testmain.go 27 | 28 | *.exe 29 | *.test 30 | *.prof 31 | 32 | # Output of the go coverage tool 33 | *.out 34 | profile.cov 35 | 36 | # we don't vendor godep _workspace 37 | **/Godeps/_workspace/** 38 | vendor/ 39 | 40 | # ignore build artifacts 41 | build/ 42 | -------------------------------------------------------------------------------- /.pluginsync.yml: -------------------------------------------------------------------------------- 1 | # File managed by pluginsync 2 | pluginsync_config: '0.1.14' 3 | managed_files: 4 | - .github 5 | - .github/ISSUE_TEMPLATE.md 6 | - .github/PULL_REQUEST_TEMPLATE.md 7 | - .gitignore 8 | - .pluginsync.yml 9 | - .travis.yml 10 | - CONTRIBUTING.md 11 | - LICENSE 12 | - scripts 13 | - scripts/build.sh 14 | - scripts/common.sh 15 | - scripts/deps.sh 16 | - scripts/large.sh 17 | - scripts/pre_deploy.sh 18 | - scripts/test 19 | - scripts/test/large_spec.rb 20 | - scripts/test/spec_helper.rb 21 | -------------------------------------------------------------------------------- /.sync.yml: -------------------------------------------------------------------------------- 1 | .travis.yml: 2 | sudo: true 3 | dist: trusty 4 | env: 5 | global: 6 | - ORG_PATH=/home/travis/gopath/src/github.com/intelsdi-x 7 | - SNAP_PLUGIN_SOURCE=/home/travis/gopath/src/github.com/${TRAVIS_REPO_SLUG} 8 | matrix: 9 | - TEST_TYPE=small 10 | # Integration test matrix 11 | # 12 | # Mesos releases occur every two months, and each minor release is supported for six months. 13 | # For more info, see: http://mesos.apache.org/documentation/latest/versioning/ 14 | # 15 | # Fortunately, Mesosphere doesn't remove previous releases from its Apt repositories. 16 | # Determine the version string to use here by running `apt-get update && apt-cache policy mesos` 17 | - TEST_TYPE=medium MESOS_RELEASE=0.28.1-2.0.20.ubuntu1404 18 | Makefile: 19 | unmanaged: true 20 | scripts/test.sh: 21 | unmanaged: true 22 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # File managed by pluginsync 2 | sudo: true 3 | dist: trusty 4 | language: go 5 | go: 6 | - 1.7.x 7 | - 1.8.x 8 | - 1.9.x 9 | env: 10 | global: 11 | - ORG_PATH=/home/travis/gopath/src/github.com/intelsdi-x 12 | - SNAP_PLUGIN_SOURCE=/home/travis/gopath/src/github.com/${TRAVIS_REPO_SLUG} 13 | matrix: 14 | - TEST_TYPE=small 15 | - TEST_TYPE=medium MESOS_RELEASE=0.28.1-2.0.20.ubuntu1404 16 | - TEST_TYPE: build 17 | matrix: 18 | exclude: 19 | - go: 1.7.x 20 | env: TEST_TYPE=build 21 | before_install: 22 | - "[[ -d $SNAP_PLUGIN_SOURCE ]] || mkdir -p $ORG_PATH && ln -s $TRAVIS_BUILD_DIR $SNAP_PLUGIN_SOURCE" 23 | install: 24 | - cd $SNAP_PLUGIN_SOURCE 25 | - make deps 26 | script: 27 | - make check 2>&1 28 | notifications: 29 | email: false 30 | slack: 31 | secure: VkbZLIc2RH8yf3PtIAxUNPdAu3rQQ7yQx0GcK124JhbEnZGaHyK615V0rbG7HcVmYKGPdB0cXqZiLBDKGqGKb2zR1NepOe1nF03jxGSpPq8jIFeEXSJGEYGL34ScDzZZGuG6qwbjFcXiW5lqn6t8igzp7v2+URYBaZo5ktCS2xY= 32 | before_deploy: 33 | - "./scripts/pre_deploy.sh" 34 | deploy: 35 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # snap plugin collector mesos 2 | 3 | 1. [Contributing Code](#contributing-code) 4 | 2. [Contributing Examples](#contributing-examples) 5 | 3. [Contribute Elsewhere](#contribute-elsewhere) 6 | 4. [Reporting Security Issues](#reporting-security-issues) 7 | 5. [Thank You](#thank-you) 8 | 9 | This repository is primarily **community supported**. We both appreciate and need your contribution to keep it stable. Thank you for being part of the community! We love you for it. 10 | 11 | ## Contributing Code 12 | **_IMPORTANT_**: We encourage contributions to the project from the community. We ask that you keep the following guidelines in mind when planning your contribution. 13 | 14 | * Whether your contribution is for a bug fix or a feature request, **create an [Issue](https://github.com/intelsdi-x/snap-plugin-collector-mesos/issues)** and let us know what you are thinking. 15 | * **For bugs**, if you have already found a fix, feel free to submit a Pull Request referencing the Issue you created. Include the `Fixes #` syntax to link it to the issue you're addressing. 16 | * **For feature requests**, we want to improve upon the library incrementally which means small changes at a time. In order to ensure your PR can be reviewed in a timely manner, please keep PRs small, e.g. <10 files and <500 lines changed. If you think this is unrealistic, then mention that within the issue and we can discuss it. 17 | 18 | Once you're ready to contribute code back to this repo, start with these steps: 19 | 20 | * Fork the appropriate sub-projects that are affected by your change. 21 | * Clone the fork to `$GOPATH/src/github.com/intelsdi-x/`: 22 | ``` 23 | $ cd "${GOPATH}/src/github.com/intelsdi-x/" 24 | $ git clone https://github.com/intelsdi-x/snap-plugin-collector-mesos.git 25 | ``` 26 | * Create a topic branch for your change and checkout that branch: 27 | ``` 28 | $ git checkout -b some-topic-branch 29 | ``` 30 | * Make your changes to the code and add tests to cover contributed code. 31 | * Validate the changes and run the test suite if one is provided. 32 | * Commit your changes and push them to your fork. 33 | * Open a pull request for the appropriate project. 34 | * Contributors will review your pull request, suggest changes, and merge it when it’s ready and/or offer feedback. 35 | 36 | If you have questions feel free to contact the [maintainers](https://github.com/intelsdi-x/snap/blob/master/docs/MAINTAINERS.md). 37 | 38 | ## Contributing Examples 39 | The most immediately helpful way you can benefit this project is by cloning the repository, adding some further examples and submitting a pull request. 40 | 41 | Have you written a blog post about how you use [Snap](http://github.com/intelsdi-x/snap) and/or this plugin? Send it to us [on Slack](http://slack.snap-telemetry.io)! 42 | 43 | ## Contribute Elsewhere 44 | This repository is one of **many** plugins in **Snap**, a powerful telemetry framework. See the full project at http://snap-telemetry.io 45 | 46 | 47 | ## Reporting Security Issues 48 | 49 | The Snap team take security very seriously. If you have any issue regarding security, 50 | please notify us by sending an email to snap-security@intel.com and not by creating a GitHub issue. 51 | We will follow up with you promptly with more information and a plan for remediation. 52 | While we are not offering a security bounty, we would love to send some Snap swag your way along with our 53 | deepest gratitude for your assistance in making Snap a more secure product. 54 | 55 | ## Thank You 56 | And **thank you!** Your contribution, through code and participation, is incredibly important to us. 57 | -------------------------------------------------------------------------------- /Godeps/Godeps.json: -------------------------------------------------------------------------------- 1 | { 2 | "ImportPath": "github.com/intelsdi-x/snap-plugin-collector-mesos", 3 | "GoVersion": "go1.5", 4 | "GodepVersion": "v62", 5 | "Deps": [ 6 | { 7 | "ImportPath": "github.com/Sirupsen/logrus", 8 | "Comment": "v0.10.0", 9 | "Rev": "4b6ea7319e214d98c938f12692336f7ca9348d6b" 10 | }, 11 | { 12 | "ImportPath": "github.com/golang/protobuf/proto", 13 | "Rev": "7cc19b78d562895b13596ddce7aafb59dd789318" 14 | }, 15 | { 16 | "ImportPath": "github.com/gopherjs/gopherjs/js", 17 | "Comment": "go1.5-43-gf10744b", 18 | "Rev": "f10744b1512e8cf31ec55b49ddeb1c93ed2de46e" 19 | }, 20 | { 21 | "ImportPath": "github.com/intelsdi-x/snap-plugin-utilities/config", 22 | "Rev": "7b3831f4053c1fe9870e68d8b7e33a2ad731e42c" 23 | }, 24 | { 25 | "ImportPath": "github.com/intelsdi-x/snap-plugin-utilities/ns", 26 | "Rev": "7b3831f4053c1fe9870e68d8b7e33a2ad731e42c" 27 | }, 28 | { 29 | "ImportPath": "github.com/intelsdi-x/snap-plugin-utilities/str", 30 | "Rev": "7b3831f4053c1fe9870e68d8b7e33a2ad731e42c" 31 | }, 32 | { 33 | "ImportPath": "github.com/intelsdi-x/snap/control/plugin", 34 | "Comment": "v0.14.0-beta", 35 | "Rev": "506318ebb84a8fefc5ef966edc4318c9cf5b2002" 36 | }, 37 | { 38 | "ImportPath": "github.com/intelsdi-x/snap/control/plugin/cpolicy", 39 | "Comment": "v0.14.0-beta", 40 | "Rev": "506318ebb84a8fefc5ef966edc4318c9cf5b2002" 41 | }, 42 | { 43 | "ImportPath": "github.com/intelsdi-x/snap/control/plugin/encoding", 44 | "Comment": "v0.14.0-beta", 45 | "Rev": "506318ebb84a8fefc5ef966edc4318c9cf5b2002" 46 | }, 47 | { 48 | "ImportPath": "github.com/intelsdi-x/snap/control/plugin/encrypter", 49 | "Comment": "v0.14.0-beta", 50 | "Rev": "506318ebb84a8fefc5ef966edc4318c9cf5b2002" 51 | }, 52 | { 53 | "ImportPath": "github.com/intelsdi-x/snap/core", 54 | "Comment": "v0.14.0-beta", 55 | "Rev": "506318ebb84a8fefc5ef966edc4318c9cf5b2002" 56 | }, 57 | { 58 | "ImportPath": "github.com/intelsdi-x/snap/core/cdata", 59 | "Comment": "v0.14.0-beta", 60 | "Rev": "506318ebb84a8fefc5ef966edc4318c9cf5b2002" 61 | }, 62 | { 63 | "ImportPath": "github.com/intelsdi-x/snap/core/ctypes", 64 | "Comment": "v0.14.0-beta", 65 | "Rev": "506318ebb84a8fefc5ef966edc4318c9cf5b2002" 66 | }, 67 | { 68 | "ImportPath": "github.com/intelsdi-x/snap/core/serror", 69 | "Comment": "v0.14.0-beta", 70 | "Rev": "506318ebb84a8fefc5ef966edc4318c9cf5b2002" 71 | }, 72 | { 73 | "ImportPath": "github.com/intelsdi-x/snap/pkg/ctree", 74 | "Comment": "v0.14.0-beta", 75 | "Rev": "506318ebb84a8fefc5ef966edc4318c9cf5b2002" 76 | }, 77 | { 78 | "ImportPath": "github.com/intelsdi-x/snap/pkg/schedule", 79 | "Comment": "v0.14.0-beta", 80 | "Rev": "506318ebb84a8fefc5ef966edc4318c9cf5b2002" 81 | }, 82 | { 83 | "ImportPath": "github.com/intelsdi-x/snap/scheduler/wmap", 84 | "Comment": "v0.14.0-beta", 85 | "Rev": "506318ebb84a8fefc5ef966edc4318c9cf5b2002" 86 | }, 87 | { 88 | "ImportPath": "github.com/jtolds/gls", 89 | "Comment": "v4.2.0", 90 | "Rev": "8ddce2a84170772b95dd5d576c48d517b22cac63" 91 | }, 92 | { 93 | "ImportPath": "github.com/oleiade/reflections", 94 | "Comment": "0.1.2-2-g632977f", 95 | "Rev": "632977f98cd34d217c4b57d0840ec188b3d3dcaf" 96 | }, 97 | { 98 | "ImportPath": "github.com/robfig/cron", 99 | "Comment": "v1-16-g0f39cf7", 100 | "Rev": "0f39cf7ebc65a602f45692f9894bd6a193faf8fa" 101 | }, 102 | { 103 | "ImportPath": "github.com/smartystreets/assertions", 104 | "Comment": "1.6.0-2-g2d74a41", 105 | "Rev": "2d74a419fbe076f8c76ed8b1c7c633a676c5dc63" 106 | }, 107 | { 108 | "ImportPath": "github.com/smartystreets/assertions/internal/go-render/render", 109 | "Comment": "1.6.0-2-g2d74a41", 110 | "Rev": "2d74a419fbe076f8c76ed8b1c7c633a676c5dc63" 111 | }, 112 | { 113 | "ImportPath": "github.com/smartystreets/assertions/internal/oglematchers", 114 | "Comment": "1.6.0-2-g2d74a41", 115 | "Rev": "2d74a419fbe076f8c76ed8b1c7c633a676c5dc63" 116 | }, 117 | { 118 | "ImportPath": "github.com/smartystreets/goconvey/convey", 119 | "Comment": "1.6.1-20-gcc5d85f", 120 | "Rev": "cc5d85f4f15bd2163abf0fe6e6753356dde72aa2" 121 | }, 122 | { 123 | "ImportPath": "github.com/smartystreets/goconvey/convey/gotest", 124 | "Comment": "1.6.1-20-gcc5d85f", 125 | "Rev": "cc5d85f4f15bd2163abf0fe6e6753356dde72aa2" 126 | }, 127 | { 128 | "ImportPath": "github.com/smartystreets/goconvey/convey/reporting", 129 | "Comment": "1.6.1-20-gcc5d85f", 130 | "Rev": "cc5d85f4f15bd2163abf0fe6e6753356dde72aa2" 131 | }, 132 | { 133 | "ImportPath": "golang.org/x/sys/unix", 134 | "Rev": "9eef40adf05b951699605195b829612bd7b69952" 135 | }, 136 | { 137 | "ImportPath": "gopkg.in/yaml.v2", 138 | "Rev": "a83829b6f1293c91addabc89d0571c246397bbf4" 139 | } 140 | ] 141 | } 142 | -------------------------------------------------------------------------------- /Godeps/Readme: -------------------------------------------------------------------------------- 1 | This directory tree is generated automatically by godep. 2 | 3 | Please do not edit. 4 | 5 | See https://github.com/tools/godep for more information. 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | -------------------------------------------------------------------------------- /METRICS.md: -------------------------------------------------------------------------------- 1 | # snap collector plugin - mesos 2 | 3 | ## Collected Metrics 4 | 5 | This plugin has the ability to gather the following metrics: 6 | 7 | Namespace | Data Type | Description 8 | ----------------------------------------------------------------------------------|-----------|--------------- 9 | /intel/mesos/agent/[framework_id]/[executor_id]/cpus_limit | | 10 | /intel/mesos/agent/[framework_id]/[executor_id]/cpus_nr_periods | | 11 | /intel/mesos/agent/[framework_id]/[executor_id]/cpus_nr_throttled | | 12 | /intel/mesos/agent/[framework_id]/[executor_id]/cpus_system_time_secs | | 13 | /intel/mesos/agent/[framework_id]/[executor_id]/cpus_throttled_time_secs | | 14 | /intel/mesos/agent/[framework_id]/[executor_id]/cpus_user_time_secs | | 15 | /intel/mesos/agent/[framework_id]/[executor_id]/mem_anon_bytes | | 16 | /intel/mesos/agent/[framework_id]/[executor_id]/mem_cache_bytes | | 17 | /intel/mesos/agent/[framework_id]/[executor_id]/mem_critical_pressure_counter | | 18 | /intel/mesos/agent/[framework_id]/[executor_id]/mem_file_bytes | | 19 | /intel/mesos/agent/[framework_id]/[executor_id]/mem_limit_bytes | | 20 | /intel/mesos/agent/[framework_id]/[executor_id]/mem_low_pressure_counter | | 21 | /intel/mesos/agent/[framework_id]/[executor_id]/mem_mapped_file_bytes | | 22 | /intel/mesos/agent/[framework_id]/[executor_id]/mem_medium_pressure_counter | | 23 | /intel/mesos/agent/[framework_id]/[executor_id]/mem_rss_bytes | | 24 | /intel/mesos/agent/[framework_id]/[executor_id]/mem_soft_limit_bytes | | 25 | /intel/mesos/agent/[framework_id]/[executor_id]/mem_swap_bytes | | 26 | /intel/mesos/agent/[framework_id]/[executor_id]/mem_total_bytes | | 27 | /intel/mesos/agent/[framework_id]/[executor_id]/mem_total_memsw_bytes | | 28 | /intel/mesos/agent/[framework_id]/[executor_id]/mem_unevictable_bytes | | 29 | /intel/mesos/agent/[framework_id]/[executor_id]/processes | | 30 | /intel/mesos/agent/[framework_id]/[executor_id]/threads | | 31 | /intel/mesos/agent/[framework_id]/[executor_id]/timestamp | | 32 | /intel/mesos/agent/containerizer/mesos/container_destroy_errors | | 33 | /intel/mesos/agent/containerizer/mesos/filesystem/containers_new_rootfs | | 34 | /intel/mesos/agent/containerizer/mesos/provisioner/bind/remove_rootfs_errors | | 35 | /intel/mesos/agent/containerizer/mesos/provisioner/remove_container_errors | | 36 | /intel/mesos/agent/slave/container_launch_errors | | 37 | /intel/mesos/agent/slave/cpus_percent | | 38 | /intel/mesos/agent/slave/cpus_revocable_percent | | 39 | /intel/mesos/agent/slave/cpus_revocable_total | | 40 | /intel/mesos/agent/slave/cpus_revocable_used | | 41 | /intel/mesos/agent/slave/cpus_total | | 42 | /intel/mesos/agent/slave/cpus_used | | 43 | /intel/mesos/agent/slave/disk_percent | | 44 | /intel/mesos/agent/slave/disk_revocable_percent | | 45 | /intel/mesos/agent/slave/disk_revocable_total | | 46 | /intel/mesos/agent/slave/disk_revocable_used | | 47 | /intel/mesos/agent/slave/disk_total | | 48 | /intel/mesos/agent/slave/disk_used | | 49 | /intel/mesos/agent/slave/executor_directory_max_allowed_age_secs | | 50 | /intel/mesos/agent/slave/executors_preempted | | 51 | /intel/mesos/agent/slave/executors_registering | | 52 | /intel/mesos/agent/slave/executors_running | | 53 | /intel/mesos/agent/slave/executors_terminated | | 54 | /intel/mesos/agent/slave/executors_terminating | | 55 | /intel/mesos/agent/slave/frameworks_active | | 56 | /intel/mesos/agent/slave/gpus_percent | | 57 | /intel/mesos/agent/slave/gpus_revocable_percent | | 58 | /intel/mesos/agent/slave/gpus_revocable_total | | 59 | /intel/mesos/agent/slave/gpus_revocable_used | | 60 | /intel/mesos/agent/slave/gpus_total | | 61 | /intel/mesos/agent/slave/gpus_used | | 62 | /intel/mesos/agent/slave/invalid_framework_messages | | 63 | /intel/mesos/agent/slave/invalid_status_updates | | 64 | /intel/mesos/agent/slave/mem_percent | | 65 | /intel/mesos/agent/slave/mem_revocable_percent | | 66 | /intel/mesos/agent/slave/mem_revocable_total | | 67 | /intel/mesos/agent/slave/mem_revocable_used | | 68 | /intel/mesos/agent/slave/mem_total | | 69 | /intel/mesos/agent/slave/mem_used | | 70 | /intel/mesos/agent/slave/recovery_errors | | 71 | /intel/mesos/agent/slave/registered | | 72 | /intel/mesos/agent/slave/tasks_failed | | 73 | /intel/mesos/agent/slave/tasks_finished | | 74 | /intel/mesos/agent/slave/tasks_gone | | 75 | /intel/mesos/agent/slave/tasks_killed | | 76 | /intel/mesos/agent/slave/tasks_killing | | 77 | /intel/mesos/agent/slave/tasks_lost | | 78 | /intel/mesos/agent/slave/tasks_running | | 79 | /intel/mesos/agent/slave/tasks_staging | | 80 | /intel/mesos/agent/slave/tasks_starting | | 81 | /intel/mesos/agent/slave/uptime_secs | | 82 | /intel/mesos/agent/slave/valid_framework_messages | | 83 | /intel/mesos/agent/slave/valid_status_updates | | 84 | /intel/mesos/agent/system/cpus_total | | 85 | /intel/mesos/agent/system/load_15min | | 86 | /intel/mesos/agent/system/load_1min | | 87 | /intel/mesos/agent/system/load_5min | | 88 | /intel/mesos/agent/system/mem_free_bytes | | 89 | /intel/mesos/agent/system/mem_total_bytes | | 90 | /intel/mesos/master/[framework_id]/offered_resources/cpus | | 91 | /intel/mesos/master/[framework_id]/offered_resources/disk | | 92 | /intel/mesos/master/[framework_id]/offered_resources/mem | | 93 | /intel/mesos/master/[framework_id]/resources/cpus | | 94 | /intel/mesos/master/[framework_id]/resources/disk | | 95 | /intel/mesos/master/[framework_id]/resources/mem | | 96 | /intel/mesos/master/[framework_id]/used_resources/cpus | | 97 | /intel/mesos/master/[framework_id]/used_resources/disk | | 98 | /intel/mesos/master/[framework_id]/used_resources/mem | | 99 | /intel/mesos/master/allocator/event_queue_dispatches | | 100 | /intel/mesos/master/allocator/mesos/allocation_run_ms/count | | 101 | /intel/mesos/master/allocator/mesos/allocation_run_ms/max | | 102 | /intel/mesos/master/allocator/mesos/allocation_run_ms/min | | 103 | /intel/mesos/master/allocator/mesos/allocation_run_ms/p50 | | 104 | /intel/mesos/master/allocator/mesos/allocation_run_ms/p90 | | 105 | /intel/mesos/master/allocator/mesos/allocation_run_ms/p95 | | 106 | /intel/mesos/master/allocator/mesos/allocation_run_ms/p9999 | | 107 | /intel/mesos/master/allocator/mesos/allocation_run_ms/p999 | | 108 | /intel/mesos/master/allocator/mesos/allocation_run_ms/p99 | | 109 | /intel/mesos/master/allocator/mesos/allocation_run_ms | | 110 | /intel/mesos/master/allocator/mesos/allocation_runs | | 111 | /intel/mesos/master/allocator/mesos/event_queue_dispatches | | 112 | /intel/mesos/master/allocator/mesos/resources/cpus/offered_or_allocated | | 113 | /intel/mesos/master/allocator/mesos/resources/cpus/total | | 114 | /intel/mesos/master/allocator/mesos/resources/disk/offered_or_allocated | | 115 | /intel/mesos/master/allocator/mesos/resources/disk/total | | 116 | /intel/mesos/master/allocator/mesos/resources/mem/offered_or_allocated | | 117 | /intel/mesos/master/allocator/mesos/resources/mem/total | | 118 | /intel/mesos/master/master/cpus_percent | | 119 | /intel/mesos/master/master/cpus_revocable_percent | | 120 | /intel/mesos/master/master/cpus_revocable_total | | 121 | /intel/mesos/master/master/cpus_revocable_used | | 122 | /intel/mesos/master/master/cpus_total | | 123 | /intel/mesos/master/master/cpus_used | | 124 | /intel/mesos/master/master/disk_percent | | 125 | /intel/mesos/master/master/disk_revocable_percent | | 126 | /intel/mesos/master/master/disk_revocable_total | | 127 | /intel/mesos/master/master/disk_revocable_used | | 128 | /intel/mesos/master/master/disk_total | | 129 | /intel/mesos/master/master/disk_used | | 130 | /intel/mesos/master/master/dropped_messages | | 131 | /intel/mesos/master/master/elected | | 132 | /intel/mesos/master/master/event_queue_dispatches | | 133 | /intel/mesos/master/master/event_queue_http_requests | | 134 | /intel/mesos/master/master/event_queue_messages | | 135 | /intel/mesos/master/master/frameworks_active | | 136 | /intel/mesos/master/master/frameworks_connected | | 137 | /intel/mesos/master/master/frameworks_disconnected | | 138 | /intel/mesos/master/master/frameworks_inactive | | 139 | /intel/mesos/master/master/gpus_percent | | 140 | /intel/mesos/master/master/gpus_revocable_percent | | 141 | /intel/mesos/master/master/gpus_revocable_total | | 142 | /intel/mesos/master/master/gpus_revocable_used | | 143 | /intel/mesos/master/master/gpus_total | | 144 | /intel/mesos/master/master/gpus_used | | 145 | /intel/mesos/master/master/invalid_executor_to_framework_messages | | 146 | /intel/mesos/master/master/invalid_framework_to_executor_messages | | 147 | /intel/mesos/master/master/invalid_status_update_acknowledgements | | 148 | /intel/mesos/master/master/invalid_status_updates | | 149 | /intel/mesos/master/master/mem_percent | | 150 | /intel/mesos/master/master/mem_revocable_percent | | 151 | /intel/mesos/master/master/mem_revocable_total | | 152 | /intel/mesos/master/master/mem_revocable_used | | 153 | /intel/mesos/master/master/mem_total | | 154 | /intel/mesos/master/master/mem_used | | 155 | /intel/mesos/master/master/messages_authenticate | | 156 | /intel/mesos/master/master/messages_deactivate_framework | | 157 | /intel/mesos/master/master/messages_decline_offers | | 158 | /intel/mesos/master/master/messages_executor_to_framework | | 159 | /intel/mesos/master/master/messages_exited_executor | | 160 | /intel/mesos/master/master/messages_framework_to_executor | | 161 | /intel/mesos/master/master/messages_kill_task | | 162 | /intel/mesos/master/master/messages_launch_tasks | | 163 | /intel/mesos/master/master/messages_reconcile_tasks | | 164 | /intel/mesos/master/master/messages_register_framework | | 165 | /intel/mesos/master/master/messages_register_slave | | 166 | /intel/mesos/master/master/messages_reregister_framework | | 167 | /intel/mesos/master/master/messages_reregister_slave | | 168 | /intel/mesos/master/master/messages_resource_request | | 169 | /intel/mesos/master/master/messages_revive_offers | | 170 | /intel/mesos/master/master/messages_status_update | | 171 | /intel/mesos/master/master/messages_status_update_acknowledgement | | 172 | /intel/mesos/master/master/messages_suppress_offers | | 173 | /intel/mesos/master/master/messages_unregister_framework | | 174 | /intel/mesos/master/master/messages_unregister_slave | | 175 | /intel/mesos/master/master/messages_update_slave | | 176 | /intel/mesos/master/master/outstanding_offers | | 177 | /intel/mesos/master/master/recovery_slave_removals | | 178 | /intel/mesos/master/master/slave_registrations | | 179 | /intel/mesos/master/master/slave_removals/reason_registered | | 180 | /intel/mesos/master/master/slave_removals/reason_unhealthy | | 181 | /intel/mesos/master/master/slave_removals/reason_unregistered | | 182 | /intel/mesos/master/master/slave_removals | | 183 | /intel/mesos/master/master/slave_reregistrations | | 184 | /intel/mesos/master/master/slave_shutdowns_canceled | | 185 | /intel/mesos/master/master/slave_shutdowns_completed | | 186 | /intel/mesos/master/master/slave_shutdowns_scheduled | | 187 | /intel/mesos/master/master/slave_unreachable_canceled | | 188 | /intel/mesos/master/master/slave_unreachable_completed | | 189 | /intel/mesos/master/master/slave_unreachable_scheduled | | 190 | /intel/mesos/master/master/slaves_active | | 191 | /intel/mesos/master/master/slaves_connected | | 192 | /intel/mesos/master/master/slaves_disconnected | | 193 | /intel/mesos/master/master/slaves_inactive | | 194 | /intel/mesos/master/master/tasks_dropped | | 195 | /intel/mesos/master/master/tasks_error | | 196 | /intel/mesos/master/master/tasks_failed | | 197 | /intel/mesos/master/master/tasks_finished | | 198 | /intel/mesos/master/master/tasks_gone | | 199 | /intel/mesos/master/master/tasks_gone_by_operator | | 200 | /intel/mesos/master/master/tasks_killed | | 201 | /intel/mesos/master/master/tasks_killing | | 202 | /intel/mesos/master/master/tasks_lost | | 203 | /intel/mesos/master/master/tasks_running | | 204 | /intel/mesos/master/master/tasks_staging | | 205 | /intel/mesos/master/master/tasks_starting | | 206 | /intel/mesos/master/master/tasks_unreachable | | 207 | /intel/mesos/master/master/uptime_secs | | 208 | /intel/mesos/master/master/valid_executor_to_framework_messages | | 209 | /intel/mesos/master/master/valid_framework_to_executor_messages | | 210 | /intel/mesos/master/master/valid_status_update_acknowledgements | | 211 | /intel/mesos/master/master/valid_status_updates | | 212 | /intel/mesos/master/registrar/log/recovered | | 213 | /intel/mesos/master/registrar/queued_operations | | 214 | /intel/mesos/master/registrar/registry_size_bytes | | 215 | /intel/mesos/master/registrar/state_fetch_ms | | 216 | /intel/mesos/master/registrar/state_store_ms/count | | 217 | /intel/mesos/master/registrar/state_store_ms/max | | 218 | /intel/mesos/master/registrar/state_store_ms/min | | 219 | /intel/mesos/master/registrar/state_store_ms/p50 | | 220 | /intel/mesos/master/registrar/state_store_ms/p90 | | 221 | /intel/mesos/master/registrar/state_store_ms/p95 | | 222 | /intel/mesos/master/registrar/state_store_ms/p9999 | | 223 | /intel/mesos/master/registrar/state_store_ms/p999 | | 224 | /intel/mesos/master/registrar/state_store_ms/p99 | | 225 | /intel/mesos/master/registrar/state_store_ms | | 226 | /intel/mesos/master/system/cpus_total | | 227 | /intel/mesos/master/system/load_15min | | 228 | /intel/mesos/master/system/load_1min | | 229 | /intel/mesos/master/system/load_5min | | 230 | /intel/mesos/master/system/mem_free_bytes | | 231 | /intel/mesos/master/system/mem_total_bytes | | 232 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | default: 2 | $(MAKE) clean 3 | $(MAKE) deps 4 | $(MAKE) all 5 | clean: 6 | -rm -rf build/ 7 | -rm -rf vendor/ 8 | -rm -rf Godeps/_workspace 9 | -rm -rf tmp/ 10 | -rm -rf profile.cov 11 | deps: 12 | bash -c "./scripts/deps.sh" 13 | test: 14 | bash -c "./scripts/test.sh $(TEST_TYPE)" 15 | test-legacy: 16 | bash -c "./scripts/test.sh legacy" 17 | test-small: 18 | bash -c "./scripts/test.sh small" 19 | test-medium: 20 | bash -c "./scripts/test.sh medium" 21 | test-large: 22 | bash -c "./scripts/test.sh large" 23 | check: 24 | $(MAKE) test 25 | all: 26 | bash -c "./scripts/build.sh $(PWD)" 27 | protobuf: 28 | curl -L -o mesos_pb2.proto https://raw.githubusercontent.com/apache/mesos/0.28.1/include/mesos/mesos.proto 29 | protoc --go_out=import_path=mesos_pb2:mesos/mesos_pb2 mesos_pb2.proto 30 | mv mesos/mesos_pb2/mesos_pb2.pb.go mesos/mesos_pb2/mesos_pb2.go 31 | rm -f mesos_pb2.proto 32 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Snap 2 | Copyright 2016 Intel Corporation 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | This software contains portions of Apache/Mesos from https://github.com/apache/mesos, licensed 9 | under an Apache License 2.0 in file mesos/mesos_pb2/mesos_pb2.go. 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | DISCONTINUATION OF PROJECT. 2 | 3 | This project will no longer be maintained by Intel. 4 | 5 | This project has been identified as having known security escapes. 6 | 7 | Intel has ceased development and contributions including, but not limited to, maintenance, bug fixes, new releases, or updates, to this project. 8 | 9 | Intel no longer accepts patches to this project. 10 | # DISCONTINUATION OF PROJECT 11 | 12 | **This project will no longer be maintained by Intel. Intel will not provide or guarantee development of or support for this project, including but not limited to, maintenance, bug fixes, new releases or updates. Patches to this project are no longer accepted by Intel. If you have an ongoing need to use this project, are interested in independently developing it, or would like to maintain patches for the community, please create your own fork of the project.** 13 | 14 | 15 | 16 | # Snap Collector Plugin — Apache Mesos 17 | 18 | [![Build Status](https://travis-ci.org/intelsdi-x/snap-plugin-collector-mesos.svg?branch=master)](https://travis-ci.org/intelsdi-x/snap-plugin-collector-mesos) 19 | [![Coverage Status](https://coveralls.io/repos/github/intelsdi-x/snap-plugin-collector-mesos/badge.svg?branch=master)](https://coveralls.io/github/intelsdi-x/snap-plugin-collector-mesos?branch=master) 20 | [![Go Report Card](https://goreportcard.com/badge/github.com/intelsdi-x/snap-plugin-collector-mesos)](https://goreportcard.com/report/github.com/intelsdi-x/snap-plugin-collector-mesos) 21 | 22 | This Snap plugin collects metrics from an [Apache Mesos][mesos-home] cluster. 23 | It gathers information about cluster resource allocation and utilization, as 24 | well as metrics about running containers. 25 | 26 | 1. [Getting Started](#getting-started) 27 | * [System Requirements](#system-requirements) 28 | * [Installation](#installation) 29 | * [Configuration and Usage](#configuration-and-usage) 30 | 2. [Documentation](#documentation) 31 | * [Collected Metrics](#collected-metrics) 32 | * [Examples](#examples) 33 | * [Known Issues and Caveats](#known-issues-and-caveats) 34 | * [Roadmap](#roadmap) 35 | 3. [Community Support](#community-support) 36 | 4. [Contributing](#contributing) 37 | 5. [License](#license) 38 | 6. [Acknowledgements](#acknowledgements) 39 | 40 | ## Getting Started 41 | ### System Requirements 42 | At a minimum, you'll need: 43 | * [Apache Mesos][mesos-home] (currently tested against 0.26.x, 0.27.x, and 0.28.x) 44 | * [Golang 1.5+][golang-dl] (only needed for building the plugin) 45 | * [Snap][snap-github] v0.14+ 46 | * Linux (amd64) 47 | * InfluxDB and the [InfluxDB publisher for Snap][snap-influxdb] v13+ (optional) 48 | * Mac OS X (x86_64) (for development/testing only) 49 | 50 | To enable metrics collection from the `perf_event` cgroups subsystem, you'll also need a kernel that supports it, 51 | and the userland tools that enable you to run the `perf` command. On Ubuntu, this includes the following packages: 52 | * `linux-tools-common` 53 | * `linux-tools-generic` 54 | * `linux-tools-$(uname -r)` 55 | 56 | *Note: see the [Caveats](#known-issues-and-caveats) section below for known issues with Mesos and the 57 | `cgroups/perf_event` isolator.* 58 | 59 | ### Installation 60 | #### Download the Mesos plugin binary 61 | You can download pre-built binaries for Linux x64 from this plugin's [GitHub Releases][releases] page. 62 | 63 | #### Building the plugin from source 64 | Clone this GitHub repository: 65 | 66 | ``` 67 | $ git clone https://github.com/intelsdi-x/snap-plugin-collector-mesos 68 | $ cd snap-plugin-collector-mesos 69 | ``` 70 | 71 | Build the plugin by running `make`: 72 | 73 | ``` 74 | $ make 75 | ``` 76 | 77 | After compilation has finished, the binary will be available in `build/rootfs/snap-plugin-collector-mesos` 78 | 79 | ### Configuration and Usage 80 | First, be sure that you've familiarized yourself with the Snap framework by reading the 81 | [Getting Started documentation][snap-getting-started]. 82 | 83 | To specify the hostname or IP address and port that this plugin should connect to, add the following to the Snap 84 | global configuration, in the `collector` object: 85 | 86 | ``` 87 | "mesos": { 88 | "all": { 89 | "master": "10.180.10.180:5050", 90 | "agent": "10.180.10.180:5051" 91 | } 92 | } 93 | ``` 94 | 95 | Typically, you should only need to provide a value for `master` _or_ `agent`, unless you're running the master and agent 96 | on the same machine (e.g. in development). This plugin will automatically determine which service(s) you provided a 97 | host/port combination for and collect metrics from the master and/or the agent. 98 | 99 | Snap and this plugin should then be deployed to each machine in the Mesos cluster. This allows collection to happen as 100 | close to the agent as possible. 101 | 102 | ![deploy-distributed](assets/deploy-distributed.png) 103 | 104 | When monitoring the Mesos masters, the plugin queries the local Mesos master to determine if it's currently the leader. 105 | If it is, metrics collection will occur normally. If the local master is not currently the leader, a message will be 106 | recorded to the plugin log, and metrics collection will not happen on that machine. 107 | 108 | Once Snap is configured and this plugin is loaded, you'll be able to start collecting metrics from the Mesos cluster. 109 | For examples on how to do this, see the [Examples](#examples) section below. 110 | 111 | ## Documentation 112 | Mesos is a complex system and its installation and administration is outside the scope of this README. There are a few 113 | resources you might want to consider taking a look at to get started with Mesos: 114 | * [Apache Mesos "Getting Started" documentation][mesos-getting-started] 115 | * [Mesosphere Downloads][mesosphere-downloads] 116 | * [scripts/provision-travis.sh](scripts/provision-travis.sh) 117 | 118 | Design documents and RFCs pertaining to this plugin are tracked via the ["RFC" label in GitHub Issues][github-rfc]. 119 | 120 | Also, you may want to consider checking out the following tests to understand how this plugin works: 121 | * [Master unit tests](mesos/master/master_test.go) 122 | * [Agent unit tests](mesos/agent/agent_test.go) 123 | * [Plugin integration tests](mesos/mesos_integration_test.go) 124 | 125 | ### Collected Metrics 126 | This plugin collects hundreds of metrics from Mesos masters and agents. As such, there are too many to list them all 127 | here, so instead we've provided a quick overview. 128 | 129 | List of collected metrics is described in [METRICS.md](METRICS.md) 130 | 131 | To get a complete list of available metrics, you can run the 132 | following commands: 133 | ``` 134 | $ snaptel plugin load snap-plugin-collector-mesos 135 | $ snaptel metric list 136 | ``` 137 | 138 | #### Mesos framework metrics 139 | At the highest (per-cluster) level, this plugin returns select metrics from the `/master/frameworks` API endpoint on 140 | the leading Mesos master. Specifically, we collect the following metrics on a per-framework basis: 141 | 142 | * Offered CPUs, memory, and disk 143 | * Allocated CPUs, memory, and disk 144 | * Used CPUs, memory, and disk 145 | 146 | 147 | #### Mesos master/agent metrics 148 | This plugin returns all available metrics from the `/metrics/snapshot` API endpoint on Mesos masters and agents. 149 | A few of the available metrics that are collected include: 150 | 151 | Masters: 152 | * `master/cpus_total` and `master/cpus_used` 153 | * `master/disk_total` and `master/disk_used` 154 | * `master/mem_total` and `master/mem_used` 155 | * `master/slaves_active` 156 | * `master/tasks_running` 157 | * `registrar/state_store_ms/p90`, `registrar/state_store_ms/min`, `registrar/state_store_ms/max` 158 | * `system/load_1min`, `system/load_5min`, `system/load_15min` 159 | 160 | Agents: 161 | * `slave/tasks_running` and `slave/tasks_failed` 162 | * `slave/cpus_total` and `slave/cpus_used` 163 | * `slave/disk_total` and `slave/disk_used` 164 | * `slave/mem_total` and `slave/mem_used` 165 | * `slave/executors_running` 166 | * `system/load_1min`, `system/load_5min`, `system/load_15min` 167 | 168 | For a complete reference, please consult the [official Mesos documentation][mesos-monitoring]. 169 | 170 | #### Mesos monitoring statistics (executor/container metrics) 171 | This plugin returns most of the available metrics from the `/monitor/statistics` API endpoint on the Mesos agent, 172 | which includes metrics about running executors (containers) on a specific Mesos agent. The metrics available via this 173 | endpoint largely depend on the features enabled in Mesos. There are far too many metrics to list here, so instead we 174 | provide links to the code that describes the structure. 175 | 176 | *Note: the available metrics are defined in the mesos.proto protobuf, which has been compiled to Golang for the purposes 177 | of this plugin. More information is available in [GitHub issue #11][github-issue-11].* 178 | 179 | * The base [`ResourceStatistics` struct][resourcestatistics-struct] is provided for each running executor on the 180 | cluster. 181 | * If Mesos was built with the `--with-network-isolator` option and the `network/port_mapping` isolator is enabled, 182 | you'll also be able to collect [various network metrics][network-usage-info]. 183 | * If the necessary perf-related packages are installed, Mesos is configured to use the `cgroups/perf_event`, and 184 | values are provided to the `--perf_events` option on the Mesos agent, you'll also be able to collect per-container 185 | perf metrics as defined in the [`PerfStatistics` struct][perfstatistics-struct]. 186 | 187 | #### Metric tags 188 | 189 | Namespace | Tag | Description 190 | ----------------------------|----------------|------------ 191 | `/intel/mesos/**` | `source` | IP and port of the Mesos master/agent that this plugin is connecting to. Depending on the network configuration of a system, the value of this tag _could_ be different than the value of the built-in `plugin_running_on` tag in Snap. 192 | `/intel/mesos/master/*/**` | `framework_id` | The UUID that the Mesos master assigned to a given framework. 193 | `/intel/mesos/agent/*/*/**` | `framework_id` | The UUID that the Mesos master assigned to a given framework. Allows executors to be grouped/queried on a per-framework basis. 194 | `/intel/mesos/agent/*/*/**` | `executor_id` | The ID that a scheduler assigned to a specific executor (container) running on a Mesos agent. 195 | 196 | ### Examples 197 | There are examples of the Snap global configuration and various tasks located in the [examples](examples) directory. 198 | Specifically, these include: 199 | 200 | * Snap global configuration 201 | * Snap tasks (publish to file and publish to InfluxDB) 202 | * Grafana dashboard for visualizing cluster telemetry 203 | 204 | *Note: these scenarios will work with the Vagrant development environment included in this repo. For more info on how 205 | to get started with Vagrant, please see [CONTRIBUTING.md](CONTRIBUTING.md).* 206 | 207 | #### Publishing metrics to a file 208 | To start collecting Mesos metrics and publish them to a file, you'll need to perform the following steps. 209 | 210 | Set up the [Snap framework](https://github.com/intelsdi-x/snap/blob/master/README.md#getting-started), 211 | run `snapteld` (in this case with logging set to 1, trust disabled and global configuration saved in snap-config-example.json): 212 | 213 | ``` 214 | $ snapteld --plugin-trust 0 --log-level 1 --config examples/configs/snap-config-example.json \ 215 | > /tmp/snap.log 2>&1 & 216 | ``` 217 | 218 | Download and load [snap-plugin-publisher-file](https://github.com/intelsdi-x/snap-plugin-publisher-file), for example: 219 | ``` 220 | $ wget https://github.com/intelsdi-x/snap-plugin-publisher-file/releases/download/2/snap-plugin-publisher-file_linux_x86_64 221 | $ snaptel plugin load snap-plugin-publisher-file_linux_x86_64 222 | ``` 223 | 224 | Build or download mesos plugin according to the [instruction](#installation) and go to directory with plugin binary file. 225 | 226 | Load mesos collector plugin: 227 | ``` 228 | $ snaptel plugin load snap-plugin-collector-mesos 229 | ``` 230 | 231 | Get the available metrics for your system: 232 | 233 | ``` 234 | $ snaptel metric list 235 | ``` 236 | 237 | Create a new Snap task: 238 | 239 | ``` 240 | $ snaptel task create -t examples/tasks/mesos-all-file.json 241 | ``` 242 | 243 | Stop the task: 244 | 245 | ``` 246 | $ snaptel task stop 247 | ``` 248 | 249 | #### Visualizing cluster telemetry with Grafana and InfluxDB 250 | To help you get up and running quickly, this repo also includes a more extensive example of how to publish Mesos cluster 251 | metrics to InfluxDB (using [snap-plugin-publisher-influxdb](https://github.com/intelsdi-x/snap-plugin-publisher-influxdb)) and visualize this data with Grafana. We assume that you already have a working InfluxDB and 252 | Grafana installation, and that you have all the necessary Snap plugins and configuration loaded. 253 | 254 | *Note: you'll need to modify the values for the `host`, `user`, and `password` options in the example tasks.* 255 | 256 | On the Mesos master(s), run the following command: 257 | 258 | ``` 259 | $ snaptel task create -t examples/tasks/mesos-master-influxdb.json 260 | ``` 261 | 262 | On the Mesos agent(s), run the following command: 263 | 264 | ``` 265 | $ snaptel task create -t examples/tasks/mesos-agent-influxdb.json 266 | ``` 267 | 268 | Finally, load the example Grafana dashboard. The following commands assume that Grafana is running at 269 | http://grafana.example.com:3000 and using the default username of `admin`, and the default password of `admin`. 270 | 271 | ``` 272 | $ GRAFANA="http://grafana.example.com:3000" 273 | $ COOKIEJAR=$(mktemp) 274 | 275 | $ curl -sH 'Content-Type: application/json; charset=UTF-8' \ 276 | --data-binary '{"user": "admin", "email": "", "password": "admin"}' \ 277 | --cookie-jar "$COOKIEJAR" "${GRAFANA}/login" 278 | 279 | $ curl -sH 'Content-Type: application/json; charset=UTF-8' --cookie "$COOKIEJAR" \ 280 | -d@mesos.json "${GRAFANA}/api/dashboards/db" 281 | ``` 282 | 283 | You should now see some basic metrics about your Mesos cluster: 284 | 285 | ![grafana-dashboard](assets/grafana-dashboard.png) 286 | 287 | ### Known Issues and Caveats 288 | * Currently, the address of the Mesos master and/or agent is provided via Snap's global configuration file. When this 289 | plugin is loaded (using `snaptel`) it plugin will query a Mesos master and/or agent for its available metrics and use 290 | the resulting JSON structure to build Snap's metrics catalog. However, a configuration change on the Mesos master or 291 | agent could alter the metrics reported by Mesos (e.g., enabling the network isolator). Therefore, if you modify the 292 | configuration of a Mesos master or agent and expect more (or fewer) metrics to be collected, you should reload this 293 | Snap plugin at the same time. 294 | * Due to a bug in Mesos, the parsing logic for the `perf` command was incorrect on certain platforms and kernels. When 295 | the `cgroups/perf_event` isolator was enabled on an agent, the `perf` object would appear in the JSON returned by the 296 | agent's `/monitor/statistics` endpoint, but it would contain no data. This issue was resolved in Mesos 0.29.0, and was 297 | backported to Mesos 0.28.2, 0.27.3, and 0.26.2. For more information, see [MESOS-4705][mesos-4705-jira]. 298 | * There is an ongoing effort to rename the Mesos "slave" service to "agent". As of Mesos 0.28.x, this work is still 299 | in progress. This plugin uses the newer "agent" terminology, but some metrics returned by Mesos may still use the 300 | older "slave" term. For more information, see [MESOS-1478][mesos-1478-jira]. 301 | 302 | ### Roadmap 303 | For version 2, we intend to support additional deployment options as documented in [GitHub issue #14][github-issue-14]. 304 | Otherwise, there isn't a formal roadmap for this plugin, but it's in active development. If you have a feature request, 305 | please [open a new issue on GitHub][github-new-issue] or [submit a pull request][github-new-pull-request]. 306 | 307 | ## Community Support 308 | This repository is one of many plugins in Snap, a powerful telemetry framework. To reach out to other users in the 309 | community, check out the full project at . 310 | 311 | ## Contributing 312 | We love contributions! 313 | 314 | There's more than one way to give back, from examples to blog posts to code updates. See our recommended process in 315 | [CONTRIBUTING.md](CONTRIBUTING.md). 316 | 317 | ## License 318 | [Snap][snap-github], along with this plugin, is open source software released 319 | under the [Apache Software License, version 2.0](LICENSE). 320 | 321 | ## Acknowledgements 322 | * Authors: [Marcin Krolik][marcin-github], [Roger Ignazio][roger-github] 323 | 324 | 325 | [github-issue-11]: https://github.com/intelsdi-x/snap-plugin-collector-mesos/issues/11 326 | [github-issue-14]: https://github.com/intelsdi-x/snap-plugin-collector-mesos/issues/14 327 | [github-new-issue]: https://github.com/intelsdi-x/snap-plugin-collector-mesos/issues/new 328 | [github-new-pull-request]: https://github.com/intelsdi-x/snap-plugin-collector-mesos/pulls 329 | [github-rfc]: https://github.com/intelsdi-x/snap-plugin-collector-mesos/issues?utf8=✓&q=is%3Aissue+is%3Aall+label%3ARFC+ 330 | [golang-dl]: https://golang.org/dl/ 331 | [marcin-github]: https://github.com/marcin-krolik 332 | [mesos-1478-jira]: https://issues.apache.org/jira/browse/MESOS-1478 333 | [mesos-4705-jira]: https://issues.apache.org/jira/browse/MESOS-4705 334 | [mesos-home]: http://mesos.apache.org 335 | [mesos-getting-started]: http://mesos.apache.org/gettingstarted/ 336 | [mesos-monitoring]: http://mesos.apache.org/documentation/latest/monitoring/ 337 | [mesosphere-downloads]: https://mesosphere.com/downloads/ 338 | [network-usage-info]: https://github.com/intelsdi-x/snap-plugin-collector-mesos/blob/master/mesos/mesos_pb2/mesos_pb2.go#L3141-L3149 339 | [perfstatistics-struct]: https://github.com/intelsdi-x/snap-plugin-collector-mesos/blob/master/mesos/mesos_pb2/mesos_pb2.go#L3541-L3610 340 | [releases]: https://github.com/intelsdi-x/snap-plugin-collector-mesos/releases 341 | [resourcestatistics-struct]: https://github.com/intelsdi-x/snap-plugin-collector-mesos/blob/master/mesos/mesos_pb2/mesos_pb2.go#L3086-L3165 342 | [roger-github]: https://github.com/rji 343 | [snap-getting-started]: https://github.com/intelsdi-x/snap/blob/master/README.md#getting-started 344 | [snap-github]: https://github.com/intelsdi-x/snap 345 | [snap-influxdb]: https://github.com/intelsdi-x/snap-plugin-publisher-influxdb 346 | -------------------------------------------------------------------------------- /VAGRANT.md: -------------------------------------------------------------------------------- 1 | ## Vagrant development environment 2 | 3 | Included in this repo is a Vagrant development environment that provisions a single-node Mesos cluster for developing 4 | and testing this plugin. To get started, you'll need to have: 5 | 6 | * VirtualBox: 7 | * Vagrant: 8 | 9 | Next, assuming that your current working directory is the root of this Git repository, simply run the following command: 10 | 11 | ``` 12 | $ vagrant up 13 | ``` 14 | 15 | The provisioning script will install Mesos, ZooKeeper, Go, and Snap. You can then connect to Mesos at the following 16 | URLs: 17 | 18 | * Mesos master: 19 | * Mesos agent: 20 | * InfluxDB: 21 | * Grafana: 22 | 23 | In order to launch a task on the cluster that you can then observe with Snap, try running the following command in a 24 | separate terminal window / SSH session: 25 | 26 | ``` 27 | $ mesos execute --master="$(mesos resolve zk://10.180.10.180:2181/mesos)" \ 28 | --name="PythonHttpServer" --command="python -m SimpleHTTPServer 9000" 29 | ``` 30 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # This Vagrant environment provisions a single virtual machine that can be 2 | # used to develop and test the Mesos plugin for the Snap telemetry framework. 3 | # 4 | # When provisioning is complete, Mesos will be listening on the following 5 | # addresses and ports: 6 | # 7 | # * Master: http://10.180.10.180:5050 8 | # * Agent: http://10.180.10.180:5051 9 | # 10 | # Set values for the three variables below to install specific versions of 11 | # Mesos, Go, and Snap. Alternately, you may set one or more of these values 12 | # to 'latest' to unpin it. 13 | # 14 | MESOS_RELEASE = '0.28.2-2.0.27.ubuntu1404' 15 | MARATHON_RELEASE = '1.1.1-1.0.472.ubuntu1404' 16 | GOLANG_RELEASE = '1.6.2' 17 | SNAP_RELEASE = 'v0.14.0-beta' 18 | IP_ADDRESS = '10.180.10.180' 19 | 20 | Vagrant.configure(2) do |config| 21 | config.vm.box = 'ubuntu/trusty64' 22 | config.vm.network 'private_network', ip: IP_ADDRESS 23 | 24 | # Forward ports from the guest to the host for ease of access. All services 25 | # should still be available at IP_ADDRESS:port as well. 26 | [3000, 5050, 5051, 8080, 8083].each do |port| 27 | config.vm.network 'forwarded_port', guest: port, host: port 28 | end 29 | 30 | config.vm.provider 'virtualbox' do |vb| 31 | vb.name = 'vagrant-snap-mesos' 32 | vb.cpus = 2 33 | vb.memory = 4096 34 | end 35 | 36 | config.vm.provision 'shell' do |sh| 37 | sh.path = 'scripts/provision-vagrant.sh' 38 | args = [ '--mesos_release', MESOS_RELEASE ] 39 | args += [ '--marathon_release', MARATHON_RELEASE ] 40 | args += [ '--golang_release', GOLANG_RELEASE ] 41 | args += [ '--snap_release', SNAP_RELEASE ] 42 | args += [ '--ip_address', IP_ADDRESS ] 43 | sh.args = args 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /assets/deploy-distributed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intelsdi-x/snap-plugin-collector-mesos/f544ec5ed003340e3b7b5a037fbe1c7de201c4fc/assets/deploy-distributed.png -------------------------------------------------------------------------------- /assets/grafana-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intelsdi-x/snap-plugin-collector-mesos/f544ec5ed003340e3b7b5a037fbe1c7de201c4fc/assets/grafana-dashboard.png -------------------------------------------------------------------------------- /examples/configs/snap-config-example.json: -------------------------------------------------------------------------------- 1 | { 2 | "control": { 3 | "plugins": { 4 | "collector": { 5 | "mesos": { 6 | "all": { 7 | "master": "10.180.10.180:5050", 8 | "agent": "10.180.10.180:5051" 9 | } 10 | } 11 | } 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /examples/grafana/mesos.json: -------------------------------------------------------------------------------- 1 | { 2 | "dashboard": { 3 | "id": null, 4 | "title": "Mesos Cluster Telemetry", 5 | "originalTitle": "Mesos Cluster Telemetry", 6 | "tags": [], 7 | "style": "dark", 8 | "timezone": "browser", 9 | "editable": true, 10 | "hideControls": false, 11 | "sharedCrosshair": false, 12 | "rows": [ 13 | { 14 | "collapse": false, 15 | "editable": true, 16 | "height": "250px", 17 | "panels": [ 18 | { 19 | "aliasColors": {}, 20 | "bars": false, 21 | "datasource": "influx", 22 | "decimals": 0, 23 | "editable": true, 24 | "error": false, 25 | "fill": 1, 26 | "grid": { 27 | "threshold1": null, 28 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 29 | "threshold2": null, 30 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 31 | }, 32 | "id": 2, 33 | "isNew": true, 34 | "legend": { 35 | "avg": false, 36 | "current": false, 37 | "max": false, 38 | "min": false, 39 | "show": true, 40 | "total": false, 41 | "values": false 42 | }, 43 | "lines": true, 44 | "linewidth": 2, 45 | "links": [], 46 | "nullPointMode": "connected", 47 | "percentage": false, 48 | "pointradius": 5, 49 | "points": false, 50 | "renderer": "flot", 51 | "seriesOverrides": [], 52 | "span": 4, 53 | "stack": false, 54 | "steppedLine": false, 55 | "targets": [ 56 | { 57 | "dsType": "influxdb", 58 | "groupBy": [], 59 | "measurement": "intel/mesos/master/master/slaves_connected", 60 | "policy": "default", 61 | "refId": "A", 62 | "resultFormat": "time_series", 63 | "select": [ 64 | [ 65 | { 66 | "params": [ 67 | "value" 68 | ], 69 | "type": "field" 70 | } 71 | ] 72 | ], 73 | "tags": [] 74 | } 75 | ], 76 | "timeFrom": null, 77 | "timeShift": null, 78 | "title": "Agents", 79 | "tooltip": { 80 | "msResolution": false, 81 | "shared": true, 82 | "value_type": "cumulative" 83 | }, 84 | "type": "graph", 85 | "xaxis": { 86 | "show": true 87 | }, 88 | "yaxes": [ 89 | { 90 | "format": "short", 91 | "label": null, 92 | "logBase": 1, 93 | "max": null, 94 | "min": 0, 95 | "show": true 96 | }, 97 | { 98 | "format": "short", 99 | "label": null, 100 | "logBase": 1, 101 | "max": null, 102 | "min": null, 103 | "show": true 104 | } 105 | ] 106 | }, 107 | { 108 | "aliasColors": { 109 | "intel/mesos/master/master/tasks_failed.count": "#890F02", 110 | "intel/mesos/master/master/tasks_failed.derivative": "#BF1B00", 111 | "intel/mesos/master/master/tasks_lost.count": "#C15C17", 112 | "intel/mesos/master/master/tasks_staging.count": "#CCA300" 113 | }, 114 | "bars": false, 115 | "datasource": "influx", 116 | "editable": true, 117 | "error": false, 118 | "fill": 1, 119 | "grid": { 120 | "threshold1": null, 121 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 122 | "threshold2": null, 123 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 124 | }, 125 | "id": 3, 126 | "isNew": true, 127 | "legend": { 128 | "avg": false, 129 | "current": false, 130 | "max": false, 131 | "min": false, 132 | "show": true, 133 | "total": false, 134 | "values": false 135 | }, 136 | "lines": true, 137 | "linewidth": 2, 138 | "links": [], 139 | "nullPointMode": "connected", 140 | "percentage": false, 141 | "pointradius": 5, 142 | "points": false, 143 | "renderer": "flot", 144 | "seriesOverrides": [], 145 | "span": 4, 146 | "stack": false, 147 | "steppedLine": false, 148 | "targets": [ 149 | { 150 | "dsType": "influxdb", 151 | "groupBy": [], 152 | "measurement": "intel/mesos/master/master/tasks_running", 153 | "policy": "default", 154 | "refId": "A", 155 | "resultFormat": "time_series", 156 | "select": [ 157 | [ 158 | { 159 | "params": [ 160 | "value" 161 | ], 162 | "type": "field" 163 | } 164 | ] 165 | ], 166 | "tags": [] 167 | }, 168 | { 169 | "dsType": "influxdb", 170 | "groupBy": [], 171 | "measurement": "intel/mesos/master/master/tasks_staging", 172 | "policy": "default", 173 | "refId": "C", 174 | "resultFormat": "time_series", 175 | "select": [ 176 | [ 177 | { 178 | "params": [ 179 | "value" 180 | ], 181 | "type": "field" 182 | } 183 | ] 184 | ], 185 | "tags": [] 186 | }, 187 | { 188 | "dsType": "influxdb", 189 | "groupBy": [], 190 | "measurement": "intel/mesos/master/master/tasks_failed", 191 | "policy": "default", 192 | "refId": "B", 193 | "resultFormat": "time_series", 194 | "select": [ 195 | [ 196 | { 197 | "params": [ 198 | "value" 199 | ], 200 | "type": "field" 201 | }, 202 | { 203 | "params": [ 204 | "10s" 205 | ], 206 | "type": "derivative" 207 | } 208 | ] 209 | ], 210 | "tags": [] 211 | }, 212 | { 213 | "dsType": "influxdb", 214 | "groupBy": [], 215 | "measurement": "intel/mesos/master/master/tasks_lost", 216 | "policy": "default", 217 | "refId": "D", 218 | "resultFormat": "time_series", 219 | "select": [ 220 | [ 221 | { 222 | "params": [ 223 | "value" 224 | ], 225 | "type": "field" 226 | }, 227 | { 228 | "params": [ 229 | "10s" 230 | ], 231 | "type": "derivative" 232 | } 233 | ] 234 | ], 235 | "tags": [] 236 | } 237 | ], 238 | "timeFrom": null, 239 | "timeShift": null, 240 | "title": "Tasks", 241 | "tooltip": { 242 | "msResolution": false, 243 | "shared": true, 244 | "value_type": "cumulative" 245 | }, 246 | "type": "graph", 247 | "xaxis": { 248 | "show": true 249 | }, 250 | "yaxes": [ 251 | { 252 | "format": "short", 253 | "label": null, 254 | "logBase": 1, 255 | "max": null, 256 | "min": 0, 257 | "show": true 258 | }, 259 | { 260 | "format": "short", 261 | "label": null, 262 | "logBase": 1, 263 | "max": null, 264 | "min": null, 265 | "show": true 266 | } 267 | ] 268 | }, 269 | { 270 | "cacheTimeout": null, 271 | "colorBackground": false, 272 | "colorValue": false, 273 | "colors": [ 274 | "rgba(245, 54, 54, 0.9)", 275 | "rgba(237, 129, 40, 0.89)", 276 | "rgba(50, 172, 45, 0.97)" 277 | ], 278 | "datasource": "influx", 279 | "editable": true, 280 | "error": false, 281 | "format": "none", 282 | "gauge": { 283 | "maxValue": 100, 284 | "minValue": 0, 285 | "show": false, 286 | "thresholdLabels": false, 287 | "thresholdMarkers": true 288 | }, 289 | "id": 4, 290 | "interval": null, 291 | "isNew": true, 292 | "links": [], 293 | "maxDataPoints": 100, 294 | "nullPointMode": "connected", 295 | "nullText": null, 296 | "postfix": "", 297 | "postfixFontSize": "50%", 298 | "prefix": "", 299 | "prefixFontSize": "50%", 300 | "span": 4, 301 | "sparkline": { 302 | "fillColor": "rgba(31, 118, 189, 0.18)", 303 | "full": false, 304 | "lineColor": "rgb(31, 120, 193)", 305 | "show": false 306 | }, 307 | "targets": [ 308 | { 309 | "dsType": "influxdb", 310 | "groupBy": [], 311 | "measurement": "intel/mesos/master/master/frameworks_connected", 312 | "policy": "default", 313 | "refId": "A", 314 | "resultFormat": "time_series", 315 | "select": [ 316 | [ 317 | { 318 | "params": [ 319 | "value" 320 | ], 321 | "type": "field" 322 | }, 323 | { 324 | "params": [], 325 | "type": "last" 326 | } 327 | ] 328 | ], 329 | "tags": [] 330 | } 331 | ], 332 | "thresholds": "", 333 | "title": "Registered Frameworks", 334 | "type": "singlestat", 335 | "valueFontSize": "80%", 336 | "valueMaps": [ 337 | { 338 | "op": "=", 339 | "text": "N/A", 340 | "value": "null" 341 | } 342 | ], 343 | "valueName": "avg" 344 | } 345 | ], 346 | "showTitle": true, 347 | "title": "Cluster Health" 348 | }, 349 | { 350 | "collapse": false, 351 | "editable": true, 352 | "height": "250px", 353 | "panels": [ 354 | { 355 | "aliasColors": { 356 | "intel/mesos/master/master/cpus_total": "#EAB839", 357 | "intel/mesos/master/master/cpus_used": "#7EB26D" 358 | }, 359 | "bars": false, 360 | "datasource": "influx", 361 | "editable": true, 362 | "error": false, 363 | "fill": 1, 364 | "grid": { 365 | "threshold1": null, 366 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 367 | "threshold2": null, 368 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 369 | }, 370 | "id": 5, 371 | "isNew": true, 372 | "legend": { 373 | "avg": false, 374 | "current": false, 375 | "max": false, 376 | "min": false, 377 | "show": true, 378 | "total": false, 379 | "values": false 380 | }, 381 | "lines": true, 382 | "linewidth": 2, 383 | "links": [], 384 | "nullPointMode": "connected", 385 | "percentage": false, 386 | "pointradius": 5, 387 | "points": false, 388 | "renderer": "flot", 389 | "seriesOverrides": [], 390 | "span": 4, 391 | "stack": false, 392 | "steppedLine": false, 393 | "targets": [ 394 | { 395 | "dsType": "influxdb", 396 | "groupBy": [], 397 | "measurement": "intel/mesos/master/master/cpus_total", 398 | "policy": "default", 399 | "refId": "A", 400 | "resultFormat": "time_series", 401 | "select": [ 402 | [ 403 | { 404 | "params": [ 405 | "value" 406 | ], 407 | "type": "field" 408 | } 409 | ] 410 | ], 411 | "tags": [] 412 | }, 413 | { 414 | "dsType": "influxdb", 415 | "groupBy": [], 416 | "measurement": "intel/mesos/master/master/cpus_used", 417 | "policy": "default", 418 | "refId": "B", 419 | "resultFormat": "time_series", 420 | "select": [ 421 | [ 422 | { 423 | "params": [ 424 | "value" 425 | ], 426 | "type": "field" 427 | } 428 | ] 429 | ], 430 | "tags": [] 431 | } 432 | ], 433 | "timeFrom": null, 434 | "timeShift": null, 435 | "title": "Cluster Utilization (CPUs)", 436 | "tooltip": { 437 | "msResolution": false, 438 | "shared": true, 439 | "value_type": "cumulative" 440 | }, 441 | "type": "graph", 442 | "xaxis": { 443 | "show": true 444 | }, 445 | "yaxes": [ 446 | { 447 | "format": "short", 448 | "label": null, 449 | "logBase": 1, 450 | "max": null, 451 | "min": null, 452 | "show": true 453 | }, 454 | { 455 | "format": "short", 456 | "label": null, 457 | "logBase": 1, 458 | "max": null, 459 | "min": null, 460 | "show": true 461 | } 462 | ] 463 | }, 464 | { 465 | "aliasColors": { 466 | "intel/mesos/master/master/mem_total": "#EAB839", 467 | "intel/mesos/master/master/mem_used": "#7EB26D" 468 | }, 469 | "bars": false, 470 | "datasource": "influx", 471 | "editable": true, 472 | "error": false, 473 | "fill": 1, 474 | "grid": { 475 | "threshold1": null, 476 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 477 | "threshold2": null, 478 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 479 | }, 480 | "id": 6, 481 | "isNew": true, 482 | "legend": { 483 | "avg": false, 484 | "current": false, 485 | "max": false, 486 | "min": false, 487 | "show": true, 488 | "total": false, 489 | "values": false 490 | }, 491 | "lines": true, 492 | "linewidth": 2, 493 | "links": [], 494 | "nullPointMode": "connected", 495 | "percentage": false, 496 | "pointradius": 5, 497 | "points": false, 498 | "renderer": "flot", 499 | "seriesOverrides": [], 500 | "span": 4, 501 | "stack": false, 502 | "steppedLine": false, 503 | "targets": [ 504 | { 505 | "dsType": "influxdb", 506 | "groupBy": [], 507 | "measurement": "intel/mesos/master/master/mem_total", 508 | "policy": "default", 509 | "refId": "A", 510 | "resultFormat": "time_series", 511 | "select": [ 512 | [ 513 | { 514 | "params": [ 515 | "value" 516 | ], 517 | "type": "field" 518 | } 519 | ] 520 | ], 521 | "tags": [] 522 | }, 523 | { 524 | "dsType": "influxdb", 525 | "groupBy": [], 526 | "measurement": "intel/mesos/master/master/mem_used", 527 | "policy": "default", 528 | "refId": "B", 529 | "resultFormat": "time_series", 530 | "select": [ 531 | [ 532 | { 533 | "params": [ 534 | "value" 535 | ], 536 | "type": "field" 537 | } 538 | ] 539 | ], 540 | "tags": [] 541 | } 542 | ], 543 | "timeFrom": null, 544 | "timeShift": null, 545 | "title": "Cluster Utilization (Memory)", 546 | "tooltip": { 547 | "msResolution": false, 548 | "shared": true, 549 | "value_type": "cumulative" 550 | }, 551 | "type": "graph", 552 | "xaxis": { 553 | "show": true 554 | }, 555 | "yaxes": [ 556 | { 557 | "format": "mbytes", 558 | "label": null, 559 | "logBase": 1, 560 | "max": null, 561 | "min": null, 562 | "show": true 563 | }, 564 | { 565 | "format": "short", 566 | "label": null, 567 | "logBase": 1, 568 | "max": null, 569 | "min": null, 570 | "show": true 571 | } 572 | ] 573 | }, 574 | { 575 | "aliasColors": { 576 | "intel/mesos/master/master/disk_total": "#EAB839", 577 | "intel/mesos/master/master/disk_used": "#7EB26D" 578 | }, 579 | "bars": false, 580 | "datasource": "influx", 581 | "editable": true, 582 | "error": false, 583 | "fill": 1, 584 | "grid": { 585 | "threshold1": null, 586 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 587 | "threshold2": null, 588 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 589 | }, 590 | "id": 7, 591 | "isNew": true, 592 | "legend": { 593 | "avg": false, 594 | "current": false, 595 | "max": false, 596 | "min": false, 597 | "show": true, 598 | "total": false, 599 | "values": false 600 | }, 601 | "lines": true, 602 | "linewidth": 2, 603 | "links": [], 604 | "nullPointMode": "connected", 605 | "percentage": false, 606 | "pointradius": 5, 607 | "points": false, 608 | "renderer": "flot", 609 | "seriesOverrides": [], 610 | "span": 4, 611 | "stack": false, 612 | "steppedLine": false, 613 | "targets": [ 614 | { 615 | "dsType": "influxdb", 616 | "groupBy": [], 617 | "measurement": "intel/mesos/master/master/disk_total", 618 | "policy": "default", 619 | "refId": "A", 620 | "resultFormat": "time_series", 621 | "select": [ 622 | [ 623 | { 624 | "params": [ 625 | "value" 626 | ], 627 | "type": "field" 628 | } 629 | ] 630 | ], 631 | "tags": [] 632 | }, 633 | { 634 | "dsType": "influxdb", 635 | "groupBy": [], 636 | "measurement": "intel/mesos/master/master/disk_used", 637 | "policy": "default", 638 | "refId": "B", 639 | "resultFormat": "time_series", 640 | "select": [ 641 | [ 642 | { 643 | "params": [ 644 | "value" 645 | ], 646 | "type": "field" 647 | } 648 | ] 649 | ], 650 | "tags": [] 651 | } 652 | ], 653 | "timeFrom": null, 654 | "timeShift": null, 655 | "title": "Cluster Utilization (Disk)", 656 | "tooltip": { 657 | "msResolution": false, 658 | "shared": true, 659 | "value_type": "cumulative" 660 | }, 661 | "type": "graph", 662 | "xaxis": { 663 | "show": true 664 | }, 665 | "yaxes": [ 666 | { 667 | "format": "mbytes", 668 | "label": null, 669 | "logBase": 1, 670 | "max": null, 671 | "min": null, 672 | "show": true 673 | }, 674 | { 675 | "format": "short", 676 | "label": null, 677 | "logBase": 1, 678 | "max": null, 679 | "min": null, 680 | "show": true 681 | } 682 | ] 683 | } 684 | ], 685 | "showTitle": true, 686 | "title": "Cluster Utilization (Overall)" 687 | }, 688 | { 689 | "collapse": false, 690 | "editable": true, 691 | "height": "250px", 692 | "panels": [ 693 | { 694 | "aliasColors": {}, 695 | "bars": false, 696 | "datasource": "influx", 697 | "editable": true, 698 | "error": false, 699 | "fill": 1, 700 | "grid": { 701 | "threshold1": null, 702 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 703 | "threshold2": null, 704 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 705 | }, 706 | "id": 8, 707 | "isNew": true, 708 | "legend": { 709 | "avg": false, 710 | "current": false, 711 | "max": false, 712 | "min": false, 713 | "show": true, 714 | "total": false, 715 | "values": false 716 | }, 717 | "lines": true, 718 | "linewidth": 2, 719 | "links": [], 720 | "nullPointMode": "connected", 721 | "percentage": false, 722 | "pointradius": 5, 723 | "points": false, 724 | "renderer": "flot", 725 | "seriesOverrides": [], 726 | "span": 4, 727 | "stack": false, 728 | "steppedLine": false, 729 | "targets": [ 730 | { 731 | "dsType": "influxdb", 732 | "groupBy": [ 733 | { 734 | "params": [ 735 | "framework_id" 736 | ], 737 | "type": "tag" 738 | } 739 | ], 740 | "measurement": "intel/mesos/master/used_resources/cpus", 741 | "policy": "default", 742 | "refId": "B", 743 | "resultFormat": "time_series", 744 | "select": [ 745 | [ 746 | { 747 | "params": [ 748 | "value" 749 | ], 750 | "type": "field" 751 | } 752 | ] 753 | ], 754 | "tags": [] 755 | } 756 | ], 757 | "timeFrom": null, 758 | "timeShift": null, 759 | "title": "CPU Utilization By Framework", 760 | "tooltip": { 761 | "msResolution": false, 762 | "shared": true, 763 | "value_type": "cumulative" 764 | }, 765 | "type": "graph", 766 | "xaxis": { 767 | "show": true 768 | }, 769 | "yaxes": [ 770 | { 771 | "format": "short", 772 | "label": null, 773 | "logBase": 1, 774 | "max": null, 775 | "min": 0, 776 | "show": true 777 | }, 778 | { 779 | "format": "short", 780 | "label": null, 781 | "logBase": 1, 782 | "max": null, 783 | "min": null, 784 | "show": true 785 | } 786 | ] 787 | }, 788 | { 789 | "aliasColors": {}, 790 | "bars": false, 791 | "datasource": "influx", 792 | "editable": true, 793 | "error": false, 794 | "fill": 1, 795 | "grid": { 796 | "threshold1": null, 797 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 798 | "threshold2": null, 799 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 800 | }, 801 | "id": 9, 802 | "isNew": true, 803 | "legend": { 804 | "avg": false, 805 | "current": false, 806 | "max": false, 807 | "min": false, 808 | "show": true, 809 | "total": false, 810 | "values": false 811 | }, 812 | "lines": true, 813 | "linewidth": 2, 814 | "links": [], 815 | "nullPointMode": "connected", 816 | "percentage": false, 817 | "pointradius": 5, 818 | "points": false, 819 | "renderer": "flot", 820 | "seriesOverrides": [], 821 | "span": 4, 822 | "stack": false, 823 | "steppedLine": false, 824 | "targets": [ 825 | { 826 | "dsType": "influxdb", 827 | "groupBy": [ 828 | { 829 | "params": [ 830 | "framework_id" 831 | ], 832 | "type": "tag" 833 | } 834 | ], 835 | "measurement": "intel/mesos/master/used_resources/mem", 836 | "policy": "default", 837 | "refId": "B", 838 | "resultFormat": "time_series", 839 | "select": [ 840 | [ 841 | { 842 | "params": [ 843 | "value" 844 | ], 845 | "type": "field" 846 | } 847 | ] 848 | ], 849 | "tags": [] 850 | } 851 | ], 852 | "timeFrom": null, 853 | "timeShift": null, 854 | "title": "Memory Utilization By Framework", 855 | "tooltip": { 856 | "msResolution": false, 857 | "shared": true, 858 | "value_type": "cumulative" 859 | }, 860 | "type": "graph", 861 | "xaxis": { 862 | "show": true 863 | }, 864 | "yaxes": [ 865 | { 866 | "format": "mbytes", 867 | "label": null, 868 | "logBase": 1, 869 | "max": null, 870 | "min": 0, 871 | "show": true 872 | }, 873 | { 874 | "format": "short", 875 | "label": null, 876 | "logBase": 1, 877 | "max": null, 878 | "min": null, 879 | "show": true 880 | } 881 | ] 882 | }, 883 | { 884 | "aliasColors": {}, 885 | "bars": false, 886 | "datasource": "influx", 887 | "editable": true, 888 | "error": false, 889 | "fill": 1, 890 | "grid": { 891 | "threshold1": null, 892 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 893 | "threshold2": null, 894 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 895 | }, 896 | "id": 10, 897 | "isNew": true, 898 | "legend": { 899 | "avg": false, 900 | "current": false, 901 | "max": false, 902 | "min": false, 903 | "show": true, 904 | "total": false, 905 | "values": false 906 | }, 907 | "lines": true, 908 | "linewidth": 2, 909 | "links": [], 910 | "nullPointMode": "connected", 911 | "percentage": false, 912 | "pointradius": 5, 913 | "points": false, 914 | "renderer": "flot", 915 | "seriesOverrides": [], 916 | "span": 4, 917 | "stack": false, 918 | "steppedLine": false, 919 | "targets": [ 920 | { 921 | "dsType": "influxdb", 922 | "groupBy": [ 923 | { 924 | "params": [ 925 | "framework_id" 926 | ], 927 | "type": "tag" 928 | } 929 | ], 930 | "measurement": "intel/mesos/master/used_resources/disk", 931 | "policy": "default", 932 | "refId": "B", 933 | "resultFormat": "time_series", 934 | "select": [ 935 | [ 936 | { 937 | "params": [ 938 | "value" 939 | ], 940 | "type": "field" 941 | } 942 | ] 943 | ], 944 | "tags": [] 945 | } 946 | ], 947 | "timeFrom": null, 948 | "timeShift": null, 949 | "title": "Disk Utilization By Framework", 950 | "tooltip": { 951 | "msResolution": false, 952 | "shared": true, 953 | "value_type": "cumulative" 954 | }, 955 | "type": "graph", 956 | "xaxis": { 957 | "show": true 958 | }, 959 | "yaxes": [ 960 | { 961 | "format": "mbytes", 962 | "label": null, 963 | "logBase": 1, 964 | "max": null, 965 | "min": 0, 966 | "show": true 967 | }, 968 | { 969 | "format": "short", 970 | "label": null, 971 | "logBase": 1, 972 | "max": null, 973 | "min": null, 974 | "show": true 975 | } 976 | ] 977 | } 978 | ], 979 | "showTitle": true, 980 | "title": "Resource Utilization (By Framework)" 981 | }, 982 | { 983 | "collapse": false, 984 | "editable": true, 985 | "height": "250px", 986 | "panels": [ 987 | { 988 | "aliasColors": {}, 989 | "bars": false, 990 | "datasource": "influx", 991 | "editable": true, 992 | "error": false, 993 | "fill": 1, 994 | "grid": { 995 | "threshold1": null, 996 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 997 | "threshold2": null, 998 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 999 | }, 1000 | "id": 11, 1001 | "isNew": true, 1002 | "legend": { 1003 | "avg": false, 1004 | "current": false, 1005 | "max": false, 1006 | "min": false, 1007 | "show": true, 1008 | "total": false, 1009 | "values": false 1010 | }, 1011 | "lines": true, 1012 | "linewidth": 2, 1013 | "links": [], 1014 | "nullPointMode": "connected", 1015 | "percentage": false, 1016 | "pointradius": 5, 1017 | "points": false, 1018 | "renderer": "flot", 1019 | "seriesOverrides": [], 1020 | "span": 4, 1021 | "stack": false, 1022 | "steppedLine": false, 1023 | "targets": [ 1024 | { 1025 | "dsType": "influxdb", 1026 | "groupBy": [ 1027 | { 1028 | "params": [ 1029 | "executor_id" 1030 | ], 1031 | "type": "tag" 1032 | } 1033 | ], 1034 | "measurement": "intel/mesos/agent/cpus_user_time_secs", 1035 | "policy": "default", 1036 | "refId": "A", 1037 | "resultFormat": "time_series", 1038 | "select": [ 1039 | [ 1040 | { 1041 | "type": "field", 1042 | "params": [ 1043 | "value" 1044 | ] 1045 | }, 1046 | { 1047 | "type": "derivative", 1048 | "params": [ 1049 | "10s" 1050 | ] 1051 | } 1052 | ] 1053 | ], 1054 | "tags": [] 1055 | } 1056 | ], 1057 | "timeFrom": null, 1058 | "timeShift": null, 1059 | "title": "CPU Utilization (User Time) By Executor", 1060 | "tooltip": { 1061 | "msResolution": true, 1062 | "shared": true, 1063 | "value_type": "cumulative" 1064 | }, 1065 | "type": "graph", 1066 | "xaxis": { 1067 | "show": true 1068 | }, 1069 | "yaxes": [ 1070 | { 1071 | "format": "short", 1072 | "label": null, 1073 | "logBase": 1, 1074 | "max": null, 1075 | "min": null, 1076 | "show": true 1077 | }, 1078 | { 1079 | "format": "short", 1080 | "label": null, 1081 | "logBase": 1, 1082 | "max": null, 1083 | "min": null, 1084 | "show": true 1085 | } 1086 | ] 1087 | }, 1088 | { 1089 | "aliasColors": {}, 1090 | "bars": false, 1091 | "datasource": "influx", 1092 | "editable": true, 1093 | "error": false, 1094 | "fill": 1, 1095 | "grid": { 1096 | "threshold1": null, 1097 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 1098 | "threshold2": null, 1099 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 1100 | }, 1101 | "id": 12, 1102 | "isNew": true, 1103 | "legend": { 1104 | "avg": false, 1105 | "current": false, 1106 | "max": false, 1107 | "min": false, 1108 | "show": true, 1109 | "total": false, 1110 | "values": false 1111 | }, 1112 | "lines": true, 1113 | "linewidth": 2, 1114 | "links": [], 1115 | "nullPointMode": "connected", 1116 | "percentage": false, 1117 | "pointradius": 5, 1118 | "points": false, 1119 | "renderer": "flot", 1120 | "seriesOverrides": [], 1121 | "span": 4, 1122 | "stack": false, 1123 | "steppedLine": false, 1124 | "targets": [ 1125 | { 1126 | "dsType": "influxdb", 1127 | "groupBy": [ 1128 | { 1129 | "params": [ 1130 | "executor_id" 1131 | ], 1132 | "type": "tag" 1133 | } 1134 | ], 1135 | "measurement": "intel/mesos/agent/mem_total_bytes", 1136 | "policy": "default", 1137 | "refId": "A", 1138 | "resultFormat": "time_series", 1139 | "select": [ 1140 | [ 1141 | { 1142 | "params": [ 1143 | "value" 1144 | ], 1145 | "type": "field" 1146 | } 1147 | ] 1148 | ], 1149 | "tags": [] 1150 | } 1151 | ], 1152 | "timeFrom": null, 1153 | "timeShift": null, 1154 | "title": "Memory Utilization By Executor", 1155 | "tooltip": { 1156 | "msResolution": true, 1157 | "shared": true, 1158 | "value_type": "cumulative" 1159 | }, 1160 | "type": "graph", 1161 | "xaxis": { 1162 | "show": true 1163 | }, 1164 | "yaxes": [ 1165 | { 1166 | "format": "bytes", 1167 | "label": null, 1168 | "logBase": 1, 1169 | "max": null, 1170 | "min": null, 1171 | "show": true 1172 | }, 1173 | { 1174 | "format": "short", 1175 | "label": null, 1176 | "logBase": 1, 1177 | "max": null, 1178 | "min": null, 1179 | "show": true 1180 | } 1181 | ] 1182 | }, 1183 | { 1184 | "aliasColors": {}, 1185 | "bars": false, 1186 | "datasource": "influx", 1187 | "editable": true, 1188 | "error": false, 1189 | "fill": 1, 1190 | "grid": { 1191 | "threshold1": null, 1192 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 1193 | "threshold2": null, 1194 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 1195 | }, 1196 | "id": 13, 1197 | "isNew": true, 1198 | "legend": { 1199 | "avg": false, 1200 | "current": false, 1201 | "max": false, 1202 | "min": false, 1203 | "show": true, 1204 | "total": false, 1205 | "values": false 1206 | }, 1207 | "lines": true, 1208 | "linewidth": 2, 1209 | "links": [], 1210 | "nullPointMode": "connected", 1211 | "percentage": false, 1212 | "pointradius": 5, 1213 | "points": false, 1214 | "renderer": "flot", 1215 | "seriesOverrides": [], 1216 | "span": 4, 1217 | "stack": false, 1218 | "steppedLine": false, 1219 | "targets": [ 1220 | { 1221 | "dsType": "influxdb", 1222 | "groupBy": [ 1223 | { 1224 | "params": [ 1225 | "executor_id" 1226 | ], 1227 | "type": "tag" 1228 | } 1229 | ], 1230 | "measurement": "intel/mesos/agent/disk_used_bytes", 1231 | "policy": "default", 1232 | "refId": "A", 1233 | "resultFormat": "time_series", 1234 | "select": [ 1235 | [ 1236 | { 1237 | "params": [ 1238 | "value" 1239 | ], 1240 | "type": "field" 1241 | } 1242 | ] 1243 | ], 1244 | "tags": [] 1245 | } 1246 | ], 1247 | "timeFrom": null, 1248 | "timeShift": null, 1249 | "title": "Disk Utilization By Executor", 1250 | "tooltip": { 1251 | "msResolution": true, 1252 | "shared": true, 1253 | "value_type": "cumulative" 1254 | }, 1255 | "type": "graph", 1256 | "xaxis": { 1257 | "show": true 1258 | }, 1259 | "yaxes": [ 1260 | { 1261 | "format": "bytes", 1262 | "label": null, 1263 | "logBase": 1, 1264 | "max": null, 1265 | "min": null, 1266 | "show": true 1267 | }, 1268 | { 1269 | "format": "short", 1270 | "label": null, 1271 | "logBase": 1, 1272 | "max": null, 1273 | "min": null, 1274 | "show": true 1275 | } 1276 | ] 1277 | } 1278 | ], 1279 | "showTitle": true, 1280 | "title": "Resource Utilization (By Executor)" 1281 | } 1282 | ], 1283 | "time": { 1284 | "from": "now-15m", 1285 | "to": "now" 1286 | }, 1287 | "timepicker": { 1288 | "refresh_intervals": [ 1289 | "5s", 1290 | "10s", 1291 | "30s", 1292 | "1m", 1293 | "5m", 1294 | "15m", 1295 | "30m", 1296 | "1h", 1297 | "2h", 1298 | "1d" 1299 | ], 1300 | "time_options": [ 1301 | "5m", 1302 | "15m", 1303 | "1h", 1304 | "6h", 1305 | "12h", 1306 | "24h", 1307 | "2d", 1308 | "7d", 1309 | "30d" 1310 | ] 1311 | }, 1312 | "templating": { 1313 | "list": [] 1314 | }, 1315 | "annotations": { 1316 | "list": [] 1317 | }, 1318 | "refresh": "30s", 1319 | "schemaVersion": 12, 1320 | "version": 2, 1321 | "links": [] 1322 | } 1323 | } 1324 | -------------------------------------------------------------------------------- /examples/tasks/mesos-agent-influxdb.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "schedule": { 4 | "type": "simple", 5 | "interval": "10s" 6 | }, 7 | "workflow": { 8 | "collect": { 9 | "metrics": { 10 | "/intel/mesos/agent/*/*/cpus_system_time_secs": {}, 11 | "/intel/mesos/agent/*/*/cpus_user_time_secs": {}, 12 | "/intel/mesos/agent/*/*/disk_limit_bytes": {}, 13 | "/intel/mesos/agent/*/*/disk_used_bytes": {}, 14 | "/intel/mesos/agent/*/*/mem_limit_bytes": {}, 15 | "/intel/mesos/agent/*/*/mem_total_bytes": {}, 16 | "/intel/mesos/agent/slave/cpus_total": {}, 17 | "/intel/mesos/agent/slave/cpus_used": {}, 18 | "/intel/mesos/agent/slave/disk_total": {}, 19 | "/intel/mesos/agent/slave/disk_used": {}, 20 | "/intel/mesos/agent/slave/executors_registering": {}, 21 | "/intel/mesos/agent/slave/executors_running": {}, 22 | "/intel/mesos/agent/slave/executors_terminated": {}, 23 | "/intel/mesos/agent/slave/executors_terminating": {}, 24 | "/intel/mesos/agent/slave/frameworks_active": {}, 25 | "/intel/mesos/agent/slave/mem_total": {}, 26 | "/intel/mesos/agent/slave/mem_used": {}, 27 | "/intel/mesos/agent/slave/tasks_failed": {}, 28 | "/intel/mesos/agent/slave/tasks_finished": {}, 29 | "/intel/mesos/agent/slave/tasks_killed": {}, 30 | "/intel/mesos/agent/slave/tasks_killing": {}, 31 | "/intel/mesos/agent/slave/tasks_lost": {}, 32 | "/intel/mesos/agent/slave/tasks_running": {}, 33 | "/intel/mesos/agent/slave/tasks_staging": {}, 34 | "/intel/mesos/agent/slave/tasks_starting": {}, 35 | "/intel/mesos/agent/system/cpus_total": {}, 36 | "/intel/mesos/agent/system/load_15min": {}, 37 | "/intel/mesos/agent/system/load_1min": {}, 38 | "/intel/mesos/agent/system/load_5min": {}, 39 | "/intel/mesos/agent/system/mem_free_bytes": {}, 40 | "/intel/mesos/agent/system/mem_total_bytes": {} 41 | 42 | }, 43 | "config": {}, 44 | "process": null, 45 | "publish": [ 46 | { 47 | "plugin_name": "influx", 48 | "config": { 49 | "host": "10.180.10.180", 50 | "port": 8086, 51 | "database": "snap", 52 | "user": "admin", 53 | "password": "admin" 54 | } 55 | } 56 | ] 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /examples/tasks/mesos-all-file.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "schedule": { 4 | "type": "simple", 5 | "interval": "15s" 6 | }, 7 | "workflow": { 8 | "collect": { 9 | "metrics": { 10 | "/intel/mesos/master/registrar/state_store_ms/p99": {}, 11 | "/intel/mesos/master/master/slaves_connected": {}, 12 | "/intel/mesos/master/master/tasks_running": {}, 13 | "/intel/mesos/master/system/load_1min": {}, 14 | "/intel/mesos/master/system/load_5min": {}, 15 | "/intel/mesos/master/system/load_15min": {}, 16 | "/intel/mesos/master/*/resources/cpus": {}, 17 | "/intel/mesos/agent/slave/cpus_percent": {}, 18 | "/intel/mesos/agent/slave/mem_percent": {}, 19 | "/intel/mesos/agent/slave/disk_percent": {}, 20 | "/intel/mesos/agent/slave/tasks_running": {}, 21 | "/intel/mesos/agent/system/load_1min": {}, 22 | "/intel/mesos/agent/system/load_5min": {}, 23 | "/intel/mesos/agent/system/load_15min": {}, 24 | "/intel/mesos/agent/*/*/cpus_system_time_secs": {}, 25 | "/intel/mesos/agent/*/*/cpus_user_time_secs": {}, 26 | "/intel/mesos/agent/*/*/mem_total_bytes": {} 27 | }, 28 | "publish": [ 29 | { 30 | "plugin_name": "file", 31 | "config": { 32 | "file": "/tmp/snap-published-metrics_mesos-all" 33 | } 34 | } 35 | ] 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /examples/tasks/mesos-master-influxdb.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "schedule": { 4 | "type": "simple", 5 | "interval": "10s" 6 | }, 7 | "workflow": { 8 | "collect": { 9 | "metrics": { 10 | "/intel/mesos/master/*/offered_resources/cpus": {}, 11 | "/intel/mesos/master/*/offered_resources/disk": {}, 12 | "/intel/mesos/master/*/offered_resources/mem": {}, 13 | "/intel/mesos/master/*/resources/cpus": {}, 14 | "/intel/mesos/master/*/resources/disk": {}, 15 | "/intel/mesos/master/*/resources/mem": {}, 16 | "/intel/mesos/master/*/used_resources/cpus": {}, 17 | "/intel/mesos/master/*/used_resources/disk": {}, 18 | "/intel/mesos/master/*/used_resources/mem": {}, 19 | "/intel/mesos/master/master/cpus_total": {}, 20 | "/intel/mesos/master/master/cpus_used": {}, 21 | "/intel/mesos/master/master/disk_total": {}, 22 | "/intel/mesos/master/master/disk_used": {}, 23 | "/intel/mesos/master/master/frameworks_connected": {}, 24 | "/intel/mesos/master/master/mem_total": {}, 25 | "/intel/mesos/master/master/mem_used": {}, 26 | "/intel/mesos/master/master/slaves_connected": {}, 27 | "/intel/mesos/master/master/tasks_error": {}, 28 | "/intel/mesos/master/master/tasks_failed": {}, 29 | "/intel/mesos/master/master/tasks_finished": {}, 30 | "/intel/mesos/master/master/tasks_killed": {}, 31 | "/intel/mesos/master/master/tasks_killing": {}, 32 | "/intel/mesos/master/master/tasks_lost": {}, 33 | "/intel/mesos/master/master/tasks_running": {}, 34 | "/intel/mesos/master/master/tasks_staging": {}, 35 | "/intel/mesos/master/master/tasks_starting": {}, 36 | "/intel/mesos/master/registrar/queued_operations": {}, 37 | "/intel/mesos/master/registrar/registry_size_bytes": {}, 38 | "/intel/mesos/master/registrar/state_fetch_ms": {}, 39 | "/intel/mesos/master/registrar/state_store_ms": {}, 40 | "/intel/mesos/master/system/load_15min": {}, 41 | "/intel/mesos/master/system/load_1min": {}, 42 | "/intel/mesos/master/system/load_5min": {}, 43 | "/intel/mesos/master/system/mem_free_bytes": {}, 44 | "/intel/mesos/master/system/mem_total_bytes": {} 45 | }, 46 | "config": {}, 47 | "process": null, 48 | "publish": [ 49 | { 50 | "plugin_name": "influx", 51 | "config": { 52 | "host": "10.180.10.180", 53 | "port": 8086, 54 | "database": "snap", 55 | "user": "admin", 56 | "password": "admin" 57 | } 58 | } 59 | ] 60 | } 61 | } 62 | } -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 Intel Corporation 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "os" 21 | 22 | "github.com/intelsdi-x/snap-plugin-collector-mesos/mesos" 23 | "github.com/intelsdi-x/snap/control/plugin" 24 | ) 25 | 26 | // plugin bootstrap 27 | func main() { 28 | plugin.Start( 29 | mesos.Meta(), 30 | mesos.NewMesosCollector(), 31 | os.Args[1], 32 | ) 33 | } 34 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | // +build small 2 | 3 | /* 4 | Copyright 2016 Intel Corporation 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package main 20 | 21 | import ( 22 | "os" 23 | "testing" 24 | 25 | . "github.com/smartystreets/goconvey/convey" 26 | ) 27 | 28 | func TestMain(t *testing.T) { 29 | Convey("ensure plugin loads and responds", t, func() { 30 | os.Args = []string{"", "{\"NoDaemon\": true}"} 31 | So(func() { main() }, ShouldNotPanic) 32 | }) 33 | } 34 | -------------------------------------------------------------------------------- /mesos/agent/agent.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 Intel Corporation 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package agent 18 | 19 | import ( 20 | "fmt" 21 | "regexp" 22 | "strings" 23 | "time" 24 | 25 | log "github.com/Sirupsen/logrus" 26 | "github.com/intelsdi-x/snap-plugin-collector-mesos/mesos/client" 27 | "github.com/intelsdi-x/snap-plugin-collector-mesos/mesos/mesos_pb2" 28 | "github.com/intelsdi-x/snap-plugin-utilities/ns" 29 | "github.com/intelsdi-x/snap-plugin-utilities/str" 30 | ) 31 | 32 | // The "/monitor/statistics" endpoint returns an array of JSON objects. Its top-level structure isn't defined by a 33 | // protobuf, but the "statistics" object (and everything under it) is. For the actual Mesos implementation, see 34 | // https://github.com/apache/mesos/blob/0.28.1/src/slave/monitor.cpp#L130-L148 35 | type Executor struct { 36 | ID string `json:"executor_id"` 37 | Name string `json:"executor_name"` 38 | Source string `json:"source"` 39 | Framework string `json:"framework_id"` 40 | Statistics *mesos_pb2.ResourceStatistics `json:"statistics"` 41 | } 42 | 43 | // The "/slave(1)/flags" endpoint on a Mesos agent returns an object that contains a single object "flags". 44 | type Flags struct { 45 | Flags map[string]string 46 | } 47 | 48 | // Get the configuration flags from the Mesos agent and return them as a map. 49 | func GetFlags(host string) (map[string]string, error) { 50 | log.Debug("Getting configuration flags from host ", host) 51 | flags := &Flags{} 52 | 53 | c := client.NewClient(host, "/slave(1)/flags", time.Duration(5)) 54 | if err := c.Fetch(&flags); err != nil { 55 | log.Error(err) 56 | return nil, err 57 | } 58 | 59 | return flags.Flags, nil 60 | } 61 | 62 | // Collect metrics from the '/metrics/snapshot' endpoint on the agent. The '/metrics/snapshot' endpoint returns JSON, 63 | // and all metrics contained in the endpoint use a string as the key, and a double (float64) for the value. For example: 64 | // 65 | // { 66 | // "slave/cpus_total": 2.0 67 | // } 68 | // 69 | // Note that, as of Mesos 0.28.x, "slave" is being renamed to "agent" and this effort isn't yet complete. For more 70 | // information, see https://issues.apache.org/jira/browse/MESOS-1478. 71 | func GetMetricsSnapshot(host string) (map[string]float64, error) { 72 | log.Debug("Getting metrics snapshot from host ", host) 73 | data := map[string]float64{} 74 | 75 | c := client.NewClient(host, "/metrics/snapshot", time.Duration(5)) 76 | if err := c.Fetch(&data); err != nil { 77 | log.Error(err) 78 | return nil, err 79 | } 80 | 81 | return data, nil 82 | } 83 | 84 | // Collect metrics from the '/monitor/statistics' endpoint on the agent. This endpoint returns JSON, and all metrics 85 | // contained in the endpoint use a string as the key. Depending on features enabled on the Mesos agent, additional 86 | // metrics might be available under either the "statistics" object, or additional nested objects (e.g. "perf") as 87 | // defined by the Executor structure, and the structures in mesos_pb2.ResourceStatistics. 88 | func GetMonitoringStatistics(host string) ([]Executor, error) { 89 | log.Debug("Getting monitoring statistics from host ", host) 90 | var executors []Executor 91 | 92 | c := client.NewClient(host, "/monitor/statistics", time.Duration(30)) 93 | if err := c.Fetch(&executors); err != nil { 94 | log.Error(err) 95 | return nil, err 96 | } 97 | 98 | return executors, nil 99 | } 100 | 101 | // Recursively traverse the Executor struct, building "/"-delimited strings that resemble snap metric types. If a given 102 | // feature is not enabled on a Mesos agent (e.g. the network isolator), then those metrics will be removed from the 103 | // metric types returned by this function. 104 | func GetMonitoringStatisticsMetricTypes(host string) ([]string, error) { 105 | log.Debug("Getting monitoring statistics metrics type from host ", host) 106 | // TODO(roger): supporting NetTrafficControlStatistics means adding another dynamic metric to the plugin. 107 | // When we're ready to do this, remove ns.InspectEmptyContainers(ns.AlwaysFalse) so this defaults to true. 108 | namespaces := []string{} 109 | err := ns.FromCompositeObject( 110 | &mesos_pb2.ResourceStatistics{}, "", &namespaces, ns.InspectEmptyContainers(ns.AlwaysFalse)) 111 | if err != nil { 112 | log.Error(err) 113 | return nil, err 114 | } 115 | 116 | // Avoid returning a metric type that is impossible to collect on this system 117 | flags, err := GetFlags(host) 118 | if err != nil { 119 | log.Error(err) 120 | return nil, err 121 | } 122 | 123 | // Isolators are defined using a comma-separated string passed to the "--isolation" flag on the Mesos agent. 124 | // See https://github.com/apache/mesos/blob/0.28.1/src/slave/containerizer/mesos/containerizer.cpp#L196-L223 125 | isolators := strings.Split(flags["isolation"], ",") 126 | 127 | if str.Contains(isolators, "cgroups/perf_event") { 128 | log.Debug("Isolator cgroups/perf_event is enabled on host ", host) 129 | // Expects a perf event from the output of `perf list`. Mesos then normalizes the event name. See 130 | // https://github.com/apache/mesos/blob/0.28.1/src/linux/perf.cpp#L65-L71 131 | namespaces = deleteFromSlice(namespaces, "^perf/.*") 132 | var normalizedPerfEvents []string 133 | perfEvents := strings.Split(flags["perf_events"], ",") 134 | 135 | for _, event := range perfEvents { 136 | log.Debug("Adding perf event ", event, " to metrics catalog") 137 | event = fmt.Sprintf("perf/%s", normalizePerfEventName(event)) 138 | normalizedPerfEvents = append(normalizedPerfEvents, event) 139 | } 140 | namespaces = append(namespaces, normalizedPerfEvents...) 141 | } else { 142 | log.Debug("Isolator cgroups/perf_event is not enabled on host ", host) 143 | namespaces = deleteFromSlice(namespaces, "^perf.*") 144 | } 145 | 146 | if !str.Contains(isolators, "posix/disk") { 147 | log.Debug("Isolator posix/disk is not enabled on host ", host) 148 | namespaces = deleteFromSlice(namespaces, "^disk_.*") 149 | } 150 | 151 | if !str.Contains(isolators, "network/port_mapping") { 152 | log.Debug("Isolator network/port_mapping is not enabled on host ", host) 153 | namespaces = deleteFromSlice(namespaces, "^net_.*") 154 | } 155 | 156 | return namespaces, nil 157 | } 158 | 159 | // Normalizes a perf event, based on https://github.com/apache/mesos/blob/0.28.1/src/linux/perf.cpp#L65-L71 160 | func normalizePerfEventName(s string) string { 161 | normalized := strings.ToLower(s) 162 | return strings.Replace(normalized, "-", "_", -1) 163 | } 164 | 165 | // Delete a given string from a slice, without returning a new slice. Regex is allowed (e.g. '^perf/.*'). 166 | func deleteFromSlice(a []string, s string) []string { 167 | for i := 0; i < len(a); i++ { 168 | matched, _ := regexp.MatchString(s, a[i]) 169 | if matched { 170 | a = append(a[:i], a[i+1:]...) 171 | i-- 172 | } 173 | } 174 | return a 175 | } 176 | -------------------------------------------------------------------------------- /mesos/agent/agent_test.go: -------------------------------------------------------------------------------- 1 | // +build small 2 | 3 | /* 4 | Copyright 2016 Intel Corporation 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package agent 20 | 21 | import ( 22 | "encoding/json" 23 | "net/http" 24 | "net/http/httptest" 25 | "net/url" 26 | "testing" 27 | 28 | "github.com/golang/protobuf/proto" 29 | "github.com/intelsdi-x/snap-plugin-collector-mesos/mesos/mesos_pb2" 30 | . "github.com/smartystreets/goconvey/convey" 31 | ) 32 | 33 | func TestGetFlags(t *testing.T) { 34 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 35 | testData := Flags{ 36 | Flags: map[string]string{ 37 | "isolation": "cgroups/cpu,cgroups/mem", 38 | "port": "5051", 39 | }, 40 | } 41 | td, err := json.Marshal(testData) 42 | if err != nil { 43 | panic(err) 44 | } 45 | 46 | w.Header().Set("Content-Type", "application/json") 47 | w.WriteHeader(200) 48 | w.Write(td) 49 | })) 50 | defer ts.Close() 51 | 52 | host, err := extractHostFromURL(ts.URL) 53 | if err != nil { 54 | panic(err) 55 | } 56 | 57 | Convey("When getting flags from the Mesos agent", t, func() { 58 | flags, err := GetFlags(host) 59 | 60 | Convey("Should return a map of the configuration flags", func() { 61 | So(err, ShouldBeNil) 62 | So(flags["isolation"], ShouldEqual, "cgroups/cpu,cgroups/mem") 63 | So(flags["port"], ShouldEqual, "5051") 64 | So(flags["perf_events"], ShouldEqual, "") 65 | }) 66 | }) 67 | } 68 | 69 | func TestGetMetricsSnapshot(t *testing.T) { 70 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 71 | td, err := json.Marshal(map[string]float64{ 72 | "containerizer/mesos/container_destroy_errors": 0.0, 73 | "slave/cpus_percent": 0.0, 74 | "system/cpus_total": 2.0, 75 | }) 76 | if err != nil { 77 | panic(err) 78 | } 79 | 80 | w.Header().Set("Content-Type", "application/json") 81 | w.WriteHeader(200) 82 | w.Write(td) 83 | })) 84 | defer ts.Close() 85 | 86 | host, err := extractHostFromURL(ts.URL) 87 | if err != nil { 88 | panic(err) 89 | } 90 | 91 | Convey("Get metrics snapshot from the master", t, func() { 92 | res, err := GetMetricsSnapshot(host) 93 | 94 | Convey("Should return a map of metrics", func() { 95 | So(len(res), ShouldEqual, 3) 96 | So(res["system/cpus_total"], ShouldEqual, 2.0) 97 | So(err, ShouldBeNil) 98 | }) 99 | }) 100 | } 101 | 102 | func TestGetMonitoringStatistics(t *testing.T) { 103 | testData := []Executor{ 104 | Executor{ 105 | ID: "id1", 106 | Name: "name1", 107 | Source: "source1", 108 | Framework: "frame1", 109 | Statistics: &mesos_pb2.ResourceStatistics{ 110 | CpusLimit: proto.Float64(1.1), 111 | MemTotalBytes: proto.Uint64(1000), 112 | Perf: &mesos_pb2.PerfStatistics{ 113 | ContextSwitches: proto.Uint64(10), 114 | }, 115 | }, 116 | }, 117 | Executor{ 118 | ID: "id2", 119 | Name: "name2", 120 | Source: "source2", 121 | Framework: "frame2", 122 | Statistics: &mesos_pb2.ResourceStatistics{ 123 | CpusLimit: proto.Float64(1.1), 124 | MemTotalBytes: proto.Uint64(2000), 125 | }, 126 | }, 127 | } 128 | 129 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 130 | td, err := json.Marshal(testData) 131 | if err != nil { 132 | panic(err) 133 | } 134 | 135 | w.Header().Set("Content-Type", "application/json") 136 | w.WriteHeader(200) 137 | w.Write(td) 138 | })) 139 | defer ts.Close() 140 | 141 | host, err := extractHostFromURL(ts.URL) 142 | if err != nil { 143 | panic(err) 144 | } 145 | 146 | Convey("When monitoring statistics are requested", t, func() { 147 | execs, err := GetMonitoringStatistics(host) 148 | 149 | Convey("Then no error should be reported", func() { 150 | So(err, ShouldBeNil) 151 | }) 152 | 153 | Convey("Then list of executors is returned", func() { 154 | So(execs, ShouldNotBeNil) 155 | So(len(execs), ShouldEqual, len(testData)) 156 | }) 157 | 158 | Convey("Then proper stats are returned", func() { 159 | for _, exec := range execs { 160 | switch exec.ID { 161 | case "id1": 162 | So(*exec.Statistics.CpusLimit, ShouldEqual, 1.1) 163 | So(*exec.Statistics.MemTotalBytes, ShouldEqual, 1000) 164 | So(*exec.Statistics.Perf.ContextSwitches, ShouldEqual, 10) 165 | case "id2": 166 | So(*exec.Statistics.CpusLimit, ShouldEqual, 1.1) 167 | So(*exec.Statistics.MemTotalBytes, ShouldEqual, 2000) 168 | } 169 | } 170 | }) 171 | }) 172 | } 173 | 174 | func Test_normalizePerfEventName(t *testing.T) { 175 | Convey("When passed a perf_event name containing mixed case and dashes", t, func() { 176 | Convey("Should return a normalized perf event name", func() { 177 | So(normalizePerfEventName("fooBarBaz"), ShouldEqual, "foobarbaz") 178 | So(normalizePerfEventName("foo-barBaz"), ShouldEqual, "foo_barbaz") 179 | So(normalizePerfEventName("foo_bar-baz"), ShouldEqual, "foo_bar_baz") 180 | }) 181 | }) 182 | } 183 | 184 | func Test_deleteFromSlice(t *testing.T) { 185 | Convey("When deleting a string or regex from a slice", t, func() { 186 | testData := []string{"foo", "bar", "foo/bar", "foo/bar_one", "foo/bar_two"} 187 | 188 | Convey("Should delete all strings matching a regex", func() { 189 | result := deleteFromSlice(testData, "^foo/bar_.*") 190 | So(result, ShouldNotContain, "foo/bar_one") 191 | So(result, ShouldNotContain, "foo/bar_two") 192 | So(result, ShouldContain, "foo") 193 | So(result, ShouldContain, "bar") 194 | So(result, ShouldContain, "foo/bar") 195 | }) 196 | Convey("Should delete a single string", func() { 197 | result := deleteFromSlice(testData, "foo") 198 | So(result, ShouldNotContain, "foo") 199 | So(result, ShouldNotContain, "foo/bar") 200 | So(result, ShouldContain, "bar") 201 | }) 202 | }) 203 | } 204 | 205 | func extractHostFromURL(u string) (string, error) { 206 | parsed, err := url.Parse(u) 207 | if err != nil { 208 | return "", err 209 | } 210 | return parsed.Host, nil 211 | } 212 | -------------------------------------------------------------------------------- /mesos/client/client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 Intel Corporation 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package client 18 | 19 | import ( 20 | "encoding/json" 21 | "fmt" 22 | "io/ioutil" 23 | "net/http" 24 | "net/url" 25 | "time" 26 | 27 | log "github.com/Sirupsen/logrus" 28 | ) 29 | 30 | // Define a client for collecting metrics from a Mesos master/agent over HTTP. 31 | type Client struct { 32 | httpClient *http.Client 33 | host string 34 | path string 35 | } 36 | 37 | // Return a new instance of Client. 38 | func NewClient(host string, path string, timeout time.Duration) *Client { 39 | log.Debug("Creating a new instance of the Mesos plugin HTTP client") 40 | return &Client{ 41 | httpClient: &http.Client{Timeout: timeout}, 42 | host: host, 43 | path: path, 44 | } 45 | } 46 | 47 | // Return the URL for this client as a string. Note that this isn't specifically required for this client, but might 48 | // be useful if you want to retrieve the actual URL for logging, etc throughout this plugin. 49 | func (c *Client) URL() string { 50 | u := url.URL{Scheme: "http", Host: c.host, Path: c.path} 51 | return u.String() 52 | } 53 | 54 | // Fetch JSON from the API endpoint, unmarshal it, and return it to the provided 'target'. 55 | func (c *Client) Fetch(target interface{}) error { 56 | log.Debug("Fetching data from ", c.URL()) 57 | resp, err := http.Get(c.URL()) 58 | if err != nil { 59 | log.Error(err) 60 | return err 61 | } 62 | defer resp.Body.Close() 63 | 64 | if resp.StatusCode != http.StatusOK { 65 | e := fmt.Errorf("fetch error: %s", resp.Status) 66 | log.Error(e) 67 | return e 68 | } 69 | 70 | b, err := ioutil.ReadAll(resp.Body) 71 | if err != nil { 72 | e := fmt.Errorf("read error: %s: %v\n", c.URL(), err) 73 | log.Error(e) 74 | return e 75 | } 76 | 77 | if err := json.Unmarshal(b, &target); err != nil { 78 | e := fmt.Errorf("unmarshal error: %s: %v\n", b, err) 79 | log.Error(e) 80 | return e 81 | } 82 | 83 | return nil 84 | } 85 | -------------------------------------------------------------------------------- /mesos/client/client_test.go: -------------------------------------------------------------------------------- 1 | // +build small 2 | 3 | /* 4 | Copyright 2016 Intel Corporation 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package client 20 | 21 | import ( 22 | "encoding/json" 23 | "net/http" 24 | "net/http/httptest" 25 | "net/url" 26 | "testing" 27 | "time" 28 | 29 | . "github.com/smartystreets/goconvey/convey" 30 | ) 31 | 32 | func TestNewClient(t *testing.T) { 33 | Convey("Should return a new client", t, func() { 34 | c := NewClient("foo.example.com", "/bar", time.Duration(1)) 35 | So(c, ShouldHaveSameTypeAs, &Client{}) 36 | }) 37 | } 38 | 39 | func TestClient_Fetch(t *testing.T) { 40 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 41 | td, err := json.Marshal(map[string]string{ 42 | "foo": "bar", 43 | }) 44 | if err != nil { 45 | panic(err) 46 | } 47 | 48 | w.Header().Set("Content-Type", "application/json") 49 | w.WriteHeader(200) 50 | w.Write(td) 51 | })) 52 | defer ts.Close() 53 | 54 | Convey("Should fetch data from the Mesos API and return", t, func() { 55 | data := map[string]string{} 56 | host, err := extractHostFromURL(ts.URL) 57 | if err != nil { 58 | panic(err) 59 | } 60 | 61 | c := NewClient(host, "/", time.Duration(1)) 62 | err = c.Fetch(&data) 63 | 64 | So(data["foo"], ShouldEqual, "bar") 65 | So(err, ShouldBeNil) 66 | }) 67 | } 68 | 69 | func TestClient_URL(t *testing.T) { 70 | Convey("Should return the URL as a string", t, func() { 71 | c := NewClient("foo.example.com", "/bar", time.Duration(1)) 72 | So(c.URL(), ShouldEqual, "http://foo.example.com/bar") 73 | }) 74 | } 75 | 76 | func extractHostFromURL(u string) (string, error) { 77 | parsed, err := url.Parse(u) 78 | if err != nil { 79 | return "", err 80 | } 81 | return parsed.Host, nil 82 | } 83 | -------------------------------------------------------------------------------- /mesos/master/master.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 Intel Corporation 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package master 18 | 19 | import ( 20 | "fmt" 21 | "net/http" 22 | "strings" 23 | "time" 24 | 25 | log "github.com/Sirupsen/logrus" 26 | "github.com/intelsdi-x/snap-plugin-collector-mesos/mesos/client" 27 | "github.com/intelsdi-x/snap-plugin-utilities/ns" 28 | ) 29 | 30 | type Frameworks struct { 31 | ActiveFrameworks []*Framework `json:"frameworks"` 32 | } 33 | 34 | type Framework struct { 35 | ID string `json:"id"` 36 | OfferedResources *Resources `json:"offered_resources"` 37 | Resources *Resources `json:"resources"` 38 | UsedResources *Resources `json:"used_resources"` 39 | } 40 | 41 | type Resources struct { 42 | CPUs float64 `json:"cpus"` 43 | Disk float64 `json:"disk"` 44 | Mem float64 `json:"mem"` 45 | } 46 | 47 | // Recursively traverse the Frameworks struct, building "/"-delimited strings that resemble snap metric types. 48 | func GetFrameworksMetricTypes() ([]string, error) { 49 | log.Debug("Getting frameworks metric types from protobuf (mesos_pb2)") 50 | namespaces := []string{} 51 | if err := ns.FromCompositeObject(Framework{}, "", &namespaces); err != nil { 52 | log.Error(err) 53 | return nil, err 54 | } 55 | for i := 0; i < len(namespaces); i++ { 56 | if namespaces[i] == "id" { 57 | namespaces = append(namespaces[:i], namespaces[i+1:]...) 58 | break 59 | } 60 | } 61 | return namespaces, nil 62 | 63 | } 64 | 65 | // Get metrics from the '/master/frameworks' endpoint on the master. This endpoint returns JSON about the overall 66 | // state and resource utilization of the frameworks running on the cluster. 67 | func GetFrameworks(host string) ([]*Framework, error) { 68 | log.Debug("Getting active frameworks resource utilization from master ", host) 69 | var frameworks Frameworks 70 | 71 | c := client.NewClient(host, "/master/frameworks", time.Duration(10)) 72 | if err := c.Fetch(&frameworks); err != nil { 73 | log.Error(err) 74 | return nil, err 75 | } 76 | 77 | return frameworks.ActiveFrameworks, nil 78 | } 79 | 80 | // Collect metrics from the '/metrics/snapshot' endpoint on the master. The '/metrics/snapshot' endpoint returns JSON, 81 | // and all metrics contained in the endpoint use a string as the key, and a double (float64) for the value. For example: 82 | // 83 | // { 84 | // "master/cpus_total": 2.0 85 | // } 86 | // 87 | func GetMetricsSnapshot(host string) (map[string]float64, error) { 88 | log.Debug("Getting metrics snapshot for host ", host) 89 | data := map[string]float64{} 90 | 91 | c := client.NewClient(host, "/metrics/snapshot", time.Duration(5)) 92 | if err := c.Fetch(&data); err != nil { 93 | log.Error(err) 94 | return nil, err 95 | } 96 | 97 | return data, nil 98 | } 99 | 100 | // Determine if a given host is currently the leader, based on the location provided by the '/master/redirect' endpoint. 101 | func IsLeader(host string) (bool, error) { 102 | log.Debug("Determining if host ", host, " is currently the leader") 103 | req, err := http.NewRequest("HEAD", "http://"+host+"/master/redirect", nil) 104 | if err != nil { 105 | e := fmt.Errorf("request error: %s", err) 106 | log.Error(e) 107 | return false, e 108 | } 109 | 110 | resp, err := http.DefaultTransport.RoundTrip(req) 111 | if err != nil { 112 | if resp.StatusCode == 307 { 113 | log.Debug("Got expected response HTTP 307") 114 | } else if resp.StatusCode != 307 { 115 | e := fmt.Errorf("error: expected HTTP 307, got %d", resp.StatusCode) 116 | log.Error(e) 117 | return false, e 118 | } else { 119 | e := fmt.Errorf("client error: %s", err) 120 | log.Error(e) 121 | return false, e 122 | } 123 | } 124 | 125 | location, err := resp.Location() 126 | if err != nil { 127 | log.Error(err) 128 | return false, err 129 | } 130 | 131 | if strings.Contains(location.Host, host) { 132 | log.Debug("Host ", host, " is currently the leader (matched ", location.Host, ")") 133 | return true, nil 134 | } else { 135 | log.Debug("Host ", host, "is not currently the leader (did not match ", location.Host, ")") 136 | return false, nil 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /mesos/master/master_test.go: -------------------------------------------------------------------------------- 1 | // +build small 2 | 3 | /* 4 | Copyright 2016 Intel Corporation 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package master 20 | 21 | import ( 22 | "encoding/json" 23 | "net/http" 24 | "net/http/httptest" 25 | "net/url" 26 | "testing" 27 | 28 | . "github.com/smartystreets/goconvey/convey" 29 | ) 30 | 31 | func TestGetFrameworks(t *testing.T) { 32 | testData := Frameworks{ActiveFrameworks: []*Framework{ 33 | &Framework{ 34 | ID: "id1", 35 | OfferedResources: &Resources{ 36 | CPUs: 1.0, 37 | Mem: 1024.0, 38 | Disk: 512.0, 39 | }, 40 | Resources: &Resources{ 41 | CPUs: 1.0, 42 | Mem: 1024.0, 43 | Disk: 512.0, 44 | }, 45 | UsedResources: &Resources{ 46 | CPUs: 1.0, 47 | Mem: 1024.0, 48 | Disk: 512.0, 49 | }, 50 | }, 51 | }, 52 | } 53 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 54 | td, err := json.Marshal(testData) 55 | if err != nil { 56 | panic(err) 57 | } 58 | 59 | w.Header().Set("Content-Type", "application/json") 60 | w.WriteHeader(200) 61 | w.Write(td) 62 | })) 63 | defer ts.Close() 64 | 65 | host, err := extractHostFromURL(ts.URL) 66 | if err != nil { 67 | panic(err) 68 | } 69 | 70 | Convey("When framework resource utilization is requested", t, func() { 71 | frameworks, err := GetFrameworks(host) 72 | 73 | Convey("Then no error should be reported", func() { 74 | So(err, ShouldBeNil) 75 | }) 76 | 77 | Convey("Then list of frameworks is returned", func() { 78 | So(frameworks, ShouldNotBeNil) 79 | So(len(frameworks), ShouldEqual, len(testData.ActiveFrameworks)) 80 | }) 81 | 82 | Convey("Then proper stats are returned", func() { 83 | for _, framework := range frameworks { 84 | switch framework.ID { 85 | case "id1": 86 | So(framework.Resources.CPUs, ShouldEqual, 1.0) 87 | So(framework.Resources.Disk, ShouldEqual, 512.0) 88 | So(framework.Resources.Mem, ShouldEqual, 1024.0) 89 | So(framework.OfferedResources.CPUs, ShouldEqual, 1.0) 90 | So(framework.OfferedResources.Disk, ShouldEqual, 512.0) 91 | So(framework.OfferedResources.Mem, ShouldEqual, 1024.0) 92 | So(framework.UsedResources.CPUs, ShouldEqual, 1.0) 93 | So(framework.UsedResources.Disk, ShouldEqual, 512.0) 94 | So(framework.UsedResources.Mem, ShouldEqual, 1024.0) 95 | 96 | } 97 | } 98 | }) 99 | }) 100 | } 101 | 102 | func TestGetFrameworksMetricTypes(t *testing.T) { 103 | Convey("When building metric types for Frameworks on the master", t, func() { 104 | namespaces, err := GetFrameworksMetricTypes() 105 | Convey("No errors should be reported", func() { 106 | So(err, ShouldBeNil) 107 | }) 108 | Convey("Valid namespace parts should be returned as a slice of strings", func() { 109 | So(len(namespaces), ShouldBeGreaterThan, 0) 110 | So(namespaces, ShouldContain, "resources/disk") 111 | So(namespaces, ShouldNotContain, "resources/foo") 112 | }) 113 | Convey("Should not contain non-metrics namespaces, e.g. 'id'", func() { 114 | So(namespaces, ShouldNotContain, "id") 115 | }) 116 | }) 117 | } 118 | 119 | func TestGetMetricsSnapshot(t *testing.T) { 120 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 121 | td, err := json.Marshal(map[string]float64{ 122 | "allocator/event_queue_dispatches": 0.0, 123 | "master/cpus_percent": 0.0, 124 | "registrar/queued_operations": 0.0, 125 | "system/cpus_total": 2.0, 126 | }) 127 | if err != nil { 128 | panic(err) 129 | } 130 | 131 | w.Header().Set("Content-Type", "application/json") 132 | w.WriteHeader(200) 133 | w.Write(td) 134 | })) 135 | defer ts.Close() 136 | 137 | host, err := extractHostFromURL(ts.URL) 138 | if err != nil { 139 | panic(err) 140 | } 141 | 142 | Convey("Get metrics snapshot from the master", t, func() { 143 | res, err := GetMetricsSnapshot(host) 144 | 145 | Convey("Should return a map of metrics", func() { 146 | So(len(res), ShouldEqual, 4) 147 | So(res["system/cpus_total"], ShouldEqual, 2.0) 148 | So(err, ShouldBeNil) 149 | }) 150 | }) 151 | } 152 | 153 | func TestIsLeader(t *testing.T) { 154 | 155 | // ts1 simulates a host that is the leader 156 | ts1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 157 | w.Header().Set("Location", r.URL.String()) 158 | w.WriteHeader(307) 159 | })) 160 | 161 | // ts2 simulates a host that is not the leader 162 | ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 163 | w.Header().Set("Location", "//mesos-master-2.example.com:5050") 164 | w.WriteHeader(307) 165 | })) 166 | 167 | defer ts1.Close() 168 | defer ts2.Close() 169 | 170 | Convey("Determine if master is leader", t, func() { 171 | host, err := extractHostFromURL(ts1.URL) 172 | if err != nil { 173 | panic(err) 174 | } 175 | 176 | Convey("No error should be reported", func() { 177 | _, err := IsLeader(host) 178 | So(err, ShouldBeNil) 179 | }) 180 | 181 | Convey("Should return true when leading", func() { 182 | hostIsLeader, err := IsLeader(host) 183 | So(hostIsLeader, ShouldBeTrue) 184 | So(err, ShouldBeNil) 185 | }) 186 | 187 | Convey("Should return false when not leading", func() { 188 | host, err := extractHostFromURL(ts2.URL) 189 | if err != nil { 190 | panic(err) 191 | } 192 | 193 | hostIsLeader, err := IsLeader(host) 194 | So(hostIsLeader, ShouldBeFalse) 195 | So(err, ShouldBeNil) 196 | }) 197 | }) 198 | } 199 | 200 | func extractHostFromURL(u string) (string, error) { 201 | parsed, err := url.Parse(u) 202 | if err != nil { 203 | return "", err 204 | } 205 | return parsed.Host, nil 206 | } 207 | -------------------------------------------------------------------------------- /mesos/mesos.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 Intel Corporation 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package mesos 18 | 19 | import ( 20 | "fmt" 21 | "strings" 22 | "time" 23 | 24 | log "github.com/Sirupsen/logrus" 25 | "github.com/intelsdi-x/snap-plugin-collector-mesos/mesos/agent" 26 | "github.com/intelsdi-x/snap-plugin-collector-mesos/mesos/master" 27 | "github.com/intelsdi-x/snap-plugin-utilities/config" 28 | "github.com/intelsdi-x/snap-plugin-utilities/ns" 29 | "github.com/intelsdi-x/snap/control/plugin" 30 | "github.com/intelsdi-x/snap/control/plugin/cpolicy" 31 | "github.com/intelsdi-x/snap/core" 32 | ) 33 | 34 | const ( 35 | pluginVendor = "intel" 36 | pluginName = "mesos" 37 | pluginVersion = 1 38 | pluginType = plugin.CollectorPluginType 39 | ) 40 | 41 | func Meta() *plugin.PluginMeta { 42 | return plugin.NewPluginMeta( 43 | pluginName, 44 | pluginVersion, 45 | pluginType, 46 | []string{plugin.SnapGOBContentType}, 47 | []string{plugin.SnapGOBContentType}) 48 | } 49 | 50 | func NewMesosCollector() *Mesos { 51 | log.Debug("Created a new instance of the Mesos collector plugin") 52 | return &Mesos{} 53 | } 54 | 55 | type Mesos struct { 56 | } 57 | 58 | func (m *Mesos) GetConfigPolicy() (*cpolicy.ConfigPolicy, error) { 59 | return cpolicy.New(), nil 60 | } 61 | 62 | func (m *Mesos) GetMetricTypes(cfg plugin.ConfigType) ([]plugin.MetricType, error) { 63 | configItems, err := getConfig(cfg) 64 | if err != nil { 65 | log.Error(err) 66 | return nil, err 67 | } 68 | 69 | metricTypes := []plugin.MetricType{} 70 | 71 | if configItems["master"] != "" { 72 | log.Info("Getting metric types for the Mesos master at ", configItems["master"]) 73 | master_mts, err := master.GetMetricsSnapshot(configItems["master"]) 74 | if err != nil { 75 | log.Error(err) 76 | return nil, err 77 | } 78 | 79 | for key, _ := range master_mts { 80 | namespace := core.NewNamespace(pluginVendor, pluginName, "master"). 81 | AddStaticElements(strings.Split(key, "/")...) 82 | log.Debug("Adding metric to catalog: ", namespace.String()) 83 | metricTypes = append(metricTypes, plugin.MetricType{Namespace_: namespace}) 84 | } 85 | 86 | framework_mts, err := master.GetFrameworksMetricTypes() 87 | if err != nil { 88 | log.Error(err) 89 | return nil, err 90 | } 91 | 92 | for _, key := range framework_mts { 93 | namespace := core.NewNamespace(pluginVendor, pluginName, "master"). 94 | AddDynamicElement("framework_id", "Framework ID"). 95 | AddStaticElements(strings.Split(key, "/")...) 96 | log.Debug("Adding metric to catalog: ", namespace.String()) 97 | metricTypes = append(metricTypes, plugin.MetricType{Namespace_: namespace}) 98 | } 99 | } 100 | 101 | if configItems["agent"] != "" { 102 | log.Info("Getting metric types for the Mesos agent at ", configItems["agent"]) 103 | agent_mts, err := agent.GetMetricsSnapshot(configItems["agent"]) 104 | if err != nil { 105 | log.Error(err) 106 | return nil, err 107 | } 108 | 109 | for key, _ := range agent_mts { 110 | namespace := core.NewNamespace(pluginVendor, pluginName, "agent"). 111 | AddStaticElements(strings.Split(key, "/")...) 112 | log.Debug("Adding metric to catalog: ", namespace.String()) 113 | metricTypes = append(metricTypes, plugin.MetricType{Namespace_: namespace}) 114 | } 115 | 116 | agent_stats, err := agent.GetMonitoringStatisticsMetricTypes(configItems["agent"]) 117 | if err != nil { 118 | log.Error(err) 119 | return nil, err 120 | } 121 | 122 | for _, key := range agent_stats { 123 | namespace := core.NewNamespace(pluginVendor, pluginName, "agent"). 124 | AddDynamicElement("framework_id", "Framework ID"). 125 | AddDynamicElement("executor_id", "Executor ID"). 126 | AddStaticElements(strings.Split(key, "/")...) 127 | log.Debug("Adding metric to catalog: ", namespace.String()) 128 | metricTypes = append(metricTypes, plugin.MetricType{Namespace_: namespace}) 129 | } 130 | } 131 | 132 | return metricTypes, nil 133 | } 134 | 135 | func (m *Mesos) CollectMetrics(mts []plugin.MetricType) ([]plugin.MetricType, error) { 136 | configItems, err := getConfig(mts[0]) 137 | if err != nil { 138 | return nil, err 139 | } 140 | 141 | requestedMaster := []core.Namespace{} 142 | requestedAgent := []core.Namespace{} 143 | 144 | for _, metricType := range mts { 145 | switch metricType.Namespace().Strings()[2] { 146 | case "master": 147 | requestedMaster = append(requestedMaster, metricType.Namespace()) 148 | case "agent": 149 | requestedAgent = append(requestedAgent, metricType.Namespace()) 150 | } 151 | } 152 | 153 | // Translate Mesos metrics into Snap PluginMetrics 154 | now := time.Now() 155 | metrics := []plugin.MetricType{} 156 | 157 | if configItems["master"] != "" && len(requestedMaster) > 0 { 158 | log.Info("Collecting ", len(requestedMaster), " metrics from the master") 159 | isLeader, err := master.IsLeader(configItems["master"]) 160 | if err != nil { 161 | log.Error(err) 162 | return nil, err 163 | } 164 | if isLeader { 165 | snapshot, err := master.GetMetricsSnapshot(configItems["master"]) 166 | if err != nil { 167 | log.Error(err) 168 | return nil, err 169 | } 170 | 171 | frameworks, err := master.GetFrameworks(configItems["master"]) 172 | if err != nil { 173 | log.Error(err) 174 | return nil, err 175 | } 176 | 177 | tags := map[string]string{"source": configItems["master"]} 178 | 179 | for _, requested := range requestedMaster { 180 | isDynamic, _ := requested.IsDynamic() 181 | if isDynamic { 182 | n := requested.Strings()[4:] 183 | 184 | // Iterate through the array of frameworks returned by GetFrameworks() 185 | for _, framework := range frameworks { 186 | val := ns.GetValueByNamespace(framework, n) 187 | if val == nil { 188 | log.Warn("Attempted to collect metric ", requested.String(), " but it returned nil!") 189 | continue 190 | } 191 | // substituting "framework" wildcard with particular framework id 192 | rendered := cloneNamespace(requested) 193 | rendered[3].Value = framework.ID 194 | // TODO(roger): units 195 | metrics = append(metrics, *plugin.NewMetricType(rendered, now, tags, "", val)) 196 | 197 | } 198 | } else { 199 | n := requested.Strings()[3:] 200 | val, ok := snapshot[strings.Join(n, "/")] 201 | if !ok { 202 | e := fmt.Errorf("error: requested metric %s not found", requested.String()) 203 | log.Error(e) 204 | return nil, e 205 | } 206 | //TODO(kromar): is it possible to provide unit NewMetricType(ns, time, tags, unit, value)? 207 | // I'm leaving empty string for now... 208 | metrics = append(metrics, *plugin.NewMetricType(requested, now, tags, "", val)) 209 | } 210 | } 211 | } else { 212 | log.Info("Attempted CollectMetrics() on ", configItems["master"], "but it isn't the leader. Skipping...") 213 | } 214 | } 215 | 216 | if configItems["agent"] != "" && len(requestedAgent) > 0 { 217 | log.Info("Collecting ", len(requestedAgent), " metrics from the agent") 218 | snapshot, err := agent.GetMetricsSnapshot(configItems["agent"]) 219 | if err != nil { 220 | log.Error(err) 221 | return nil, err 222 | } 223 | 224 | executors, err := agent.GetMonitoringStatistics(configItems["agent"]) 225 | if err != nil { 226 | log.Error(err) 227 | return nil, err 228 | } 229 | 230 | tags := map[string]string{"source": configItems["agent"]} 231 | 232 | for _, requested := range requestedAgent { 233 | n := requested.Strings()[5:] 234 | isDynamic, _ := requested.IsDynamic() 235 | if isDynamic { 236 | // Iterate through the array of executors returned by GetMonitoringStatistics() 237 | for _, exec := range executors { 238 | val := ns.GetValueByNamespace(exec.Statistics, n) 239 | if val == nil { 240 | log.Warn("Attempted to collect metric ", requested.String(), " but it returned nil!") 241 | continue 242 | } 243 | rendered := cloneNamespace(requested) 244 | // substituting "framework" wildcard with particular framework id 245 | rendered[3].Value = exec.Framework 246 | // substituting "executor" wildcard with particular executor id 247 | rendered[4].Value = exec.ID 248 | // TODO(roger): units 249 | metrics = append(metrics, *plugin.NewMetricType(rendered, now, tags, "", val)) 250 | 251 | } 252 | } else { 253 | // Get requested metrics from the snapshot map 254 | n := requested.Strings()[3:] 255 | val, ok := snapshot[strings.Join(n, "/")] 256 | if !ok { 257 | e := fmt.Errorf("error: requested metric %v not found", requested.String()) 258 | log.Error(e) 259 | return nil, e 260 | } 261 | 262 | //TODO(kromar): units here also? 263 | metrics = append(metrics, *plugin.NewMetricType(requested, now, tags, "", val)) 264 | } 265 | } 266 | } 267 | 268 | log.Debug("Collected a total of ", len(metrics), " metrics.") 269 | return metrics, nil 270 | } 271 | 272 | func getConfig(cfg interface{}) (map[string]string, error) { 273 | items := make(map[string]string) 274 | var ok bool 275 | 276 | // Note: although config.GetConfigItems can accept multiple config parameter names, it appears that if 277 | // any of those names are missing, GetConfigItems() will `return nil, err`. Since this plugin will work 278 | // individually with master or agent (or both), we break this up into two separate lookups and then 279 | // test for the existence of the configuration parameter to determine which metric types are available. 280 | 281 | // We expect the value of "master" in the global config to follow the convention "192.168.99.100:5050" 282 | master_cfg, master_err := config.GetConfigItem(cfg, "master") 283 | 284 | // We expect the value of "agent" in the global config to follow the convention "192.168.99.100:5051" 285 | agent_cfg, agent_err := config.GetConfigItem(cfg, "agent") 286 | 287 | if master_err != nil && agent_err != nil { 288 | e := fmt.Errorf("error: no global config specified for 'master' and 'agent'.") 289 | log.Error(e) 290 | return items, e 291 | } 292 | 293 | items["master"], ok = master_cfg.(string) 294 | if !ok { 295 | log.Warn("No global config specified for 'master', only 'agent' metrics will be collected.") 296 | } 297 | 298 | items["agent"], ok = agent_cfg.(string) 299 | if !ok { 300 | log.Warn("No global config specified for 'agent', only 'master' metrics will be collected.") 301 | } 302 | 303 | return items, nil 304 | } 305 | 306 | func cloneNamespace(ns core.Namespace) core.Namespace { 307 | nsCopy := make(core.Namespace, len(ns)) 308 | copy(nsCopy, ns) 309 | 310 | return nsCopy 311 | } 312 | -------------------------------------------------------------------------------- /mesos/mesos_integration_test.go: -------------------------------------------------------------------------------- 1 | // +build medium 2 | 3 | /* 4 | Copyright 2016 Intel Corporation 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package mesos 20 | 21 | import ( 22 | "bytes" 23 | "fmt" 24 | "net/http" 25 | "net/url" 26 | "os" 27 | "os/exec" 28 | "testing" 29 | "time" 30 | 31 | "github.com/intelsdi-x/snap-plugin-collector-mesos/mesos/agent" 32 | "github.com/intelsdi-x/snap-plugin-collector-mesos/mesos/master" 33 | "github.com/intelsdi-x/snap-plugin-utilities/config" 34 | "github.com/intelsdi-x/snap/control/plugin" 35 | "github.com/intelsdi-x/snap/core" 36 | "github.com/intelsdi-x/snap/core/cdata" 37 | "github.com/intelsdi-x/snap/core/ctypes" 38 | . "github.com/smartystreets/goconvey/convey" 39 | ) 40 | 41 | func TestMesos_GetMetricTypes(t *testing.T) { 42 | cfg := setupCfg() 43 | mc := NewMesosCollector() 44 | mts, err := mc.GetMetricTypes(cfg) 45 | if err != nil { 46 | panic(err) 47 | } 48 | 49 | var namespaces []string 50 | for i := 0; i < len(mts); i++ { 51 | namespaces = append(namespaces, mts[i].Namespace().String()) 52 | } 53 | 54 | Convey("Should return metric types for the master and agent", t, func() { 55 | So(err, ShouldBeNil) 56 | So(len(mts), ShouldBeGreaterThan, 1) 57 | So(namespaces, ShouldContain, "/intel/mesos/master/system/cpus_total") 58 | So(namespaces, ShouldContain, "/intel/mesos/master/*/resources/cpus") 59 | So(namespaces, ShouldContain, "/intel/mesos/agent/system/cpus_total") 60 | So(namespaces, ShouldContain, "/intel/mesos/agent/*/*/cpus_limit") 61 | }) 62 | 63 | // In both the Vagrant and Travis provisioning scripts, "network/port_mapping" is not 64 | // enabled because Mesos hasn't been compiled with that isolator. If that changes, this 65 | // test case will need to be updated. 66 | Convey("Should only return metrics for available features on a Mesos agent", t, func() { 67 | So(namespaces, ShouldNotContain, "/intel/mesos/agent/*/*/net_rx_bytes") 68 | }) 69 | } 70 | 71 | func TestMesos_CollectMetrics(t *testing.T) { 72 | cfg := setupCfg() 73 | cfgItems, err := config.GetConfigItems(cfg, []string{"master", "agent"}...) 74 | if err != nil { 75 | panic(err) 76 | } 77 | 78 | // Clean slate 79 | teardown(cfgItems["master"].(string)) 80 | launchTasks(cfgItems["master"].(string), cfgItems["agent"].(string)) 81 | 82 | Convey("Collect metrics from a Mesos master and agent", t, func() { 83 | mc := NewMesosCollector() 84 | _, err := mc.GetMetricTypes(cfg) 85 | if err != nil { 86 | panic(err) 87 | } 88 | 89 | Convey("Should collect requested metrics from the master", func() { 90 | mts := []plugin.MetricType{ 91 | plugin.MetricType{ 92 | Namespace_: core.NewNamespace("intel", "mesos", "master", "master", "tasks_running"), 93 | Config_: cfg.ConfigDataNode, 94 | }, 95 | plugin.MetricType{ 96 | Namespace_: core.NewNamespace("intel", "mesos", "master", "registrar", "state_store_ms", "p99"), 97 | Config_: cfg.ConfigDataNode, 98 | }, 99 | plugin.MetricType{ 100 | Namespace_: core.NewNamespace("intel", "mesos", "master", "system", "load_5min"), 101 | Config_: cfg.ConfigDataNode, 102 | }, 103 | plugin.MetricType{ 104 | Namespace_: core.NewNamespace("intel", "mesos", "master"). 105 | AddDynamicElement("framework_id", "Framework ID"). 106 | AddStaticElements("used_resources", "cpus"), 107 | Config_: cfg.ConfigDataNode, 108 | }, 109 | } 110 | 111 | metrics, err := mc.CollectMetrics(mts) 112 | So(err, ShouldBeNil) 113 | So(metrics, ShouldNotBeNil) 114 | So(len(metrics), ShouldEqual, 5) 115 | So(metrics[0].Namespace().String(), ShouldEqual, "/intel/mesos/master/master/tasks_running") 116 | So(metrics[1].Namespace().String(), ShouldEqual, "/intel/mesos/master/registrar/state_store_ms/p99") 117 | So(metrics[2].Namespace().String(), ShouldEqual, "/intel/mesos/master/system/load_5min") 118 | So(metrics[3].Namespace().Strings()[4], ShouldEqual, "used_resources") 119 | }) 120 | 121 | // NOTE: in future versions of Mesos, the term "slave" will change to "agent". This has the potential 122 | // to break this test if the Mesos version is bumped in CI and this test isn't updated at the same time. 123 | Convey("Should collect requested metrics from the agent", func() { 124 | mts := []plugin.MetricType{ 125 | plugin.MetricType{ 126 | Namespace_: core.NewNamespace("intel", "mesos", "agent", "slave", "tasks_running"), 127 | Config_: cfg.ConfigDataNode, 128 | }, 129 | plugin.MetricType{ 130 | Namespace_: core.NewNamespace("intel", "mesos", "agent", "system", "load_5min"), 131 | Config_: cfg.ConfigDataNode, 132 | }, 133 | plugin.MetricType{ 134 | Namespace_: core.NewNamespace("intel", "mesos", "agent"). 135 | AddDynamicElement("framework_id", "Framework ID"). 136 | AddDynamicElement("executor_id", "Executor ID"). 137 | AddStaticElement("cpus_system_time_secs"), 138 | Config_: cfg.ConfigDataNode, 139 | }, 140 | plugin.MetricType{ 141 | Namespace_: core.NewNamespace("intel", "mesos", "agent"). 142 | AddDynamicElement("framework_id", "Framework ID"). 143 | AddDynamicElement("executor_id", "Executor ID"). 144 | AddStaticElement("disk_used_bytes"), 145 | Config_: cfg.ConfigDataNode, 146 | }, 147 | plugin.MetricType{ 148 | Namespace_: core.NewNamespace("intel", "mesos", "agent"). 149 | AddDynamicElement("framework_id", "Framework ID"). 150 | AddDynamicElement("executor_id", "Executor ID"). 151 | AddStaticElement("mem_total_bytes"), 152 | Config_: cfg.ConfigDataNode, 153 | }, 154 | } 155 | 156 | metrics, err := mc.CollectMetrics(mts) 157 | So(err, ShouldBeNil) 158 | So(metrics, ShouldNotBeNil) 159 | So(len(metrics), ShouldEqual, 8) 160 | So(metrics[0].Namespace().String(), ShouldEqual, "/intel/mesos/agent/slave/tasks_running") 161 | So(metrics[1].Namespace().String(), ShouldEqual, "/intel/mesos/agent/system/load_5min") 162 | So(metrics[2].Namespace().Strings()[5], ShouldEqual, "cpus_system_time_secs") 163 | So(metrics[4].Namespace().Strings()[5], ShouldEqual, "disk_used_bytes") 164 | So(metrics[6].Namespace().Strings()[5], ShouldEqual, "mem_total_bytes") 165 | }) 166 | 167 | Convey("Should return an error if an invalid metric was requested", func() { 168 | mts := []plugin.MetricType{ 169 | plugin.MetricType{ 170 | Namespace_: core.NewNamespace("intel", "mesos", "master", "foo", "bar", "baz"), 171 | Config_: cfg.ConfigDataNode, 172 | }, 173 | } 174 | 175 | metrics, err := mc.CollectMetrics(mts) 176 | So(metrics, ShouldBeNil) 177 | So(err, ShouldNotBeNil) 178 | }) 179 | }) 180 | 181 | teardown(cfgItems["master"].(string)) 182 | } 183 | 184 | // setupCfg builds a new ConfigDataNode that specifies the Mesos master and agent host / port 185 | // to use in the integration test(s). 186 | func setupCfg() plugin.ConfigType { 187 | master := os.Getenv("SNAP_MESOS_MASTER") 188 | if master == "" { 189 | master = "127.0.0.1:5050" 190 | } 191 | 192 | agent := os.Getenv("SNAP_MESOS_AGENT") 193 | if agent == "" { 194 | agent = "127.0.0.1:5051" 195 | } 196 | 197 | node := cdata.NewNode() 198 | node.AddItem("master", ctypes.ConfigValueStr{Value: master}) 199 | node.AddItem("agent", ctypes.ConfigValueStr{Value: agent}) 200 | 201 | return plugin.ConfigType{ConfigDataNode: node} 202 | 203 | } 204 | 205 | // Launch some Mesos tasks 206 | func launchTasks(masterHost string, agentHost string) { 207 | cmd := "mesos" 208 | id := time.Now().Unix() 209 | task1Args := []string{ 210 | "execute", fmt.Sprintf("--master=%s", masterHost), 211 | fmt.Sprintf("--name=sleep.%v", id), 212 | "--resources=cpus:0.5;mem:64;disk:32", 213 | "--command=date && sleep 60", 214 | } 215 | task2Args := []string{ 216 | "execute", fmt.Sprintf("--master=%s", masterHost), 217 | fmt.Sprintf("--name=sleep: %v", id), 218 | "--resources=cpus:0.5;mem:64;disk:32", 219 | "--command=date && sleep 60", 220 | } 221 | 222 | launch := func(cmd *exec.Cmd) { 223 | var stderr bytes.Buffer 224 | cmd.Stderr = &stderr 225 | err := cmd.Run() 226 | if err != nil { 227 | fmt.Println(stderr.String()) 228 | panic(err) 229 | } 230 | } 231 | 232 | go launch(exec.Command(cmd, task1Args...)) 233 | go launch(exec.Command(cmd, task2Args...)) 234 | 235 | // There is a delay until disk usage information is available for a newly-launched executor. See: 236 | // * https://github.com/apache/mesos/blob/0.28.1/src/slave/containerizer/mesos/isolators/posix/disk.cpp#L352-L357 237 | // * https://github.com/apache/mesos/blob/0.28.1/src/tests/disk_quota_tests.cpp#L496-L514 238 | // 239 | // Therefore, this function needs to fetch statistics for the new executors it just created and block until 240 | // the disk usage metrics are available. Otherwise, we'll see some flakiness in the integration tests: 241 | // 242 | // * /home/vagrant/work/src/github.com/intelsdi-x/snap-plugin-collector-mesos/mesos/mesos_integration_test.go 243 | // Line 157: 244 | // Expected: '8' 245 | // Actual: '6' 246 | // (Should be equal) 247 | // 248 | done := map[string]bool{} 249 | for len(done) != 2 { 250 | executors, err := agent.GetMonitoringStatistics(agentHost) 251 | if err != nil { 252 | panic(err) 253 | } 254 | if len(executors) != 2 { 255 | time.Sleep(1) 256 | continue 257 | } 258 | for _, exec := range executors { 259 | if done[exec.ID] != true { 260 | if exec.Statistics.DiskUsedBytes != nil { 261 | done[exec.ID] = true 262 | } else { 263 | time.Sleep(1) 264 | } 265 | } 266 | } 267 | } 268 | } 269 | 270 | // Get the system to a clean state by tearing down all active frameworks on the Mesos master, thus killing all tasks. 271 | func teardown(host string) { 272 | u := url.URL{Scheme: "http", Host: host, Path: "/master/teardown"} 273 | frameworks, err := master.GetFrameworks(host) 274 | if err != nil { 275 | panic(err) 276 | } 277 | 278 | for _, framework := range frameworks { 279 | formData := url.Values{} 280 | formData.Add("frameworkId", framework.ID) 281 | resp, err := http.PostForm(u.String(), formData) 282 | if err != nil { 283 | panic(err) 284 | } 285 | if resp.StatusCode != http.StatusOK { 286 | panic(fmt.Errorf("Expected HTTP 200, got %d", resp.StatusCode)) 287 | } 288 | } 289 | time.Sleep(1) 290 | } 291 | -------------------------------------------------------------------------------- /mesos/mesos_test.go: -------------------------------------------------------------------------------- 1 | // +build small 2 | 3 | /* 4 | Copyright 2016 Intel Corporation 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package mesos 20 | 21 | import ( 22 | "testing" 23 | 24 | log "github.com/Sirupsen/logrus" 25 | "github.com/intelsdi-x/snap/control/plugin" 26 | "github.com/intelsdi-x/snap/core/cdata" 27 | "github.com/intelsdi-x/snap/core/ctypes" 28 | . "github.com/smartystreets/goconvey/convey" 29 | ) 30 | 31 | func TestMesosPlugin(t *testing.T) { 32 | Convey("Meta should return metadata for the plugin", t, func() { 33 | meta := Meta() 34 | So(meta.Name, ShouldResemble, pluginName) 35 | So(meta.Version, ShouldResemble, pluginVersion) 36 | So(meta.Type, ShouldResemble, pluginType) 37 | }) 38 | 39 | Convey("Create Mesos collector", t, func() { 40 | mc := NewMesosCollector() 41 | Convey("MesosCollector should not be nil", func() { 42 | So(mc, ShouldNotBeNil) 43 | }) 44 | Convey("MesosCollector should be of type Mesos", func() { 45 | So(mc, ShouldHaveSameTypeAs, &Mesos{}) 46 | }) 47 | }) 48 | } 49 | 50 | func TestMesos_getConfig(t *testing.T) { 51 | log.SetLevel(log.ErrorLevel) // Suppress warning messages from getConfig 52 | 53 | Convey("Get plugin configuration from snap global config", t, func() { 54 | Convey("When only a master is provided, getConfig() should return only the master value", func() { 55 | node := cdata.NewNode() 56 | node.AddItem("master", ctypes.ConfigValueStr{Value: "mesos-master.example.com:5050"}) 57 | snapCfg := plugin.ConfigType{ConfigDataNode: node} 58 | 59 | parsedCfg, err := getConfig(snapCfg) 60 | 61 | So(parsedCfg["master"], ShouldEqual, "mesos-master.example.com:5050") 62 | So(parsedCfg["agent"], ShouldEqual, "") 63 | So(err, ShouldBeNil) 64 | }) 65 | 66 | Convey("When only an agent is provided, getConfig() should return only the agent value", func() { 67 | node := cdata.NewNode() 68 | node.AddItem("agent", ctypes.ConfigValueStr{Value: "mesos-agent.example.com:5051"}) 69 | snapCfg := plugin.ConfigType{ConfigDataNode: node} 70 | 71 | parsedCfg, err := getConfig(snapCfg) 72 | 73 | So(parsedCfg["master"], ShouldEqual, "") 74 | So(parsedCfg["agent"], ShouldEqual, "mesos-agent.example.com:5051") 75 | So(err, ShouldBeNil) 76 | }) 77 | 78 | Convey("When both a master and an agent are provided, getConfig() should return both values", func() { 79 | node := cdata.NewNode() 80 | node.AddItem("master", ctypes.ConfigValueStr{Value: "mesos-master.example.com:5050"}) 81 | node.AddItem("agent", ctypes.ConfigValueStr{Value: "mesos-agent.example.com:5051"}) 82 | snapCfg := plugin.ConfigType{ConfigDataNode: node} 83 | 84 | parsedCfg, err := getConfig(snapCfg) 85 | 86 | So(len(parsedCfg), ShouldEqual, 2) 87 | So(err, ShouldBeNil) 88 | }) 89 | 90 | Convey("When both master and agent are missing, getConfig() should return an error", func() { 91 | node := cdata.NewNode() 92 | node.AddItem("foo", ctypes.ConfigValueStr{Value: "bar"}) 93 | snapCfg := plugin.ConfigType{ConfigDataNode: node} 94 | 95 | parsedCfg, err := getConfig(snapCfg) 96 | 97 | So(len(parsedCfg), ShouldEqual, 0) 98 | So(err, ShouldNotBeNil) 99 | }) 100 | }) 101 | } 102 | -------------------------------------------------------------------------------- /scripts/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # File managed by pluginsync 3 | 4 | # http://www.apache.org/licenses/LICENSE-2.0.txt 5 | # 6 | # 7 | # Copyright 2016 Intel Corporation 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | set -e 22 | set -u 23 | set -o pipefail 24 | 25 | __dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 26 | __proj_dir="$(dirname "$__dir")" 27 | 28 | # shellcheck source=scripts/common.sh 29 | . "${__dir}/common.sh" 30 | 31 | plugin_name=${__proj_dir##*/} 32 | build_dir="${__proj_dir}/build" 33 | go_build=(go build -ldflags "-w") 34 | 35 | _info "project path: ${__proj_dir}" 36 | _info "plugin name: ${plugin_name}" 37 | 38 | export CGO_ENABLED=0 39 | 40 | # rebuild binaries: 41 | _debug "removing: ${build_dir:?}/*" 42 | rm -rf "${build_dir:?}/"* 43 | 44 | _info "building plugin: ${plugin_name}" 45 | export GOOS=linux 46 | export GOARCH=amd64 47 | mkdir -p "${build_dir}/${GOOS}/x86_64" 48 | "${go_build[@]}" -o "${build_dir}/${GOOS}/x86_64/${plugin_name}" . || exit 1 49 | -------------------------------------------------------------------------------- /scripts/common.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # File managed by pluginsync 3 | 4 | # http://www.apache.org/licenses/LICENSE-2.0.txt 5 | # 6 | # 7 | # Copyright 2016 Intel Corporation 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | set -e 22 | set -u 23 | set -o pipefail 24 | 25 | LOG_LEVEL="${LOG_LEVEL:-6}" 26 | NO_COLOR="${NO_COLOR:-}" 27 | NO_GO_TEST=${NO_GO_TEST:-'-not -path "./.*" -not -path "*/_*" -not -path "./Godeps/*" -not -path "./vendor/*"'} 28 | 29 | trap_exitcode() { 30 | exit $? 31 | } 32 | 33 | trap trap_exitcode SIGINT 34 | 35 | _fmt () { 36 | local color_debug="\x1b[35m" 37 | local color_info="\x1b[32m" 38 | local color_notice="\x1b[34m" 39 | local color_warning="\x1b[33m" 40 | local color_error="\x1b[31m" 41 | local colorvar=color_$1 42 | 43 | local color="${!colorvar:-$color_error}" 44 | local color_reset="\x1b[0m" 45 | if [ "${NO_COLOR}" = "true" ] || [[ "${TERM:-}" != "xterm"* ]] || [ -t 1 ]; then 46 | # Don't use colors on pipes or non-recognized terminals 47 | color=""; color_reset="" 48 | fi 49 | echo -e "$(date -u +"%Y-%m-%d %H:%M:%S UTC") ${color}$(printf "[%9s]" "${1}")${color_reset}"; 50 | } 51 | 52 | _debug () { [ "${LOG_LEVEL}" -ge 7 ] && echo "$(_fmt debug) ${*}" 1>&2 || true; } 53 | _info () { [ "${LOG_LEVEL}" -ge 6 ] && echo "$(_fmt info) ${*}" 1>&2 || true; } 54 | _notice () { [ "${LOG_LEVEL}" -ge 5 ] && echo "$(_fmt notice) ${*}" 1>&2 || true; } 55 | _warning () { [ "${LOG_LEVEL}" -ge 4 ] && echo "$(_fmt warning) ${*}" 1>&2 || true; } 56 | _error () { [ "${LOG_LEVEL}" -ge 3 ] && echo "$(_fmt error) ${*}" 1>&2 || true; exit 1; } 57 | 58 | _test_files() { 59 | local test_files=$(sh -c "find . -type f -name '*.go' ${NO_GO_TEST} -print") 60 | _debug "go source files ${test_files}" 61 | echo "${test_files}" 62 | } 63 | 64 | _test_dirs() { 65 | local test_dirs=$(sh -c "find . -type f -name '*.go' ${NO_GO_TEST} -print0" | xargs -0 -n1 dirname | sort -u) 66 | _debug "go code directories ${test_dirs}" 67 | echo "${test_dirs}" 68 | } 69 | 70 | _go_get() { 71 | local _url=$1 72 | local _util 73 | 74 | _util=$(basename "${_url}") 75 | 76 | type -p "${_util}" > /dev/null || go get "${_url}" && _debug "go get ${_util} ${_url}" 77 | } 78 | 79 | _gofmt() { 80 | test -z "$(gofmt -l -d $(_test_files) | tee /dev/stderr)" 81 | } 82 | 83 | _goimports() { 84 | _go_get golang.org/x/tools/cmd/goimports 85 | test -z "$(goimports -l -d $(_test_files) | tee /dev/stderr)" 86 | } 87 | 88 | _golint() { 89 | _go_get github.com/golang/lint/golint 90 | golint ./... 91 | } 92 | 93 | _go_vet() { 94 | go vet $(_test_dirs) 95 | } 96 | 97 | _go_race() { 98 | go test -race ./... 99 | } 100 | 101 | _go_test() { 102 | _info "running test type: ${TEST_TYPE}" 103 | # Standard go tooling behavior is to ignore dirs with leading underscors 104 | for dir in $(_test_dirs); 105 | do 106 | if [[ -z ${go_cover+x} ]]; then 107 | _debug "running go test with cover in ${dir}" 108 | go test -v --tags="${TEST_TYPE}" -covermode=count -coverprofile="${dir}/profile.tmp" "${dir}" 109 | if [ -f "${dir}/profile.tmp" ]; then 110 | tail -n +2 "${dir}/profile.tmp" >> profile.cov 111 | rm "${dir}/profile.tmp" 112 | fi 113 | else 114 | _debug "running go test without cover in ${dir}" 115 | go test -v --tags="${TEST_TYPE}" "${dir}" 116 | fi 117 | done 118 | } 119 | 120 | _go_cover() { 121 | go tool cover -func profile.cov 122 | } 123 | -------------------------------------------------------------------------------- /scripts/deps.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # File managed by pluginsync 3 | 4 | # http://www.apache.org/licenses/LICENSE-2.0.txt 5 | # 6 | # 7 | # Copyright 2016 Intel Corporation 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | set -e 22 | set -u 23 | set -o pipefail 24 | 25 | __dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 26 | __proj_dir="$(dirname "$__dir")" 27 | 28 | # shellcheck source=scripts/common.sh 29 | . "${__dir}/common.sh" 30 | 31 | detect_go_dep() { 32 | [[ -f "${__proj_dir}/Godeps/Godeps.json" ]] && _dep='godep' 33 | [[ -f "${__proj_dir}/glide.yaml" ]] && _dep='glide' 34 | [[ -f "${__proj_dir}/vendor/vendor.json" ]] && _dep='govendor' 35 | _info "golang dependency tool: ${_dep}" 36 | echo "${_dep}" 37 | } 38 | 39 | install_go_dep() { 40 | local _dep=${_dep:=$(_detect_dep)} 41 | _info "ensuring ${_dep} is available" 42 | case $_dep in 43 | godep) 44 | _go_get github.com/tools/godep 45 | ;; 46 | glide) 47 | _go_get github.com/Masterminds/glide 48 | ;; 49 | govendor) 50 | _go_get github.com/kardianos/govendor 51 | ;; 52 | esac 53 | } 54 | 55 | restore_go_dep() { 56 | local _dep=${_dep:=$(_detect_dep)} 57 | _info "restoring dependency with ${_dep}" 58 | case $_dep in 59 | godep) 60 | (cd "${__proj_dir}" && godep restore) 61 | ;; 62 | glide) 63 | (cd "${__proj_dir}" && glide install) 64 | ;; 65 | govendor) 66 | (cd "${__proj_dir}" && govendor sync) 67 | ;; 68 | esac 69 | } 70 | 71 | _dep=$(detect_go_dep) 72 | install_go_dep 73 | restore_go_dep 74 | -------------------------------------------------------------------------------- /scripts/large.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # File managed by pluginsync 3 | 4 | set -e 5 | set -u 6 | set -o pipefail 7 | 8 | __dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 9 | __proj_dir="$(dirname "$__dir")" 10 | __proj_name="$(basename "$__proj_dir")" 11 | 12 | # shellcheck source=scripts/common.sh 13 | . "${__dir}/common.sh" 14 | 15 | _verify_docker() { 16 | type -p docker > /dev/null 2>&1 || _error "docker needs to be installed" 17 | type -p docker-compose > /dev/null 2>&1 || _error "docker-compose needs to be installed" 18 | docker version >/dev/null 2>&1 || _error "docker needs to be configured/running" 19 | } 20 | 21 | _verify_docker 22 | 23 | [[ -f "${__proj_dir}/build/linux/x86_64/${__proj_name}" ]] || (cd "${__proj_dir}" && make) 24 | 25 | SNAP_VERSION=${SNAP_VERSION:-"latest"} 26 | OS=${OS:-"alpine"} 27 | PLUGIN_PATH=${PLUGIN_PATH:-"${__proj_dir}"} 28 | DEMO=${DEMO:-"false"} 29 | TASK=${TASK:-""} 30 | 31 | if [[ ${DEBUG:-} == "true" ]]; then 32 | cmd="cd /plugin/scripts && rescue rspec ./test/*_spec.rb" 33 | else 34 | cmd="cd /plugin/scripts && rspec ./test/*_spec.rb" 35 | fi 36 | 37 | _info "running large test" 38 | docker run -v /var/run/docker.sock:/var/run/docker.sock -v "${__proj_dir}":/plugin -e DEMO="${DEMO}" -e TASK="${TASK}" -e PLUGIN_PATH="${PLUGIN_PATH}" -e SNAP_VERSION="${SNAP_VERSION}" -e OS="${OS}" -ti intelsdi/serverspec:alpine /bin/sh -c "${cmd}" 39 | -------------------------------------------------------------------------------- /scripts/medium.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -u 5 | set -o pipefail 6 | 7 | TRAVIS=${TRAVIS:-} 8 | 9 | if [[ $TRAVIS == "true" ]]; then 10 | _info "Provisioning Mesos master and agent..." 11 | export SNAP_MESOS_MASTER="127.0.0.1:5050" 12 | export SNAP_MESOS_AGENT="127.0.0.1:5051" 13 | 14 | sudo ./scripts/provision-travis.sh --mesos_release "${MESOS_RELEASE}" --ip_address 127.0.0.1 15 | 16 | UNIT_TEST="go_test" 17 | test_unit 18 | else 19 | _info "Not running in Travis CI. Skipping medium test." 20 | fi 21 | -------------------------------------------------------------------------------- /scripts/pre_deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # File managed by pluginsync 3 | 4 | # http://www.apache.org/licenses/LICENSE-2.0.txt 5 | # 6 | # 7 | # Copyright 2016 Intel Corporation 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | set -e 22 | set -u 23 | set -o pipefail 24 | 25 | __dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 26 | __proj_dir="$(dirname "$__dir")" 27 | 28 | # shellcheck source=scripts/common.sh 29 | . "${__dir}/common.sh" 30 | 31 | build_path="${__proj_dir}/build" 32 | _info "build_path: ${build_path}" 33 | _debug "$(find "${build_path}")" 34 | 35 | plugin_name="${__proj_dir##*/}" 36 | git_sha=$(git log --pretty=format:"%H" -1) 37 | s3_path="${__proj_dir}/s3/${plugin_name}" 38 | 39 | set +u 40 | if [ -z "$TRAVIS_TAG" ]; then 41 | set -u 42 | git_path="${s3_path}/${git_sha}" 43 | latest_path="${s3_path}/latest_build" 44 | mkdir -p "${git_path}" 45 | mkdir -p "${latest_path}" 46 | 47 | _info "copying plugin binaries to ${git_path}" 48 | cp -rp "${build_path}/"* "${git_path}" 49 | _info "copying plugin binaries to ${latest_path}" 50 | cp -rp "${build_path}/"* "${latest_path}" 51 | else 52 | set -u 53 | tag_path="${s3_path}/${TRAVIS_TAG}" 54 | latest_path="${s3_path}/latest" 55 | mkdir -p "${tag_path}" 56 | mkdir -p "${latest_path}" 57 | 58 | _info "copying plugin binaries to ${tag_path}" 59 | cp -rp "${build_path}/"* "${tag_path}" 60 | _info "copying plugin binaries to ${latest_path}" 61 | cp -rp "${build_path}/"* "${latest_path}" 62 | fi 63 | 64 | release_path="${SNAP_PATH:-"${__proj_dir}/release"}" 65 | mkdir -p "${release_path}" 66 | 67 | _info "moving plugin binaries to ${release_path}" 68 | 69 | for file in "${build_path}"/**/*/snap-plugin-* ; do 70 | filename="${file##*/}" 71 | parent="${file%/*}" 72 | arch="${parent##*/}" 73 | parent="${parent%/*}" 74 | os="${parent##*/}" 75 | cp "${file}" "${release_path}/${filename}_${os}_${arch}" 76 | done 77 | 78 | _debug "$(find "${build_path}")" 79 | _debug "$(find "${s3_path}")" 80 | _debug "$(find "${release_path}")" 81 | -------------------------------------------------------------------------------- /scripts/provision-travis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # This script provisions a single-node Mesos cluster that can be used for 3 | # running integration tests for the Mesos plugin in Travis CI. 4 | 5 | if [[ $(id -u) -ne 0 ]]; then 6 | echo "Please re-run this script as root." 7 | exit 1 8 | fi 9 | 10 | DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]') 11 | CODENAME=$(lsb_release -cs) 12 | 13 | function parse_args { 14 | while [[ $# > 1 ]]; do 15 | case "$1" in 16 | --mesos_release) MESOS_RELEASE="$2" ; shift ;; 17 | --ip_address) IP_ADDRESS="${2:-127.0.0.1}" ; shift ;; 18 | --*) echo "Error: invalid option '$1'" ; exit 1 ;; 19 | esac 20 | shift 21 | done 22 | } 23 | 24 | function _install_pkg_with_version { 25 | local name="$1" 26 | local ver="$2" 27 | 28 | if [[ $ver =~ "latest" ]]; then 29 | echo "Installing ${name}..." 30 | apt-get -y install "${name}" 31 | else 32 | echo "Installing ${name} version ${ver}..." 33 | apt-get -y install "${name}=${ver}" 34 | fi 35 | } 36 | 37 | function install_prereqs { 38 | echo "Updating metadata and installing prerequisites..." 39 | apt-get -y update 40 | apt-get -y install apt-transport-https ca-certificates git \ 41 | linux-tools-common linux-tools-generic linux-tools-$(uname -r) 42 | 43 | } 44 | 45 | function install_zookeeper { 46 | echo "Installing ZooKeeper..." 47 | apt-get -y install zookeeperd 48 | echo "1" > /etc/zookeeper/conf/myid 49 | service zookeeper restart 50 | } 51 | 52 | function install_mesos { 53 | echo "Installing Mesosphere repository..." 54 | apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF 55 | echo "deb http://repos.mesosphere.io/${DISTRO} ${CODENAME} main" \ 56 | | tee /etc/apt/sources.list.d/mesosphere.list 57 | 58 | echo "Refreshing metadata..." 59 | apt-get -y update 60 | 61 | _install_pkg_with_version mesos $MESOS_RELEASE 62 | } 63 | 64 | function configure_mesos { 65 | mkdir -p /etc/{mesos,mesos-master,mesos-slave} 66 | 67 | # Master 68 | echo "zk://${IP_ADDRESS}:2181/mesos" > /etc/mesos/zk 69 | echo "${IP_ADDRESS}" > /etc/mesos-master/hostname 70 | echo "${IP_ADDRESS}" > /etc/mesos-master/ip 71 | service mesos-master restart 72 | 73 | # Agent 74 | # Note: there is a known bug in Mesos when using the 'cgroups/perf_event' isolator 75 | # on specific kernels and platforms. For more info, see MESOS-4705. 76 | echo "1secs" > /etc/mesos-slave/container_disk_watch_interval 77 | echo "mesos" > /etc/mesos-slave/containerizers 78 | echo "${IP_ADDRESS}" > /etc/mesos-slave/hostname 79 | echo "${IP_ADDRESS}" > /etc/mesos-slave/ip 80 | echo "cgroups/cpu,cgroups/mem,cgroups/perf_event,posix/disk" > /etc/mesos-slave/isolation 81 | echo "cpu-clock,task-clock,context-switches" > /etc/mesos-slave/perf_events 82 | echo "/var/lib/mesos" > /etc/mesos-slave/work_dir 83 | service mesos-slave restart 84 | 85 | cat << END 86 | -------------------------------------------------- 87 | Mesos version ${MESOS_RELEASE} has been installed. 88 | 89 | * Master: http://${IP_ADDRESS}:5050 90 | * Agent: http://${IP_ADDRESS}:5051 91 | 92 | -------------------------------------------------- 93 | END 94 | } 95 | 96 | function main { 97 | parse_args "$@" 98 | install_prereqs 99 | 100 | install_zookeeper 101 | install_mesos 102 | configure_mesos 103 | } 104 | 105 | main "$@" 106 | -------------------------------------------------------------------------------- /scripts/provision-vagrant.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This script provisions a single-node Mesos cluster that can be used for 3 | # developing and testing the Mesos plugin for the Snap telemetry framework. 4 | set -e 5 | 6 | if [[ $(id -u) -ne 0 ]]; then 7 | echo "Please re-run this script as root." 8 | exit 1 9 | fi 10 | 11 | DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]') 12 | CODENAME=$(lsb_release -cs) 13 | 14 | function parse_args { 15 | while [[ $# > 1 ]]; do 16 | case "$1" in 17 | --mesos_release) MESOS_RELEASE="$2" ; shift ;; 18 | --marathon_release) MARATHON_RELEASE="$2" ; shift ;; 19 | --golang_release) GOLANG_RELEASE="$2" ; shift ;; 20 | --snap_release) SNAP_RELEASE="$2" ; shift ;; 21 | --ip_address) IP_ADDRESS="${2:-127.0.0.1}" ; shift ;; 22 | --*) echo "Error: invalid option '$1'" ; exit 1 ;; 23 | esac 24 | shift 25 | done 26 | } 27 | 28 | function _install_pkg_with_version { 29 | local name="$1" 30 | local ver="$2" 31 | 32 | if [[ $ver =~ "latest" ]]; then 33 | echo "Installing ${name}..." 34 | apt-get -y install "${name}" 35 | else 36 | echo "Installing ${name} version ${ver}..." 37 | apt-get -y install "${name}=${ver}" 38 | fi 39 | } 40 | 41 | function install_prereqs { 42 | echo "Updating metadata and installing prerequisites..." 43 | apt-get -y update 44 | apt-get -y install apt-transport-https ca-certificates git \ 45 | linux-tools-common linux-tools-generic linux-tools-$(uname -r) 46 | echo debconf shared/accepted-oracle-license-v1-1 select true | sudo debconf-set-selections 47 | echo debconf shared/accepted-oracle-license-v1-1 seen true | sudo debconf-set-selections 48 | apt-get install -y oracle-java8-installer oracle-java8-set-default 49 | 50 | } 51 | 52 | function configure_repos { 53 | echo "Installing Mesosphere repository..." 54 | # We use hkp://keyserver.ubuntu.com:80 to work around corporate firewalls 55 | # that block the native HKP port 11371. Since HKP is a higher-level protocol 56 | # over HTTP, this should be fine, but *could* pose a problem if deep packet 57 | # inspection is enabled. -- roger, 2016/06/02 58 | apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E56151BF 59 | echo "deb http://repos.mesosphere.io/${DISTRO} ${CODENAME} main" \ 60 | | tee /etc/apt/sources.list.d/mesosphere.list 61 | 62 | echo "Installing InfluxDB repository..." 63 | curl -ssL https://repos.influxdata.com/influxdb.key | apt-key add - 64 | echo "deb https://repos.influxdata.com/${DISTRO} ${CODENAME} stable" \ 65 | | tee /etc/apt/sources.list.d/influxdb.list 66 | 67 | echo "Installing Grafana repository..." 68 | curl -ssL https://packagecloud.io/gpg.key | apt-key add - 69 | echo "deb https://packagecloud.io/grafana/stable/debian/ wheezy main" \ 70 | | tee /etc/apt/sources.list.d/grafana.list 71 | 72 | echo "Installing Java repo" 73 | add-apt-repository -y ppa:webupd8team/java 74 | echo "Refreshing metadata..." 75 | apt-get -y update 76 | } 77 | 78 | function install_zookeeper { 79 | echo "Installing ZooKeeper..." 80 | apt-get -y install zookeeperd 81 | echo "1" > /etc/zookeeper/conf/myid 82 | service zookeeper restart 83 | } 84 | 85 | function install_mesos { 86 | _install_pkg_with_version mesos $MESOS_RELEASE 87 | } 88 | 89 | function install_marathon { 90 | echo "Installing Marathon ..." 91 | _install_pkg_with_version marathon $MARATHON_RELEASE 92 | service marathon restart 93 | } 94 | 95 | function configure_mesos { 96 | mkdir -p /etc/{mesos,mesos-master,mesos-slave} 97 | 98 | # Master 99 | echo "zk://${IP_ADDRESS}:2181/mesos" > /etc/mesos/zk 100 | echo "${IP_ADDRESS}" > /etc/mesos-master/hostname 101 | echo "${IP_ADDRESS}" > /etc/mesos-master/ip 102 | service mesos-master restart 103 | 104 | # Agent 105 | # Note: there is a known bug in Mesos when using the 'cgroups/perf_event' isolator 106 | # on specific kernels and platforms. For more info, see MESOS-4705. 107 | echo "1secs" > /etc/mesos-slave/container_disk_watch_interval 108 | echo "mesos" > /etc/mesos-slave/containerizers 109 | echo "${IP_ADDRESS}" > /etc/mesos-slave/hostname 110 | echo "${IP_ADDRESS}" > /etc/mesos-slave/ip 111 | echo "cgroups/cpu,cgroups/mem,cgroups/perf_event,posix/disk" > /etc/mesos-slave/isolation 112 | echo "cpu-clock,task-clock,context-switches" > /etc/mesos-slave/perf_events 113 | echo "/var/lib/mesos" > /etc/mesos-slave/work_dir 114 | service mesos-slave restart 115 | 116 | # Add environment variables needed for local integration testing 117 | echo "export SNAP_MESOS_MASTER=${IP_ADDRESS}:5050" >> /etc/profile 118 | echo "export SNAP_MESOS_AGENT=${IP_ADDRESS}:5051" >> /etc/profile 119 | 120 | cat << END 121 | -------------------------------------------------- 122 | Mesos version ${MESOS_RELEASE} has been installed. 123 | 124 | * Master: http://${IP_ADDRESS}:5050 125 | * Agent: http://${IP_ADDRESS}:5051 126 | 127 | -------------------------------------------------- 128 | END 129 | } 130 | 131 | function install_golang { 132 | local GOROOT="/usr/local" 133 | 134 | if [[ -d "${GOROOT}/go" ]]; then 135 | echo "Found an existing Go installation at ${GOROOT}/go. Skipping install..." 136 | return 137 | fi 138 | 139 | echo "Installing Go ${GOLANG_RELEASE}..." 140 | local GOLANG_URL="https://storage.googleapis.com/golang" 141 | local GOLANG_FILENAME="go${GOLANG_RELEASE}.linux-amd64.tar.gz" 142 | 143 | local GOPATH="/home/vagrant/work" 144 | 145 | curl -sLO "${GOLANG_URL}/${GOLANG_FILENAME}" 146 | tar zxf $GOLANG_FILENAME -C $GOROOT 147 | 148 | echo "export GOPATH=${GOPATH}" >> /etc/profile 149 | echo "export PATH=\${PATH}:\${GOPATH}/bin:${GOROOT}/go/bin" >> /etc/profile 150 | echo "export PATH=\${PATH}:\${GOPATH}/bin" >> /etc/profile 151 | 152 | mkdir -p "${GOPATH}/src/github.com/intelsdi-x" && chown -R vagrant:vagrant $GOPATH 153 | ln -fs /vagrant "${GOPATH}/src/github.com/intelsdi-x/snap-plugin-collector-mesos" 154 | 155 | # Bring in godep so we don't have to do this manually each time 156 | . /etc/profile 157 | go get github.com/tools/godep 158 | 159 | cat << END 160 | -------------------------------------------------------------- 161 | Go ${GOLANG_RELEASE} has been installed to /usr/local/go. 162 | /usr/local/go/bin has been appended to \$PATH in /etc/profile. 163 | /home/vagrant/work has been set as the \$GOPATH. 164 | 165 | For more information on getting started with Go, see 166 | https://golang.org/doc/code.html 167 | 168 | -------------------------------------------------------------- 169 | END 170 | } 171 | 172 | function install_snap { 173 | local SNAP_PATH="/usr/local/snap" 174 | if [[ -d $SNAP_PATH ]]; then 175 | echo "Found an existing Snap installation at ${SNAP_PATH}. Skipping install..." 176 | return 177 | fi 178 | 179 | echo "Installing snap ${SNAP_RELEASE}..." 180 | local SNAP_URL="https://github.com/intelsdi-x/snap/releases/download" 181 | local SNAP_FILENAME="snap-${SNAP_RELEASE}-linux-amd64.tar.gz" 182 | local SNAP_PLUGINS_FILENAME="snap-plugins-${SNAP_RELEASE}-linux-amd64.tar.gz" 183 | curl -sLO "${SNAP_URL}/${SNAP_RELEASE}/${SNAP_FILENAME}" 184 | curl -sLO "${SNAP_URL}/${SNAP_RELEASE}/${SNAP_PLUGINS_FILENAME}" 185 | 186 | echo "export SNAP_PATH=${SNAP_PATH}" >> /etc/profile 187 | echo 'export PATH=${PATH}:${SNAP_PATH}/bin' >> /etc/profile 188 | mkdir -p $SNAP_PATH 189 | tar zxf $SNAP_FILENAME --strip-components=1 -C $SNAP_PATH 190 | tar zxf $SNAP_PLUGINS_FILENAME --strip-components=1 -C $SNAP_PATH 191 | 192 | cat << END 193 | ------------------------------------------------------------------------ 194 | Snap version ${SNAP_RELEASE} has been installed to ${SNAP_PATH}. 195 | ${SNAP_PATH}/bin has been appended to \$PATH in /etc/profile. 196 | The Snap plugins have also been installed to ${SNAP_PATH}/plugin. 197 | 198 | When you're ready, you can start the snap daemon by running: 199 | 200 | snapteld --plugin-trust 0 --log-level 1 --auto-discover \\ 201 | "${SNAP_PATH}/plugin" >> /var/log/snap.log 2>&1 & 202 | 203 | or something similar. 204 | 205 | ------------------------------------------------------------------------ 206 | END 207 | } 208 | 209 | function install_influxdb { 210 | apt-get install influxdb 211 | service influxdb restart 212 | 213 | # Wait for InfluxDB to start before proceeding 214 | echo "Waiting for InfluxDB to start..." 215 | while ! curl -sG "http://localhost:8086/query?u=admin&p=admin" \ 216 | --data-urlencode "q=SHOW DATABASES" > /dev/null 2>&1; do 217 | echo -n "." 218 | sleep 1 219 | done 220 | echo 221 | 222 | echo "Creating 'snap' database in InfluxDB..." 223 | # Create snap database in InfluxDB 224 | curl -sG -X POST "http://localhost:8086/query?u=admin&p=admin" \ 225 | --data-urlencode "q=CREATE DATABASE snap" 226 | } 227 | 228 | function install_grafana { 229 | apt-get install grafana 230 | update-rc.d grafana-server defaults 95 10 231 | service grafana-server restart 232 | 233 | echo "Configuring Grafana..." 234 | local COOKIEJAR=$(mktemp) 235 | 236 | curl -s -H 'Content-Type: application/json; charset=UTF-8' \ 237 | --data-binary '{"user": "admin", "email": "", "password": "admin"}' \ 238 | --cookie-jar "$COOKIEJAR" http://localhost:3000/login 239 | echo 240 | 241 | echo "Adding InfluxDB datasource 'influx' ..." 242 | curl -s -H 'Content-Type: application/json; charset=UTF-8' \ 243 | --cookie "$COOKIEJAR" --data-binary '{"name": "influx", "type": "influxdb", "url": "http://localhost:8086", "access": "proxy", "database": "snap"}"' \ 244 | http://localhost:3000/api/datasources 245 | echo 246 | 247 | echo "Importing Grafana dashboard 'mesos.json' ..." 248 | curl -s -H 'Content-Type: application/json; charset=UTF-8' --cookie "$COOKIEJAR" \ 249 | --data @/vagrant/examples/grafana/mesos.json http://localhost:3000/api/dashboards/db 250 | echo 251 | } 252 | 253 | function main { 254 | parse_args "$@" 255 | configure_repos 256 | install_prereqs 257 | 258 | install_zookeeper 259 | install_mesos 260 | configure_mesos 261 | 262 | install_golang 263 | install_snap 264 | 265 | install_influxdb 266 | install_grafana 267 | 268 | install_marathon 269 | } 270 | 271 | main "$@" 272 | -------------------------------------------------------------------------------- /scripts/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # The script does automatic checking on a Go package and its sub-packages, including: 3 | # - gofmt (https://golang.org/cmd/gofmt) 4 | # - goimports (https://godoc.org/cmd/goimports) 5 | # - go vet (https://golang.org/cmd/vet) 6 | # - test coverage (https://blog.golang.org/cover) 7 | 8 | COVERALLS_MAX_ATTEMPTS=5 9 | TEST_DIRS="main.go mesos/" 10 | PKG_DIRS=". ./mesos/..." 11 | IGNORE_PKGS="mesos_pb2" 12 | 13 | function _gofmt { 14 | echo "Running 'gofmt'" 15 | test -z "$(gofmt -l -d $TEST_DIRS | tee /dev/stderr)" 16 | } 17 | 18 | function _goimports { 19 | echo "Running 'goimports'" 20 | go get golang.org/x/tools/cmd/goimports 21 | test -z "$(goimports -l -d $TEST_DIRS | tee /dev/stderr)" 22 | } 23 | 24 | function _govet { 25 | echo "Running 'go vet'" 26 | go vet $PKG_DIRS 27 | } 28 | 29 | function _unit_test_with_coverage { 30 | echo "Running unit tests..." 31 | 32 | go get github.com/smartystreets/goconvey/convey 33 | go get golang.org/x/tools/cmd/cover 34 | 35 | # As of Go 1.6, we cannot use the test profile flag with multiple packages. 36 | # Therefore, we run 'go test' for each package, and concatenate the results 37 | # into 'profile.cov'. 38 | echo "mode: count" > profile.cov 39 | mkdir -p ./tmp 40 | 41 | for import_path in $(go list -f={{.ImportPath}} ${PKG_DIRS}); do 42 | package=$(basename ${import_path}) 43 | [[ "$IGNORE_PKGS" =~ $package ]] && continue 44 | go test -v --tags=small -covermode=count -coverprofile=./tmp/profile_${package}.cov $import_path 45 | done 46 | 47 | for f in ./tmp/profile_*.cov; do 48 | tail -n +2 ${f} >> profile.cov 49 | done 50 | 51 | rm -rf ./tmp 52 | go tool cover -func profile.cov 53 | } 54 | 55 | function _submit_to_coveralls { 56 | # Only submit to Coveralls.io if we're running in Travis CI. We don't want 57 | # this happening on dev machines! Note that the Coveralls repo token is 58 | # available via the $COVERALLS_REPO_TOKEN environment variable, which is 59 | # configured for the project in the Travis CI web interface. 60 | if [[ $TRAVIS == "true" && $COVERALLS_REPO_TOKEN != "" ]]; then 61 | go get github.com/mattn/goveralls 62 | 63 | for attempt in {1..${COVERALLS_MAX_ATTEMPTS}}; do 64 | echo "Posting test coverage to Coveralls, attempt ${attempt} of ${COVERALLS_MAX_ATTEMPTS}" 65 | goveralls -v -coverprofile=profile.cov -service=travis-ci -repotoken ${COVERALLS_REPO_TOKEN} && break 66 | done 67 | else 68 | echo "Not running in Travis CI (environment variables unset), not posting test coverage to Coveralls!" 69 | fi 70 | } 71 | 72 | function _integration_test { 73 | echo "Running integration tests..." 74 | 75 | if [[ $TRAVIS == "true" ]]; then 76 | echo "Detected that we're running in Travis CI. Provisioning Mesos master and agent..." 77 | export SNAP_MESOS_MASTER="127.0.0.1:5050" 78 | export SNAP_MESOS_AGENT="127.0.0.1:5051" 79 | 80 | sudo ./scripts/provision-travis.sh --mesos_release ${MESOS_RELEASE} --ip_address 127.0.0.1 81 | 82 | sleep 10 83 | set +e 84 | sudo service mesos-master status 85 | sudo service mesos-slave status 86 | sudo tail -100 /var/log/syslog 87 | curl http://127.0.0.1:5050 88 | curl http://127.0.0.1:5051 89 | set -e 90 | else 91 | echo "Detected that we aren't running in Travis CI. Skipping provisioning of Mesos master and agent..." 92 | fi 93 | 94 | go test -v --tags=medium ./... 95 | } 96 | 97 | function main { 98 | TEST_SUITE="$1" 99 | 100 | if [[ $TEST_SUITE == "small" ]]; then 101 | _gofmt 102 | _goimports 103 | _govet 104 | _unit_test_with_coverage 105 | _submit_to_coveralls 106 | elif [[ $TEST_SUITE == "medium" ]]; then 107 | _integration_test 108 | elif [[ $TEST_SUITE == "build" ]]; then 109 | ./scripts/build.sh 110 | else 111 | echo "Error: unknown test suite ${TEST_SUITE}" 112 | exit 1 113 | fi 114 | } 115 | 116 | main "$@" 117 | -------------------------------------------------------------------------------- /scripts/test/large_spec.rb: -------------------------------------------------------------------------------- 1 | # File managed by pluginsync 2 | 3 | require_relative './spec_helper' 4 | require 'specinfra/backend/docker_compose' 5 | 6 | compose_yml = File.expand_path(File.join(__FILE__, "../docker-compose.yml")) 7 | raise(Exception, "Missing docker-compose file: #{compose_yml}") unless File.exists? compose_yml 8 | 9 | # NOTE: scan docker compose file and pull latest containers: 10 | images = File.readlines(compose_yml).select {|l| l =~ /^\s*image:/} 11 | images = images.collect{|l| l.split('image:').last.strip }.uniq 12 | images.each do |i| 13 | puts `docker pull #{i}` 14 | end 15 | 16 | set :docker_compose_container, :snap 17 | 18 | describe docker_compose(compose_yml) do 19 | 20 | # NOTE: If you need to wait for a service or create a database perform it in setup.rb 21 | setup = File.expand_path(File.join(__FILE__, '../setup.rb')) 22 | eval File.read setup if File.exists? setup 23 | 24 | its_container(:snap) do 25 | describe 'docker-compose.yml run' do 26 | TIMEOUT = 60 27 | 28 | describe "download Snap" do 29 | it { 30 | expect(cmd_with_retry("/opt/snap/bin/snaptel --version", :timeout => TIMEOUT).exit_status).to eq 0 31 | expect(cmd_with_retry("/opt/snap/sbin/snapteld --version", :timeout => TIMEOUT).exit_status).to eq 0 32 | } 33 | end 34 | 35 | if os[:family] == 'alpine' 36 | describe port(8181) do 37 | it { should be_listening } 38 | end 39 | end 40 | 41 | context "load Snap plugins" do 42 | describe command("snaptel plugin list") do 43 | it { load_all_plugins } 44 | its(:exit_status) { should eq 0 } 45 | its(:stdout) { 46 | plugins.each do |p| 47 | _ , name = p 48 | should contain(/#{name}/) 49 | end 50 | } 51 | end 52 | end 53 | 54 | describe file("/opt/snap/sbin/snapteld") do 55 | it { should be_file } 56 | it { should be_executable } 57 | end 58 | 59 | describe file("/opt/snap/bin/snaptel") do 60 | it { should be_file } 61 | it { should be_executable } 62 | end 63 | 64 | describe command("snapteld --version") do 65 | its(:exit_status) { should eq 0 } 66 | its(:stdout) { should contain(/#{ENV['SNAP_VERSION']}/) } 67 | end if ENV['SNAP_VERSION'] =~ /^\d+.\d+.\d+$/ 68 | 69 | SnapUtils.tasks.each do |t| 70 | context "Snap task #{t}" do 71 | task_id = nil 72 | 73 | describe command("snaptel task create -t /plugin/examples/tasks/#{t}") do 74 | its(:exit_status) { should eq 0 } 75 | its(:stdout) { should contain(/Task created/) } 76 | it { 77 | id = subject.stdout.split("\n").find{|l|l=~/^ID:/} 78 | task_id = $1 if id.match(/^ID: (.*)$/) 79 | expect(task_id).to_not be_nil 80 | # NOTE we need a short pause before checking task state in case it fails: 81 | sleep 3 82 | } 83 | end 84 | 85 | describe command("snaptel task list") do 86 | its(:exit_status) { should eq 0 } 87 | its(:stdout) { should contain(/Running/) } 88 | end 89 | 90 | describe "Metrics in running tasks" do 91 | it { 92 | binding.pry if ENV["DEMO"] == "true" 93 | 94 | data = curl_json_api("http://127.0.0.1:8181/v1/tasks") 95 | task = data["body"]["ScheduledTasks"].find{|i| i['id'] == task_id} 96 | expect(task['id']).to eq task_id 97 | data = curl_json_api(task['href']) 98 | collect_metrics = data["body"]["workflow"]["collect"]["metrics"].collect{|k,v| k} 99 | 100 | config = load_yaml(SnapUtils.examples/"tasks/#{t}") 101 | config_metrics = config['workflow']['collect']['metrics'].collect{|k,v| k} 102 | 103 | config_metrics.each do |m| 104 | expect(collect_metrics).to include(m) 105 | end 106 | } 107 | end 108 | 109 | # NOTE: can not use the normal describe command(...) since we need to access task_id 110 | describe "Stop task" do 111 | it { 112 | c = command("snaptel task stop #{task_id}") 113 | expect(c.exit_status).to eq 0 114 | expect(c.stdout).to match(/Task stopped/) 115 | } 116 | end 117 | 118 | describe "Remove task" do 119 | it { 120 | c = command("snaptel task remove #{task_id}") 121 | expect(c.exit_status).to eq 0 122 | expect(c.stdout).to match(/Task removed/) 123 | } 124 | end 125 | end 126 | end 127 | end 128 | end 129 | 130 | # NOTE: If you need to perform additional checks such as database verification it be done at the end: 131 | verify = File.expand_path(File.join(__FILE__, '../verify.rb')) 132 | eval File.read verify if File.exists? verify 133 | end 134 | -------------------------------------------------------------------------------- /scripts/test/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # File managed by pluginsync 2 | 3 | require 'hashie' 4 | require 'json' 5 | require 'pathname' 6 | require 'yaml' 7 | require 'rspec/retry' 8 | require 'dockerspec/serverspec' 9 | 10 | begin 11 | require 'pry' 12 | rescue LoadError 13 | end 14 | 15 | module SnapUtils 16 | def sh(arg) 17 | c = command(arg) 18 | puts c.stderr 19 | puts c.stdout 20 | end 21 | 22 | def build_path 23 | File.expand_path(File.join(__FILE__, '../../../build/linux/x86_64')) 24 | end 25 | 26 | def local_plugins 27 | Dir.chdir(build_path) 28 | @local_plugins ||= Dir.glob("snap-plugin-*") 29 | end 30 | 31 | def load_plugin(type, name, version="latest") 32 | plugin_name = "snap-plugin-#{type}-#{name}" 33 | 34 | # NOTE: use mock2 plugin when mock is requested, in general we should avoid mock collector. 35 | case name 36 | when 'mock' # TODO: revisit how we handle mock plugins 37 | url = "https://s3-us-west-2.amazonaws.com/snap.ci.snap-telemetry.io/snap/#{version}/linux/x86_64/#{plugin_name}2" 38 | when 'mock1', 'mock2', 'mock2-grpc', 'passthru', 'passthru-grpc', 'mock-file', 'mock-file-grpc' 39 | url = "https://s3-us-west-2.amazonaws.com/snap.ci.snap-telemetry.io/snap/#{version}/linux/x86_64/#{plugin_name}" 40 | else 41 | url = "https://s3-us-west-2.amazonaws.com/snap.ci.snap-telemetry.io/plugins/#{plugin_name}/#{version}/linux/x86_64/#{plugin_name}" 42 | end 43 | 44 | if local_plugins.include? plugin_name 45 | command("snaptel plugin load #{build_path}/#{plugin_name}").exit_status 46 | else 47 | command("curl -sfL #{url} -o /opt/snap/plugins/#{plugin_name}").exit_status 48 | command("snaptel plugin load /opt/snap/plugins/#{plugin_name}").exit_status 49 | end 50 | end 51 | 52 | def cmd_with_retry(arg, opt={ :timeout => 30 }) 53 | cmd = command(arg) 54 | while cmd.exit_status != 0 or cmd.stdout == '' and opt[:timeout] > 0 55 | sleep 5 56 | opt[:timeout] -= 5 57 | cmd = command(arg) 58 | end 59 | return cmd 60 | end 61 | 62 | def curl_json_api(url) 63 | output = cmd_with_retry("curl #{url}").stdout 64 | if output.size > 0 65 | JSON.parse(output) 66 | else 67 | {} 68 | end 69 | end 70 | 71 | def load_json(file) 72 | file = File.expand_path file 73 | raise ArgumentError, "Invalid json file path: #{file}" unless File.exist? file 74 | JSON.parse(gsub_env(File.read file)) 75 | end 76 | 77 | def load_yaml(file) 78 | file = File.expand_path file 79 | raise ArgumentError, "Invalid json file path: #{file}" unless File.exist? file 80 | YAML.load(gsub_env(File.read file)) 81 | end 82 | 83 | def gsub_env(content) 84 | content.gsub(/\$([a-zA-Z_]+[a-zA-Z0-9_]*)|\$\{(.+)\}/) { ENV[$1 || $2] } 85 | end 86 | 87 | def self.examples 88 | Pathname.new(File.expand_path(File.join(__FILE__,'../../../examples'))) 89 | end 90 | 91 | def self.tasks 92 | if ENV["TASK"] != "" 93 | pattern="#{examples}/tasks/#{ENV["TASK"]}" 94 | else 95 | pattern="#{examples}/tasks/*.y{a,}ml" 96 | end 97 | Dir.glob(pattern).collect{|f| File.basename f} 98 | end 99 | 100 | def add_plugins(plugins, type) 101 | plugins.flatten.compact.uniq.each do |name| 102 | @plugins.add([type, name]) 103 | end 104 | end 105 | 106 | def parse_task(t) 107 | t.extend Hashie::Extensions::DeepFetch 108 | t.extend Hashie::Extensions::DeepFind 109 | 110 | m = t.deep_fetch("workflow", "collect", "metrics"){ |k| {} } 111 | collectors = m.collect do |k, v| 112 | k.match(/^\/intel\/(.*?)\/(.*?)/) 113 | # NOTE: procfs/* doesn't follow the convention, nor does disk/smart. 114 | if $1 == 'procfs' || $1 == 'disk' 115 | case $2 116 | when 'iface' 117 | 'interface' 118 | when 'filesystem' 119 | 'df' 120 | else 121 | $2 122 | end 123 | else 124 | $1 125 | end 126 | end 127 | add_plugins(collectors, 'collector') 128 | 129 | p = t.deep_find_all("process") || {} 130 | processors = p.collect do |i| 131 | if i.is_a? ::Array 132 | i.collect{|j| j["plugin_name"] if j.include? "plugin_name"} 133 | end 134 | end 135 | add_plugins(processors, 'processor') 136 | 137 | p = t.deep_find_all("publish") || {} 138 | publishers = p.collect do |i| 139 | if i.is_a? ::Array 140 | i.collect{|j| j["plugin_name"] if j.include? "plugin_name"} 141 | end 142 | end 143 | add_plugins(publishers, 'publisher') 144 | end 145 | 146 | def plugins 147 | @plugins ||= load_tasks 148 | end 149 | 150 | def load_tasks 151 | @plugins = Set.new 152 | SnapUtils.tasks.each do |t| 153 | y = load_yaml SnapUtils.examples/"tasks/#{t}" 154 | parse_task(y) 155 | end 156 | @plugins 157 | end 158 | 159 | def load_all_plugins 160 | plugins.each do |i| 161 | type, name = i 162 | load_plugin(type, name) 163 | end 164 | end 165 | end 166 | 167 | RSpec.configure do |c| 168 | c.formatter = 'documentation' 169 | c.mock_framework = :rspec 170 | c.verbose_retry = true 171 | c.order = 'default' 172 | c.include SnapUtils 173 | if ENV["DEMO"] == "true" then 174 | Pry.config.pager = false 175 | 176 | Pry.hooks.add_hook(:before_session, "notice") do |output, binding, pry| 177 | output.puts "Setup complete for DEMO mode. When you are finished checking out Snap please type 'exit-program' to shutdown containers." 178 | end 179 | end 180 | end 181 | --------------------------------------------------------------------------------