├── .github └── workflows │ └── codeql-analysis.yml ├── .gitignore ├── .travis.yml ├── CODE_OF_CONDUCT.md ├── Dockerfile ├── Gemfile ├── LICENSE.txt ├── README.md ├── Rakefile ├── bin ├── console ├── deadman-check └── setup ├── deadman_check.gemspec ├── lib ├── deadman_check.rb ├── deadman_check │ └── version.rb ├── deadman_check_global.rb ├── deadman_check_keyset.rb └── deadman_check_switch.rb └── test ├── deadman_check_test.rb ├── fixtures ├── job1.nomad └── job2.nomad └── test_helper.rb /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ master ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ master ] 20 | schedule: 21 | - cron: '45 9 * * 6' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'ruby' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://git.io/codeql-language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v3 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v1 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 52 | 53 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 54 | # If this step fails, then you should remove it and run the build manually (see below) 55 | - name: Autobuild 56 | uses: github/codeql-action/autobuild@v1 57 | 58 | # ℹ️ Command-line programs to run using the OS shell. 59 | # 📚 https://git.io/JvXDl 60 | 61 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 62 | # and modify them (or add more) to build your code if your project 63 | # uses a compiled language 64 | 65 | #- run: | 66 | # make bootstrap 67 | # make release 68 | 69 | - name: Perform CodeQL Analysis 70 | uses: github/codeql-action/analyze@v1 71 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /Gemfile.lock 4 | /_yardoc/ 5 | /coverage/ 6 | /doc/ 7 | /pkg/ 8 | /spec/reports/ 9 | /tmp/ 10 | *.gem 11 | .slackapi 12 | 13 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | language: ruby 3 | rvm: 4 | - 2.4.5 5 | - 2.5.7 6 | - 2.7.0 7 | install: 8 | - gem install bundler 9 | - bundle install 10 | script: 11 | - bundle exec rake test 12 | deploy: 13 | provider: rubygems 14 | gem: deadman_check 15 | gemspec: deadman_check.gemspec 16 | api_key: 17 | secure: rcQ8I4cBn+UpROxhBUsldKbpiiZ9QNUEep8fLGbmcU6QqN5NA2VrdW/DDzXrgXPPLoTDxRF32K2clIjT6WHYopVOvCeAA0+kNjACtGEV8DKL5VlBz0yWvLqM5L7y6ZhmiL/XUibmN7jcvqqI96h2QRr3qeO4a3XUlAIBqotjMopkqqBcHfq9+19p8CS3yS9Y85c1mM5EDcP7E/3Sa3umK7vvoMh+ulz1mic/CNKSnJOdhicqspvpPLUP0vOZ4Jlnn8I41clkBA0FcbOvEmEU2GNtMR8xDS9e0t+aVBAD92AiKYlHNzWp/84r3ZNjMURmltSLRHOQKYQhFdNJUNxZj9f44mPZcMIqD1kkKFkrdzjRzqpGxldrlUwgGkdiFev9d3x8XYMybDI3SmFDPNfa8d7DeW/AYIwkW6j7rdaFQQLzG4AZnoaW6FOmldePk0hPDT47wdpVy5nItH8ZQax8qJmDOX46IRr4fZ6piXiWZlSIfdwUtdBD47fmZS9xxi+mKTggZYiy+th3txqaXORJCCaaJVo5Yg1hbMN73+yKtLtzmsw/nKr1yVVX3KhYQkNix3ZskNhuOJ2aN2stnVsYKbL0Y924sVp7OrSyg5cWqXEXYcWGpXOKlaABa593QFGwQGl6wo7JiFO7giMQw0n6B4xkRv910BBXSVzafHk+jwk= 18 | on: 19 | tags: true 20 | env: 21 | global: 22 | secure: rcQ8I4cBn+UpROxhBUsldKbpiiZ9QNUEep8fLGbmcU6QqN5NA2VrdW/DDzXrgXPPLoTDxRF32K2clIjT6WHYopVOvCeAA0+kNjACtGEV8DKL5VlBz0yWvLqM5L7y6ZhmiL/XUibmN7jcvqqI96h2QRr3qeO4a3XUlAIBqotjMopkqqBcHfq9+19p8CS3yS9Y85c1mM5EDcP7E/3Sa3umK7vvoMh+ulz1mic/CNKSnJOdhicqspvpPLUP0vOZ4Jlnn8I41clkBA0FcbOvEmEU2GNtMR8xDS9e0t+aVBAD92AiKYlHNzWp/84r3ZNjMURmltSLRHOQKYQhFdNJUNxZj9f44mPZcMIqD1kkKFkrdzjRzqpGxldrlUwgGkdiFev9d3x8XYMybDI3SmFDPNfa8d7DeW/AYIwkW6j7rdaFQQLzG4AZnoaW6FOmldePk0hPDT47wdpVy5nItH8ZQax8qJmDOX46IRr4fZ6piXiWZlSIfdwUtdBD47fmZS9xxi+mKTggZYiy+th3txqaXORJCCaaJVo5Yg1hbMN73+yKtLtzmsw/nKr1yVVX3KhYQkNix3ZskNhuOJ2aN2stnVsYKbL0Y924sVp7OrSyg5cWqXEXYcWGpXOKlaABa593QFGwQGl6wo7JiFO7giMQw0n6B4xkRv910BBXSVzafHk+jwk= 23 | '': 24 | travis: 25 | yml: 26 | secure: sJE+8NjpQarrV2j9CXM9mu29iTVRX9rF1BPwdbnmZAYFlnTnUH9mlDbHGJCp4jN3nbcFgki428AAjREt11kGasj6aCqzjfE+4l6owlyckJdGtfUmivKe5KSXJbn6Rnt+w5jZzI9oybRy8Q6D6Yb7DSLUWU7+GDgzAdyScc+keM2k6TIBKaIM/lFzQbRAh2f8B1CryU8VdSbGNoAxuG6WSa9E5Zp9DQyT7fgwt5LUQXi7ulrR7IloJ58HSE7cg9Kt3h918gxQKrqUMhIjHxkhb6bOzXjdZNeOObG5MhKfVS4YxlggGLSFBJ45wDF+CH2CMFO/wz1Z3bASkRmOsSas8NZqE8I6FRJD0E65mtrqHtAh7qZuVmqEF8ks37ZGQLPRVIjQnW386Kgxbrdv0NEIBUWpywR3csTuVOlemg9dyI73X3YjuZYXl0UwStmIbUhFEo5MdcyC3SERp3OJtniwt1f2x7ZCbLCK/Ba5E4CBcxJ5tqxoSkcXsjm2UU7fq3doIv44VHqTkdpo54E+NvjBoPx5ZqrPWmDm9JebMXCxI3HO0LiHSDW0NH5VrhG1h/ps/fVQSOSzq8geP25UqG6GvhM/U7VB+NsYEZpe76rbc4mbwt72FB3xn0xrBEZX3aQGY+uyrp4qRLTPutm+wS4zEjkP0SF5bxoLFIBpbd049XQ= 27 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, gender identity and expression, level of experience, 9 | nationality, personal appearance, race, religion, or sexual identity and 10 | orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at zane.williamson@gmail.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at [http://contributor-covenant.org/version/1/4][version] 72 | 73 | [homepage]: http://contributor-covenant.org 74 | [version]: http://contributor-covenant.org/version/1/4/ 75 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.9.5 2 | MAINTAINER Zane Williamson 3 | 4 | # Install apk packages 5 | RUN apk update && \ 6 | apk add build-base \ 7 | git \ 8 | openssl \ 9 | ruby \ 10 | ruby-dev \ 11 | ruby-bundler \ 12 | ruby-rdoc \ 13 | ruby-bigdecimal \ 14 | ruby-irb \ 15 | ruby-json \ 16 | ruby-io-console \ 17 | --no-cache 18 | 19 | ADD . /app/ 20 | ADD lib /app/lib 21 | 22 | VOLUME /app 23 | WORKDIR /app 24 | 25 | RUN gem install bundler 26 | RUN bundle install 27 | RUN rake install 28 | 29 | RUN ruby -v 30 | 31 | ENTRYPOINT ["deadman-check"] 32 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in deadman_check.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017 Zane Williamson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Deadman Check 2 | 3 | ![CodeQL](https://github.com/sepulworld/deadman-check/workflows/CodeQL/badge.svg) 4 | [![Gem Version](https://badge.fury.io/rb/deadman_check.svg)](http://badge.fury.io/rb/deadman_check) 5 | 6 | A monitoring sidecar for Nomad periodic [jobs](https://www.nomadproject.io/docs/job-specification/periodic.html) that alerts if the periodic job isn't 7 | running at the expected interval. 8 | 9 | 10 | # Overview 11 | 1. [Requierments](#requirements) 12 | 2. [Monitoring Modes](#monitoringmodes) 13 | 3. [Alert Options](#alertoptions) 14 | 4. [Example Usage](#exampleusage) 15 | 5. [Alerting Setup](#alertingsetup) 16 | 6. [CLI Usage](#cliusage) 17 | 7. [Local System Installation Option](#localsysteminstallationoption) 18 | 8. [Run deadman-check via Docker Option](#rundeadmanviadockeroption) 19 | 9. [CLI Command Help](#clicommandhelp) 20 | 1. [Usage for key_set command](#usageforkeysetcommand) 21 | 2. [Usage for switch_monitor command](#usageforswitchmonitorcommand) 22 | 10. [Development](#development) 23 | 11. [Contributing](#contributing) 24 | 12. [License](#license) 25 | 26 | ## Requirements 27 | 28 | * [Consul](https://www.consul.io/) 29 | 30 | 31 | ## Monitoring Modes 32 | 33 | 1. Run with the Nomad periodic job as an additional [task](https://www.nomadproject.io/docs/job-specification/task.html). In this mode deadman-check will leverage a Consul key store to evaluate task frequency requirements. It uses [Epoch time](https://en.wikipedia.org/wiki/Unix_time) to verify task is running within time frequency required. 34 | 35 | 2. Run as a stand alone process that can monitor a large grouping of jobs which are reporting time frequency values into a Consul key. 36 | 37 | 38 | ## Alert Options 39 | 40 | * [Slack](https://slack.com/) 41 | screen shot 2017-03-26 at 3 29 28 pm 42 | 43 | * [AWS SNS](https://aws.amazon.com/documentation/sns/) - Broadcasting alerts and/or triggering [AWS Lambda functions](https://docs.aws.amazon.com/sns/latest/dg/sns-lambda.html) to run code 44 | screen shot 2017-08-04 at 11 39 12 am 45 | 46 | ## Example Usage 47 | 48 | Let's say I have a Nomad periodic job that is set to run every 10 minutes. The Nomad configuration looks like this: 49 | 50 | ```hcl 51 | job "SilverBulletPeriodic" { 52 | datacenters = ["dc1"] 53 | type = "batch" 54 | 55 | periodic { 56 | cron = "*/10 * * * * *" 57 | prohibit_overlap = true 58 | } 59 | 60 | group "utility" { 61 | task "SilverBulletPeriodicProcess" { 62 | driver = "docker" 63 | config { 64 | image = "silverbullet:build_1" 65 | work_dir = "/utility/silverbullet" 66 | command = "blaster" 67 | } 68 | resources { 69 | cpu = 100 70 | memory = 500 71 | } 72 | } 73 | } 74 | } 75 | ``` 76 | 77 | To monitor the SilverBulletPeriodicProcess task let's add a deadmad-check task. The host input is the Consul endpoint required by deadman-check (In this case 10.0.0.10) 78 | 79 | ```hcl 80 | job "SilverBulletPeriodic" { 81 | datacenters = ["dc1"] 82 | type = "batch" 83 | 84 | periodic { 85 | cron = "*/10 * * * * *" 86 | prohibit_overlap = true 87 | } 88 | 89 | group "silverbullet" { 90 | task "SilverBulletPeriodicProcess" { 91 | driver = "docker" 92 | config { 93 | image = "silverbullet:build_1" 94 | work_dir = "/utility/silverbullet" 95 | command = "blaster" 96 | } 97 | resources { 98 | cpu = 100 99 | memory = 500 100 | } 101 | } 102 | task "DeadmanSetSilverBulletPeriodicProcess" { 103 | driver = "docker" 104 | config { 105 | image = "sepulworld/deadman-check" 106 | command = "key_set" 107 | args = [ 108 | "--host", 109 | "10.0.0.10", 110 | "--port", 111 | "8500", 112 | "--key", 113 | "deadman/SilverBulletPeriodicProcess", 114 | "--frequency", 115 | "700"] 116 | } 117 | resources { 118 | cpu = 100 119 | memory = 256 120 | } 121 | } 122 | } 123 | } 124 | ``` 125 | screen shot 2017-04-23 at 11 14 36 pm 126 | 127 | The Consul key, deadman/SilverBulletPeriodicProcess, at 10.0.0.10 will be updated with 128 | the Epoch time for each SilverBulletPeriodic job run. If the job hangs or fails to run 129 | the job frequency calculation will be in an alerting state. 130 | 131 | Next we need a job that will run to monitor this key. 132 | 133 | ```hcl 134 | job "DeadmanMonitoring" { 135 | datacenters = ["dc1"] 136 | type = "service" 137 | 138 | group "monitor" { 139 | task "DeadmanMonitorSilverBulletPeriodicProcess" { 140 | driver = "docker" 141 | config { 142 | image = "sepulworld/deadman-check" 143 | command = "switch_monitor" 144 | args = [ 145 | "--host", 146 | "10.0.0.10", 147 | "--port", 148 | "8500", 149 | "--key", 150 | "deadman/SilverBulletPeriodicProcess", 151 | "--alert-to-slack", 152 | "slackroom", 153 | "--daemon", 154 | "--daemon-sleep", 155 | "900"] 156 | } 157 | resources { 158 | cpu = 100 159 | memory = 256 160 | } 161 | env { 162 | SLACK_API_TOKEN = "YourSlackApiToken" 163 | } 164 | } 165 | } 166 | } 167 | ``` 168 | 169 | Monitor a Consul key that contains an Epoch time entry. Send a Slack message if Epoch age hits given frequency threshold 170 | 171 | screen shot 2017-03-26 at 3 29 28 pm 172 | 173 | If you have multiple periodic jobs that need to be monitored then use the ```--key-path``` argument instead of ```--key```. Be sure to ```key_set``` all under the same Consul key path. 174 | 175 | screen shot 2017-04-23 at 11 17 29 pm 176 | 177 | To monitor the above you would just use the ```--key-path``` argument instead of ```--key``` and AWS SNS for alerting endpoint 178 | 179 | ```hcl 180 | job "DeadmanMonitoring" { 181 | datacenters = ["dc1"] 182 | type = "service" 183 | 184 | group "monitor" { 185 | task "DeadmanMonitorSilverBulletPeriodicProcesses" { 186 | driver = "docker" 187 | config { 188 | image = "sepulworld/deadman-check" 189 | command = "switch_monitor" 190 | args = [ 191 | "--host", 192 | "10.0.0.1", 193 | "--port", 194 | "8500", 195 | "--key-path", 196 | "deadman/", 197 | "--alert-to-sns", 198 | "arn:aws:sns:us-east-1:123412345678:deadman-check", 199 | "--alert-to-sns-region", 200 | "us-east-1", 201 | "--daemon", 202 | "--daemon-sleep", 203 | "900"] 204 | } 205 | resources { 206 | cpu = 100 207 | memory = 256 208 | } 209 | env { 210 | AWS_ACCESS_KEY_ID = "YourAWSKEY" 211 | AWS_SECRET_ACCESS_KEY = "YourAWSSecret" 212 | } 213 | } 214 | } 215 | } 216 | ``` 217 | 218 | ## Alerting Setup 219 | 220 | * Slack alerting requires a SLACK_API_TOKEN environment variable to be set (use [Slack Bot integration](https://my.slack.com/services/new/bot)) (optional) 221 | 222 | * [AWS SNS](https://aws.amazon.com/documentation/sns/) alerting requires appropreiate AWS IAM access to target SNS topic. One of the following can be used for authentication. IAM policy access to publish to the topic will be required 223 | - ENV['AWS_ACCESS_KEY_ID'] and ENV['AWS_SECRET_ACCESS_KEY'] 224 | - The shared credentials ini file at ~/.aws/credentials (more information) 225 | - From an [instance profile](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html) when running on EC2 226 | 227 | 228 | # CLI Usage: 229 | 230 | ## Local System Installation Option 231 | 232 | ``` 233 | gem install deadman_check 234 | ``` 235 | 236 | ## Run deadman-check via Docker Option 237 | 238 | ``` 239 | $ alias deadman-check='\ 240 | docker run \ 241 | -it --rm --name=deadman-check \ 242 | sepulworld/deadman-check' 243 | ``` 244 | 245 | 246 | ## CLI Command Help 247 | 248 | ```bash 249 | $ deadman-check -h 250 | NAME: 251 | 252 | deadman-check 253 | 254 | DESCRIPTION: 255 | 256 | Monitor a Consul key or key-path that contains an EPOCH time entry and frequency. Send Slack message if EPOCH age is greater than given frequency 257 | 258 | COMMANDS: 259 | 260 | help Display global or [command] help documentation 261 | key_set Update a given Consul key with current EPOCH 262 | switch_monitor Target a Consul key to monitor 263 | 264 | GLOBAL OPTIONS: 265 | 266 | -h, --help 267 | Display help documentation 268 | 269 | -v, --version 270 | Display version information 271 | 272 | -t, --trace 273 | Display backtrace when an error occurs 274 | ``` 275 | 276 | ### Usage for key_set command 277 | 278 | ```bash 279 | $ deadman-check key_set -h 280 | 281 | NAME: 282 | 283 | key_set 284 | 285 | SYNOPSIS: 286 | 287 | deadman-check key_set [options] 288 | 289 | DESCRIPTION: 290 | 291 | key_set will set a consul key that contains the current epoch and time frequency that job should be running at, example key {"epoch":1493010437,"frequency":"300"} 292 | 293 | EXAMPLES: 294 | 295 | # Update a Consul key deadman/myservice, with current EPOCH time 296 | deadman-check key_set --host 127.0.0.1 --port 8500 --key deadman/myservice --frequency 300 297 | 298 | OPTIONS: 299 | 300 | --host HOST 301 | IP address or hostname of Consul system 302 | 303 | --port PORT 304 | port Consul is listening on 305 | 306 | --key KEY 307 | Consul key to report EPOCH time and frequency for service 308 | 309 | --frequency FREQUENCY 310 | Frequency at which this key should be updated in seconds 311 | 312 | --consul-token TOKEN 313 | Consul KV access token (optional) 314 | ``` 315 | 316 | ### Usage for switch_monitor command 317 | 318 | ```bash 319 | $ deadman-check switch_monitor -h 320 | 321 | NAME: 322 | 323 | switch_monitor 324 | 325 | SYNOPSIS: 326 | 327 | deadman-check switch_monitor [options] 328 | 329 | DESCRIPTION: 330 | 331 | switch_monitor will monitor either a given key which contains a services last epoch checkin and frequency, or a series of services that set keys 332 | under a given key-path in Consul 333 | 334 | EXAMPLES: 335 | 336 | # Target a Consul key deadman/myservice, and this key has an EPOCH value to check looking to alert 337 | deadman-check switch_monitor --host 127.0.0.1 --port 8500 --key deadman/myservice --alert-to-slack my-slack-monitor-channel 338 | 339 | # Target a Consul key path deadman/, which contains 2 or more service keys to monitor, i.e. deadman/myservice1, deadman/myservice2, 340 | deadmman/myservice3 all fall under the path deadman/ 341 | deadman-check switch_monitor --host 127.0.0.1 --port 8500 --key-path deadman/ --alert-to-slack my-slack-monitor-channel 342 | 343 | # Target a Consul key path deadman/, alert to Amazon SNS, i.e. deadman/myservice1, deadman/myservice2, deadmman/myservice3 all fall under the path 344 | deadman/ 345 | deadman-check switch_monitor --host 127.0.0.1 --port 8500 --key-path deadman/ --alert-to-sns arn:aws:sns:*:123456789012:my_corporate_topic 346 | 347 | OPTIONS: 348 | 349 | --host HOST 350 | IP address or hostname of Consul system 351 | 352 | --port PORT 353 | port Consul is listening on 354 | 355 | --key-path KEYPATH 356 | Consul key path to monitor, performs a recursive key lookup at given path. 357 | 358 | --key KEY 359 | Consul key to monitor, provide this or --key-path if you have multiple keys in a given path. 360 | 361 | --alert-to-slack SLACKCHANNEL 362 | Slack channel to send alert, don't include the # tag in name 363 | 364 | --alert-to-sns SNSARN 365 | Amazon Web Services SNS arn to send alert, example arn arn:aws:sns:*:123456789012:my_corporate_topic 366 | 367 | --alert-to-sns-region AWSREGION 368 | Amazon Web Services region the SNS topic is in, defaults to us-west-2 369 | 370 | --daemon 371 | Run as a daemon, otherwise will run check just once 372 | 373 | --daemon-sleep SECONDS 374 | Set the number of seconds to sleep in between switch checks, default 300 375 | 376 | --consul-token TOKEN 377 | Consul KV access token (optional) 378 | ``` 379 | 380 | ### Development 381 | 382 | After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. 383 | 384 | To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org). 385 | 386 | ### Contributing 387 | 388 | Bug reports and pull requests are welcome on GitHub at https://github.com/sepulworld/deadman_check. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct. 389 | 390 | 391 | ### License 392 | 393 | The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT). 394 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require "rake/testtask" 3 | 4 | Rake::TestTask.new(:test) do |t| 5 | t.libs << "test" 6 | t.libs << "lib" 7 | t.test_files = FileList['test/**/*_test.rb'] 8 | t.warning = false 9 | end 10 | 11 | task :default => :test 12 | 13 | desc "Docker build image" 14 | task :docker_build do 15 | sh %{docker build -t sepulworld/deadman-check .} 16 | end 17 | 18 | desc "Push Docker image to Docker Hub" 19 | task :docker_push do 20 | sh %{docker push sepulworld/deadman-check} 21 | end 22 | 23 | desc "Pull Docker image to Docker Hub" 24 | task :docker_pull do 25 | sh %{docker pull sepulworld/deadman-check} 26 | end 27 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "bundler/setup" 4 | require "deadman_check" 5 | 6 | # You can add fixtures and/or initialization code here to make experimenting 7 | # with your gem easier. You can also use a different console, if you like. 8 | 9 | # (If you use this, don't forget to add pry to your Gemfile!) 10 | # require "pry" 11 | # Pry.start 12 | 13 | require "irb" 14 | IRB.start 15 | -------------------------------------------------------------------------------- /bin/deadman-check: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'rubygems' 4 | require 'commander/import' 5 | require 'deadman_check' 6 | require 'daemons' 7 | 8 | program :name, 'deadman-check' 9 | program :version, DeadmanCheck::VERSION 10 | program :description, %q{Monitor a Consul key or key-path that contains an EPOCH time entry and frequency. Send Slack message if EPOCH age is greater than given frequency} 11 | 12 | command :switch_monitor do |c| 13 | c.syntax = 'deadman-check switch_monitor [options]' 14 | c.summary = 'Target a Consul key to monitor' 15 | c.description = 'switch_monitor will monitor either a given key which contains a services last epoch checkin and frequency, or a series of services that set keys under a given key-path in Consul' 16 | c.example %q{Target a Consul key deadman/myservice, and this key has an EPOCH value to check looking to alert}, 17 | %q{deadman-check switch_monitor --host 127.0.0.1 --port 8500 --key deadman/myservice --alert-to-slack my-slack-monitor-channel} 18 | c.example %q{Target a Consul key path deadman/, which contains 2 or more service keys to monitor, i.e. deadman/myservice1, deadman/myservice2, deadmman/myservice3 all fall under the path deadman/}, 19 | %q{deadman-check switch_monitor --host 127.0.0.1 --port 8500 --key-path deadman/ --alert-to-slack my-slack-monitor-channel} 20 | c.example %q{Target a Consul key path deadman/, alert to Amazon SNS, i.e. deadman/myservice1, deadman/myservice2, deadmman/myservice3 all fall under the path deadman/}, 21 | %q{deadman-check switch_monitor --host 127.0.0.1 --port 8500 --key-path deadman/ --alert-to-sns arn:aws:sns:*:123456789012:my_corporate_topic} 22 | c.option '--host HOST', String, 'IP address or hostname of Consul system' 23 | c.option '--port PORT', String, 'port Consul is listening on' 24 | c.option '--key-path KEYPATH', String, 'Consul key path to monitor, performs a recursive key lookup at given path.' 25 | c.option '--key KEY', String, 'Consul key to monitor, provide this or --key-path if you have multiple keys in a given path.' 26 | c.option '--alert-to-slack SLACKCHANNEL', String, 'Slack channel to send alert, don\'t include the # tag in name' 27 | c.option '--alert-to-sns SNSARN', String, 'Amazon Web Services SNS arn to send alert, example arn arn:aws:sns:*:123456789012:my_corporate_topic' 28 | c.option '--alert-to-sns-region AWSREGION', String, 'Amazon Web Services region the SNS topic is in, defaults to us-west-2' 29 | c.option '--daemon', 'Run as a daemon, otherwise will run check just once' 30 | c.option '--daemon-sleep SECONDS', String, 'Set the number of seconds to sleep in between switch checks, default 300' 31 | c.option '--consul-token TOKEN', String, 'Consul KV access token' 32 | c.action do |args, options| 33 | options.default :daemon_sleep => 300, 34 | :alert_to_sns_region => 'us-west-2', 35 | :alert_to_sns => nil, 36 | :alert_to_slack => nil, 37 | :consul_token => "" 38 | 39 | if options.key_path && options.key 40 | abort("Specify --key-path or --key, don't specify both") 41 | end 42 | if options.key 43 | target = options.key 44 | recurse = false 45 | else 46 | target = options.key_path 47 | recurse = true 48 | end 49 | switch_monitor = DeadmanCheck::SwitchMonitor.new( 50 | options.host, options.port, 51 | target, options.alert_to_slack, options.alert_to_sns, 52 | options.alert_to_sns_region, recurse, options.daemon_sleep, options.consul_token) 53 | if options.daemon 54 | Daemons.run(switch_monitor.run_check_daemon) 55 | else 56 | switch_monitor.run_check_once 57 | end 58 | end 59 | end 60 | 61 | command :key_set do |c| 62 | c.syntax = 'deadman-check key_set [options]' 63 | c.summary = 'Update a given Consul key with current EPOCH' 64 | c.description = 'key_set will set a consul key that contains the current epoch and time frequency for which the job being monitored runs at, example key {"epoch":1493010437,"frequency":"300"}' 65 | c.example %q{Update a Consul key deadman/myservice, with current EPOCH time}, 66 | %q{deadman-check key_set --host 127.0.0.1 --port 8500 --key deadman/myservice --frequency 300} 67 | c.option '--host HOST', String, 'IP address or hostname of Consul system' 68 | c.option '--port PORT', String, 'port Consul is listening on' 69 | c.option '--key KEY', String, 'Consul key to report EPOCH time and frequency for service' 70 | c.option '--frequency FREQUENCY', String, 'Frequency at which this key should be updated in seconds' 71 | c.option '--consul-token TOKEN', String, 'Consul KV access token' 72 | c.action do |args, options| 73 | options.default :consul_token => "" 74 | 75 | if options.frequency.nil? 76 | abort("Specify --frequency at which this key should be updated by the service") 77 | end 78 | if options.key.nil? 79 | abort("Must specify a --key") 80 | end 81 | key_set = DeadmanCheck::KeySet.new(options.host, options.port, options.key, 82 | options.frequency, options.consul_token) 83 | key_set.run_consul_key_update 84 | end 85 | end 86 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | set -vx 5 | 6 | bundle install 7 | 8 | # Do any other automated setup that you need to do here 9 | -------------------------------------------------------------------------------- /deadman_check.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'deadman_check/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "deadman_check" 8 | spec.version = DeadmanCheck::VERSION 9 | spec.authors = ["zane"] 10 | spec.email = ["zane.williamson@gmail.com"] 11 | 12 | spec.summary = %q{Monitor a Consul key that contains an EPOCH time entry. 13 | Send email if EPOCH age hits given threshold} 14 | spec.description = %q{A script to check a given Consul key EPOCH for 15 | freshness. Good for monitoring cron jobs or batch jobs. Have the last step 16 | of the job post the EPOCH time to target Consul key. This script will monitor 17 | it for a given freshness value (difference in time now to posted EPOCH)} 18 | spec.homepage = "https://github.com/sepulworld/deadman-check" 19 | spec.license = "MIT" 20 | 21 | if spec.respond_to?(:metadata) 22 | spec.metadata['allowed_push_host'] = "https://rubygems.org" 23 | spec.metadata['optional_gems'] = "keyring" 24 | end 25 | 26 | spec.files = `git ls-files -z`.split("\x0").reject do |f| 27 | f.match(%r{^(test|spec|features)/}) 28 | end 29 | spec.bindir = "bin" 30 | spec.executables = "deadman-check" 31 | spec.require_paths = ["lib"] 32 | 33 | spec.add_development_dependency "bundler", "~> 2.1" 34 | spec.add_development_dependency "rake", "~> 13.0" 35 | spec.add_development_dependency "minitest", "~> 5.0" 36 | spec.add_development_dependency "webmock", "~> 3.0" 37 | 38 | spec.add_dependency 'commander', '~> 4.4', '>= 4.4.3' 39 | spec.add_dependency 'diplomat', '~> 2.2.5', '>= 2.0.0' 40 | spec.add_dependency 'slack-ruby-client', '~> 0.8.0' 41 | spec.add_dependency 'daemons', '~> 1.2.4', '>=1.2.4' 42 | spec.add_dependency 'aws-sdk-sns', '~> 1' 43 | end 44 | -------------------------------------------------------------------------------- /lib/deadman_check.rb: -------------------------------------------------------------------------------- 1 | require "deadman_check/version" 2 | 3 | module DeadmanCheck 4 | Dir[File.dirname(__FILE__) + '/*.rb'].each do |file| 5 | require file 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /lib/deadman_check/version.rb: -------------------------------------------------------------------------------- 1 | module DeadmanCheck 2 | VERSION = "0.3.1" 3 | end 4 | -------------------------------------------------------------------------------- /lib/deadman_check_global.rb: -------------------------------------------------------------------------------- 1 | require 'deadman_check/version' 2 | require 'diplomat' 3 | 4 | module DeadmanCheck 5 | class DeadmanCheckGlobal 6 | 7 | def get_epoch_time 8 | epoch_time_now = Time.now.to_i 9 | return epoch_time_now 10 | end 11 | 12 | def configure_diplomat(host, port, consul_token) 13 | Diplomat.configure do |config| 14 | config.url = "http://#{host}:#{port}" 15 | if consul_token != "" 16 | config.options = {headers: {"X-Consul-Token" => consul_token}} 17 | end 18 | end 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /lib/deadman_check_keyset.rb: -------------------------------------------------------------------------------- 1 | require 'deadman_check/version' 2 | require 'deadman_check_global' 3 | require 'diplomat' 4 | require 'json' 5 | 6 | module DeadmanCheck 7 | # KeySet Class 8 | class KeySet 9 | attr_accessor :host, :port, :key, :frequency, :consul_token 10 | 11 | def initialize(host, port, key, frequency, consul_token) 12 | @host = host 13 | @port = port 14 | @key = key 15 | @frequency = frequency 16 | @consul_token = consul_token 17 | end 18 | 19 | def run_consul_key_update 20 | update_consul_key(@host, @port, @key, @frequency, @consul_token) 21 | end 22 | 23 | private 24 | def generate_json(epoch, frequency) 25 | consul_key = { :epoch => epoch, :frequency => frequency } 26 | consul_key.to_json 27 | end 28 | 29 | def update_consul_key(host, port, key, frequency, consul_token) 30 | DeadmanCheck::DeadmanCheckGlobal.new.configure_diplomat(host, port, consul_token) 31 | epoch_time_now = DeadmanCheck::DeadmanCheckGlobal.new.get_epoch_time 32 | Diplomat::Kv.put(key, "#{generate_json(epoch_time_now, frequency)}") 33 | puts "Consul key #{key} updated EPOCH to #{epoch_time_now}" 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/deadman_check_switch.rb: -------------------------------------------------------------------------------- 1 | require 'deadman_check/version' 2 | require 'deadman_check_global' 3 | require 'diplomat' 4 | require 'slack-ruby-client' 5 | require 'json' 6 | require 'aws-sdk-sns' 7 | 8 | module DeadmanCheck 9 | # Switch class 10 | class SwitchMonitor 11 | attr_accessor :host, :port, :target, :alert_to_slack, 12 | :alert_to_sns, :alert_to_sns_region, :recurse, :daemon_sleep 13 | 14 | def initialize(host, port, target, alert_to_slack, alert_to_sns, 15 | alert_to_sns_region, recurse, daemon_sleep, consul_token) 16 | @host = host 17 | @port = port 18 | @target = target 19 | @alert_to_slack = alert_to_slack 20 | @alert_to_sns = alert_to_sns 21 | @alert_to_sns_region = alert_to_sns_region 22 | @recurse = recurse 23 | @daemon_sleep = daemon_sleep.to_i 24 | @consul_token = consul_token 25 | 26 | unless @alert_to_slack.nil? 27 | Slack.configure do |config| 28 | config.token = ENV['SLACK_API_TOKEN'] 29 | end 30 | end 31 | 32 | unless @alert_to_sns.nil? 33 | @sns = Aws::SNS::Client.new( 34 | region: @alert_to_sns_region 35 | ) 36 | end 37 | end 38 | 39 | def run_check_once 40 | recorded_epochs = get_recorded_epochs(@host, @port, @target, @recurse) 41 | current_epoch = DeadmanCheck::DeadmanCheckGlobal.new.get_epoch_time.to_i 42 | if @recurse 43 | check_recursive_recorded_epochs(recorded_epochs, current_epoch) 44 | else 45 | record = parse_recorded_epoch(recorded_epochs) 46 | check_recorded_epoch(record, current_epoch) 47 | end 48 | end 49 | 50 | def run_check_daemon 51 | loop do 52 | run_check_once 53 | sleep(@daemon_sleep) 54 | end 55 | end 56 | 57 | private 58 | def diff_epoch(current_epoch, recorded_epoch) 59 | epoch_difference = current_epoch - recorded_epoch 60 | return epoch_difference 61 | end 62 | 63 | def get_recorded_epochs(host, port, target, recurse) 64 | DeadmanCheck::DeadmanCheckGlobal.new.configure_diplomat(host, port, @consul_token) 65 | recorded_epochs = Diplomat::Kv.get(target, recurse: recurse) 66 | return recorded_epochs 67 | end 68 | 69 | def check_recursive_recorded_epochs(recorded_epochs, current_epoch) 70 | recorded_epochs.each do |recorded_service| 71 | value_json = JSON.parse(recorded_service[:value]) 72 | frequency = value_json["frequency"].to_i 73 | epoch = value_json["epoch"].to_i 74 | epoch_diff = diff_epoch(current_epoch, epoch) 75 | alert_if_epoch_greater_than_frequency(epoch_diff, 76 | recorded_service[:key], 77 | frequency) 78 | end 79 | end 80 | 81 | def parse_recorded_epoch(recorded_epochs) 82 | # {"epoch":1493000501,"frequency":"300"} 83 | value_json = JSON.parse(recorded_epochs) 84 | frequency = value_json["frequency"] 85 | epoch = value_json["epoch"] 86 | return epoch, frequency 87 | end 88 | 89 | def check_recorded_epoch(parse_recorded_epoch, current_epoch) 90 | recorded_epoch = parse_recorded_epoch[0].to_i 91 | frequency = parse_recorded_epoch[1].to_i 92 | epoch_diff = diff_epoch(current_epoch, recorded_epoch) 93 | alert_if_epoch_greater_than_frequency(epoch_diff, @target, frequency) 94 | end 95 | 96 | def alert_if_epoch_greater_than_frequency(epoch_diff, target, frequency) 97 | if epoch_diff > frequency 98 | slack_alert( 99 | @alert_to_slack, target, epoch_diff) unless @alert_to_slack.nil? 100 | sns_alert( 101 | @alert_to_sns, target, epoch_diff) unless @alert_to_sns.nil? 102 | end 103 | end 104 | 105 | def slack_alert(alert_to_slack, target, epoch_diff) 106 | client = Slack::Web::Client.new 107 | client.chat_postMessage(channel: "\##{alert_to_slack}", 108 | text: "Alert: Deadman Switch 109 | Triggered for #{target}, with #{epoch_diff} seconds since last run", 110 | username: 'deadman') 111 | end 112 | 113 | def sns_alert(alert_to_sns, target, epoch_diff) 114 | @sns.publish( 115 | target_arn: @alert_to_sns, 116 | message_structure: 'json', 117 | message: { 118 | :default => "Alert: Deadman Switch triggered for #{target}", 119 | :email => "Alert: Deadman Switch triggered for #{target}, with 120 | #{epoch_diff} seconds since last run", 121 | :sms => "Alert: Deadman Switch for #{target}" 122 | }.to_json 123 | ) 124 | end 125 | end 126 | end 127 | -------------------------------------------------------------------------------- /test/deadman_check_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class DeadmanCheckTest < Minitest::Test 4 | 5 | @@standalone_key_response = "[ 6 | { 7 | \"LockIndex\": \"0\", 8 | \"Key\": \"deadman/myservice1\", 9 | \"Flags\": \"0\", 10 | \"Value\": \"eyJlcG9jaCI6MTQ5MzAxMDQzNywiZnJlcXVlbmN5IjoiMzAwIn0=\", 11 | \"CreateIndex\": \"9\", 12 | \"ModifyIndex\": \"465\" 13 | } 14 | ]" 15 | 16 | @@recursive_key_response = "[ 17 | { 18 | \"LockIndex\": \"0\", 19 | \"Key\": \"deadman/myservice\", 20 | \"Flags\": \"0\", 21 | \"Value\": \"eyJlcG9jaCI6MTQ5Mjk2NDU0NSwiZnJlcXVlbmN5IjoiMzAwIn0=\", 22 | \"CreateIndex\": \"8\", 23 | \"ModifyIndex\": \"10\" 24 | }, 25 | { 26 | \"LockIndex\": \"0\", 27 | \"Key\": \"deadman/myservice2\", 28 | \"Flags\": \"0\", 29 | \"Value\": \"eyJlcG9jaCI6MTQ5Mjk2NTI5MiwiZnJlcXVlbmN5IjoiMzAwIn0=\", 30 | \"CreateIndex\": \"58\", 31 | \"ModifyIndex\": \"58\" 32 | }, 33 | { 34 | \"LockIndex\": \"0\", 35 | \"Key\": \"deadman/myservice3\", 36 | \"Flags\": \"0\", 37 | \"Value\": \"eyJlcG9jaCI6MTQ5Mjk2NTMxMSwiZnJlcXVlbmN5IjoiMzAwIn0=\", 38 | \"CreateIndex\": \"61\", 39 | \"ModifyIndex\": \"61\" 40 | }, 41 | { 42 | \"LockIndex\": \"0\", 43 | \"Key\": \"deadman/myservice4\", 44 | \"Flags\": \"0\", 45 | \"Value\": \"eyJlcG9jaCI6MTQ5Mjk2NTI5NywiZnJlcXVlbmN5IjoiMzAwIn0=\", 46 | \"CreateIndex\": \"59\", 47 | \"ModifyIndex\": \"59\" 48 | } 49 | ]" 50 | 51 | def test_that_it_has_a_version_number 52 | refute_nil ::DeadmanCheck::VERSION 53 | end 54 | 55 | def test_consul_key_update_slack 56 | stub_request(:put, "http://127.0.0.1:8500/v1/kv/test"). 57 | with(body: "{\"epoch\":#{Time.now.to_i},\"frequency\":\"300\"}"). 58 | to_return(status: 200, body: "", headers: {}) 59 | key_set = DeadmanCheck::KeySet.new('127.0.0.1', '8500', 'test', 60 | '300', '') 61 | key_set.run_consul_key_update 62 | end 63 | 64 | def test_recursive_key_lookup_slack 65 | stub_request(:get, "http://127.0.0.1:8500/v1/kv/deadman/?recurse"). 66 | to_return(status: 200, body: @@recursive_key_response, headers: {}) 67 | stub_request(:post, "https://slack.com/api/chat.postMessage"). 68 | to_return(status: 200, body: "{\"ok\":true}", headers: {}) 69 | switch_monitor = DeadmanCheck::SwitchMonitor.new('127.0.0.1', '8500', 70 | 'deadman/', 'monitoroom', nil, nil, true, '30', '') 71 | switch_monitor.run_check_once 72 | end 73 | 74 | def test_standalone_key_lookup_slack 75 | stub_request(:get, "http://127.0.0.1:8500/v1/kv/deadman/myservice1"). 76 | to_return(status: 200, body: @@standalone_key_response, headers: {}) 77 | stub_request(:post, "https://slack.com/api/chat.postMessage"). 78 | to_return(status: 200, body: "{\"ok\":true}", headers: {}) 79 | switch_monitor = DeadmanCheck::SwitchMonitor.new('127.0.0.1', '8500', 80 | 'deadman/myservice1', 'monitoroom', nil, nil, false, '30', '') 81 | switch_monitor.run_check_once 82 | end 83 | end 84 | -------------------------------------------------------------------------------- /test/fixtures/job1.nomad: -------------------------------------------------------------------------------- 1 | job "DeadmanMonitoring" { 2 | type = "service" 3 | datacenters = ["dc1"] 4 | 5 | group "monitor" { 6 | task "DeadmanMonitorSilverBulletPeriodicProcess" { 7 | driver = "docker" 8 | config { 9 | image = "sepulworld/deadman-check" 10 | command = "switch_monitor" 11 | args = [ 12 | "--host", 13 | "192.168.43.145", 14 | "--port", 15 | "8500", 16 | "--key", 17 | "deadman/SilverBulletPeriodicProcess", 18 | "--freshness", 19 | "80", 20 | "--alert-to", 21 | "random", 22 | "--daemon", 23 | "--daemon-sleep", 24 | "30"] 25 | } 26 | resources { 27 | cpu = 100 28 | memory = 500 29 | } 30 | env { 31 | SLACK_API_TOKEN = "YourSlackKeyHere" 32 | } 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /test/fixtures/job2.nomad: -------------------------------------------------------------------------------- 1 | job "SilverBulletPeriodic" { 2 | type = "batch" 3 | datacenters = ["dc1"] 4 | 5 | periodic { 6 | cron = "*/1 * * * * *" 7 | prohibit_overlap = true 8 | } 9 | 10 | group "silverbullet" { 11 | task "DeadmanSetSilverBulletPeriodicProcess" { 12 | driver = "docker" 13 | config { 14 | image = "sepulworld/deadman-check" 15 | command = "key_set" 16 | args = [ 17 | "--host", 18 | "192.168.43.145", 19 | "--port", 20 | "8500", 21 | "--key", 22 | "deadman/SilverBulletPeriodicProcess"] 23 | } 24 | resources { 25 | cpu = 100 26 | memory = 256 27 | } 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) 2 | require 'deadman_check' 3 | 4 | require 'minitest/autorun' 5 | require 'webmock/minitest' 6 | --------------------------------------------------------------------------------