2 |
54 |
55 |
56 |
57 | Your changes have been saved.
58 |
59 |
60 |
156 |
157 |
158 |
159 |
(in minutes)
160 |
161 |
--------------------------------------------------------------------------------
/resources/public/css/styles.css:
--------------------------------------------------------------------------------
1 | /*
2 | * Style resets.
3 | */
4 | * {
5 | margin: 0;
6 | padding: 0;
7 | }
8 |
9 | li { list-style-type: none; }
10 |
11 | /*
12 | * General styles
13 | */
14 | a { color: #1987C1; }
15 | a:active {
16 | background-color: #1987C1;
17 | color: white;
18 | }
19 |
20 | h1 { margin: 16px 0; }
21 |
22 | h2 {
23 | margin-top: 20px;
24 | margin-bottom: 10px;
25 | font-weight: normal;
26 | }
27 |
28 | p { margin: 16px 0; }
29 |
30 | input[type=text] {
31 | width: 100%;
32 | padding: 5px 5px;
33 | }
34 |
35 | button, .button, input[type=submit] {
36 | display: inline-block;
37 | width: 180px;
38 | padding: 12px 16px;
39 | font-size: 16px;
40 | background-color: #ccc;
41 | background-color: #F6CC25;
42 | color: black;
43 | border: 0;
44 | border-radius:4px;
45 | box-shadow: 0px 3px 0px #999;
46 | text-decoration: none;
47 | box-sizing: border-box;
48 | text-align: center;
49 | height: 40px;
50 | }
51 |
52 | button:active, .button:active, input[type=submit]:active {
53 | position: relative;
54 | top: 2px;
55 | box-shadow: 0px 1px 0px #999;
56 | height: 38px;
57 | outline: none;
58 | }
59 |
60 | i[class^="icon-"] { text-decoration: none; }
61 |
62 | .example {
63 | font-size: 80%;
64 | color: #888;
65 | font-weight: normal;
66 | }
67 |
68 | /*
69 | * Display tables.
70 | */
71 | table.display {
72 | border-collapse: collapse;
73 | width: 100%;
74 | }
75 |
76 | table.display th, table.display td {
77 | padding: 0 10px;
78 | padding-top: 10px;
79 | width: 40%;
80 | vertical-align: bottom;
81 | }
82 |
83 | table.display th {
84 | text-align: left;
85 | padding-bottom: 10px;
86 | border-bottom: 1px solid #ccc;
87 | }
88 |
89 | table.display td.check-path {
90 | width: 60%;
91 | }
92 |
93 | table.display th:first-of-type, table.display td:first-of-type {
94 | padding-left: 0;
95 | }
96 |
97 | table.display th:last-of-type, table.display td:last-of-type {
98 | text-align: right;
99 | }
100 |
101 | html { height: 100%; }
102 | body {
103 | background-color: #f9f9f9;
104 | font-family: "Helvetica Neue", "Lucida Grande", Helvetica, Arial, Verdana, sans-serif;
105 | min-height: 100%;
106 | display: flex;
107 | }
108 |
109 | header {
110 | display: block;
111 | padding: 40px;
112 | padding-bottom: 0;
113 | margin-bottom: 40px;
114 | width: 100%;
115 | }
116 |
117 | #inner-body {
118 | min-width: 800px;
119 | flex: 2;
120 | box-sizing: border-box;
121 | padding: 0 40px 100px 40px;
122 | }
123 |
124 | #watchman-text {
125 | position: absolute;
126 | padding: 0 30px;
127 | right: 160px;
128 | line-height: 160px;
129 | font-size: 120px;
130 | -webkit-transform-origin: top right;
131 | -webkit-transform: rotate(-90deg);
132 | -moz-transform-origin: top right;
133 | -moz-transform: rotate(-90deg);
134 | color: #F6CC25; /* Yellow */
135 | font-family: Impact, "Helvetica Neue", "Lucida Grande", Helvetica, Arial, sans-serif;
136 | font-weight: bold;
137 | -moz-user-select: none;
138 | -webkit-user-select: none;
139 | -ms-user-select: none;
140 | }
141 |
142 | aside.side-column {
143 | flex: 1;
144 | /* max-width is set so that the middle column can expand further for larger displays. */
145 | max-width: 300px;
146 | z-index: -1;
147 | }
148 |
149 | #black-edge {
150 | /* Make sure the black edge is always tall enough to contain watchman-text. */
151 | min-height: 700px;
152 | position: relative;
153 | background-color: black;
154 | }
155 |
156 | nav li { display: inline-block; }
157 | nav li a {
158 | display: inline-block;
159 | font-weight: bold;
160 | font-size: 20px;
161 | margin-right: 20px;
162 | padding-bottom: 15px;
163 | text-decoration: none;
164 | color: black;
165 | }
166 | nav li a:active {
167 | color: #666;
168 | background: none;
169 | }
170 | nav li.selected a { border-bottom: 3px solid #F6CC25; }
171 | nav a:hover, nav li.selected a:hover { border-bottom: 3px solid black; }
172 |
173 | /*
174 | * Index page.
175 | */
176 | .check-status .status a { display: block; }
177 | .check-status .up { background-color: #69D166; }
178 | .check-status .up a { color: black; }
179 | .check-status .down { background-color: #D15335; }
180 | .check-status .down a { color: yellow; }
181 | .check-status .unknown { background-color: #e2e2e2; }
182 | .check-status .status a i { display: none; }
183 | .check-status .up a i.icon-ok { display: inline; }
184 | .check-status .down a i.icon-exclamation { display: inline; }
185 | .check-status .unknown a i.icon-question { display: inline; }
186 |
187 | table#check-statuses td {
188 | border-bottom: 1px solid #ddd;
189 | padding: 10px;
190 | font-size: 14px;
191 | white-space: nowrap;
192 | }
193 | table#check-statuses td:first-of-type { padding-left: 0; }
194 |
195 | table td.host { width: 100%; }
196 | table#check-statuses .last-checked, table#check-statuses .status-last-changed { text-align: right; }
197 |
198 | table#check-statuses td.status { text-align: center; }
199 | table#check-statuses tr[data-state="paused"] a .icon-pause { display: none; }
200 | table#check-statuses tr[data-state="enabled"] a .icon-play { display: none; }
201 | table#check-statuses tr[data-state="paused"] .status {
202 | background-color: #ddd;
203 | color: #999;
204 | }
205 |
206 | table#check-statuses th.pause, table#check-statuses td.pause { border: 0; }
207 | table#check-statuses td.pause a { color: #aaa; }
208 |
209 | table#check-statuses tr.failure-message { background-color: #eee; }
210 | table#check-statuses tr.failure-message td {
211 | /* This is a workaround to prevent long failure messages from distorting the table's cell spacing. */
212 | max-width: 1px;
213 | text-align: left;
214 | }
215 | /* This response body can be long and overflow the table. */
216 | table#check-statuses tr.failure-message pre {
217 | max-width: 690px;
218 | overflow-x: scroll;
219 | }
220 |
221 | /*
222 | * Roles page.
223 | */
224 | #roles-page li { margin: 8px 0; }
225 |
226 | /*
227 | * Roles-edit page
228 | */
229 | #flash-message {
230 | padding: 10px;
231 | background-color: #FFEFC2;
232 | }
233 | #roles-edit-page th span.example { margin-left: 8px; }
234 | #roles-edit-page table.display td:last-of-type { vertical-align: middle; }
235 |
236 | #roles-edit-page a.add, #roles-edit-page a.remove { text-decoration: none; }
237 | #roles-edit-page a.remove { color: #cc3333; }
238 | #roles-edit-page a:hover { color: black; }
239 | #roles-edit-page a:active { color: white; }
240 |
241 | #roles-edit-page th { white-space:nowrap; }
242 |
243 | #roles-edit-page input[name="name"] {
244 | width: 200px;
245 | margin-left: 10px;
246 | }
247 |
248 | #roles-edit-page input.path { width: 100%; }
249 |
250 | #roles-edit-page input.hostname { width: 400px; }
251 |
252 | #roles-edit-page input.expected-status-code, #roles-edit-page input.timeout,
253 |
254 | #roles-edit-page input.max-retries { width: 40px; }
255 |
256 | #roles-edit-page input[name="email"] {
257 | width: 200px;
258 | margin: 15px 0 0 10px;
259 | }
260 |
261 | span#snooze-until, span.snooze-message {
262 | font-size: 80%;
263 | color: #A33333;
264 | font-weight: normal;
265 | }
266 |
267 | span.hidden { display: none; }
268 |
269 | #roles-edit-page input.snooze-duration {
270 | width: 200px;
271 | margin-left: 10px;
272 | }
273 |
274 | #roles-edit-page p.snooze-desc {
275 | position: relative;
276 | left: 195px;
277 | }
278 |
279 | form.update { margin-bottom: 20px; }
280 |
281 | .optional { margin: 8px 0 }
282 |
283 | #roles-edit-page td.check-send-email { text-align:center; vertical-align:middle; }
284 |
--------------------------------------------------------------------------------
/src/watchman/models.clj:
--------------------------------------------------------------------------------
1 | (ns watchman.models
2 | (:require [clj-time.core :as time-core]
3 | [clj-time.coerce :as time-coerce]
4 | [clj-time.format :as time-format]
5 | [clojure.core.incubator :refer [-?>]]
6 | [clojure.string :as string]
7 | [korma.db :refer [transaction]]
8 | [korma.incubator.core :as k :refer [belongs-to defentity has-many many-to-many]]
9 | [watchman.utils :refer [sget sget-in role-snoozed?]]
10 | [korma.db :refer :all]))
11 |
12 | (defdb watchman-db (postgres {:host (or (System/getenv "WATCHMAN_DB_HOST") "localhost")
13 | :db (or (System/getenv "WATCHMAN _DB_NAME") "watchman")
14 | :user (or (System/getenv "WATCHMAN_DB_USER") (System/getenv "USER"))
15 | :password (or (System/getenv "WATCHMAN_DB_PASS") "")}))
16 |
17 | (declare check-statuses)
18 |
19 | (def underscorize
20 | "Takes a hyphenated keyword (or string) and returns an underscored keyword.
21 | Appends the optional suffix string arg if supplied."
22 | (memoize (fn [hyphenated-keyword & {:keys [suffix]}]
23 | (-> hyphenated-keyword name (string/replace "-" "_") (str suffix) keyword))))
24 |
25 | (defmacro defentity2
26 | "Like defentity, but automatically sets the table name to the underscorized version of the entity name"
27 | [entity-var & body]
28 | (let [table-name (underscorize entity-var)]
29 | `(defentity ~entity-var
30 | (k/table ~table-name)
31 | ~@body)))
32 |
33 | ; A URL to check on one or more hosts.
34 | ; - path
35 | ; - nickname
36 | ; - timeout
37 | ; - interval: How frequently to run this check. Defaults to 60s.
38 | ; - max_retries: how many times to retry before considering the check a failure.
39 | ; - expected_status_code
40 | ; - expected_response_contents
41 | (defentity2 checks
42 | (has-many check-statuses {:fk :check_id}))
43 |
44 | ; A host to check, which can belong to one or more roles.
45 | ; - hostname
46 | ; - nickname
47 | (defentity2 hosts)
48 |
49 | ; The status of a check for a given host.
50 | ; - host_id
51 | ; - check_id
52 | ; - state: enabled or paused.
53 | ; - last_checked_at
54 | ; - last_response_status_code
55 | ; - last_response_body
56 | ; - status: either unknown (i.e. recently created), up or down
57 | (defentity2 check-statuses
58 | (belongs-to checks {:fk :check_id})
59 | (belongs-to hosts {:fk :host_id}))
60 |
61 | ; A role is a set of checks and a set of hosts to apply them to.
62 | ; - name
63 | (defentity2 roles
64 | (has-many checks {:fk :role_id})
65 | (many-to-many hosts :roles_hosts {:lfk :role_id :rfk :host_id}))
66 |
67 | ; A join table for roles-hosts.
68 | (defentity2 roles-hosts)
69 |
70 | ; A URL to which to post updates to check-statuses
71 | ; - url
72 | (defentity2 webhooks)
73 |
74 | (defn upsert
75 | "Updates rows which match the given where-map. If no row matches this map, insert one first.
76 | This isn't a truly robust or performant upsert, but it should be sufficient for our purposes."
77 | ([korma-entity where-map]
78 | (upsert korma-entity where-map {}))
79 | ([korma-entity where-map additional-fields]
80 | (let [row-values (merge where-map additional-fields)]
81 | (if (empty? (k/select korma-entity (k/where where-map)))
82 | (k/insert korma-entity (k/values row-values))
83 | (k/update korma-entity
84 | (k/set-fields row-values)
85 | (k/where where-map))))))
86 |
87 | (defn delete-check [check-id]
88 | (transaction
89 | (k/delete check-statuses (k/where {:check_id check-id}))
90 | (k/delete checks (k/where {:id check-id}))))
91 |
92 | (defn add-check-to-role [check-id role-id]
93 | (transaction
94 | (let [host-ids (->> (k/select roles-hosts (k/where {:role_id role-id}))
95 | (map :host_id))]
96 | (doseq [host-id host-ids]
97 | (upsert check-statuses {:host_id host-id :check_id check-id})))))
98 |
99 | (defn add-host-to-role [host-id role-id]
100 | (transaction
101 | (upsert roles-hosts {:role_id role-id :host_id host-id})
102 | ;; Create a check-status entry for every host+check pair.
103 | (let [check-ids (->> (k/select checks (k/where {:role_id role-id}))
104 | (map :id))]
105 | (doseq [check-id check-ids]
106 | (upsert check-statuses {:host_id host-id :check_id check-id})))))
107 |
108 | (defn remove-host-from-role [host-id role-id]
109 | (transaction
110 | (let [check-ids (->> (k/select checks (k/where {:role_id role-id}))
111 | (map :id))]
112 | (k/delete check-statuses (k/where {:host_id host-id :check_id [in check-ids]}))
113 | (k/delete roles-hosts (k/where {:host_id host-id :role_id role-id})))))
114 |
115 | (defn snooze-role
116 | "Snoozes a role for the given duration in minutes. Returns the snooze_until parameter set for the role."
117 | [role-id snooze-duration]
118 | (let [snooze-until (->> snooze-duration
119 | time-core/minutes
120 | (time-core/plus (time-core/now))
121 | time-coerce/to-timestamp)]
122 | (k/update roles
123 | (k/set-fields {:snooze_until snooze-until})
124 | (k/where {:id role-id}))
125 | snooze-until))
126 |
127 | (defn get-check-statuses-with-hosts-and-checks
128 | "Returns a sequence of all check-status records in the database with their associated :hosts and :checks
129 | relations eagerly loaded."
130 | []
131 | ; Korma.incubator's 'with-object' unfortunately does not eagerly load associations.
132 | ; See: https://github.com/korma/korma.incubator/issues/7
133 | (letfn [(in-memory-join [model foreign-key relation check-statuses]
134 | (let [ids (->> check-statuses (map #(sget % foreign-key)) set)
135 | id->object (->> (k/select model (k/where {:id [in ids]}))
136 | (reduce (fn [m obj] (assoc m (sget obj :id) obj)) {}))]
137 | (map (fn [check-status]
138 | (assoc check-status relation (sget id->object (sget check-status foreign-key))))
139 | check-statuses)))]
140 | (->> (k/select check-statuses)
141 | (in-memory-join hosts :host_id :hosts)
142 | (in-memory-join checks :check_id :checks))))
143 |
144 | (defn get-check-status-by-id [id]
145 | (first (k/select check-statuses
146 | (k/where {:id id})
147 | (k/with-object hosts)
148 | (k/with-object checks))))
149 |
150 | (defn get-role-by-id [id]
151 | (first (k/select roles
152 | (k/where {:id id})
153 | (k/with-object hosts)
154 | (k/with-object checks))))
155 |
156 | (defn get-host-by-id [id]
157 | (first (k/select hosts (k/where {:id id}))))
158 |
159 | (defn get-hosts-in-role [role-id]
160 | (k/select hosts
161 | (k/join roles-hosts (= :roles_hosts.host_id :id))
162 | (k/where {:roles_hosts.role_id role-id})))
163 |
164 | (defn get-host-by-hostname-in-role
165 | "Returns a host record by hostname iff it is assigned to the specified role."
166 | [hostname role-id]
167 | (first (k/select hosts
168 | (k/join roles-hosts (= :roles_hosts.host_id :id))
169 | (k/where {:hostname hostname
170 | :roles_hosts.role_id role-id}))))
171 |
172 | (defn get-host-by-hostname [hostname]
173 | (first (k/select hosts (k/where {:hostname hostname}))))
174 |
175 | (defn get-check-display-name [check-status]
176 | (or (sget check-status :nickname)
177 | (sget check-status :path)))
178 |
179 | (defn extract-subdomain-from-hostname
180 | "Returns the subdomain of the given fully-qualified domain name.
181 | Given hostname subdomain1.subdomain2.example.com, returns subdomain1.subdomain2.
182 | Given example.com, returns example.com."
183 | [hostname]
184 | (let [parts (string/split hostname #"\.")]
185 | (if (< (count parts) 3)
186 | hostname
187 | (string/join "." (take (- (count parts) 2) parts)))))
188 |
189 | (defn get-host-display-name [host]
190 | (or (sget host :nickname)
191 | (-> (sget host :hostname)
192 | extract-subdomain-from-hostname)))
193 |
194 | (defn get-url-of-check-status
195 | "The URL (hostname plus path) that a check-status checks."
196 | [check-status]
197 | (str "http://" (sget-in check-status [:hosts :hostname]) (sget-in check-status [:checks :path])))
198 |
199 | (defn get-webhooks []
200 | (k/select webhooks))
201 |
202 | (defn create-role [fields]
203 | (k/insert roles (k/values fields)))
204 |
205 | (defn create-check [fields]
206 | (k/insert checks (k/values fields)))
207 |
208 | (defn create-host [fields]
209 | (k/insert hosts (k/values fields)))
210 |
211 | (defn find-or-create-host [hostname]
212 | (or (first (k/select hosts
213 | (k/where {:hostname hostname})))
214 | (k/insert hosts (k/values {:hostname hostname}))))
215 |
216 | (defn update-check-status [id fields]
217 | {:pre [(number? id)]}
218 | (k/update check-statuses
219 | (k/set-fields fields)
220 | (k/where {:id id})))
221 |
222 | (defn ready-to-perform
223 | "True if enough time has elapsed since we last checked this alert and the alert's role isn't
224 | snoozed."
225 | [check-status]
226 | (let [cur-time (time-core/now)
227 | role-id (-> check-status :checks :role_id)
228 | last-checked-at (-?> (sget check-status :last_checked_at)
229 | time-coerce/to-date-time)]
230 | (when (not (role-snoozed? (get-role-by-id role-id) cur-time))
231 | (or (nil? last-checked-at)
232 | (->> (sget-in check-status [:checks :interval])
233 | time-core/secs
234 | (time-core/plus last-checked-at)
235 | (time-core/after? cur-time))))))
236 |
--------------------------------------------------------------------------------
/src/watchman/pinger.clj:
--------------------------------------------------------------------------------
1 | (ns watchman.pinger
2 | (:require [cheshire.core :as json]
3 | [clojure.core.incubator :refer [-?>]]
4 | [clj-time.core :as time-core]
5 | [clj-time.coerce :as time-coerce]
6 | [clj-time.format :as time-format]
7 | [clj-http.client :as http]
8 | [clojure.string :as string]
9 | [korma.incubator.core :as k]
10 | [overtone.at-at :as at-at]
11 | [watchman.models :as models]
12 | [net.cgrand.enlive-html :refer [content deftemplate do-> html-content set-attr substitute]]
13 | [watchman.utils :refer [get-env-var log-exception log-info sget sget-in truncate-string]]
14 | [postal.core :as postal])
15 | (:import org.apache.http.conn.ConnectTimeoutException
16 | java.net.SocketTimeoutException))
17 |
18 | (def log-emails-without-sending (= "true" (System/getenv "WATCHMAN_LOG_EMAILS_WITHOUT_SENDING")))
19 |
20 | (def smtp-credentials {:user (get-env-var "WATCHMAN_SMTP_USERNAME")
21 | :pass (get-env-var "WATCHMAN_SMTP_PASSWORD")
22 | :host (or (get-env-var "WATCHMAN_SMTP_HOST")
23 | ; Default to Amazon's Simple Email Service.
24 | "email-smtp.us-east-1.amazonaws.com")
25 | :port 587
26 | :tls true})
27 |
28 | (def from-email-address (get-env-var "WATCHMAN_FROM_EMAIL_ADDRESS"))
29 | (def to-email-address (get-env-var "WATCHMAN_TO_EMAIL_ADDRESS"))
30 |
31 | (def watchman-host (or (System/getenv "WATCHMAN_HOST") "localhost:8130"))
32 |
33 | (def polling-frequency-ms 5000)
34 |
35 | (def response-body-size-limit-chars 10000)
36 |
37 | (def at-at-pool (at-at/mk-pool))
38 |
39 | (def checks-in-progress
40 | "A map check-status-id -> {:in-progress, :attempt-number}. This is a bookkeeping map, for handling retries."
41 | (atom {}))
42 |
43 | (defn- add-role-to-email-address
44 | "Adds the name of the role to the email address using this format: username+role@domain.com. This makes
45 | filtering emails for a given role easier."
46 | [role-name email-address]
47 | (let [[username domain] (string/split email-address #"@")
48 | ; Make sure the role name is suitable for being embedded in an email address.
49 | escaped-role-name (-> role-name
50 | (string/replace #"\s+" "_")
51 | (string/replace #"[^\w]" "")
52 | string/lower-case)]
53 | (format "%s+%s@%s" username escaped-role-name domain)))
54 |
55 | (deftemplate alert-email-html-template "alert_email.html"
56 | [check-status response-body]
57 | [:#check-url] (do-> (set-attr "href" (models/get-url-of-check-status check-status))
58 | (content (models/get-url-of-check-status check-status)))
59 | [:#ssh-link] (set-attr "href" (format "http://%s/ssh_redirect?host_id=%s" watchman-host
60 | (sget check-status :host_id)))
61 | [:#status] (substitute (sget check-status :status))
62 | [:#http-status] (substitute (str (sget check-status :last_response_status_code)))
63 | [:.unique-message-id] (content (->> (time-core/now)
64 | time-coerce/to-date-time
65 | (time-format/unparse (:date-time time-format/formatters))))
66 | [:#additional-details] (if (= (sget check-status :status) "down")
67 | identity
68 | (substitute nil))
69 | [:#response-body] (if (= (sget check-status :status) "up")
70 | (substitute nil)
71 | (html-content response-body)))
72 |
73 | (defn- escape-html-chars [s]
74 | (-> s (string/replace "&" "&") (string/replace "<" "<") (string/replace ">" ">")))
75 |
76 | (defn- alert-email-html [check-status]
77 | (let [response-body (-> check-status
78 | (sget :last_response_body)
79 | str
80 | (truncate-string response-body-size-limit-chars))
81 | response-body (if (= (sget check-status :last_response_content_type) "text/html")
82 | response-body
83 | (-> response-body escape-html-chars (string/replace "\n" "