├── .gitignore
├── .travis.yml
├── CODEOWNERS
├── LICENSE
├── README.md
├── jenkins
└── deploy.sh
├── project.clj
├── src
└── puppetlabs
│ └── structured_logging
│ ├── core.clj
│ └── protocols.clj
└── test
└── puppetlabs
└── structured_logging
└── core_test.clj
/.gitignore:
--------------------------------------------------------------------------------
1 | /target
2 | /classes
3 | /checkouts
4 | pom.xml
5 | pom.xml.asc
6 | *.jar
7 | *.class
8 | /.lein-*
9 | /.nrepl-port
10 | .hgignore
11 | .hg/
12 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: clojure
2 |
3 | jdk:
4 | - openjdk8
5 | - oraclejdk8
6 |
7 | notifications:
8 | email: false
9 |
--------------------------------------------------------------------------------
/CODEOWNERS:
--------------------------------------------------------------------------------
1 | * @puppetlabs/dumpling
2 | * @puppetlabs/skeletor
3 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 |
2 | Apache License
3 | Version 2.0, January 2004
4 | http://www.apache.org/licenses/
5 |
6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7 |
8 | 1. Definitions.
9 |
10 | "License" shall mean the terms and conditions for use, reproduction,
11 | and distribution as defined by Sections 1 through 9 of this document.
12 |
13 | "Licensor" shall mean the copyright owner or entity authorized by
14 | the copyright owner that is granting the License.
15 |
16 | "Legal Entity" shall mean the union of the acting entity and all
17 | other entities that control, are controlled by, or are under common
18 | control with that entity. For the purposes of this definition,
19 | "control" means (i) the power, direct or indirect, to cause the
20 | direction or management of such entity, whether by contract or
21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
22 | outstanding shares, or (iii) beneficial ownership of such entity.
23 |
24 | "You" (or "Your") shall mean an individual or Legal Entity
25 | exercising permissions granted by this License.
26 |
27 | "Source" form shall mean the preferred form for making modifications,
28 | including but not limited to software source code, documentation
29 | source, and configuration files.
30 |
31 | "Object" form shall mean any form resulting from mechanical
32 | transformation or translation of a Source form, including but
33 | not limited to compiled object code, generated documentation,
34 | and conversions to other media types.
35 |
36 | "Work" shall mean the work of authorship, whether in Source or
37 | Object form, made available under the License, as indicated by a
38 | copyright notice that is included in or attached to the work
39 | (an example is provided in the Appendix below).
40 |
41 | "Derivative Works" shall mean any work, whether in Source or Object
42 | form, that is based on (or derived from) the Work and for which the
43 | editorial revisions, annotations, elaborations, or other modifications
44 | represent, as a whole, an original work of authorship. For the purposes
45 | of this License, Derivative Works shall not include works that remain
46 | separable from, or merely link (or bind by name) to the interfaces of,
47 | the Work and Derivative Works thereof.
48 |
49 | "Contribution" shall mean any work of authorship, including
50 | the original version of the Work and any modifications or additions
51 | to that Work or Derivative Works thereof, that is intentionally
52 | submitted to Licensor for inclusion in the Work by the copyright owner
53 | or by an individual or Legal Entity authorized to submit on behalf of
54 | the copyright owner. For the purposes of this definition, "submitted"
55 | means any form of electronic, verbal, or written communication sent
56 | to the Licensor or its representatives, including but not limited to
57 | communication on electronic mailing lists, source code control systems,
58 | and issue tracking systems that are managed by, or on behalf of, the
59 | Licensor for the purpose of discussing and improving the Work, but
60 | excluding communication that is conspicuously marked or otherwise
61 | designated in writing by the copyright owner as "Not a Contribution."
62 |
63 | "Contributor" shall mean Licensor and any individual or Legal Entity
64 | on behalf of whom a Contribution has been received by Licensor and
65 | subsequently incorporated within the Work.
66 |
67 | 2. Grant of Copyright License. Subject to the terms and conditions of
68 | this License, each Contributor hereby grants to You a perpetual,
69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70 | copyright license to reproduce, prepare Derivative Works of,
71 | publicly display, publicly perform, sublicense, and distribute the
72 | Work and such Derivative Works in Source or Object form.
73 |
74 | 3. Grant of Patent License. Subject to the terms and conditions of
75 | this License, each Contributor hereby grants to You a perpetual,
76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77 | (except as stated in this section) patent license to make, have made,
78 | use, offer to sell, sell, import, and otherwise transfer the Work,
79 | where such license applies only to those patent claims licensable
80 | by such Contributor that are necessarily infringed by their
81 | Contribution(s) alone or by combination of their Contribution(s)
82 | with the Work to which such Contribution(s) was submitted. If You
83 | institute patent litigation against any entity (including a
84 | cross-claim or counterclaim in a lawsuit) alleging that the Work
85 | or a Contribution incorporated within the Work constitutes direct
86 | or contributory patent infringement, then any patent licenses
87 | granted to You under this License for that Work shall terminate
88 | as of the date such litigation is filed.
89 |
90 | 4. Redistribution. You may reproduce and distribute copies of the
91 | Work or Derivative Works thereof in any medium, with or without
92 | modifications, and in Source or Object form, provided that You
93 | meet the following conditions:
94 |
95 | (a) You must give any other recipients of the Work or
96 | Derivative Works a copy of this License; and
97 |
98 | (b) You must cause any modified files to carry prominent notices
99 | stating that You changed the files; and
100 |
101 | (c) You must retain, in the Source form of any Derivative Works
102 | that You distribute, all copyright, patent, trademark, and
103 | attribution notices from the Source form of the Work,
104 | excluding those notices that do not pertain to any part of
105 | the Derivative Works; and
106 |
107 | (d) If the Work includes a "NOTICE" text file as part of its
108 | distribution, then any Derivative Works that You distribute must
109 | include a readable copy of the attribution notices contained
110 | within such NOTICE file, excluding those notices that do not
111 | pertain to any part of the Derivative Works, in at least one
112 | of the following places: within a NOTICE text file distributed
113 | as part of the Derivative Works; within the Source form or
114 | documentation, if provided along with the Derivative Works; or,
115 | within a display generated by the Derivative Works, if and
116 | wherever such third-party notices normally appear. The contents
117 | of the NOTICE file are for informational purposes only and
118 | do not modify the License. You may add Your own attribution
119 | notices within Derivative Works that You distribute, alongside
120 | or as an addendum to the NOTICE text from the Work, provided
121 | that such additional attribution notices cannot be construed
122 | as modifying the License.
123 |
124 | You may add Your own copyright statement to Your modifications and
125 | may provide additional or different license terms and conditions
126 | for use, reproduction, or distribution of Your modifications, or
127 | for any such Derivative Works as a whole, provided Your use,
128 | reproduction, and distribution of the Work otherwise complies with
129 | the conditions stated in this License.
130 |
131 | 5. Submission of Contributions. Unless You explicitly state otherwise,
132 | any Contribution intentionally submitted for inclusion in the Work
133 | by You to the Licensor shall be under the terms and conditions of
134 | this License, without any additional terms or conditions.
135 | Notwithstanding the above, nothing herein shall supersede or modify
136 | the terms of any separate license agreement you may have executed
137 | with Licensor regarding such Contributions.
138 |
139 | 6. Trademarks. This License does not grant permission to use the trade
140 | names, trademarks, service marks, or product names of the Licensor,
141 | except as required for reasonable and customary use in describing the
142 | origin of the Work and reproducing the content of the NOTICE file.
143 |
144 | 7. Disclaimer of Warranty. Unless required by applicable law or
145 | agreed to in writing, Licensor provides the Work (and each
146 | Contributor provides its Contributions) on an "AS IS" BASIS,
147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 | implied, including, without limitation, any warranties or conditions
149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 | PARTICULAR PURPOSE. You are solely responsible for determining the
151 | appropriateness of using or redistributing the Work and assume any
152 | risks associated with Your exercise of permissions under this License.
153 |
154 | 8. Limitation of Liability. In no event and under no legal theory,
155 | whether in tort (including negligence), contract, or otherwise,
156 | unless required by applicable law (such as deliberate and grossly
157 | negligent acts) or agreed to in writing, shall any Contributor be
158 | liable to You for damages, including any direct, indirect, special,
159 | incidental, or consequential damages of any character arising as a
160 | result of this License or out of the use or inability to use the
161 | Work (including but not limited to damages for loss of goodwill,
162 | work stoppage, computer failure or malfunction, or any and all
163 | other commercial damages or losses), even if such Contributor
164 | has been advised of the possibility of such damages.
165 |
166 | 9. Accepting Warranty or Additional Liability. While redistributing
167 | the Work or Derivative Works thereof, You may choose to offer,
168 | and charge a fee for, acceptance of support, warranty, indemnity,
169 | or other liability obligations and/or rights consistent with this
170 | License. However, in accepting such obligations, You may act only
171 | on Your own behalf and on Your sole responsibility, not on behalf
172 | of any other Contributor, and only if You agree to indemnify,
173 | defend, and hold each Contributor harmless for any liability
174 | incurred by, or claims asserted against, such Contributor by reason
175 | of your accepting any such warranty or additional liability.
176 |
177 | END OF TERMS AND CONDITIONS
178 |
179 | APPENDIX: How to apply the Apache License to your work.
180 |
181 | To apply the Apache License to your work, attach the following
182 | boilerplate notice, with the fields enclosed by brackets "[]"
183 | replaced with your own identifying information. (Don't include
184 | the brackets!) The text should be enclosed in the appropriate
185 | comment syntax for the file format. We also recommend that a
186 | file or class name and description of purpose be included on the
187 | same "printed page" as the copyright notice for easier
188 | identification within third-party archives.
189 |
190 | Copyright [yyyy] [name of copyright owner]
191 |
192 | Licensed under the Apache License, Version 2.0 (the "License");
193 | you may not use this file except in compliance with the License.
194 | You may obtain a copy of the License at
195 |
196 | http://www.apache.org/licenses/LICENSE-2.0
197 |
198 | Unless required by applicable law or agreed to in writing, software
199 | distributed under the License is distributed on an "AS IS" BASIS,
200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201 | See the License for the specific language governing permissions and
202 | limitations under the License.
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # puppetlabs/structured-logging [](https://clojars.org/puppetlabs/structured-logging) [](https://travis-ci.org/puppetlabs/structured-logging)
2 |
3 | `structured-logging` is a library that helps you to:
4 | - write arbitrary JSON to your logs, making it easier to interface with log analysis tools.
5 | - write messages to arbitrarily named loggers, instead of just the one named
6 | after the namespace in which the log statement appears.
7 |
8 | It is built on `clojure.tools.logging`, but it only works with logback.
9 |
10 | ## Usage
11 |
12 | ### maplog
13 |
14 | Structured log entries are created by calling `maplog` like this:
15 |
16 | (maplog :warn {:user-service "https://...", :status 503, :elapsed 27}
17 | #(format "Failed to query user-service %s. Response: status %d"
18 | (:user-service %) (:status %)))
19 |
20 | The second parameter is the structured data you want to log, and any
21 | clojure map is ok, as long as
22 | [Cheshire](https://github.com/dakrone/cheshire) can handle it.
23 |
24 | The log message itself is generated by passing the structured data to
25 | the function provided as the final argument. This provides a general
26 | way to support arbitrary formatters, including some like those in
27 | [clj-i18n](https://github.com/puppetlabs/clj-i18n), which require
28 | that their invocations appear literally, at each translation call
29 | site, i.e.:
30 |
31 | (maplog :info {:temperature t :stamp (now)}
32 | #(i18n/trs "Temperature has exceeded {0} at {1}"
33 | (:temperature %) (:stamp %)))
34 |
35 | ### Named loggers
36 |
37 | When you would pass a log-level parameter, you may instead supply a vector of
38 | `[:custom-logger :level]`. This will log a message to the logger named
39 | "custom-logger". Then you can specifically address that logger in your logback
40 | configuration (to redirect it to a different log file, for example).
41 |
42 | ## Common fields
43 |
44 | If you use the recommended logger configuration, as described below, you will
45 | see the following fields in each JSON log message:
46 |
47 | * @timestamp
48 | * message
49 | * logger_name
50 | * thread_name
51 | * level
52 | * level_value (numeric, suitable for sorting)
53 | * stack_trace
54 |
55 | Additional relevant fields may be added in any given message, but this base set
56 | will always be present.
57 |
58 | ## Configuration
59 | ### JSON text
60 |
61 | If you want to log JSON data where you would otherwise log regular text, replace the `encoder` element in
62 | your `logback.xml` with this one:
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 | Even though this says 'logstash' on it, it works completely independently from
78 | any log aggregation system. The final `` element is
79 | important - that inserts our custom properties into each json message.
80 |
81 | ## Logstash integration
82 |
83 | You can also log directly to logstash with an appender configured like this:
84 |
85 |
86 | my.great.logstash.server.com
87 | 4560
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 | You will also need to add a reference to the appender from the `` element:
104 |
105 |
106 | ...
107 |
108 |
109 |
110 |
111 |
112 | ## Log Output
113 |
114 | We are using a json encoder that comes with the `logstash-logback` integration
115 | library. (https://github.com/logstash/logstash-logback-encoder) This can be used
116 | in conjunction with the logstash tcp appender, or in conjunction with a typical
117 | file appender. In either case, the json for the above log message looks
118 | something like this:
119 |
120 | {
121 | "@timestamp": "...",
122 | "level": "WARN",
123 | "user-service": "https://...",
124 | "status": 503,
125 | "elapsed": 27
126 | }
127 |
128 | The exact fields that show up depend on how you've configured the encoder;
129 | '@timestamp' is a logstash convention.
130 |
131 | ## Answers to expected questions
132 |
133 | *Can I embed arbitrary structures in there?*
134 |
135 | You can log anything that cheshire can serialize, but you may wish to stick
136 | to simple key-value formats to keep your logs easy to analyze.
137 |
138 | *What happens when I log a map with :level as the key?*
139 |
140 | Don't do that. It's not yet clear how this case should be handled. For now,
141 | avoid such keys. Namespacing your map keys may be prudent if you're worried
142 | about collisions.
143 |
144 | *Does this library require logstash?*
145 |
146 | No. It does depend on the json encoder that comes with the logback-logstash
147 | integration library, but you can point the json it spits out at any old logback
148 | appender. Critically, it has a feature that allows us to edit the json before it
149 | gets written, which is where we add our custom information. We *could* write
150 | such an encoder ourselves, but they already wrote it.
151 |
152 | *Why didn't you use one of the other logback-json encoders?*
153 |
154 | Because those just encode the normal log event fields as json; the useful
155 | feature here is the ability to include arbitrary data as part of the json.
156 |
157 | ## Tips
158 |
159 | Structured logging is a slippery slope and should be applied where it has
160 | clear benefits. While we've streamlined it a lot, this is heavier than regular
161 | logging, and it's still non-application code that you have to scatter throughout
162 | your program.
163 |
164 | ## License
165 |
166 | Copyright © 2015 Puppet Labs
167 |
168 | Distributed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html)
169 |
--------------------------------------------------------------------------------
/jenkins/deploy.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | set -e
4 | set -x
5 |
6 | git fetch --tags
7 |
8 | lein test
9 | echo "Tests passed!"
10 |
11 | lein release
12 | echo "Release plugin successful, pushing changes to git"
13 |
14 | git push origin --tags HEAD:master
15 |
16 | echo "git push successful."
17 |
--------------------------------------------------------------------------------
/project.clj:
--------------------------------------------------------------------------------
1 | (defproject puppetlabs/structured-logging "0.2.1-SNAPSHOT"
2 | :description "Write data structures to your logs from clojure."
3 | :url "https://github.com/puppetlabs/structured-logging"
4 | :license {:name "Apache License, Version 2.0"
5 | :url "http://www.apache.org/licenses/LICENSE-2.0.html"}
6 | :dependencies [[org.clojure/clojure "1.8.0"]
7 | [net.logstash.logback/logstash-logback-encoder "4.6"]
8 | [org.clojure/tools.logging "0.3.1"]
9 | [cheshire "5.6.3"]
10 | [org.slf4j/slf4j-api "1.7.21"]]
11 | :plugins [[lein-release "1.0.5"]]
12 | :deploy-repositories [["releases" {:url "https://clojars.org/repo"
13 | :username :env/clojars_jenkins_username
14 | :password :env/clojars_jenkins_password
15 | :sign-releases false}]]
16 | :lein-release {:scm :git
17 | :deploy-via :lein-deploy})
18 |
--------------------------------------------------------------------------------
/src/puppetlabs/structured_logging/core.clj:
--------------------------------------------------------------------------------
1 | (ns puppetlabs.structured-logging.core
2 | (:require [clojure.tools.logging :refer [*logger-factory*]]
3 | [clojure.tools.logging.impl :as impl]
4 | [puppetlabs.structured-logging.protocols :refer :all]
5 | [cheshire.core :as cheshire])
6 | (:import [net.logstash.logback.marker Markers LogstashMarker]
7 | [com.fasterxml.jackson.core JsonGenerator]))
8 |
9 | (definterface ISemlogMarker
10 | (semlogMap [] "Returns the semlog map for this marker."))
11 |
12 | (defn- merge-clojure-map-marker
13 | "Create a marker that, when written to the LogStash json encoder, will
14 | json-encode the given map `m` and merge it with any already-created json.
15 |
16 | Use the following encoder configuration inside your logback appender
17 | configuration to write your log messages as json:
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 | "
32 | [m]
33 | (proxy [LogstashMarker ISemlogMarker] ["SEMLOG_MAP"]
34 | (writeTo [^JsonGenerator generator]
35 | (binding [cheshire/*generator* generator]
36 | ;; `::none` is the 'wholeness' parameter to cheshire, which indicates which
37 | ;; start- and end-object markers to write. In this case we don't want any
38 | ;; of them, so we'll pass `::none`. This is not on the list of supported values,
39 | ;; but the fact that it's not equal to any of them means it will work.
40 | (cheshire/write m ::none)))
41 | (semlogMap [] m)))
42 |
43 | (extend-protocol MarkerLogger
44 | org.slf4j.Logger
45 | (write-with-marker! [logger level e msg marker-map]
46 | (let [^String msg (str msg)
47 | marker (merge-clojure-map-marker marker-map)]
48 | (if e
49 | (case level
50 | :trace (.trace logger marker msg e)
51 | :debug (.debug logger marker msg e)
52 | :info (.info logger marker msg e)
53 | :warn (.warn logger marker msg e)
54 | :error (.error logger marker msg e)
55 | :fatal (.error logger marker msg e)
56 | (throw (IllegalArgumentException. (str level))))
57 | (case level
58 | :trace (.trace logger marker msg)
59 | :debug (.debug logger marker msg)
60 | :info (.info logger marker msg)
61 | :warn (.warn logger marker msg)
62 | :error (.error logger marker msg)
63 | :fatal (.error logger marker msg)
64 | (throw (IllegalArgumentException. (str level))))))))
65 |
66 | (defmacro maplog
67 | "Logs an event with ctx-map as an slf4j event Marker.
68 | The logger-level parameter may be either a log level keyword like
69 | :error or a vector of a custom logger and the log level, like
70 | [:sync :error]. Calls (create-message ctx-map) to generate the
71 | log message string.
72 |
73 | Examples:
74 |
75 | (maplog :info {:status 200} #(str \"Received success status \" (:status %))
76 |
77 | (maplog [:sync :warn] {:remote ..., :response ...}
78 | #(format \"Failed to pull record from remote %s. Response: status %s\"
79 | (:remote %) (:status %)))
80 |
81 | (maplog [:sync :info] {:remote ...}
82 | #(format \"Finished pull from %s in %s seconds\"
83 | sync-time (:remote %)))
84 |
85 | (maplog :info {:status 200}
86 | #(i18n/trs \"Received success status {0}\" (:status %1)))"
87 | ([logger-level ctx-map create-message]
88 | `(maplog ~logger-level nil ~ctx-map ~create-message))
89 | ([logger-level throwable ctx-map create-message]
90 | `(let [logger-level# ~logger-level
91 | [ns# level#] (if (coll? logger-level#)
92 | logger-level#
93 | [~*ns* logger-level#])
94 | ns# (if (keyword? ns#) (name ns#) ns#)
95 | logger# (impl/get-logger *logger-factory* ns#)
96 | ctx-map# ~ctx-map]
97 | (when (impl/enabled? logger# level#)
98 | (write-with-marker! logger# level#
99 | ~throwable
100 | (~create-message ctx-map#)
101 | ctx-map#)))))
102 |
--------------------------------------------------------------------------------
/src/puppetlabs/structured_logging/protocols.clj:
--------------------------------------------------------------------------------
1 | (ns puppetlabs.structured-logging.protocols)
2 |
3 | (defprotocol MarkerLogger
4 | (write-with-marker! [^org.slf4j.Logger logger level ^Throwable e msg marker]))
5 |
--------------------------------------------------------------------------------
/test/puppetlabs/structured_logging/core_test.clj:
--------------------------------------------------------------------------------
1 | (ns puppetlabs.structured-logging.core-test
2 | (:require [clojure.test :refer :all]
3 | [clojure.tools.logging :refer [*logger-factory*]]
4 | [clojure.tools.logging.impl :as impl]
5 | [puppetlabs.structured-logging.core :refer :all]
6 | [puppetlabs.structured-logging.protocols :refer :all]))
7 |
8 | (def ^:private stns *ns*)
9 |
10 | (defn make-log-record [logger-ns level ex message marker]
11 | (letfn [(maybe-add-exception [rec]
12 | (if ex (assoc rec :throwable ex) rec))
13 | (maybe-add-marker [rec]
14 | (if marker
15 | (assoc rec :marker marker)
16 | rec))]
17 | (-> {:ns logger-ns, :level level, :message message}
18 | maybe-add-exception
19 | maybe-add-marker)))
20 |
21 | (defn atom-logger-factory [log-atom]
22 | (reify impl/LoggerFactory
23 | (name [_] "atomLogger")
24 | (get-logger [_ logger-ns]
25 | (reify
26 | impl/Logger
27 | (enabled? [_ _] true)
28 | (write! [_ level ex message]
29 | (swap! log-atom conj (make-log-record logger-ns level ex message nil)))
30 | MarkerLogger
31 | (write-with-marker! [logger level ex message marker]
32 | (swap! log-atom conj (make-log-record logger-ns level ex message marker)))))))
33 |
34 | (defn expect-log [f expected-log]
35 | (let [log (atom [])]
36 | (binding [*logger-factory* (atom-logger-factory log)]
37 | (f))
38 | (is (= expected-log @log))))
39 |
40 | (def throwable (Exception. "ex"))
41 |
42 | (deftest maplog-behavior
43 | (are [f expected] (expect-log f expected)
44 | #(maplog :error {:x 1} (fn [m] (str "Test " (:x m))))
45 | [{:ns stns, :level :error, :message "Test 1" :marker {:x 1}}]
46 |
47 | #(maplog [:sync :error] {:x 2} (fn [m] (str "Test " (:x m))))
48 | [{:ns "sync", :level :error, :message "Test 2" :marker {:x 2}}]
49 |
50 | #(maplog [:sync :error] throwable {:x 3} (fn [m] (str "Test " (:x m))))
51 | [{:ns "sync", :level :error, :message "Test 3", :throwable throwable
52 | :marker {:x 3}}]))
53 |
--------------------------------------------------------------------------------