├── .gitignore ├── Package.swift ├── README.md ├── Sources └── StackdriverLogging │ └── StackdriverLogHandler.swift └── Tests └── StackdriverLoggingTests └── StackdriverLoggerTests.swift /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | /.build 3 | /Packages 4 | /*.xcodeproj 5 | Package.resolved 6 | .swiftpm 7 | -------------------------------------------------------------------------------- /Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:5.8 2 | // The swift-tools-version declares the minimum version of Swift required to build this package. 3 | 4 | import PackageDescription 5 | 6 | let package = Package( 7 | name: "StackdriverLogging", 8 | platforms: [.macOS(.v13), .iOS(.v16)], 9 | products: [ 10 | .library(name: "StackdriverLogging", targets: ["StackdriverLogging"]), 11 | ], 12 | dependencies: [ 13 | // Swift logging API 14 | .package(url: "https://github.com/apple/swift-log.git", from: "1.5.0"), 15 | 16 | // Used for threadPool 17 | .package(url: "https://github.com/apple/swift-nio.git", from: "2.64.0"), 18 | 19 | // Used for fileIO 20 | .package(url: "https://github.com/apple/swift-system.git", from: "1.2.1"), 21 | ], 22 | targets: [ 23 | .target( 24 | name: "StackdriverLogging", 25 | dependencies: [ 26 | .product(name: "Logging", package: "swift-log"), 27 | .product(name: "NIO", package: "swift-nio"), 28 | .product(name: "SystemPackage", package: "swift-system"), 29 | ] 30 | ), 31 | .testTarget(name: "StackdriverLoggingTests", dependencies: ["StackdriverLogging"]), 32 | ] 33 | ) 34 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # StackdriverLogging 2 | A [SwiftLog](https://github.com/apple/swift-log) `LogHandler` that logs GCP Stackdriver formatted JSON. 3 | 4 | For more information on Stackdriver structured logging, see: https://cloud.google.com/logging/docs/structured-logging and [LogEntry](https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry) 5 | 6 | ## Dependencies 7 | This Stackdriver `LogHandler` depends on [SwiftNIO](https://github.com/apple/swift-nio) which is used to create and save your new log entries in a non-blocking fashion. 8 | 9 | ## How to install 10 | 11 | ### Swift Package Manager 12 | 13 | ```swift 14 | .package(url: "https://github.com/Brainfinance/StackdriverLogging.git", from: "4.0.0"), 15 | ``` 16 | In your target's dependencies add `"StackdriverLogging"` e.g. like this: 17 | ```swift 18 | .target(name: "App", dependencies: ["StackdriverLogging"]), 19 | ``` 20 | 21 | ### Vapor 4 22 | Here's a bootstrapping example for a standard Vapor 4 application. 23 | ```swift 24 | import App 25 | import Vapor 26 | 27 | var env = try Environment.detect() 28 | try LoggingSystem.bootstrap(from: &env) { (logLevel) -> (String) -> LogHandler in 29 | return { label -> LogHandler in 30 | var logger = StackdriverLogHandler(destination: .stdout) 31 | logger.logLevel = logLevel 32 | return logger 33 | } 34 | } 35 | let app = Application(env) 36 | defer { app.shutdown() } 37 | try configure(app) 38 | try app.run() 39 | ``` 40 | 41 | ## Logging JSON values using `Logger.MetadataValue` 42 | To log metadata values as JSON, simply log all JSON values other than `String` as a `Logger.MetadataValue.stringConvertible` and, instead of the usual conversion of your value to a `String` in the log entry, it will keep the original JSON type of your values whenever possible. 43 | 44 | For example: 45 | ```Swift 46 | var logger = Logger(label: "Stackdriver") 47 | logger[metadataKey: "jsonpayload-example-object"] = [ 48 | "json-null": .stringConvertible(NSNull()), 49 | "json-bool": .stringConvertible(true), 50 | "json-integer": .stringConvertible(1), 51 | "json-float": .stringConvertible(1.5), 52 | "json-string": .string("Example"), 53 | "stackdriver-timestamp": .stringConvertible(Date()), 54 | "json-array-of-numbers": [.stringConvertible(1), .stringConvertible(5.8)], 55 | "json-object": [ 56 | "key": "value" 57 | ] 58 | ] 59 | logger.info("test") 60 | ``` 61 | Will log the non pretty-printed representation of: 62 | ```json 63 | { 64 | "sourceLocation":{ 65 | "function":"boot(_:)", 66 | "file":"\/Sources\/App\/boot.swift", 67 | "line":25 68 | }, 69 | "jsonpayload-example-object":{ 70 | "json-bool":true, 71 | "json-float":1.5, 72 | "json-string":"Example", 73 | "json-object":{ 74 | "key":"value" 75 | }, 76 | "json-null":null, 77 | "json-integer":1, 78 | "json-array-of-numbers":[ 79 | 1, 80 | 5.8 81 | ], 82 | "stackdriver-timestamp":"2019-07-15T21:21:02.451Z" 83 | }, 84 | "message":"test", 85 | "severity":"INFO" 86 | } 87 | ``` 88 | 89 | ## Logging from a managed platform 90 | If your app is running inside a managed environment such as Google Cloud Run or a container based Compute Engine, logging to stdout should get you up and running automatically. 91 | 92 | ## Stackdriver logging agent + fluentd config 93 | If you prefer logging to a file, you can use a file destination `StackdriverLogHandler.Destination.file` in combination with the Stackdriver logging agent https://cloud.google.com/logging/docs/agent/installation and a matching json format 94 | google-fluentd config (/etc/google-fluentd/config.d/example.conf) to automatically send your JSON logs to Stackdriver for you. 95 | 96 | Here's an example google-fluentd conf file that monitors a json based logfile and send new log entries to Stackdriver: 97 | ``` 98 | 99 | @type tail 100 | # Format 'JSON' indicates the log is structured (JSON). 101 | format json 102 | # The path of the log file. 103 | path /var/log/example.log 104 | # The path of the position file that records where in the log file 105 | # we have processed already. This is useful when the agent 106 | # restarts. 107 | pos_file /var/lib/google-fluentd/pos/example-log.pos 108 | read_from_head true 109 | # The log tag for this log input. 110 | tag exampletag 111 | 112 | ``` 113 | -------------------------------------------------------------------------------- /Sources/StackdriverLogging/StackdriverLogHandler.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import Logging 3 | import NIO 4 | import SystemPackage 5 | 6 | /// `LogHandler` to log JSON to GCP Stackdriver using a fluentd config and the GCP logging-assistant. 7 | /// Use the `MetadataValue.stringConvertible` case to log non-string JSON values supported by JSONSerializer such as NSNull, Bool, Int, Float/Double, NSNumber, etc. 8 | /// The `MetadataValue.stringConvertible` type will also take care of automatically logging `Date` as an iso8601 timestamp and `Data` as a base64 9 | /// encoded `String`. 10 | /// 11 | /// The log entry format matches https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry 12 | /// 13 | /// ** Use the `StackdriverLogHandler.Factory` to instantiate new `StackdriverLogHandler` instances. 14 | public struct StackdriverLogHandler: LogHandler { 15 | /// A `StackdriverLogHandler` output destination, can be either the standard output or a file. 16 | public struct Destination: CustomStringConvertible { 17 | internal enum Kind { 18 | case file(_ filepath: String) 19 | case stdout 20 | } 21 | 22 | internal var kind: Kind 23 | internal var fd: SystemPackage.FileDescriptor 24 | 25 | public static func file(_ filepath: String) throws -> Destination { 26 | return .init( 27 | kind: .file(filepath), 28 | fd: try SystemPackage.FileDescriptor.open( 29 | FilePath(filepath), 30 | .writeOnly, 31 | options: [.append, .create], 32 | permissions: FilePermissions(rawValue: 0o644) 33 | ) 34 | ) 35 | } 36 | 37 | public static var stdout: Destination { 38 | return .init( 39 | kind: .stdout, 40 | fd: .standardOutput 41 | ) 42 | } 43 | 44 | public var description: String { 45 | switch kind { 46 | case .stdout: 47 | return "standard output" 48 | case .file(let filePath): 49 | return URL(fileURLWithPath: filePath).description 50 | } 51 | } 52 | } 53 | 54 | public var metadata: Logger.Metadata = .init() 55 | public var metadataProvider: Logger.MetadataProvider? = nil 56 | 57 | public var logLevel: Logger.Level = .info 58 | 59 | private var destination: Destination 60 | private var threadPool: NIOThreadPool 61 | private var sourceLocationLogLevel: Logger.Level 62 | 63 | /// Create a new StackdriverLogHandler 64 | /// - Parameters: 65 | /// - destination: The ``Destination`` to write the log entries to, such as a file or stdout 66 | /// - threadPool: The thread pool to use for writing log entries. Defaults to the singleton `NIOThreadPool`. 67 | /// - sourceLocationLogLevel: The level to log the source location at. Defaults to `.trace` - i.e. all levels will log the source location. 68 | public init(destination: Destination, threadPool: NIOThreadPool = .singleton, sourceLocationLogLevel: Logger.Level = .trace) { 69 | self.destination = destination 70 | self.threadPool = threadPool 71 | self.sourceLocationLogLevel = sourceLocationLogLevel 72 | } 73 | 74 | public subscript(metadataKey key: String) -> Logger.Metadata.Value? { 75 | get { 76 | return metadata[key] 77 | } 78 | set(newValue) { 79 | metadata[key] = newValue 80 | } 81 | } 82 | 83 | public func log(level: Logger.Level, message: Logger.Message, metadata: Logger.Metadata?, source: String, file: String, function: String, line: UInt) { 84 | let providerMetadata = self.metadataProvider?.get() 85 | let now = Date() 86 | 87 | // run in threadpool or immediately (when threadpool is inactive) 88 | threadPool.submit { _ in 89 | // JSONSerialization and its internal JSONWriter calls seem to leak significant memory, especially when 90 | // called recursively or in loops. Wrapping the calls in an autoreleasepool fixes the problems entirely on Darwin. 91 | // see: https://bugs.swift.org/browse/SR-5501 92 | withAutoReleasePool { 93 | var entryMetadata: Logger.Metadata = [:] 94 | if let providerMetadata = providerMetadata { 95 | entryMetadata.merge(providerMetadata) { $1 } 96 | } 97 | entryMetadata.merge(self.metadata) { $1 } 98 | if let metadata = metadata { 99 | entryMetadata.merge(metadata) { $1 } 100 | } 101 | 102 | var json = Self.unpackMetadata(.dictionary(entryMetadata)) as! [String: Any] 103 | assert(json["message"] == nil, "'message' is a metadata field reserved by Stackdriver, your custom 'message' metadata value will be overriden in production") 104 | assert(json["severity"] == nil, "'severity' is a metadata field reserved by Stackdriver, your custom 'severity' metadata value will be overriden in production") 105 | assert(json["sourceLocation"] == nil, "'sourceLocation' is a metadata field reserved by Stackdriver, your custom 'sourceLocation' metadata value will be overriden in production") 106 | assert(json["timestamp"] == nil, "'timestamp' is a metadata field reserved by Stackdriver, your custom 'timestamp' metadata value will be overriden in production") 107 | 108 | json["message"] = message.description 109 | json["severity"] = Severity.fromLoggerLevel(level).rawValue 110 | if level >= sourceLocationLogLevel { 111 | json["sourceLocation"] = [ 112 | "file": Self.conciseSourcePath(file), 113 | "line": line, 114 | "function": function, 115 | "source": source, 116 | ] 117 | } 118 | json["timestamp"] = Self.iso8601DateFormatter.string(from: now) 119 | 120 | let entry: Data 121 | do { 122 | var _entry = try JSONSerialization.data(withJSONObject: json, options: []) 123 | _entry.append(0x0A) // Appends a new line at the end of the entry 124 | entry = _entry 125 | } catch { 126 | print("Failed to serialize your log entry metadata to JSON with error: '\(error.localizedDescription)'") 127 | return 128 | } 129 | 130 | do { 131 | try self.destination.fd.writeAll(entry) 132 | } catch { 133 | print("Failed to write logfile entry to '\(self.destination)' with error: '\(error.localizedDescription)'") 134 | } 135 | } 136 | } 137 | } 138 | 139 | /// ISO 8601 `DateFormatter` which is the accepted format for timestamps in Stackdriver 140 | private static var iso8601DateFormatter: DateFormatter { 141 | let key = "StackdriverLogHandler_iso8601DateFormatter" 142 | let threadLocal = Thread.current.threadDictionary 143 | if let value = threadLocal[key] { 144 | return value as! DateFormatter 145 | } 146 | 147 | let formatter = DateFormatter() 148 | formatter.calendar = Calendar(identifier: .iso8601) 149 | formatter.locale = Locale(identifier: "en_US_POSIX") 150 | formatter.timeZone = TimeZone(secondsFromGMT: 0) 151 | formatter.dateFormat = "yyyy-MM-dd'T'HH:mm:ss.SSSXXXXX" 152 | 153 | threadLocal[key] = formatter 154 | return formatter 155 | } 156 | 157 | private static func unpackMetadata(_ value: Logger.MetadataValue) -> Any { 158 | /// Based on the core-foundation implementation of `JSONSerialization.isValidObject`, but optimized to reduce the amount of comparisons done per validation. 159 | /// https://github.com/apple/swift-corelibs-foundation/blob/9e505a94e1749d329563dac6f65a32f38126f9c5/Foundation/JSONSerialization.swift#L52 160 | func isValidJSONValue(_ value: CustomStringConvertible) -> Bool { 161 | if value is Int || value is Bool || value is NSNull || 162 | (value as? Double)?.isFinite ?? false || 163 | (value as? Float)?.isFinite ?? false || 164 | (value as? Decimal)?.isFinite ?? false || 165 | value is UInt || 166 | value is Int8 || value is Int16 || value is Int32 || value is Int64 || 167 | value is UInt8 || value is UInt16 || value is UInt32 || value is UInt64 || 168 | value is String { 169 | return true 170 | } 171 | 172 | // Using the official `isValidJSONObject` call for NSNumber since `JSONSerialization.isValidJSONObject` uses internal/private functions to validate them... 173 | if let number = value as? NSNumber { 174 | return JSONSerialization.isValidJSONObject([number]) 175 | } 176 | 177 | return false 178 | } 179 | 180 | switch value { 181 | case .string(let value): 182 | return value 183 | case .stringConvertible(let value): 184 | if isValidJSONValue(value) { 185 | return value 186 | } else if let date = value as? Date { 187 | return iso8601DateFormatter.string(from: date) 188 | } else if let data = value as? Data { 189 | return data.base64EncodedString() 190 | } else { 191 | return value.description 192 | } 193 | case .array(let value): 194 | return value.map { Self.unpackMetadata($0) } 195 | case .dictionary(let value): 196 | return value.mapValues { Self.unpackMetadata($0) } 197 | } 198 | } 199 | 200 | private static func conciseSourcePath(_ path: String) -> String { 201 | return path.split(separator: "/") 202 | .split(separator: "Sources") 203 | .last? 204 | .joined(separator: "/") ?? path 205 | } 206 | 207 | } 208 | 209 | // Internal Stackdriver and related mapping from `Logger.Level` 210 | extension StackdriverLogHandler { 211 | /// The Stackdriver internal `Severity` levels 212 | fileprivate enum Severity: String { 213 | /// (0) The log entry has no assigned severity level. 214 | case `default` = "DEFAULT" 215 | 216 | /// (100) Debug or trace information. 217 | case debug = "DEBUG" 218 | 219 | /// (200) Routine information, such as ongoing status or performance. 220 | case info = "INFO" 221 | 222 | /// (300) Normal but significant events, such as start up, shut down, or a configuration change. 223 | case notice = "NOTICE" 224 | 225 | /// (400) Warning events might cause problems. 226 | case warning = "WARNING" 227 | 228 | /// (500) Error events are likely to cause problems. 229 | case error = "ERROR" 230 | 231 | /// (600) Critical events cause more severe problems or outages. 232 | case critical = "CRITICAL" 233 | 234 | /// (700) A person must take an action immediately. 235 | case alert = "ALERT" 236 | 237 | /// (800) One or more systems are unusable. 238 | case emergency = "EMERGENCY" 239 | 240 | static func fromLoggerLevel(_ level: Logger.Level) -> Self { 241 | switch level { 242 | case .trace, .debug: 243 | return .debug 244 | case .info: 245 | return .info 246 | case .notice: 247 | return .notice 248 | case .warning: 249 | return .warning 250 | case .error: 251 | return .error 252 | case .critical: 253 | return .critical 254 | } 255 | } 256 | } 257 | } 258 | 259 | // Stackdriver related metadata helpers 260 | extension Logger { 261 | /// Set the metadata for a Stackdriver formatted "LogEntryOperation", i.e used to give a unique tag to all the log entries related to some, potentially long running, operation 262 | /// https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#logentryoperation 263 | public mutating func setLogEntryOperationMetadata(id: String, producer: String?, first: Bool? = nil, last: Bool? = nil) { 264 | var metadataValue: Logger.Metadata = [:] 265 | metadataValue["id"] = .optionalString(id) 266 | metadataValue["producer"] = .optionalString(producer) 267 | metadataValue["first"] = .optionalStringConvertible(first) 268 | metadataValue["last"] = .optionalStringConvertible(last) 269 | self[metadataKey: "operation"] = .dictionary(metadataValue) 270 | } 271 | /// Set the metadata for a Stackdriver formatted "HTTPRequest", i.e to associated a particular HTTPRequest with your log entries. 272 | /// https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#httprequest 273 | public mutating func setHTTPRequestMetadata(requestMethod: String?, 274 | requestUrl: String?, 275 | requestSize: String? = nil, 276 | status: Int? = nil, 277 | responseSize: String? = nil, 278 | userAgent: String? = nil, 279 | remoteIp: String? = nil, 280 | serverIp: String? = nil, 281 | referer: String? = nil, 282 | latency: String? = nil, 283 | cacheLookup: Bool? = nil, 284 | cacheHit: Bool? = nil, 285 | cacheValidatedWithOriginServer: Bool? = nil, 286 | cacheFillBytes: String? = nil, 287 | protocol: String? = nil) { 288 | var metadataValue: Logger.Metadata = [:] 289 | metadataValue["requestMethod"] = .optionalString(requestMethod) 290 | metadataValue["requestUrl"] = .optionalString(requestUrl) 291 | metadataValue["requestSize"] = .optionalString(requestSize) 292 | metadataValue["status"] = .optionalStringConvertible(status) 293 | metadataValue["responseSize"] = .optionalString(responseSize) 294 | metadataValue["userAgent"] = .optionalString(userAgent) 295 | metadataValue["remoteIp"] = .optionalString(remoteIp) 296 | metadataValue["serverIp"] = .optionalString(serverIp) 297 | metadataValue["referer"] = .optionalString(referer) 298 | metadataValue["latency"] = .optionalString(latency) 299 | metadataValue["cacheLookup"] = .optionalStringConvertible(cacheLookup) 300 | metadataValue["cacheHit"] = .optionalStringConvertible(cacheHit) 301 | metadataValue["cacheValidatedWithOriginServer"] = .optionalStringConvertible(cacheValidatedWithOriginServer) 302 | metadataValue["cacheFillBytes"] = .optionalString(cacheFillBytes) 303 | metadataValue["protocol"] = .optionalString(`protocol`) 304 | self[metadataKey: "httpRequest"] = .dictionary(metadataValue) 305 | } 306 | } 307 | 308 | extension Logger.MetadataValue { 309 | fileprivate static func optionalString(_ value: String?) -> Logger.MetadataValue? { 310 | guard let value = value else { 311 | return nil 312 | } 313 | return .string(value) 314 | } 315 | fileprivate static func optionalStringConvertible(_ value: CustomStringConvertible?) -> Logger.MetadataValue? { 316 | guard let value = value else { 317 | return nil 318 | } 319 | return .stringConvertible(value) 320 | } 321 | } 322 | 323 | private func withAutoReleasePool(_ execute: () throws -> T) rethrows -> T { 324 | #if os(Linux) 325 | return try execute() 326 | #else 327 | return try autoreleasepool { 328 | try execute() 329 | } 330 | #endif 331 | } 332 | -------------------------------------------------------------------------------- /Tests/StackdriverLoggingTests/StackdriverLoggerTests.swift: -------------------------------------------------------------------------------- 1 | import NIO 2 | import StackdriverLogging 3 | import XCTest 4 | 5 | final class StackdriverLoggingTests: XCTestCase { 6 | func testStdout() { 7 | let handler = StackdriverLogHandler(destination: .stdout) 8 | handler.log( 9 | level: .error, 10 | message: "test error log 1", 11 | metadata: [ 12 | "test-metadata": "hello", 13 | ], 14 | source: "StackdriverLoggingTests", 15 | file: #file, 16 | function: #function, 17 | line: #line 18 | ) 19 | handler.log( 20 | level: .error, 21 | message: "test error log 2", 22 | metadata: nil, 23 | source: "StackdriverLoggingTests", 24 | file: #file, 25 | function: #function, 26 | line: #line 27 | ) 28 | } 29 | 30 | func testFile() throws { 31 | let tmpPath = NSTemporaryDirectory() + "/\(Self.self)+\(UUID()).log" 32 | 33 | let inactiveTP = NIOThreadPool(numberOfThreads: 1) 34 | let handler = try StackdriverLogHandler(destination: .file(tmpPath), threadPool: inactiveTP) 35 | handler.log( 36 | level: .error, 37 | message: "test error log 1", 38 | metadata: nil, 39 | source: "StackdriverLoggingTests", 40 | file: #file, 41 | function: #function, 42 | line: #line 43 | ) 44 | 45 | for (i, line) in try String(contentsOfFile: tmpPath).split(separator: "\n").enumerated() { 46 | XCTAssertTrue(line.contains("test error log \(i + 1)")) 47 | } 48 | 49 | handler.log( 50 | level: .error, 51 | message: "test error log 2", 52 | metadata: nil, 53 | source: "StackdriverLoggingTests", 54 | file: #file, 55 | function: #function, 56 | line: #line 57 | ) 58 | 59 | for (i, line) in try String(contentsOfFile: tmpPath).split(separator: "\n").enumerated() { 60 | XCTAssertTrue(line.contains("test error log \(i + 1)")) 61 | } 62 | 63 | try FileManager.default.removeItem(atPath: tmpPath) 64 | } 65 | 66 | func testSourceLocationLogLevel() throws { 67 | let tmpPath = NSTemporaryDirectory() + "/\(Self.self)+\(UUID()).log" 68 | let handler = try StackdriverLogHandler(destination: .file(tmpPath), sourceLocationLogLevel: .error) 69 | handler.log(level: .warning, message: "Some message", metadata: nil, source: "StackdriverLoggingTests", file: #file, function: #function, line: #line) 70 | 71 | for line in try String(contentsOfFile: tmpPath).split(separator: "\n") { 72 | XCTAssertFalse(line.contains("sourceLocation")) 73 | } 74 | 75 | handler.log(level: .error, message: "Some message", metadata: nil, source: "StackdriverLoggingTests", file: #file, function: #function, line: #line) 76 | 77 | for line in try String(contentsOfFile: tmpPath).split(separator: "\n") { 78 | XCTAssertTrue(line.contains("sourceLocation")) 79 | } 80 | } 81 | } 82 | --------------------------------------------------------------------------------