├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── cmd └── cloudsqltail │ └── main.go ├── go.mod ├── go.sum └── run.sh /.gitignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.13-alpine3.10 AS build 2 | 3 | RUN mkdir /src 4 | WORKDIR /src 5 | 6 | # Do go mod stuff first to cache if it hasn't changed 7 | COPY go.mod . 8 | COPY go.sum . 9 | RUN go mod download 10 | 11 | COPY cmd/ ./cmd 12 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /go/bin/cloudsqltail ./cmd/cloudsqltail 13 | 14 | # runtime container 15 | FROM alpine:3.10 16 | 17 | RUN apk add --update ca-certificates 18 | # honeytail was compiled against libc, not musl, but they're compatible. 19 | RUN mkdir /lib64 && ln -s /lib/libc.musl-x86_64.so.1 /lib64/ld-linux-x86-64.so.2 20 | 21 | RUN mkdir /app 22 | WORKDIR /app 23 | COPY --from=build /go/bin/cloudsqltail ./cloudsqltail 24 | COPY run.sh . 25 | RUN wget -q -O honeytail https://honeycomb.io/download/honeytail/v1.1.4/honeytail-linux-amd64 && \ 26 | echo 'd23e6f1808bdf5f99a2fd4183d143ffaf81f5b41af26c5e0e11a2d45ed04f277 honeytail' | sha256sum -c && \ 27 | chmod 755 ./honeytail 28 | CMD ["ash", "/app/run.sh"] 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2019 Dark, Inc 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY : build 2 | build: dist/cloudsqltail 3 | 4 | dist/cloudsqltail: cmd/cloudsqltail/*.go 5 | go build -o $@ ./cmd/cloudsqltail 6 | 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | CloudSQL to Honeycomb 2 | ===================== 3 | 4 | This is a docker image, to be run in a k8s cluster, to ingest Postgres logs from 5 | [CloudSQL](https://cloud.google.com/sql/docs/postgres/) and send them off to 6 | [honeycomb](https://www.honeycomb.io/). 7 | 8 | The basic infrastructure to accomplish this requires using Cloud Pub/Sub to 9 | send messages between CloudSQL and this program, which sends them to Honeycomb. 10 | 11 | That is: 12 | 13 | CloudSQL -> Stackdriver Logs -> Cloud Pub/Sub -> 14 | cloudsqltail (this) -> honeytail -> honeycomb 15 | 16 | Running the default command of the provided docker image will run a pipeline of 17 | both cloudsqltail and honeytail, with the effect of reading from a specified 18 | Pub/Sub Subscription and writing the resulting events into Honeycomb. 19 | 20 | ## Implementation 21 | 22 | `cloudsqltail` is a small golang program that reads log messages from a Pub/Sub 23 | Subscription, attempting to correctly order them and write them to STDOUT in a 24 | format consumable by `honeytail`. The two details to note here are the 25 | reordering and reformatting. 26 | 27 | Reordering: messages in a Subscription are not guaranteed to be delivered in 28 | any order, which is problematic because Postgres query logs can span multiple 29 | lines. Honeytail expects a strict ordering of log lines to be able to make 30 | sense of the queries. `cloudsqltail` attempts to best-effort order the log 31 | messages it pulls from the Subscription by buffering messages in memory for 32 | some amount of time, sorting them by their nanosecond-resolution timestamp, 33 | then outputing the sorted list. This is not perfect, but is generally good 34 | enough that the mangled or missing query noise becomes statistically 35 | insignificant. See the `-flush-interval` flag of `cloudsqltail` for fine tuning 36 | the time between buffer sort/flush. 37 | 38 | Reformatting: `honeytail` requires a timestamp for each logged query for 39 | accurate event time, which is normally accomplished by modifying the Postgres 40 | `log_line_prefix` configuration to add a timestamp. However, CloudSQL does not 41 | allow configuration of this parameter. Instead, `cloudsqltail` will prepend the 42 | timestamp associated with the Pub/Sub message to the appropriate postgres log 43 | lines, just as if it had been added by Postgres. 44 | 45 | ## CloudSQL configuration 46 | 47 | You'll want to turn on query logging. Assuming no custom database flags are set 48 | on your cloudsql instance, you can run: 49 | 50 | ``` 51 | gcloud sql instances patch --database-flags log_min_duration_statement=0 52 | ``` 53 | 54 | Caveat 1: this requires restarting your cloudsql instance, which gcloud will do for 55 | you. 56 | 57 | To see what flags are set: `gcloud sql instances describe ` and 58 | look for the top-level key `databaseFlags`. (You'll want to include existing 59 | flags in your `gcloud sql instances patch` command, if any are set.) 60 | 61 | ## Stackdriver Configuration 62 | 63 | Ingestion is done via [Cloud Pub/Sub](https://cloud.google.com/pubsub/). 64 | 65 | To use this, [set up a Stackdriver 66 | sink](https://cloud.google.com/logging/docs/export/configure_export_v2) of your 67 | Postgres logs to a Pub/Sub topic. You will also need to create a 68 | [subscription](https://cloud.google.com/pubsub/docs/subscriber) to that topic. 69 | Note the subscription name you provide, as it is necessary to configure 70 | cloudsqltail. 71 | 72 | ## Docker 73 | 74 | ``` 75 | docker build -t postgres-honeytail . 76 | ``` 77 | 78 | Then push to your preferred docker registry. 79 | 80 | By default, the container will run a wrapper script `run.sh`, which takes the 81 | output of `cloudsqltail` and sends it along to honeycomb via `honeytail`. 82 | 83 | ## Config and deploy 84 | 85 | ### Required environment variables 86 | 87 | - `GOOGLE_APPLICATION_CREDENTIALS_JSON` is used to auth to gcloud; the service 88 | account must have read access to the pubsub subscription. 89 | - `PROJECT_ID` the gcloud project the pubsub subscription is in 90 | - `SUBSCRIPTION_NAME` the pubsub subscription name 91 | - `HONEYCOMB_WRITEKEY` (not required if `DEBUG` is set, see below) 92 | 93 | ### Optional environment variables 94 | 95 | - `DEBUG` runs `honeytail` with the flags `--debug` (setting the log level to 96 | DEBUG) and `--debug_stdout` (writing events to stdout instead of sending to 97 | honeycomb). The latter flag also means that `HONEYCOMB_WRITEKEY` is not 98 | verified, so it can be left unset. 99 | - `DATASET` the honeycomb dataset to write to; defaults to `postgres` 100 | 101 | ## Development 102 | 103 | ### Go 104 | 105 | The `cloudsqltail` command uses go modules to track it's depedencies, so you'll 106 | need at least Go 1.11 or higher, with module mode enabled via `GO111MODULE=on`. 107 | With >=1.12, `make` should work out of the box with no additional configuration 108 | necessary. 109 | 110 | ### Running in Docker 111 | 112 | `cloudtailsql` needs Google Cloud access credentials (to fetch the Pub/Sub 113 | Subscription). To get this locally, run docker with 114 | 115 | ``` 116 | -v $HOME/.config/gcloud:/root/.config/gcloud 117 | ``` 118 | 119 | -------------------------------------------------------------------------------- /cmd/cloudsqltail/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "flag" 7 | "fmt" 8 | "os" 9 | "runtime" 10 | "sort" 11 | "sync" 12 | "time" 13 | 14 | "cloud.google.com/go/pubsub" 15 | ) 16 | 17 | // flags used for configuration 18 | var ( 19 | flagProject = flag.String("project", "", "Google project ID") 20 | flagSubscription = flag.String("subscription", "", "Google PubSub subscription name") 21 | 22 | flagReceiveGoroutines = flag.Int( 23 | "recv-routines", 24 | runtime.NumCPU(), 25 | "Number of goroutines to use to receive messages from the pubsub Subscription. default: runtime.NumCPUs()", 26 | ) 27 | 28 | // flagFlushInterval is the duration used for ticker that flushes the 29 | // buffer to STDOUT. A smaller duration will mean less memory usage but a 30 | // higher probability of out-of-order messages in the output. 31 | // 32 | // Importantly, this duration also controls the data loss you are willing 33 | // to tolerate in the event of an unclean shutdown. Messages are acked once 34 | // put into this buffer, so if the program exits unexpectedly, any messages 35 | // in buffer and not flushed to STDOUT will be lost. 36 | flagFlushInterval = flag.Duration( 37 | "flush-interval", 38 | 5*time.Second, 39 | "Time between flushes of message buffer to STDOUT.", 40 | ) 41 | ) 42 | 43 | // mx is the mutex used to proect the global message buffer 44 | var mx sync.Mutex 45 | 46 | // globalBuffer is the buffer used to store messages until flush 47 | var globalBuffer []ParsedMessage 48 | 49 | // pgTimestamp is the format that postgres usually uses for timestamps, and 50 | // consequently what honeytail expects. It's almost but not quite 51 | // time.RFC3339Nano. 52 | const pgTimestamp = "2006-01-02 15:04:05.999999999 UTC" 53 | 54 | // ParsedMessage holds the relevant details from a parsed Cloud Subscription message. 55 | type ParsedMessage struct { 56 | TextPayload string `json:"textPayload"` 57 | Timestamp time.Time `json:"timestamp"` 58 | } 59 | 60 | func main() { 61 | flag.Parse() 62 | 63 | switch "" { 64 | case *flagProject: 65 | fmt.Fprintf(os.Stderr, "must provide -project") 66 | os.Exit(1) 67 | case *flagSubscription: 68 | fmt.Fprintf(os.Stderr, "must provide -subscription") 69 | os.Exit(1) 70 | } 71 | if *flagFlushInterval < 1 { 72 | fmt.Fprintf(os.Stderr, "flush internal '%s' must be > 0", *flagFlushInterval) 73 | os.Exit(1) 74 | } else if *flagFlushInterval < time.Second { 75 | fmt.Fprintf( 76 | os.Stderr, 77 | `WARNING: using an small flush interval may result in more out-of-order output. Are you sure you didn't mean "%ds"?`, 78 | *flagFlushInterval) 79 | } 80 | 81 | ctx := context.Background() 82 | c, err := pubsub.NewClient(ctx, *flagProject) 83 | if err != nil { 84 | fmt.Fprintf(os.Stderr, "%+v\n", err) 85 | os.Exit(1) 86 | } 87 | 88 | sub := c.Subscription(*flagSubscription) 89 | sub.ReceiveSettings.NumGoroutines = *flagReceiveGoroutines 90 | 91 | // this is a forever ticker that periodically 92 | go flushBuffer(*flagFlushInterval) 93 | 94 | err = sub.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) { 95 | bufferMessage(msg.Data) 96 | msg.Ack() 97 | }) 98 | if err != nil { 99 | fmt.Fprintf(os.Stderr, "%+v\n", err) 100 | os.Exit(1) 101 | } 102 | } 103 | 104 | // bufferMessage takes a message's unparsed JSON data, parses it, and appends a 105 | // ParsedMessage to the global message buffer. 106 | func bufferMessage(data []byte) { 107 | var pm ParsedMessage 108 | if err := json.Unmarshal(data, &pm); err != nil { 109 | fmt.Fprintf(os.Stderr, "error unparsing msg %s\n", data) 110 | return 111 | } 112 | 113 | mx.Lock() 114 | globalBuffer = append(globalBuffer, pm) 115 | mx.Unlock() 116 | } 117 | 118 | // flushBuffer sets up a ticker to flush the message buffer every 'dur'. 119 | // Each tick locks then sorts the message buffer based on the Timestamp, 120 | // then outputs the buffer sequentially to STDOUT. Some notes: 121 | // 122 | // Subscriptions make no guarantee about the ordering of delivered 123 | // messages, so we sort all buffered messages before output. This does 124 | // not guarantee that we don't ever miss or mangle messages, but it 125 | // gets it mostly correct. 126 | // 127 | // Postgres query logs can span multiple lines. 128 | // The first line always has a prefix, which we know starts with the 129 | // [PID] in brackets, while the following lines are \t indented. 130 | // Add the timestamp from the pubsub message to initial lines and 131 | // pass-through remaining lines unmodified. 132 | func flushBuffer(dur time.Duration) { 133 | tick := time.NewTicker(dur) 134 | for { 135 | <-tick.C 136 | mx.Lock() 137 | 138 | sort.Slice(globalBuffer, func(i, j int) bool { 139 | return globalBuffer[i].Timestamp.Before(globalBuffer[j].Timestamp) 140 | }) 141 | 142 | for i := range globalBuffer { 143 | fmt.Println(globalBuffer[i].TextPayload) 144 | } 145 | 146 | // Reset the global entries slice, pre-allocating enough capacity to 147 | // fit the same number of messages as we saw last time. This trades 148 | // some memory usage for not having to copy the slice a bunch when we 149 | // append() in bufferMessage. 150 | globalBuffer = make([]ParsedMessage, 0, len(globalBuffer)) 151 | mx.Unlock() 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/darklang/cloudsql-to-honeytail 2 | 3 | go 1.13 4 | 5 | require cloud.google.com/go/pubsub v1.0.1 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= 4 | cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= 5 | cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= 6 | cloud.google.com/go v0.45.1 h1:lRi0CHyU+ytlvylOlFKKq0af6JncuyoRh1J+QJBqQx0= 7 | cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= 8 | cloud.google.com/go/bigquery v1.0.1 h1:hL+ycaJpVE9M7nLoiXb/Pn10ENE2u+oddxbD8uu0ZVU= 9 | cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= 10 | cloud.google.com/go/datastore v1.0.0 h1:Kt+gOPPp2LEPWp8CSfxhsM8ik9CcyE/gYu+0r+RnZvM= 11 | cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= 12 | cloud.google.com/go/pubsub v1.0.1 h1:W9tAK3E57P75u0XLLR82LZyw8VpAnhmyTOxW9qzmyj8= 13 | cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= 14 | github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= 15 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 16 | github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= 17 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 18 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= 19 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 20 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 21 | github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 22 | github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= 23 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 24 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 25 | github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= 26 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 27 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 28 | github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 29 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 30 | github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= 31 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 32 | github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= 33 | github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= 34 | github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 35 | github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 36 | github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= 37 | github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= 38 | github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= 39 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 40 | github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= 41 | github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 42 | github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024 h1:rBMNdlhTLzJjJSDIjNEXX1Pz3Hmwmz91v+zycvx9PJc= 43 | github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= 44 | go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= 45 | go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= 46 | go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= 47 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 48 | golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 49 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 50 | golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522 h1:OeRHuibLsmZkFj773W4LcfAGsSxJgfPONhr8cmO+eLA= 51 | golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= 52 | golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= 53 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 54 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 55 | golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 56 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 57 | golang.org/x/lint v0.0.0-20190409202823-959b441ac422 h1:QzoH/1pFpZguR8NrRHLcO6jKqfv2zpuSqZLgdm7ZmjI= 58 | golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 59 | golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= 60 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 61 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 62 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 63 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 64 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 65 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 66 | golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 67 | golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 68 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 69 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= 70 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 71 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 72 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 73 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= 74 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 75 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 76 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 77 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 78 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 79 | golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= 80 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 81 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 82 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 83 | golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 84 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 85 | golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 86 | golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 87 | golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 88 | golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc= 89 | golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 90 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 91 | golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 92 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 93 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 94 | golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 95 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 96 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 97 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 98 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 99 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 100 | golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 101 | golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 102 | golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 103 | golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 104 | golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 105 | golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0 h1:Dh6fw+p6FyRl5x/FvNswO1ji0lIGzm3KP8Y9VkS9PTE= 106 | golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 107 | google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= 108 | google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= 109 | google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 110 | google.golang.org/api v0.9.0 h1:jbyannxz0XFD3zdjgrSUsaJbgpH4eTrkdhRChkHPfO8= 111 | google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 112 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 113 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 114 | google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 115 | google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= 116 | google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= 117 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 118 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 119 | google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 120 | google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 121 | google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 122 | google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 123 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= 124 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 125 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 126 | google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= 127 | google.golang.org/grpc v1.21.1 h1:j6XxA85m/6txkUCHvzlV5f+HBNl/1r5cZ2A/3IEFOO8= 128 | google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= 129 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 130 | honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 131 | honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a h1:LJwr7TCTghdatWv40WobzlKXc9c4s8oGa7QKJUtHhWA= 132 | honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 133 | rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= 134 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | set -euo pipefail 2 | 3 | # For testing, you can run with DEBUG=1, and data will not be sent to honeycomb 4 | # (--debug sets the log level, --debug_stdout says to write events to stdout 5 | # instead of sending to honeycomb) 6 | if ! [[ -z "${DEBUG:-}" ]]; then 7 | HTDEBUG="--debug --debug_stdout" 8 | fi 9 | 10 | # In-cluster gcloud auth 11 | if [[ "${GOOGLE_APPLICATION_CREDENTIALS_JSON:-}" != "" ]]; then 12 | echo $GOOGLE_APPLICATION_CREDENTIALS_JSON > service-account.key 13 | export GOOGLE_APPLICATION_CREDENTIALS=service-account.key 14 | fi 15 | 16 | ./cloudsqltail -project ${PROJECT_ID} -subscription ${SUBSCRIPTION_NAME} | \ 17 | ./honeytail ${HTDEBUG:-} \ 18 | -k="${HONEYCOMB_WRITEKEY:-unset}" \ 19 | --dataset="${DATASET:-postgres}" \ 20 | --parser=postgresql \ 21 | --postgresql.log_line_prefix='%m [%p]: [%l-1] db=%d,user=%u' \ 22 | -f - 23 | --------------------------------------------------------------------------------