├── .gitignore ├── .goreleaser.yml ├── Gopkg.lock ├── Gopkg.toml ├── Makefile ├── README.md ├── examples └── simple │ └── main.tf ├── main.go └── topic ├── provider.go ├── provider_test.go ├── resource.go └── resource_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | peers.json 2 | store.db 3 | vendor/* 4 | !vendor/vendor.json 5 | .idea 6 | *.log 7 | *.index 8 | NOTES.org 9 | .dir-locals.el 10 | .#* 11 | .DS_Store 12 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | snapshot: 2 | name_template: SNAPSHOT-{{.Commit}} 3 | 4 | builds: 5 | - main: main.go 6 | binary: terraform-provider-kafka 7 | goos: 8 | - darwin 9 | - linux 10 | - windows 11 | - freebsd 12 | goarch: 13 | - amd64 14 | -------------------------------------------------------------------------------- /Gopkg.lock: -------------------------------------------------------------------------------- 1 | # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. 2 | 3 | 4 | [[projects]] 5 | name = "github.com/Shopify/sarama" 6 | packages = ["."] 7 | revision = "3c763ff04e6daa57d4a4614e5bcd908f2527c989" 8 | 9 | [[projects]] 10 | name = "github.com/agext/levenshtein" 11 | packages = ["."] 12 | revision = "5f10fee965225ac1eecdc234c09daf5cd9e7f7b6" 13 | version = "v1.2.1" 14 | 15 | [[projects]] 16 | branch = "master" 17 | name = "github.com/apparentlymart/go-cidr" 18 | packages = ["cidr"] 19 | revision = "2bd8b58cf4275aeb086ade613de226773e29e853" 20 | 21 | [[projects]] 22 | branch = "master" 23 | name = "github.com/apparentlymart/go-textseg" 24 | packages = ["textseg"] 25 | revision = "b836f5c4d331d1945a2fead7188db25432d73b69" 26 | 27 | [[projects]] 28 | branch = "master" 29 | name = "github.com/armon/go-radix" 30 | packages = ["."] 31 | revision = "1fca145dffbcaa8fe914309b1ec0cfc67500fe61" 32 | 33 | [[projects]] 34 | name = "github.com/aws/aws-sdk-go" 35 | packages = [ 36 | "aws", 37 | "aws/awserr", 38 | "aws/awsutil", 39 | "aws/client", 40 | "aws/client/metadata", 41 | "aws/corehandlers", 42 | "aws/credentials", 43 | "aws/credentials/ec2rolecreds", 44 | "aws/credentials/endpointcreds", 45 | "aws/credentials/stscreds", 46 | "aws/defaults", 47 | "aws/ec2metadata", 48 | "aws/endpoints", 49 | "aws/request", 50 | "aws/session", 51 | "aws/signer/v4", 52 | "internal/sdkio", 53 | "internal/sdkrand", 54 | "internal/shareddefaults", 55 | "private/protocol", 56 | "private/protocol/query", 57 | "private/protocol/query/queryutil", 58 | "private/protocol/rest", 59 | "private/protocol/restxml", 60 | "private/protocol/xml/xmlutil", 61 | "service/s3", 62 | "service/sts" 63 | ] 64 | revision = "c7cd1ebe87257cde9b65112fc876b0339ea0ac30" 65 | version = "v1.13.49" 66 | 67 | [[projects]] 68 | branch = "master" 69 | name = "github.com/bgentry/go-netrc" 70 | packages = ["netrc"] 71 | revision = "9fd32a8b3d3d3f9d43c341bfe098430e07609480" 72 | 73 | [[projects]] 74 | name = "github.com/bgentry/speakeasy" 75 | packages = ["."] 76 | revision = "4aabc24848ce5fd31929f7d1e4ea74d3709c14cd" 77 | version = "v0.1.0" 78 | 79 | [[projects]] 80 | name = "github.com/blang/semver" 81 | packages = ["."] 82 | revision = "2ee87856327ba09384cabd113bc6b5d174e9ec0f" 83 | version = "v3.5.1" 84 | 85 | [[projects]] 86 | name = "github.com/davecgh/go-spew" 87 | packages = ["spew"] 88 | revision = "346938d642f2ec3594ed81d874461961cd0faa76" 89 | version = "v1.1.0" 90 | 91 | [[projects]] 92 | name = "github.com/eapache/go-resiliency" 93 | packages = ["breaker"] 94 | revision = "ea41b0fad31007accc7f806884dcdf3da98b79ce" 95 | version = "v1.1.0" 96 | 97 | [[projects]] 98 | branch = "master" 99 | name = "github.com/eapache/go-xerial-snappy" 100 | packages = ["."] 101 | revision = "bb955e01b9346ac19dc29eb16586c90ded99a98c" 102 | 103 | [[projects]] 104 | name = "github.com/eapache/queue" 105 | packages = ["."] 106 | revision = "44cc805cf13205b55f69e14bcb69867d1ae92f98" 107 | version = "v1.1.0" 108 | 109 | [[projects]] 110 | name = "github.com/fatih/color" 111 | packages = ["."] 112 | revision = "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4" 113 | version = "v1.7.0" 114 | 115 | [[projects]] 116 | name = "github.com/go-ini/ini" 117 | packages = ["."] 118 | revision = "6529cf7c58879c08d927016dde4477f18a0634cb" 119 | version = "v1.36.0" 120 | 121 | [[projects]] 122 | name = "github.com/golang/protobuf" 123 | packages = [ 124 | "proto", 125 | "ptypes", 126 | "ptypes/any", 127 | "ptypes/duration", 128 | "ptypes/timestamp" 129 | ] 130 | revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" 131 | version = "v1.1.0" 132 | 133 | [[projects]] 134 | branch = "master" 135 | name = "github.com/golang/snappy" 136 | packages = ["."] 137 | revision = "553a641470496b2327abcac10b36396bd98e45c9" 138 | 139 | [[projects]] 140 | branch = "master" 141 | name = "github.com/hashicorp/errwrap" 142 | packages = ["."] 143 | revision = "7554cd9344cec97297fa6649b055a8c98c2a1e55" 144 | 145 | [[projects]] 146 | branch = "master" 147 | name = "github.com/hashicorp/go-cleanhttp" 148 | packages = ["."] 149 | revision = "d5fe4b57a186c716b0e00b8c301cbd9b4182694d" 150 | 151 | [[projects]] 152 | branch = "master" 153 | name = "github.com/hashicorp/go-getter" 154 | packages = [ 155 | ".", 156 | "helper/url" 157 | ] 158 | revision = "3f60ec5cfbb2a39731571b9ddae54b303bb0a969" 159 | 160 | [[projects]] 161 | branch = "master" 162 | name = "github.com/hashicorp/go-hclog" 163 | packages = ["."] 164 | revision = "69ff559dc25f3b435631604f573a5fa1efdb6433" 165 | 166 | [[projects]] 167 | branch = "master" 168 | name = "github.com/hashicorp/go-multierror" 169 | packages = ["."] 170 | revision = "b7773ae218740a7be65057fc60b366a49b538a44" 171 | 172 | [[projects]] 173 | branch = "master" 174 | name = "github.com/hashicorp/go-plugin" 175 | packages = ["."] 176 | revision = "e8d22c780116115ae5624720c9af0c97afe4f551" 177 | 178 | [[projects]] 179 | branch = "master" 180 | name = "github.com/hashicorp/go-safetemp" 181 | packages = ["."] 182 | revision = "b1a1dbde6fdc11e3ae79efd9039009e22d4ae240" 183 | 184 | [[projects]] 185 | branch = "master" 186 | name = "github.com/hashicorp/go-uuid" 187 | packages = ["."] 188 | revision = "27454136f0364f2d44b1276c552d69105cf8c498" 189 | 190 | [[projects]] 191 | branch = "master" 192 | name = "github.com/hashicorp/go-version" 193 | packages = ["."] 194 | revision = "23480c0665776210b5fbbac6eaaee40e3e6a96b7" 195 | 196 | [[projects]] 197 | branch = "master" 198 | name = "github.com/hashicorp/hcl" 199 | packages = [ 200 | ".", 201 | "hcl/ast", 202 | "hcl/parser", 203 | "hcl/scanner", 204 | "hcl/strconv", 205 | "hcl/token", 206 | "json/parser", 207 | "json/scanner", 208 | "json/token" 209 | ] 210 | revision = "ef8a98b0bbce4a65b5aa4c368430a80ddc533168" 211 | 212 | [[projects]] 213 | branch = "master" 214 | name = "github.com/hashicorp/hcl2" 215 | packages = [ 216 | "gohcl", 217 | "hcl", 218 | "hcl/hclsyntax", 219 | "hcl/json", 220 | "hcldec", 221 | "hclparse" 222 | ] 223 | revision = "9db880accff19d9bd953958df738ef0c02b4a311" 224 | 225 | [[projects]] 226 | branch = "master" 227 | name = "github.com/hashicorp/hil" 228 | packages = [ 229 | ".", 230 | "ast", 231 | "parser", 232 | "scanner" 233 | ] 234 | revision = "fa9f258a92500514cc8e9c67020487709df92432" 235 | 236 | [[projects]] 237 | branch = "master" 238 | name = "github.com/hashicorp/logutils" 239 | packages = ["."] 240 | revision = "0dc08b1671f34c4250ce212759ebd880f743d883" 241 | 242 | [[projects]] 243 | name = "github.com/hashicorp/terraform" 244 | packages = [ 245 | "config", 246 | "config/configschema", 247 | "config/hcl2shim", 248 | "config/module", 249 | "dag", 250 | "flatmap", 251 | "helper/config", 252 | "helper/hashcode", 253 | "helper/hilmapstructure", 254 | "helper/logging", 255 | "helper/resource", 256 | "helper/schema", 257 | "httpclient", 258 | "moduledeps", 259 | "plugin", 260 | "plugin/discovery", 261 | "registry", 262 | "registry/regsrc", 263 | "registry/response", 264 | "svchost", 265 | "svchost/auth", 266 | "svchost/disco", 267 | "terraform", 268 | "tfdiags", 269 | "version" 270 | ] 271 | revision = "41e50bd32a8825a84535e353c3674af8ce799161" 272 | version = "v0.11.7" 273 | 274 | [[projects]] 275 | branch = "master" 276 | name = "github.com/hashicorp/yamux" 277 | packages = ["."] 278 | revision = "2658be15c5f05e76244154714161f17e3e77de2e" 279 | 280 | [[projects]] 281 | name = "github.com/jmespath/go-jmespath" 282 | packages = ["."] 283 | revision = "0b12d6b5" 284 | 285 | [[projects]] 286 | name = "github.com/mattn/go-colorable" 287 | packages = ["."] 288 | revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072" 289 | version = "v0.0.9" 290 | 291 | [[projects]] 292 | name = "github.com/mattn/go-isatty" 293 | packages = ["."] 294 | revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39" 295 | version = "v0.0.3" 296 | 297 | [[projects]] 298 | branch = "master" 299 | name = "github.com/mitchellh/cli" 300 | packages = ["."] 301 | revision = "c48282d14eba4b0817ddef3f832ff8d13851aefd" 302 | 303 | [[projects]] 304 | branch = "master" 305 | name = "github.com/mitchellh/copystructure" 306 | packages = ["."] 307 | revision = "d23ffcb85de31694d6ccaa23ccb4a03e55c1303f" 308 | 309 | [[projects]] 310 | branch = "master" 311 | name = "github.com/mitchellh/go-homedir" 312 | packages = ["."] 313 | revision = "b8bc1bf767474819792c23f32d8286a45736f1c6" 314 | 315 | [[projects]] 316 | branch = "master" 317 | name = "github.com/mitchellh/go-testing-interface" 318 | packages = ["."] 319 | revision = "a61a99592b77c9ba629d254a693acffaeb4b7e28" 320 | 321 | [[projects]] 322 | branch = "master" 323 | name = "github.com/mitchellh/go-wordwrap" 324 | packages = ["."] 325 | revision = "ad45545899c7b13c020ea92b2072220eefad42b8" 326 | 327 | [[projects]] 328 | branch = "master" 329 | name = "github.com/mitchellh/hashstructure" 330 | packages = ["."] 331 | revision = "2bca23e0e452137f789efbc8610126fd8b94f73b" 332 | 333 | [[projects]] 334 | branch = "master" 335 | name = "github.com/mitchellh/mapstructure" 336 | packages = ["."] 337 | revision = "bb74f1db0675b241733089d5a1faa5dd8b0ef57b" 338 | 339 | [[projects]] 340 | branch = "master" 341 | name = "github.com/mitchellh/reflectwalk" 342 | packages = ["."] 343 | revision = "63d60e9d0dbc60cf9164e6510889b0db6683d98c" 344 | 345 | [[projects]] 346 | name = "github.com/oklog/run" 347 | packages = ["."] 348 | revision = "4dadeb3030eda0273a12382bb2348ffc7c9d1a39" 349 | version = "v1.0.0" 350 | 351 | [[projects]] 352 | name = "github.com/pierrec/lz4" 353 | packages = ["."] 354 | revision = "2fcda4cb7018ce05a25959d2fe08c83e3329f169" 355 | version = "v1.1" 356 | 357 | [[projects]] 358 | name = "github.com/pierrec/xxHash" 359 | packages = ["xxHash32"] 360 | revision = "f051bb7f1d1aaf1b5a665d74fb6b0217712c69f7" 361 | version = "v0.1.1" 362 | 363 | [[projects]] 364 | name = "github.com/posener/complete" 365 | packages = [ 366 | ".", 367 | "cmd", 368 | "cmd/install", 369 | "match" 370 | ] 371 | revision = "98eb9847f27ba2008d380a32c98be474dea55bdf" 372 | version = "v1.1.1" 373 | 374 | [[projects]] 375 | branch = "master" 376 | name = "github.com/rcrowley/go-metrics" 377 | packages = ["."] 378 | revision = "e2704e165165ec55d062f5919b4b29494e9fa790" 379 | 380 | [[projects]] 381 | name = "github.com/ulikunitz/xz" 382 | packages = [ 383 | ".", 384 | "internal/hash", 385 | "internal/xlog", 386 | "lzma" 387 | ] 388 | revision = "0c6b41e72360850ca4f98dc341fd999726ea007f" 389 | version = "v0.5.4" 390 | 391 | [[projects]] 392 | branch = "master" 393 | name = "github.com/zclconf/go-cty" 394 | packages = [ 395 | "cty", 396 | "cty/convert", 397 | "cty/function", 398 | "cty/function/stdlib", 399 | "cty/gocty", 400 | "cty/json", 401 | "cty/set" 402 | ] 403 | revision = "d006e4534bc4fbc512383aa98d04d641ea951ba5" 404 | 405 | [[projects]] 406 | branch = "master" 407 | name = "golang.org/x/crypto" 408 | packages = [ 409 | "bcrypt", 410 | "blowfish", 411 | "cast5", 412 | "openpgp", 413 | "openpgp/armor", 414 | "openpgp/elgamal", 415 | "openpgp/errors", 416 | "openpgp/packet", 417 | "openpgp/s2k" 418 | ] 419 | revision = "1a580b3eff7814fc9b40602fd35256c63b50f491" 420 | 421 | [[projects]] 422 | branch = "master" 423 | name = "golang.org/x/net" 424 | packages = [ 425 | "context", 426 | "html", 427 | "html/atom", 428 | "http/httpguts", 429 | "http2", 430 | "http2/hpack", 431 | "idna", 432 | "internal/timeseries", 433 | "trace" 434 | ] 435 | revision = "2491c5de3490fced2f6cff376127c667efeed857" 436 | 437 | [[projects]] 438 | branch = "master" 439 | name = "golang.org/x/sys" 440 | packages = ["unix"] 441 | revision = "7c87d13f8e835d2fb3a70a2912c811ed0c1d241b" 442 | 443 | [[projects]] 444 | name = "golang.org/x/text" 445 | packages = [ 446 | "collate", 447 | "collate/build", 448 | "internal/colltab", 449 | "internal/gen", 450 | "internal/tag", 451 | "internal/triegen", 452 | "internal/ucd", 453 | "language", 454 | "secure/bidirule", 455 | "transform", 456 | "unicode/bidi", 457 | "unicode/cldr", 458 | "unicode/norm", 459 | "unicode/rangetable" 460 | ] 461 | revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" 462 | version = "v0.3.0" 463 | 464 | [[projects]] 465 | branch = "master" 466 | name = "google.golang.org/genproto" 467 | packages = ["googleapis/rpc/status"] 468 | revision = "7bb2a897381c9c5ab2aeb8614f758d7766af68ff" 469 | 470 | [[projects]] 471 | name = "google.golang.org/grpc" 472 | packages = [ 473 | ".", 474 | "balancer", 475 | "balancer/base", 476 | "balancer/roundrobin", 477 | "channelz", 478 | "codes", 479 | "connectivity", 480 | "credentials", 481 | "encoding", 482 | "encoding/proto", 483 | "grpclb/grpc_lb_v1/messages", 484 | "grpclog", 485 | "health", 486 | "health/grpc_health_v1", 487 | "internal", 488 | "keepalive", 489 | "metadata", 490 | "naming", 491 | "peer", 492 | "resolver", 493 | "resolver/dns", 494 | "resolver/passthrough", 495 | "stats", 496 | "status", 497 | "tap", 498 | "transport" 499 | ] 500 | revision = "41344da2231b913fa3d983840a57a6b1b7b631a1" 501 | version = "v1.12.0" 502 | 503 | [solve-meta] 504 | analyzer-name = "dep" 505 | analyzer-version = 1 506 | inputs-digest = "5a6a625dd74f9c3753b1dec58a711a46d25c7c247b4dd1eaa8e67a148cf83854" 507 | solver-name = "gps-cdcl" 508 | solver-version = 1 509 | -------------------------------------------------------------------------------- /Gopkg.toml: -------------------------------------------------------------------------------- 1 | # Gopkg.toml example 2 | # 3 | # Refer to https://golang.github.io/dep/docs/Gopkg.toml.html 4 | # for detailed Gopkg.toml documentation. 5 | # 6 | # required = ["github.com/user/thing/cmd/thing"] 7 | # ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] 8 | # 9 | # [[constraint]] 10 | # name = "github.com/user/project" 11 | # version = "1.0.0" 12 | # 13 | # [[constraint]] 14 | # name = "github.com/user/project2" 15 | # branch = "dev" 16 | # source = "github.com/myfork/project2" 17 | # 18 | # [[override]] 19 | # name = "github.com/x/y" 20 | # version = "2.4.0" 21 | # 22 | # [prune] 23 | # non-go = false 24 | # go-tests = true 25 | # unused-packages = true 26 | 27 | 28 | [[constraint]] 29 | name = "github.com/Shopify/sarama" 30 | revision = "3c763ff04e6daa57d4a4614e5bcd908f2527c989" 31 | 32 | [[constraint]] 33 | name = "github.com/hashicorp/terraform" 34 | version = "0.11.7" 35 | 36 | [prune] 37 | go-tests = true 38 | unused-packages = true 39 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Metadata about this makefile and position 2 | MKFILE_PATH := $(lastword $(MAKEFILE_LIST)) 3 | CURRENT_DIR := $(patsubst %/,%,$(dir $(realpath $(MKFILE_PATH)))) 4 | 5 | # Ensure GOPATH 6 | GOPATH ?= $(HOME)/go 7 | 8 | # List all our actual files, excluding vendor 9 | GOFILES ?= $(shell go list $(TEST) | grep -v /vendor/) 10 | 11 | # Tags specific for building 12 | GOTAGS ?= 13 | 14 | # Number of procs to use 15 | GOMAXPROCS ?= 4 16 | 17 | # Get the project metadata 18 | GOVERSION := 1.10.1 19 | PROJECT := $(CURRENT_DIR:$(GOPATH)/src/%=%) 20 | OWNER := $(notdir $(patsubst %/,%,$(dir $(PROJECT)))) 21 | NAME := $(notdir $(PROJECT)) 22 | VERSION := 0.1.0 23 | EXTERNAL_TOOLS = \ 24 | github.com/golang/dep/cmd/dep 25 | 26 | # Current system information 27 | GOOS ?= $(shell go env GOOS) 28 | GOARCH ?= $(shell go env GOARCH) 29 | 30 | # Default os-arch combination to build 31 | XC_OS ?= darwin freebsd linux openbsd solaris windows 32 | XC_ARCH ?= 386 amd64 arm 33 | XC_EXCLUDE ?= darwin/386 darwin/arm solaris/386 solaris/arm windows/arm 34 | 35 | # GPG Signing key (blank by default, means no GPG signing) 36 | GPG_KEY ?= 37 | 38 | # List of ldflags 39 | LD_FLAGS ?= \ 40 | -s \ 41 | -w 42 | 43 | # List of tests to run 44 | TEST ?= ./... 45 | 46 | # Path to Terraform plugins 47 | PLUGIN_PATH ?= "${HOME}/.terraform.d/plugins" 48 | 49 | # Create a cross-compile target for every os-arch pairing. This will generate 50 | # a make target for each os/arch like "make linux/amd64" as well as generate a 51 | # meta target (build) for compiling everything. 52 | define make-xc-target 53 | $1/$2: 54 | ifneq (,$(findstring ${1}/${2},$(XC_EXCLUDE))) 55 | @printf "%s%20s %s\n" "-->" "${1}/${2}:" "${PROJECT} (excluded)" 56 | else 57 | @printf "%s%20s %s\n" "-->" "${1}/${2}:" "${PROJECT}" 58 | @docker run \ 59 | --interactive \ 60 | --rm \ 61 | --dns="8.8.8.8" \ 62 | --volume="${CURRENT_DIR}:/go/src/${PROJECT}" \ 63 | --workdir="/go/src/${PROJECT}" \ 64 | "golang:${GOVERSION}" \ 65 | env \ 66 | CGO_ENABLED="0" \ 67 | GOOS="${1}" \ 68 | GOARCH="${2}" \ 69 | go build \ 70 | -a \ 71 | -o="pkg/${1}_${2}/${NAME}${3}" \ 72 | -ldflags "${LD_FLAGS}" \ 73 | -tags "${GOTAGS}" 74 | endif 75 | .PHONY: $1/$2 76 | 77 | $1:: $1/$2 78 | .PHONY: $1 79 | 80 | build:: $1/$2 81 | .PHONY: build 82 | endef 83 | $(foreach goarch,$(XC_ARCH),$(foreach goos,$(XC_OS),$(eval $(call make-xc-target,$(goos),$(goarch),$(if $(findstring windows,$(goos)),.exe,))))) 84 | 85 | # deps updates all dependencies 86 | deps: 87 | @dep ensure -update 88 | 89 | # dev builds and installs the plugin into ~/.terraform.d 90 | dev: 91 | @go install \ 92 | -ldflags "${LD_FLAGS}" \ 93 | -tags "${GOTAGS}" 94 | 95 | # dist builds the binaries and then signs and packages them for distribution 96 | dist: 97 | ifndef GPG_KEY 98 | @echo "==> ERROR: No GPG key specified! Without a GPG key, this release cannot" 99 | @echo " be signed. Set the environment variable GPG_KEY to the ID of" 100 | @echo " the GPG key to continue." 101 | @exit 127 102 | else 103 | @$(MAKE) -f "${MKFILE_PATH}" _cleanup 104 | @$(MAKE) -f "${MKFILE_PATH}" -j4 build 105 | @$(MAKE) -f "${MKFILE_PATH}" _compress _checksum _sign 106 | endif 107 | .PHONY: dist 108 | 109 | # test runs the tests 110 | test: 111 | @go test -v $(TEST) 112 | 113 | # _cleanup removes any previous binaries 114 | _cleanup: 115 | @rm -rf "${CURRENT_DIR}/pkg/" 116 | @rm -rf "${CURRENT_DIR}/bin/" 117 | .PHONY: _cleanup 118 | 119 | # _compress compresses all the binaries in pkg/* as tarball and zip. 120 | _compress: 121 | @mkdir -p "${CURRENT_DIR}/pkg/dist" 122 | @for platform in $$(find ./pkg -mindepth 1 -maxdepth 1 -type d); do \ 123 | osarch=$$(basename "$$platform"); \ 124 | if [ "$$osarch" = "dist" ]; then \ 125 | continue; \ 126 | fi; \ 127 | \ 128 | ext=""; \ 129 | if test -z "$${osarch##*windows*}"; then \ 130 | ext=".exe"; \ 131 | fi; \ 132 | cd "$$platform"; \ 133 | tar -czf "${CURRENT_DIR}/pkg/dist/${NAME}_${VERSION}_$${osarch}.tgz" "${NAME}$${ext}"; \ 134 | zip -q "${CURRENT_DIR}/pkg/dist/${NAME}_${VERSION}_$${osarch}.zip" "${NAME}$${ext}"; \ 135 | cd - &>/dev/null; \ 136 | done 137 | .PHONY: _compress 138 | 139 | # _checksum produces the checksums for the binaries in pkg/dist 140 | _checksum: 141 | @cd "${CURRENT_DIR}/pkg/dist" && \ 142 | shasum --algorithm 256 * > ${CURRENT_DIR}/pkg/dist/${NAME}_${VERSION}_SHA256SUMS && \ 143 | cd - &>/dev/null 144 | .PHONY: _checksum 145 | 146 | # _sign signs the binaries using the given GPG_KEY. This should not be called 147 | # as a separate function. 148 | _sign: 149 | @echo "==> Signing ${PROJECT} at v${VERSION}" 150 | @gpg \ 151 | --default-key "${GPG_KEY}" \ 152 | --detach-sig "${CURRENT_DIR}/pkg/dist/${NAME}_${VERSION}_SHA256SUMS" 153 | @git commit \ 154 | --allow-empty \ 155 | --gpg-sign="${GPG_KEY}" \ 156 | --message "Release v${VERSION}" \ 157 | --quiet \ 158 | --signoff 159 | @git tag \ 160 | --annotate \ 161 | --create-reflog \ 162 | --local-user "${GPG_KEY}" \ 163 | --message "Version ${VERSION}" \ 164 | --sign \ 165 | "v${VERSION}" master 166 | @echo "--> Do not forget to run:" 167 | @echo "" 168 | @echo " git push && git push --tags" 169 | @echo "" 170 | @echo "And then upload the binaries in dist/!" 171 | .PHONY: _sign 172 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Terraform Kafka Topic Provider 2 | 3 | This is a Terraform provider for managing Kafka topics with 4 | Terraform. 5 | 6 | Why use this Kafka provider? 7 | 8 | - Supports adding partitions and altering configs 9 | - Supports TLS/SASL 10 | - Uses Kafka's new admin APIs rather than shelling out to old bash scripts 11 | 12 | ## Installation 13 | 14 | 1. Download the latest compiled binary from [GitHub releases](https://github.com/travisjeffery/terraform-provider-kafka/releases). 15 | 16 | 1. Unzip/untar the archive. 17 | 18 | 1. Move it into `$HOME/.terraform.d/plugins`: 19 | 20 | ```sh 21 | $ mkdir -p $HOME/.terraform.d/plugins 22 | $ mv terraform-provider-kafka $HOME/.terraform.d/plugins/terraform-provider-kafka 23 | ``` 24 | 25 | 1. Create your Terraform configurations as normal, and run `terraform init`: 26 | 27 | ```sh 28 | $ terraform init 29 | ``` 30 | 31 | This will find the plugin locally. 32 | 33 | 34 | ## Usage 35 | 36 | 1. Create a Terraform configuration file: 37 | 38 | ```hcl 39 | provider "kafka" { 40 | hosts = ["localhost:9092"] 41 | } 42 | 43 | resource "kafka_topic" "example" { 44 | name: "example" 45 | num_partitions: "8" 46 | replication_factor: "1" 47 | config_entries: { 48 | retention.bytes: "102400" 49 | cleanup.policy: "compact 50 | } 51 | } 52 | ``` 53 | 54 | [There's parameters to set if you use TLS/SASL](https://github.com/travisjeffery/terraform-provider-kafka/blob/58dfc2e47748eb6a4f817a3e93d9848c1668c164/topic/provider.go#L18-L46). 55 | 56 | 1. Run `terraform init` to pull in the provider: 57 | 58 | ```sh 59 | $ terraform init 60 | ``` 61 | 62 | 1. Run `terraform plan` and `terraform apply` to interact with the filesystem: 63 | 64 | ```sh 65 | $ terraform plan 66 | 67 | $ terraform apply 68 | ``` 69 | 70 | ## Importing topics 71 | 72 | This provider supports importing externally created topics by their name. Assuming you've already created a topic declaration like the one above, you can get Terraform to manage the state of the existing topic: 73 | 74 | ```sh 75 | $ terraform import kafka_topic.example example 76 | ``` 77 | 78 | ## Examples 79 | 80 | For more examples, please see the [examples](https://github.com/travisjeffery/terraform-provider-kafka/tree/master/examples) folder in this 81 | repository. 82 | 83 | ## License 84 | 85 | MIT 86 | 87 | --- 88 | 89 | - [travisjeffery.com](http://travisjeffery.com) 90 | - GitHub [@travisjeffery](https://github.com/travisjeffery) 91 | - Twitter [@travisjeffery](https://twitter.com/travisjeffery) 92 | - Medium [@travisjeffery](https://medium.com/@travisjeffery) 93 | -------------------------------------------------------------------------------- /examples/simple/main.tf: -------------------------------------------------------------------------------- 1 | provider "kafka" { 2 | hosts = ["localhost:9092"] 3 | } 4 | 5 | resource "kafka_topic" "example" { 6 | name = "example" 7 | num_partitions = 8 8 | replication_factor = 1 9 | 10 | config_entries = { 11 | "retention.bytes" = 102400 12 | "cleanup.policy" = "compact" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/hashicorp/terraform/plugin" 5 | "github.com/travisjeffery/terraform-provider-kafka/topic" 6 | ) 7 | 8 | func main() { 9 | plugin.Serve(&plugin.ServeOpts{ 10 | ProviderFunc: topic.Provider, 11 | }) 12 | } 13 | -------------------------------------------------------------------------------- /topic/provider.go: -------------------------------------------------------------------------------- 1 | package topic 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "strings" 8 | "sync" 9 | 10 | "github.com/Shopify/sarama" 11 | "github.com/hashicorp/terraform/helper/schema" 12 | "github.com/hashicorp/terraform/terraform" 13 | ) 14 | 15 | type threadsafeClient struct { 16 | sarama.Client 17 | *sync.Mutex 18 | } 19 | 20 | // Provider returns the actual provider instance. 21 | func Provider() terraform.ResourceProvider { 22 | return &schema.Provider{ 23 | Schema: map[string]*schema.Schema{ 24 | "hosts": { 25 | Type: schema.TypeList, 26 | Elem: &schema.Schema{Type: schema.TypeString}, 27 | Optional: true, 28 | Description: "Your Kafka host addresses.", 29 | DefaultFunc: func() (interface{}, error) { 30 | return getHosts() 31 | }, 32 | }, 33 | "tls_enable": { 34 | Type: schema.TypeBool, 35 | Description: "Whether or not to use TLS when connecting to the broker.", 36 | Optional: true, 37 | }, 38 | "sasl_enable": { 39 | Type: schema.TypeBool, 40 | Description: "Whether or not to use SASL auth when connecting to the broker.", 41 | Optional: true, 42 | }, 43 | "sasl_username": { 44 | Type: schema.TypeString, 45 | Description: "Username for SASL/Plain authentication.", 46 | Optional: true, 47 | }, 48 | "sasl_password": { 49 | Type: schema.TypeString, 50 | Description: "Password for SASL/Plain authentication.", 51 | Optional: true, 52 | }, 53 | }, 54 | ResourcesMap: map[string]*schema.Resource{ 55 | "kafka_topic": resource(), 56 | }, 57 | ConfigureFunc: configure, 58 | } 59 | } 60 | 61 | func configure(d *schema.ResourceData) (interface{}, error) { 62 | cfg := sarama.NewConfig() 63 | cfg.Version = sarama.V1_0_0_0 64 | 65 | if v, ok := d.GetOk("tls_enable"); ok { 66 | cfg.Net.TLS.Enable = v.(bool) 67 | } 68 | 69 | if v, ok := d.GetOk("sasl_enable"); ok { 70 | cfg.Net.SASL.Enable = v.(bool) 71 | } 72 | 73 | if v, ok := d.GetOk("sasl_username"); ok { 74 | cfg.Net.SASL.User = v.(string) 75 | } 76 | 77 | if v, ok := d.GetOk("sasl_password"); ok { 78 | cfg.Net.SASL.Password = v.(string) 79 | } 80 | 81 | var hosts []string 82 | for _, host := range d.Get("hosts").([]interface{}) { 83 | hosts = append(hosts, host.(string)) 84 | } 85 | if hosts == nil { 86 | hosts, _ = getHosts() 87 | } 88 | 89 | log.Printf("[INFO] Initializing Kafka client with hosts: %v\n", hosts) 90 | 91 | client, err := sarama.NewClient(hosts, cfg) 92 | if err != nil { 93 | return nil, fmt.Errorf("failed to create kafka client: %s", err) 94 | } 95 | 96 | return &threadsafeClient{client, new(sync.Mutex)}, nil 97 | } 98 | 99 | func getHosts() ([]string, error) { 100 | hosts := os.Getenv("KAFKA_HOSTS") 101 | log.Printf("[INFO] hosts: %v\n", hosts) 102 | if hosts == "" { 103 | return []string{}, nil 104 | } 105 | return strings.Split(hosts, ","), nil 106 | } 107 | -------------------------------------------------------------------------------- /topic/provider_test.go: -------------------------------------------------------------------------------- 1 | package topic 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/hashicorp/terraform/config" 7 | "github.com/hashicorp/terraform/helper/schema" 8 | "github.com/hashicorp/terraform/terraform" 9 | ) 10 | 11 | var testAccProviders map[string]terraform.ResourceProvider 12 | var testAccProvider *schema.Provider 13 | 14 | func init() { 15 | testAccProvider = Provider().(*schema.Provider) 16 | testAccProviders = map[string]terraform.ResourceProvider{ 17 | "kafka": testAccProvider, 18 | } 19 | } 20 | 21 | func TestProvider(t *testing.T) { 22 | if err := Provider().(*schema.Provider).InternalValidate(); err != nil { 23 | t.Fatal(err) 24 | } 25 | } 26 | 27 | func TestProvider_impl(t *testing.T) { 28 | var _ terraform.ResourceProvider = Provider() 29 | } 30 | 31 | func testAccPreCheck(t *testing.T) { 32 | c, err := config.NewRawConfig(map[string]interface{}{ 33 | "hosts": []string{"localhost:9092"}, 34 | }) 35 | if err != nil { 36 | t.Fatal(err) 37 | } 38 | err = testAccProvider.Configure(terraform.NewResourceConfig(c)) 39 | if err != nil { 40 | t.Fatal(err) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /topic/resource.go: -------------------------------------------------------------------------------- 1 | package topic 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/Shopify/sarama" 8 | "github.com/hashicorp/terraform/helper/schema" 9 | ) 10 | 11 | func resource() *schema.Resource { 12 | return &schema.Resource{ 13 | Create: create, 14 | Update: update, 15 | Read: read, 16 | Delete: delete, 17 | Importer: &schema.ResourceImporter{State: importTopic}, 18 | 19 | Schema: map[string]*schema.Schema{ 20 | "name": &schema.Schema{ 21 | Type: schema.TypeString, 22 | Description: "Name of the topic", 23 | ForceNew: true, 24 | Required: true, 25 | }, 26 | 27 | "num_partitions": &schema.Schema{ 28 | Type: schema.TypeInt, 29 | Description: "Number of partitions.", 30 | Required: true, 31 | }, 32 | 33 | "replication_factor": &schema.Schema{ 34 | Type: schema.TypeInt, 35 | Description: "Replication factor.", 36 | Required: true, 37 | }, 38 | 39 | "config_entries": &schema.Schema{ 40 | Type: schema.TypeMap, 41 | Description: "Config entries.", 42 | Optional: true, 43 | }, 44 | }, 45 | } 46 | } 47 | 48 | func create(d *schema.ResourceData, meta interface{}) error { 49 | c, err := client(meta) 50 | if err != nil { 51 | return err 52 | } 53 | 54 | topic := d.Get("name").(string) 55 | 56 | d.SetId(topic) 57 | 58 | topicDetail := &sarama.TopicDetail{} 59 | topicDetail.NumPartitions = int32(d.Get("num_partitions").(int)) 60 | topicDetail.ReplicationFactor = int16(d.Get("replication_factor").(int)) 61 | topicDetail.ConfigEntries = make(map[string]*string) 62 | 63 | for name, value := range d.Get("config_entries").(map[string]interface{}) { 64 | strval := value.(string) 65 | topicDetail.ConfigEntries[name] = &strval 66 | } 67 | 68 | topicDetails := make(map[string]*sarama.TopicDetail) 69 | topicDetails[topic] = topicDetail 70 | 71 | response, err := c.CreateTopics(&sarama.CreateTopicsRequest{ 72 | TopicDetails: topicDetails, 73 | Timeout: time.Second * 15, 74 | }) 75 | if err != nil || response.TopicErrors == nil { 76 | return err 77 | } 78 | if err := response.TopicErrors[topic]; err.Err != sarama.ErrNoError { 79 | return fmt.Errorf("topic error: %v", err) 80 | } 81 | 82 | return read(d, meta) 83 | } 84 | 85 | func update(d *schema.ResourceData, meta interface{}) error { 86 | c, err := client(meta) 87 | if err != nil { 88 | return err 89 | } 90 | 91 | topic := d.Get("name").(string) 92 | 93 | if d.HasChange("replication_factor") { 94 | return fmt.Errorf("can't update the replication factor currently") 95 | } 96 | 97 | if d.HasChange("num_partitions") { 98 | old, new := d.GetChange("num_partitions") 99 | if new.(int) < old.(int) { 100 | return fmt.Errorf("new num_partitions must be >= old num_partitions") 101 | } 102 | response, err := c.CreatePartitions(&sarama.CreatePartitionsRequest{ 103 | Timeout: time.Second * 15, 104 | TopicPartitions: map[string]*sarama.TopicPartition{ 105 | topic: { 106 | Count: int32(new.(int)), 107 | }, 108 | }, 109 | }) 110 | if err != nil || response.TopicPartitionErrors == nil { 111 | return err 112 | } 113 | if err := response.TopicPartitionErrors[topic]; err.Err != sarama.ErrNoError { 114 | return fmt.Errorf("topic partition error: %v", err) 115 | } 116 | } 117 | 118 | if d.HasChange("config_entries") { 119 | _, new := d.GetChange("config_entries") 120 | 121 | configs := make(map[string]*string) 122 | 123 | for name, value := range new.(map[string]interface{}) { 124 | strval := value.(string) 125 | configs[name] = &strval 126 | } 127 | 128 | response, err := c.AlterConfigs(&sarama.AlterConfigsRequest{ 129 | Resources: []*sarama.AlterConfigsResource{{ 130 | Type: sarama.TopicResource, 131 | Name: topic, 132 | ConfigEntries: configs, 133 | }}, 134 | }) 135 | if err != nil { 136 | return err 137 | } 138 | for _, resource := range response.Resources { 139 | if resource.ErrorCode != int16(sarama.ErrNoError) { 140 | return fmt.Errorf( 141 | "resource error: code: %d, message: %s", 142 | resource.ErrorCode, 143 | resource.ErrorMsg, 144 | ) 145 | } 146 | } 147 | } 148 | 149 | return read(d, meta) 150 | } 151 | 152 | func read(d *schema.ResourceData, meta interface{}) error { 153 | c, err := client(meta) 154 | if err != nil { 155 | return err 156 | } 157 | 158 | metadata, err := c.GetMetadata(&sarama.MetadataRequest{Topics: []string{d.Get("name").(string)}}) 159 | if err != nil { 160 | return err 161 | } 162 | if len(metadata.Topics) != 1 { 163 | return fmt.Errorf("expected 1 topic in metadata") 164 | } 165 | 166 | topic := metadata.Topics[0] 167 | 168 | d.Set("name", topic.Name) 169 | d.Set("num_partitions", len(topic.Partitions)) 170 | d.Set("replication_factor", len(topic.Partitions[0].Replicas)) // this work? 171 | 172 | if old, ok := d.GetOk("config_entries"); ok { 173 | read, err := configs(c, topic.Name) 174 | if err != nil { 175 | return err 176 | } 177 | new := make(map[string]interface{}) 178 | for name, value := range read { 179 | if _, ok := old.(map[string]interface{})[name]; ok { 180 | new[name] = value 181 | } 182 | } 183 | d.Set("config_entries", new) 184 | } 185 | 186 | return nil 187 | } 188 | 189 | func delete(d *schema.ResourceData, meta interface{}) error { 190 | c, err := client(meta) 191 | if err != nil { 192 | return err 193 | } 194 | 195 | topic := d.Get("name").(string) 196 | 197 | response, err := c.DeleteTopics(&sarama.DeleteTopicsRequest{ 198 | Topics: []string{topic}, 199 | Timeout: time.Second * 15, 200 | }) 201 | if err != nil || response.TopicErrorCodes == nil { 202 | return err 203 | } 204 | if errCode := response.TopicErrorCodes[topic]; errCode != sarama.ErrNoError { 205 | return fmt.Errorf("topic error code: %s", errCode) 206 | } 207 | return nil 208 | } 209 | 210 | func importTopic(d *schema.ResourceData, meta interface{}) ([]*schema.ResourceData, error) { 211 | if err := read(d, meta); err != nil { 212 | return nil, err 213 | } 214 | return []*schema.ResourceData{d}, nil 215 | } 216 | 217 | func client(meta interface{}) (*sarama.Broker, error) { 218 | client := meta.(*threadsafeClient) 219 | client.Lock() 220 | defer client.Unlock() 221 | controller, err := client.Controller() 222 | if err != nil { 223 | return nil, err 224 | } 225 | if ok, err := controller.Connected(); err != nil { 226 | return nil, err 227 | } else if ok { 228 | return controller, nil 229 | } 230 | if err = controller.Open(client.Config()); err != nil { 231 | return nil, err 232 | } 233 | return controller, nil 234 | } 235 | 236 | func configs(c *sarama.Broker, topic string) (map[string]string, error) { 237 | response, err := c.DescribeConfigs(&sarama.DescribeConfigsRequest{ 238 | Resources: []*sarama.ConfigResource{{ 239 | Type: sarama.TopicResource, 240 | Name: topic, 241 | }}}, 242 | ) 243 | if err != nil { 244 | return nil, err 245 | } 246 | if len(response.Resources) != 1 { 247 | return nil, fmt.Errorf("expected 1 resource in response") 248 | } 249 | resource := response.Resources[0] 250 | if resource.ErrorCode != int16(sarama.ErrNoError) { 251 | return nil, fmt.Errorf( 252 | "resource error: code: %d, message: %s", 253 | resource.ErrorCode, 254 | resource.ErrorMsg, 255 | ) 256 | } 257 | 258 | configs := make(map[string]string) 259 | for _, config := range resource.Configs { 260 | configs[config.Name] = config.Value 261 | } 262 | 263 | return configs, nil 264 | } 265 | -------------------------------------------------------------------------------- /topic/resource_test.go: -------------------------------------------------------------------------------- 1 | package topic 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "testing" 7 | "time" 8 | 9 | "github.com/Shopify/sarama" 10 | tfresource "github.com/hashicorp/terraform/helper/resource" 11 | "github.com/hashicorp/terraform/terraform" 12 | ) 13 | 14 | var ( 15 | createConfig = ` 16 | resource "kafka_topic" "example" { 17 | name = "example" 18 | num_partitions = "3" 19 | replication_factor = "1" 20 | config_entries = { 21 | retention.bytes = "102400" 22 | cleanup.policy = "compact" 23 | } 24 | } 25 | ` 26 | 27 | updateConfig = ` 28 | resource "kafka_topic" "example" { 29 | name = "example" 30 | num_partitions = "4" 31 | replication_factor = "1" 32 | config_entries = { 33 | retention.bytes = "1024" 34 | cleanup.policy = "compact" 35 | } 36 | } 37 | ` 38 | ) 39 | 40 | func TestTopic(t *testing.T) { 41 | t.Run("local", func(t *testing.T) { 42 | t.Parallel() 43 | 44 | tfresource.UnitTest(t, tfresource.TestCase{ 45 | IsUnitTest: true, 46 | PreCheck: func() { testAccPreCheck(t) }, 47 | Providers: testAccProviders, 48 | Steps: []tfresource.TestStep{ 49 | { 50 | Config: createConfig, 51 | Check: func(s *terraform.State) error { 52 | attrs := s.RootModule().Resources["kafka_topic.example"].Primary.Attributes 53 | 54 | if act, exp := attrs["name"], "example"; act != exp { 55 | t.Errorf("expected %q to be %q", act, exp) 56 | } 57 | if act, exp := attrs["num_partitions"], "3"; act != exp { 58 | t.Errorf("expected %q to be %q", act, exp) 59 | } 60 | if act, exp := attrs["replication_factor"], "1"; act != exp { 61 | t.Errorf("expected %q to be %q", act, exp) 62 | } 63 | if act, exp := attrs["config_entries"], map[string]interface{}{ 64 | "retention.bytes": "102400", 65 | "cleanup.policy": "compact", 66 | }; reflect.DeepEqual(act, exp) { 67 | t.Errorf("expected %v to be %v", act, exp) 68 | } 69 | 70 | return nil 71 | }, 72 | }, 73 | { 74 | Config: updateConfig, 75 | Check: func(s *terraform.State) error { 76 | attrs := s.RootModule().Resources["kafka_topic.example"].Primary.Attributes 77 | 78 | if act, exp := attrs["name"], "example"; act != exp { 79 | t.Errorf("expected %q to be %q", act, exp) 80 | } 81 | if act, exp := attrs["num_partitions"], "4"; act != exp { 82 | t.Errorf("expected %q to be %q", act, exp) 83 | } 84 | if act, exp := attrs["replication_factor"], "1"; act != exp { 85 | t.Errorf("expected %q to be %q", act, exp) 86 | } 87 | if act, exp := attrs["config_entries"], map[string]interface{}{ 88 | "retention.bytes": "1024", 89 | "cleanup.policy": "compact", 90 | }; reflect.DeepEqual(act, exp) { 91 | t.Errorf("expected %v to be %v", act, exp) 92 | } 93 | 94 | return nil 95 | }, 96 | }, 97 | }, 98 | CheckDestroy: func(*terraform.State) error { 99 | cfg := sarama.NewConfig() 100 | cfg.Version = sarama.V1_0_0_0 101 | client, err := sarama.NewClient([]string{"localhost:9092"}, cfg) 102 | if err != nil { 103 | return fmt.Errorf("failed to create kafka client: %s", err) 104 | } 105 | for count := 0; count < 3; count++ { 106 | if err := client.RefreshMetadata(); err != nil { 107 | continue 108 | } 109 | topics, err := client.Topics() 110 | if err != nil { 111 | continue 112 | } 113 | var found bool 114 | for _, topic := range topics { 115 | if topic == "example" { 116 | found = true 117 | } 118 | } 119 | if !found { 120 | return nil 121 | } 122 | time.Sleep(1 * time.Second) 123 | } 124 | return fmt.Errorf("topic wasn't removed") 125 | }, 126 | }) 127 | }) 128 | } 129 | --------------------------------------------------------------------------------