├── .gitignore ├── BUILD.bazel ├── README.md ├── WORKSPACE ├── _config.yml ├── factory ├── BUILD.bazel ├── config │ ├── BUILD.baze │ └── base.json ├── main.go └── templates │ ├── BUILD.bazel │ └── things.tmpl ├── rules └── factory.bzl └── stone ├── BUILD.bazel └── config └── config.json /.gitignore: -------------------------------------------------------------------------------- 1 | /bazel-* 2 | -------------------------------------------------------------------------------- /BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@bazel_gazelle//:def.bzl", "gazelle") 2 | 3 | # gazelle:prefix github.com/linzhp/codegen_example 4 | gazelle(name = "gazelle") 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # How to define a Bazel rule to generate Go code 2 | ## Introduction 3 | At Uber, we use Bazel, an open source build system from Google, to build our [Go monorepo](https://eng.uber.com/go-monorepo-bazel/), which heavily uses generated code. A simple approach to manage generated code is to check them into the source repository and build them in the same way as manually written code. However, this approach would increase the size of the source repository, leading to performance issues. In addition, the generated code can often get outdated, leading to unexpected build failures. To avoid these issues, we decided to generate code as part of Bazel builds. With build cache, Bazel will automatically decide whether the generated code needs to be updated. To tell Bazel how to generate code, we need to define the code generation process as [rules](https://docs.bazel.build/versions/master/skylark/rules.html). 4 | 5 | The open source community has developed Bazel rules for some commonly used code generators. Most notable ones are [go_proto_library](https://github.com/bazelbuild/rules_go/blob/master/proto/core.rst#go_proto_library) for protoc, [gomock](https://github.com/jmhodges/bazel_gomock) for mockgen. However, there are cases when people want to use their own code generators. This article is a step-by-step tutorial on how to write a Bazel rule to generate and build code with a custom code generator. 6 | 7 | This article assumes you are in a repository with Bazel Go rules and Gazelle properly set up. If not, please follow the [Setup](https://github.com/bazelbuild/bazel-gazelle#setup) section of Gazelle to do so. 8 | 9 | ## Toy Code Generator 10 | Let’s first create a toy code generator, which reads a configuration file to fill a template, then write the result into a Go file. The code generator looks like this: 11 | 12 | ```go 13 | func main() { 14 | pkg := flag.String("package", "codegen", "the package name in the generated code file") 15 | tmplPath := flag.String("tmpl", "factory/templates/things.tmpl", "the template file") 16 | configPath := flag.String("config", "factory/config/base.json", "the configuration file") 17 | outPath := flag.String("out", "out.go", "the output file") 18 | flag.Parse() 19 | file, err := os.Open(*configPath) 20 | check(err) 21 | decoder := json.NewDecoder(file) 22 | var config Configuration 23 | if err = decoder.Decode(&config); err != nil { 24 | log.Fatal(err) 25 | } 26 | config.Package = *pkg 27 | 28 | rawBytes, err := ioutil.ReadFile(*tmplPath) 29 | check(err) 30 | tmpl, err := template.New("thing").Parse(string(rawBytes)) 31 | check(err) 32 | out, err := os.Create(*outPath) 33 | check(err) 34 | err = tmpl.Execute(out, config) 35 | check(err) 36 | } 37 | 38 | type Configuration struct { 39 | Package string 40 | Count int 41 | Material string 42 | } 43 | ``` 44 | 45 | The template file looks like this: 46 | 47 | ```go 48 | // Generated code. DO NOT EDIT 49 | package {{.Package}} 50 | 51 | func String() string { 52 | return "{{.Count}} items are made of {{.Material}}" 53 | } 54 | ``` 55 | 56 | The configuration file: 57 | 58 | ```json 59 | { 60 | "Material": "wool", 61 | "Count": 17 62 | } 63 | ``` 64 | 65 | Now if we run Gazelle with `bazel run //:gazelle`, a BUILD.bazel file will be generated for the code generator with a go_binary rule in it. Now we add a data parameter to the go_binary rule: 66 | 67 | ```python 68 | data = glob([ 69 | "config/*.json", 70 | "templates/*.tmpl", 71 | ]), 72 | ``` 73 | 74 | With this parameter, we are able to run the code generator (which is under a directory called `factory`) like this: 75 | 76 | ```bash 77 | $ bazel run //factory:factory -- -out=/tmp/out.go -config factory/config/base.json -tmpl factory/templates/things.tmpl 78 | ``` 79 | 80 | If it succeeds, we can find /tmp/out.go is generated. As the output itself is a Go file, we want to also compile the Go file too. Of course, we can copy the Go file into some directory in the repository, and use Bazel to compile it. However, how do we make Bazel both generate the file and compile it with one single “bazel build” command? We need to define a Bazel rule on our own. 81 | 82 | ## Declaring a rule 83 | 84 | Bazel has a go_library rule to compile Go files, so we don’t need to worry about compilation. However, for the toy code generator we just invented, we need to write a Bazel rule from scratch. A rule definition consists of at least two parts: a declaration and an implementation. We will start with a rule declaration in this section and cover the implementation in the next. 85 | 86 | ```python 87 | load("@io_bazel_rules_go//go:def.bzl", "go_rule") 88 | 89 | def _codegen_impl(ctx): 90 | # to be done in the next section 91 | pass 92 | 93 | factory = go_rule( 94 | _codegen_impl, 95 | attrs = { 96 | "_template": attr.label( 97 | allow_single_file = True, 98 | default = "//factory/templates:things.tmpl", 99 | ), 100 | "config": attr.label( 101 | allow_single_file = True, 102 | ), 103 | "package": attr.string( 104 | doc = "the package name for the generated Go file", 105 | ), 106 | "_generator": attr.label( 107 | executable = True, 108 | cfg = "host", 109 | default = "//factory", 110 | ), 111 | }, 112 | ) 113 | ``` 114 | 115 | In this code snippet, we declare a Bazel rule called `factory`. As part of the declaration, we provide it with an implementation function of the rule call `_codegen_impl`. Then we declare all build targets this rule depends on, such as executable targets and files. In this example, the only executable is the `//factory:factory` target we created in the last section. If the executable calls other tools such as gofmt programmatically after generating the Go file, gofmt has to be specified here too. Similarly, all the files `//factory:factory` needs to read during its execution have to be declared in the rule declaration, even if the file path is hard coded in the code instead of passed in as a command line argument to the code generator. 116 | 117 | For some rule attributes that we don’t want users to customize, they can have their name starting with an underscore. In this example, we don’t want users to customize template and the code generator location, but they need to be declared as attributes because they are inputs to the rule, so we declare them as `_template` and `_generator`, and assign a default value to each of them. 118 | 119 | More information about the attributes can be found in the official [Bazel document](https://docs.bazel.build/versions/master/skylark/rules.html#attributes). 120 | 121 | ## Rule implementation 122 | Rule implementation is the actual code that calls the code generator, reads the input, and generates the Go file, according to the values of rule attributes users pass in. 123 | 124 | ```python 125 | def _codegen_impl(ctx): 126 | out = ctx.actions.declare_file("out.go") 127 | args = ctx.actions.args() 128 | args.add("-config", ctx.file.config.path) 129 | args.add("-out", out.path) 130 | args.add("-package", ctx.attr.package) 131 | ctx.actions.run( 132 | inputs = [ctx.file._template, ctx.file.config], 133 | outputs = [out], 134 | executable = ctx.executable._generator, 135 | tools = [ctx.executable._generator], 136 | arguments = [args], 137 | mnemonic = "SmallFactory", 138 | ) 139 | return [ 140 | DefaultInfo(files = depset([out])), 141 | ] 142 | ``` 143 | 144 | In the code snippet above, we first declare that the rule is going to generate a file called `out.go`. 145 | Then we construct the command line arguments to the code generator. Some of the values are from the rule attributes declared in the rule. Note that we do not pass the template, and instead let the code generator use its default template. The function `ctx.actions.run` is the one that executes the code generator. Although we do not pass the template in the command line arguments, we still have to specify it as one of the inputs to the action. The `executable` parameter specifies the main executable of the action. However, the main executable may programmatically call other executables. All of these executables have to be specified in the `tools` parameter. 146 | 147 | Finally, the rule returns some [providers](https://docs.bazel.build/versions/master/skylark/rules.html#providers) that other rules may need. In our example, the only output is the generated Go file. Some other rules may also compile the Go file too, making the binary file the final output. If this is the case, it's better to return an extra provider called `OutputGroupInfo`, and put the generated Go file in an output group called `go_generated_srcs`, similar to [go_proto_library](https://github.com/bazelbuild/rules_go/blob/ac1b0e0544de55a1ef2cbd37b30503c9e860f795/proto/def.bzl#L127-L139): 148 | 149 | ```python 150 | return [ 151 | DefaultInfo(files = depset([ctx.outputs.out])), 152 | OutputGroupInfo( 153 | go_generated_srcs = [ctx.outputs.out], 154 | ), 155 | ] 156 | ``` 157 | 158 | So we can generate the Go code without other unnecessary steps by: 159 | 160 | ```bash 161 | $ bazel build --output_groups=go_generated_srcs //some/code/gen:target 162 | ``` 163 | 164 | ## Using the factory rule 165 | 166 | Now we can use the newly created `factory` rule to generate code. After loading the rule, we pass a new configuration file, a package name for the generated code, and the output file name. 167 | 168 | ```python 169 | load("//:rules/factory.bzl", "factory") 170 | 171 | factory( 172 | name = "go_factory", 173 | config = "config/config.json", 174 | package = "main", 175 | ) 176 | ``` 177 | 178 | Assuming we save this rule in `stone/BUILD.bazel`, we can execute the rule using the following Bazel build command: 179 | 180 | ```bash 181 | $ bazel build //stone:go_factory 182 | INFO: Analyzed target //stone:go_factory (1 packages loaded, 2 targets configured). 183 | INFO: Found 1 target... 184 | Target //stone:go_factory up-to-date: 185 | bazel-bin/stone/stone.go 186 | INFO: Elapsed time: 0.138s, Critical Path: 0.00s 187 | INFO: 0 processes. 188 | INFO: Build completed successfully, 1 total action 189 | ``` 190 | 191 | We can see that a Go file has been generated at `bazel-bin/stone/stone.go`: 192 | 193 | ```go 194 | // Generated code. DO NOT EDIT 195 | package main 196 | 197 | func String() string { 198 | return "17 items are made of stone" 199 | } 200 | ``` 201 | 202 | ## Compiling the generated code 203 | 204 | Now we can compile this generated code along with other regular Go files. Let’s first create a `print.go` file to call the generated function `String()`: 205 | 206 | ```go 207 | package main 208 | 209 | import "fmt" 210 | 211 | func main() { 212 | fmt.Println(String()) 213 | } 214 | ``` 215 | 216 | After putting this new Go file along side with `stone/BUILD.bazel`, and run Gazelle (`bazel run //:gazelle`), a `go_library` and a `go_binary` rule will be generated. In order to compile the generated code, we need to add the code generation target to the `srcs` of `go_library`: 217 | 218 | ```python 219 | go_library( 220 | name = "go_default_library", 221 | srcs = [ 222 | "print.go", 223 | ":go_factory", # keep 224 | ], 225 | importpath = "github.com/linzhp/codegen_example/stone", 226 | visibility = ["//visibility:public"], 227 | ) 228 | 229 | go_binary( 230 | name = "stone", 231 | embed = [":go_default_library"], 232 | visibility = ["//visibility:public"], 233 | ) 234 | ``` 235 | 236 | Note that we also put a `# keep` directive besides the `:go_factory` target, so future run of Gazelle will preserve the manually added target. When we pass the `:go_factory` target into `srcs`, we are actually passing the files in the `DefaultInfo` provider returned by `_codegen_impl`. 237 | 238 | Now we can build a binary from both Go files: 239 | 240 | ```bash 241 | $ bazel build //stone:stone 242 | INFO: Analyzed target //stone:stone (0 packages loaded, 4 targets configured). 243 | INFO: Found 1 target... 244 | Target //stone:stone up-to-date: 245 | bazel-bin/stone/darwin_amd64_stripped/stone 246 | INFO: Elapsed time: 0.122s, Critical Path: 0.00s 247 | INFO: 0 processes. 248 | INFO: Build completed successfully, 1 total action 249 | ``` 250 | 251 | ## All in one step 252 | As we promised, we want to do the code generation, compilation with one Bazel command. We can actually go one step further and run the final binary. As we had run some build commands before, let’s run `bazel clean` command to remove all the artifacts generated by previous commands and make sure that both `bazel-bin/stone/stone.go` and `bazel-bin/stone/darwin_amd64_stripped/stone` are gone. Now we can do all the steps in a Bazel run command: 253 | 254 | ```bash 255 | $ bazel run //stone 256 | INFO: Analyzed target //stone:stone (29 packages loaded, 6670 targets configured). 257 | INFO: Found 1 target... 258 | Target //stone:stone up-to-date: 259 | bazel-bin/stone/darwin_amd64_stripped/stone 260 | INFO: Elapsed time: 3.475s, Critical Path: 1.47s 261 | INFO: 6 processes: 6 darwin-sandbox. 262 | INFO: Build completed successfully, 12 total actions 263 | INFO: Build completed successfully, 12 total actions 264 | 17 items are made of stone 265 | ``` 266 | 267 | This command first tries to build `//stone:stone`, and finds that its dependencies have not built yet. So it builds the dependencies first, which includes the code generation. After the binary is built, the command also executes the binary, which prints out the string from the generated code: "17 items are made of stone." 268 | 269 | The full working example can be found at https://github.com/linzhp/codegen_example. Thanks for reading. 270 | -------------------------------------------------------------------------------- /WORKSPACE: -------------------------------------------------------------------------------- 1 | load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") 2 | 3 | http_archive( 4 | name = "io_bazel_rules_go", 5 | urls = [ 6 | "https://storage.googleapis.com/bazel-mirror/github.com/bazelbuild/rules_go/releases/download/v0.20.3/rules_go-v0.20.3.tar.gz", 7 | "https://github.com/bazelbuild/rules_go/releases/download/v0.20.3/rules_go-v0.20.3.tar.gz", 8 | ], 9 | sha256 = "e88471aea3a3a4f19ec1310a55ba94772d087e9ce46e41ae38ecebe17935de7b", 10 | ) 11 | 12 | http_archive( 13 | name = "bazel_gazelle", 14 | urls = [ 15 | "https://storage.googleapis.com/bazel-mirror/github.com/bazelbuild/bazel-gazelle/releases/download/v0.19.1/bazel-gazelle-v0.19.1.tar.gz", 16 | "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.19.1/bazel-gazelle-v0.19.1.tar.gz", 17 | ], 18 | sha256 = "86c6d481b3f7aedc1d60c1c211c6f76da282ae197c3b3160f54bd3a8f847896f", 19 | ) 20 | 21 | load("@io_bazel_rules_go//go:deps.bzl", "go_rules_dependencies", "go_register_toolchains") 22 | 23 | go_rules_dependencies() 24 | 25 | go_register_toolchains() 26 | 27 | load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") 28 | 29 | gazelle_dependencies() -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-minimal -------------------------------------------------------------------------------- /factory/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["main.go"], 6 | importpath = "github.com/linzhp/codegen_example/factory", 7 | visibility = ["//visibility:private"], 8 | ) 9 | 10 | go_binary( 11 | name = "factory", 12 | data = glob([ 13 | "config/*.json", 14 | "templates/*.tmpl", 15 | ]), 16 | embed = [":go_default_library"], 17 | visibility = ["//visibility:public"], 18 | ) 19 | -------------------------------------------------------------------------------- /factory/config/BUILD.baze: -------------------------------------------------------------------------------- 1 | filegroup( 2 | name = "config", 3 | srcs = glob(["*.json"]), 4 | ) -------------------------------------------------------------------------------- /factory/config/base.json: -------------------------------------------------------------------------------- 1 | { 2 | "Material": "wool", 3 | "Count": 17 4 | } -------------------------------------------------------------------------------- /factory/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "flag" 6 | "html/template" 7 | "io/ioutil" 8 | "log" 9 | "os" 10 | ) 11 | 12 | func main() { 13 | pkg := flag.String("package", "codegen", "the package name in the generated code file") 14 | tmplPath := flag.String("tmpl", "factory/templates/things.tmpl", "the template file") 15 | configPath := flag.String("config", "factory/config/base.json", "the configuration file") 16 | outPath := flag.String("out", "out.go", "the output file") 17 | flag.Parse() 18 | file, err := os.Open(*configPath) 19 | check(err) 20 | decoder := json.NewDecoder(file) 21 | var config Configuration 22 | if err = decoder.Decode(&config); err != nil { 23 | log.Fatal(err) 24 | } 25 | config.Package = *pkg 26 | 27 | rawBytes, err := ioutil.ReadFile(*tmplPath) 28 | check(err) 29 | tmpl, err := template.New("thing").Parse(string(rawBytes)) 30 | check(err) 31 | out, err := os.Create(*outPath) 32 | check(err) 33 | err = tmpl.Execute(out, config) 34 | check(err) 35 | } 36 | 37 | type Configuration struct { 38 | Package string 39 | Count int 40 | Material string 41 | } 42 | 43 | func check(err error) { 44 | if err != nil { 45 | log.Fatal(err) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /factory/templates/BUILD.bazel: -------------------------------------------------------------------------------- 1 | exports_files( 2 | glob(["*.tmpl"]), 3 | visibility = ["//visibility:public"], 4 | ) 5 | -------------------------------------------------------------------------------- /factory/templates/things.tmpl: -------------------------------------------------------------------------------- 1 | // Generated code. DO NOT EDIT 2 | package {{.Package}} 3 | 4 | func String() string { 5 | return "{{.Count}} items are made of {{.Material}}" 6 | } -------------------------------------------------------------------------------- /rules/factory.bzl: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_rule") 2 | 3 | def _codegen_impl(ctx): 4 | out = ctx.actions.declare_file("out.go") 5 | args = ctx.actions.args() 6 | args.add("-config", ctx.file.config.path) 7 | args.add("-out", out.path) 8 | args.add("-package", ctx.attr.package) 9 | ctx.actions.run( 10 | inputs = [ctx.file._template, ctx.file.config], 11 | outputs = [out], 12 | executable = ctx.executable._generator, 13 | tools = [ctx.executable._generator], 14 | arguments = [args], 15 | mnemonic = "SmallFactory", 16 | ) 17 | return [ 18 | DefaultInfo(files = depset([out])), 19 | ] 20 | 21 | factory = go_rule( 22 | _codegen_impl, 23 | attrs = { 24 | "_template": attr.label( 25 | allow_single_file = True, 26 | default = "//factory/templates:things.tmpl", 27 | ), 28 | "config": attr.label( 29 | allow_single_file = True, 30 | ), 31 | "package": attr.string( 32 | doc = "the package name for the generated Go file", 33 | ), 34 | "_generator": attr.label( 35 | executable = True, 36 | cfg = "host", 37 | default = "//factory", 38 | ), 39 | }, 40 | ) 41 | -------------------------------------------------------------------------------- /stone/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library") 2 | load("//:rules/factory.bzl", "factory") 3 | 4 | factory( 5 | name = "go_factory", 6 | config = "config/config.json", 7 | package = "stone", 8 | ) 9 | 10 | go_library( 11 | name = "go_generated_library", 12 | srcs = [":go_factory"], 13 | importpath = "github.com/linzhp/codegen_example/stone", 14 | visibility = ["//visibility:public"], 15 | ) 16 | -------------------------------------------------------------------------------- /stone/config/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "Material": "stone", 3 | "Count": 17 4 | } --------------------------------------------------------------------------------