├── LICENSE ├── Makefile ├── README.md ├── WORKSPACE ├── barn ├── BUILD.bazel ├── Makefile ├── k8s │ ├── BUILD.bazel │ └── namespace.yaml ├── redis │ └── k8s │ │ ├── BUILD.bazel │ │ ├── master.yaml │ │ ├── sentinel.yaml │ │ └── service.yaml ├── scheduler │ ├── BUILD.bazel │ ├── k8s │ │ ├── BUILD.bazel │ │ ├── deploy.yaml │ │ └── service.yaml │ └── structure-tests.yaml ├── server │ ├── BUILD.bazel │ ├── k8s │ │ ├── BUILD.bazel │ │ ├── configmap.yaml │ │ ├── deploy.yaml │ │ └── service.yaml │ └── structure-tests.yaml ├── worker │ ├── BUILD.bazel │ ├── k8s │ │ ├── BUILD.bazel │ │ └── deploy.yaml │ └── structure-tests.yaml └── workspace.bzl ├── bazelrc ├── farm ├── BUILD.bazel ├── Makefile ├── k8s │ ├── BUILD.bazel │ └── namespace.yaml ├── local_repository.bzl ├── server │ ├── BUILD.bazel │ ├── k8s │ │ ├── BUILD.bazel │ │ ├── deploy.yaml │ │ └── service.yaml │ ├── server.config │ └── structure-tests.yaml ├── worker │ ├── BUILD.bazel │ ├── k8s │ │ ├── BUILD.bazel │ │ └── deploy.yaml │ ├── structure-tests.yaml │ └── worker.config └── workspace.bzl ├── grid ├── BUILD.bazel ├── Makefile ├── k8s │ ├── BUILD.bazel │ └── namespace.yaml ├── server │ ├── BUILD.bazel │ ├── k8s │ │ ├── BUILD.bazel │ │ ├── deploy.yaml │ │ └── service.yaml │ ├── server.conf │ └── structure-tests.yaml ├── structure-tests.yaml ├── worker │ ├── BUILD.bazel │ ├── k8s │ │ ├── BUILD.bazel │ │ └── deploy.yaml │ ├── structure-tests.yaml │ └── worker.conf └── workspace.bzl └── rbe └── Makefile /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2016 PubRef.org 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); you 4 | may not use this file except in compliance with the License. You may 5 | obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | abseil_clone: 2 | (cd /tmp && git clone https://github.com/abseil/abseil-cpp.git) 3 | 4 | abseil_clean: 5 | (cd /tmp/abseil-cpp && bazel clean) 6 | 7 | abseil: 8 | cp ./bazelrc /tmp 9 | (cd /tmp/abseil-cpp && bazel \ 10 | --bazelrc=/tmp/bazelrc \ 11 | build //absl/... \ 12 | --remote_instance_name=main \ 13 | --config=remote) 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # buildkube 2 | 3 | 4 | 9 | 15 | 21 | 22 | 23 | 24 | 25 |
5 | 6 | 7 | 8 | 10 | 11 | 13 | 14 | 16 | 17 | 19 | 20 |
BazelREAPIKubernetes
26 | 27 | buildkube uses [rules_docker] and [rules_k8s] to build and deploy 28 | [bazel-buildfarm] (java), [bazel-buildbarn] (golang) and/or [buildgrid] (python) 29 | into an existing kubernetes cluster. These are the 3 known open-source 30 | server-side implementations of the [remote-execution-api] (REAPI), plus the 31 | closed source google Remote Build Execution 32 | ([RBE](https://groups.google.com/forum/#!forum/rbe-alpha-customers)) service 33 | (alpha). 34 | 35 | Known clients of the REAPI include [bazel](https://github.com/bazelbuild/bazel) 36 | itself, [recc](https://gitlab.com/bloomberg/recc), and possibly 37 | [pants](https://github.com/pantsbuild/pants/pull/4910). 38 | 39 | ## INSTRUCTIONS 40 | 41 | 1. Clone this repository 42 | 2. Edit the `WORKSPACE` file `k8s_defaults` rule to point to your kubernetes 43 | cluster (should match `$ kubectl config current-context`) 44 | 3. Build and deploy an implementation: for example: `$ (cd farm/ && make 45 | install)` 46 | 4. In a separate terminal, establish port-forwarding to the server 47 | implementation `$ (cd farm/ && make port-forward)` 48 | 5. Clone the abseil repository as a test case: `$ make abseil_clone` 49 | 6. Compile abseil remotely: `$ make abseil` 50 | 51 | ## NOTES 52 | 53 | * Bazel 0.17.1 or higher is required (primarily tested on 0.17.2 on an ubuntu 54 | laptop). 55 | * Run all tests via `$ bazel test //...`. 56 | * Each implementation goes in its own namespace. `$ kubectl get pods 57 | --all-namespaces` to see all. 58 | * Consider adjusting `replicas` in the `deploy.yaml` files and/or `bazelrc` 59 | file. 60 | 61 | ## OBSERVATIONS 62 | 63 | ### General 64 | 65 | * Logging in all 3 implementations is scant and makes debugging difficult. 66 | Prometheus metrics are available in the barn impl (not examined thus far). 67 | 68 | ### BuildFarm 69 | 70 | * BuildFarm worker does not detect if server goes down. Must manually `kubectl 71 | delete pod --selector=k8s-app=worker` when re-installing or updating server 72 | deployment. 73 | 74 | * When a worker registers itself with the server (operation-queue), it provides 75 | a dict of key:value pairs that must match the action execution requirements. 76 | In particular, the `worker.config` `container-image` key MUST be exactly 77 | matching the rbe_ubuntu image tag. 78 | 79 | ### BuildBarn 80 | 81 | * After spinning up a new install, the service seems flaky at first. Tend to 82 | get several errors like: `/tmp/abseil-cpp/absl/utility/BUILD.bazel:22:1: C++ 83 | compilation of rule '//absl/utility:utility_test' failed (Exit 34). Note: 84 | Remote connection/protocol failed with: execution failed catastrophically`. 85 | 86 | > NOTE(@EdShoueten): There are three ways that can be used to alleviate this 87 | > issue: 88 | > - Spawn more workers on your cluster. 89 | > - Pass in an explicit --jobs= to the build that is the same order of magnitude 90 | > as the number of workers. 91 | > - Tune this flag on the scheduler process: 92 | > https://github.com/EdSchouten/bazel-buildbarn/blob/master/cmd/bbb_scheduler/main.go#L22 93 | > 94 | > [More details](https://groups.google.com/forum/#!topic/bazel-discuss/pPNIc9-liCE) 95 | 96 | ### BuildGrid 97 | 98 | * Worker does not auto-reconnect to a new server (like buildfarm). 99 | * Instance name (`main`) must match across the `bazelrc` `--instance_name=main`, 100 | server args `-scheduler main|ubuntu-scheduler:8981`, and worker args `bot 101 | --remote=http://server:8980 --parent=main host-tools` 102 | * Overall robustness to changes (increases) in job size and worker size is low. 103 | Seems to require resetting the server/workers in some cases. Seems happiest 104 | when job size matches worker replicas. 105 | 106 | [rules_docker]: https://github.com/bazelbuild/rules_docker 107 | [rules_k8s]: https://github.com/bazelbuild/rules_k8s 108 | [bazel-buildfarm]: https://github.com/bazelbuild/bazel-buildfarm/ 109 | [bazel-buildbarn]: https://github.com/EdSchouten/bazel-buildbarn/ 110 | [buildgrid]: https://gitlab.com/BuildGrid/buildgrid 111 | [remote-execution-api]: https://github.com/bazelbuild/remote-apis 112 | -------------------------------------------------------------------------------- /WORKSPACE: -------------------------------------------------------------------------------- 1 | workspace(name = "com_github_stackb_buildkube") 2 | 3 | load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") 4 | 5 | ##################################################################### 6 | # rules_go 7 | ##################################################################### 8 | 9 | RULES_GO_VERSION = "43d7595b0123b5f0cc35bd45c084582b3eb3198b" 10 | RULES_GO_SHA256 = "31ac06c3101ae13c226cdbccefd5f14577249c7e78bb344aabfb8e5d29366079" 11 | 12 | http_archive( 13 | name = "io_bazel_rules_go", 14 | strip_prefix = "rules_go-" + RULES_GO_VERSION, 15 | urls = ["https://github.com/bazelbuild/rules_go/archive/%s.tar.gz" % RULES_GO_VERSION], 16 | sha256 = RULES_GO_SHA256, 17 | ) 18 | 19 | load("@io_bazel_rules_go//go:def.bzl", "go_rules_dependencies", "go_register_toolchains") 20 | 21 | go_rules_dependencies() 22 | 23 | go_register_toolchains() 24 | 25 | 26 | ##################################################################### 27 | # bazel_skylib 28 | ##################################################################### 29 | 30 | http_archive( 31 | name = "bazel_skylib", 32 | sha256 = "b5f6abe419da897b7901f90cbab08af958b97a8f3575b0d3dd062ac7ce78541f", 33 | strip_prefix = "bazel-skylib-0.5.0", 34 | urls = ["https://github.com/bazelbuild/bazel-skylib/archive/0.5.0.tar.gz"], 35 | ) 36 | 37 | 38 | ##################################################################### 39 | # rules_docker 40 | ##################################################################### 41 | 42 | RULES_DOCKER_VERSION = "c9065d170c076d540166f068aec0e04039a10e66" 43 | RULES_DOCKER_SHA256 = "e1403c24f894b49bfd64f47b74a594687567c0180eddf43d014a565b3c5552e6" 44 | 45 | http_archive( 46 | name = "io_bazel_rules_docker", 47 | sha256 = RULES_DOCKER_SHA256, 48 | strip_prefix = "rules_docker-" + RULES_DOCKER_VERSION, 49 | urls = ["https://github.com/bazelbuild/rules_docker/archive/%s.tar.gz" % RULES_DOCKER_VERSION], 50 | ) 51 | 52 | 53 | ############################################################# 54 | # worker execution container 55 | ############################################################# 56 | 57 | load( 58 | "@io_bazel_rules_docker//container:container.bzl", 59 | container_repositories = "repositories", 60 | "container_pull", 61 | ) 62 | 63 | RBE_UBUNTU_REGISTRY = "gcr.io" 64 | RBE_UBUNTU_REPOSITORY = "cloud-marketplace/google/rbe-ubuntu16-04" 65 | RBE_UBUNTU_DIGEST = "sha256:9bd8ba020af33edb5f11eff0af2f63b3bcb168cd6566d7b27c6685e717787928" 66 | RBE_UBUNTU_TAG = "%s/%s@%s" % (RBE_UBUNTU_REGISTRY, RBE_UBUNTU_REPOSITORY, RBE_UBUNTU_DIGEST) 67 | 68 | container_pull( 69 | name = "rbe_ubuntu", 70 | registry = RBE_UBUNTU_REGISTRY, 71 | repository = RBE_UBUNTU_REPOSITORY, 72 | digest = RBE_UBUNTU_DIGEST, 73 | ) 74 | 75 | ############################################################# 76 | # KUBERNETES 77 | ############################################################# 78 | 79 | RULES_K8S_VERSION = "62ae7911ef60f91ed32fdd48a6b837287a626a80" 80 | RULES_K8S_SHA256 = "9bf9974199b3908a78638d3c7bd688bc2a69b3ddc857bd160399c58ca7fc18ea" 81 | 82 | http_archive( 83 | name = "io_bazel_rules_k8s", 84 | url = "https://github.com/bazelbuild/rules_k8s/archive/%s.zip" % RULES_K8S_VERSION, 85 | strip_prefix = "rules_k8s-" + RULES_K8S_VERSION, 86 | sha256 = RULES_K8S_SHA256, 87 | ) 88 | 89 | load("@io_bazel_rules_k8s//k8s:k8s.bzl", "k8s_repositories") 90 | 91 | k8s_repositories() 92 | 93 | load("@io_bazel_rules_k8s//k8s:k8s.bzl", "k8s_defaults") 94 | 95 | k8s_defaults( 96 | name = "k8s_deploy", 97 | kind = "deployment", 98 | cluster = "gke_stackb-151821_us-central1-a_remote-execution", 99 | ) 100 | 101 | 102 | ##################################################################### 103 | # BUILDFARM 104 | ##################################################################### 105 | 106 | load( 107 | "@io_bazel_rules_docker//java:image.bzl", 108 | java_image_repositories = "repositories", 109 | ) 110 | 111 | java_image_repositories() 112 | 113 | load("//farm:workspace.bzl", "buildfarm_repository") 114 | 115 | BUILDFARM_VERSION = "7f8f4c407041b5cd6795b28121cef7d3a3cbab2b" 116 | 117 | buildfarm_repository( 118 | name = "build_buildfarm", 119 | commit = BUILDFARM_VERSION, 120 | ) 121 | 122 | # Switch to the following for local buildfarm development 123 | # 124 | # load("//farm:local_repository.bzl", "local_buildfarm_repository") 125 | # local_buildfarm_repository( 126 | # path = "../../bazelbuild/bazel-buildfarm", 127 | # ) 128 | # load("@build_buildfarm//3rdparty:workspace.bzl", "maven_dependencies", "declare_maven") 129 | # maven_dependencies(declare_maven) 130 | 131 | 132 | ##################################################################### 133 | # BUILDGRID 134 | ##################################################################### 135 | 136 | 137 | BUILDGRID_VERSION = "a49581a60a595fcca0ddb7beec958cf943f09cf7" 138 | 139 | load("//grid:workspace.bzl", "buildgrid_repository") 140 | 141 | buildgrid_repository( 142 | name = "buildgrid_server", 143 | commit = BUILDGRID_VERSION, 144 | ) 145 | 146 | buildgrid_repository( 147 | name = "buildgrid_worker", 148 | commit = BUILDGRID_VERSION, 149 | dockerfile = """ 150 | FROM {base_image_tag} 151 | RUN python3 -m pip install --upgrade setuptools pip 152 | WORKDIR /app 153 | COPY . . 154 | RUN pip install --user --editable . 155 | """.format(base_image_tag = RBE_UBUNTU_TAG), 156 | ) 157 | 158 | 159 | ##################################################################### 160 | # BUILDBARN 161 | ##################################################################### 162 | 163 | 164 | http_archive( 165 | name = "bazel_gazelle", 166 | sha256 = "bc653d3e058964a5a26dcad02b6c72d7d63e6bb88d94704990b908a1445b8758", 167 | urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/0.13.0/bazel-gazelle-0.13.0.tar.gz"], 168 | ) 169 | 170 | load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") 171 | 172 | gazelle_dependencies() 173 | 174 | load("//barn:workspace.bzl", "buildbarn_repositories") 175 | 176 | buildbarn_repositories() 177 | 178 | BUILDBARN_VERSION = "e4c05f8003ae7a9f80876ed8fe61cf9b0e4b0784" 179 | BUILDBARN_SHA256 = "e4f4abc2fa5ddcd50c1652d21a28973113408b50f5151fcbe570d985f8bc7599" 180 | 181 | http_archive( 182 | name = "buildbarn", 183 | strip_prefix = "bazel-buildbarn-" + BUILDBARN_VERSION, 184 | urls = ["https://github.com/EdSchouten/bazel-buildbarn/archive/%s.tar.gz" % BUILDBARN_VERSION], 185 | sha256 = BUILDBARN_SHA256, 186 | patch_cmds = [ 187 | # Expose the go_library targets so we can build our own binaries / images 188 | "sed -i 's|//visibility:private|//visibility:public|g' cmd/bbb_frontend/BUILD.bazel", 189 | "sed -i 's|//visibility:private|//visibility:public|g' cmd/bbb_scheduler/BUILD.bazel", 190 | "sed -i 's|//visibility:private|//visibility:public|g' cmd/bbb_worker/BUILD.bazel", 191 | ], 192 | ) 193 | 194 | # local_repository( 195 | # name = "buildbarn", 196 | # path = "../../EdShouten/bazel-buildbarn", 197 | # ) 198 | 199 | load( 200 | "@io_bazel_rules_docker//go:image.bzl", 201 | go_image_repositories = "repositories", 202 | ) 203 | 204 | go_image_repositories() 205 | -------------------------------------------------------------------------------- /barn/BUILD.bazel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackb/buildkube/6ac7de40337b07ce61d378068088ac3fde978c32/barn/BUILD.bazel -------------------------------------------------------------------------------- /barn/Makefile: -------------------------------------------------------------------------------- 1 | install: 2 | bazel build \ 3 | //barn/k8s:k8s.apply \ 4 | //barn/redis/k8s:k8s.apply \ 5 | //barn/server/k8s:k8s.apply \ 6 | //barn/scheduler/k8s:k8s.apply \ 7 | //barn/worker/k8s:k8s.apply 8 | 9 | bazel run //barn/k8s:k8s.apply 10 | bazel run //barn/redis/k8s:k8s.apply 11 | bazel run //barn/server/k8s:k8s.apply 12 | bazel run //barn/scheduler/k8s:k8s.apply 13 | bazel run //barn/worker/k8s:k8s.apply 14 | 15 | uninstall: 16 | bazel run //barn/k8s:k8s.delete 17 | 18 | port-forward: 19 | kubectl -n barn port-forward \ 20 | `kubectl -n barn get pods --selector=k8s-app=server -o jsonpath='{.items[0].metadata.name}'` \ 21 | 8980 22 | 23 | -------------------------------------------------------------------------------- /barn/k8s/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_k8s//k8s:objects.bzl", "k8s_objects") 2 | load("@k8s_deploy//:defaults.bzl", "k8s_deploy") 3 | 4 | k8s_deploy( 5 | name = "namespace", 6 | template = "namespace.yaml", 7 | ) 8 | 9 | k8s_objects( 10 | name = "k8s", 11 | objects = [ 12 | ":namespace", 13 | ], 14 | ) 15 | -------------------------------------------------------------------------------- /barn/k8s/namespace.yaml: -------------------------------------------------------------------------------- 1 | kind: Namespace 2 | apiVersion: v1 3 | metadata: 4 | name: barn 5 | labels: 6 | name: barn -------------------------------------------------------------------------------- /barn/redis/k8s/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_k8s//k8s:objects.bzl", "k8s_objects") 2 | load("@k8s_deploy//:defaults.bzl", "k8s_deploy") 3 | 4 | k8s_deploy( 5 | name = "master", 6 | template = "master.yaml", 7 | ) 8 | 9 | k8s_deploy( 10 | name = "service", 11 | template = "service.yaml", 12 | ) 13 | 14 | k8s_deploy( 15 | name = "sentinel", 16 | template = "sentinel.yaml", 17 | ) 18 | 19 | k8s_objects( 20 | name = "k8s", 21 | objects = [ 22 | ":master", 23 | ":service", 24 | ":sentinel", 25 | ], 26 | visibility = ["//visibility:public"], 27 | ) 28 | -------------------------------------------------------------------------------- /barn/redis/k8s/master.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | labels: 5 | name: redis 6 | redis-sentinel: "true" 7 | role: master 8 | name: redis-master 9 | namespace: barn 10 | spec: 11 | containers: 12 | - name: master 13 | image: k8s.gcr.io/redis:v1 14 | env: 15 | - name: MASTER 16 | value: "true" 17 | ports: 18 | - containerPort: 6379 19 | resources: 20 | limits: 21 | cpu: "0.1" 22 | volumeMounts: 23 | - mountPath: /redis-master-data 24 | name: data 25 | - name: sentinel 26 | image: kubernetes/redis:v1 27 | env: 28 | - name: SENTINEL 29 | value: "true" 30 | ports: 31 | - containerPort: 26379 32 | volumes: 33 | - name: data 34 | emptyDir: {} 35 | --- 36 | apiVersion: v1 37 | kind: ReplicationController 38 | metadata: 39 | name: redis 40 | namespace: barn 41 | spec: 42 | replicas: 3 43 | selector: 44 | name: redis 45 | template: 46 | metadata: 47 | labels: 48 | name: redis 49 | role: master 50 | spec: 51 | containers: 52 | - name: redis 53 | image: k8s.gcr.io/redis:v1 54 | ports: 55 | - containerPort: 6379 56 | resources: 57 | limits: 58 | cpu: "0.1" 59 | volumeMounts: 60 | - mountPath: /redis-master-data 61 | name: data 62 | volumes: 63 | - name: data 64 | -------------------------------------------------------------------------------- /barn/redis/k8s/sentinel.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ReplicationController 3 | metadata: 4 | name: redis-sentinel 5 | namespace: barn 6 | spec: 7 | replicas: 3 8 | selector: 9 | redis-sentinel: "true" 10 | template: 11 | metadata: 12 | labels: 13 | name: redis-sentinel 14 | redis-sentinel: "true" 15 | role: sentinel 16 | spec: 17 | containers: 18 | - name: sentinel 19 | image: k8s.gcr.io/redis:v1 20 | env: 21 | - name: SENTINEL 22 | value: "true" 23 | ports: 24 | - containerPort: 26379 25 | --- 26 | apiVersion: v1 27 | kind: Service 28 | metadata: 29 | labels: 30 | name: sentinel 31 | role: service 32 | name: redis-sentinel 33 | namespace: barn 34 | spec: 35 | ports: 36 | - port: 26379 37 | targetPort: 26379 38 | selector: 39 | redis-sentinel: "true" -------------------------------------------------------------------------------- /barn/redis/k8s/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: redis 5 | namespace: barn 6 | spec: 7 | selector: 8 | name: redis 9 | role: master 10 | ports: 11 | - port: 6379 12 | targetPort: 6379 -------------------------------------------------------------------------------- /barn/scheduler/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | load("@io_bazel_rules_docker//go:image.bzl", "go_image") 3 | load("@io_bazel_rules_docker//contrib:test.bzl", "container_test") 4 | 5 | go_binary( 6 | name = "scheduler", 7 | goos = "linux", 8 | goarch = "amd64", 9 | pure = "on", 10 | embed = ["@buildbarn//cmd/bbb_scheduler:go_default_library"], 11 | ) 12 | 13 | go_image( 14 | name = "image", 15 | binary = ":scheduler", 16 | visibility = ["//barn/scheduler:__subpackages__"], 17 | ) 18 | 19 | container_test( 20 | name = "structure", 21 | image = ":image", 22 | configs = [ 23 | "structure-tests.yaml", 24 | ], 25 | size = "small", 26 | ) 27 | -------------------------------------------------------------------------------- /barn/scheduler/k8s/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_k8s//k8s:objects.bzl", "k8s_objects") 2 | load("@k8s_deploy//:defaults.bzl", "k8s_deploy") 3 | 4 | k8s_deploy( 5 | name = "service", 6 | template = "service.yaml", 7 | ) 8 | 9 | k8s_deploy( 10 | name = "deploy", 11 | images = { 12 | "gcr.io/stack-build/buildkube/barn/scheduler:latest": "//barn/scheduler:image", 13 | }, 14 | template = "deploy.yaml", 15 | ) 16 | 17 | k8s_objects( 18 | name = "k8s", 19 | objects = [ 20 | ":service", 21 | ":deploy", 22 | ], 23 | ) 24 | -------------------------------------------------------------------------------- /barn/scheduler/k8s/deploy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | namespace: barn 5 | name: ubuntu-scheduler 6 | spec: 7 | replicas: 1 8 | template: 9 | metadata: 10 | labels: 11 | k8s-app: ubuntu-scheduler 12 | spec: 13 | containers: 14 | - image: gcr.io/stack-build/buildkube/barn/scheduler:latest 15 | name: ubuntu-scheduler 16 | ports: 17 | - containerPort: 8981 18 | resources: 19 | limits: 20 | cpu: "1" 21 | memory: 512Mi 22 | requests: 23 | cpu: 250m 24 | memory: 128Mi -------------------------------------------------------------------------------- /barn/scheduler/k8s/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | namespace: barn 5 | name: ubuntu-scheduler 6 | spec: 7 | selector: 8 | k8s-app: ubuntu-scheduler 9 | ports: 10 | - port: 8981 11 | targetPort: 8981 -------------------------------------------------------------------------------- /barn/scheduler/structure-tests.yaml: -------------------------------------------------------------------------------- 1 | 2 | schemaVersion: '2.0.0' 3 | 4 | metadataTest: 5 | cmd: [] 6 | entrypoint: ["/app/barn/scheduler/scheduler"] 7 | 8 | fileExistenceTests: 9 | - name: "scheduler binary" 10 | path: "/app/barn/scheduler/scheduler.runfiles/com_github_stackb_buildkube/barn/scheduler/linux_amd64_pure_stripped/scheduler" 11 | shouldExist: true 12 | permissions: "-r-xr-xr-x" 13 | 14 | -------------------------------------------------------------------------------- /barn/server/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | load("@io_bazel_rules_docker//go:image.bzl", "go_image") 3 | load("@io_bazel_rules_docker//contrib:test.bzl", "container_test") 4 | 5 | go_binary( 6 | name = "server", 7 | goos = "linux", 8 | goarch = "amd64", 9 | pure = "on", 10 | embed = ["@buildbarn//cmd/bbb_frontend:go_default_library"], 11 | ) 12 | 13 | go_image( 14 | name = "image", 15 | binary = ":server", 16 | visibility = ["//barn/server:__subpackages__"], 17 | ) 18 | 19 | container_test( 20 | name = "structure", 21 | image = ":image", 22 | configs = [ 23 | "structure-tests.yaml", 24 | ], 25 | size = "small", 26 | ) 27 | -------------------------------------------------------------------------------- /barn/server/k8s/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_k8s//k8s:objects.bzl", "k8s_objects") 2 | load("@k8s_deploy//:defaults.bzl", "k8s_deploy") 3 | 4 | k8s_deploy( 5 | name = "service", 6 | template = "service.yaml", 7 | ) 8 | 9 | k8s_deploy( 10 | name = "configmap", 11 | template = "configmap.yaml", 12 | ) 13 | 14 | k8s_deploy( 15 | name = "deploy", 16 | images = { 17 | "gcr.io/stack-build/buildkube/barn/server:latest": "//barn/server:image", 18 | }, 19 | template = "deploy.yaml", 20 | ) 21 | 22 | k8s_objects( 23 | name = "k8s", 24 | objects = [ 25 | ":service", 26 | ":configmap", 27 | ":deploy", 28 | ], 29 | ) 30 | -------------------------------------------------------------------------------- /barn/server/k8s/configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: config 5 | namespace: barn 6 | data: 7 | blobstore.conf: | 8 | content_addressable_storage { 9 | redis { 10 | endpoint: "redis:6379" 11 | db: 0 12 | } 13 | } 14 | action_cache { 15 | redis { 16 | endpoint: "redis:6379" 17 | db: 1 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /barn/server/k8s/deploy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: server 5 | namespace: barn 6 | spec: 7 | replicas: 1 8 | template: 9 | metadata: 10 | labels: 11 | k8s-app: server 12 | spec: 13 | containers: 14 | - name: server 15 | image: gcr.io/stack-build/buildkube/barn/server:latest 16 | args: 17 | - -scheduler 18 | - main|ubuntu-scheduler:8981 19 | ports: 20 | - containerPort: 8980 21 | resources: 22 | limits: 23 | cpu: "1" 24 | memory: 2Gi 25 | requests: 26 | cpu: 250m 27 | memory: 512Mi 28 | volumeMounts: 29 | - mountPath: /config 30 | name: config 31 | volumes: 32 | - configMap: 33 | defaultMode: 400 34 | name: config 35 | name: config -------------------------------------------------------------------------------- /barn/server/k8s/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: server 5 | namespace: barn 6 | spec: 7 | selector: 8 | k8s-app: server 9 | ports: 10 | - port: 8980 11 | targetPort: 8980 -------------------------------------------------------------------------------- /barn/server/structure-tests.yaml: -------------------------------------------------------------------------------- 1 | 2 | schemaVersion: '2.0.0' 3 | 4 | metadataTest: 5 | entrypoint: ["/app/barn/server/server"] 6 | 7 | fileExistenceTests: 8 | - name: "server binary" 9 | path: "/app/barn/server/server.runfiles/com_github_stackb_buildkube/barn/server/linux_amd64_pure_stripped/server" 10 | shouldExist: true 11 | permissions: "-r-xr-xr-x" 12 | 13 | -------------------------------------------------------------------------------- /barn/worker/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | load("@io_bazel_rules_docker//go:image.bzl", "go_image") 3 | load("@io_bazel_rules_docker//contrib:test.bzl", "container_test") 4 | 5 | go_binary( 6 | name = "worker", 7 | goos = "linux", 8 | goarch = "amd64", 9 | pure = "on", 10 | embed = ["@buildbarn//cmd/bbb_worker:go_default_library"], 11 | ) 12 | 13 | go_image( 14 | name = "image", 15 | base = "@rbe_ubuntu//image", 16 | binary = ":worker", 17 | visibility = ["//barn/worker:__subpackages__"], 18 | ) 19 | 20 | container_test( 21 | name = "structure", 22 | image = ":image", 23 | configs = [ 24 | "structure-tests.yaml", 25 | ], 26 | size = "small", 27 | ) 28 | -------------------------------------------------------------------------------- /barn/worker/k8s/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_k8s//k8s:objects.bzl", "k8s_objects") 2 | load("@k8s_deploy//:defaults.bzl", "k8s_deploy") 3 | 4 | k8s_deploy( 5 | name = "deploy", 6 | images = { 7 | "gcr.io/stack-build/buildkube/barn/ubuntu-worker:latest": "//barn/worker:image", 8 | }, 9 | template = "deploy.yaml", 10 | ) 11 | 12 | k8s_objects( 13 | name = "k8s", 14 | objects = [ 15 | ":deploy", 16 | ], 17 | ) 18 | -------------------------------------------------------------------------------- /barn/worker/k8s/deploy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: ubuntu-worker 5 | namespace: barn 6 | spec: 7 | replicas: 16 8 | template: 9 | metadata: 10 | labels: 11 | k8s-app: ubuntu-worker 12 | spec: 13 | containers: 14 | - name: ubuntu-worker 15 | image: gcr.io/stack-build/buildkube/barn/ubuntu-worker:latest 16 | args: 17 | - -scheduler 18 | - ubuntu-scheduler:8981 19 | volumeMounts: 20 | - mountPath: /config 21 | name: config 22 | volumes: 23 | - configMap: 24 | defaultMode: 400 25 | name: config 26 | name: config -------------------------------------------------------------------------------- /barn/worker/structure-tests.yaml: -------------------------------------------------------------------------------- 1 | 2 | schemaVersion: '2.0.0' 3 | 4 | metadataTest: 5 | entrypoint: ["/app/barn/worker/worker"] 6 | 7 | fileExistenceTests: 8 | - name: "worker binary" 9 | path: "/app/barn/worker/worker.runfiles/com_github_stackb_buildkube/barn/worker/linux_amd64_pure_stripped/worker" 10 | shouldExist: true 11 | permissions: "-r-xr-xr-x" 12 | 13 | -------------------------------------------------------------------------------- /barn/workspace.bzl: -------------------------------------------------------------------------------- 1 | load("@bazel_gazelle//:deps.bzl", "go_repository") 2 | 3 | def buildbarn_repositories(): 4 | go_repository( 5 | name = "com_github_aws_aws_sdk_go", 6 | importpath = "github.com/aws/aws-sdk-go", 7 | sha256 = "b6cd9c78df8aeb973f8d9b01d11c1d1e5096850614b3a3e0b4111ec747d811d3", 8 | strip_prefix = "aws-sdk-go-bc3f534c19ffdf835e524e11f0f825b3eaf541c3", 9 | urls = ["https://github.com/aws/aws-sdk-go/archive/bc3f534c19ffdf835e524e11f0f825b3eaf541c3.tar.gz"], 10 | ) 11 | 12 | go_repository( 13 | name = "com_github_beorn7_perks", 14 | commit = "3a771d992973f24aa725d07868b467d1ddfceafb", 15 | importpath = "github.com/beorn7/perks", 16 | ) 17 | 18 | go_repository( 19 | name = "com_github_go_ini_ini", 20 | commit = "358ee7663966325963d4e8b2e1fbd570c5195153", 21 | importpath = "github.com/go-ini/ini", 22 | ) 23 | 24 | go_repository( 25 | name = "com_github_golang_protobuf", 26 | commit = "b4deda0973fb4c70b50d226b1af49f3da59f5265", 27 | importpath = "github.com/golang/protobuf", 28 | ) 29 | 30 | go_repository( 31 | name = "com_github_jmespath_go_jmespath", 32 | commit = "0b12d6b5", 33 | importpath = "github.com/jmespath/go-jmespath", 34 | ) 35 | 36 | go_repository( 37 | name = "com_github_matttproud_golang_protobuf_extensions", 38 | commit = "c12348ce28de40eed0136aa2b644d0ee0650e56c", 39 | importpath = "github.com/matttproud/golang_protobuf_extensions", 40 | ) 41 | 42 | go_repository( 43 | name = "com_github_prometheus_client_golang", 44 | commit = "c5b7fccd204277076155f10851dad72b76a49317", 45 | importpath = "github.com/prometheus/client_golang", 46 | ) 47 | 48 | go_repository( 49 | name = "com_github_prometheus_client_model", 50 | commit = "5c3871d89910bfb32f5fcab2aa4b9ec68e65a99f", 51 | importpath = "github.com/prometheus/client_model", 52 | ) 53 | 54 | go_repository( 55 | name = "com_github_prometheus_common", 56 | commit = "7600349dcfe1abd18d72d3a1770870d9800a7801", 57 | importpath = "github.com/prometheus/common", 58 | ) 59 | 60 | go_repository( 61 | name = "com_github_prometheus_procfs", 62 | commit = "ae68e2d4c00fed4943b5f6698d504a5fe083da8a", 63 | importpath = "github.com/prometheus/procfs", 64 | ) 65 | 66 | go_repository( 67 | name = "com_github_satori_go_uuid", 68 | commit = "f58768cc1a7a7e77a3bd49e98cdd21419399b6a3", 69 | importpath = "github.com/satori/go.uuid", 70 | ) 71 | 72 | go_repository( 73 | name = "org_golang_google_genproto", 74 | commit = "e92b116572682a5b432ddd840aeaba2a559eeff1", 75 | importpath = "google.golang.org/genproto", 76 | ) 77 | 78 | go_repository( 79 | name = "org_golang_google_grpc", 80 | commit = "168a6198bcb0ef175f7dacec0b8691fc141dc9b8", 81 | importpath = "google.golang.org/grpc", 82 | ) 83 | 84 | go_repository( 85 | name = "org_golang_x_net", 86 | commit = "039a4258aec0ad3c79b905677cceeab13b296a77", 87 | importpath = "golang.org/x/net", 88 | ) 89 | 90 | go_repository( 91 | name = "org_golang_x_text", 92 | commit = "f21a4dfb5e38f5895301dc265a8def02365cc3d0", 93 | importpath = "golang.org/x/text", 94 | ) 95 | 96 | go_repository( 97 | name = "com_github_grpc_ecosystem_go_grpc_prometheus", 98 | commit = "c225b8c3b01faf2899099b768856a9e916e5087b", 99 | importpath = "github.com/grpc-ecosystem/go-grpc-prometheus", 100 | ) 101 | 102 | go_repository( 103 | name = "com_github_go_redis_redis", 104 | commit = "480db94d33e6088e08d628833b6c0705451d24bb", 105 | importpath = "github.com/go-redis/redis", 106 | ) 107 | 108 | go_repository( 109 | name = "com_github_bazelbuild_remote_apis", 110 | commit = "6130f7e23ae157d5cf12c5d6af325a1dae57e235", 111 | importpath = "github.com/bazelbuild/remote-apis", 112 | ) 113 | -------------------------------------------------------------------------------- /bazelrc: -------------------------------------------------------------------------------- 1 | build:remote --jobs=300 2 | 3 | build:remote --host_javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:jdk8 4 | build:remote --javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:jdk8 5 | build:remote --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 6 | build:remote --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 7 | build:remote --crosstool_top=@bazel_toolchains//configs/ubuntu16_04_clang/1.1/bazel_0.17.1/default:toolchain 8 | build:remote --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 9 | 10 | build:remote --extra_toolchains=@bazel_toolchains//configs/ubuntu16_04_clang/1.1/bazel_0.17.1/cpp:cc-toolchain-clang-x86_64-default 11 | build:remote --extra_execution_platforms=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:rbe_ubuntu1604 12 | build:remote --host_platform=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:rbe_ubuntu1604 13 | build:remote --platforms=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:rbe_ubuntu1604 14 | 15 | build:remote --spawn_strategy=remote 16 | build:remote --strategy=Javac=remote 17 | build:remote --strategy=Closure=remote 18 | build:remote --genrule_strategy=remote 19 | build:remote --define=EXECUTOR=remote 20 | 21 | build:remote --remote_cache=localhost:8980 22 | build:remote --remote_executor=localhost:8980 23 | build:remote --experimental_strict_action_env=true 24 | build:remote --remote_timeout=3600 -------------------------------------------------------------------------------- /farm/BUILD.bazel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stackb/buildkube/6ac7de40337b07ce61d378068088ac3fde978c32/farm/BUILD.bazel -------------------------------------------------------------------------------- /farm/Makefile: -------------------------------------------------------------------------------- 1 | install: 2 | bazel build \ 3 | //farm/k8s:k8s.apply \ 4 | //farm/server/k8s:k8s.apply \ 5 | //farm/worker/k8s:k8s.apply 6 | bazel run //farm/k8s:k8s.apply 7 | bazel run //farm/server/k8s:k8s.apply 8 | bazel run //farm/worker/k8s:k8s.apply 9 | 10 | uninstall: 11 | bazel run //farm/k8s:k8s.delete 12 | 13 | port-forward: 14 | kubectl -n farm port-forward \ 15 | `kubectl -n farm get pods --selector=k8s-app=server -o jsonpath='{.items[0].metadata.name}'` \ 16 | 8980 -------------------------------------------------------------------------------- /farm/k8s/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_k8s//k8s:objects.bzl", "k8s_objects") 2 | load("@k8s_deploy//:defaults.bzl", "k8s_deploy") 3 | 4 | k8s_deploy( 5 | name = "namespace", 6 | template = "namespace.yaml", 7 | ) 8 | 9 | k8s_objects( 10 | name = "k8s", 11 | objects = [ 12 | ":namespace", 13 | ], 14 | ) 15 | -------------------------------------------------------------------------------- /farm/k8s/namespace.yaml: -------------------------------------------------------------------------------- 1 | kind: Namespace 2 | apiVersion: v1 3 | metadata: 4 | name: farm 5 | labels: 6 | name: farm -------------------------------------------------------------------------------- /farm/local_repository.bzl: -------------------------------------------------------------------------------- 1 | 2 | def local_buildfarm_repository(path): 3 | 4 | native.local_repository( 5 | name = "build_buildfarm", 6 | path = path, 7 | ) 8 | 9 | native.new_http_archive( 10 | name = "googleapis", 11 | sha256 = "7b6ea252f0b8fb5cd722f45feb83e115b689909bbb6a393a873b6cbad4ceae1d", 12 | url = "https://github.com/googleapis/googleapis/archive/143084a2624b6591ee1f9d23e7f5241856642f4d.zip", 13 | strip_prefix = "googleapis-143084a2624b6591ee1f9d23e7f5241856642f4d", 14 | build_file = "@build_buildfarm//:BUILD.googleapis", 15 | ) 16 | 17 | # The API that we implement. 18 | native.new_http_archive( 19 | name = "remote_apis", 20 | sha256 = "865c6950a64b859cf211761330e5d13e6c4b54e22a454ae1195238594299de34", 21 | url = "https://github.com/bazelbuild/remote-apis/archive/fdeb922b595df28650d12fc2335c4426df2fc726.zip", 22 | strip_prefix = "remote-apis-fdeb922b595df28650d12fc2335c4426df2fc726", 23 | build_file = "@build_buildfarm//:BUILD.remote_apis", 24 | ) 25 | 26 | native.http_archive( 27 | name = "grpc_java", 28 | sha256 = "20a35772b20d8194854f6d149324f971472b7acc1a76a0969a048c4c02a1da0d", 29 | strip_prefix = "grpc-java-1.8.0", 30 | urls = ["https://github.com/grpc/grpc-java/archive/v1.8.0.zip"], 31 | ) 32 | -------------------------------------------------------------------------------- /farm/server/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_docker//container:image.bzl", "container_image") 2 | load("@io_bazel_rules_docker//contrib:test.bzl", "container_test") 3 | 4 | container_image( 5 | name = "image", 6 | base = "@java_image_base//image", 7 | cmd = [ 8 | "/buildfarm-server_deploy.jar", 9 | "/server.config", 10 | ], 11 | files = [ 12 | "server.config", 13 | "@build_buildfarm//:server", 14 | 15 | # Use this one in conjunction with local_repository.bzl 16 | #"@build_buildfarm//src/main/java/build/buildfarm:buildfarm-server_deploy.jar", 17 | ], 18 | ports = [ 19 | "8980", 20 | ], 21 | visibility = ["//farm/server:__subpackages__"], 22 | ) 23 | 24 | container_test( 25 | name = "structure", 26 | image = ":image", 27 | configs = [ 28 | "structure-tests.yaml", 29 | ], 30 | size = "small", 31 | ) 32 | -------------------------------------------------------------------------------- /farm/server/k8s/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_k8s//k8s:objects.bzl", "k8s_objects") 2 | load("@k8s_deploy//:defaults.bzl", "k8s_deploy") 3 | 4 | k8s_deploy( 5 | name = "service", 6 | template = "service.yaml", 7 | ) 8 | 9 | k8s_deploy( 10 | name = "deploy", 11 | images = { 12 | "gcr.io/stack-build/buildkube/farm/server:latest": "//farm/server:image", 13 | }, 14 | template = "deploy.yaml", 15 | ) 16 | 17 | k8s_objects( 18 | name = "k8s", 19 | objects = [ 20 | ":service", 21 | ":deploy", 22 | ], 23 | ) -------------------------------------------------------------------------------- /farm/server/k8s/deploy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: server 5 | namespace: farm 6 | spec: 7 | replicas: 1 8 | template: 9 | metadata: 10 | labels: 11 | k8s-app: server 12 | spec: 13 | containers: 14 | - name: server 15 | image: gcr.io/stack-build/buildkube/farm/server:latest 16 | ports: 17 | - containerPort: 8980 -------------------------------------------------------------------------------- /farm/server/k8s/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: operation-queue 5 | namespace: farm 6 | spec: 7 | selector: 8 | k8s-app: server 9 | ports: 10 | - port: 8980 11 | targetPort: 8980 12 | --- 13 | apiVersion: v1 14 | kind: Service 15 | metadata: 16 | name: cas 17 | namespace: farm 18 | spec: 19 | selector: 20 | k8s-app: server 21 | ports: 22 | - port: 8980 23 | targetPort: 8980 24 | --- 25 | apiVersion: v1 26 | kind: Service 27 | metadata: 28 | name: action-cache 29 | namespace: farm 30 | spec: 31 | selector: 32 | k8s-app: server 33 | ports: 34 | - port: 8980 35 | targetPort: 8980 -------------------------------------------------------------------------------- /farm/server/server.config: -------------------------------------------------------------------------------- 1 | # an instance specification 2 | instances { 3 | name: "main" 4 | 5 | # the digest function for this instance, required 6 | # to match out of band between the client and server 7 | # since resource names must be determined on the client 8 | # for a valid upload 9 | digest_function: SHA256 10 | 11 | # the implicit type specifier for this instance 12 | # a memory instance is volatile and has no persistent 13 | # footprint. all bulk storage is in terms of the CAS 14 | # with proxy maps for action cache/operations 15 | memory_instance_config: { 16 | # Operations#listOperations request limits 17 | list_operations_default_page_size: 1024 18 | list_operations_max_page_size: 16384 19 | 20 | # ContentAddressableStorage#getTree request limits 21 | tree_default_page_size: 1024 22 | tree_max_page_size: 16384 23 | 24 | # the maximum time after dispatch of an operation until 25 | # the worker must poll to indicate continued work, after 26 | # which the operation will be requeued 27 | operation_poll_timeout: { 28 | seconds: 30 29 | nanos: 0 30 | } 31 | 32 | # the delay after an action timeout before an action is 33 | # automatically considered to have failed with no results 34 | # and a timeout exceeded failure condition 35 | operation_completed_delay: { 36 | seconds: 10 37 | nanos: 0 38 | } 39 | 40 | cas_config: { 41 | memory: { 42 | # limit for CAS total content size in bytes 43 | max_size_bytes: 1073741824 # 1024 * 1024 * 1024 44 | } 45 | } 46 | 47 | action_cache_config: { 48 | # retain a local map of actionKeys and retain actions in CAS 49 | delegate_cas: {} 50 | } 51 | 52 | # an imposed action-key-invariant timeout used in the unspecified timeout case 53 | default_action_timeout: { 54 | seconds: 600 55 | nanos: 0 56 | } 57 | 58 | # a limit on the action timeout specified in the action, above which 59 | # the operation will report a failed result immediately 60 | maximum_action_timeout: { 61 | seconds: 3600 62 | nanos: 0 63 | } 64 | } 65 | } 66 | 67 | # the listening port of the buildfarm grpc server 68 | port: 8980 69 | 70 | # the instance to which all requests with an empty/missing 71 | # instance name are routed 72 | # 73 | # this can be empty as well, to indicate that there is no 74 | # default instance 75 | default_instance_name: "main" 76 | -------------------------------------------------------------------------------- /farm/server/structure-tests.yaml: -------------------------------------------------------------------------------- 1 | 2 | schemaVersion: '2.0.0' 3 | 4 | # 5 | # Structure tests specific to server 6 | # 7 | 8 | metadataTest: 9 | cmd: ["/buildfarm-server_deploy.jar", "/server.config"] 10 | entrypoint: ["/usr/bin/java", "-jar"] 11 | 12 | fileExistenceTests: 13 | - name: "server config file" 14 | path: "/server.config" 15 | shouldExist: true 16 | permissions: "-r-xr-xr-x" 17 | 18 | - name: "server jar" 19 | path: "/buildfarm-server_deploy.jar" 20 | shouldExist: true 21 | permissions: "-r-xr-xr-x" 22 | 23 | - name: "server config" 24 | path: "/server.config" 25 | shouldExist: true 26 | permissions: "-r-xr-xr-x" -------------------------------------------------------------------------------- /farm/worker/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_docker//container:image.bzl", "container_image") 2 | load("@io_bazel_rules_docker//contrib:test.bzl", "container_test") 3 | 4 | container_image( 5 | name = "image", 6 | base = "@rbe_ubuntu//image", 7 | entrypoint = ["/usr/bin/java", "-jar"], 8 | cmd = [ 9 | "/buildfarm-worker_deploy.jar", 10 | "/worker.config", 11 | ], 12 | files = [ 13 | "worker.config", 14 | "@build_buildfarm//:worker", 15 | 16 | # Use this one with local_repository.bzl 17 | #"@build_buildfarm//src/main/java/build/buildfarm:buildfarm-worker_deploy.jar", 18 | ], 19 | visibility = ["//farm/worker:__subpackages__"], 20 | ) 21 | 22 | container_test( 23 | name = "structure", 24 | image = ":image", 25 | configs = [ 26 | "structure-tests.yaml", 27 | ], 28 | size = "small", 29 | ) 30 | -------------------------------------------------------------------------------- /farm/worker/k8s/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_k8s//k8s:objects.bzl", "k8s_objects") 2 | load("@k8s_deploy//:defaults.bzl", "k8s_deploy") 3 | 4 | k8s_deploy( 5 | name = "deploy", 6 | images = { 7 | "gcr.io/stack-build/buildkube/farm/worker:latest": "//farm/worker:image", 8 | }, 9 | template = "deploy.yaml", 10 | ) 11 | 12 | k8s_objects( 13 | name = "k8s", 14 | objects = [ 15 | ":deploy", 16 | ], 17 | ) 18 | -------------------------------------------------------------------------------- /farm/worker/k8s/deploy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: worker 5 | namespace: farm 6 | spec: 7 | replicas: 16 8 | template: 9 | metadata: 10 | labels: 11 | k8s-app: worker 12 | spec: 13 | containers: 14 | - name: worker 15 | image: gcr.io/stack-build/buildkube/farm/worker:latest -------------------------------------------------------------------------------- /farm/worker/structure-tests.yaml: -------------------------------------------------------------------------------- 1 | 2 | schemaVersion: '2.0.0' 3 | 4 | # 5 | # Structure tests specific to worker 6 | # 7 | 8 | metadataTest: 9 | cmd: ["/buildfarm-worker_deploy.jar", "/worker.config"] 10 | entrypoint: ["/usr/bin/java", "-jar"] 11 | 12 | fileExistenceTests: 13 | - name: "worker config file" 14 | path: "/worker.config" 15 | shouldExist: true 16 | permissions: "-r-xr-xr-x" 17 | 18 | - name: "worker jar" 19 | path: "/buildfarm-worker_deploy.jar" 20 | shouldExist: true 21 | permissions: "-r-xr-xr-x" 22 | -------------------------------------------------------------------------------- /farm/worker/worker.config: -------------------------------------------------------------------------------- 1 | digest_function: SHA256 2 | 3 | # the endpoint used to execute operations 4 | operation_queue: { 5 | target: "operation-queue:8980" 6 | 7 | # the instance domain that this worker will execute work in 8 | # all requests will be tagged with this instance name 9 | instance_name: "main" 10 | } 11 | 12 | # the endpoint used for cas interactions 13 | content_addressable_storage: { 14 | target: "cas:8980" 15 | 16 | # the instance domain that this worker will make resource requests in 17 | # all requests will be tagged with this instance name 18 | instance_name: "main" 19 | } 20 | 21 | # the endpoint used for action cache interactions 22 | action_cache: { 23 | target: "action-cache:8980" 24 | 25 | # the instance domain that this worker will make resource requests in 26 | # all requests will be tagged with this instance name 27 | instance_name: "main" 28 | } 29 | 30 | # all content for the operations will be stored under this path 31 | root: "/tmp/worker" 32 | 33 | # the local cache location relative to the 'root', or absolute 34 | cas_cache_directory: "cache" 35 | 36 | # total size in bytes of inline content for action results 37 | # output files, stdout, and stderr content, in that order 38 | # will be inlined if their cumulative size does not exceed this limit. 39 | inline_content_limit: 1048567 # 1024 * 1024 40 | 41 | # whether the stdout of running processes should be streamed 42 | stream_stdout: true 43 | 44 | # whether to insert stdout into the CAS, can be: 45 | # ALWAYS_INSERT: stdout is always inserted into the CAS 46 | # INSERT_ABOVE_LIMIT: stdout is inserted into the CAS when it exceeds the inline limit above 47 | stdout_cas_policy: ALWAYS_INSERT 48 | 49 | # whether the stderr of running processes should be streamed 50 | stream_stderr: true 51 | 52 | # whether to insert stderr into the CAS, can be: 53 | # ALWAYS_INSERT: stderr is always inserted into the CAS 54 | # INSERT_ABOVE_LIMIT: stderr is inserted into the CAS when it exceeds the inline limit above 55 | stderr_cas_policy: ALWAYS_INSERT 56 | 57 | # whether to insert output files into the CAS, can be: 58 | # ALWAYS_INSERT: output files are always inserted into the CAS 59 | # INSERT_ABOVE_LIMIT: output files are inserted into the CAS when it exceeds the inline limit above 60 | file_cas_policy: ALWAYS_INSERT 61 | 62 | # the worker will take it upon itself to requeue (exceptionally) 63 | # failed operations via the OperationQueue#put method with queued 64 | # status. 65 | requeue_on_failure: true 66 | 67 | # ContentAddressableStorage#getTree per-page directory count 68 | # value of '0' means let the server decide 69 | tree_page_size: 0 70 | 71 | # the period between poll operations at any stage 72 | operation_poll_period: { 73 | seconds: 1 74 | nanos: 0 75 | } 76 | 77 | # key/value set of definining capabilities of this worker 78 | # all execute requests must match perfectly with workers which 79 | # provide capabilities 80 | # so an action with a required platform: { arch: "x86_64" } must 81 | # match with a worker with at least { arch: "x86_64" } here 82 | platform: { 83 | # commented out here for illustrative purposes, a default empty 84 | # 'platform' is a sufficient starting point without specifying 85 | # any platform requirements on the actions' side 86 | properties: { 87 | name: "container-image" 88 | value: "docker://gcr.io/cloud-marketplace/google/rbe-ubuntu16-04@sha256:9bd8ba020af33edb5f11eff0af2f63b3bcb168cd6566d7b27c6685e717787928" 89 | } 90 | } 91 | 92 | # limit for contents of files retained 93 | # from CAS in the cache 94 | cas_cache_max_size_bytes: 2147483648 # 2 * 1024 * 1024 * 1024 95 | 96 | # the number of concurrently available slots in the execute phase 97 | execute_stage_width: 1 98 | 99 | # an imposed action-key-invariant timeout used in the unspecified timeout case 100 | default_action_timeout: { 101 | seconds: 600 102 | nanos: 0 103 | } 104 | 105 | # a limit on the action timeout specified in the action, above which 106 | # the operation will report a failed result immediately 107 | maximum_action_timeout: { 108 | seconds: 3600 109 | nanos: 0 110 | } 111 | 112 | # prefix command executions with this path 113 | #execution_policies: { 114 | # name: "test" 115 | # wrapper: { 116 | # path: "/path/to/execution/wrapper" 117 | # } 118 | #} 119 | -------------------------------------------------------------------------------- /farm/workspace.bzl: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_docker//container:load.bzl", "container_load") 2 | 3 | BUILD_BAZEL = """ 4 | java_import( 5 | name = "server", 6 | jars = ["buildfarm-server_deploy.jar"], 7 | visibility = ["//visibility:public"], 8 | ) 9 | java_import( 10 | name = "worker", 11 | jars = ["buildfarm-worker_deploy.jar"], 12 | visibility = ["//visibility:public"], 13 | ) 14 | """ 15 | 16 | def modify(repository_ctx, filename, directive): 17 | args = ["sed", "-i", directive, filename] 18 | result = repository_ctx.execute(args) 19 | if result.return_code: 20 | fail("%r failed: %s" % (args, result.stderr)) 21 | 22 | 23 | def _buildfarm_repository_impl(repository_ctx): 24 | commit = repository_ctx.attr.commit 25 | 26 | url = repository_ctx.attr.remote.format( 27 | commit = commit, 28 | ) 29 | 30 | # Download and unarchive it! 31 | repository_ctx.download_and_extract(url, 32 | stripPrefix = "-".join(["bazel-buildfarm", commit]), 33 | ) 34 | 35 | # Apply mods if requested 36 | for filename, commands in repository_ctx.attr.modifications.items(): 37 | for command in commands: 38 | modify(repository_ctx, filename, command) 39 | 40 | result = repository_ctx.execute(["bazel", "build", 41 | "//src/main/java/build/buildfarm:buildfarm-server_deploy.jar", 42 | "//src/main/java/build/buildfarm:buildfarm-worker_deploy.jar", 43 | ], quiet = False) 44 | if result.return_code: 45 | fail("bazel build failed: %s" % result.stderr) 46 | 47 | result = repository_ctx.execute(["cp", "bazel-bin/src/main/java/build/buildfarm/buildfarm-server_deploy.jar", "."]) 48 | if result.return_code: 49 | fail("copy failed: %s" % result.stderr) 50 | result = repository_ctx.execute(["cp", "bazel-bin/src/main/java/build/buildfarm/buildfarm-worker_deploy.jar", "."]) 51 | if result.return_code: 52 | fail("copy failed: %s" % result.stderr) 53 | 54 | repository_ctx.file("BUILD.bazel", BUILD_BAZEL) 55 | 56 | 57 | buildfarm_repository = repository_rule( 58 | implementation = _buildfarm_repository_impl, 59 | attrs = { 60 | "remote": attr.string( 61 | default = "https://github.com/bazelbuild/bazel-buildfarm/archive/{commit}.tar.gz", 62 | ), 63 | "commit": attr.string( 64 | mandatory = True, 65 | ), 66 | "modifications": attr.string_list_dict( 67 | doc = "Optional sed modifications to apply", 68 | ), 69 | } 70 | ) 71 | -------------------------------------------------------------------------------- /grid/BUILD.bazel: -------------------------------------------------------------------------------- 1 | filegroup( 2 | name = "structure-tests", 3 | srcs = ["structure-tests.yaml"], 4 | visibility = ["//grid:__subpackages__"], 5 | ) -------------------------------------------------------------------------------- /grid/Makefile: -------------------------------------------------------------------------------- 1 | install: 2 | bazel build \ 3 | //grid/k8s:k8s.apply \ 4 | //grid/server/k8s:k8s.apply \ 5 | //grid/worker/k8s:k8s.apply 6 | 7 | bazel run //grid/k8s:k8s.apply 8 | bazel run //grid/server/k8s:k8s.apply 9 | bazel run //grid/worker/k8s:k8s.apply 10 | 11 | uninstall: 12 | bazel run //grid/k8s:k8s.delete 13 | 14 | port-forward: 15 | kubectl -n grid port-forward \ 16 | `kubectl -n grid get pods --selector=k8s-app=server -o jsonpath='{.items[0].metadata.name}'` \ 17 | 8980 18 | -------------------------------------------------------------------------------- /grid/k8s/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_k8s//k8s:objects.bzl", "k8s_objects") 2 | load("@k8s_deploy//:defaults.bzl", "k8s_deploy") 3 | 4 | k8s_deploy( 5 | name = "namespace", 6 | template = "namespace.yaml", 7 | ) 8 | 9 | k8s_objects( 10 | name = "k8s", 11 | objects = [ 12 | ":namespace", 13 | ], 14 | ) 15 | -------------------------------------------------------------------------------- /grid/k8s/namespace.yaml: -------------------------------------------------------------------------------- 1 | kind: Namespace 2 | apiVersion: v1 3 | metadata: 4 | name: grid 5 | labels: 6 | name: grid -------------------------------------------------------------------------------- /grid/server/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_docker//container:image.bzl", "container_image") 2 | load("@io_bazel_rules_docker//contrib:test.bzl", "container_test") 3 | 4 | container_image( 5 | name = "image", 6 | base = "@buildgrid_server//image", 7 | entrypoint = ["/root/.local/bin/bgd"], 8 | cmd = ["server", "start", "/server.conf"], 9 | files = ["server.conf"], 10 | ports = ["8980"], 11 | workdir = "/app", 12 | visibility = ["//grid/server:__subpackages__"], 13 | ) 14 | 15 | container_test( 16 | name = "structure", 17 | image = ":image", 18 | configs = [ 19 | "//grid:structure-tests", 20 | "structure-tests.yaml", 21 | ], 22 | size = "small", 23 | ) 24 | -------------------------------------------------------------------------------- /grid/server/k8s/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_k8s//k8s:objects.bzl", "k8s_objects") 2 | load("@k8s_deploy//:defaults.bzl", "k8s_deploy") 3 | 4 | k8s_deploy( 5 | name = "service", 6 | template = "service.yaml", 7 | ) 8 | 9 | k8s_deploy( 10 | name = "deploy", 11 | images = { 12 | "gcr.io/stack-build/buildkube/grid/server:latest": "//grid/server:image", 13 | }, 14 | template = "deploy.yaml", 15 | ) 16 | 17 | k8s_objects( 18 | name = "k8s", 19 | objects = [ 20 | ":service", 21 | ":deploy", 22 | ], 23 | ) 24 | -------------------------------------------------------------------------------- /grid/server/k8s/deploy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: server 5 | namespace: grid 6 | spec: 7 | replicas: 1 8 | template: 9 | metadata: 10 | labels: 11 | k8s-app: server 12 | spec: 13 | containers: 14 | - name: server 15 | image: gcr.io/stack-build/buildkube/grid/server:latest 16 | ports: 17 | - containerPort: 8980 -------------------------------------------------------------------------------- /grid/server/k8s/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: server 5 | namespace: grid 6 | spec: 7 | selector: 8 | k8s-app: server 9 | ports: 10 | - port: 8980 11 | targetPort: 8980 12 | -------------------------------------------------------------------------------- /grid/server/server.conf: -------------------------------------------------------------------------------- 1 | server: 2 | - !channel 3 | port: 8980 4 | insecure_mode: true 5 | 6 | instances: 7 | - name: main 8 | 9 | storages: 10 | - !lru-storage &main-storage 11 | size: 512MB 12 | 13 | services: 14 | - !action-cache &main-action 15 | storage: *main-storage 16 | max_cached_refs: 256 17 | allow_updates: true 18 | - !execution 19 | storage: *main-storage 20 | action_cache: *main-action 21 | - !cas 22 | storage: *main-storage 23 | - !bytestream 24 | storage: *main-storage 25 | -------------------------------------------------------------------------------- /grid/server/structure-tests.yaml: -------------------------------------------------------------------------------- 1 | schemaVersion: '2.0.0' 2 | 3 | # 4 | # Structure tests specific to server 5 | # 6 | 7 | metadataTest: 8 | cmd: ["server", "start", "/server.conf"] 9 | exposedPorts: ["8980"] 10 | entrypoint: ["/root/.local/bin/bgd"] 11 | 12 | fileExistenceTests: 13 | - name: "server config file" 14 | path: "/server.conf" 15 | shouldExist: true 16 | permissions: "-r-xr-xr-x" 17 | 18 | -------------------------------------------------------------------------------- /grid/structure-tests.yaml: -------------------------------------------------------------------------------- 1 | schemaVersion: '2.0.0' 2 | 3 | # 4 | # Structure tests common to both server/worker 5 | # 6 | 7 | metadataTest: 8 | entrypoint: ["/root/.local/bin/bgd"] 9 | workdir: "" 10 | 11 | fileExistenceTests: 12 | - name: "bgd tool" 13 | path: "/root/.local/bin/bgd" 14 | shouldExist: true 15 | permissions: "-rwxr-xr-x" 16 | 17 | commandTests: 18 | - name: "list operations" 19 | command: "/root/.local/bin/bgd" 20 | expectedOutput: [".*Start a local server instance.*"] 21 | -------------------------------------------------------------------------------- /grid/worker/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_docker//container:image.bzl", "container_image") 2 | load("@io_bazel_rules_docker//contrib:test.bzl", "container_test") 3 | 4 | container_image( 5 | name = "image", 6 | base = "@buildgrid_worker//image", 7 | entrypoint = ["/root/.local/bin/bgd"], 8 | workdir = "/app", 9 | visibility = ["//grid/worker:__subpackages__"], 10 | ) 11 | 12 | container_test( 13 | name = "structure", 14 | image = ":image", 15 | configs = [ 16 | "//grid:structure-tests", 17 | "structure-tests.yaml", 18 | ], 19 | size = "small", 20 | ) 21 | -------------------------------------------------------------------------------- /grid/worker/k8s/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_k8s//k8s:objects.bzl", "k8s_objects") 2 | load("@k8s_deploy//:defaults.bzl", "k8s_deploy") 3 | 4 | k8s_deploy( 5 | name = "deploy", 6 | images = { 7 | "gcr.io/stack-build/buildkube/grid/worker:latest": "//grid/worker:image", 8 | }, 9 | template = "deploy.yaml", 10 | ) 11 | 12 | k8s_objects( 13 | name = "k8s", 14 | objects = [ 15 | ":deploy", 16 | ], 17 | ) 18 | -------------------------------------------------------------------------------- /grid/worker/k8s/deploy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: worker 5 | namespace: grid 6 | spec: 7 | replicas: 16 8 | template: 9 | metadata: 10 | labels: 11 | k8s-app: worker 12 | spec: 13 | containers: 14 | - name: worker 15 | image: gcr.io/stack-build/buildkube/grid/worker:latest 16 | args: 17 | - bot 18 | - --remote=http://server:8980 19 | - --parent=main 20 | - host-tools 21 | -------------------------------------------------------------------------------- /grid/worker/structure-tests.yaml: -------------------------------------------------------------------------------- 1 | schemaVersion: '2.0.0' 2 | 3 | metadataTest: 4 | workdir: "/app" -------------------------------------------------------------------------------- /grid/worker/worker.conf: -------------------------------------------------------------------------------- 1 | server: 2 | - !channel 3 | port: 8980 4 | insecure_mode: true 5 | 6 | instances: 7 | - name: main 8 | 9 | storages: 10 | - !lru-storage &main-storage 11 | size: 512MB 12 | 13 | services: 14 | - !action-cache &main-action 15 | storage: *main-storage 16 | max_cached_refs: 256 17 | allow_updates: true 18 | - !execution 19 | storage: *main-storage 20 | action_cache: *main-action 21 | - !cas 22 | storage: *main-storage 23 | - !bytestream 24 | storage: *main-storage 25 | -------------------------------------------------------------------------------- /grid/workspace.bzl: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_docker//container:load.bzl", "container_load") 2 | 3 | BUILD_BAZEL = """ 4 | exports_files(["image.tar"]) 5 | """ 6 | 7 | 8 | def _buildgrid_repository_impl(repository_ctx): 9 | commit = repository_ctx.attr.commit 10 | tag = "buildgrid-" + commit 11 | 12 | # Build the gitlab url where buildgrid is installed 13 | url = repository_ctx.attr.remote + "?ref=" + commit 14 | 15 | print("Installing " + url) 16 | 17 | # Download and unarchive it! 18 | repository_ctx.download_and_extract(url, 19 | stripPrefix = "-".join(["buildgrid", commit, commit]), 20 | ) 21 | 22 | if repository_ctx.attr.dockerfile: 23 | repository_ctx.file("Dockerfile", repository_ctx.attr.dockerfile) 24 | 25 | result = repository_ctx.execute(["docker", "build", "-t", tag, "."], quiet = False) 26 | if result.return_code: 27 | fail("docker build failed: %s" % result.stderr) 28 | 29 | result = repository_ctx.execute(["docker", "save", "-o", "image.tar", tag], quiet = False) 30 | if result.return_code: 31 | fail("docker build failed: %s" % result.stderr) 32 | 33 | repository_ctx.file("BUILD.bazel", BUILD_BAZEL) 34 | 35 | 36 | _buildgrid_repository = repository_rule( 37 | implementation = _buildgrid_repository_impl, 38 | attrs = { 39 | "remote": attr.string( 40 | default = "https://gitlab.com/BuildGrid/buildgrid/repository/archive.tar.gz", 41 | ), 42 | "dockerfile": attr.string( 43 | doc = "The Dockerfile content to use when building the image.", 44 | ), 45 | "commit": attr.string( 46 | mandatory = True, 47 | ), 48 | } 49 | ) 50 | 51 | def buildgrid_repository(**kwargs): 52 | name = kwargs.pop("name") 53 | name_tar = name + "_tar" 54 | kwargs["name"] = name_tar 55 | 56 | _buildgrid_repository(**kwargs) 57 | 58 | container_load( 59 | name = name, 60 | file = "@%s//:image.tar" % name_tar, 61 | ) 62 | -------------------------------------------------------------------------------- /rbe/Makefile: -------------------------------------------------------------------------------- 1 | 2 | setup: 3 | curl -X GET -o /tmp/key.json "https://www.googleapis.com/storage/v1/b/bazel_key/o/key.json?alt=media" 4 | 5 | abseil: 6 | cp ./bazelrc /tmp 7 | (cd /tmp/abseil-cpp && bazel \ 8 | --bazelrc=/tmp/bazelrc \ 9 | build //absl/... \ 10 | --config=remote --jobs=300 \ 11 | --remote_instance_name=projects/bazelcon18-rbe-shared/instances/default_instance \ 12 | --auth_credentials=/tmp/key.json) 13 | --------------------------------------------------------------------------------