├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── workflows │ └── go.yml ├── .gitignore ├── .travis.yml ├── AUTHORS ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── cloudprober.go ├── cloudprober_test.go ├── cmd ├── cloudprober.go └── cloudprober_test.cfg ├── common ├── file │ ├── file.go │ └── file_test.go ├── iputils │ ├── iputils.go │ └── iputils_test.go ├── message │ ├── message.go │ ├── message_test.go │ └── proto │ │ ├── message.pb.go │ │ └── message.proto ├── metadata │ └── metadata.go ├── oauth │ ├── bearer.go │ ├── bearer_test.go │ ├── oauth.go │ ├── oauth_test.go │ └── proto │ │ ├── config.pb.go │ │ └── config.proto └── tlsconfig │ ├── proto │ ├── config.pb.go │ └── config.proto │ └── tlsconfig.go ├── config ├── config.go ├── config_test.go ├── proto │ ├── config.pb.go │ └── config.proto └── runconfig │ ├── runconfig.go │ └── runconfig_test.go ├── contrib ├── Dockerfile ├── Makefile └── gcp │ ├── bigquery │ ├── bigquery.go │ └── bigquery_test.go │ └── cmd │ └── bigquery_probe.go ├── docs ├── Makefile ├── archetypes │ └── default.md ├── config.toml ├── content │ ├── _index.md │ ├── _readme.md │ ├── concepts │ │ ├── probe.md │ │ └── targets.md │ ├── getting-started.md │ ├── how-to │ │ ├── additional-labels.md │ │ ├── built-in-servers.md │ │ ├── extensions.md │ │ ├── external-probe.md │ │ ├── percentiles.md │ │ ├── run-on-kubernetes.md │ │ └── validators.md │ └── surfacers │ │ ├── cloudwatch.md │ │ ├── overview.md │ │ └── stackdriver.md ├── layouts │ └── shortcodes │ │ └── content.html ├── static │ ├── CNAME │ └── diagrams │ │ ├── cloudprober_use_case.svg │ │ ├── cloudprober_use_case.xml │ │ ├── external_probe_server.svg │ │ ├── latency_distribution.png │ │ ├── metrics_explorer_percentile.png │ │ ├── rds_targets.png │ │ └── redis_probe_screenshot.png └── themes │ └── hugo-material-docs │ ├── CHANGELOG.md │ ├── LICENSE.md │ ├── README.md │ ├── archetypes │ └── default.md │ ├── exampleSite │ ├── config.toml │ ├── content │ │ ├── adding-content │ │ │ └── index.md │ │ ├── getting-started │ │ │ └── index.md │ │ ├── index.md │ │ ├── license │ │ │ └── index.md │ │ └── roadmap │ │ │ └── index.md │ └── static │ │ └── .gitkeep │ ├── images │ ├── screenshot.png │ └── tn.png │ ├── layouts │ ├── 404.html │ ├── _default │ │ ├── __list.html │ │ └── single.html │ ├── index.html │ ├── partials │ │ ├── drawer.html │ │ ├── footer.html │ │ ├── footer_js.html │ │ ├── head.html │ │ ├── header.html │ │ ├── nav.html │ │ └── nav_link.html │ └── shortcodes │ │ ├── note.html │ │ └── warning.html │ ├── static │ ├── fonts │ │ ├── icon.eot │ │ ├── icon.svg │ │ ├── icon.ttf │ │ └── icon.woff │ ├── images │ │ ├── colors.png │ │ ├── favicon.ico │ │ ├── logo.png │ │ └── screen.png │ ├── javascripts │ │ ├── application.js │ │ └── modernizr.js │ └── stylesheets │ │ ├── application.css │ │ ├── palettes.css │ │ ├── syntax.css │ │ └── temporary.css │ └── theme.toml ├── examples ├── additional_label │ └── cloudprober.cfg ├── extensions │ └── myprober │ │ ├── myprobe │ │ ├── myprobe.go │ │ ├── myprobe.pb.go │ │ └── myprobe.proto │ │ ├── myprober │ │ ├── myprober.cfg │ │ └── myprober.go ├── external │ ├── cloudprober.cfg │ ├── cloudprober_aggregate.cfg │ ├── cloudprober_server.cfg │ └── redis_probe.go ├── file_based_targets │ ├── cloudprober.cfg │ └── targets.textpb ├── surfacers │ ├── default_surfacers.cfg │ └── stackdriver_surfacer.cfg ├── templates │ ├── README.md │ └── cloudprober.cfg └── validators │ └── cloudprober_validator.cfg ├── go.mod ├── go.sum ├── logger ├── logger.go └── logger_test.go ├── metrics ├── dist.go ├── dist_test.go ├── eventmetrics.go ├── eventmetrics_test.go ├── float.go ├── int.go ├── map.go ├── map_test.go ├── metrics.go ├── metrics_test.go ├── payload │ ├── payload.go │ ├── payload_test.go │ └── proto │ │ ├── config.pb.go │ │ └── config.proto ├── proto │ ├── dist.pb.go │ └── dist.proto ├── string.go ├── string_test.go └── testutils │ ├── testutils.go │ └── testutils_test.go ├── prober ├── cmd │ └── client.go ├── prober.go ├── proto │ ├── service.pb.go │ └── service.proto ├── serviceimpl.go └── serviceimpl_test.go ├── probes ├── common │ └── statskeeper │ │ ├── statskeeper.go │ │ └── statskeeper_test.go ├── dns │ ├── cmd │ │ └── dns.go │ ├── dns.go │ ├── dns_test.cfg │ ├── dns_test.go │ └── proto │ │ ├── config.pb.go │ │ └── config.proto ├── external │ ├── cmd │ │ └── external.go │ ├── external.go │ ├── external_test.go │ ├── proto │ │ ├── config.pb.go │ │ ├── config.proto │ │ ├── server.pb.go │ │ └── server.proto │ └── serverutils │ │ └── serverutils.go ├── grpc │ ├── grpc.go │ ├── grpc_test.go │ └── proto │ │ ├── config.pb.go │ │ └── config.proto ├── http │ ├── cmd │ │ └── http.go │ ├── http.go │ ├── http_test.cfg │ ├── http_test.go │ ├── proto │ │ ├── config.pb.go │ │ └── config.proto │ ├── request.go │ └── request_test.go ├── options │ ├── labels.go │ ├── labels_test.go │ ├── options.go │ └── options_test.go ├── ping │ ├── cmd │ │ └── ping.go │ ├── icmpconn_nonunix.go │ ├── icmpconn_unix.go │ ├── ping.go │ ├── ping_test.go │ ├── pingutils.go │ ├── pingutils_test.go │ └── proto │ │ ├── config.pb.go │ │ └── config.proto ├── probes.go ├── probes_status_tmpl.go ├── probes_test.go ├── probeutils │ ├── probeutils.go │ └── probeutils_test.go ├── proto │ ├── config.pb.go │ └── config.proto ├── testdata │ ├── testdata.pb.go │ └── testdata.proto ├── udp │ ├── cmd │ │ └── udp.go │ ├── proto │ │ ├── config.pb.go │ │ └── config.proto │ ├── udp.go │ └── udp_test.go └── udplistener │ ├── proto │ ├── config.pb.go │ └── config.proto │ ├── udplistener.go │ └── udplistener_test.go ├── rds ├── client │ ├── client.go │ ├── client_test.go │ ├── cmd │ │ └── client.go │ ├── proto │ │ ├── config.pb.go │ │ └── config.proto │ ├── srvlist.go │ └── srvlist_test.go ├── examples │ └── cloudprober_rds.cfg ├── file │ ├── file.go │ ├── file_test.go │ ├── proto │ │ ├── config.pb.go │ │ └── config.proto │ └── testdata │ │ ├── targets.json │ │ ├── targets1.textpb │ │ └── targets2.textpb ├── gcp │ ├── forwarding_rules.go │ ├── gce_instances.go │ ├── gce_instances_test.go │ ├── gcp.go │ ├── gcp_test.go │ ├── proto │ │ ├── config.pb.go │ │ └── config.proto │ ├── pubsub.go │ ├── pubsub_test.go │ ├── rtc_variables.go │ ├── rtc_variables_test.go │ └── testdata │ │ ├── instances.json │ │ └── zones.json ├── kubernetes │ ├── client.go │ ├── client_test.go │ ├── endpoints.go │ ├── endpoints_test.go │ ├── ingresses.go │ ├── ingresses_test.go │ ├── kubernetes.go │ ├── kubernetes_test.go │ ├── pods.go │ ├── pods_test.go │ ├── proto │ │ ├── config.pb.go │ │ └── config.proto │ ├── services.go │ ├── services_test.go │ └── testdata │ │ ├── endpoints.json │ │ ├── ingresses.json │ │ ├── pods.json │ │ └── services.json ├── proto │ ├── rds.pb.go │ └── rds.proto └── server │ ├── cmd │ └── server.go │ ├── filter │ ├── filter.go │ ├── filter_test.go │ ├── utils.go │ └── utils_test.go │ ├── proto │ ├── config.pb.go │ └── config.proto │ ├── server.go │ └── server_test.go ├── servers ├── README.md ├── external │ ├── external.go │ └── proto │ │ ├── config.pb.go │ │ └── config.proto ├── grpc │ ├── grpc.go │ ├── grpc_test.go │ └── proto │ │ ├── config.pb.go │ │ ├── config.proto │ │ ├── grpcservice.pb.go │ │ └── grpcservice.proto ├── http │ ├── http.go │ ├── http_test.go │ └── proto │ │ ├── config.pb.go │ │ └── config.proto ├── proto │ ├── config.pb.go │ └── config.proto ├── servers.go └── udp │ ├── cmd │ └── udp.go │ ├── proto │ ├── config.pb.go │ └── config.proto │ ├── udp.go │ └── udp_test.go ├── surfacers ├── cloudwatch │ ├── cloudwatch.go │ ├── cloudwatch_test.go │ └── proto │ │ ├── config.pb.go │ │ └── config.proto ├── common │ ├── compress │ │ ├── compress.go │ │ └── compress_test.go │ ├── options │ │ ├── options.go │ │ └── options_test.go │ └── transform │ │ ├── transform.go │ │ └── transform_test.go ├── datadog │ ├── client.go │ ├── client_test.go │ ├── datadog.go │ ├── datadog_test.go │ └── proto │ │ ├── config.pb.go │ │ └── config.proto ├── file │ ├── file.go │ ├── file_test.go │ └── proto │ │ ├── config.pb.go │ │ └── config.proto ├── postgres │ ├── postgres.go │ ├── postgres_test.go │ └── proto │ │ ├── config.pb.go │ │ └── config.proto ├── prometheus │ ├── prometheus.go │ ├── prometheus_test.go │ └── proto │ │ ├── config.pb.go │ │ └── config.proto ├── proto │ ├── config.pb.go │ └── config.proto ├── pubsub │ ├── proto │ │ ├── config.pb.go │ │ └── config.proto │ ├── pubsub.go │ └── pubsub_test.go ├── stackdriver │ ├── proto │ │ ├── config.pb.go │ │ └── config.proto │ ├── resource.go │ ├── stackdriver.go │ └── stackdriver_test.go ├── surfacers.go └── surfacers_test.go ├── sysvars ├── runtime.go ├── runtime_linux.go ├── runtime_nonlinux.go ├── runtime_test.go ├── sysvars.go ├── sysvars_ec2.go ├── sysvars_gce.go └── sysvars_test.go ├── targets ├── endpoint │ ├── endpoint.go │ └── endpoint_test.go ├── file │ ├── file.go │ ├── file_test.go │ └── proto │ │ ├── config.pb.go │ │ └── config.proto ├── gce │ ├── forwarding_rules.go │ ├── gce.go │ ├── gce_test.go │ ├── gce_utils.go │ ├── gce_utils_test.go │ └── proto │ │ ├── config.pb.go │ │ └── config.proto ├── lameduck │ ├── lameduck.go │ ├── lameduck_test.go │ └── proto │ │ ├── config.pb.go │ │ └── config.proto ├── proto │ ├── targets.pb.go │ └── targets.proto ├── resolver │ ├── resolver.go │ └── resolver_test.go ├── rtc │ └── rtcservice │ │ ├── rtcservice.go │ │ ├── rtcservice_stub.go │ │ └── rtcservice_test.go ├── statictargets.go ├── statictargets_test.go ├── targets.go ├── targets_test.go └── testdata │ ├── testdata.pb.go │ └── testdata.proto ├── tools ├── build.sh ├── cloudprober_startup.sh └── gen_pb_go.sh ├── validators ├── http │ ├── http.go │ ├── http_test.go │ └── proto │ │ ├── config.pb.go │ │ └── config.proto ├── integrity │ ├── integrity.go │ ├── integrity_test.go │ └── proto │ │ ├── config.pb.go │ │ └── config.proto ├── proto │ ├── config.pb.go │ └── config.proto ├── regex │ ├── regex.go │ └── regex_test.go ├── validators.go └── validators_test.go └── web ├── formatutils └── formatutils.go ├── status_tmpl.go └── web.go /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **Cloudprober Version** 14 | 15 | **To Reproduce** 16 | Steps to reproduce the behavior: 17 | 1. Go to '...' 18 | 2. Click on '....' 19 | 3. Scroll down to '....' 20 | 4. See error 21 | 22 | **Additional context** 23 | Add any other context about the problem here. 24 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: File a feature request 4 | title: '' 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the feature you'd like and the problem it will solve** 11 | Description of what you want to happen and what problem will it solve for you. 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | docs/public 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | matrix: 2 | include: 3 | - os: linux 4 | dist: trusty 5 | - os: osx 6 | env: EXTRA_TEST_FLAGS="-tags skip_udp_probe_test" 7 | 8 | language: go 9 | 10 | go: 11 | - "1.14" 12 | 13 | install: 14 | - go get -t ./... 15 | 16 | script: 17 | - go test ${EXTRA_TEST_FLAGS} -v -race -covermode=atomic ./... 18 | 19 | go_import_path: github.com/google/cloudprober 20 | 21 | services: 22 | - docker 23 | 24 | after_success: 25 | - make cloudprober 26 | - cp /etc/ssl/certs/ca-certificates.crt . 27 | - test "$TRAVIS_BRANCH" = "master" && test "$TRAVIS_EVENT_TYPE" = "push" && DOCKER_VERSION=latest 28 | - test -n "$TRAVIS_TAG" && DOCKER_VERSION=$TRAVIS_TAG 29 | - test -n "$DOCKER_IMAGE" && test -n "$DOCKER_VERSION" && test "$TRAVIS_OS_NAME" = "linux" && DOCKER_VERSION=${DOCKER_VERSION} make docker_build && docker login -u ${DOCKER_USER} -p ${DOCKER_PASS} && docker push ${DOCKER_IMAGE} 30 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # This is the list of Cloudprober authors for copyright purposes. 2 | # 3 | # This does not list everyone who has contributed code. To see the full list of 4 | # contributors, see the revision history in source control. 5 | 6 | Google Inc. 7 | 8 | # Primary author: 9 | Manu Garg 10 | 11 | # Partial list of contributors: 12 | Lenin Singaravelu 13 | Nerijus Bendžiūnas 14 | Rob Pickerill 15 | Christopher Broglie 16 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution, 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult [GitHub 22 | Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | 25 | ## Source of Truth (SoT) and Commit Process 26 | 27 | Cloudprober's source of truth is Google's internal copy of the code for various 28 | reasons. To keep it that way, we first import the proposed pull request into 29 | the internal copy of the code, get it reviewed internally, and then export it 30 | back to Github. As a result of this, original PR is eventually discarded and a 31 | new PR is merged into the master branch. All through this process, author 32 | attribution is not lost. Final change will still appear to be done by you. 33 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # This Dockerfile expects "cloudprober" binary and ca-certificates to exist 2 | # in the working directory. 3 | # 4 | # Docker image built using this can executed in the following manner: 5 | # docker run --net host -v $PWD/cloudprober.cfg:/etc/cloudprober.cfg \ 6 | # cloudprober/cloudprober 7 | FROM busybox 8 | ADD cloudprober /cloudprober 9 | COPY ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 10 | 11 | # Metadata params 12 | ARG BUILD_DATE 13 | ARG VERSION 14 | ARG VCS_REF 15 | 16 | # Metadata 17 | LABEL org.label-schema.build-date=$BUILD_DATE \ 18 | org.label-schema.name="Cloudprober" \ 19 | org.label-schema.vcs-url="https://github.com/google/cloudprober" \ 20 | org.label-schema.vcs-ref=$VCS_REF \ 21 | org.label-schema.version=$VERSION \ 22 | com.microscaling.license="Apache-2.0" 23 | 24 | ENTRYPOINT ["/cloudprober", "--logtostderr"] 25 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | VERSION ?= $(shell git describe --tags) 2 | DOCKER_VERSION ?= $(VERSION) 3 | GIT_COMMIT = $(strip $(shell git rev-parse --short HEAD)) 4 | GOBIN ?= ${GOPATH}/bin 5 | BINARY ?= cloudprober 6 | DOCKER_IMAGE ?= cloudprober/cloudprober 7 | CACERTS ?= /etc/ssl/certs/ca-certificates.crt 8 | SOURCES := $(shell find . -name '*.go') 9 | 10 | test: 11 | go test -v -race -covermode=atomic ./... 12 | 13 | $(BINARY): $(SOURCES) 14 | CGO_ENABLED=0 go build -o cloudprober -ldflags "-X main.version=$(VERSION) -extldflags -static" ./cmd/cloudprober.go 15 | 16 | ca-certificates.crt: $(CACERTS) 17 | cp $(CACERTS) ca-certificates.crt 18 | 19 | docker_build: $(BINARY) ca-certificates.crt Dockerfile 20 | docker build \ 21 | --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \ 22 | --build-arg VERSION=$(VERSION) \ 23 | --build-arg VCS_REF=$(GIT_COMMIT) \ 24 | -t $(DOCKER_IMAGE) . 25 | 26 | docker_push: 27 | docker tag $(DOCKER_IMAGE) $(DOCKER_IMAGE):$(DOCKER_VERSION) 28 | docker login -u "${DOCKER_USER}" -p "${DOCKER_PASS}" 29 | docker push $(DOCKER_IMAGE):$(DOCKER_VERSION) 30 | 31 | docker_push_tagged: 32 | docker tag $(DOCKER_IMAGE) $(DOCKER_IMAGE):$(DOCKER_VERSION) 33 | docker tag $(DOCKER_IMAGE) $(DOCKER_IMAGE):latest 34 | docker login -u "${DOCKER_USER}" -p "${DOCKER_PASS}" 35 | docker image push --all-tags $(DOCKER_IMAGE) 36 | 37 | install: 38 | GOBIN=$(GOBIN) CGO_ENABLED=0 go install -ldflags "-X main.version=$(VERSION) -extldflags -static" ./cmd/cloudprober.go 39 | 40 | clean: 41 | rm cloudprober 42 | go get -u ./... 43 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Active development of this repository has now moved to [github.com/cloudprober/cloudprober](http://github.com/cloudprober/cloudprober). 2 | -------------------------------------------------------------------------------- /cloudprober_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cloudprober 16 | 17 | import ( 18 | "os" 19 | "testing" 20 | 21 | configpb "github.com/google/cloudprober/config/proto" 22 | "google.golang.org/protobuf/proto" 23 | ) 24 | 25 | func TestGetDefaultServerPort(t *testing.T) { 26 | tests := []struct { 27 | desc string 28 | configPort int32 29 | envVar string 30 | wantPort int 31 | wantErr bool 32 | }{ 33 | { 34 | desc: "use port from config", 35 | configPort: 9316, 36 | envVar: "3141", 37 | wantPort: 9316, 38 | }, 39 | { 40 | desc: "use default port", 41 | configPort: 0, 42 | envVar: "", 43 | wantPort: DefaultServerPort, 44 | }, 45 | { 46 | desc: "use port from env", 47 | configPort: 0, 48 | envVar: "3141", 49 | wantPort: 3141, 50 | }, 51 | { 52 | desc: "ignore kubernetes port", 53 | configPort: 0, 54 | envVar: "tcp://100.101.102.103:3141", 55 | wantPort: 9313, 56 | }, 57 | { 58 | desc: "error due to bad env var", 59 | configPort: 0, 60 | envVar: "a3141", 61 | wantErr: true, 62 | }, 63 | } 64 | 65 | for _, test := range tests { 66 | t.Run(test.desc, func(t *testing.T) { 67 | os.Setenv(ServerPortEnvVar, test.envVar) 68 | port, err := getDefaultServerPort(&configpb.ProberConfig{ 69 | Port: proto.Int32(test.configPort), 70 | }, nil) 71 | 72 | if err != nil { 73 | if !test.wantErr { 74 | t.Errorf("Got unexpected error: %v", err) 75 | } else { 76 | return 77 | } 78 | } 79 | 80 | if port != test.wantPort { 81 | t.Errorf("got port: %d, want port: %d", port, test.wantPort) 82 | } 83 | }) 84 | } 85 | 86 | } 87 | -------------------------------------------------------------------------------- /cmd/cloudprober_test.cfg: -------------------------------------------------------------------------------- 1 | {{with $shards := mkSlice "00" "01"}} 2 | {{range $_, $shard := $shards}} 3 | probe { 4 | type: PING 5 | name: "vm-to-google-{{$shard}}" 6 | targets { 7 | host_names: "8.8.8.8,8.8.4.4,www.google.com" 8 | } 9 | ping_probe { 10 | use_datagram_socket: true 11 | } 12 | } 13 | {{end}} 14 | {{end}} 15 | 16 | {{ $shard := "ig-us-east1-a-02-afgx" | extractSubstring "[^-]+-[^-]+-[^-]+-[^-]+-([^-]+)-.*" 1 }} 17 | {{ $targets := printf "^ig.*-([^-]+-[^-]+-[^-]+)-%s-[^-]+$" $shard }} 18 | 19 | probe { 20 | type: PING 21 | name: "vm-to-vm-{{$shard}}" 22 | targets { 23 | gce_targets { 24 | project: "google.com:bbmc-testing-prod" 25 | instances {} 26 | } 27 | regex: "{{$targets}}" 28 | } 29 | ping_probe {} 30 | } 31 | 32 | probe { 33 | type: HTTP 34 | name: "vm-to-google-http-{{$shard}}" 35 | targets { 36 | host_names: "www.google.com,{{.hostname}}" 37 | } 38 | http_probe { 39 | protocol: HTTP 40 | relative_url: "/healthz" 41 | } 42 | } 43 | 44 | 45 | probe { 46 | type: DNS 47 | name: "vm-to-public-dns" 48 | targets { 49 | host_names: "8.8.8.8" 50 | } 51 | dns_probe { 52 | resolved_domain: "www.google.com." 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /common/file/file_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package file 16 | 17 | import ( 18 | "io/ioutil" 19 | "testing" 20 | ) 21 | 22 | func createTempFile(t *testing.T, b []byte) string { 23 | tmpfile, err := ioutil.TempFile("", "") 24 | if err != nil { 25 | t.Fatal(err) 26 | return "" 27 | } 28 | 29 | defer tmpfile.Close() 30 | if _, err := tmpfile.Write(b); err != nil { 31 | t.Fatal(err) 32 | } 33 | 34 | return tmpfile.Name() 35 | } 36 | 37 | func testReadFile(path string) ([]byte, error) { 38 | return []byte("content-for-" + path), nil 39 | } 40 | 41 | func TestReadFile(t *testing.T) { 42 | prefixToReadfunc["test://"] = testReadFile 43 | 44 | // Virtual file 45 | testPath := "test://test-file" 46 | 47 | // Disk file 48 | tempContent := "temp-content" 49 | tempPath := createTempFile(t, []byte(tempContent)) 50 | 51 | testData := map[string]string{ 52 | testPath: "content-for-test-file", 53 | tempPath: tempContent, 54 | } 55 | 56 | for path, expectedContent := range testData { 57 | t.Run("ReadFile("+path+")", func(t *testing.T) { 58 | b, err := ReadFile(path) 59 | if err != nil { 60 | t.Fatalf("Error while reading the file: %s", path) 61 | } 62 | 63 | if string(b) != expectedContent { 64 | t.Errorf("ReadFile(%s) = %s, expected=%s", path, string(b), expectedContent) 65 | } 66 | }) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /common/iputils/iputils_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package iputils 16 | 17 | import ( 18 | "net" 19 | "testing" 20 | ) 21 | 22 | func TestIPVersion(t *testing.T) { 23 | rows := []struct { 24 | ip string 25 | ipVer int 26 | }{ 27 | {"1.1.1.1", 4}, 28 | {"::1", 6}, 29 | } 30 | 31 | for _, r := range rows { 32 | ipVer := IPVersion(net.ParseIP(r.ip)) 33 | 34 | if ipVer != r.ipVer { 35 | t.Errorf("Unexpected IPVersion want=%d, got=%d", r.ipVer, ipVer) 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /common/message/proto/message.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package message; 4 | 5 | option go_package = "github.com/google/cloudprober/common/message/proto"; 6 | 7 | // Constants defines constants with default values. 8 | message Constants { 9 | optional uint64 magic = 1 [default = 0xea74cace87ea]; 10 | } 11 | 12 | // Datanode is something that see's a message AND can modify it. 13 | message DataNode { 14 | enum Type { 15 | UNKNOWN = 0; 16 | CLIENT = 1; 17 | SERVER = 2; 18 | } 19 | optional Type type = 1 [default = CLIENT]; 20 | optional string name = 2; 21 | optional string port = 4; 22 | 23 | // 8 bytes of timestamp in pcap-friendly network byte order. 24 | optional bytes timestamp_usec = 3; 25 | } 26 | 27 | // Msg is a message sent over the network. 28 | // magic, seq, src and dst are required fields. 29 | message Msg { 30 | optional fixed64 magic = 1; // required. 31 | 32 | // 8 bytes of sequence in pcap-friendly network byte order. 33 | optional bytes seq = 2; // required. 34 | 35 | // Datanodes seen by this message. 36 | optional DataNode src = 3; // required. 37 | optional DataNode dst = 4; // required. 38 | repeated DataNode nodes = 5; // Intermediate nodes. 39 | 40 | // Optional payload. 41 | optional bytes payload = 99; 42 | } 43 | -------------------------------------------------------------------------------- /common/metadata/metadata.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /* 16 | Package metadata implements metadata related utilities. 17 | */ 18 | package metadata 19 | 20 | import ( 21 | "io/ioutil" 22 | "os" 23 | ) 24 | 25 | // IsKubernetes return true if running on Kubernetes. 26 | // It uses the environment variable KUBERNETES_SERVICE_HOST to decide if we 27 | // we are running Kubernetes. 28 | func IsKubernetes() bool { 29 | return os.Getenv("KUBERNETES_SERVICE_HOST") != "" 30 | } 31 | 32 | // KubernetesNamespace returns the Kubernetes namespace. It returns an empty 33 | // string if there is an error in retrieving the namespace. 34 | func KubernetesNamespace() string { 35 | namespaceBytes, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") 36 | if err == nil { 37 | return string(namespaceBytes) 38 | } 39 | return "" 40 | } 41 | -------------------------------------------------------------------------------- /common/oauth/oauth.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /* 16 | Package oauth implements OAuth related utilities for Cloudprober. 17 | */ 18 | package oauth 19 | 20 | import ( 21 | "context" 22 | "fmt" 23 | 24 | "github.com/google/cloudprober/common/file" 25 | configpb "github.com/google/cloudprober/common/oauth/proto" 26 | "github.com/google/cloudprober/logger" 27 | "golang.org/x/oauth2" 28 | "golang.org/x/oauth2/google" 29 | ) 30 | 31 | // TokenSourceFromConfig builds a oauth2.TokenSource from the provided config. 32 | func TokenSourceFromConfig(c *configpb.Config, l *logger.Logger) (oauth2.TokenSource, error) { 33 | switch c.Type.(type) { 34 | 35 | case *configpb.Config_BearerToken: 36 | return newBearerTokenSource(c.GetBearerToken(), l) 37 | 38 | case *configpb.Config_GoogleCredentials: 39 | f := c.GetGoogleCredentials().GetJsonFile() 40 | 41 | // If JSON file is not provided, try default credentials. 42 | if f == "" { 43 | creds, err := google.FindDefaultCredentials(context.Background(), c.GetGoogleCredentials().GetScope()...) 44 | if err != nil { 45 | return nil, err 46 | } 47 | return creds.TokenSource, nil 48 | } 49 | 50 | jsonKey, err := file.ReadFile(f) 51 | if err != nil { 52 | return nil, fmt.Errorf("error reading Google Credentials file (%s): %v", f, err) 53 | } 54 | 55 | aud := c.GetGoogleCredentials().GetAudience() 56 | if aud != "" || c.GetGoogleCredentials().GetJwtAsAccessToken() { 57 | if !c.GetGoogleCredentials().GetJwtAsAccessToken() { 58 | return nil, fmt.Errorf("oauth: audience (%s) should only be set if jwt_as_access_token is set to true", aud) 59 | } 60 | return google.JWTAccessTokenSourceFromJSON(jsonKey, aud) 61 | } 62 | 63 | creds, err := google.CredentialsFromJSON(context.Background(), jsonKey, c.GetGoogleCredentials().GetScope()...) 64 | if err != nil { 65 | return nil, fmt.Errorf("error parsing Google Credentials file (%s): %v", f, err) 66 | } 67 | return creds.TokenSource, nil 68 | } 69 | 70 | return nil, fmt.Errorf("unknown oauth credentials type: %v", c.Type) 71 | } 72 | -------------------------------------------------------------------------------- /common/oauth/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.oauth; 4 | 5 | option go_package = "github.com/google/cloudprober/common/oauth/proto"; 6 | 7 | message Config { 8 | oneof type { 9 | BearerToken bearer_token = 1; 10 | GoogleCredentials google_credentials = 2; 11 | } 12 | } 13 | 14 | // Bearer token is added to the HTTP request through an HTTP header: 15 | // "Authorization: Bearer " 16 | message BearerToken { 17 | oneof source { 18 | // Path to token file. 19 | string file = 1; 20 | 21 | // Run a comand to obtain the token, e.g. 22 | // cat /var/lib/myapp/token, or 23 | // /var/lib/run/get_token.sh 24 | string cmd = 2; 25 | 26 | // GCE metadata token 27 | string gce_service_account = 3; 28 | } 29 | 30 | // How often to refresh token. As OAuth token usually expire, we need to 31 | // refresh them on a regular interval. If set to 0, caching is disabled. 32 | optional float refresh_interval_sec = 90 [default = 60]; 33 | } 34 | 35 | // Google credentials in JSON format. We simply use oauth2/google package to 36 | // use these credentials. 37 | message GoogleCredentials { 38 | optional string json_file = 1; 39 | repeated string scope = 2; 40 | 41 | // Use encoded JWT directly as access token, instead of implementing the whole 42 | // OAuth2.0 flow. 43 | optional bool jwt_as_access_token = 4; 44 | 45 | // Audience works only if jwt_as_access_token is true. 46 | optional string audience = 3; 47 | } 48 | -------------------------------------------------------------------------------- /common/tlsconfig/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.tlsconfig; 4 | 5 | option go_package = "github.com/google/cloudprober/common/tlsconfig/proto"; 6 | 7 | message TLSConfig { 8 | // CA certificate file to verify certificates provided by the other party. 9 | optional string ca_cert_file = 1; 10 | 11 | // Local certificate file. 12 | optional string tls_cert_file = 2; 13 | 14 | // Private key file corresponding to the certificate above. 15 | optional string tls_key_file = 3; 16 | 17 | // Whether to ignore the cert validation. 18 | optional bool disable_cert_validation = 4; 19 | 20 | // ServerName override 21 | optional string server_name = 5; 22 | } 23 | -------------------------------------------------------------------------------- /common/tlsconfig/tlsconfig.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package tlsconfig implements utilities to parse TLSConfig. 16 | package tlsconfig 17 | 18 | import ( 19 | "crypto/tls" 20 | "crypto/x509" 21 | "fmt" 22 | 23 | "github.com/google/cloudprober/common/file" 24 | configpb "github.com/google/cloudprober/common/tlsconfig/proto" 25 | ) 26 | 27 | // UpdateTLSConfig parses the provided protobuf and updates the tls.Config object. 28 | func UpdateTLSConfig(tlsConfig *tls.Config, c *configpb.TLSConfig, addClientCACerts bool) error { 29 | if c.GetDisableCertValidation() { 30 | tlsConfig.InsecureSkipVerify = true 31 | } 32 | 33 | if c.GetCaCertFile() != "" { 34 | caCert, err := file.ReadFile(c.GetCaCertFile()) 35 | if err != nil { 36 | return fmt.Errorf("common/tlsconfig: error reading CA cert file (%s): %v", c.GetCaCertFile(), err) 37 | } 38 | caCertPool := x509.NewCertPool() 39 | if !caCertPool.AppendCertsFromPEM(caCert) { 40 | return fmt.Errorf("error while adding CA certs from: %s", c.GetCaCertFile()) 41 | } 42 | 43 | tlsConfig.RootCAs = caCertPool 44 | // Client CA certs are used by servers to authenticate clients. 45 | if addClientCACerts { 46 | tlsConfig.ClientCAs = caCertPool 47 | } 48 | } 49 | 50 | if c.GetTlsCertFile() != "" { 51 | certPEMBlock, err := file.ReadFile(c.GetTlsCertFile()) 52 | if err != nil { 53 | return fmt.Errorf("common/tlsconfig: error reading TLS cert file (%s): %v", c.GetTlsCertFile(), err) 54 | } 55 | keyPEMBlock, err := file.ReadFile(c.GetTlsKeyFile()) 56 | if err != nil { 57 | return fmt.Errorf("common/tlsconfig: error reading TLS key file (%s): %v", c.GetTlsKeyFile(), err) 58 | } 59 | 60 | cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock) 61 | if err != nil { 62 | return fmt.Errorf("common/tlsconfig: error initializing cert from cert key pair: %v", err) 63 | } 64 | tlsConfig.Certificates = append(tlsConfig.Certificates, cert) 65 | } 66 | 67 | if c.GetServerName() != "" { 68 | tlsConfig.ServerName = c.GetServerName() 69 | } 70 | 71 | return nil 72 | } 73 | -------------------------------------------------------------------------------- /config/runconfig/runconfig_test.go: -------------------------------------------------------------------------------- 1 | package runconfig 2 | 3 | import ( 4 | "testing" 5 | 6 | "google.golang.org/grpc" 7 | ) 8 | 9 | func TestRunConfig(t *testing.T) { 10 | if srv := DefaultGRPCServer(); srv != nil { 11 | t.Fatalf("RunConfig has server unexpectedly set. Got %v Want nil", srv) 12 | } 13 | testSrv := grpc.NewServer() 14 | if testSrv == nil { 15 | t.Fatal("Unable to create a test gRPC server") 16 | } 17 | if err := SetDefaultGRPCServer(testSrv); err != nil { 18 | t.Fatalf("Unable to set default gRPC server: %v", err) 19 | } 20 | if srv := DefaultGRPCServer(); srv != testSrv { 21 | t.Fatalf("Error retrieving stored service. Got %v Want %v", srv, testSrv) 22 | } 23 | if err := SetDefaultGRPCServer(testSrv); err == nil { 24 | t.Errorf("RunConfig allowed overriding of an already set variable.") 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /contrib/Dockerfile: -------------------------------------------------------------------------------- 1 | # This Dockerfile packages contrib binaries alongwith cloudprober 2 | # and ca certificates. 3 | # 4 | # Docker image built using this can executed in the following manner: 5 | # docker run -v $PWD/cloudprober.cfg:/etc/cloudprober.cfg \ 6 | # cloudprober/cloudprober 7 | FROM busybox 8 | ADD cloudprober /cloudprober 9 | ADD bigquery_probe /contrib/bigquery_probe 10 | COPY ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 11 | 12 | # Metadata params 13 | ARG BUILD_DATE 14 | ARG VERSION 15 | ARG VCS_REF 16 | 17 | # Metadata 18 | LABEL org.label-schema.build-date=$BUILD_DATE \ 19 | org.label-schema.name="Cloudprober" \ 20 | org.label-schema.vcs-url="https://github.com/google/cloudprober" \ 21 | org.label-schema.vcs-ref=$VCS_REF \ 22 | org.label-schema.version=$VERSION \ 23 | com.microscaling.license="Apache-2.0" 24 | 25 | ENTRYPOINT ["/cloudprober", "--logtostderr"] 26 | -------------------------------------------------------------------------------- /contrib/Makefile: -------------------------------------------------------------------------------- 1 | VERSION ?= $(shell git describe --tags) 2 | DOCKER_VERSION ?= $(VERSION) 3 | GIT_COMMIT = $(strip $(shell git rev-parse --short HEAD)) 4 | GOBIN ?= ${GOPATH}/bin 5 | DOCKER_IMAGE ?= cloudprober/cloudprober-contrib 6 | CACERTS ?= /etc/ssl/certs/ca-certificates.crt 7 | SOURCES := $(shell find . -name '*.go') 8 | 9 | test: 10 | go test -v -race -covermode=atomic ./... 11 | 12 | cloudprober: 13 | make -C .. cloudprober 14 | cp ../cloudprober . 15 | 16 | bigquery_probe: $(SOURCES) 17 | CGO_ENABLED=0 go build -o bigquery_probe -ldflags "-X main.version=$(VERSION) -extldflags -static" ./gcp/cmd/bigquery_probe.go 18 | 19 | ca-certificates.crt: $(CACERTS) 20 | cp $(CACERTS) ca-certificates.crt 21 | 22 | docker_build: cloudprober bigquery_probe ca-certificates.crt Dockerfile 23 | docker build \ 24 | --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \ 25 | --build-arg VERSION=$(VERSION) \ 26 | --build-arg VCS_REF=$(GIT_COMMIT) \ 27 | -t $(DOCKER_IMAGE) . 28 | 29 | docker_push: 30 | docker tag $(DOCKER_IMAGE) $(DOCKER_IMAGE):$(DOCKER_VERSION) 31 | docker login -u "${DOCKER_USER}" -p "${DOCKER_PASS}" 32 | docker push $(DOCKER_IMAGE):$(DOCKER_VERSION) 33 | 34 | docker_push_tagged: 35 | docker tag $(DOCKER_IMAGE) $(DOCKER_IMAGE):$(DOCKER_VERSION) 36 | docker tag $(DOCKER_IMAGE) $(DOCKER_IMAGE):latest 37 | docker login -u "${DOCKER_USER}" -p "${DOCKER_PASS}" 38 | docker image push --all-tags $(DOCKER_IMAGE) 39 | 40 | clean: 41 | rm cloudprober bigquery_probe 42 | go get -u ./... 43 | -------------------------------------------------------------------------------- /contrib/gcp/bigquery/bigquery_test.go: -------------------------------------------------------------------------------- 1 | package bigquery 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "testing" 7 | ) 8 | 9 | type stubRunner struct { 10 | result, query string 11 | err error 12 | } 13 | 14 | func (f *stubRunner) Query(ctx context.Context, query string) (string, error) { 15 | f.query = query 16 | return f.result, f.err 17 | } 18 | 19 | func TestProbe(t *testing.T) { 20 | probeTests := []struct { 21 | table, expQuery string 22 | result string 23 | err error 24 | expMetrics string 25 | }{ 26 | {"", "SELECT 1", "1", nil, "bigquery_connect 1"}, 27 | {"ds.table", "SELECT COUNT(*) FROM ds.table", "500", nil, "row_count 500"}, 28 | {"", "SELECT 1", "", errors.New("connection error"), ""}, 29 | } 30 | 31 | for _, pt := range probeTests { 32 | f := stubRunner{ 33 | result: pt.result, 34 | err: pt.err, 35 | } 36 | metrics, err := Probe(context.Background(), &f, pt.table) 37 | if err != pt.err { 38 | t.Errorf("Probe(table=%#v): mismatched error, got %#v want %#v", pt.table, err, pt.err) 39 | } 40 | if metrics != pt.expMetrics { 41 | t.Errorf("Probe(table=%#v) = %#v, want %#v", pt.table, metrics, pt.expMetrics) 42 | } 43 | if f.query != pt.expQuery { 44 | t.Errorf("Probe(table=%#v): unexpected BQL, got: %#v, want %#v", pt.table, f.query, pt.expQuery) 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all deploy clean 2 | 3 | all: public public/index.html 4 | 5 | public: 6 | git worktree add public gh-pages 7 | 8 | public/index.html: public 9 | hugo 10 | 11 | deploy: 12 | cd public && \ 13 | git add --all && \ 14 | git commit -m "Deploy to gh-pages" && \ 15 | git push origin gh-pages 16 | 17 | # Removing the actual public directory confuses git and will require a git 18 | # worktree prune to fix 19 | clean: 20 | rm -rf public/* 21 | -------------------------------------------------------------------------------- /docs/archetypes/default.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "{{ replace .TranslationBaseName "-" " " | title }}" 3 | date: {{ .Date }} 4 | draft: true 5 | --- 6 | 7 | -------------------------------------------------------------------------------- /docs/config.toml: -------------------------------------------------------------------------------- 1 | baseURL = "https://cloudprober.org/" 2 | languageCode = "en-us" 3 | title = "Cloudprober" 4 | theme = "hugo-material-docs" 5 | googleAnalytics = "UA-79661-8" 6 | pygmentsUseClasses = true 7 | canonifyURLs = true 8 | [params] 9 | provider = "GitHub" 10 | repo_url = "https://github.com/google/cloudprober" 11 | 12 | [markup.goldmark.renderer] 13 | unsafe= true 14 | -------------------------------------------------------------------------------- /docs/content/_index.md: -------------------------------------------------------------------------------- 1 | --- 2 | menu: 3 | main: 4 | name: "Overview" 5 | weight: -100 6 | title: Cloudprober 7 | type: index 8 | weight: -100 9 | --- 10 | 11 | {{% content "_readme.md" %}} 12 | -------------------------------------------------------------------------------- /docs/content/how-to/additional-labels.md: -------------------------------------------------------------------------------- 1 | --- 2 | menu: 3 | main: 4 | parent: "How-Tos" 5 | weight: 26 6 | title: "Additional Labels" 7 | date: 2021-06-28T17:24:32-07:00 8 | --- 9 | ## Adding additional labels 10 | 11 | You can add additional labels to probe metrics using a probe-level field: `additional_label`. An additional label's value can be static, or it can be determined at the run-time: from the environment that the probe is running in (e.g. GCE instance labels), or target's labels. 12 | 13 | Example config [here](https://github.com/google/cloudprober/blob/master/examples/additional_label/cloudprober.cfg) demonstrates adding various types of additional labels to probe metrics. For this config (also listed below for quick rerefence): 14 | 15 | * if ingress target has label "`fqdn:app.example.com`", 16 | * and prober is running in the GCE zone `us-east1-c`, 17 | * and prober's GCE instance has label `env:prod`. 18 | 19 | Probe metrics will look like the following: 20 | ``` 21 | total{probe="my_ingress",ptype="http",metrictype="prober",env="prod",src_zone="us-east1-c",host="app.example.com"}: 90 22 | success{probe="my_ingress",ptype="http",metrictype="prober",env="prod",src_zone="us-east1-c",host="app.example.com"}: 80 23 | ``` 24 | 25 | 26 | 27 | ```bash 28 | probe { 29 | name: "my_ingress" 30 | type: HTTP 31 | 32 | targets { 33 | rds_targets { 34 | resource_path: "k8s://ingresses" 35 | filter { 36 | key: "namespace" 37 | value: "default" 38 | } 39 | } 40 | } 41 | 42 | # Static label 43 | additional_label { 44 | key: "metrictype" 45 | value: "prober" 46 | } 47 | 48 | # Label is configured at the run-time, based on the prober instance label (GCE). 49 | additional_label { 50 | key: "env" 51 | value: "{{.label_env}}" 52 | } 53 | 54 | # Label is configured at the run-time, based on the prober environment (GCE). 55 | additional_label { 56 | key: "src_zone" 57 | value: "{{.zone}}" 58 | } 59 | 60 | # Label is configured based on the target's labels. 61 | additional_label { 62 | key: "host" 63 | value: "@target.label.fqdn@" 64 | } 65 | 66 | http_probe {} 67 | } 68 | ``` 69 | (Listing source: [examples/additional_label/cloudprober.cfg](https://github.com/google/cloudprober/blob/master/examples/additional_label/cloudprober.cfg)) 70 | 71 | ## Adding your own metrics 72 | For external probes, Cloudprober also allows external programs to provide additional metrics. 73 | See [External Probe](https://cloudprober.org/how-to/external-probe) for more details. 74 | 75 | -------------------------------------------------------------------------------- /docs/content/how-to/built-in-servers.md: -------------------------------------------------------------------------------- 1 | --- 2 | menu: 3 | main: 4 | parent: "How-Tos" 5 | weight: 27 6 | title: "Built-in Servers" 7 | date: 2019-10-11T17:48:13-05:00 8 | --- 9 | Cloudprober has a few built in servers. This is useful when you are probing that 10 | a connection is working, or as a baseline to compare the probing results from 11 | your actual service to. 12 | 13 | ## HTTP 14 | 15 | {{< highlight shell >}} 16 | server { 17 | type: HTTP 18 | http_server { 19 | port: 8080 20 | } 21 | } 22 | {{< / highlight >}} 23 | 24 | This creates an HTTP server that responds on port `8080`. By default it will 25 | respond to the following endpoints: 26 | 27 | * `/healthcheck` 28 | * `/lameduck` 29 | 30 | {{< highlight shell >}} 31 | server { 32 | type: HTTP 33 | http_server { 34 | port: 8080 35 | pattern_data_handler { 36 | response_size: 1024 37 | } 38 | pattern_data_handler { 39 | response_size: 4 40 | pattern: "four" 41 | } 42 | } 43 | } 44 | {{< / highlight >}} 45 | 46 | This adds two endpoints to the HTTP server: 47 | 48 | * `/data_1024` which responds with 1024 bytes of 49 | `cloudprobercloudprobercloudprober`. 50 | * `/data_4` which responds with `four`. 51 | 52 | See 53 | [servers/http/proto/config.go](https://github.com/google/cloudprober/blob/master/servers/http/proto/config.proto) 54 | for all HTTP server configuration options. 55 | 56 | ## UDP 57 | 58 | A Cloudprober UDP server can be configured to either echo or discard packets it 59 | receives. 60 | 61 | {{< highlight shell >}} 62 | server { 63 | type: UDP 64 | udp_server { 65 | port: 85 66 | type: ECHO 67 | } 68 | } 69 | 70 | server { 71 | type: UDP 72 | udp_server { 73 | port: 90 74 | type: DISCARD 75 | } 76 | } 77 | {{< / highlight >}} 78 | 79 | See 80 | [servers/udp/proto/config.go](https://github.com/google/cloudprober/blob/master/servers/udp/proto/config.proto) 81 | for all UDP server configuration options. 82 | 83 | ## GRPC 84 | 85 | See 86 | [servers/grpc/proto/config.go](https://github.com/google/cloudprober/blob/master/servers/grpc/proto/config.proto) 87 | for all GRPC server configuration options. 88 | -------------------------------------------------------------------------------- /docs/layouts/shortcodes/content.html: -------------------------------------------------------------------------------- 1 | {{$file := .Get 0}} 2 | {{ $file | readFile | markdownify }} 3 | -------------------------------------------------------------------------------- /docs/static/CNAME: -------------------------------------------------------------------------------- 1 | cloudprober.org 2 | -------------------------------------------------------------------------------- /docs/static/diagrams/latency_distribution.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cloudprober/91ca11d906769633d418a758de0d97b59bfa0556/docs/static/diagrams/latency_distribution.png -------------------------------------------------------------------------------- /docs/static/diagrams/metrics_explorer_percentile.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cloudprober/91ca11d906769633d418a758de0d97b59bfa0556/docs/static/diagrams/metrics_explorer_percentile.png -------------------------------------------------------------------------------- /docs/static/diagrams/rds_targets.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cloudprober/91ca11d906769633d418a758de0d97b59bfa0556/docs/static/diagrams/rds_targets.png -------------------------------------------------------------------------------- /docs/static/diagrams/redis_probe_screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cloudprober/91ca11d906769633d418a758de0d97b59bfa0556/docs/static/diagrams/redis_probe_screenshot.png -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | 4 | ### 8th April 2017 5 | 6 | `.Now` has been deprecated. Hence the required minimum version of Hugo is v0.20. 7 | 8 | ### 11th May 2016 9 | 10 | #### Add templates for section lists 11 | 12 | Sections such as www.example.com/foo/ will now be rendered with a list of all pages that are part of this section. The list shows the pages' title and a summary of their content. 13 | 14 | [Show me the diff](https://github.com/digitalcraftsman/hugo-material-docs/commit/1f8393a8d4ce1b8ee3fc7d87be05895c12810494) 15 | 16 | ### 22nd March 2016 17 | 18 | #### Changing setup for Google Analytics 19 | 20 | Formerly, the tracking id for Google Analytics was set like below: 21 | 22 | ```toml 23 | [params] 24 | google_analytics = ["UA-XXXXXXXX-X", "auto"] 25 | ``` 26 | 27 | Now the theme uses Hugo's own Google Analytics config option. The variable moved outside the scope of `params` and the setup requires only the tracking id as a string: 28 | 29 | ```toml 30 | googleAnalytics = "UA-XXXXXXXX-X" 31 | ``` 32 | 33 | [Show me the diff](https://github.com/digitalcraftsman/hugo-material-docs/commit/fa10c8eef935932426d46b662a51f29a5e0d48e2) 34 | -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 Digitalcraftsman
2 | Copyright (c) 2016 Martin Donath 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including without limitation the 7 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | sell copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | IN THE SOFTWARE. -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/README.md: -------------------------------------------------------------------------------- 1 | # Material Docs 2 | 3 | A material design theme for [Hugo](https://gohugo.io). 4 | 5 | [![Screenshot](https://raw.githubusercontent.com/digitalcraftsman/hugo-material-docs/master/static/images/screen.png)](https://digitalcraftsman.github.io/hugo-material-docs/) 6 | 7 | ## Quick start 8 | 9 | Install with `git`: 10 | 11 | 12 | git clone https://github.com/digitalcraftsman/hugo-material-docs.git themes/hugo-material-docs 13 | 14 | 15 | Next, take a look in the `exampleSite` folder at. This directory contains an example config file and the content for the demo. It serves as an example setup for your documentation. 16 | 17 | Copy at least the `config.toml` in the root directory of your website. Overwrite the existing config file if necessary. 18 | 19 | Hugo includes a development server, so you can view your changes as you go - 20 | very handy. Spin it up with the following command: 21 | 22 | ``` sh 23 | hugo server 24 | ``` 25 | 26 | Now you can go to [localhost:1313](http://localhost:1313) and the Material 27 | theme should be visible. For detailed installation instructions visit the [demo](http://themes.gohugo.io/theme/material-docs/). 28 | 29 | Noteworthy changes of this theme are listed in the [changelog](https://github.com/digitalcraftsman/hugo-material-docs/blob/master/CHANGELOG.md). 30 | 31 | ## Acknowledgements 32 | 33 | A big thank you to [Martin Donath](https://github.com/squidfunk). He created the original [Material theme](https://github.com/squidfunk/mkdocs-material) for Hugo's companion [MkDocs](http://www.mkdocs.org/). This port wouldn't be possible without him. 34 | 35 | Furthermore, thanks to [Steve Francia](https://gihub.com/spf13) for creating Hugo and the [awesome community](https://github.com/spf13/hugo/graphs/contributors) around the project. 36 | 37 | ## License 38 | 39 | The theme is released under the MIT license. Read the [license](https://github.com/digitalcraftsman/hugo-material-docs/blob/master/LICENSE.md) for more information. 40 | 41 | -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/archetypes/default.md: -------------------------------------------------------------------------------- 1 | --- 2 | --- -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/exampleSite/config.toml: -------------------------------------------------------------------------------- 1 | baseurl = "https://example.org/" 2 | languageCode = "en-us" 3 | title = "Material Docs" 4 | theme = "hugo-material-docs" 5 | metadataformat = "yaml" 6 | canonifyurls = true 7 | # Enable Google Analytics by entering your tracking id 8 | googleAnalytics = "" 9 | 10 | [params] 11 | # General information 12 | author = "Digitalcraftsman" 13 | description = "A material design theme for documentations." 14 | copyright = "Released under the MIT license" 15 | 16 | # Repository 17 | provider = "GitHub" 18 | repo_url = "https://github.com/digitalcraftsman/hugo-material-docs" 19 | 20 | version = "1.0.0" 21 | logo = "images/logo.png" 22 | favicon = "" 23 | 24 | permalink = "#" 25 | 26 | # Custom assets 27 | custom_css = [] 28 | custom_js = [] 29 | 30 | # Syntax highlighting theme 31 | highlight_css = "" 32 | 33 | [params.palette] 34 | primary = "red" 35 | accent = "teal" 36 | 37 | [params.font] 38 | text = "Ubuntu" 39 | code = "Ubuntu Mono" 40 | 41 | 42 | [social] 43 | twitter = "" 44 | github = "digitalcraftsman" 45 | email = "hello@email.com" 46 | 47 | 48 | [[menu.main]] 49 | name = "Material" 50 | url = "/" 51 | weight = 0 52 | 53 | [[menu.main]] 54 | name = "Getting started" 55 | url = "getting-started/" 56 | weight = 10 57 | 58 | [[menu.main]] 59 | name = "Adding content" 60 | url = "adding-content/" 61 | weight = 20 62 | 63 | [[menu.main]] 64 | name = "Roadmap" 65 | url = "roadmap/" 66 | weight = 30 67 | 68 | [[menu.main]] 69 | name = "License" 70 | url = "license/" 71 | weight = 40 72 | 73 | 74 | [blackfriday] 75 | smartypants = true 76 | fractions = true 77 | smartDashes = true 78 | plainIDAnchors = true 79 | -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/exampleSite/content/adding-content/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | date: 2016-03-09T19:56:50+01:00 3 | title: Adding content 4 | weight: 20 5 | --- 6 | 7 | ## Hello world 8 | 9 | Let's create our first content file for your documentation. Open a terminal and add the following command for each new file you want to add. Replace `` with a general term that describes your document in detail. 10 | 11 | ```sh 12 | hugo new /filename.md 13 | ``` 14 | 15 | Visitors of your website will find the final document under `www.example.com//filename/`. 16 | 17 | Since it's possible to have multiple content files in the same section I recommend to create at least one `index.md` file per section. This ensures that users will find an index page under `www.example.com/`. 18 | 19 | ## Homepage 20 | 21 | To add content to the homepage you need to add a small indicator to the frontmatter of the content file: 22 | 23 | ```toml 24 | type: index 25 | ``` 26 | 27 | Otherwise the theme will not be able to find the corresponding content file. 28 | 29 | ## Table of contents 30 | 31 | You maybe noticed that the menu on the left contains a small table of contents of the current page. All `

` tags (`## Headline` in Markdown) will be added automatically. 32 | 33 | ## Admonitions 34 | 35 | Admonition is a handy feature that adds block-styled side content to your documentation, for example hints, notes or warnings. It can be enabled by using the corresponding [shortcodes](http://gohugo.io/extras/shortcodes/) inside your content: 36 | 37 | ```go 38 | {{}} 39 | Nothing to see here, move along. 40 | {{}} 41 | ``` 42 | 43 | This will print the following block: 44 | 45 | {{< note title="Note" >}} 46 | Nothing to see here, move along. 47 | {{< /note >}} 48 | 49 | The shortcode adds a neutral color for the note class and a red color for the warning class. You can also add a custom title: 50 | 51 | ```go 52 | {{}} 53 | Nothing to see here, move along. 54 | {{}} 55 | ``` 56 | 57 | This will print the following block: 58 | 59 | {{< warning title="Don't try this at home" >}} 60 | Nothing to see here, move along. 61 | {{< /warning >}} 62 | -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/exampleSite/content/license/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | date: 2016-03-09T20:10:46+01:00 3 | title: License 4 | weight: 40 5 | --- 6 | 7 | Copyright (c) 2016 Digitalcraftsman
8 | Copyright (c) 2016 Martin Donath 9 | 10 | Permission is hereby granted, free of charge, to any person obtaining a copy 11 | of this software and associated documentation files (the "Software"), to 12 | deal in the Software without restriction, including without limitation the 13 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 14 | sell copies of the Software, and to permit persons to whom the Software is 15 | furnished to do so, subject to the following conditions: 16 | 17 | The above copyright notice and this permission notice shall be included in 18 | all copies or substantial portions of the Software. 19 | 20 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22 | FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE 23 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 24 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 25 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 26 | IN THE SOFTWARE. 27 | 28 | -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/exampleSite/content/roadmap/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | date: 2016-03-09T20:08:11+01:00 3 | title: Roadmap 4 | weight: 30 5 | --- 6 | 7 | Quo vadis? The port of the original [Material theme](https://github.com/squidfunk/mkdocs-material) has replicated nearly all of its features. A few are still missing but I've good news: the Hugo community is actively working on this issues. Maybe with the next release of Hugo we can abandon this list. Stay tuned. 8 | 9 | ## Localization 10 | 11 | Currently, it is possible to collect all strings in a single place for easy customization. However, this only enables you to define all strings in a single language. This approach is quite limiting in terms of localization support. Therefore, I decided to wait for a native integration. This way we can avoid a second setup of all strings in your website. 12 | 13 | Keep an eye on [#1734](https://github.com/spf13/hugo/issues/1734). 14 | 15 | ## Search 16 | 17 | Beside third-party services, some hacky workarounds and Grunt-/Gulp-based scripts that only require unnecessary dependencies, future versions of Hugo will support the generation of a content index as a core feature. 18 | 19 | This approach plays well with this theme since MkDocs does the same. 20 | 21 | Keep an eye on [#1853](https://github.com/spf13/hugo/pull/1853). 22 | 23 | ## Contributing 24 | 25 | Did you found an bug or you would like to suggest a new feature? I'm open for feedback. Please open a new [issue](https://github.com/digitalcraftsman/hugo-material-docs/issues) and let me know. 26 | 27 | You're also welcome to contribute with [pull requests](https://github.com/digitalcraftsman/hugo-material-docs/pulls). 28 | -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/exampleSite/static/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cloudprober/91ca11d906769633d418a758de0d97b59bfa0556/docs/themes/hugo-material-docs/exampleSite/static/.gitkeep -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/images/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cloudprober/91ca11d906769633d418a758de0d97b59bfa0556/docs/themes/hugo-material-docs/images/screenshot.png -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/images/tn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cloudprober/91ca11d906769633d418a758de0d97b59bfa0556/docs/themes/hugo-material-docs/images/tn.png -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/layouts/404.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cloudprober/91ca11d906769633d418a758de0d97b59bfa0556/docs/themes/hugo-material-docs/layouts/404.html -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/layouts/_default/__list.html: -------------------------------------------------------------------------------- 1 | {{ partial "head" . }} 2 | 3 |
4 |
5 |
6 | 7 | 8 | 9 | 10 | 11 |
12 | {{ partial "header" . }} 13 |
14 | 15 |
16 |
17 | {{ partial "drawer" . }} 18 |
19 | 20 |
21 |
22 |

Pages in {{ .Title | singularize }}

23 | 24 | {{ range .Data.Pages }} 25 | 26 |

{{ .Title }}

27 | 28 | 29 |
30 | {{ printf "%s" .Summary | markdownify }} 31 | 32 |
33 | {{ end }} 34 | 35 | 44 |
45 |
46 | 47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 | 57 | {{ partial "footer_js" . }} 58 | -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/layouts/_default/single.html: -------------------------------------------------------------------------------- 1 | {{ partial "head" . }} 2 | 3 | {{ if (eq (trim .Site.Params.provider " " | lower) "github") | and (isset .Site.Params "repo_url") }} 4 | {{ $repo_id := replace .Site.Params.repo_url "https://github.com/" ""}} 5 | {{ .Scratch.Set "repo_id" $repo_id }} 6 | {{ end }} 7 | 8 |
9 |
10 |
11 | 12 | 13 | 14 | 15 | 16 |
17 | {{ partial "header" . }} 18 |
19 | 20 |
21 |
22 | {{ partial "drawer" . }} 23 |
24 | 25 |
26 |
27 |

{{ .Title }} {{ if .IsDraft }} (Draft){{ end }}

28 | 29 | {{ .Content }} 30 | 31 | 40 | 41 |
42 | {{ partial "footer" . }} 43 |
44 |
45 |
46 | 47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 | 57 | {{ partial "footer_js" . }} 58 | -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/layouts/index.html: -------------------------------------------------------------------------------- 1 | {{ partial "head" . }} 2 | 3 | {{ if (eq (trim .Site.Params.provider " " | lower) "github") | and (isset .Site.Params "repo_url") }} 4 | {{ $repo_id := replace .Site.Params.repo_url "https://github.com/" ""}} 5 | {{ .Scratch.Set "repo_id" $repo_id }} 6 | {{ end }} 7 | 8 |
9 |
10 |
11 | 12 | 13 | 14 | 15 | 16 |
17 | {{ partial "header" . }} 18 |
19 | 20 |
21 |
22 | {{ partial "drawer" . }} 23 |
24 | 25 |
26 |
27 | {{ range where .Site.Pages "Type" "index" }} 28 |

{{ .Title }} {{ if .IsDraft }} (Draft){{ end }}

29 | 30 | {{ .Content }} 31 | {{ end }} 32 | 33 | 42 | 43 |
44 | {{ partial "footer" . }} 45 |
46 |
47 |
48 | 49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 | 59 | {{ partial "footer_js" . }} 60 | -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/layouts/partials/footer.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cloudprober/91ca11d906769633d418a758de0d97b59bfa0556/docs/themes/hugo-material-docs/layouts/partials/footer.html -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/layouts/partials/header.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/layouts/partials/nav.html: -------------------------------------------------------------------------------- 1 | {{ $currentNode := . }} 2 | 3 | {{ range .Site.Menus.main.ByWeight }} 4 | 5 | {{ $.Scratch.Set "currentMenuEntry" . }} 6 |
  • 7 | {{ if .HasChildren }} 8 | {{ .Name | title }} 9 |
      10 | {{ range .Children }} 11 | {{ $.Scratch.Set "currentMenuEntry" . }} 12 | {{ partial "nav_link" $currentNode }} 13 | {{ end }} 14 |
    15 | {{ else }} 16 | {{ partial "nav_link" $currentNode }} 17 | {{ end }} 18 |
    • 19 | {{ end }} 20 | -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/layouts/partials/nav_link.html: -------------------------------------------------------------------------------- 1 | {{ $currentMenuEntry := .Scratch.Get "currentMenuEntry" }} 2 | {{ $isCurrent := eq (.Permalink | relURL) ($currentMenuEntry.URL | relURL) }} 3 | 4 | 5 | 6 | {{ $currentMenuEntry.Pre }} 7 | {{ $currentMenuEntry.Name }} 8 | 9 | 10 | {{ if $isCurrent }} 11 |
      12 |
    13 | {{ end }} 14 | -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/layouts/shortcodes/note.html: -------------------------------------------------------------------------------- 1 |
    2 |

    {{ .Get "title" }}

    3 |

    {{ printf "%s" .Inner | markdownify }}

    4 |
    -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/layouts/shortcodes/warning.html: -------------------------------------------------------------------------------- 1 |
    2 |

    {{ .Get "title" }}

    3 |

    {{ printf "%s" .Inner | markdownify }}

    4 |
    -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/static/fonts/icon.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cloudprober/91ca11d906769633d418a758de0d97b59bfa0556/docs/themes/hugo-material-docs/static/fonts/icon.eot -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/static/fonts/icon.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cloudprober/91ca11d906769633d418a758de0d97b59bfa0556/docs/themes/hugo-material-docs/static/fonts/icon.ttf -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/static/fonts/icon.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cloudprober/91ca11d906769633d418a758de0d97b59bfa0556/docs/themes/hugo-material-docs/static/fonts/icon.woff -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/static/images/colors.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cloudprober/91ca11d906769633d418a758de0d97b59bfa0556/docs/themes/hugo-material-docs/static/images/colors.png -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/static/images/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cloudprober/91ca11d906769633d418a758de0d97b59bfa0556/docs/themes/hugo-material-docs/static/images/favicon.ico -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/static/images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cloudprober/91ca11d906769633d418a758de0d97b59bfa0556/docs/themes/hugo-material-docs/static/images/logo.png -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/static/images/screen.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cloudprober/91ca11d906769633d418a758de0d97b59bfa0556/docs/themes/hugo-material-docs/static/images/screen.png -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/static/stylesheets/temporary.css: -------------------------------------------------------------------------------- 1 | /* This file only exists (temporarily) until the 2 | custom styling can be replaced with the 3 | implementation of the upstream project. 4 | */ 5 | 6 | blockquote { 7 | padding: 0 20px; 8 | margin: 0 0 20px; 9 | font-size: inherit; 10 | border-left: 5px solid #eee; 11 | } 12 | -------------------------------------------------------------------------------- /docs/themes/hugo-material-docs/theme.toml: -------------------------------------------------------------------------------- 1 | name = "Material Docs" 2 | license = "MIT" 3 | licenselink = "https://github.com/digitalcraftsman/hugo-material-docs/blob/master/LICENSE.md" 4 | description = "A material design theme for documentations." 5 | homepage = "https://github.com/digitalcraftsman/hugo-material-docs" 6 | tags = ["material", "documentation", "docs", "google analytics", "responsive"] 7 | features = [] 8 | min_version = 0.20 9 | 10 | [author] 11 | name = "Digitalcraftsman" 12 | homepage = "https://github.com/digitalcraftsman" 13 | 14 | # If porting an existing theme 15 | [original] 16 | name = "Martin Donath" 17 | homepage = "http://struct.cc/" 18 | repo = "https://github.com/squidfunk/mkdocs-material" 19 | -------------------------------------------------------------------------------- /examples/additional_label/cloudprober.cfg: -------------------------------------------------------------------------------- 1 | # You can add additional labels to the probe results using the probe-level field, 2 | # "additional_label". An additional label's value can be static or it can be 3 | # derived from the target labels. 4 | # 5 | # For the following config: 6 | # if ingress target has label "fqdn:app.example.com", 7 | # and prober is running in the GCE zone "us-east1-c", 8 | # and prober's GCE instance has label "env:prod". 9 | # 10 | # Probe results will look like the following: 11 | # total{probe="my_ingress",ptype="http",metrictype="prober",env="prod",src_zone="us-east1-c",host="app.example.com"}: 90 12 | # success{probe="my_ingress",ptype="http",metrictype="prober",env="prod",src_zone="us-east1-c",host="app.example.com"}: 80 13 | probe { 14 | name: "my_ingress" 15 | type: HTTP 16 | 17 | targets { 18 | rds_targets { 19 | resource_path: "k8s://ingresses" 20 | filter { 21 | key: "namespace" 22 | value: "default" 23 | } 24 | } 25 | } 26 | 27 | # Static label 28 | additional_label { 29 | key: "metrictype" 30 | value: "prober" 31 | } 32 | 33 | # Label is configured at the run time, based on the prober instance label (GCE). 34 | additional_label { 35 | key: "env" 36 | value: "{{.label_env}}" 37 | } 38 | 39 | # Label is configured at the run time, based on the prober environment (GCE). 40 | additional_label { 41 | key: "src_zone" 42 | value: "{{.zone}}" 43 | } 44 | 45 | # Label is configured based on the target labels. 46 | additional_label { 47 | key: "host" 48 | value: "@target.label.fqdn@" 49 | } 50 | 51 | http_probe {} 52 | } 53 | -------------------------------------------------------------------------------- /examples/extensions/myprober/myprobe/myprobe.proto: -------------------------------------------------------------------------------- 1 | // This protobuf defines a new cloudprober probe type. 2 | syntax = "proto2"; 3 | 4 | package myprober; 5 | 6 | option go_package="github.com/google/cloudprober/examples/extensions/myprober/myprobe"; 7 | 8 | import "github.com/google/cloudprober/probes/proto/config.proto"; 9 | 10 | message ProbeConf { 11 | // Redis operation 12 | enum Op { 13 | GET = 0; 14 | SET = 1; 15 | DELETE = 2; 16 | } 17 | required Op op = 1; 18 | 19 | // Key and value for the redis operation 20 | required string key = 2; 21 | optional string value = 3; 22 | } 23 | 24 | extend cloudprober.probes.ProbeDef { 25 | optional ProbeConf redis_probe = 200; 26 | } 27 | -------------------------------------------------------------------------------- /examples/extensions/myprober/myprober: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cloudprober/91ca11d906769633d418a758de0d97b59bfa0556/examples/extensions/myprober/myprober -------------------------------------------------------------------------------- /examples/extensions/myprober/myprober.cfg: -------------------------------------------------------------------------------- 1 | probe { 2 | name: "redis_set" 3 | type: EXTENSION 4 | interval_msec: 10000 5 | timeout_msec: 5000 6 | targets { 7 | host_names: "localhost:6379" 8 | } 9 | [myprober.redis_probe] { 10 | op: SET 11 | key: "testkey" 12 | value: "testval" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /examples/extensions/myprober/myprober.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "io/ioutil" 7 | "os" 8 | 9 | "cloud.google.com/go/compute/metadata" 10 | "github.com/golang/glog" 11 | "github.com/google/cloudprober" 12 | "github.com/google/cloudprober/config" 13 | "github.com/google/cloudprober/examples/extensions/myprober/myprobe" 14 | "github.com/google/cloudprober/probes" 15 | "github.com/google/cloudprober/web" 16 | ) 17 | 18 | var ( 19 | configFile = flag.String("config_file", "", "Config file") 20 | ) 21 | 22 | const ( 23 | configMetadataKeyName = "cloudprober_config" 24 | defaultConfigFile = "/etc/cloudprober.cfg" 25 | ) 26 | 27 | func configFileToString(fileName string) string { 28 | b, err := ioutil.ReadFile(fileName) 29 | if err != nil { 30 | glog.Exitf("Failed to read the config file: %v", err) 31 | } 32 | return string(b) 33 | } 34 | 35 | func getConfig() string { 36 | if *configFile != "" { 37 | return configFileToString(*configFile) 38 | } 39 | // On GCE first check if there is a config in custom metadata 40 | // attributes. 41 | if metadata.OnGCE() { 42 | if config, err := config.ReadFromGCEMetadata(configMetadataKeyName); err != nil { 43 | glog.Infof("Error reading config from metadata. Err: %v", err) 44 | } else { 45 | return config 46 | } 47 | } 48 | // If config not found in metadata, check default config on disk 49 | if _, err := os.Stat(defaultConfigFile); !os.IsNotExist(err) { 50 | return configFileToString(defaultConfigFile) 51 | } 52 | glog.Warningf("Config file %s not found. Using default config.", defaultConfigFile) 53 | return config.DefaultConfig() 54 | } 55 | 56 | func main() { 57 | flag.Parse() 58 | 59 | // Register stubby probe type 60 | probes.RegisterProbeType(int(myprobe.E_RedisProbe.Field), 61 | func() probes.Probe { return &myprobe.Probe{} }) 62 | 63 | err := cloudprober.InitFromConfig(getConfig()) 64 | if err != nil { 65 | glog.Exitf("Error initializing cloudprober. Err: %v", err) 66 | } 67 | 68 | // web.Init sets up web UI for cloudprober. 69 | web.Init() 70 | 71 | cloudprober.Start(context.Background()) 72 | 73 | // Wait forever 74 | select {} 75 | } 76 | -------------------------------------------------------------------------------- /examples/external/cloudprober.cfg: -------------------------------------------------------------------------------- 1 | # Run an external probe that executes a command from the current working 2 | # directory. 3 | probe { 4 | name: "redis_probe" 5 | type: EXTERNAL 6 | targets { dummy_targets {} } 7 | external_probe { 8 | mode: ONCE 9 | command: "./redis_probe" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /examples/external/cloudprober_aggregate.cfg: -------------------------------------------------------------------------------- 1 | # Run an external probe and aggregate metrics in cloudprober. 2 | probe { 3 | name: "redis_probe" 4 | type: EXTERNAL 5 | targets { dummy_targets {} } 6 | external_probe { 7 | mode: ONCE 8 | command: "./redis_probe" 9 | 10 | output_metrics_options { 11 | aggregate_in_cloudprober: true 12 | 13 | # Create distributions for get_latency_ms and set_latency_ms. 14 | dist_metric { 15 | key: "get_latency_ms" 16 | value: { 17 | explicit_buckets: "0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0,1.5,2.0" 18 | } 19 | } 20 | dist_metric { 21 | key: "set_latency_ms" 22 | value: { 23 | explicit_buckets: "0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0,1.5,2.0" 24 | } 25 | } 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /examples/external/cloudprober_server.cfg: -------------------------------------------------------------------------------- 1 | # Run an external probe and aggregate metrics in cloudprober. 2 | probe { 3 | name: "redis_probe" 4 | type: EXTERNAL 5 | targets { dummy_targets {} } 6 | external_probe { 7 | mode: SERVER 8 | command: "./redis_probe --server" 9 | 10 | output_metrics_options { 11 | aggregate_in_cloudprober: true 12 | 13 | # Create distributions for get_latency_ms and set_latency_ms. 14 | dist_metric { 15 | key: "get_latency_ms" 16 | value: { 17 | explicit_buckets: "0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0,1.5,2.0" 18 | } 19 | } 20 | dist_metric { 21 | key: "set_latency_ms" 22 | value: { 23 | explicit_buckets: "0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0,1.5,2.0" 24 | } 25 | } 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /examples/file_based_targets/cloudprober.cfg: -------------------------------------------------------------------------------- 1 | # This config demonstrates the use of file based targets. 2 | # 3 | # Probes in this config use targets defined in the "targets.textpb" file. 4 | # targets.textpb lists targets in the RDS resource format. You "don't" need to 5 | # restart cloudprober after updating the targets file; cloudprober will 6 | # automatically reload the file at the given interval (re_eval_sec). 7 | # 8 | # cloudprober --config_file=cloudprober.cfg 9 | # 10 | # Probe results will look like the following: 11 | # 12 | # total{ptype="http",probe="all-endpoints",dst="web-aa-01",cluster="aa"} 20 13 | # total{ptype="http",probe="all-endpoints",dst="web-aa-02",cluster="aa"} 20 14 | # total{ptype="http",probe="all-endpoints",dst="web-bb-01",cluster="bb"} 20 15 | # total{ptype="http",probe="dc-aa-endpoints",dst="web-aa-01"} 20 16 | # total{ptype="http",probe="dc-aa-endpoints",dst="web-aa-02"} 20 17 | 18 | probe { 19 | name: "all-endpoints" 20 | type: HTTP 21 | 22 | additional_label { 23 | key: "cluster" 24 | value: "@target.label.dc@" 25 | } 26 | 27 | targets { 28 | file_targets { 29 | file_path: "targets.textpb" 30 | re_eval_sec: 300 31 | } 32 | } 33 | } 34 | 35 | probe { 36 | name: "dc-aa-endpoints" 37 | type: HTTP 38 | 39 | targets { 40 | file_targets { 41 | file_path: "targets.textpb" 42 | filter { 43 | key: "labels.dc" 44 | value: "aa" 45 | } 46 | re_eval_sec: 300 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /examples/file_based_targets/targets.textpb: -------------------------------------------------------------------------------- 1 | # This file lists targets in the RDS resource format: 2 | # https://github.com/google/cloudprober/blob/master/rds/proto/rds.proto#L77 3 | # See cloudprober.cfg in the same directory for more details on how this file 4 | # is used. 5 | resource { 6 | name: "web-aa-01" 7 | ip: "10.11.112.3" 8 | port: 8080 9 | labels { 10 | key: "dc" 11 | value: "aa" 12 | } 13 | } 14 | 15 | resource { 16 | name: "web-aa-02" 17 | ip: "10.11.112.4" 18 | port: 8080 19 | labels { 20 | key: "dc" 21 | value: "aa" 22 | } 23 | } 24 | 25 | resource { 26 | name: "web-bb-01" 27 | ip: "10.16.110.12" 28 | port: 8080 29 | labels { 30 | key: "dc" 31 | value: "bb" 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /examples/surfacers/default_surfacers.cfg: -------------------------------------------------------------------------------- 1 | # This config file doesn't specify any surfacers. In this case, following two 2 | # surfacers are added automatically: 3 | # 4 | # surfacer { 5 | # type: FILE 6 | # } 7 | # 8 | # sufacer { 9 | # type: PROMETHEUS 10 | # } 11 | # 12 | # Note: If any surfacer is configured explicitly, default surfacers will not be 13 | # added automatically. 14 | # 15 | # File surfacer writes metrics to stdout (you can specify a file as well) in 16 | # the following format: 17 | # https://github.com/google/cloudprober/blob/ffb22ec61142a65180c42dce9b78c635e596e73f/metrics/eventmetrics.go#L192 18 | # 19 | # Prometheus surfacer exports metrics in prometheus-compatible format at the 20 | # URL: http://:/metrics 21 | # 22 | probe { 23 | name: "http_google" 24 | type: HTTP 25 | targets { 26 | host_names: "www.google.com" 27 | } 28 | interval_msec: 10000 # Probe every 10s 29 | timeout_msec: 1000 30 | } 31 | -------------------------------------------------------------------------------- /examples/surfacers/stackdriver_surfacer.cfg: -------------------------------------------------------------------------------- 1 | # This config file adds two surfacers: prometheus and stackdriver. Prometheus 2 | # surfacer exports data for all probes, while stackdriver exports data only 3 | # from one probe. 4 | # 5 | # NOTE: We need to explicitly specify prometheus surfacer since stackdriver 6 | # surfacer is configured. 7 | probe { 8 | name: "http_google" 9 | type: HTTP 10 | targets { 11 | host_names: "www.google.com" 12 | } 13 | } 14 | probe { 15 | name: "http_aws" 16 | type: HTTP 17 | targets { 18 | host_names: "aws.amazon.com" 19 | } 20 | } 21 | surfacer { 22 | type: PROMETHEUS 23 | } 24 | 25 | # Surfacer to export data to Google Cloud Monitoring (Stackdriver). It assumes 26 | # that cloudprober is running on GCP (that's why no project specification), and 27 | # VMs or GKE nodes have write access for Stackdriver. 28 | # More on stackdriver surfacer: 29 | # https://cloudprober.org/surfacers/stackdriver/ 30 | surfacer { 31 | type: STACKDRIVER 32 | 33 | # Exports data only from the "http_aws" probe. 34 | stackdriver_surfacer { 35 | allowed_metrics_regex: ".*\/http_aws\/.*" 36 | } 37 | } -------------------------------------------------------------------------------- /examples/templates/README.md: -------------------------------------------------------------------------------- 1 | ## Go templates based configs 2 | 3 | Cloudprober supports [Go text templates](https://golang.org/pkg/text/template/) based configs, bringing in some programming capabilities to the config language. This directory contains example configs that demonstrate this capability. 4 | -------------------------------------------------------------------------------- /examples/templates/cloudprober.cfg: -------------------------------------------------------------------------------- 1 | # .region is a special variable that expands to the region where Cloudprober is 2 | # running. Other such variables are: zone, project, instance, etc. More 3 | # varibles can be found here: 4 | # https://github.com/google/cloudprober/blob/master/sysvars/sysvars_gce.go 5 | {{$localregion := .region}} 6 | 7 | # Intra-region HTTP probe everywhere except in us-west1. 8 | {{if ne $localregion "us-west1"}} 9 | probe { 10 | name: "intraregion_http" 11 | type: HTTP 12 | targets { 13 | # Forwarding rules with names matching "web-fr-{{$localregion}}.*" 14 | gce_targets { 15 | forwarding_rules {} 16 | } 17 | regex: "^web-fr-{{$localregion}}.*" 18 | } 19 | interval_msec: 5000 20 | timeout_msec: 1000 21 | http_probe { 22 | port: 8080 23 | } 24 | } 25 | {{end}} 26 | -------------------------------------------------------------------------------- /examples/validators/cloudprober_validator.cfg: -------------------------------------------------------------------------------- 1 | # Following probe demonstrates the use of data integrity validator. It probes 2 | # the local HTTP server (configured below) and verifies that the response is 3 | # made up of the repeated pattern "cloudprober". 4 | # 5 | # Since the HTTP server below is configured to use the pattern "cloudprobe", 6 | # we'll see validation failures. Generated monitoring data and logs will look 7 | # like this: 8 | # labels=ptype=http,probe=http_localhost,dst=localhost total=5 success=0 latency=0.000 timeouts=0 resp-code=map:code,200:5 resp-body=map:resp validation_failure=map:validator,data-integrity:5 9 | # E1130 14:33:20.131399 27197 integrity.go:71] bytes are not in the expected format. payload[0-Replica]=[99 108 111 117 100 112 114 111 98 101 99], pattern=[99 108 111 117 100 112 114 111 98 101 114] 10 | probe { 11 | name: "http_localhost" 12 | type: HTTP 13 | targets { 14 | host_names: "localhost" 15 | } 16 | interval_msec: 10000 # Probe every 10s 17 | timeout_msec: 1000 18 | 19 | http_probe { 20 | relative_url: "/data_4096" 21 | port: 3141 22 | } 23 | # This validator will fail as HTTP server is configured to use the pattern 24 | # "cloudprobe". 25 | validator { 26 | name: "data-integrity" 27 | integrity_validator { 28 | pattern_string: "cloudprober" 29 | } 30 | } 31 | } 32 | 33 | server { 34 | type: HTTP 35 | http_server { 36 | port: 3141 37 | 38 | # Return a repeated pattern of "cloudprobe" at the URL /data_4096. 39 | pattern_data_handler { 40 | response_size: 4096 41 | pattern: "cloudprobe" 42 | } 43 | } 44 | } 45 | 46 | # Following probe demonstrates the use of HTTP status code and regex 47 | # validators. 48 | probe { 49 | name: "http_google" 50 | type: HTTP 51 | targets { 52 | host_names: "www.google.com" 53 | } 54 | interval_msec: 10000 # Probe every 10s 55 | timeout_msec: 1000 56 | 57 | # This validator should succeed. 58 | validator { 59 | name: "status_code_2xx_and_HSTS" 60 | http_validator { 61 | success_status_codes: "200-299" 62 | success_header: { 63 | name: "Strict-Transport-Security" 64 | value_regex: "max-age=31536000" 65 | } 66 | } 67 | } 68 | # This validator will fail, notice missing 'o' in our regex. 69 | validator { 70 | name: "gogle_re" 71 | regex: "gogle" 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/google/cloudprober 2 | 3 | go 1.15 4 | 5 | require ( 6 | cloud.google.com/go v0.81.0 7 | cloud.google.com/go/bigquery v1.8.0 8 | cloud.google.com/go/logging v1.4.2 9 | cloud.google.com/go/pubsub v1.8.1 10 | github.com/DataDog/datadog-api-client-go v1.2.0 11 | github.com/aws/aws-sdk-go v1.35.7 12 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b 13 | github.com/golang/protobuf v1.5.2 14 | github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 15 | github.com/hashicorp/golang-lru v0.5.3 // indirect 16 | github.com/hoisie/redis v0.0.0-20160730154456-b5c6e81454e0 17 | github.com/jstemmer/go-junit-report v0.9.1 // indirect 18 | github.com/kylelemons/godebug v1.1.0 19 | github.com/lib/pq v1.8.0 20 | github.com/miekg/dns v1.1.33 21 | golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee // indirect 22 | golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420 23 | golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c 24 | golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324 25 | google.golang.org/api v0.46.0 26 | google.golang.org/appengine v1.6.7 // indirect 27 | google.golang.org/genproto v0.0.0-20210517163617-5e0236093d7a 28 | google.golang.org/grpc v1.37.1 29 | google.golang.org/protobuf v1.26.0 30 | ) 31 | -------------------------------------------------------------------------------- /logger/logger_test.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | ) 8 | 9 | func TestEnvVarSet(t *testing.T) { 10 | varName := "TEST_VAR" 11 | 12 | testRows := []struct { 13 | v string 14 | expected bool 15 | }{ 16 | {"1", true}, 17 | {"yes", true}, 18 | {"not_set", false}, 19 | {"no", false}, 20 | {"false", false}, 21 | } 22 | 23 | for _, row := range testRows { 24 | t.Run(fmt.Sprintf("Val: %s, should be set: %v", row.v, row.expected), func(t *testing.T) { 25 | os.Unsetenv(varName) 26 | if row.v != "not_set" { 27 | os.Setenv(varName, row.v) 28 | } 29 | 30 | got := envVarSet(varName) 31 | if got != row.expected { 32 | t.Errorf("Variable set: got=%v, expected=%v", got, row.expected) 33 | } 34 | }) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /metrics/metrics.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package metrics implements data types for probes generated data. 16 | package metrics 17 | 18 | import ( 19 | "errors" 20 | "strconv" 21 | "strings" 22 | ) 23 | 24 | // Value represents any metric value 25 | type Value interface { 26 | Clone() Value 27 | Add(delta Value) error 28 | AddInt64(i int64) 29 | AddFloat64(f float64) 30 | String() string 31 | 32 | // SubtractCounter subtracts the provided "lastVal", assuming that value 33 | // represents a counter, i.e. if "value" is less than the "lastVal", we 34 | // assume that counter has been reset and don't subtract. 35 | SubtractCounter(last Value) (wasReset bool, err error) 36 | } 37 | 38 | // NumValue represents any numerical metric value, e.g. Int, Float. 39 | // It's a superset of Value interface. 40 | type NumValue interface { 41 | Value 42 | Inc() 43 | Int64() int64 44 | Float64() float64 45 | IncBy(delta NumValue) 46 | } 47 | 48 | // ParseValueFromString parses a value from its string representation 49 | func ParseValueFromString(val string) (Value, error) { 50 | c := val[0] 51 | switch { 52 | // A float value 53 | case '0' <= c && c <= '9': 54 | f, err := strconv.ParseFloat(val, 64) 55 | if err != nil { 56 | return nil, err 57 | } 58 | return NewFloat(f), nil 59 | 60 | // A map value 61 | case c == 'm': 62 | if !strings.HasPrefix(val, "map") { 63 | break 64 | } 65 | return ParseMapFromString(val) 66 | 67 | // A string value 68 | case c == '"': 69 | return NewString(strings.Trim(val, "\"")), nil 70 | 71 | // A distribution value 72 | case c == 'd': 73 | if !strings.HasPrefix(val, "dist") { 74 | break 75 | } 76 | distVal, err := ParseDistFromString(val) 77 | if err != nil { 78 | return nil, err 79 | } 80 | return distVal, nil 81 | } 82 | 83 | return nil, errors.New("unknown value type") 84 | } 85 | -------------------------------------------------------------------------------- /metrics/metrics_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package metrics 16 | 17 | import ( 18 | "testing" 19 | ) 20 | 21 | func TestParseValueFromString(t *testing.T) { 22 | var val string 23 | 24 | // Bad value, should return an error 25 | val = "234g" 26 | _, err := ParseValueFromString(val) 27 | if err == nil { 28 | t.Errorf("ParseValueFromString(%s) returned no error", val) 29 | } 30 | 31 | // Float value 32 | val = "234" 33 | v, err := ParseValueFromString(val) 34 | if err != nil { 35 | t.Errorf("ParseValueFromString(%s) returned error: %v", val, err) 36 | } 37 | if _, ok := v.(*Float); !ok { 38 | t.Errorf("ParseValueFromString(%s) returned a non-float: %v", val, v) 39 | } 40 | 41 | // String value, aggregation disabled 42 | val = "\"234\"" 43 | v, err = ParseValueFromString(val) 44 | if err != nil { 45 | t.Errorf("ParseValueFromString(%s) returned error: %v", val, err) 46 | } 47 | if _, ok := v.(String); !ok { 48 | t.Errorf("ParseValueFromString(%s) returned a non-string: %v", val, v) 49 | } 50 | 51 | // Map value 52 | val = "map:code 200:10 404:1" 53 | v, err = ParseValueFromString(val) 54 | if err != nil { 55 | t.Errorf("ParseValueFromString(%s) returned error: %v", val, err) 56 | } 57 | if _, ok := v.(*Map); !ok { 58 | t.Errorf("ParseValueFromString(%s) returned a non-map: %v", val, v) 59 | } 60 | 61 | // Dist value 62 | val = "dist:sum:899|count:221|lb:-Inf,0.5,2,7.5|bc:34,54,121,12" 63 | v, err = ParseValueFromString(val) 64 | if err != nil { 65 | t.Errorf("ParseValueFromString(%s) returned error: %v", val, err) 66 | } 67 | if _, ok := v.(*Distribution); !ok { 68 | t.Errorf("ParseValueFromString(%s) returned a non-dist: %v", val, v) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /metrics/payload/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.metrics.payload; 4 | 5 | import "github.com/google/cloudprober/metrics/proto/dist.proto"; 6 | 7 | option go_package = "github.com/google/cloudprober/metrics/payload/proto"; 8 | 9 | message OutputMetricsOptions { 10 | // MetricsKind specifies whether to treat output metrics as GAUGE or 11 | // CUMULATIVE. If left unspecified, metrics from ONCE mode probes are treated 12 | // as GAUGE and metrics from SERVER mode probes are treated as CUMULATIVE. 13 | enum MetricsKind { 14 | UNDEFINED = 0; 15 | GAUGE = 1; 16 | CUMULATIVE = 2; 17 | } 18 | optional MetricsKind metrics_kind = 1; 19 | 20 | // Additional labels (comma-separated) to attach to the output metrics, e.g. 21 | // "region=us-east1,zone=us-east1-d". ptype="external" and probe="" 22 | // are attached automatically. 23 | optional string additional_labels = 2; 24 | 25 | // Whether to aggregate metrics in Cloudprober. If enabled, Cloudprober 26 | // aggregates the metrics returned by the external probe process -- external 27 | // probe process should return metrics only since the last probe run. 28 | // Note that this option is mutually exclusive with GAUGE metrics and 29 | // cloudprober will fail during initialization if both options are enabled. 30 | optional bool aggregate_in_cloudprober = 3 [default = false]; 31 | 32 | // Metrics that should be treated as distributions. These metrics are exported 33 | // by the external probe program as comma-separated list of values, for 34 | // example: "op_latency 4.7,5.6,5.9,6.1,4.9". To be able to build distribution 35 | // from these values, these metrics should be pre-configured in external 36 | // probe: 37 | // dist_metric { 38 | // key: "op_latency" 39 | // value { 40 | // explicit_buckets: "1,2,4,8,16,32,64,128,256" 41 | // } 42 | // } 43 | map dist_metric = 4; 44 | } 45 | -------------------------------------------------------------------------------- /metrics/proto/dist.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.metrics; 4 | 5 | option go_package = "github.com/google/cloudprober/metrics/proto"; 6 | 7 | // Dist defines a Distribution data type. 8 | message Dist { 9 | oneof buckets { 10 | // Comma-separated list of lower bounds, where each lower bound is a float 11 | // value. Example: 0.5,1,2,4,8. 12 | string explicit_buckets = 1; 13 | 14 | // Exponentially growing buckets 15 | ExponentialBuckets exponential_buckets = 2; 16 | } 17 | } 18 | 19 | // ExponentialBucket defines a set of num_buckets+2 buckets: 20 | // bucket[0] covers (−Inf, 0) 21 | // bucket[1] covers [0, scale_factor) 22 | // bucket[2] covers [scale_factor, scale_factor*base) 23 | // ... 24 | // bucket[i] covers [scale_factor*base^(i−2), scale_factor*base^(i−1)) 25 | // ... 26 | // bucket[num_buckets+1] covers [scale_factor*base^(num_buckets−1), +Inf) 27 | // NB: Base must be at least 1.01. 28 | message ExponentialBuckets { 29 | optional float scale_factor = 1 [default = 1.0]; 30 | optional float base = 2 [default = 2]; 31 | optional uint32 num_buckets = 3 [default = 20]; 32 | } 33 | -------------------------------------------------------------------------------- /metrics/string.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2019 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package metrics 16 | 17 | import "errors" 18 | 19 | // String implements a value type with string storage. 20 | // It satisfies the Value interface. 21 | type String struct { 22 | s string 23 | } 24 | 25 | // NewString returns a new String with the given string value. 26 | func NewString(s string) String { 27 | return String{s: s} 28 | } 29 | 30 | // Add isn't supported for the String type, this is only to satisfy the Value 31 | // interface. 32 | func (s String) Add(val Value) error { 33 | return errors.New("string value type doesn't support Add() operation") 34 | } 35 | 36 | // SubtractCounter isn't supported for the String type, this is only to satisfy 37 | // the Value interface. 38 | func (s String) SubtractCounter(val Value) (bool, error) { 39 | return false, errors.New("string value type doesn't support SubtractCounter() operation") 40 | } 41 | 42 | // AddInt64 generates a panic for the String type. This is added only to satisfy 43 | // the Value interface. 44 | func (s String) AddInt64(i int64) { 45 | panic("String type doesn't implement AddInt64()") 46 | } 47 | 48 | // AddFloat64 generates a panic for the String type. This is added only to 49 | // satisfy the Value interface. 50 | func (s String) AddFloat64(f float64) { 51 | panic("String type doesn't implement AddFloat64()") 52 | } 53 | 54 | // String simply returns the stored string. 55 | func (s String) String() string { 56 | return "\"" + s.s + "\"" 57 | } 58 | 59 | // Clone returns the copy of receiver String. 60 | func (s String) Clone() Value { 61 | return String{s: s.s} 62 | } 63 | 64 | // IsString checks if the given value is a string. 65 | func IsString(v Value) bool { 66 | if v == nil { 67 | return false 68 | } 69 | _, ok := v.(String) 70 | return ok 71 | } 72 | -------------------------------------------------------------------------------- /metrics/string_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package metrics 16 | 17 | import ( 18 | "testing" 19 | ) 20 | 21 | func TestIsString(t *testing.T) { 22 | for _, test := range []struct { 23 | desc string 24 | input Value 25 | want bool 26 | }{ 27 | { 28 | desc: "Test with float", 29 | input: NewFloat(45.0), 30 | want: false, 31 | }, 32 | { 33 | desc: "Test with string", 34 | input: NewString("test-string"), 35 | want: true, 36 | }, 37 | { 38 | desc: "Test with nil", 39 | want: false, 40 | }, 41 | } { 42 | t.Run(test.desc, func(t *testing.T) { 43 | isString := IsString(test.input) 44 | if isString != test.want { 45 | t.Errorf("IsString(%v)=%v, want=%v", test.input, isString, test.want) 46 | } 47 | }) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /metrics/testutils/testutils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /* 16 | Package testutils provides utilities for tests. 17 | */ 18 | package testutils 19 | 20 | import ( 21 | "context" 22 | "fmt" 23 | "time" 24 | 25 | "github.com/google/cloudprober/metrics" 26 | ) 27 | 28 | // MetricsFromChannel reads metrics.EventMetrics from dataChannel with a timeout 29 | func MetricsFromChannel(dataChan chan *metrics.EventMetrics, num int, timeout time.Duration) (results []*metrics.EventMetrics, err error) { 30 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 31 | defer cancel() 32 | 33 | var timedout bool 34 | 35 | for i := 0; i < num && !timedout; i++ { 36 | select { 37 | case em := <-dataChan: 38 | results = append(results, em) 39 | case <-ctx.Done(): 40 | timedout = true 41 | } 42 | } 43 | 44 | if timedout { 45 | err = fmt.Errorf("timed out while waiting for data from dataChannel, got only %d metrics out of %d", len(results), num) 46 | } 47 | return 48 | } 49 | 50 | // MetricsMap rearranges a list of metrics into a map of map. 51 | // { 52 | // "m1": { 53 | // "target1": [val1, val2..], 54 | // "target2": [val1], 55 | // }, 56 | // "m2": { 57 | // ... 58 | // } 59 | // } 60 | func MetricsMap(ems []*metrics.EventMetrics) map[string]map[string][]*metrics.EventMetrics { 61 | results := make(map[string]map[string][]*metrics.EventMetrics) 62 | for _, em := range ems { 63 | target := em.Label("dst") 64 | for _, m := range em.MetricsKeys() { 65 | if results[m] == nil { 66 | results[m] = make(map[string][]*metrics.EventMetrics) 67 | } 68 | results[m][target] = append(results[m][target], em) 69 | } 70 | } 71 | return results 72 | } 73 | -------------------------------------------------------------------------------- /prober/cmd/client.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // This binary implements a Cloudprober gRPC client. This binary is to only 16 | // demonstrate how cloudprober can be programmed dynamically. 17 | // 18 | // go run ./cmd/client.go --server localhost:9314 --add_probe newprobe.cfg 19 | // go run ./cmd/client.go --server localhost:9314 --rm_probe newprobe 20 | package main 21 | 22 | import ( 23 | "context" 24 | "io/ioutil" 25 | 26 | "flag" 27 | "github.com/golang/glog" 28 | "github.com/golang/protobuf/proto" 29 | pb "github.com/google/cloudprober/prober/proto" 30 | spb "github.com/google/cloudprober/prober/proto" 31 | configpb "github.com/google/cloudprober/probes/proto" 32 | "google.golang.org/grpc" 33 | ) 34 | 35 | var ( 36 | server = flag.String("server", "", "gRPC server address") 37 | addProbe = flag.String("add_probe", "", "Path to probe config to add") 38 | rmProbe = flag.String("rm_probe", "", "Probe name to remove") 39 | ) 40 | 41 | func main() { 42 | flag.Parse() 43 | 44 | conn, err := grpc.Dial(*server, grpc.WithInsecure()) 45 | if err != nil { 46 | glog.Fatal(err) 47 | } 48 | 49 | client := spb.NewCloudproberClient(conn) 50 | 51 | if *addProbe != "" && *rmProbe != "" { 52 | glog.Fatal("Options --add_probe and --rm_probe cannot be specified at the same time") 53 | } 54 | 55 | if *rmProbe != "" { 56 | _, err := client.RemoveProbe(context.Background(), &pb.RemoveProbeRequest{ProbeName: rmProbe}) 57 | if err != nil { 58 | glog.Exit(err) 59 | } 60 | } 61 | 62 | if *addProbe != "" { 63 | b, err := ioutil.ReadFile(*addProbe) 64 | if err != nil { 65 | glog.Exitf("Failed to read the config file: %v", err) 66 | } 67 | 68 | glog.Infof("Read probe config: %s", string(b)) 69 | 70 | cfg := &configpb.ProbeDef{} 71 | if err := proto.UnmarshalText(string(b), cfg); err != nil { 72 | glog.Exit(err) 73 | } 74 | 75 | _, err = client.AddProbe(context.Background(), &pb.AddProbeRequest{ProbeConfig: cfg}) 76 | if err != nil { 77 | glog.Exit(err) 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /prober/proto/service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober; 4 | 5 | import "github.com/google/cloudprober/probes/proto/config.proto"; 6 | 7 | option go_package = "github.com/google/cloudprober/prober/proto"; 8 | 9 | service Cloudprober { 10 | // AddProbe adds a probe to cloudprober. Error is returned if probe is already 11 | // defined or there is an error during initialization of the probe. 12 | rpc AddProbe(AddProbeRequest) returns (AddProbeResponse) {} 13 | 14 | // RemoveProbe stops the probe and removes it from the in-memory database. 15 | rpc RemoveProbe(RemoveProbeRequest) returns (RemoveProbeResponse) {} 16 | 17 | // ListProbes lists active probes. 18 | rpc ListProbes(ListProbesRequest) returns (ListProbesResponse) {} 19 | } 20 | 21 | message AddProbeRequest { 22 | optional probes.ProbeDef probe_config = 1; 23 | } 24 | 25 | message AddProbeResponse {} 26 | 27 | message RemoveProbeRequest { 28 | optional string probe_name = 1; 29 | } 30 | 31 | message RemoveProbeResponse {} 32 | 33 | message ListProbesRequest {} 34 | 35 | message Probe { 36 | optional string name = 1; 37 | optional probes.ProbeDef config = 2; 38 | } 39 | 40 | message ListProbesResponse { 41 | repeated Probe probe = 1; 42 | } 43 | -------------------------------------------------------------------------------- /probes/dns/dns_test.cfg: -------------------------------------------------------------------------------- 1 | resolved_domain: "www.google.com." 2 | -------------------------------------------------------------------------------- /probes/dns/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.probes.dns; 4 | 5 | option go_package = "github.com/google/cloudprober/probes/dns/proto"; 6 | 7 | // DNS query types from https://en.wikipedia.org/wiki/List_of_DNS_record_types 8 | enum QueryType { 9 | NONE = 0; 10 | A = 1; 11 | NS = 2; 12 | CNAME = 5; 13 | SOA = 6; 14 | PTR = 12; 15 | MX = 15; 16 | TXT = 16; 17 | RP = 17; 18 | AFSDB = 18; 19 | SIG = 24; 20 | KEY = 25; 21 | AAAA = 28; 22 | LOC = 29; 23 | SRV = 33; 24 | NAPTR = 35; 25 | KX = 36; 26 | CERT = 37; 27 | DNAME = 39; 28 | APL = 42; 29 | DS = 43; 30 | SSHFP = 44; 31 | IPSECKEY = 45; 32 | RRSIG = 46; 33 | NSEC = 47; 34 | DNSKEY = 48; 35 | DHCID = 49; 36 | NSEC3 = 50; 37 | NSEC3PARAM = 51; 38 | TLSA = 52; 39 | HIP = 55; 40 | CDS = 59; 41 | CDNSKEY = 60; 42 | OPENPGPKEY = 61; 43 | TKEY = 249; 44 | TSIG = 250; 45 | URI = 256; 46 | CAA = 257; 47 | TA = 32768; 48 | DLV = 32769; 49 | } 50 | 51 | message ProbeConf { 52 | // Domain to use when making DNS queries 53 | optional string resolved_domain = 1 [default = "www.google.com."]; 54 | 55 | // DNS Query Type 56 | optional QueryType query_type = 3 [default = MX]; 57 | 58 | // Minimum number of answers expected. Default behavior is to return success 59 | // if DNS response status is NOERROR. 60 | optional uint32 min_answers = 4 [default = 0]; 61 | 62 | // Whether to resolve the target before making the request. If set to false, 63 | // we hand over the target directly to the DNS client. Otherwise, we resolve 64 | // the target first to an IP address. 65 | optional bool resolve_first = 5 [default = false]; 66 | } 67 | -------------------------------------------------------------------------------- /probes/external/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.probes.external; 4 | 5 | import "github.com/google/cloudprober/metrics/payload/proto/config.proto"; 6 | 7 | option go_package = "github.com/google/cloudprober/probes/external/proto"; 8 | 9 | message ProbeConf { 10 | // External probes support two mode: ONCE and SERVER. In ONCE mode, external 11 | // command is re-executed for each probe run, while in SERVER mode, command 12 | // is run in server mode, re-executed only if not running already. 13 | enum Mode { 14 | ONCE = 0; 15 | SERVER = 1; 16 | } 17 | optional Mode mode = 1 [default = ONCE]; 18 | 19 | // Command. For ONCE probes, arguments are processed for the following field 20 | // substitutions: 21 | // @probe@ Name of the probe 22 | // @target@ Hostname of the target 23 | // @address@ IP address of the target 24 | // 25 | // For example, for target ig-us-central1-a, /tools/recreate_vm -vm @target@ 26 | // will get converted to: /tools/recreate_vm -vm ig-us-central1-a 27 | required string command = 2; 28 | 29 | // Options for the SERVER mode probe requests. These options are passed on to 30 | // the external probe server as part of the ProbeRequest. Values are 31 | // substituted similar to command arguments for the ONCE mode probes. 32 | message Option { 33 | optional string name = 1; 34 | optional string value = 2; 35 | } 36 | repeated Option options = 3; 37 | 38 | // Export output as metrics, where output is the output returned by the 39 | // external probe process, over stdout for ONCE probes, and through ProbeReply 40 | // for SERVER probes. Cloudprober expects variables to be in the following 41 | // format in the output: 42 | // var1 value1 (for example: total_errors 589) 43 | optional bool output_as_metrics = 4 [default = true]; 44 | optional metrics.payload.OutputMetricsOptions output_metrics_options = 5; 45 | } 46 | -------------------------------------------------------------------------------- /probes/external/proto/server.proto: -------------------------------------------------------------------------------- 1 | // This package defines protocol for use with cloudprober's external 2 | // probe (in server mode). 3 | syntax = "proto2"; 4 | 5 | package cloudprober; 6 | 7 | option go_package = "github.com/google/cloudprober/probes/external/proto"; 8 | 9 | // ProbeRequest is the message that cloudprober sends to the external probe 10 | // server. 11 | message ProbeRequest { 12 | // The unique identifier for this request. This is unique across 13 | // an execution of the probe server. It starts at 1. 14 | required int32 request_id = 1; 15 | 16 | // How long to allow for the execution of this request, in 17 | // milliseconds. If the time limit is exceeded, the server 18 | // should abort the request, but *not* send back a reply. The 19 | // client will have to do timeouts anyway. 20 | required int32 time_limit = 2; 21 | 22 | message Option { 23 | required string name = 1; 24 | required string value = 2; 25 | } 26 | repeated Option options = 3; 27 | } 28 | 29 | // ProbeReply is the message that external probe server sends back to the 30 | // cloudprober. 31 | message ProbeReply { 32 | // The sequence number for this request. 33 | required int32 request_id = 1; 34 | 35 | // For a normal result, this is not present. 36 | // If it is present, it indicates that the probe failed. 37 | optional string error_message = 2; 38 | 39 | // The result of the probe. Cloudprober parses the payload to retrieve 40 | // variables from it. It expects variables in the following format: 41 | // var1 value1 (for example: total_errors 589) 42 | // TODO(manugarg): Add an option to export mapped variables, for example: 43 | // client-errors map:lang java:200 python:20 golang:3 44 | optional string payload = 3; 45 | } 46 | -------------------------------------------------------------------------------- /probes/grpc/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.probes.grpc; 4 | 5 | import "github.com/google/cloudprober/common/oauth/proto/config.proto"; 6 | 7 | option go_package = "github.com/google/cloudprober/probes/grpc/proto"; 8 | 9 | message ProbeConf { 10 | // Optional oauth config. For GOOGLE_DEFAULT_CREDENTIALS, use: 11 | // oauth_config: { bearer_token { gce_service_account: "default" } } 12 | optional oauth.Config oauth_config = 1; 13 | 14 | // ALTS is a gRPC security method supported by some Google services. 15 | // If enabled, peers, with the help of a handshaker service (e.g. metadata 16 | // server of GCE instances), use credentials attached to the service accounts 17 | // to authenticate each other. See 18 | // https://cloud.google.com/security/encryption-in-transit/#service_integrity_encryption 19 | // for more details. 20 | message ALTSConfig { 21 | // If provided, ALTS verifies that peer is using one of the given service 22 | // accounts. 23 | repeated string target_service_account = 1; 24 | 25 | // Handshaker service address. Default is to use the local metadata server. 26 | // For most of the ALTS use cases, default address should be okay. 27 | optional string handshaker_service_address = 2; 28 | } 29 | // If alts_config is provided, gRPC client uses ALTS for authentication and 30 | // encryption. For default alts configs, use: 31 | // alts_config: {} 32 | optional ALTSConfig alts_config = 2; 33 | 34 | enum MethodType { 35 | ECHO = 1; 36 | READ = 2; 37 | WRITE = 3; 38 | } 39 | optional MethodType method = 3 [default = ECHO]; 40 | 41 | optional int32 blob_size = 4 [default = 1024]; 42 | optional int32 num_conns = 5 [default = 2]; 43 | optional bool keep_alive = 6 [default = true]; 44 | 45 | // If connect_timeout is not specified, reuse probe timeout. 46 | optional int32 connect_timeout_msec = 7; 47 | 48 | // URI scheme allows gRPC to use different resolvers 49 | // Example URI scheme: "google-c2p:///" 50 | // See https://github.com/grpc/grpc/blob/master/doc/naming.md for more details 51 | optional string uri_scheme = 8; 52 | } 53 | -------------------------------------------------------------------------------- /probes/http/cmd/http.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2018 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // This program implements a stand-alone http prober binary using the 16 | // cloudprober/http package. It's intended to help prototype the http package 17 | // quickly and doesn't provide the facilities that cloudprober provides. 18 | package main 19 | 20 | import ( 21 | "context" 22 | "fmt" 23 | "io/ioutil" 24 | "time" 25 | 26 | "flag" 27 | "github.com/golang/glog" 28 | "github.com/golang/protobuf/proto" 29 | "github.com/google/cloudprober/metrics" 30 | "github.com/google/cloudprober/probes/http" 31 | configpb "github.com/google/cloudprober/probes/http/proto" 32 | "github.com/google/cloudprober/probes/options" 33 | "github.com/google/cloudprober/targets" 34 | ) 35 | 36 | var ( 37 | config = flag.String("config_file", "", "Config file (ProbeConf)") 38 | intervalF = flag.Duration("interval", time.Second*2, "Interval between probes") 39 | timeoutF = flag.Duration("timeout", time.Second*1, "Per-probe timeout") 40 | targetsF = flag.String("targets", "www.google.com", "Static host targets.") 41 | ) 42 | 43 | func main() { 44 | flag.Parse() 45 | 46 | c := &configpb.ProbeConf{} 47 | 48 | // If we are given a config file, read it. If not, use defaults. 49 | if *config != "" { 50 | b, err := ioutil.ReadFile(*config) 51 | if err != nil { 52 | glog.Exit(err) 53 | } 54 | if err = proto.UnmarshalText(string(b), c); err != nil { 55 | glog.Exitf("Error while parsing config protobuf %s: Err: %v", string(b), err) 56 | } 57 | } 58 | 59 | opts := &options.Options{ 60 | Interval: *intervalF, 61 | Timeout: *timeoutF, 62 | Targets: targets.StaticTargets(*targetsF), 63 | ProbeConf: c, 64 | } 65 | 66 | hp := &http.Probe{} 67 | if err := hp.Init("http_test", opts); err != nil { 68 | glog.Exitf("Error in initializing probe %s from the config. Err: %v", "http_test", err) 69 | } 70 | dataChan := make(chan *metrics.EventMetrics, 1000) 71 | go hp.Start(context.Background(), dataChan) 72 | 73 | for { 74 | em := <-dataChan 75 | fmt.Println(em.String()) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /probes/http/http_test.cfg: -------------------------------------------------------------------------------- 1 | protocol: HTTP 2 | relative_url: "/healthz" 3 | -------------------------------------------------------------------------------- /probes/ping/cmd/ping.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // This program implements a stand-alone ping prober binary using the 16 | // cloudprober/ping package. It's intended to help prototype the ping package 17 | // quickly and doesn't provide the facilities that cloudprober provides. 18 | package main 19 | 20 | import ( 21 | "context" 22 | "fmt" 23 | "io/ioutil" 24 | "time" 25 | 26 | "flag" 27 | "github.com/golang/glog" 28 | "github.com/golang/protobuf/proto" 29 | "github.com/google/cloudprober/metrics" 30 | "github.com/google/cloudprober/probes/options" 31 | "github.com/google/cloudprober/probes/ping" 32 | configpb "github.com/google/cloudprober/probes/ping/proto" 33 | "github.com/google/cloudprober/targets" 34 | ) 35 | 36 | var ( 37 | targetsF = flag.String("targets", "www.google.com,www.yahoo.com", "Comma separated list of targets") 38 | config = flag.String("config_file", "", "Config file to change ping probe options (see probes/ping/config.proto for default options)") 39 | ipVer = flag.Int("ip", 0, "IP version") 40 | ) 41 | 42 | func main() { 43 | flag.Parse() 44 | 45 | probeConfig := &configpb.ProbeConf{} 46 | if *config != "" { 47 | b, err := ioutil.ReadFile(*config) 48 | if err != nil { 49 | glog.Exit(err) 50 | } 51 | if err = proto.UnmarshalText(string(b), probeConfig); err != nil { 52 | glog.Exitf("error while parsing config: %v", err) 53 | } 54 | } 55 | 56 | opts := &options.Options{ 57 | Targets: targets.StaticTargets(*targetsF), 58 | Interval: 2 * time.Second, 59 | Timeout: time.Second, 60 | LatencyUnit: 1 * time.Millisecond, 61 | ProbeConf: probeConfig, 62 | LogMetrics: func(*metrics.EventMetrics) {}, 63 | IPVersion: *ipVer, 64 | } 65 | p := &ping.Probe{} 66 | if err := p.Init("ping", opts); err != nil { 67 | glog.Exitf("error initializing ping probe from config: %v", err) 68 | } 69 | 70 | dataChan := make(chan *metrics.EventMetrics, 1000) 71 | go p.Start(context.Background(), dataChan) 72 | 73 | for { 74 | em := <-dataChan 75 | fmt.Println(em.String()) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /probes/ping/icmpconn_nonunix.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !aix && !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd && !solaris 16 | // +build !aix,!darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris 17 | 18 | package ping 19 | 20 | import ( 21 | "net" 22 | "strconv" 23 | "time" 24 | 25 | "golang.org/x/net/icmp" 26 | ) 27 | 28 | type icmpPacketConn struct { 29 | c *icmp.PacketConn 30 | } 31 | 32 | func newICMPConn(sourceIP net.IP, ipVer int, datagramSocket bool) (icmpConn, error) { 33 | network := map[int]string{ 34 | 4: "ip4:icmp", 35 | 6: "ip6:ipv6-icmp", 36 | }[ipVer] 37 | 38 | if datagramSocket { 39 | network = "udp" + strconv.Itoa(ipVer) 40 | } 41 | 42 | c, err := icmp.ListenPacket(network, sourceIP.String()) 43 | if err != nil { 44 | return nil, err 45 | } 46 | return &icmpPacketConn{c}, nil 47 | } 48 | 49 | func (ipc *icmpPacketConn) read(buf []byte) (int, net.Addr, time.Time, error) { 50 | n, addr, err := ipc.c.ReadFrom(buf) 51 | return n, addr, time.Now(), err 52 | } 53 | 54 | func (ipc *icmpPacketConn) write(buf []byte, peer net.Addr) (int, error) { 55 | return ipc.c.WriteTo(buf, peer) 56 | } 57 | 58 | func (ipc *icmpPacketConn) setReadDeadline(deadline time.Time) { 59 | ipc.c.SetReadDeadline(deadline) 60 | } 61 | 62 | func (ipc *icmpPacketConn) close() { 63 | ipc.c.Close() 64 | } 65 | -------------------------------------------------------------------------------- /probes/ping/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.probes.ping; 4 | 5 | option go_package = "github.com/google/cloudprober/probes/ping/proto"; 6 | 7 | // Next tag: 1 8 | message ProbeConf { 9 | // Packets per probe 10 | optional int32 packets_per_probe = 6 [default = 2]; 11 | // How long to wait between two packets to the same target 12 | optional int32 packets_interval_msec = 7 [default = 25]; 13 | // Resolve targets after these many probes 14 | optional int32 resolve_targets_interval = 9 [default = 5]; // =10s 15 | // Ping payload size in bytes. It cannot be smaller than 8, number of bytes 16 | // required for the nanoseconds timestamp. 17 | optional int32 payload_size = 10 [default = 56]; 18 | // Use datagram socket for ICMP. 19 | // This option enables unprivileged pings (that is, you don't require root 20 | // privilege to send ICMP packets). Note that most of the Linux distributions 21 | // don't allow unprivileged pings by default. To enable unprivileged pings on 22 | // some Linux distributions, you may need to run the following command: 23 | // sudo sysctl -w net.ipv4.ping_group_range="0 " 24 | // net.ipv4.ping_group_range system setting takes two integers that specify 25 | // the group id range that is allowed to execute the unprivileged pings. Note 26 | // that the same setting (with ipv4 in the path) applies to IPv6 as well. 27 | optional bool use_datagram_socket = 12 [default = true]; 28 | // Disable integrity checks. To detect data courruption in the network, we 29 | // craft the outgoing ICMP packet payload in a certain format and verify that 30 | // the reply payload matches the same format. 31 | optional bool disable_integrity_check = 13 [default = false]; 32 | } 33 | -------------------------------------------------------------------------------- /probes/probes_status_tmpl.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package probes 16 | 17 | import "html/template" 18 | 19 | // StatusTmpl variable stores the HTML template suitable to generate the 20 | // probes' status for cloudprober's /status page. It expects an array of 21 | // ProbeInfo objects as input. 22 | var StatusTmpl = template.Must(template.New("statusTmpl").Parse(` 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | {{ range . }} 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 49 | 50 | 51 | 52 | 53 | {{ end }} 54 |
    NameTypeIntervalTimeoutTargetsProbe ConfLatency UnitLatency Distribution Lower Bounds (if configured)
    {{.Name}}{{.Type}}{{.Interval}}{{.Timeout}}
    {{.TargetsDesc}}
    		 43 | {{if .ProbeConf}} 44 | 
    {{.ProbeConf}}
    45 | {{else}} 46 | default 47 | {{end}} 48 |     		{{.LatencyUnit}}
    {{.LatencyDistLB}}
    55 | `)) 56 | -------------------------------------------------------------------------------- /probes/probeutils/probeutils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2019 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /* 16 | Package probeutils implements utilities that are shared across multiple probe 17 | types. 18 | */ 19 | package probeutils 20 | 21 | import ( 22 | "bytes" 23 | "fmt" 24 | ) 25 | 26 | // PatternPayload builds a payload that can be verified using VerifyPayloadPattern. 27 | // It repeats the pattern to fill the payload []byte slice. Last remaining 28 | // bytes (len(payload) mod patternSize) are left unpopulated (hence set to 0 29 | // bytes). 30 | func PatternPayload(payload, pattern []byte) { 31 | patternSize := len(pattern) 32 | for i := 0; i < len(payload); i += patternSize { 33 | copy(payload[i:], pattern) 34 | } 35 | } 36 | 37 | // VerifyPayloadPattern verifies the payload built using PatternPayload. 38 | func VerifyPayloadPattern(payload, pattern []byte) error { 39 | patternSize := len(pattern) 40 | nReplica := len(payload) / patternSize 41 | 42 | for i := 0; i < nReplica; i++ { 43 | bN := payload[0:patternSize] // Next pattern sized bytes 44 | payload = payload[patternSize:] // Shift payload for next iteration 45 | 46 | if !bytes.Equal(bN, pattern) { 47 | return fmt.Errorf("bytes are not in the expected format. payload[%d-Replica]=%v, pattern=%v", i, bN, pattern) 48 | } 49 | } 50 | 51 | if !bytes.Equal(payload, pattern[:len(payload)]) { 52 | return fmt.Errorf("last %d bytes are not in the expected format. payload=%v, expected=%v", len(payload), payload, pattern[:len(payload)]) 53 | } 54 | 55 | return nil 56 | } 57 | -------------------------------------------------------------------------------- /probes/probeutils/probeutils_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2019 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package probeutils 16 | 17 | import ( 18 | "reflect" 19 | "testing" 20 | ) 21 | 22 | func TestPayloadVerification(t *testing.T) { 23 | testBytes := []byte("test bytes") 24 | 25 | // Verify that for the larger payload sizes we get replicas of the same 26 | // bytes. 27 | for _, size := range []int{4, 256, 999, 2048, 4 * len(testBytes)} { 28 | payload := make([]byte, size) 29 | 30 | PatternPayload(payload, testBytes) 31 | 32 | var expectedBuf []byte 33 | for i := 0; i < size/len(testBytes); i++ { 34 | expectedBuf = append(expectedBuf, testBytes...) 35 | } 36 | // Remaining bytes 37 | expectedBuf = append(expectedBuf, testBytes[:size-len(expectedBuf)]...) 38 | if !reflect.DeepEqual(payload, expectedBuf) { 39 | t.Errorf("Bytes array:\n%o\n\nExpected:\n%o", payload, expectedBuf) 40 | } 41 | 42 | // Verify payload. 43 | err := VerifyPayloadPattern(payload, testBytes) 44 | if err != nil { 45 | t.Errorf("Data verification error: %v", err) 46 | } 47 | } 48 | } 49 | 50 | func benchmarkVerifyPayloadPattern(size int, b *testing.B) { 51 | testBytes := []byte("test bytes") 52 | payload := make([]byte, size) 53 | PatternPayload(payload, testBytes) 54 | 55 | for n := 0; n < b.N; n++ { 56 | VerifyPayloadPattern(payload, testBytes) 57 | } 58 | } 59 | 60 | func BenchmarkVerifyPayloadPattern56(b *testing.B) { benchmarkVerifyPayloadPattern(56, b) } 61 | func BenchmarkVerifyPayloadPattern256(b *testing.B) { benchmarkVerifyPayloadPattern(256, b) } 62 | func BenchmarkVerifyPayloadPattern1999(b *testing.B) { benchmarkVerifyPayloadPattern(1999, b) } 63 | func BenchmarkVerifyPayloadPattern9999(b *testing.B) { benchmarkVerifyPayloadPattern(9999, b) } 64 | -------------------------------------------------------------------------------- /probes/testdata/testdata.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.probes.testdata; 4 | 5 | import "github.com/google/cloudprober/probes/proto/config.proto"; 6 | 7 | option go_package = "github.com/google/cloudprober/probes/testdata"; 8 | 9 | message FancyProbe { 10 | optional string name = 1; 11 | } 12 | 13 | message AnotherFancyProbe { 14 | optional string name = 1; 15 | } 16 | 17 | extend probes.ProbeDef { 18 | optional FancyProbe fancy_probe = 200; 19 | optional AnotherFancyProbe another_fancy_probe = 201; 20 | } 21 | -------------------------------------------------------------------------------- /probes/udp/cmd/udp.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Udp_bin implements a stand-alone udp prober binary using the 16 | // cloudprober/probes/udp package. It's intended to help prototype the udp package 17 | // quickly and doesn't provide the facilities that cloudprober provides. 18 | package main 19 | 20 | import ( 21 | "context" 22 | "fmt" 23 | "io/ioutil" 24 | "time" 25 | 26 | "flag" 27 | "github.com/golang/glog" 28 | "github.com/golang/protobuf/proto" 29 | "github.com/google/cloudprober/metrics" 30 | "github.com/google/cloudprober/probes/options" 31 | "github.com/google/cloudprober/probes/udp" 32 | configpb "github.com/google/cloudprober/probes/udp/proto" 33 | "github.com/google/cloudprober/targets" 34 | ) 35 | 36 | var ( 37 | config = flag.String("config_file", "", "Config file (UDPProbe)") 38 | intervalF = flag.Duration("interval", 2*time.Second, "Interval between probes") 39 | timeoutF = flag.Duration("timeout", 1*time.Second, "Per-probe timeout") 40 | targetsF = flag.String("targets", "", "Static host targets.") 41 | ) 42 | 43 | func main() { 44 | flag.Parse() 45 | 46 | c := &configpb.ProbeConf{} 47 | 48 | // If we are given a config file, read it. If not, use defaults. 49 | if *config != "" { 50 | b, err := ioutil.ReadFile(*config) 51 | if err != nil { 52 | glog.Exit(err) 53 | } 54 | if err := proto.UnmarshalText(string(b), c); err != nil { 55 | glog.Exitf("Error while parsing config protobuf %s: Err: %v", string(b), err) 56 | } 57 | } 58 | 59 | opts := &options.Options{ 60 | Targets: targets.StaticTargets(*targetsF), 61 | Interval: *intervalF, 62 | Timeout: *timeoutF, 63 | ProbeConf: c, 64 | } 65 | 66 | up := &udp.Probe{} 67 | if err := up.Init("udp_test", opts); err != nil { 68 | glog.Exitf("Error in initializing probe %s from the config. Err: %v", "udp_test", err) 69 | } 70 | dataChan := make(chan *metrics.EventMetrics, 1000) 71 | go up.Start(context.Background(), dataChan) 72 | 73 | for { 74 | em := <-dataChan 75 | fmt.Println(em.String()) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /probes/udp/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.probes.udp; 4 | 5 | option go_package = "github.com/google/cloudprober/probes/udp/proto"; 6 | 7 | message ProbeConf { 8 | // Port to send UDP Ping to (UDP Echo). If running with the UDP server that 9 | // comes with cloudprober, it should be same as 10 | // ProberConfig.udp_echo_server_port. 11 | optional int32 port = 3 [default = 31122]; 12 | 13 | // Number of sending side ports to use. 14 | optional int32 num_tx_ports = 4 [default = 16]; 15 | 16 | // message max to account for MTU. 17 | optional int32 max_length = 5 [default = 1300]; 18 | 19 | // Payload size 20 | optional int32 payload_size = 6; 21 | 22 | // Changes the exported monitoring streams to be per port: 23 | // 1. Changes the streams names to total-per-port, success-per-port etc. 24 | // 2. Adds src_port and dst_port as stream labels. 25 | // Note that the field name is experimental and may change in the future. 26 | optional bool export_metrics_by_port = 7 [default = false]; 27 | 28 | // Whether to use all transmit ports per probe, per target. 29 | // Default is to probe each target once per probe and round-robin through the 30 | // source ports. 31 | // Setting this field to true changes the behavior to send traffic from all 32 | // ports to all targets in each probe. 33 | // For example, if num_tx_ports is set to 16, in every probe cycle, we'll send 34 | // 16 packets to every target (1 per tx port). 35 | // Note that setting this field to true will increase the probe traffic. 36 | optional bool use_all_tx_ports_per_probe = 8 [default = false]; 37 | } 38 | -------------------------------------------------------------------------------- /probes/udplistener/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.probes.udplistener; 4 | 5 | option go_package = "github.com/google/cloudprober/probes/udplistener/proto"; 6 | 7 | message ProbeConf { 8 | // Port to listen. 9 | optional int32 port = 3 [default = 32212]; 10 | 11 | // Probe response to an incoming packet: echo back or discard. 12 | enum Type { 13 | INVALID = 0; 14 | ECHO = 1; 15 | DISCARD = 2; 16 | } 17 | optional Type type = 4; 18 | 19 | // Number of packets sent in a single probe. 20 | optional int32 packets_per_probe = 5 [default = 1]; 21 | } 22 | -------------------------------------------------------------------------------- /rds/client/proto/config.proto: -------------------------------------------------------------------------------- 1 | // Configuration proto for RDS targets. 2 | // Example: 3 | // { 4 | // request { 5 | // resource_uri: "gcp://gce_instances/google.com:bbmc-stackdriver/*" 6 | // } 7 | // } 8 | syntax = "proto2"; 9 | 10 | package cloudprober.rds; 11 | 12 | import "github.com/google/cloudprober/common/oauth/proto/config.proto"; 13 | import "github.com/google/cloudprober/common/tlsconfig/proto/config.proto"; 14 | import "github.com/google/cloudprober/rds/proto/rds.proto"; 15 | 16 | option go_package = "github.com/google/cloudprober/rds/client/proto"; 17 | 18 | // ClientConf represents resource discovery service (RDS) based targets. 19 | // Next tag: 6 20 | message ClientConf { 21 | message ServerOptions { 22 | optional string server_address = 1; 23 | 24 | // Optional oauth config for authentication. 25 | optional oauth.Config oauth_config = 2; 26 | 27 | // TLS config, it can be used to: 28 | // - Specify a CA cert for server cert verification: 29 | // tls_config { 30 | // ca_cert_file: "...." 31 | // } 32 | // 33 | // - Specify client's TLS cert and key: 34 | // tls_config { 35 | // tls_cert_file: "..." 36 | // tls_key_file: "..." 37 | // } 38 | optional tlsconfig.TLSConfig tls_config = 3; 39 | } 40 | optional ServerOptions server_options = 1; 41 | 42 | required ListResourcesRequest request = 2; 43 | 44 | // How often targets should be evaluated. Any number less than or equal to 0 45 | // will result in no target caching (targets will be reevaluated on demand). 46 | // Note that individual target types may have their own caches implemented 47 | // (specifically GCE instances/forwarding rules). This does not impact those 48 | // caches. 49 | optional int32 re_eval_sec = 3 [default = 30]; 50 | } 51 | -------------------------------------------------------------------------------- /rds/examples/cloudprober_rds.cfg: -------------------------------------------------------------------------------- 1 | # Run cloudprober in RDS (resource discovery service) server mode. 2 | 3 | rds_server { 4 | provider { 5 | id: "gcp" 6 | gcp_config { 7 | project: "google.com:test-project-1" 8 | project: "google.com:test-project-2" 9 | 10 | gce_instances { 11 | zone_filter: "name = us-central1-*" 12 | re_eval_sec: 60 13 | } 14 | 15 | rtc_variables { 16 | rtc_config { 17 | name: "lame-duck-targets" 18 | } 19 | } 20 | 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /rds/file/proto/config.proto: -------------------------------------------------------------------------------- 1 | // Configuration proto for Kubernetes provider. 2 | // 3 | // Example provider config: 4 | // { 5 | // pods {} 6 | // } 7 | // 8 | // In probe config: 9 | // probe { 10 | // targets{ 11 | // rds_targets { 12 | // resource_path: "k8s://pods" 13 | // filter { 14 | // key: "namespace" 15 | // value: "default" 16 | // } 17 | // filter { 18 | // key: "name" 19 | // value: "cloudprober.*" 20 | // } 21 | // } 22 | // } 23 | // } 24 | syntax = "proto2"; 25 | 26 | package cloudprober.rds.file; 27 | 28 | import "github.com/google/cloudprober/rds/proto/rds.proto"; 29 | 30 | option go_package = "github.com/google/cloudprober/rds/file/proto"; 31 | 32 | // File provider config. 33 | message ProviderConfig { 34 | // File that contains resources in either textproto or json format. 35 | // Example in textproto format: 36 | // 37 | // resource { 38 | // name: "switch-xx-01" 39 | // ip: "10.11.112.3" 40 | // port: 8080 41 | // labels { 42 | // key: "device_type" 43 | // value: "switch" 44 | // } 45 | // } 46 | // resource { 47 | // name: "switch-yy-01" 48 | // ip: "10.16.110.12" 49 | // port: 8080 50 | // } 51 | repeated string file_path = 1; 52 | 53 | enum Format { 54 | UNSPECIFIED = 0; // Determine format using file extension/ 55 | TEXTPB = 1; // Text proto format (.textpb). 56 | JSON = 2; // JSON proto format (.json). 57 | } 58 | optional Format format = 2; 59 | 60 | // If specified, file will be re-read at the given interval. 61 | optional int32 re_eval_sec = 3; 62 | 63 | // Whenever possible, we reload a file only if it has been modified since the 64 | // last load. If following option is set, mod time check is disabled. 65 | // Note that mod-time check doesn't work for GCS. 66 | optional bool disable_modified_time_check = 4; 67 | } 68 | 69 | message FileResources { 70 | repeated .cloudprober.rds.Resource resource = 1; 71 | } 72 | -------------------------------------------------------------------------------- /rds/file/testdata/targets.json: -------------------------------------------------------------------------------- 1 | { 2 | "resource": [ 3 | { 4 | "name": "switch-xx-1", 5 | "ip": "10.1.1.1", 6 | "port": 8080, 7 | "labels": { 8 | "device_type": "switch", 9 | "cluster": "xx" 10 | } 11 | }, 12 | { 13 | "name": "switch-xx-2", 14 | "ip": "10.1.1.2", 15 | "port": 8081, 16 | "labels": { 17 | "cluster": "xx" 18 | } 19 | }, 20 | { 21 | "name": "switch-yy-1", 22 | "ip": "10.1.2.1", 23 | "port": 8080 24 | }, 25 | { 26 | "name": "switch-zz-1", 27 | "ip": "::aaa:1", 28 | "port": 8080 29 | } 30 | ] 31 | } 32 | -------------------------------------------------------------------------------- /rds/file/testdata/targets1.textpb: -------------------------------------------------------------------------------- 1 | resource { 2 | name: "switch-xx-1" 3 | ip: "10.1.1.1" 4 | port: 8080 5 | labels { 6 | key: "device_type" 7 | value: "switch" 8 | } 9 | labels { 10 | key: "cluster" 11 | value: "xx" 12 | } 13 | } 14 | 15 | resource { 16 | name: "switch-xx-2" 17 | ip: "10.1.1.2" 18 | port: 8081 19 | labels { 20 | key: "cluster" 21 | value: "xx" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /rds/file/testdata/targets2.textpb: -------------------------------------------------------------------------------- 1 | resource { 2 | name: "switch-yy-1" 3 | ip: "10.1.2.1" 4 | port: 8080 5 | } 6 | 7 | resource { 8 | name: "switch-zz-1" 9 | ip: "::aaa:1" 10 | port: 8080 11 | } 12 | -------------------------------------------------------------------------------- /rds/gcp/pubsub_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package gcp 16 | 17 | import ( 18 | "testing" 19 | "time" 20 | 21 | "github.com/golang/protobuf/proto" 22 | pb "github.com/google/cloudprober/rds/proto" 23 | ) 24 | 25 | func TestListMessages(t *testing.T) { 26 | lister := &pubsubMsgsLister{ 27 | cache: make(map[string]map[string]time.Time), 28 | } 29 | lister.cache["s1"] = map[string]time.Time{ 30 | "m1": time.Now().Add(-6 * time.Minute), 31 | "m2": time.Now().Add(-1 * time.Minute), 32 | } 33 | lister.cache["s2"] = map[string]time.Time{ 34 | "m3": time.Now().Add(-1 * time.Minute), 35 | } 36 | 37 | // No filter 38 | want := []string{"m1", "m2", "m3"} 39 | resources, err := lister.listResources(nil) 40 | if err != nil { 41 | t.Errorf("Got error while listing resources: %v", err) 42 | } 43 | compareResources(t, resources, want) 44 | 45 | // Subscription s1, updated within 5m 46 | want = []string{"m2"} 47 | resources, err = lister.listResources(&pb.ListResourcesRequest{ 48 | Filter: []*pb.Filter{ 49 | &pb.Filter{ 50 | Key: proto.String("subscription"), 51 | Value: proto.String("s1"), 52 | }, 53 | &pb.Filter{ 54 | Key: proto.String("updated_within"), 55 | Value: proto.String("5m"), 56 | }, 57 | }, 58 | }) 59 | 60 | if err != nil { 61 | t.Errorf("Got error while listing resources: %v", err) 62 | } 63 | compareResources(t, resources, want) 64 | 65 | // Subscription s1 and s2 66 | want = []string{"m1", "m2", "m3"} 67 | resources, _ = lister.listResources(&pb.ListResourcesRequest{ 68 | Filter: []*pb.Filter{ 69 | &pb.Filter{ 70 | Key: proto.String("subscription"), 71 | Value: proto.String("s"), 72 | }, 73 | }, 74 | }) 75 | 76 | if err != nil { 77 | t.Errorf("Got error while listing resources: %v", err) 78 | } 79 | compareResources(t, resources, want) 80 | } 81 | -------------------------------------------------------------------------------- /rds/gcp/testdata/instances.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "instance-list", 3 | "items": [ 4 | { 5 | "name": "ig-us-central1-a-00-abcd", 6 | "labels": { 7 | "app": "cloudprober", 8 | "shard": "00" 9 | }, 10 | "networkInterfaces": [ 11 | { 12 | "accessConfigs": [ 13 | { 14 | "kind": "compute#accessConfig", 15 | "name": "external-nat", 16 | "natIP": "194.197.208.201", 17 | "networkTier": "PREMIUM", 18 | "type": "ONE_TO_ONE_NAT" 19 | } 20 | ], 21 | "ipv6AccessConfigs": [ 22 | { 23 | "externalIpv6": "2600:2d00:4030:a47:c0a8:2110:1:0" 24 | } 25 | ], 26 | "fingerprint": "DxvgIvO1QQ8=", 27 | "kind": "compute#networkInterface", 28 | "name": "nic0", 29 | "network": "https://www.googleapis.com/compute/v1/projects/proj1/global/networks/default", 30 | "networkIP": "10.0.0.2", 31 | "ipv6Address": "2600:2d00:4030:a47:c0a8:2110:0:0", 32 | "subnetwork": "https://www.googleapis.com/compute/v1/projects/proj1/regions/us-central1/subnetworks/default" 33 | }, 34 | { 35 | "accessConfigs": [ 36 | { 37 | "natIP": "194.197.208.202" 38 | } 39 | ], 40 | "networkIP": "10.0.0.3", 41 | "ipv6Address": "2600:2d00:4030:a47:c0a8:2110:0:1" 42 | } 43 | ] 44 | }, 45 | { 46 | "name": "ig-us-central1-a-01-efgh", 47 | "labels": { 48 | "app": "cloudprober", 49 | "shard": "01" 50 | }, 51 | "networkInterfaces": [ 52 | { 53 | "accessConfigs": [ 54 | { 55 | "natIP": "194.197.209.202" 56 | } 57 | ], 58 | "networkIP": "10.0.1.3" 59 | } 60 | ] 61 | } 62 | ] 63 | } 64 | -------------------------------------------------------------------------------- /rds/gcp/testdata/zones.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "zones-list", 3 | "items": [ 4 | { 5 | "name": "us-central1-a" 6 | }, 7 | { 8 | "name": "us-central1-b" 9 | } 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /rds/kubernetes/kubernetes_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package kubernetes 16 | 17 | import ( 18 | "testing" 19 | ) 20 | 21 | func TestHTTPRequest(t *testing.T) { 22 | c := &client{ 23 | bearer: "testtoken", 24 | apiHost: "testHost", 25 | } 26 | 27 | testURL := "/test-url" 28 | 29 | req, err := c.httpRequest(testURL) 30 | 31 | if err != nil { 32 | t.Errorf("Unexpected error while creating HTTP request from URL (%s): %v", testURL, err) 33 | } 34 | 35 | if req.Host != c.apiHost { 36 | t.Errorf("Got host = %s, expected = %s", req.Host, c.apiHost) 37 | } 38 | 39 | if req.URL.Path != testURL { 40 | t.Errorf("Got URL path = %s, expected = %s", req.URL.Path, testURL) 41 | } 42 | 43 | if req.Header.Get("Authorization") != c.bearer { 44 | t.Errorf("Got Authorization Header = %s, expected = %s", req.Header.Get("Authorization"), c.bearer) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /rds/kubernetes/proto/config.proto: -------------------------------------------------------------------------------- 1 | // Configuration proto for Kubernetes provider. 2 | // 3 | // Example provider config: 4 | // { 5 | // pods {} 6 | // } 7 | // 8 | // In probe config: 9 | // probe { 10 | // targets{ 11 | // rds_targets { 12 | // resource_path: "k8s://pods" 13 | // filter { 14 | // key: "namespace" 15 | // value: "default" 16 | // } 17 | // filter { 18 | // key: "name" 19 | // value: "cloudprober.*" 20 | // } 21 | // } 22 | // } 23 | // } 24 | syntax = "proto2"; 25 | 26 | package cloudprober.rds.kubernetes; 27 | 28 | import "github.com/google/cloudprober/common/tlsconfig/proto/config.proto"; 29 | 30 | option go_package = "github.com/google/cloudprober/rds/kubernetes/proto"; 31 | 32 | message Pods {} 33 | 34 | message Endpoints {} 35 | 36 | message Services {} 37 | 38 | message Ingresses {} 39 | 40 | // Kubernetes provider config. 41 | message ProviderConfig { 42 | // Namespace to list resources for. If not specified, we default to all 43 | // namespaces. 44 | optional string namespace = 1; 45 | 46 | // Pods discovery options. This field should be declared for the pods 47 | // discovery to be enabled. 48 | optional Pods pods = 2; 49 | 50 | // Endpoints discovery options. This field should be declared for the 51 | // endpoints discovery to be enabled. 52 | optional Endpoints endpoints = 3; 53 | 54 | // Services discovery options. This field should be declared for the 55 | // services discovery to be enabled. 56 | optional Services services = 4; 57 | 58 | // Ingresses discovery options. This field should be declared for the 59 | // ingresses discovery to be enabled. 60 | // Note: Ingress support is experimental and may change in future. 61 | optional Ingresses ingresses = 5; 62 | 63 | // Kubernetes API server address. If not specified, we assume in-cluster mode 64 | // and get it from the local environment variables. 65 | optional string api_server_address = 91; 66 | 67 | // TLS config to authenticate communication with the API server. 68 | optional tlsconfig.TLSConfig tls_config = 93; 69 | 70 | // How often resources should be evaluated/expanded. 71 | optional int32 re_eval_sec = 99 [default = 60]; // default 1 min 72 | } 73 | -------------------------------------------------------------------------------- /rds/server/cmd/server.go: -------------------------------------------------------------------------------- 1 | // This binary implements a stand-alone ResourceDiscovery server. 2 | package main 3 | 4 | import ( 5 | "context" 6 | "io/ioutil" 7 | "net" 8 | 9 | "flag" 10 | "github.com/golang/glog" 11 | "github.com/golang/protobuf/proto" 12 | "github.com/google/cloudprober/logger" 13 | "github.com/google/cloudprober/rds/server" 14 | configpb "github.com/google/cloudprober/rds/server/proto" 15 | "google.golang.org/grpc" 16 | "google.golang.org/grpc/credentials" 17 | ) 18 | 19 | var ( 20 | config = flag.String("config_file", "", "Config file (ServerConf)") 21 | addr = flag.String("addr", ":0", "Port for the gRPC server") 22 | tlsCertFile = flag.String("tls_cert_file", ":0", "Port for the gRPC server") 23 | tlsKeyFile = flag.String("tls_key_file", ":0", "Port for the gRPC server") 24 | ) 25 | 26 | func main() { 27 | flag.Parse() 28 | 29 | c := &configpb.ServerConf{} 30 | 31 | // If we are given a config file, read it. If not, use defaults. 32 | if *config != "" { 33 | b, err := ioutil.ReadFile(*config) 34 | if err != nil { 35 | glog.Exit(err) 36 | } 37 | if err := proto.UnmarshalText(string(b), c); err != nil { 38 | glog.Exitf("Error while parsing config protobuf %s: Err: %v", string(b), err) 39 | } 40 | } 41 | 42 | grpcLn, err := net.Listen("tcp", *addr) 43 | if err != nil { 44 | glog.Exitf("error while creating listener for default gRPC server: %v", err) 45 | } 46 | 47 | // Create a gRPC server for RDS service. 48 | var serverOpts []grpc.ServerOption 49 | 50 | if *tlsCertFile != "" { 51 | creds, err := credentials.NewServerTLSFromFile(*tlsCertFile, *tlsKeyFile) 52 | if err != nil { 53 | glog.Exitf("error initializing gRPC server TLS credentials: %v", err) 54 | } 55 | serverOpts = append(serverOpts, grpc.Creds(creds)) 56 | } 57 | 58 | grpcServer := grpc.NewServer(serverOpts...) 59 | srv, err := server.New(context.Background(), c, nil, &logger.Logger{}) 60 | if err != nil { 61 | glog.Exit(err) 62 | } 63 | srv.RegisterWithGRPC(grpcServer) 64 | 65 | grpcServer.Serve(grpcLn) 66 | } 67 | -------------------------------------------------------------------------------- /rds/server/filter/filter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2019 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package filter 16 | 17 | import ( 18 | "testing" 19 | ) 20 | 21 | func TestLabelsFilter(t *testing.T) { 22 | for _, testData := range []struct { 23 | testLabels []map[string]string 24 | labelFilters map[string]string 25 | expectError bool 26 | matchCount int 27 | }{ 28 | { 29 | testLabels: []map[string]string{ 30 | {"lA": "vAA", "lB": "vB"}, 31 | {"lA": "vAB", "lB": "vB"}, 32 | }, 33 | labelFilters: map[string]string{"lA": "vA.", "lB": "vB"}, 34 | expectError: false, 35 | matchCount: 2, 36 | }, 37 | { 38 | testLabels: []map[string]string{ 39 | {"lA": "vAA", "lB": "vB"}, // Only this will match. 40 | {"lA": "vBB", "lB": "vB"}, 41 | }, 42 | labelFilters: map[string]string{"lA": "vA.", "lB": "vB"}, 43 | expectError: false, 44 | matchCount: 1, 45 | }, 46 | { 47 | // Label lC not on any instance, no match. 48 | testLabels: []map[string]string{ 49 | {"lA": "vAA", "lB": "vB"}, 50 | {"lA": "vBB", "lB": "vB"}, 51 | }, 52 | labelFilters: map[string]string{"lC": "vC.", "lB": "vB"}, 53 | expectError: false, 54 | matchCount: 0, 55 | }, 56 | } { 57 | lf, err := NewLabelsFilter(testData.labelFilters) 58 | if err != nil { 59 | t.Errorf("Got unexpected error while adding a label filter: %v", err) 60 | } 61 | 62 | var testMatchCount int 63 | for _, testInstance := range testData.testLabels { 64 | t.Logf("Matching instance with labels: %v", testInstance) 65 | if lf.Match(testInstance, nil) { 66 | testMatchCount++ 67 | } 68 | } 69 | 70 | if testMatchCount != testData.matchCount { 71 | t.Errorf("Match count doesn't match with expected match count. Got=%d, Expected=%d", testMatchCount, testData.matchCount) 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /rds/server/filter/utils.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | pb "github.com/google/cloudprober/rds/proto" 8 | ) 9 | 10 | // Filters encapsulates all types of filters. Its main purpose is to serve as a 11 | // return type for ParseFilters. 12 | type Filters struct { 13 | RegexFilters map[string]*RegexFilter 14 | *LabelsFilter 15 | *FreshnessFilter 16 | } 17 | 18 | // ParseFilters parses filter protobufs into Filters struct. Filters are parsed 19 | // based on the following criteria: 20 | // - There can be multiple regex filters. Keys for these filters should be 21 | // provided through the regexFilterKeys argument. 22 | // - Labels filter keys always starts with the prefix "labels.". 23 | // - There can be only one freshness filter, key for which should be provided 24 | // through the freshnessFilterKey argument. 25 | func ParseFilters(filters []*pb.Filter, regexFilterKeys []string, freshnessFilterKey string) (*Filters, error) { 26 | r := &Filters{ 27 | RegexFilters: make(map[string]*RegexFilter), 28 | } 29 | 30 | // Initialize r.RegexFilters with expected regex filter keys for quick lookup. 31 | for _, k := range regexFilterKeys { 32 | r.RegexFilters[k] = nil 33 | } 34 | 35 | labels := make(map[string]string) 36 | 37 | for _, f := range filters { 38 | 39 | // If we expect this filter to be a regex filter. 40 | if _, ok := r.RegexFilters[f.GetKey()]; ok { 41 | rf, err := NewRegexFilter(f.GetValue()) 42 | if err != nil { 43 | return nil, fmt.Errorf("filter: error creating regex filter from: %s=%s, err: %v", f.GetKey(), f.GetValue(), err) 44 | } 45 | r.RegexFilters[f.GetKey()] = rf 46 | continue 47 | } 48 | 49 | // If we expect this filter to be a freshness filter. 50 | if f.GetKey() == freshnessFilterKey { 51 | ff, err := NewFreshnessFilter(f.GetValue()) 52 | if err != nil { 53 | return nil, fmt.Errorf("filter: error creating freshness filter from: %s=%s, err: %v", f.GetKey(), f.GetValue(), err) 54 | } 55 | r.FreshnessFilter = ff 56 | continue 57 | } 58 | 59 | // If it is a labels filter (starting with labels.). 60 | // Note: labels. format matches with gcloud's filter options. 61 | if strings.HasPrefix(f.GetKey(), "labels.") { 62 | labels[strings.TrimPrefix(f.GetKey(), "labels.")] = f.GetValue() 63 | continue 64 | } 65 | 66 | // Unexpected filter key. 67 | return nil, fmt.Errorf("unsupported filter key: %s", f.GetKey()) 68 | } 69 | 70 | if len(labels) != 0 { 71 | var err error 72 | if r.LabelsFilter, err = NewLabelsFilter(labels); err != nil { 73 | return nil, fmt.Errorf("filter: error creating labels filter from: %v, err: %v", labels, err) 74 | } 75 | } 76 | 77 | return r, nil 78 | } 79 | -------------------------------------------------------------------------------- /rds/server/filter/utils_test.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "reflect" 5 | "sort" 6 | "testing" 7 | 8 | "github.com/golang/protobuf/proto" 9 | pb "github.com/google/cloudprober/rds/proto" 10 | ) 11 | 12 | func TestParseFilters(t *testing.T) { 13 | tests := []struct { 14 | desc string 15 | reqFilters map[string]string 16 | regexFilterKeys []string 17 | freshnessFilterKey string 18 | wantReFilters []string 19 | wantLabelsFilter bool 20 | wantFreshnessFilter bool 21 | wantErr bool 22 | }{ 23 | { 24 | desc: "Error invalid filter key", 25 | reqFilters: map[string]string{"random_key": "random_value"}, 26 | regexFilterKeys: []string{"name"}, 27 | wantErr: true, 28 | }, 29 | { 30 | desc: "Pass with 2 regex filters, labels filter and a freshness filter", 31 | reqFilters: map[string]string{ 32 | "name": "x.*", 33 | "namespace": "y", 34 | "labels.app": "cloudprober", 35 | "updated_within": "5m", 36 | }, 37 | regexFilterKeys: []string{"name", "namespace"}, 38 | freshnessFilterKey: "updated_within", 39 | wantReFilters: []string{"name", "namespace"}, 40 | wantLabelsFilter: true, 41 | wantFreshnessFilter: true, 42 | }, 43 | } 44 | 45 | for _, test := range tests { 46 | t.Run(test.desc, func(t *testing.T) { 47 | var reqFiltersPB []*pb.Filter 48 | for k, v := range test.reqFilters { 49 | reqFiltersPB = append(reqFiltersPB, &pb.Filter{Key: proto.String(k), Value: proto.String(v)}) 50 | } 51 | 52 | allFilters, err := ParseFilters(reqFiltersPB, test.regexFilterKeys, test.freshnessFilterKey) 53 | 54 | if err != nil { 55 | if !test.wantErr { 56 | t.Errorf("Got unexpected error while parsing filters: %v", err) 57 | } 58 | return 59 | } 60 | 61 | var reFilterKeys []string 62 | for k := range allFilters.RegexFilters { 63 | reFilterKeys = append(reFilterKeys, k) 64 | } 65 | 66 | sort.Strings(reFilterKeys) 67 | sort.Strings(test.wantReFilters) 68 | if !reflect.DeepEqual(reFilterKeys, test.wantReFilters) { 69 | t.Errorf("regex filters, got=%v, want=%v", reFilterKeys, test.wantReFilters) 70 | } 71 | 72 | if (allFilters.LabelsFilter != nil) != test.wantLabelsFilter { 73 | t.Errorf("labels filters, got=%v, wantLabelsFilter=%v", allFilters.LabelsFilter, test.wantLabelsFilter) 74 | } 75 | 76 | if (allFilters.FreshnessFilter != nil) != test.wantFreshnessFilter { 77 | t.Errorf("freshness filter filters, got=%v, wantFreshnessFilter=%v", allFilters.FreshnessFilter, test.wantFreshnessFilter) 78 | } 79 | }) 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /rds/server/proto/config.proto: -------------------------------------------------------------------------------- 1 | // Configuration proto for ResourceDiscovery (rds) server. 2 | // Example config: 3 | // 4 | // provider { 5 | // id: "gcp" 6 | // gcp_config { 7 | // project_id: 'test-project-id' 8 | // gce_instances {} 9 | // forwarding_rules {} 10 | // } 11 | // } 12 | syntax = "proto2"; 13 | 14 | package cloudprober.rds; 15 | 16 | import "github.com/google/cloudprober/rds/file/proto/config.proto"; 17 | import "github.com/google/cloudprober/rds/gcp/proto/config.proto"; 18 | import "github.com/google/cloudprober/rds/kubernetes/proto/config.proto"; 19 | 20 | option go_package = "github.com/google/cloudprober/rds/server/proto"; 21 | 22 | message ServerConf { 23 | // List of providers that server supports. 24 | repeated Provider provider = 1; 25 | } 26 | 27 | message Provider { 28 | // Provider identifier, e.g. "gcp". Server routes incoming requests to various 29 | // providers based on this id. 30 | optional string id = 1; 31 | 32 | oneof config { 33 | file.ProviderConfig file_config = 4; 34 | gcp.ProviderConfig gcp_config = 2; 35 | kubernetes.ProviderConfig kubernetes_config = 3; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /rds/server/server_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "reflect" 6 | "testing" 7 | 8 | "github.com/golang/protobuf/proto" 9 | pb "github.com/google/cloudprober/rds/proto" 10 | ) 11 | 12 | type testProvider struct { 13 | resources []*pb.Resource 14 | } 15 | 16 | func (tp *testProvider) ListResources(*pb.ListResourcesRequest) (*pb.ListResourcesResponse, error) { 17 | return &pb.ListResourcesResponse{ 18 | Resources: tp.resources, 19 | }, nil 20 | } 21 | 22 | func TestListResources(t *testing.T) { 23 | testResources := []*pb.Resource{ 24 | &pb.Resource{ 25 | Name: proto.String("testR1"), 26 | Ip: proto.String("IP1"), 27 | }, 28 | &pb.Resource{ 29 | Name: proto.String("testR2"), 30 | Ip: proto.String("IP2"), 31 | }, 32 | } 33 | srv := &Server{ 34 | providers: map[string]Provider{ 35 | "test_provider": &testProvider{ 36 | resources: testResources, 37 | }, 38 | }, 39 | } 40 | res, err := srv.ListResources(context.Background(), &pb.ListResourcesRequest{ 41 | Provider: proto.String("test_provider"), 42 | ResourcePath: proto.String("rp"), 43 | }) 44 | if err != nil { 45 | t.Errorf("Got unexpected error while listing test resources: %v", err) 46 | } 47 | if !reflect.DeepEqual(res.Resources, testResources) { 48 | t.Errorf("Didn't get expected resource. Got=%v, Want=%v", res.Resources, testResources) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /servers/README.md: -------------------------------------------------------------------------------- 1 | This directory is for helper servers available for use with Cloudprober. 2 | Cloudprober itself is a prober daemon with primary purpose to run probes but 3 | since it runs on all of our probing VMs, it provides a perfect platform to run 4 | servers that can act as the backend for probes that need to access something on 5 | cloud. 6 | -------------------------------------------------------------------------------- /servers/external/external.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package external adds support for an external server. 16 | package external 17 | 18 | import ( 19 | "context" 20 | "fmt" 21 | "os/exec" 22 | 23 | "github.com/google/cloudprober/logger" 24 | "github.com/google/cloudprober/metrics" 25 | configpb "github.com/google/cloudprober/servers/external/proto" 26 | "github.com/google/shlex" 27 | ) 28 | 29 | // Server implements a external command runner. 30 | type Server struct { 31 | c *configpb.ServerConf 32 | l *logger.Logger 33 | 34 | cmdName string 35 | cmdArgs []string 36 | cmd *exec.Cmd 37 | } 38 | 39 | // TODO 40 | // 1. Export health status (pid-file OR by monitoring the process started.) 41 | // 2. Add support for command-line substitution. 42 | // 2. Add auto-restart support. 43 | 44 | // New creates a new external server. 45 | func New(initCtx context.Context, c *configpb.ServerConf, l *logger.Logger) (*Server, error) { 46 | cmdParts, err := shlex.Split(c.GetCommand()) 47 | if err != nil { 48 | return nil, fmt.Errorf("error parsing command line (%s): %v", c.GetCommand(), err) 49 | } 50 | 51 | return &Server{ 52 | c: c, 53 | l: l, 54 | cmdName: cmdParts[0], 55 | cmdArgs: cmdParts[1:len(cmdParts)], 56 | }, nil 57 | } 58 | 59 | // Start runs the external command. 60 | func (s *Server) Start(ctx context.Context, dataChan chan<- *metrics.EventMetrics) error { 61 | s.cmd = exec.CommandContext(ctx, s.cmdName, s.cmdArgs...) 62 | go func() { 63 | err := s.cmd.Run() 64 | s.l.Infof("Command %s started. Err status: %v", s.c.GetCommand(), err) 65 | }() 66 | 67 | return nil 68 | } 69 | -------------------------------------------------------------------------------- /servers/external/proto/config.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto2"; 16 | 17 | package cloudprober.servers.external; 18 | 19 | option go_package = "github.com/google/cloudprober/servers/external/proto"; 20 | 21 | // Next available tag = 2 22 | message ServerConf { 23 | optional string command = 1; 24 | } 25 | -------------------------------------------------------------------------------- /servers/grpc/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.servers.grpc; 4 | 5 | option go_package = "github.com/google/cloudprober/servers/grpc/proto"; 6 | 7 | message ServerConf { 8 | optional int32 port = 1 [default = 3142]; 9 | 10 | // Enables gRPC reflection for publicly visible services, allowing grpc_cli to 11 | // work. See https://grpc.io/grpc/core/md_doc_server_reflection_tutorial.html. 12 | optional bool enable_reflection = 2 [default = false]; 13 | 14 | // If use_dedicated_server is set to true, then create a new gRPC server 15 | // to handle probes. Otherwise, attempt to reuse gRPC server from runconfig 16 | // if that was set. 17 | optional bool use_dedicated_server = 3 [default = true]; 18 | } 19 | -------------------------------------------------------------------------------- /servers/grpc/proto/grpcservice.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.servers.grpc; 4 | 5 | option go_package = "github.com/google/cloudprober/servers/grpc/proto"; 6 | 7 | message EchoMessage { 8 | optional bytes blob = 1; 9 | } 10 | 11 | message StatusRequest { 12 | optional string client_name = 1; 13 | } 14 | 15 | message StatusResponse { 16 | optional int64 uptime_us = 1; 17 | } 18 | 19 | message BlobReadRequest { 20 | optional int32 size = 1; 21 | } 22 | 23 | message BlobReadResponse { 24 | optional bytes blob = 1; 25 | } 26 | 27 | message BlobWriteRequest { 28 | optional bytes blob = 1; 29 | } 30 | 31 | message BlobWriteResponse { 32 | optional int32 size = 1; 33 | } 34 | 35 | service Prober { 36 | // Echo echoes back incoming messages. 37 | rpc Echo(EchoMessage) returns (EchoMessage) {} 38 | // BlobRead returns a blob of bytes to the prober. 39 | rpc BlobRead(BlobReadRequest) returns (BlobReadResponse) {} 40 | // ServerStatus returns the current server status. 41 | rpc ServerStatus(StatusRequest) returns (StatusResponse) {} 42 | // BlobWrite allows client to write a blob to the server. 43 | rpc BlobWrite(BlobWriteRequest) returns (BlobWriteResponse) {} 44 | } 45 | -------------------------------------------------------------------------------- /servers/http/proto/config.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2019 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto2"; 16 | 17 | package cloudprober.servers.http; 18 | 19 | option go_package = "github.com/google/cloudprober/servers/http/proto"; 20 | 21 | // Next available tag = 10 22 | message ServerConf { 23 | optional int32 port = 1 [default = 3141]; 24 | 25 | // tls_cert_file and tls_key_file field should be set for HTTPS. 26 | enum ProtocolType { 27 | HTTP = 0; 28 | HTTPS = 1; 29 | } 30 | optional ProtocolType protocol = 6 [default = HTTP]; 31 | 32 | // Maximum duration for reading the entire request, including the body. 33 | optional int32 read_timeout_ms = 2 [default = 10000]; // default: 10s 34 | 35 | // Maximum duration before timing out writes of the response. 36 | optional int32 write_timeout_ms = 3 [default = 10000]; // default: 10s 37 | 38 | // Maximum amount of time to wait for the next request when keep-alives are 39 | // enabled. 40 | optional int32 idle_timeout_ms = 4 [default = 60000]; // default: 1m 41 | 42 | // Certificate file to use for HTTPS servers. 43 | optional string tls_cert_file = 7; 44 | 45 | // Private key file corresponding to the certificate above. 46 | optional string tls_key_file = 8; 47 | 48 | // Disable HTTP/2 for HTTPS servers. 49 | optional bool disable_http2 = 9; 50 | 51 | message PatternDataHandler { 52 | // Response sizes to server, e.g. 1024. 53 | required int32 response_size = 1; 54 | // Pattern is repeated to build the response, with "response_size mod 55 | // pattern_size" filled by '0' bytes. 56 | optional string pattern = 2 [default = "cloudprober"]; 57 | } 58 | // Pattern data handler returns pattern data at the url /data_, 59 | // e.g. "/data_2048". 60 | repeated PatternDataHandler pattern_data_handler = 5; 61 | } 62 | -------------------------------------------------------------------------------- /servers/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.servers; 4 | 5 | import "github.com/google/cloudprober/servers/grpc/proto/config.proto"; 6 | import "github.com/google/cloudprober/servers/http/proto/config.proto"; 7 | import "github.com/google/cloudprober/servers/udp/proto/config.proto"; 8 | import "github.com/google/cloudprober/servers/external/proto/config.proto"; 9 | 10 | option go_package = "github.com/google/cloudprober/servers/proto"; 11 | 12 | message ServerDef { 13 | enum Type { 14 | HTTP = 0; 15 | UDP = 1; 16 | GRPC = 2; 17 | EXTERNAL = 3; 18 | } 19 | required Type type = 1; 20 | 21 | oneof server { 22 | http.ServerConf http_server = 2; 23 | udp.ServerConf udp_server = 3; 24 | grpc.ServerConf grpc_server = 4; 25 | external.ServerConf external_server = 5; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /servers/udp/cmd/udp.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /* 16 | This binary implements a stand-alone UDP server using the 17 | cloudprober/servers/udp/udp package. 18 | */ 19 | package main 20 | 21 | import ( 22 | "context" 23 | 24 | "github.com/golang/protobuf/proto" 25 | "github.com/google/cloudprober/logger" 26 | "github.com/google/cloudprober/servers/udp" 27 | configpb "github.com/google/cloudprober/servers/udp/proto" 28 | 29 | "flag" 30 | "github.com/golang/glog" 31 | ) 32 | 33 | var ( 34 | port = flag.Int("port", 31122, "Port to listen on") 35 | responseType = flag.String("type", "echo", "Server type: echo|discard") 36 | ) 37 | 38 | func main() { 39 | flag.Parse() 40 | 41 | l, err := logger.New(context.Background(), "UDP_"+*responseType) 42 | if err != nil { 43 | glog.Fatal(err) 44 | } 45 | 46 | config := &configpb.ServerConf{ 47 | Port: proto.Int32(int32(*port)), 48 | Type: configpb.ServerConf_DISCARD.Enum(), 49 | } 50 | server, err := udp.New(context.Background(), config, l) 51 | if err != nil { 52 | glog.Fatalf("Error creating a new UDP server: %v", err) 53 | } 54 | glog.Fatal(server.Start(context.Background(), nil)) 55 | } 56 | -------------------------------------------------------------------------------- /servers/udp/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.servers.udp; 4 | 5 | option go_package = "github.com/google/cloudprober/servers/udp/proto"; 6 | 7 | message ServerConf { 8 | required int32 port = 1; 9 | 10 | enum Type { 11 | // Echos the incoming packet back. 12 | // Note that UDP echo server limits reads to 4098 bytes. For messages longer 13 | // than 4098 bytes it won't work as expected. 14 | ECHO = 0; 15 | 16 | // Discard the incoming packet. Return nothing. 17 | DISCARD = 1; 18 | } 19 | required Type type = 2; 20 | } 21 | -------------------------------------------------------------------------------- /surfacers/cloudwatch/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.surfacer.cloudwatch; 4 | 5 | option go_package = "github.com/google/cloudprober/surfacers/cloudwatch/proto"; 6 | 7 | message SurfacerConf { 8 | // The cloudwatch metric namespace 9 | optional string namespace = 1 [default = "cloudprober"]; 10 | 11 | // The cloudwatch resolution value, lowering this below 60 will incur 12 | // additional charges as the metrics will be charged at a high resolution 13 | // rate. 14 | optional int64 resolution = 2 [default = 60]; 15 | 16 | // The AWS Region, used to create a CloudWatch session. 17 | // The order of fallback for evaluating the AWS Region: 18 | // 1. This config value. 19 | // 2. EC2 metadata endpoint, via cloudprober sysvars. 20 | // 3. AWS_REGION environment value. 21 | // 4. AWS_DEFAULT_REGION environment value, if AWS_SDK_LOAD_CONFIG is set. 22 | // https://docs.aws.amazon.com/sdk-for-go/api/aws/session/ 23 | optional string region = 3; 24 | } 25 | -------------------------------------------------------------------------------- /surfacers/common/transform/transform.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package transform implements some transformations for metrics before we 16 | // export them. 17 | package transform 18 | 19 | import ( 20 | "fmt" 21 | 22 | "github.com/google/cloudprober/logger" 23 | "github.com/google/cloudprober/metrics" 24 | ) 25 | 26 | // AddFailureMetric adds failure metric to the EventMetrics based on the 27 | // config options. 28 | func AddFailureMetric(em *metrics.EventMetrics) error { 29 | tv, sv, fv := em.Metric("total"), em.Metric("success"), em.Metric("failure") 30 | // If there is already a failure metric, or if "total" and "success" metrics 31 | // are not available, don't compute failure metric. 32 | if fv != nil || tv == nil || sv == nil { 33 | return nil 34 | } 35 | 36 | total, totalOk := tv.(metrics.NumValue) 37 | success, successOk := sv.(metrics.NumValue) 38 | if !totalOk || !successOk { 39 | return fmt.Errorf("total (%v) and success (%v) values are not numeric, this should never happen", tv, sv) 40 | } 41 | 42 | em.AddMetric("failure", metrics.NewInt(total.Int64()-success.Int64())) 43 | return nil 44 | } 45 | 46 | // CumulativeToGauge creates a "gauge" EventMetrics from a "cumulative" 47 | // EventMetrics using a cache. It looks for the EventMetrics in the given cache 48 | // and if it exists already, it subtracts the current values from the cached 49 | // values. 50 | func CumulativeToGauge(em *metrics.EventMetrics, lvCache map[string]*metrics.EventMetrics, l *logger.Logger) (*metrics.EventMetrics, error) { 51 | key := em.Key() 52 | 53 | lastEM, ok := lvCache[key] 54 | // Cache a copy of "em" as some fields like maps and dist can be shared 55 | // across successive "em" writes. 56 | lvCache[key] = em.Clone() 57 | 58 | // If it is the first time for this EventMetrics, return it as it is. 59 | if !ok { 60 | return em, nil 61 | } 62 | 63 | gaugeEM, err := em.SubtractLast(lastEM) 64 | if err != nil { 65 | return nil, fmt.Errorf("error subtracting cached metrics from current metrics: %v", err) 66 | } 67 | 68 | return gaugeEM, nil 69 | } 70 | -------------------------------------------------------------------------------- /surfacers/datadog/datadog_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package datadog 16 | 17 | import ( 18 | "context" 19 | "reflect" 20 | "testing" 21 | "time" 22 | 23 | "github.com/google/cloudprober/logger" 24 | "github.com/google/cloudprober/metrics" 25 | ) 26 | 27 | func newTestDDSurfacer() DDSurfacer { 28 | l, _ := logger.New(context.TODO(), "test-logger") 29 | 30 | return DDSurfacer{ 31 | l: l, 32 | prefix: "testPrefix", 33 | } 34 | } 35 | 36 | func TestEmLabelsToTags(t *testing.T) { 37 | timestamp := time.Now() 38 | 39 | tests := map[string]struct { 40 | em *metrics.EventMetrics 41 | want []string 42 | }{ 43 | "no label": { 44 | em: metrics.NewEventMetrics(timestamp), 45 | want: []string{}, 46 | }, 47 | "one label": { 48 | em: metrics.NewEventMetrics(timestamp).AddLabel("ptype", "sysvars"), 49 | want: []string{"ptype:sysvars"}, 50 | }, 51 | "three labels": { 52 | em: metrics.NewEventMetrics(timestamp).AddLabel("label1", "value1"). 53 | AddLabel("label2", "value2"). 54 | AddLabel("label3", "value3"), 55 | want: []string{"label1:value1", "label2:value2", "label3:value3"}, 56 | }, 57 | } 58 | 59 | for name, tc := range tests { 60 | t.Run(name, func(t *testing.T) { 61 | got := emLabelsToTags(tc.em) 62 | if !reflect.DeepEqual(got, tc.want) { 63 | // if got != tc.want { 64 | t.Errorf("got: %v, want %v %v %v", got, tc.want, reflect.TypeOf(got), reflect.TypeOf(tc.want)) 65 | } 66 | }) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /surfacers/datadog/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.surfacer.datadog; 4 | 5 | option go_package = "github.com/google/cloudprober/surfacers/datadog/proto"; 6 | 7 | // Surfacer config for datadog surfacer. 8 | message SurfacerConf { 9 | // Prefix to add to all metrics. 10 | optional string prefix = 1 [default = "cloudprober"]; 11 | 12 | // Datadog API key. If not set, DD_API_KEY env variable is used. 13 | optional string api_key = 2; 14 | 15 | // Datadog APP key. If not set, DD_APP_KEY env variable is used. 16 | optional string app_key = 3; 17 | 18 | // Datadog server, default: "api.datadoghq.com" 19 | optional string server = 4; 20 | } 21 | -------------------------------------------------------------------------------- /surfacers/file/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.surfacer.file; 4 | 5 | option go_package = "github.com/google/cloudprober/surfacers/file/proto"; 6 | 7 | message SurfacerConf { 8 | // Where to write the results. If left unset, file surfacer writes to the 9 | // standard output. 10 | optional string file_path = 1; 11 | optional string prefix = 2 [default = "cloudprober"]; 12 | 13 | // Compress data before writing to the file. 14 | optional bool compression_enabled = 3 [default = false]; 15 | } 16 | -------------------------------------------------------------------------------- /surfacers/postgres/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.surfacer.postgres; 4 | 5 | option go_package = "github.com/google/cloudprober/surfacers/postgres/proto"; 6 | 7 | message SurfacerConf { 8 | required string connection_string = 1; 9 | required string metrics_table_name = 2; 10 | optional int64 metrics_buffer_size = 3 [default = 10000]; 11 | } 12 | -------------------------------------------------------------------------------- /surfacers/prometheus/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.surfacer.prometheus; 4 | 5 | option go_package = "github.com/google/cloudprober/surfacers/prometheus/proto"; 6 | 7 | message SurfacerConf { 8 | // How many metrics entries (EventMetrics) to buffer. Incoming metrics 9 | // processing is paused while serving data to prometheus. This buffer is to 10 | // make writes to prometheus surfacer non-blocking. 11 | // NOTE: This field is confusing for users and will be removed from the config 12 | // after v0.10.3. 13 | optional int64 metrics_buffer_size = 1 [default = 10000]; 14 | 15 | // Whether to include timestamps in metrics. If enabled (default) each metric 16 | // string includes the metric timestamp as recorded in the EventMetric. 17 | // Prometheus associates the scraped values with this timestamp. If disabled, 18 | // i.e. timestamps are not exported, prometheus associates scraped values with 19 | // scrape timestamp. 20 | optional bool include_timestamp = 2 [default = true]; 21 | 22 | // URL that prometheus scrapes metrics from. 23 | optional string metrics_url = 3 [default = "/metrics"]; 24 | 25 | // Prefix to add to all metric names. For example setting this field to 26 | // "cloudprober_" will result in metrics with names: 27 | // cloudprober_total, cloudprober_success, cloudprober_latency, .. 28 | optional string metrics_prefix = 4; 29 | } 30 | -------------------------------------------------------------------------------- /surfacers/pubsub/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.surfacer.pubsub; 4 | 5 | option go_package = "github.com/google/cloudprober/surfacers/pubsub/proto"; 6 | 7 | message SurfacerConf { 8 | // GCP project name for pubsub. If not specified and running on GCP, 9 | // project is used. 10 | optional string project = 1; 11 | 12 | // Pubsub topic name. 13 | // Default is cloudprober-{hostname} 14 | optional string topic_name = 2; 15 | 16 | // Compress data before writing to pubsub. 17 | optional bool compression_enabled = 4 [default = false]; 18 | } 19 | -------------------------------------------------------------------------------- /surfacers/stackdriver/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.surfacer.stackdriver; 4 | 5 | option go_package = "github.com/google/cloudprober/surfacers/stackdriver/proto"; 6 | 7 | message SurfacerConf { 8 | // GCP project name for stackdriver. If not specified and running on GCP, 9 | // project is used. 10 | optional string project = 1; 11 | 12 | // How often to export metrics to stackdriver. 13 | optional uint64 batch_timer_sec = 2 [default = 10]; 14 | 15 | // If allowed_metrics_regex is specified, only metrics matching the given 16 | // regular expression will be exported to stackdriver. Since probe type and 17 | // probe name are part of the metric name, you can use this field to restrict 18 | // stackdriver metrics to a particular probe. 19 | // Example: 20 | // allowed_metrics_regex: ".*(http|ping).*(success|validation_failure).*" 21 | optional string allowed_metrics_regex = 3; 22 | 23 | // Monitoring URL base. Full metric URL looks like the following: 24 | // /// 25 | // Example: 26 | // custom.googleapis.com/cloudprober/http/google-homepage/latency 27 | optional string monitoring_url = 4 28 | [default = "custom.googleapis.com/cloudprober/"]; 29 | 30 | // How many metrics entries to buffer. Incoming metrics 31 | // processing is paused while serving data to Stackdriver. This buffer is to 32 | // make writes to Stackdriver surfacer non-blocking. 33 | optional int64 metrics_buffer_size = 5 [default = 10000]; 34 | } 35 | -------------------------------------------------------------------------------- /sysvars/runtime_linux.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2021 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build linux 16 | // +build linux 17 | 18 | package sysvars 19 | 20 | import ( 21 | "time" 22 | 23 | "github.com/google/cloudprober/logger" 24 | "github.com/google/cloudprober/metrics" 25 | "golang.org/x/sys/unix" 26 | ) 27 | 28 | func osRuntimeVars(dataChan chan *metrics.EventMetrics, l *logger.Logger) { 29 | em := metrics.NewEventMetrics(time.Now()). 30 | AddLabel("ptype", "sysvars"). 31 | AddLabel("probe", "sysvars") 32 | 33 | // CPU usage 34 | var timeSpec unix.Timespec 35 | if err := unix.ClockGettime(unix.CLOCK_PROCESS_CPUTIME_ID, &timeSpec); err != nil { 36 | l.Warningf("Error while trying to get CPU usage: %v", err) 37 | } else { 38 | em.AddMetric("cpu_usage_msec", metrics.NewFloat(float64(timeSpec.Nano())/1e6)) 39 | } 40 | 41 | dataChan <- em 42 | l.Debug(em.String()) 43 | } 44 | -------------------------------------------------------------------------------- /sysvars/runtime_nonlinux.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2021 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !linux 16 | // +build !linux 17 | 18 | package sysvars 19 | 20 | import ( 21 | "github.com/google/cloudprober/logger" 22 | "github.com/google/cloudprober/metrics" 23 | ) 24 | 25 | // osRuntimeVars doesn't anything for the non-Linux systems yet. We have it 26 | // here to make sysvars package compilation work on non-Linux systems. 27 | func osRuntimeVars(dataChan chan *metrics.EventMetrics, l *logger.Logger) { 28 | } 29 | -------------------------------------------------------------------------------- /sysvars/runtime_test.go: -------------------------------------------------------------------------------- 1 | package sysvars 2 | 3 | import ( 4 | "runtime" 5 | "testing" 6 | "time" 7 | 8 | "github.com/google/cloudprober/logger" 9 | "github.com/google/cloudprober/metrics" 10 | ) 11 | 12 | func TestCounterRuntimeVars(t *testing.T) { 13 | dataChan := make(chan *metrics.EventMetrics, 1) 14 | l := &logger.Logger{} 15 | m := &runtime.MemStats{} 16 | runtime.ReadMemStats(m) 17 | ts := time.Now() 18 | 19 | counterRuntimeVars(dataChan, ts, m, l) 20 | em := <-dataChan 21 | 22 | if em.Timestamp != ts { 23 | t.Errorf("em.Timestamp=%v, want=%v", em.Timestamp, ts) 24 | } 25 | 26 | if em.Kind != metrics.CUMULATIVE { 27 | t.Errorf("Metrics kind is not cumulative.") 28 | } 29 | 30 | for _, name := range []string{"uptime_msec", "gc_time_msec", "mallocs", "frees"} { 31 | if em.Metric(name) == nil { 32 | t.Errorf("Expected metric \"%s\" not defined in EventMetrics: %s", name, em.String()) 33 | } 34 | } 35 | } 36 | 37 | func TestGaugeRuntimeVars(t *testing.T) { 38 | dataChan := make(chan *metrics.EventMetrics, 1) 39 | l := &logger.Logger{} 40 | m := &runtime.MemStats{} 41 | runtime.ReadMemStats(m) 42 | ts := time.Now() 43 | 44 | gaugeRuntimeVars(dataChan, ts, m, l) 45 | em := <-dataChan 46 | 47 | if em.Timestamp != ts { 48 | t.Errorf("em.Timestamp=%v, want=%v", em.Timestamp, ts) 49 | } 50 | 51 | if em.Kind != metrics.GAUGE { 52 | t.Errorf("Metrics kind is not gauge.") 53 | } 54 | 55 | for _, name := range []string{"goroutines", "mem_stats_sys_bytes"} { 56 | if em.Metric(name) == nil { 57 | t.Errorf("Expected metric \"%s\" not defined in EventMetrics: %s", name, em.String()) 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /sysvars/sysvars_ec2.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2020 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package sysvars 16 | 17 | import ( 18 | "fmt" 19 | 20 | "github.com/aws/aws-sdk-go/aws" 21 | "github.com/aws/aws-sdk-go/aws/ec2metadata" 22 | "github.com/aws/aws-sdk-go/aws/session" 23 | "github.com/google/cloudprober/logger" 24 | ) 25 | 26 | var ec2Vars = func(sysVars map[string]string, l *logger.Logger) (bool, error) { 27 | s, err := session.NewSession(&aws.Config{ 28 | MaxRetries: aws.Int(0), 29 | }) 30 | if err != nil { 31 | // We ignore session errors. It's not clear what can cause them. 32 | l.Warningf("sysvars_ec2: could not create AWS session: %v", err) 33 | return false, nil 34 | } 35 | 36 | md := ec2metadata.New(s) 37 | // Doing the availability check in module since we need a session 38 | if md.Available() == false { 39 | return false, nil 40 | } 41 | 42 | id, err := md.GetInstanceIdentityDocument() 43 | if err != nil { 44 | sysVars["EC2_METADATA_Available"] = "false" 45 | return true, fmt.Errorf("sysvars_ec2: could not get instance identity document %v", err) 46 | } 47 | 48 | sysVars["EC2_METADATA_Available"] = "true" 49 | sysVars["EC2_AvailabilityZone"] = id.AvailabilityZone 50 | sysVars["EC2_PrivateIP"] = id.PrivateIP 51 | sysVars["EC2_Region"] = id.Region 52 | sysVars["EC2_InstanceID"] = id.InstanceID 53 | sysVars["EC2_InstanceType"] = id.InstanceType 54 | sysVars["EC2_ImageID"] = id.ImageID 55 | sysVars["EC2_KernelID"] = id.KernelID 56 | sysVars["EC2_RamdiskID"] = id.RamdiskID 57 | sysVars["EC2_Architecture"] = id.Architecture 58 | return true, nil 59 | } 60 | -------------------------------------------------------------------------------- /targets/endpoint/endpoint.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package endpoint provides the type Endpoint, to be used with the 16 | // targets.Targets interface. 17 | package endpoint 18 | 19 | import ( 20 | "sort" 21 | "strconv" 22 | "strings" 23 | "time" 24 | ) 25 | 26 | // Endpoint represents a targets and associated parameters. 27 | type Endpoint struct { 28 | Name string 29 | Labels map[string]string 30 | LastUpdated time.Time 31 | Port int 32 | } 33 | 34 | // Key returns a string key that uniquely identifies that endpoint. 35 | // Endpoint key consists of endpoint name, port and labels. 36 | func (ep *Endpoint) Key() string { 37 | labelSlice := make([]string, len(ep.Labels)) 38 | i := 0 39 | for k, v := range ep.Labels { 40 | labelSlice[i] = k + ":" + v 41 | i++ 42 | } 43 | sort.Strings(labelSlice) 44 | 45 | return strings.Join(append([]string{ep.Name, strconv.Itoa(ep.Port)}, labelSlice...), "_") 46 | } 47 | 48 | // Lister should implement the ListEndpoints method. 49 | type Lister interface { 50 | // ListEndpoints returns list of endpoints (name, port tupples). 51 | ListEndpoints() []Endpoint 52 | } 53 | 54 | // EndpointsFromNames is convenience function to build a list of endpoints 55 | // from only names. It leaves the Port field in Endpoint unset and initializes 56 | // Labels field to an empty map. 57 | func EndpointsFromNames(names []string) []Endpoint { 58 | result := make([]Endpoint, len(names)) 59 | for i, name := range names { 60 | result[i].Name = name 61 | result[i].Labels = make(map[string]string) 62 | } 63 | return result 64 | } 65 | 66 | // NamesFromEndpoints is convenience function to build a list of names 67 | // from endpoints. 68 | func NamesFromEndpoints(endpoints []Endpoint) []string { 69 | result := make([]string, len(endpoints)) 70 | for i, ep := range endpoints { 71 | result[i] = ep.Name 72 | } 73 | return result 74 | } 75 | -------------------------------------------------------------------------------- /targets/endpoint/endpoint_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package endpoint 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | ) 21 | 22 | func TestEndpointsFromNames(t *testing.T) { 23 | names := []string{"targetA", "targetB", "targetC"} 24 | endpoints := EndpointsFromNames(names) 25 | 26 | for i := range names { 27 | ep := endpoints[i] 28 | 29 | if ep.Name != names[i] { 30 | t.Errorf("Endpoint.Name=%s, want=%s", ep.Name, names[i]) 31 | } 32 | if ep.Port != 0 { 33 | t.Errorf("Endpoint.Port=%d, want=0", ep.Port) 34 | } 35 | if len(ep.Labels) != 0 { 36 | t.Errorf("Endpoint.Labels=%v, want={}", ep.Labels) 37 | } 38 | } 39 | } 40 | 41 | func TestKey(t *testing.T) { 42 | for _, test := range []struct { 43 | name string 44 | port int 45 | labels map[string]string 46 | key string 47 | }{ 48 | { 49 | name: "t1", 50 | port: 80, 51 | key: "t1_80", 52 | }, 53 | { 54 | name: "t1", 55 | port: 80, 56 | labels: map[string]string{"app": "cloudprober", "dc": "xx"}, 57 | key: "t1_80_app:cloudprober_dc:xx", 58 | }, 59 | { 60 | name: "t1", 61 | port: 80, 62 | labels: map[string]string{"dc": "xx", "app": "cloudprober"}, 63 | key: "t1_80_app:cloudprober_dc:xx", 64 | }, 65 | } { 66 | ep := Endpoint{ 67 | Name: test.name, 68 | Port: test.port, 69 | Labels: test.labels, 70 | } 71 | t.Run(fmt.Sprintf("%v", ep), func(t *testing.T) { 72 | key := ep.Key() 73 | if key != test.key { 74 | t.Errorf("Got key: %s, want: %s", key, test.key) 75 | } 76 | }) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /targets/file/file.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020-2021 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /* 16 | Package file implements a file-based targets for cloudprober. 17 | */ 18 | package file 19 | 20 | import ( 21 | "context" 22 | 23 | "github.com/google/cloudprober/logger" 24 | "github.com/google/cloudprober/rds/client" 25 | client_configpb "github.com/google/cloudprober/rds/client/proto" 26 | "github.com/google/cloudprober/rds/file" 27 | file_configpb "github.com/google/cloudprober/rds/file/proto" 28 | rdspb "github.com/google/cloudprober/rds/proto" 29 | configpb "github.com/google/cloudprober/targets/file/proto" 30 | dnsRes "github.com/google/cloudprober/targets/resolver" 31 | ) 32 | 33 | // New returns new file targets. 34 | func New(opts *configpb.TargetsConf, res *dnsRes.Resolver, l *logger.Logger) (*client.Client, error) { 35 | lister, err := file.New(&file_configpb.ProviderConfig{ 36 | FilePath: []string{opts.GetFilePath()}, 37 | }, l) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | clientConf := &client_configpb.ClientConf{ 43 | Request: &rdspb.ListResourcesRequest{Filter: opts.GetFilter()}, 44 | ReEvalSec: opts.ReEvalSec, 45 | } 46 | 47 | return client.New(clientConf, func(_ context.Context, req *rdspb.ListResourcesRequest) (*rdspb.ListResourcesResponse, error) { 48 | return lister.ListResources(req) 49 | }, l) 50 | } 51 | -------------------------------------------------------------------------------- /targets/file/proto/config.proto: -------------------------------------------------------------------------------- 1 | // Configuration proto for File targets. 2 | syntax = "proto2"; 3 | 4 | package cloudprober.targets.file; 5 | 6 | import "github.com/google/cloudprober/rds/file/proto/config.proto"; 7 | import "github.com/google/cloudprober/rds/proto/rds.proto"; 8 | 9 | option go_package = "github.com/google/cloudprober/targets/file/proto"; 10 | 11 | message TargetsConf { 12 | // File that contains resources in either textproto or json format. 13 | // Example in textproto format: 14 | // 15 | // resource { 16 | // name: "switch-xx-01" 17 | // ip: "10.11.112.3" 18 | // port: 8080 19 | // labels { 20 | // key: "device_type" 21 | // value: "switch" 22 | // } 23 | // } 24 | // resource { 25 | // name: "switch-yy-01" 26 | // ip: "10.16.110.12" 27 | // port: 8080 28 | // } 29 | optional string file_path = 1; 30 | 31 | repeated .cloudprober.rds.Filter filter = 2; 32 | 33 | optional .cloudprober.rds.file.ProviderConfig.Format format = 3; 34 | 35 | // If specified, file will be re-read at the given interval. 36 | optional int32 re_eval_sec = 4; 37 | } 38 | -------------------------------------------------------------------------------- /targets/gce/gce_utils_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2019 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package gce 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/golang/protobuf/proto" 21 | rdspb "github.com/google/cloudprober/rds/proto" 22 | ) 23 | 24 | func TestParseLabels(t *testing.T) { 25 | tests := []struct { 26 | desc string 27 | label string 28 | shouldFail bool 29 | want []*rdspb.Filter 30 | }{ 31 | { 32 | "Valid label should succeed", 33 | "k:v", 34 | false, 35 | []*rdspb.Filter{ 36 | { 37 | Key: proto.String("labels.k"), 38 | Value: proto.String("v"), 39 | }, 40 | }, 41 | }, 42 | { 43 | "Multiple separators should fail", 44 | "k:v:t", 45 | true, 46 | nil, 47 | }, 48 | { 49 | "No separator should fail", 50 | "kv", 51 | true, 52 | nil, 53 | }, 54 | } 55 | 56 | for _, test := range tests { 57 | t.Run(test.desc, func(t *testing.T) { 58 | got, err := parseLabels([]string{test.label}) 59 | if test.shouldFail && err == nil { 60 | t.Errorf("parseLabels() error got:nil, want:error") 61 | } else if !test.shouldFail && err != nil { 62 | t.Errorf("parseLabels() error got:%s, want:nil", err) 63 | } 64 | 65 | eq := len(got) == len(test.want) 66 | for i := 0; i < len(got) && i < len(test.want); i++ { 67 | eq = eq && proto.Equal(got[i], test.want[i]) 68 | } 69 | if !eq { 70 | t.Errorf("parseLabels() got:%s, want:%s", got, test.want) 71 | } 72 | }) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /targets/lameduck/proto/config.proto: -------------------------------------------------------------------------------- 1 | // Provides all configuration necesary to list targets for a cloudprober probe. 2 | syntax = "proto2"; 3 | 4 | package cloudprober.targets.lameduck; 5 | 6 | import "github.com/google/cloudprober/rds/client/proto/config.proto"; 7 | 8 | option go_package = "github.com/google/cloudprober/targets/lameduck/proto"; 9 | 10 | message Options { 11 | // How often to check for lame-ducked targets 12 | optional int32 re_eval_sec = 1 [default = 10]; 13 | 14 | // Runtime config project. If running on GCE, this defaults to the project 15 | // containing the VM. 16 | optional string runtimeconfig_project = 2; 17 | 18 | // Lame duck targets runtime config name. An operator will create a variable 19 | // here to mark a target as lame-ducked. 20 | optional string runtimeconfig_name = 3 [default = "lame-duck-targets"]; 21 | 22 | // Lame duck targets pubsub topic name. An operator will create a message 23 | // here to mark a target as lame-ducked. 24 | optional string pubsub_topic = 7; 25 | 26 | // Lame duck expiration time. We ignore variables (targets) that have been 27 | // updated more than these many seconds ago. This is a safety mechanism for 28 | // failing to cleanup. Also, the idea is that if a target has actually 29 | // disappeared, automatic targets expansion will take care of that some time 30 | // during this expiration period. 31 | optional int32 expiration_sec = 4 [default = 300]; 32 | 33 | // Use an RDS client to get lame-duck-targets. 34 | // This option is always true now and will be removed after v0.10.7. 35 | optional bool use_rds = 5 [deprecated = true]; 36 | 37 | // RDS server options, for example: 38 | // rds_server_options { 39 | // server_address: "rds-server.xyz:9314" 40 | // oauth_config: { 41 | // ... 42 | // } 43 | optional rds.ClientConf.ServerOptions rds_server_options = 6; 44 | } 45 | -------------------------------------------------------------------------------- /targets/statictargets_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package targets 16 | 17 | import ( 18 | "reflect" 19 | "testing" 20 | 21 | "github.com/google/cloudprober/targets/endpoint" 22 | ) 23 | 24 | func TestStaticTargets(t *testing.T) { 25 | for _, test := range []struct { 26 | desc string 27 | hosts string 28 | wantNames []string 29 | wantPorts []int 30 | wantErr bool 31 | }{ 32 | { 33 | desc: "valid hosts (extra space)", 34 | hosts: "www.google.com ,127.0.0.1, 2001::2001", 35 | wantNames: []string{"www.google.com", "127.0.0.1", "2001::2001"}, 36 | wantPorts: []int{0, 0, 0}, 37 | }, 38 | { 39 | desc: "Ports in name", 40 | hosts: "www.google.com:80,127.0.0.1:8080,[2001::2001]:8081", 41 | wantNames: []string{"www.google.com", "127.0.0.1", "2001::2001"}, 42 | wantPorts: []int{80, 8080, 8081}, 43 | }, 44 | { 45 | desc: "invalid host, IPv6 port in name without brackets", 46 | hosts: "www.google.com,127.0.0.1:8080,0:0:0:0:0:1:8081", 47 | wantErr: true, 48 | }, 49 | { 50 | desc: "invalid hosts,URL in name", 51 | hosts: "www.google.com/url1,127.0.0.1,2001::2001", 52 | wantErr: true, 53 | }, 54 | } { 55 | t.Run(test.desc, func(t *testing.T) { 56 | tgts, err := staticTargets(test.hosts) 57 | if err != nil { 58 | if !test.wantErr { 59 | t.Errorf("Unexpected error: %v", err) 60 | } 61 | return 62 | } 63 | if test.wantErr { 64 | t.Errorf("wanted error, but didn't get one") 65 | } 66 | wantEp := make([]endpoint.Endpoint, len(test.wantNames)) 67 | for i := 0; i < len(wantEp); i++ { 68 | wantEp[i] = endpoint.Endpoint{Name: test.wantNames[i], Port: test.wantPorts[i]} 69 | } 70 | got := tgts.ListEndpoints() 71 | if !reflect.DeepEqual(got, wantEp) { 72 | t.Errorf("staticTargets: got=%v, wanted: %v", got, wantEp) 73 | } 74 | }) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /targets/testdata/testdata.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.targets.testdata; 4 | 5 | import "github.com/google/cloudprober/targets/proto/targets.proto"; 6 | 7 | option go_package = "github.com/google/cloudprober/targets/testdata"; 8 | 9 | message FancyTargets { 10 | optional string name = 1; 11 | } 12 | 13 | message AnotherFancyTargets { 14 | optional string name = 1; 15 | } 16 | 17 | extend targets.TargetsDef { 18 | optional FancyTargets fancy_targets = 200; 19 | optional AnotherFancyTargets another_fancy_targets = 201; 20 | } 21 | -------------------------------------------------------------------------------- /tools/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eu 2 | # 3 | # Copyright 2017 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # This script builds cloudprober from source. It expects protobuf's Go code to 18 | # be already available (can be done using tools/gen_pb_go.sh). 19 | 20 | PROTOC_VERSION="3.3.0" 21 | PROJECT="github.com/google/cloudprober" 22 | 23 | GOPATH=$(go env GOPATH) 24 | 25 | if [ -z "$GOPATH" ]; then 26 | echo "Go environment is not setup correctly. Please look at" 27 | echo "https://golang.org/doc/code.html to set up Go environment." 28 | exit 1 29 | fi 30 | 31 | # Change directory to the project workspace in GOPATH 32 | project_dir="${GOPATH}/src/${PROJECT}" 33 | 34 | if [ ! -d "${project_dir}" ];then 35 | echo "${PROJECT} not found under Go workspace: ${GOPATH}/src. Please download" 36 | echo " cloudprober source code from github.com/google/cloudprober and set it " 37 | echo "such that it's available at ${project_dir}." 38 | exit 1 39 | fi 40 | 41 | cd ${project_dir} 42 | # Get all dependencies 43 | echo "Getting all the dependencies.." 44 | echo "======================================================================" 45 | go get -t ./... 46 | 47 | # Build everything 48 | echo "Build everything..." 49 | echo "======================================================================" 50 | go build ./... 51 | 52 | # Run tests 53 | echo "Running tests..." 54 | echo "======================================================================" 55 | go test ./... 56 | 57 | # Install cloudprober 58 | echo "Build static cloudprober binary.." 59 | echo "======================================================================" 60 | CGO_ENABLED=0 go build -ldflags '-extldflags "-static"' ./cmd/cloudprober.go 61 | -------------------------------------------------------------------------------- /tools/cloudprober_startup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eu 2 | # 3 | # Copyright 2017 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # This startup_script makes it easy to run cloudprober inside a docker 18 | # image. 19 | while getopts 'i:j:v:' flag; do 20 | case "$flag" in 21 | i) IMAGE=${OPTARG} ;; 22 | j) JOB=${OPTARG} ;; 23 | v) VERSION=${OPTARG} ;; 24 | esac 25 | done 26 | 27 | shift $(($OPTIND - 1)) 28 | cmd=$1 29 | if [ "$cmd" != "start" ]; then 30 | # First execution by cloud-init. Make the script accessible to all. 31 | chmod a+rx "$0" 32 | exit 33 | fi 34 | 35 | if [ -z "${JOB}" ]; then 36 | echo "-j is a required parameter" 37 | exit 1 38 | fi 39 | 40 | [[ -f "/etc/default/${JOB}" ]] && . "/etc/default/${JOB}" 41 | 42 | VERSION=${VERSION:-latest} 43 | KERNEL_VERSION=$(uname -r) 44 | GOOGLE_RELEASE=$(grep GOOGLE_RELEASE /etc/lsb-release|cut -d"=" -f2) 45 | 46 | # Make sure that when this script exits, all child processes are exited as well 47 | # Sending a SIGHUP/SIGTERM to bash kills the shell but leaves the child processes 48 | # running. We don't want that behavior. 49 | # "kill -- -$$" sends a SIGTERM to the process group. 50 | # "trap - SIGTERM" removes the trap for SIGTERM to avoid recursion. 51 | trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT 52 | # Get credentials to fetch docker image from gcr.io 53 | . /usr/share/google/dockercfg_update.sh 54 | docker pull "${IMAGE}:${VERSION}" 55 | DIGEST=$(docker inspect --format "{{.Id}}" "${IMAGE}:${VERSION}") 56 | VARS="kernel=${KERNEL_VERSION},google_release=${GOOGLE_RELEASE}" 57 | VARS="${VARS},${JOB}_tag=${VERSION},${JOB}_version=${DIGEST:0:12}" 58 | 59 | docker run -e "SYSVARS=${VARS}" --env-file <(env | grep CLOUDPROBER_) \ 60 | --net host --privileged -v /tmp:/tmp "${IMAGE}:${VERSION}" 61 | -------------------------------------------------------------------------------- /validators/http/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.validators.http; 4 | 5 | option go_package = "github.com/google/cloudprober/validators/http/proto"; 6 | 7 | // HTTP validator configuration. For HTTP validator to succeed, all conditions 8 | // specified in the validator should succeed. Note that failures conditions are 9 | // evaluated before success conditions. 10 | message Validator { 11 | // Comma-separated list of success status codes and code ranges. 12 | // Example: success_stauts_codes: 200-299,301,302 13 | optional string success_status_codes = 1; 14 | 15 | // Comma-separated list of failure status codes and code ranges. If HTTP 16 | // status code matches failure_status_codes, validator fails. 17 | optional string failure_status_codes = 2; 18 | 19 | message Header { 20 | // Header name to look for 21 | optional string name = 1; 22 | // Header value to match. If omited - check for header existense 23 | optional string value_regex = 2; 24 | } 25 | 26 | // Header based validations. 27 | // TODO(manugarg): Add support for specifying multiple success and failure 28 | // headers. 29 | // 30 | // Success Header: 31 | // If specified, HTTP response headers should match the success_header for 32 | // validation to succeed. Example: 33 | // success_header: { 34 | // name: "Strict-Transport-Security" 35 | // value_regex: "max-age=31536000" 36 | // } 37 | optional Header success_header = 3; 38 | 39 | // Failure Header: 40 | // If HTTP response headers match failure_header, validation fails. 41 | optional Header failure_header = 4; 42 | } 43 | -------------------------------------------------------------------------------- /validators/integrity/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.validators.integrity; 4 | 5 | option go_package = "github.com/google/cloudprober/validators/integrity/proto"; 6 | 7 | message Validator { 8 | // Validate the data integrity of the response using a pattern that is 9 | // repeated throughout the length of the response, with last len(response) % 10 | // len(pattern) bytes being zero bytes. 11 | // 12 | // For example if response length is 100 bytes and pattern length is 8 bytes, 13 | // first 96 bytes of the response should be pattern repeated 12 times, and 14 | // last 4 bytes should be set to zero byte ('\0') 15 | oneof pattern { 16 | // Pattern string for pattern repetition based integrity checks. 17 | // For example, cloudprobercloudprobercloudprober... 18 | string pattern_string = 1; 19 | 20 | // Pattern is derived from the first few bytes of the payload. This is 21 | // useful when pattern is not known in advance, for example cloudprober's 22 | // ping probe repeates the timestamp (8 bytes) in the packet payload. 23 | // An error is returned if response is smaller than pattern_num_bytes. 24 | int32 pattern_num_bytes = 2; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /validators/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package cloudprober.validators; 4 | 5 | import "github.com/google/cloudprober/validators/http/proto/config.proto"; 6 | import "github.com/google/cloudprober/validators/integrity/proto/config.proto"; 7 | 8 | option go_package = "github.com/google/cloudprober/validators/proto"; 9 | 10 | message Validator { 11 | required string name = 1; 12 | oneof type { 13 | http.Validator http_validator = 2; 14 | 15 | // Data integrity validator 16 | integrity.Validator integrity_validator = 3; 17 | 18 | // Regex validator 19 | string regex = 4; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /validators/regex/regex.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package regex provides regex validator for the Cloudprober's 16 | // validator framework. 17 | package regex 18 | 19 | import ( 20 | "errors" 21 | "fmt" 22 | "regexp" 23 | 24 | "github.com/google/cloudprober/logger" 25 | ) 26 | 27 | // Validator implements a regex validator. 28 | type Validator struct { 29 | r *regexp.Regexp 30 | l *logger.Logger 31 | } 32 | 33 | // Init initializes the regex validator. 34 | // It compiles the regex in the configuration and returns an error if regex 35 | // doesn't compile for some reason. 36 | func (v *Validator) Init(config interface{}, l *logger.Logger) error { 37 | regexStr, ok := config.(string) 38 | if !ok { 39 | return fmt.Errorf("%v is not a valid regex validator config", config) 40 | } 41 | if regexStr == "" { 42 | return errors.New("validator regex string cannot be empty") 43 | } 44 | 45 | r, err := regexp.Compile(regexStr) 46 | if err != nil { 47 | return fmt.Errorf("error compiling the given regex (%s): %v", regexStr, err) 48 | } 49 | 50 | v.r = r 51 | v.l = l 52 | return nil 53 | } 54 | 55 | // Validate the provided responseBody and return true if responseBody matches 56 | // the configured regex. 57 | func (v *Validator) Validate(responseBody []byte) (bool, error) { 58 | return v.r.Match(responseBody), nil 59 | } 60 | -------------------------------------------------------------------------------- /validators/regex/regex_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package regex 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/google/cloudprober/logger" 21 | ) 22 | 23 | func TestInvalidConfig(t *testing.T) { 24 | // Empty config 25 | testConfig := "" 26 | v := Validator{} 27 | err := v.Init(testConfig, &logger.Logger{}) 28 | if err == nil { 29 | t.Errorf("v.Init(%s, l): expected error but got nil", testConfig) 30 | } 31 | 32 | // Invalid regex as Go regex doesn't support negative lookaheads. 33 | testConfig = "(?!cloudprober)" 34 | v = Validator{} 35 | err = v.Init(testConfig, &logger.Logger{}) 36 | if err == nil { 37 | t.Errorf("v.Init(%s, l): expected error but got nil", testConfig) 38 | } 39 | } 40 | 41 | func verifyValidate(t *testing.T, respBody []byte, regexStr string, expected bool) { 42 | t.Helper() 43 | // Test initializing with pattern string. 44 | v := Validator{} 45 | err := v.Init(regexStr, &logger.Logger{}) 46 | if err != nil { 47 | t.Errorf("v.Init(%s, l): got error: %v", regexStr, err) 48 | } 49 | 50 | result, err := v.Validate(respBody) 51 | if err != nil { 52 | t.Errorf("v.Validate(nil, %s): got error: %v", string(respBody), err) 53 | } 54 | 55 | if result != expected { 56 | if err != nil { 57 | t.Errorf("v.Validate(nil, %s): result: %v, expected: %v", string(respBody), result, expected) 58 | } 59 | } 60 | } 61 | 62 | func TestPatternString(t *testing.T) { 63 | rows := []struct { 64 | regex string 65 | respBody []byte 66 | expected bool 67 | }{ 68 | { 69 | regex: "cloud.*", 70 | respBody: []byte("cloudprober"), 71 | expected: true, 72 | }, 73 | { 74 | regex: "[Cc]loud.*", 75 | respBody: []byte("Cloudprober"), 76 | expected: false, 77 | }, 78 | } 79 | 80 | for _, r := range rows { 81 | verifyValidate(t, r.respBody, r.regex, r.expected) 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /validators/validators_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package validators 16 | 17 | import ( 18 | "reflect" 19 | "testing" 20 | 21 | "github.com/google/cloudprober/logger" 22 | ) 23 | 24 | type testValidator struct { 25 | Succeed bool 26 | } 27 | 28 | func (tv *testValidator) Init(config interface{}, l *logger.Logger) error { 29 | return nil 30 | } 31 | 32 | func (tv *testValidator) Validate(responseObject interface{}, responseBody []byte) (bool, error) { 33 | if tv.Succeed { 34 | return true, nil 35 | } 36 | return false, nil 37 | } 38 | 39 | var testValidators = []*Validator{ 40 | { 41 | Name: "test-v1", 42 | Validate: func(input *Input) (bool, error) { return true, nil }, 43 | }, 44 | { 45 | Name: "test-v2", 46 | Validate: func(input *Input) (bool, error) { return false, nil }, 47 | }, 48 | } 49 | 50 | func TestRunValidators(t *testing.T) { 51 | vfMap := ValidationFailureMap(testValidators) 52 | failures := RunValidators(testValidators, &Input{}, vfMap, nil) 53 | 54 | if vfMap.GetKey("test-v1").Int64() != 0 { 55 | t.Error("Got unexpected test-v1 validation failure.") 56 | } 57 | 58 | if vfMap.GetKey("test-v2").Int64() != 1 { 59 | t.Errorf("Didn't get expected test-v2 validation failure.") 60 | } 61 | 62 | if !reflect.DeepEqual(failures, []string{"test-v2"}) { 63 | t.Errorf("Didn't get expected validation failures. Expected: {\"test-v2\"}, Got: %v", failures) 64 | } 65 | } 66 | 67 | func TestValidatorFailureMap(t *testing.T) { 68 | vfMap := ValidationFailureMap(testValidators) 69 | 70 | expectedKeys := []string{"test-v1", "test-v3"} 71 | if reflect.DeepEqual(vfMap.Keys(), expectedKeys) { 72 | t.Errorf("Didn't get expected keys in the mao. Got: %s, Expected: %v", vfMap.Keys(), expectedKeys) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /web/formatutils/formatutils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package formatutils provides web related utils for the cloudprober 16 | // sub-packages. 17 | package formatutils 18 | 19 | import ( 20 | "fmt" 21 | 22 | "github.com/golang/protobuf/proto" 23 | ) 24 | 25 | // ConfToString tries to convert the given conf object into a string. 26 | func ConfToString(conf interface{}) string { 27 | if msg, ok := conf.(proto.Message); ok { 28 | return proto.MarshalTextString(msg) 29 | } 30 | if stringer, ok := conf.(fmt.Stringer); ok { 31 | return stringer.String() 32 | } 33 | return "" 34 | } 35 | -------------------------------------------------------------------------------- /web/status_tmpl.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | var statusTmpl = ` 4 | 5 | 6 | 7 | 29 | 30 | 31 | Started: {{.StartTime}} -- up {{.Uptime}}
    32 | Version: {{.Version}}
    33 | Config: /config
    34 | 35 |

    Probes:

    36 | {{.ProbesStatus}} 37 | 38 |

    Surfacers:

    39 | {{.SurfacersStatus}} 40 | 41 |

    Servers:

    42 | {{.ServersStatus}} 43 | ` 44 | -------------------------------------------------------------------------------- /web/web.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Cloudprober Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package web provides web interface for cloudprober. 16 | package web 17 | 18 | import ( 19 | "bytes" 20 | "fmt" 21 | "html/template" 22 | "net/http" 23 | "time" 24 | 25 | "github.com/google/cloudprober" 26 | "github.com/google/cloudprober/config/runconfig" 27 | "github.com/google/cloudprober/probes" 28 | "github.com/google/cloudprober/servers" 29 | "github.com/google/cloudprober/surfacers" 30 | "github.com/google/cloudprober/sysvars" 31 | ) 32 | 33 | func execTmpl(tmpl *template.Template, v interface{}) template.HTML { 34 | var statusBuf bytes.Buffer 35 | err := tmpl.Execute(&statusBuf, v) 36 | if err != nil { 37 | return template.HTML(template.HTMLEscapeString(err.Error())) 38 | } 39 | return template.HTML(statusBuf.String()) 40 | } 41 | 42 | // Status returns cloudprober status string. 43 | func Status() string { 44 | var statusBuf bytes.Buffer 45 | 46 | probeInfo, surfacerInfo, serverInfo := cloudprober.GetInfo() 47 | startTime := sysvars.StartTime() 48 | uptime := time.Since(startTime) 49 | 50 | tmpl, _ := template.New("statusTmpl").Parse(statusTmpl) 51 | tmpl.Execute(&statusBuf, struct { 52 | Version, StartTime, Uptime, ProbesStatus, ServersStatus, SurfacersStatus interface{} 53 | }{ 54 | Version: runconfig.Version(), 55 | StartTime: startTime.Format(time.RFC1123), 56 | Uptime: uptime.String(), 57 | ProbesStatus: execTmpl(probes.StatusTmpl, probeInfo), 58 | SurfacersStatus: execTmpl(surfacers.StatusTmpl, surfacerInfo), 59 | ServersStatus: execTmpl(servers.StatusTmpl, serverInfo), 60 | }) 61 | 62 | return statusBuf.String() 63 | } 64 | 65 | func configHandler(w http.ResponseWriter, r *http.Request) { 66 | fmt.Fprint(w, cloudprober.GetTextConfig()) 67 | } 68 | 69 | func statusHandler(w http.ResponseWriter, r *http.Request) { 70 | fmt.Fprintf(w, Status()) 71 | } 72 | 73 | // Init initializes cloudprober web interface handler. 74 | func Init() { 75 | http.HandleFunc("/config", configHandler) 76 | http.HandleFunc("/status", statusHandler) 77 | } 78 | --------------------------------------------------------------------------------