├── .go-version ├── .golangci.yml ├── .buildkite ├── hooks │ └── pre-command ├── scripts │ ├── build.sh │ ├── check.sh │ ├── test.sh │ ├── install-prereq.sh │ ├── package.sh │ └── publish.sh ├── pull-requests.json ├── README.md └── pipeline.yml ├── assetbeat.spec ├── conftest.py ├── .github ├── CODEOWNERS ├── dependabot.yml └── workflows │ ├── post-dependabot.yml │ └── docker.yml ├── .gitignore ├── internal ├── tools │ ├── README.md │ └── tools.go ├── notice │ ├── dependencies.csv.tmpl │ ├── rules.json │ ├── NOTICE.txt.tmpl │ └── overrides.json └── dev-tools │ ├── sha.go │ ├── package_types.go │ ├── build_platforms.go │ ├── notice.go │ ├── tar.go │ ├── build.go │ └── package.go ├── make.bat ├── tests ├── files │ ├── config2.yml │ └── config.yml └── e2e │ ├── assets_aws_test.go │ ├── assets_gcp_test.go │ └── assets_k8s_test.go ├── assetbeat.yml ├── LICENSE.txt ├── Makefile ├── version ├── version.go └── helper.go ├── input ├── internal │ ├── ecs.go │ ├── config.go │ ├── ecs_test.go │ ├── config_test.go │ └── publish.go ├── aws │ ├── asset.go │ ├── asset_test.go │ ├── ec2.go │ ├── aws_test.go │ ├── vpc.go │ └── ec2_test.go ├── gcp │ ├── asset.go │ ├── asset_test.go │ ├── gcp_test.go │ ├── compute.go │ ├── util.go │ └── util_test.go ├── testutil │ ├── in_memory_publisher_test.go │ └── in_memory_publisher.go ├── default-inputs │ └── inputs.go ├── azure │ ├── azure_test.go │ └── README.md ├── k8s │ ├── pods_test.go │ ├── containers_test.go │ ├── containers.go │ ├── pods.go │ ├── nodes_test.go │ ├── util.go │ └── nodes.go ├── README.md └── hostdata │ ├── hostdata_test.go │ └── hostdata.go ├── util └── functional.go ├── Dockerfile.reference ├── assetbeat.spec.yml ├── channel ├── interface.go ├── factory.go ├── connector.go ├── outlet.go ├── util_test.go └── util.go ├── catalog-info.yaml ├── main.go ├── cmd └── root.go ├── README.md ├── main_test.go ├── beater ├── signalwait.go ├── channels.go └── crawler.go ├── .mergify.yml ├── deploy └── assetbeat-kubernetes-manifest.yml └── config ├── config.go └── config_test.go /.go-version: -------------------------------------------------------------------------------- 1 | 1.20 -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | timeout: 5m 3 | -------------------------------------------------------------------------------- /.buildkite/hooks/pre-command: -------------------------------------------------------------------------------- 1 | export GO_VERSION=$(cat .go-version) -------------------------------------------------------------------------------- /assetbeat.spec: -------------------------------------------------------------------------------- 1 | {"BinaryPath":"assetbeat","Args":["-e"],"Configurable":"grpc"} 2 | -------------------------------------------------------------------------------- /conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | sys.path.append(os.path.join(os.path.dirname(__file__), '../libbeat/tests/system')) 5 | -------------------------------------------------------------------------------- /.buildkite/scripts/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -uxo pipefail 3 | 4 | # Install prerequirements (go, mage...) 5 | source .buildkite/scripts/install-prereq.sh 6 | 7 | # Build 8 | mage build -------------------------------------------------------------------------------- /.buildkite/scripts/check.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -uxo pipefail 3 | 4 | # Install prerequirements (go, mage...) 5 | source .buildkite/scripts/install-prereq.sh 6 | 7 | # Check 8 | mage check -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # GitHub CODEOWNERS definition 2 | # See: https://help.github.com/articles/about-codeowners/ 3 | 4 | # The assetbeat repository is owned by the @elastic/assetbeat team. 5 | * @elastic/assetbeat 6 | -------------------------------------------------------------------------------- /.buildkite/scripts/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -uxo pipefail 3 | 4 | # Install prerequirements (go, mage...) 5 | source .buildkite/scripts/install-prereq.sh 6 | 7 | # Unit tests 8 | mage unitTest 9 | 10 | # End to end tests 11 | mage e2eTest -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /_meta/filebeat.dev.yml 2 | .vagrant 3 | /docs/html_docs 4 | assetbeat-dev.yml 5 | 6 | assetbeat 7 | data 8 | 9 | build 10 | _meta/module.generated 11 | _meta/beat.yml 12 | _meta/beat.reference.yml 13 | /tests/load/logs 14 | coverage* 15 | .tools 16 | 17 | .idea 18 | .DS_Store 19 | .fleet/ 20 | logs/ 21 | -------------------------------------------------------------------------------- /internal/tools/README.md: -------------------------------------------------------------------------------- 1 | # Internal tools 2 | 3 | The purpose of this package is: 4 | 5 | 1. To allow efficient caching of build/testing tools in CI jobs. 6 | 2. To pin these tools to specific versions to improve consistency & reliability. 7 | 8 | You don't need to do anything to 'enable' this, it's handled in the build file (`magefile.go`). -------------------------------------------------------------------------------- /make.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | REM Windows wrapper for Mage (https://magefile.org/) that installs it 4 | REM to %GOPATH%\bin from the Beats vendor directory. 5 | REM 6 | REM After running this once you may invoke mage.exe directly. 7 | 8 | WHERE mage 9 | IF %ERRORLEVEL% NEQ 0 go get github.com/magefile/mage 10 | 11 | mage %* 12 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: github-actions 4 | directory: / 5 | schedule: 6 | interval: daily 7 | - package-ecosystem: gomod 8 | directory: / 9 | schedule: 10 | interval: daily 11 | - package-ecosystem: gomod 12 | directory: /internal/tools 13 | schedule: 14 | interval: daily 15 | -------------------------------------------------------------------------------- /tests/files/config2.yml: -------------------------------------------------------------------------------- 1 | assetbeat: 2 | inputs: 3 | - type: assets_aws 4 | regions: 5 | - us-east-1 6 | - us-west-1 7 | access_key_id: blahblahblah 8 | secret_access_key: youllneverknow 9 | period: 1h 10 | 11 | output: 12 | elasticsearch: 13 | enabled: true 14 | hosts: ["192.168.99.100:9200"] 15 | -------------------------------------------------------------------------------- /internal/notice/dependencies.csv.tmpl: -------------------------------------------------------------------------------- 1 | {{- define "depInfo" -}} 2 | {{- range $i, $dep := . }} 3 | {{ $dep.Name }},{{ $dep.URL }},{{ $dep.Version | canonicalVersion }},{{ $dep.Version | revision }},{{ $dep.LicenceType }}, 4 | {{- end -}} 5 | {{- end -}} 6 | 7 | name,url,version,revision,license,sourceURL{{ template "depInfo" .Direct }}{{ template "depInfo" .Indirect }} -------------------------------------------------------------------------------- /internal/notice/rules.json: -------------------------------------------------------------------------------- 1 | { 2 | "allowlist": [ 3 | "Apache-2.0", 4 | "BSD", 5 | "BSD-2-Clause", 6 | "BSD-2-Clause-FreeBSD", 7 | "BSD-3-Clause", 8 | "CC0-1.0", 9 | "Elastic", 10 | "ISC", 11 | "MIT", 12 | "MPL-2.0", 13 | "Public Domain", 14 | "Unlicense", 15 | "Zlib" 16 | ], 17 | "maybelist": [ 18 | "EPL-1.0", 19 | "GPL-3.0" 20 | ] 21 | } -------------------------------------------------------------------------------- /tests/files/config.yml: -------------------------------------------------------------------------------- 1 | assetbeat: 2 | inputs: 3 | - type: assets_aws 4 | regions: 5 | - us-east-1 6 | - us-west-1 7 | - eu-west-2 8 | - eu-north-1 9 | access_key_id: blahblahblah 10 | secret_access_key: youllneverknow 11 | period: 1h 12 | 13 | output: 14 | elasticsearch: 15 | enabled: true 16 | hosts: ["192.168.99.100:9200"] 17 | -------------------------------------------------------------------------------- /.buildkite/scripts/install-prereq.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euxo pipefail 3 | 4 | # Install mage 5 | if ! command -v mage &>/dev/null; then 6 | go install github.com/magefile/mage@latest 7 | else 8 | echo "Mage is already installed. Skipping installation..." 9 | fi 10 | 11 | # To avoid the following error while executing go build 12 | # " error obtaining VCS status: exit status 128. Use -buildvcs=false to disable VCS stamping. " 13 | go env -w GOFLAGS="-buildvcs=false" -------------------------------------------------------------------------------- /.buildkite/scripts/package.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -uxo pipefail 3 | 4 | export PLATFORMS="linux/amd64 linux/arm64" 5 | export TYPES="tar.gz" 6 | 7 | WORKFLOW=$1 8 | 9 | if [ "$WORKFLOW" = "snapshot" ] ; then 10 | export SNAPSHOT="true" 11 | fi 12 | 13 | 14 | # Install prerequirements (go, mage...) 15 | source .buildkite/scripts/install-prereq.sh 16 | 17 | # Download Go dependencies 18 | go mod download 19 | 20 | # Packaging the assetbeat binary 21 | mage package 22 | 23 | # Generate the CSV dependency report 24 | mage dependencyReport -------------------------------------------------------------------------------- /assetbeat.yml: -------------------------------------------------------------------------------- 1 | assetbeat.inputs: 2 | - type: assets_aws 3 | regions: 4 | - us-east-1 5 | - us-west-1 6 | - eu-west-2 7 | access_key_id: 8 | secret_access_key: 9 | session_token: 10 | period: 600s 11 | 12 | output.elasticsearch: 13 | hosts: ["localhost:9200"] 14 | protocol: "https" 15 | username: "elastic" 16 | password: "changeme" 17 | ssl.verification_mode: "none" 18 | 19 | 20 | logging.level: info 21 | logging.to_files: false 22 | logging.to_stderr: true 23 | logging.selectors: ["*"] 24 | -------------------------------------------------------------------------------- /.buildkite/pull-requests.json: -------------------------------------------------------------------------------- 1 | { 2 | "jobs": [ 3 | { 4 | "enabled": true, 5 | "pipeline_slug": "assetbeat", 6 | "allow_org_users": true, 7 | "allowed_repo_permissions": ["admin", "write"], 8 | "build_on_commit": true, 9 | "build_on_comment": true, 10 | "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))", 11 | "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))", 12 | "skip_ci_labels": ["skip-ci"], 13 | "skip_ci_on_only_changed": ["\\.md$"] 14 | } 15 | ] 16 | } -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2023 Elastic 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BEAT_NAME?=assetbeat 2 | BEAT_TITLE?=Inputruner 3 | SYSTEM_TESTS?=true 4 | TEST_ENVIRONMENT?=true 5 | GOX_FLAGS=-arch="amd64 386 arm ppc64 ppc64le" 6 | ES_BEATS?=.. 7 | EXCLUDE_COMMON_UPDATE_TARGET=true 8 | 9 | include ${ES_BEATS}/libbeat/scripts/Makefile 10 | 11 | .PHONY: update 12 | update: mage 13 | mage update 14 | 15 | # Creates a new module. Requires the params MODULE. 16 | .PHONY: create-module 17 | create-module: mage 18 | mage generate:module 19 | 20 | # Creates a new fileset. Requires the params MODULE and FILESET. 21 | .PHONY: create-fileset 22 | create-fileset: mage 23 | mage generate:fileset 24 | 25 | # Creates a fields.yml based on a pipeline.json file. Requires the params MODULE and FILESET. 26 | .PHONY: create-fields 27 | create-fields: mage 28 | mage generate:fields 29 | -------------------------------------------------------------------------------- /internal/notice/NOTICE.txt.tmpl: -------------------------------------------------------------------------------- 1 | {{- define "depInfo" -}} 2 | {{- range $i, $dep := . }} 3 | {{ "-" | line }} 4 | Dependency : {{ $dep.Name }} 5 | Version: {{ $dep.Version }} 6 | Licence type (autodetected): {{ $dep.LicenceType }} 7 | {{ "-" | line }} 8 | 9 | {{ $dep | licenceText }} 10 | {{ end }} 11 | {{- end -}} 12 | 13 | Elastic Assetbeat 14 | Copyright 2023-{{ currentYear }} Elasticsearch BV 15 | 16 | This product includes software developed by The Apache Software 17 | Foundation (http://www.apache.org/). 18 | 19 | {{ "=" | line }} 20 | Third party libraries used by the Elastic Assetbeat project: 21 | {{ "=" | line }} 22 | 23 | {{ template "depInfo" .Direct }} 24 | 25 | {{ if .Indirect }} 26 | {{ "=" | line }} 27 | Indirect dependencies 28 | 29 | {{ template "depInfo" .Indirect -}} 30 | {{- end}} -------------------------------------------------------------------------------- /internal/tools/tools.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | //go:build tools 19 | 20 | package tools 21 | 22 | import ( 23 | _ "github.com/elastic/go-licenser" 24 | _ "go.elastic.co/go-licence-detector" 25 | ) 26 | -------------------------------------------------------------------------------- /version/version.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package version 19 | 20 | // defaultVersion is the current release version of assetbeat, this version must match the Elastic Agent version 21 | const defaultVersion = "8.12.0" 22 | -------------------------------------------------------------------------------- /input/internal/ecs.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package internal 19 | 20 | import ( 21 | "github.com/elastic/beats/v7/libbeat/beat" 22 | ) 23 | 24 | func WithCloudInstanceId(instanceId string) AssetOption { 25 | return func(e beat.Event) beat.Event { 26 | e.Fields["cloud.instance.id"] = instanceId 27 | return e 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /input/aws/asset.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package aws 19 | 20 | import ( 21 | "github.com/elastic/assetbeat/input/internal" 22 | "github.com/elastic/elastic-agent-libs/mapstr" 23 | ) 24 | 25 | func WithAssetTags(value mapstr.M) internal.AssetOption { 26 | return internal.WithAssetMetadata(mapstr.M{ 27 | "tags": value, 28 | }) 29 | } 30 | -------------------------------------------------------------------------------- /input/gcp/asset.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package gcp 19 | 20 | import ( 21 | "github.com/elastic/assetbeat/input/internal" 22 | "github.com/elastic/elastic-agent-libs/mapstr" 23 | ) 24 | 25 | func WithAssetLabels(value mapstr.M) internal.AssetOption { 26 | return internal.WithAssetMetadata(mapstr.M{ 27 | "labels": value, 28 | }) 29 | } 30 | -------------------------------------------------------------------------------- /util/functional.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package util 19 | 20 | // Map applies a function to each element in a slice, returning a new slice with the results. 21 | func Map[I any, O any](mapper func(I) O, iter []I) []O { 22 | out := make([]O, len(iter)) 23 | for i, in := range iter { 24 | out[i] = mapper(in) 25 | } 26 | return out 27 | } 28 | -------------------------------------------------------------------------------- /Dockerfile.reference: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | 3 | RUN mkdir -p /usr/share/assetbeat 4 | COPY assetbeat /usr/share/assetbeat/assetbeat 5 | 6 | RUN mkdir -p /usr/share/assetbeat/data /usr/share/assetbeat/logs && \ 7 | chown -R root:root /usr/share/assetbeat && \ 8 | find /usr/share/assetbeat -type d -exec chmod 0755 {} \; && \ 9 | find /usr/share/assetbeat -type f -exec chmod 0644 {} \; && \ 10 | chmod 0775 /usr/share/assetbeat/data /usr/share/assetbeat/logs 11 | 12 | 13 | RUN chmod 0755 /usr/share/assetbeat/assetbeat 14 | RUN for iter in {1..10}; do \ 15 | apt-get update -y && \ 16 | DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --yes ca-certificates curl coreutils gawk libcap2-bin xz-utils && \ 17 | apt-get clean all && \ 18 | exit_code=0 && break || exit_code=$? && echo "apt-get error: retry $iter in 10s" && sleep 10; \ 19 | done; \ 20 | (exit $exit_code) 21 | 22 | 23 | RUN groupadd --gid 1000 assetbeat 24 | RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home /usr/share/assetbeat assetbeat 25 | USER assetbeat 26 | 27 | WORKDIR /usr/share/assetbeat 28 | CMD [ "/bin/bash", "-c", "./assetbeat", "run" ] -------------------------------------------------------------------------------- /input/internal/config.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package internal 19 | 20 | import ( 21 | "time" 22 | ) 23 | 24 | type BaseConfig struct { 25 | Period time.Duration `config:"period"` 26 | AssetTypes []string `config:"asset_types"` 27 | } 28 | 29 | func IsTypeEnabled(configuredTypes []string, currentType string) bool { 30 | if len(configuredTypes) == 0 { 31 | return true 32 | } 33 | 34 | for _, t := range configuredTypes { 35 | if currentType == t { 36 | return true 37 | } 38 | } 39 | 40 | return false 41 | } 42 | -------------------------------------------------------------------------------- /assetbeat.spec.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | inputs: 3 | - name: assetbeat 4 | description: "Assetbeat" 5 | platforms: &platforms 6 | - linux/amd64 7 | - linux/arm64 8 | outputs: &outputs 9 | - elasticsearch 10 | command: &command 11 | restart_monitoring_period: 5s 12 | maximum_restarts_per_period: 1 13 | timeouts: 14 | restart: 1s 15 | args: 16 | - "-E" 17 | - "setup.ilm.enabled=false" 18 | - "-E" 19 | - "setup.template.enabled=false" 20 | - "-E" 21 | - "management.enabled=true" 22 | - "-E" 23 | - "management.restart_on_output_change=true" 24 | - "-E" 25 | - "logging.level=info" 26 | - "-E" 27 | - "logging.to_stderr=true" 28 | - "-E" 29 | - "gc_percent=${ASSETBEAT_GOGC:100}" 30 | - name: assets_k8s 31 | description: "Kubernetes assets" 32 | platforms: *platforms 33 | outputs: *outputs 34 | command: *command 35 | - name: assets_aws 36 | description: "AWS assets" 37 | platforms: *platforms 38 | outputs: *outputs 39 | command: *command 40 | - name: assets_gcp 41 | description: "GCP Assets" 42 | platforms: *platforms 43 | outputs: *outputs 44 | command: *command 45 | - name: assets_azure 46 | description: "Azure Assets" 47 | platforms: *platforms 48 | outputs: *outputs 49 | command: *command -------------------------------------------------------------------------------- /version/helper.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package version 19 | 20 | import "os" 21 | 22 | // This variable needs to be overriden at build time, if a version qualifier is provided 23 | var buildQualifier = "" 24 | 25 | var Qualifier, HasQualifier = os.LookupEnv("VERSION_QUALIFIER") 26 | 27 | func GetBuildVersion() string { 28 | if buildQualifier == "" { 29 | return defaultVersion 30 | } 31 | return defaultVersion + "-" + buildQualifier 32 | } 33 | 34 | func GetVersion() string { 35 | if Qualifier == "" { 36 | return defaultVersion 37 | } 38 | return defaultVersion + "-" + Qualifier 39 | } 40 | -------------------------------------------------------------------------------- /input/testutil/in_memory_publisher_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package testutil 19 | 20 | import ( 21 | "testing" 22 | 23 | "github.com/elastic/beats/v7/libbeat/beat" 24 | "github.com/elastic/elastic-agent-libs/mapstr" 25 | "github.com/stretchr/testify/assert" 26 | ) 27 | 28 | func TestInMemoryPublisher(t *testing.T) { 29 | p := NewInMemoryPublisher() 30 | 31 | event := beat.Event{Fields: mapstr.M{ 32 | "hello": "world", 33 | }} 34 | 35 | assert.Equal(t, 0, len(p.Events)) 36 | p.Publish(event) 37 | assert.Equal(t, 1, len(p.Events)) 38 | assert.Equal(t, event, p.Events[0]) 39 | 40 | p.Publish(event) 41 | assert.Equal(t, 2, len(p.Events)) 42 | } 43 | -------------------------------------------------------------------------------- /input/testutil/in_memory_publisher.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package testutil 19 | 20 | import ( 21 | "sync" 22 | 23 | "github.com/elastic/beats/v7/libbeat/beat" 24 | ) 25 | 26 | // InMemoryPublisher is a publisher which stores events in memory, to be used 27 | // in unit tests 28 | type InMemoryPublisher struct { 29 | mu sync.Mutex 30 | Events []beat.Event 31 | } 32 | 33 | // NewInMemoryPublisher creates a new instance of InMemoryPublisher 34 | func NewInMemoryPublisher() *InMemoryPublisher { 35 | return &InMemoryPublisher{} 36 | } 37 | 38 | // Publish stores a new event in memory 39 | func (p *InMemoryPublisher) Publish(e beat.Event) { 40 | p.mu.Lock() 41 | defer p.mu.Unlock() 42 | 43 | p.Events = append(p.Events, e) 44 | } 45 | -------------------------------------------------------------------------------- /.buildkite/README.md: -------------------------------------------------------------------------------- 1 | # Buildkite 2 | 3 | This README provides an overview of the Buildkite pipeline used to automate the build and publish process for Assetbeat artifacts. 4 | 5 | **_NOTE_: The pipeline is still a work in progress and in testing phase. Frequent changes are expected** 6 | 7 | ## Artifacts 8 | 9 | The pipeline generates the following artifacts: 10 | 11 | - **assetbeat-ASSETBEAT_VERSION-WORKFLOW-GOOS-GOARCH.tar.gz**: This tarball includes the `assetbeat` binary and other related files (e.g LICENSE, assetbeat.yml, etc.). The supported platforms for the artifacts are linux/amd64 and linux/arm64. 12 | - **assetbeat-ASSETBEAT_VERSION-WORKFLOW-GOOS-GOARCH.tar.gz.sha512** The sha512 hash of the above tarball. 13 | 14 | ## Triggering the Pipeline 15 | 16 | The pipeline is triggered in the following scenarios: 17 | 18 | - **Snapshot Builds**: A snapshot build is triggered when a pull request (PR) is opened and also when it is merged into the 'main' branch. 19 | 20 | ## Pipeline Configuration 21 | 22 | To view the pipeline and its configuration, click [here](https://buildkite.com/elastic/assetbeat). 23 | 24 | ## Test pipeline changes locally 25 | 26 | Buildkite provides a command line tool, named `bk`, to run pipelines locally. To perform a local run, you need to 27 | 28 | 1. [Install Buildkite agent.](https://buildkite.com/docs/agent/v3/installation) 29 | 2. [Install `bk` cli](https://github.com/buildkite/cli) 30 | 3. Execute `bk local run` inside this repo. 31 | 32 | For more information, please click [here](https://buildkite.com/changelog/44-run-pipelines-locally-with-bk-cli) 33 | -------------------------------------------------------------------------------- /channel/interface.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package channel 19 | 20 | import ( 21 | "github.com/elastic/beats/v7/libbeat/beat" 22 | conf "github.com/elastic/elastic-agent-libs/config" 23 | ) 24 | 25 | // Factory is used to create a new Outlet instance 26 | type Factory func(beat.PipelineConnector) Connector 27 | 28 | // Connector creates an Outlet connecting the event publishing with some internal pipeline. 29 | // type Connector func(*conf.C, *mapstr.Pointer) (Outleter, error) 30 | type Connector interface { 31 | Connect(*conf.C) (Outleter, error) 32 | ConnectWith(*conf.C, beat.ClientConfig) (Outleter, error) 33 | } 34 | 35 | // Outleter is the outlet for an input 36 | type Outleter interface { 37 | Close() error 38 | Done() <-chan struct{} 39 | OnEvent(beat.Event) bool 40 | } 41 | -------------------------------------------------------------------------------- /catalog-info.yaml: -------------------------------------------------------------------------------- 1 | # Declare a Backstage Component that represents your application. 2 | --- 3 | # yaml-language-server: $schema=https://json.schemastore.org/catalog-info.json 4 | apiVersion: backstage.io/v1alpha1 5 | kind: Component 6 | metadata: 7 | name: assetbeat 8 | 9 | spec: 10 | type: application 11 | owner: group:observability-asset-management 12 | lifecycle: beta 13 | 14 | --- 15 | # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json 16 | apiVersion: backstage.io/v1alpha1 17 | kind: Resource 18 | metadata: 19 | name: buildkite-pipeline-assetbeat 20 | description: Buildkite Pipeline for assetbeat 21 | links: 22 | - title: Pipeline 23 | url: https://buildkite.com/elastic/assetbeat 24 | 25 | spec: 26 | type: buildkite-pipeline 27 | owner: group:observability-asset-management 28 | system: buildkite 29 | implementation: 30 | apiVersion: buildkite.elastic.dev/v1 31 | kind: Pipeline 32 | metadata: 33 | name: assetbeat 34 | spec: 35 | repository: elastic/assetbeat 36 | pipeline_file: ".buildkite/pipeline.yml" 37 | env: 38 | ELASTIC_PR_COMMENTS_ENABLED: 'true' 39 | provider_settings: 40 | build_pull_requests: true 41 | publish_commit_status: true 42 | publish_blocked_as_pending: true 43 | cancel_deleted_branch_builds: true 44 | teams: 45 | observability-asset-management: 46 | access_level: MANAGE_BUILD_AND_READ 47 | everyone: 48 | access_level: READ_ONLY 49 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package main 19 | 20 | import ( 21 | "os" 22 | 23 | "github.com/elastic/assetbeat/cmd" 24 | 25 | inputs "github.com/elastic/assetbeat/input/default-inputs" 26 | ) 27 | 28 | // The basic model of execution: 29 | // - input: finds files in paths/globs to harvest, starts harvesters 30 | // - harvester: reads a file, sends events to the spooler 31 | // - spooler: buffers events until ready to flush to the publisher 32 | // - publisher: writes to the network, notifies registrar 33 | // - registrar: records positions of files read 34 | // Finally, input uses the registrar information, on restart, to 35 | // determine where in each file to restart a harvester. 36 | func main() { 37 | if err := cmd.Assetbeat(inputs.Init, cmd.AssetbeatSettings()).Execute(); err != nil { 38 | os.Exit(1) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /channel/factory.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package channel 19 | 20 | import ( 21 | "github.com/elastic/beats/v7/libbeat/beat" 22 | ) 23 | 24 | type OutletFactory struct { 25 | done <-chan struct{} 26 | } 27 | 28 | // NewOutletFactory creates a new outlet factory for 29 | // connecting an input to the publisher pipeline. 30 | func NewOutletFactory(done <-chan struct{}) *OutletFactory { 31 | o := &OutletFactory{ 32 | done: done, 33 | } 34 | 35 | return o 36 | } 37 | 38 | // Create builds a new Outleter, while applying common input settings. 39 | // Inputs and all harvesters use the same pipeline client instance. 40 | // This guarantees ordering between events as required by the registrar for 41 | // file.State updates 42 | func (f *OutletFactory) Create(p beat.PipelineConnector) Connector { 43 | return &pipelineConnector{parent: f, pipeline: p} 44 | } 45 | -------------------------------------------------------------------------------- /.github/workflows/post-dependabot.yml: -------------------------------------------------------------------------------- 1 | # Follow-on actions relating to dependabot PRs. In elastic/beats, any changes to 2 | # dependencies contained in go.mod requires the change to be reflected in the 3 | # NOTICE.txt file. When dependabot creates a branch for a go_modules change this 4 | # will update the NOTICE.txt file for that change. 5 | name: post-dependabot 6 | 7 | on: 8 | push: 9 | branches: 10 | - 'dependabot/go_modules/**' 11 | 12 | env: 13 | DEFAULT_GO_VERSION: "1.20" 14 | jobs: 15 | update-notice: 16 | permissions: 17 | # Allow job to write to the branch. 18 | contents: write 19 | runs-on: ubuntu-latest 20 | steps: 21 | - uses: actions/checkout@v4 22 | 23 | - name: Set up Go 24 | uses: actions/setup-go@v4 25 | with: 26 | go-version-file: .go-version 27 | 28 | - name: update notice 29 | uses: magefile/mage-action@v3 30 | with: 31 | version: latest 32 | args: update 33 | 34 | - name: check for modified NOTICE.txt 35 | id: notice-check 36 | run: echo "modified=$(if git status --porcelain --untracked-files=no | grep -q -E ' NOTICE.txt$'; then echo "true"; else echo "false"; fi)" >> $GITHUB_OUTPUT 37 | 38 | - name: commit NOTICE.txt 39 | if: steps.notice-check.outputs.modified == 'true' 40 | run: | 41 | git config --global user.name 'dependabot[bot]' 42 | git config --global user.email 'dependabot[bot]@users.noreply.github.com' 43 | git add NOTICE.txt 44 | git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} 45 | git commit -m "Update NOTICE.txt" 46 | git push -------------------------------------------------------------------------------- /.buildkite/scripts/publish.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | WORKFLOW=$1 5 | # The "branch" here selects which "$BRANCH.gradle" file of release manager is used 6 | export VERSION=$(grep defaultVersion version/version.go | cut -f2 -d "\"" | tail -n1) 7 | MAJOR=$(echo $VERSION | awk -F. '{ print $1 }') 8 | MINOR=$(echo $VERSION | awk -F. '{ print $2 }') 9 | if [ -n "$(git ls-remote --heads origin $MAJOR.$MINOR)" ] ; then 10 | BRANCH=$MAJOR.$MINOR 11 | elif [ -n "$(git ls-remote --heads origin $MAJOR.x)" ] ; then 12 | BRANCH=$MAJOR.x 13 | else 14 | BRANCH=main 15 | fi 16 | 17 | # Download artifacts from other stages 18 | echo "Downloading artifacts..." 19 | buildkite-agent artifact download "build/distributions/*" "." --step package-"${WORKFLOW}" 20 | 21 | # Fix file permissions 22 | chmod -R a+r build/* 23 | chmod -R a+w build 24 | 25 | # Shared secret path containing the dra creds for project teams 26 | echo "Retrieving DRA crededentials..." 27 | DRA_CREDS=$(vault kv get -field=data -format=json kv/ci-shared/release/dra-role) 28 | 29 | # Run release-manager 30 | echo "Running release-manager container..." 31 | IMAGE="docker.elastic.co/infra/release-manager:latest" 32 | docker run --rm \ 33 | --name release-manager \ 34 | -e VAULT_ADDR=$(echo $DRA_CREDS | jq -r '.vault_addr') \ 35 | -e VAULT_ROLE_ID=$(echo $DRA_CREDS | jq -r '.role_id') \ 36 | -e VAULT_SECRET_ID=$(echo $DRA_CREDS | jq -r '.secret_id') \ 37 | --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \ 38 | "$IMAGE" \ 39 | cli collect \ 40 | --project assetbeat \ 41 | --branch "${BRANCH}" \ 42 | --commit "${BUILDKITE_COMMIT}" \ 43 | --workflow "${WORKFLOW}" \ 44 | --version "${VERSION}" \ 45 | --artifact-set main -------------------------------------------------------------------------------- /input/default-inputs/inputs.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package inputs 19 | 20 | import ( 21 | "github.com/elastic/assetbeat/beater" 22 | "github.com/elastic/assetbeat/input/aws" 23 | "github.com/elastic/assetbeat/input/azure" 24 | "github.com/elastic/assetbeat/input/gcp" 25 | "github.com/elastic/assetbeat/input/hostdata" 26 | "github.com/elastic/assetbeat/input/k8s" 27 | v2 "github.com/elastic/beats/v7/filebeat/input/v2" 28 | "github.com/elastic/beats/v7/libbeat/beat" 29 | "github.com/elastic/elastic-agent-libs/logp" 30 | ) 31 | 32 | func Init(info beat.Info, log *logp.Logger, components beater.StateStore) []v2.Plugin { 33 | return genericInputs(log, components) 34 | } 35 | 36 | func genericInputs(log *logp.Logger, components beater.StateStore) []v2.Plugin { 37 | return []v2.Plugin{ 38 | aws.Plugin(), 39 | gcp.Plugin(), 40 | azure.Plugin(), 41 | hostdata.Plugin(), 42 | k8s.Plugin(), 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /internal/dev-tools/sha.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package dev_tools 19 | 20 | import ( 21 | "crypto/sha512" 22 | "encoding/hex" 23 | "fmt" 24 | "io" 25 | "os" 26 | "path/filepath" 27 | ) 28 | 29 | // CreateSHA512File computes the sha512 sum of the specified file the writes 30 | // a sidecar file containing the hash and filename. 31 | func CreateSHA512File(file string) error { 32 | fmt.Printf("Creating SHA512 hash... Filepath: %s\n", file+".sha512") 33 | f, err := os.Open(file) 34 | if err != nil { 35 | return fmt.Errorf("failed to open file for sha512 summing. Error %s", err) 36 | } 37 | defer f.Close() 38 | 39 | sum := sha512.New() 40 | if _, err := io.Copy(sum, f); err != nil { 41 | return fmt.Errorf("failed reading from input file. Error %s", err) 42 | } 43 | 44 | computedHash := hex.EncodeToString(sum.Sum(nil)) 45 | out := fmt.Sprintf("%v %v", computedHash, filepath.Base(file)) 46 | 47 | return os.WriteFile(file+".sha512", []byte(out), 0644) 48 | } 49 | -------------------------------------------------------------------------------- /input/internal/ecs_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package internal 19 | 20 | import ( 21 | "testing" 22 | 23 | "github.com/stretchr/testify/assert" 24 | 25 | "github.com/elastic/assetbeat/input/testutil" 26 | "github.com/elastic/beats/v7/libbeat/beat" 27 | "github.com/elastic/elastic-agent-libs/mapstr" 28 | ) 29 | 30 | func TestECS_WithCloudInstanceId(t *testing.T) { 31 | for _, tt := range []struct { 32 | name string 33 | assetOp AssetOption 34 | expected beat.Event 35 | }{ 36 | { 37 | name: "instance Id provided", 38 | assetOp: WithCloudInstanceId("i-0699b78f46f0fa248"), 39 | expected: beat.Event{ 40 | Fields: mapstr.M{"cloud.instance.id": "i-0699b78f46f0fa248"}, 41 | Meta: mapstr.M{"index": GetDefaultIndexName()}, 42 | }, 43 | }, 44 | } { 45 | t.Run(tt.name, func(t *testing.T) { 46 | publisher := testutil.NewInMemoryPublisher() 47 | 48 | Publish(publisher, nil, tt.assetOp) 49 | 50 | assert.Equal(t, 1, len(publisher.Events)) 51 | assert.Equal(t, tt.expected, publisher.Events[0]) 52 | }) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /cmd/root.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package cmd 19 | 20 | import ( 21 | "github.com/elastic/assetbeat/version" 22 | "github.com/spf13/pflag" 23 | 24 | "github.com/elastic/assetbeat/beater" 25 | "github.com/elastic/beats/v7/libbeat/cmd" 26 | "github.com/elastic/beats/v7/libbeat/cmd/instance" 27 | ) 28 | 29 | // Name of this beat 30 | const Name = "assetbeat" 31 | 32 | // RootCmd to handle beats cli 33 | var RootCmd *cmd.BeatsRootCmd 34 | 35 | // AssetbeatSettings contains the default settings for assetbeat 36 | func AssetbeatSettings() instance.Settings { 37 | runFlags := pflag.NewFlagSet(Name, pflag.ExitOnError) 38 | return instance.Settings{ 39 | RunFlags: runFlags, 40 | Name: Name, 41 | Version: version.GetBuildVersion(), 42 | HasDashboards: false, 43 | } 44 | } 45 | 46 | // Assetbeat builds the beat root command for executing assetbeat and it's subcommands. 47 | func Assetbeat(inputs beater.PluginFactory, settings instance.Settings) *cmd.BeatsRootCmd { 48 | command := cmd.GenRootCmdWithSettings(beater.New(inputs), settings) 49 | return command 50 | } 51 | -------------------------------------------------------------------------------- /.buildkite/pipeline.yml: -------------------------------------------------------------------------------- 1 | env: 2 | GO_AGENT_IMAGE: golang:${GO_VERSION} 3 | steps: 4 | - label: ":white_check_mark: Check" 5 | key: check 6 | command: 7 | - .buildkite/scripts/check.sh 8 | agents: 9 | image: ${GO_AGENT_IMAGE} 10 | cpu: "8" 11 | memory: 8G 12 | - label: ":building_construction: Build" 13 | key: build 14 | command: 15 | - .buildkite/scripts/build.sh 16 | agents: 17 | image: ${GO_AGENT_IMAGE} 18 | cpu: "8" 19 | memory: 4G 20 | - label: ":test_tube: Test" 21 | key: test 22 | command: 23 | - .buildkite/scripts/test.sh 24 | agents: 25 | image: ${GO_AGENT_IMAGE} 26 | cpu: "8" 27 | memory: 4G 28 | - label: ":package: Package Assetbeat - Snapshot" 29 | key: package-snapshot 30 | depends_on: 31 | - check 32 | - build 33 | - test 34 | command: ./.buildkite/scripts/package.sh snapshot 35 | artifact_paths: build/distributions/* 36 | agents: 37 | image: ${GO_AGENT_IMAGE} 38 | cpu: "8" 39 | memory: 4G 40 | - label: ":rocket: Publishing Snapshot DRA artifacts" 41 | if: build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9]+\$/ 42 | depends_on: package-snapshot 43 | command: ./.buildkite/scripts/publish.sh snapshot 44 | agents: 45 | provider: gcp 46 | - label: ":package: Package Assetbeat - Staging" 47 | key: package-staging 48 | depends_on: 49 | - check 50 | - build 51 | - test 52 | command: ./.buildkite/scripts/package.sh staging 53 | artifact_paths: build/distributions/* 54 | agents: 55 | image: ${GO_AGENT_IMAGE} 56 | cpu: "8" 57 | memory: 4G 58 | - label: ":rocket: Publishing Staging DRA artifacts" 59 | if: build.branch =~ /^[0-9]+\.[0-9]+\$/ 60 | depends_on: package-staging 61 | command: ./.buildkite/scripts/publish.sh staging 62 | agents: 63 | provider: gcp 64 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build status](https://badge.buildkite.com/c02badfd4c5a879748bbc27ecfc19147c296af2410391dc749.svg?branch=main)](https://buildkite.com/elastic/assetbeat) 2 | [![Go Report Card](https://goreportcard.com/badge/github.com/elastic/assetbeat)](https://goreportcard.com/report/github.com/elastic/assetbeat) 3 | 4 | # Assetbeat 5 | 6 | Assetbeat is a small binary for collecting information about infrastructure "assets". Assets are defined as elements within your infrastructure, such as containers, machines, pods, clusters, etc. 7 | 8 | **Note:** Assetbeat is currently in technical preview and may be subject to frequent changes. Elastic will apply best effort to fix any issues, but features in technical preview are not subject to the support SLA of official GA features. 9 | 10 | ## Inputs 11 | 12 | Documentation for each input can be found in the relevant directory (e.g. input/aws). 13 | 14 | ## Development 15 | 16 | Requirements: 17 | - go 1.20+ 18 | - [Mage](https://magefile.org/) 19 | 20 | Mage targets are self-explanatory and can be listed with `mage -l`. 21 | 22 | Build the assetbeat binary with `mage build`, and run it locally with `./assetbeat`. 23 | See `./assetbeat -h` for more detail on configuration options. 24 | 25 | Run `mage update` before creating new PRs. This command automatically updates `go.mod`, add license headers to any new *.go files and re-generate 26 | NOTICE.txt. Also double-check that `mage check` returns with no errors, as the PR CI will fail otherwise. 27 | 28 | Please aim for 100% unit test coverage on new code. 29 | You can view the HTML coverage report by running `mage unitTest && [xdg-]open ./coverage.html`. 30 | 31 | ### Requirements for inputs 32 | 33 | - Compatible with [Elastic Agent v2](https://github.com/elastic/elastic-agent/blob/main/docs/architecture.md) 34 | - No [Cgo](https://pkg.go.dev/cmd/cgo) allowed 35 | - Stateless (including publisher) 36 | - Config must be compatible with Elastic Agent 37 | -------------------------------------------------------------------------------- /internal/notice/overrides.json: -------------------------------------------------------------------------------- 1 | {"name": "github.com/elastic/beats/v7", "licenceType": "Elastic"} 2 | {"name": "github.com/elastic/elastic-agent-client/v7", "licenceType": "Elastic"} 3 | {"name": "github.com/elastic/elastic-agent-shipper-client", "licenceType": "Elastic"} 4 | {"name": "github.com/gorhill/cronexpr", "licenceType": "Apache-2.0", "licenceFile":"APLv2"} 5 | {"name": "github.com/hashicorp/cronexpr", "licenceType": "Apache-2.0", "licenceFile":"APLv2"} 6 | {"name": "github.com/miekg/dns", "licenceType": "BSD"} 7 | {"name": "github.com/kr/logfmt", "licenceFile": "Readme", "licenceType": "MIT"} 8 | {"name": "github.com/samuel/go-parser", "licenceType": "BSD-3-Clause"} 9 | {"name": "github.com/xeipuuv/gojsonpointer", "licenceFile": "LICENSE-APACHE-2.0.txt", "licenceType": "Apache-2.0"} 10 | {"name": "github.com/xeipuuv/gojsonreference", "licenceFile": "LICENSE-APACHE-2.0.txt", "licenceType": "Apache-2.0"} 11 | {"name": "github.com/xeipuuv/gojsonschema", "licenceFile": "LICENSE-APACHE-2.0.txt", "licenceType": "Apache-2.0"} 12 | {"name": "github.com/chzyer/logex", "licenceType": "MIT"} 13 | {"name": "github.com/munnerz/goautoneg", "licenceType": "BSD-3-Clause"} 14 | {"name": "github.com/pelletier/go-buffruneio", "licenceType": "MIT"} 15 | {"name": "github.com/urso/magetools", "licenceType": "Apache-2.0"} 16 | {"name": "kernel.org/pub/linux/libs/security/libcap/cap", "licenceType": "BSD-3-Clause", "note": "dual licensed as GPL-v2 and BSD"} 17 | {"name": "kernel.org/pub/linux/libs/security/libcap/psx", "licenceType": "BSD-3-Clause", "note": "dual licensed as GPL-v2 and BSD"} 18 | {"name": "github.com/awslabs/kinesis-aggregation/go/v2", "licenceType": "Apache-2.0", "url": "https://github.com/awslabs/kinesis-aggregation/blob/master/LICENSE.txt"} 19 | {"name": "github.com/dnaeon/go-vcr", "licenceType": "BSD-2-Clause"} 20 | {"name": "github.com/JohnCGriffin/overflow", "licenceType": "MIT"} 21 | {"name": "github.com/tenntenn/text/transform", "licenceType": "GPL-3.0"} 22 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package main 19 | 20 | // This file is mandatory as otherwise the packetbeat.test binary is not generated correctly. 21 | 22 | import ( 23 | "flag" 24 | "os" 25 | "testing" 26 | 27 | ircmd "github.com/elastic/assetbeat/cmd" 28 | inputs "github.com/elastic/assetbeat/input/default-inputs" 29 | "github.com/elastic/beats/v7/libbeat/cmd" 30 | "github.com/elastic/beats/v7/libbeat/tests/system/template" 31 | ) 32 | 33 | var ( 34 | systemTest *bool 35 | irCommand *cmd.BeatsRootCmd 36 | ) 37 | 38 | func init() { 39 | testing.Init() 40 | systemTest = flag.Bool("systemTest", false, "Set to true when running system tests") 41 | irCommand = ircmd.Assetbeat(inputs.Init, ircmd.AssetbeatSettings()) 42 | irCommand.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("systemTest")) 43 | irCommand.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("test.coverprofile")) 44 | } 45 | 46 | // Test started when the test binary is started. Only calls main. 47 | func TestSystem(t *testing.T) { 48 | if *systemTest { 49 | if err := irCommand.Execute(); err != nil { 50 | os.Exit(1) 51 | } 52 | } 53 | } 54 | 55 | func TestTemplate(t *testing.T) { 56 | template.TestTemplate(t, irCommand.Name(), false) 57 | } 58 | -------------------------------------------------------------------------------- /input/internal/config_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package internal 19 | 20 | import ( 21 | "github.com/stretchr/testify/assert" 22 | "testing" 23 | ) 24 | 25 | func TestAssets_IsTypeEnabled(t *testing.T) { 26 | for _, tt := range []struct { 27 | name string 28 | shouldBeEnabled bool 29 | configuredTypes []string 30 | currentType string 31 | }{ 32 | { 33 | name: "always enabled when config field is nil", 34 | shouldBeEnabled: true, 35 | configuredTypes: nil, 36 | currentType: "pod", // doesn't matter 37 | }, 38 | { 39 | name: "always enabled when config field is empty", 40 | shouldBeEnabled: true, 41 | configuredTypes: []string{}, 42 | currentType: "ec2", // doesn't matter 43 | }, 44 | { 45 | name: "enabled for listed type", 46 | shouldBeEnabled: true, 47 | configuredTypes: []string{"vpc"}, 48 | currentType: "vpc", 49 | }, 50 | { 51 | name: "disabled when type isn't in the list", 52 | shouldBeEnabled: false, 53 | configuredTypes: []string{"eks"}, 54 | currentType: "node", 55 | }, 56 | } { 57 | t.Run(tt.name, func(t *testing.T) { 58 | assert.Equal(t, tt.shouldBeEnabled, IsTypeEnabled(tt.configuredTypes, tt.currentType)) 59 | }) 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /input/azure/azure_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package azure 19 | 20 | import ( 21 | "context" 22 | "github.com/elastic/assetbeat/input/testutil" 23 | v2 "github.com/elastic/beats/v7/filebeat/input/v2" 24 | "github.com/elastic/elastic-agent-libs/logp" 25 | "github.com/stretchr/testify/assert" 26 | "sync" 27 | "testing" 28 | "time" 29 | ) 30 | 31 | func TestPlugin(t *testing.T) { 32 | p := Plugin() 33 | assert.Equal(t, "assets_azure", p.Name) 34 | assert.NotNil(t, p.Manager) 35 | } 36 | 37 | func TestAssetsAzure_Run(t *testing.T) { 38 | publisher := testutil.NewInMemoryPublisher() 39 | 40 | ctx, cancel := context.WithCancel(context.Background()) 41 | inputCtx := v2.Context{ 42 | Logger: logp.NewLogger("test"), 43 | Cancelation: ctx, 44 | } 45 | 46 | input, err := newAssetsAzure(defaultConfig()) 47 | assert.NoError(t, err) 48 | 49 | var wg sync.WaitGroup 50 | wg.Add(1) 51 | go func() { 52 | defer wg.Done() 53 | err = input.Run(inputCtx, publisher) 54 | assert.NoError(t, err) 55 | }() 56 | 57 | time.Sleep(time.Second) 58 | cancel() 59 | timeout := time.After(time.Second) 60 | closeCh := make(chan struct{}) 61 | go func() { 62 | defer close(closeCh) 63 | wg.Wait() 64 | }() 65 | select { 66 | case <-timeout: 67 | t.Fatal("Test timed out") 68 | case <-closeCh: 69 | // Waitgroup finished in time, nothing to do 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /beater/signalwait.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package beater 19 | 20 | import ( 21 | "time" 22 | 23 | "github.com/elastic/elastic-agent-libs/logp" 24 | ) 25 | 26 | type signalWait struct { 27 | count int // number of potential 'alive' signals 28 | signals chan struct{} 29 | } 30 | 31 | type signaler func() 32 | 33 | func newSignalWait() *signalWait { 34 | return &signalWait{ 35 | signals: make(chan struct{}, 1), 36 | } 37 | } 38 | 39 | func (s *signalWait) Wait() { 40 | if s.count == 0 { 41 | return 42 | } 43 | 44 | <-s.signals 45 | s.count-- 46 | } 47 | 48 | func (s *signalWait) Add(fn signaler) { 49 | s.count++ 50 | go func() { 51 | fn() 52 | var v struct{} 53 | s.signals <- v 54 | }() 55 | } 56 | 57 | func (s *signalWait) AddChan(c <-chan struct{}) { 58 | s.Add(waitChannel(c)) 59 | } 60 | 61 | func (s *signalWait) AddTimer(t *time.Timer) { 62 | s.Add(waitTimer(t)) 63 | } 64 | 65 | func (s *signalWait) AddTimeout(d time.Duration) { 66 | s.Add(waitDuration(d)) 67 | } 68 | 69 | func (s *signalWait) Signal() { 70 | s.Add(func() {}) 71 | } 72 | 73 | func waitChannel(c <-chan struct{}) signaler { 74 | return func() { <-c } 75 | } 76 | 77 | func waitTimer(t *time.Timer) signaler { 78 | return func() { <-t.C } 79 | } 80 | 81 | func waitDuration(d time.Duration) signaler { 82 | return waitTimer(time.NewTimer(d)) 83 | } 84 | 85 | func withLog(s signaler, msg string) signaler { 86 | return func() { 87 | s() 88 | logp.Info("%v", msg) 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /channel/connector.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package channel 19 | 20 | import ( 21 | "github.com/elastic/beats/v7/libbeat/beat" 22 | conf "github.com/elastic/elastic-agent-libs/config" 23 | ) 24 | 25 | // ConnectorFunc is an adapter for using ordinary functions as Connector. 26 | type ConnectorFunc func(*conf.C, beat.ClientConfig) (Outleter, error) 27 | 28 | type pipelineConnector struct { 29 | parent *OutletFactory 30 | pipeline beat.PipelineConnector 31 | } 32 | 33 | // Connect passes the cfg and the zero value of beat.ClientConfig to the underlying function. 34 | func (fn ConnectorFunc) Connect(cfg *conf.C) (Outleter, error) { 35 | return fn(cfg, beat.ClientConfig{}) 36 | } 37 | 38 | // ConnectWith passes the configuration and the pipeline connection setting to the underlying function. 39 | func (fn ConnectorFunc) ConnectWith(cfg *conf.C, clientCfg beat.ClientConfig) (Outleter, error) { 40 | return fn(cfg, clientCfg) 41 | } 42 | 43 | func (c *pipelineConnector) Connect(cfg *conf.C) (Outleter, error) { 44 | return c.ConnectWith(cfg, beat.ClientConfig{}) 45 | } 46 | 47 | func (c *pipelineConnector) ConnectWith(cfg *conf.C, clientCfg beat.ClientConfig) (Outleter, error) { 48 | // connect with updated configuration 49 | client, err := c.pipeline.ConnectWith(clientCfg) 50 | if err != nil { 51 | return nil, err 52 | } 53 | 54 | outlet := newOutlet(client) 55 | if c.parent.done != nil { 56 | return CloseOnSignal(outlet, c.parent.done), nil 57 | } 58 | return outlet, nil 59 | } 60 | -------------------------------------------------------------------------------- /internal/dev-tools/package_types.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package dev_tools 19 | 20 | import ( 21 | "fmt" 22 | "golang.org/x/exp/slices" 23 | "os" 24 | "strings" 25 | ) 26 | 27 | var supportedPackageTypes = []string{"docker", "tar.gz"} 28 | 29 | // GetPackageTypes returns the list of package types to use for packaging/release distribution. 30 | // By default, it returns the list of supported package types. It can be overridden by setting the TYPES 31 | // environment variable. 32 | func GetPackageTypes() []string { 33 | var packageTypesList []string 34 | types, ok := os.LookupEnv("TYPES") 35 | if ok { 36 | packageTypesList = getPackageTypesList(types) 37 | } else { 38 | fmt.Println("TYPES env variable not defined.") 39 | packageTypesList = append(packageTypesList, supportedPackageTypes...) 40 | } 41 | fmt.Printf("PackageTypes: %+v\n", packageTypesList) 42 | return packageTypesList 43 | } 44 | 45 | // getPackageTypesList returns a list of package types from a space-delimited string of package types 46 | // If the package type is not supported, it is discarded from the returned list 47 | func getPackageTypesList(types string) []string { 48 | var typesList []string 49 | inputTypesList := strings.Split(types, " ") 50 | for _, packageType := range inputTypesList { 51 | if slices.Contains(supportedPackageTypes, packageType) { 52 | typesList = append(typesList, packageType) 53 | } else { 54 | fmt.Printf("Unsupported packageType %s. Skipping...", packageType) 55 | } 56 | } 57 | return typesList 58 | } 59 | -------------------------------------------------------------------------------- /tests/e2e/assets_aws_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | //go:build e2e 19 | 20 | package e2e 21 | 22 | import ( 23 | "context" 24 | "sync" 25 | "testing" 26 | "time" 27 | 28 | "github.com/elastic/assetbeat/input/testutil" 29 | v2 "github.com/elastic/beats/v7/filebeat/input/v2" 30 | stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" 31 | 32 | "github.com/stretchr/testify/assert" 33 | 34 | "github.com/elastic/assetbeat/input/aws" 35 | "github.com/elastic/elastic-agent-libs/config" 36 | "github.com/elastic/elastic-agent-libs/logp" 37 | ) 38 | 39 | func TestAssetsAWS_Run_startsAndStopsTheInput(t *testing.T) { 40 | publisher := testutil.NewInMemoryPublisher() 41 | 42 | ctx, cancel := context.WithCancel(context.Background()) 43 | inputCtx := v2.Context{ 44 | Logger: logp.NewLogger("test"), 45 | Cancelation: ctx, 46 | } 47 | 48 | input, err := aws.Plugin().Manager.(stateless.InputManager).Configure(config.NewConfig()) 49 | assert.NoError(t, err) 50 | 51 | var wg sync.WaitGroup 52 | wg.Add(1) 53 | go func() { 54 | defer wg.Done() 55 | err = input.Run(inputCtx, publisher) 56 | assert.NoError(t, err) 57 | }() 58 | 59 | time.Sleep(time.Millisecond) 60 | cancel() 61 | 62 | timeout := time.After(time.Second) 63 | closeCh := make(chan struct{}) 64 | go func() { 65 | defer close(closeCh) 66 | wg.Wait() 67 | }() 68 | 69 | select { 70 | case <-timeout: 71 | t.Fatal("Test timed out") 72 | case <-closeCh: 73 | // Waitgroup finished in time, nothing to do 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /tests/e2e/assets_gcp_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | //go:build e2e 19 | 20 | package e2e 21 | 22 | import ( 23 | "context" 24 | "sync" 25 | "testing" 26 | "time" 27 | 28 | "github.com/elastic/assetbeat/input/gcp" 29 | "github.com/elastic/assetbeat/input/testutil" 30 | stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" 31 | 32 | "github.com/stretchr/testify/assert" 33 | 34 | v2 "github.com/elastic/beats/v7/filebeat/input/v2" 35 | "github.com/elastic/elastic-agent-libs/config" 36 | "github.com/elastic/elastic-agent-libs/logp" 37 | ) 38 | 39 | func TestAssetsGCP_Run_startsAndStopsTheInput(t *testing.T) { 40 | publisher := testutil.NewInMemoryPublisher() 41 | 42 | ctx, cancel := context.WithCancel(context.Background()) 43 | inputCtx := v2.Context{ 44 | Logger: logp.NewLogger("test"), 45 | Cancelation: ctx, 46 | } 47 | 48 | input, err := gcp.Plugin().Manager.(stateless.InputManager).Configure(config.NewConfig()) 49 | assert.NoError(t, err) 50 | 51 | var wg sync.WaitGroup 52 | wg.Add(1) 53 | go func() { 54 | defer wg.Done() 55 | err = input.Run(inputCtx, publisher) 56 | assert.NoError(t, err) 57 | }() 58 | 59 | time.Sleep(time.Millisecond) 60 | cancel() 61 | 62 | timeout := time.After(time.Second) 63 | closeCh := make(chan struct{}) 64 | go func() { 65 | defer close(closeCh) 66 | wg.Wait() 67 | }() 68 | 69 | select { 70 | case <-timeout: 71 | t.Fatal("Test timed out") 72 | case <-closeCh: 73 | // Waitgroup finished in time, nothing to do 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /channel/outlet.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package channel 19 | 20 | import ( 21 | "github.com/elastic/beats/v7/libbeat/beat" 22 | "github.com/elastic/beats/v7/libbeat/common/atomic" 23 | ) 24 | 25 | type outlet struct { 26 | client beat.Client 27 | isOpen atomic.Bool 28 | done chan struct{} 29 | } 30 | 31 | func newOutlet(client beat.Client) *outlet { 32 | o := &outlet{ 33 | client: client, 34 | isOpen: atomic.MakeBool(true), 35 | done: make(chan struct{}), 36 | } 37 | return o 38 | } 39 | 40 | func (o *outlet) Close() error { 41 | isOpen := o.isOpen.Swap(false) 42 | if isOpen { 43 | close(o.done) 44 | return o.client.Close() 45 | } 46 | return nil 47 | } 48 | 49 | func (o *outlet) Done() <-chan struct{} { 50 | return o.done 51 | } 52 | 53 | func (o *outlet) OnEvent(event beat.Event) bool { 54 | if !o.isOpen.Load() { 55 | return false 56 | } 57 | 58 | o.client.Publish(event) 59 | 60 | // Note: race condition on shutdown: 61 | // The underlying beat.Client is asynchronous. Without proper ACK 62 | // handler we can not tell if the event made it 'through' or the client 63 | // close has been completed before sending. In either case, 64 | // we report 'false' here, indicating the event eventually being dropped. 65 | // Returning false here, prevents the harvester from updating the state 66 | // to the most recently published events. Therefore, on shutdown the harvester 67 | // might report an old/outdated state update to the registry, overwriting the 68 | // most recently 69 | // published offset in the registry on shutdown. 70 | return o.isOpen.Load() 71 | } 72 | -------------------------------------------------------------------------------- /tests/e2e/assets_k8s_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | //go:build e2e 19 | 20 | package e2e 21 | 22 | import ( 23 | "context" 24 | "sync" 25 | "testing" 26 | "time" 27 | 28 | "github.com/stretchr/testify/assert" 29 | k8sfake "k8s.io/client-go/kubernetes/fake" 30 | 31 | "github.com/elastic/assetbeat/input/k8s" 32 | "github.com/elastic/assetbeat/input/testutil" 33 | v2 "github.com/elastic/beats/v7/filebeat/input/v2" 34 | stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" 35 | "github.com/elastic/elastic-agent-libs/config" 36 | "github.com/elastic/elastic-agent-libs/logp" 37 | ) 38 | 39 | func TestAssetsK8s_Run_startsAndStopsTheInput(t *testing.T) { 40 | publisher := testutil.NewInMemoryPublisher() 41 | 42 | ctx, cancel := context.WithCancel(context.Background()) 43 | inputCtx := v2.Context{ 44 | Logger: logp.NewLogger("test"), 45 | Cancelation: ctx, 46 | } 47 | input, err := k8s.Plugin().Manager.(stateless.InputManager).Configure(config.NewConfig()) 48 | assert.NoError(t, err) 49 | client := k8sfake.NewSimpleClientset() 50 | if err := k8s.SetClient(client, input); err != nil { 51 | t.Fatalf("Test failed: %s", err) 52 | } 53 | 54 | var wg sync.WaitGroup 55 | wg.Add(1) 56 | go func() { 57 | defer wg.Done() 58 | err = input.Run(inputCtx, publisher) 59 | assert.NoError(t, err) 60 | }() 61 | 62 | time.Sleep(time.Millisecond) 63 | cancel() 64 | 65 | timeout := time.After(10 * time.Second) 66 | closeCh := make(chan struct{}) 67 | go func() { 68 | defer close(closeCh) 69 | wg.Wait() 70 | }() 71 | 72 | select { 73 | case <-timeout: 74 | t.Fatal("Test timed out") 75 | case <-closeCh: 76 | // Waitgroup finished in time, nothing to do 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | name: docker 2 | 3 | on: 4 | merge_group: 5 | branches: [ "main" ] 6 | 7 | concurrency: 8 | group: ${{ github.workflow }}-${{ github.ref }} 9 | cancel-in-progress: true 10 | 11 | env: 12 | PLATFORMS: linux/amd64 13 | TYPES: docker 14 | DOCKER_REGISTRY: docker.elastic.co 15 | DOCKER_IMG: docker.elastic.co/observability/assetbeat 16 | DOCKER_IMG_TAG_LATEST: docker.elastic.co/observability/assetbeat:latest 17 | jobs: 18 | pkg-docker: 19 | runs-on: ubuntu-20.04 20 | steps: 21 | - uses: actions/checkout@v4 22 | - name: Package assetbeat in a Dockerfile 23 | uses: magefile/mage-action@v3 24 | with: 25 | version: latest 26 | args: package 27 | - uses: hashicorp/vault-action@v2.7.3 28 | with: 29 | url: ${{ secrets.VAULT_ADDR }} 30 | method: approle 31 | roleId: ${{ secrets.VAULT_ROLE_ID }} 32 | secretId: ${{ secrets.VAULT_SECRET_ID }} 33 | secrets: | 34 | secret/observability-team/ci/docker-registry/prod username | DOCKER_USERNAME ; 35 | secret/observability-team/ci/docker-registry/prod password | DOCKER_PASSWORD 36 | - name: Log in to the Container registry 37 | uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d 38 | with: 39 | registry: ${{ env.DOCKER_REGISTRY }} 40 | username: ${{ env.DOCKER_USERNAME }} 41 | password: ${{ env.DOCKER_PASSWORD }} 42 | - name: Reset environment 43 | shell: bash 44 | run: | 45 | echo "DOCKER_USERNAME=" >> $GITHUB_ENV 46 | echo "DOCKER_PASSWORD=" >> $GITHUB_ENV 47 | - name: Set Version 48 | id: set-version 49 | uses: magefile/mage-action@v3 50 | env: 51 | GITHUB_OUTPUT: $GITHUB_OUTPUT 52 | with: 53 | version: latest 54 | args: writeversiontogithuboutput 55 | - name: Build and push image with version 56 | uses: docker/build-push-action@v5 57 | with: 58 | context: ./build/package/assetbeat/assetbeat-linux-x86_64.docker/docker-build 59 | push: true 60 | tags: ${{ env.DOCKER_IMG }}:${{ steps.set-version.outputs.VERSION }}-SNAPSHOT 61 | - name: Build and push with latest tag 62 | if: ${{ success() }} 63 | uses: docker/build-push-action@v5 64 | with: 65 | context: ./build/package/assetbeat/assetbeat-linux-x86_64.docker/docker-build 66 | push: true 67 | tags: ${{ env.DOCKER_IMG_TAG_LATEST }} 68 | -------------------------------------------------------------------------------- /channel/util_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package channel 19 | 20 | import ( 21 | "testing" 22 | 23 | "github.com/stretchr/testify/assert" 24 | 25 | "github.com/elastic/beats/v7/libbeat/beat" 26 | "github.com/elastic/beats/v7/libbeat/tests/resources" 27 | ) 28 | 29 | type dummyOutletter struct { 30 | closed bool 31 | c chan struct{} 32 | } 33 | 34 | func (o *dummyOutletter) OnEvent(event beat.Event) bool { 35 | return true 36 | } 37 | 38 | func (o *dummyOutletter) Close() error { 39 | o.closed = true 40 | close(o.c) 41 | return nil 42 | } 43 | 44 | func (o *dummyOutletter) Done() <-chan struct{} { 45 | return o.c 46 | } 47 | 48 | func TestCloseOnSignal(t *testing.T) { 49 | goroutines := resources.NewGoroutinesChecker() 50 | defer goroutines.Check(t) 51 | 52 | o := &dummyOutletter{c: make(chan struct{})} 53 | sig := make(chan struct{}) 54 | CloseOnSignal(o, sig) 55 | close(sig) 56 | } 57 | 58 | func TestCloseOnSignalClosed(t *testing.T) { 59 | goroutines := resources.NewGoroutinesChecker() 60 | defer goroutines.Check(t) 61 | 62 | o := &dummyOutletter{c: make(chan struct{})} 63 | sig := make(chan struct{}) 64 | c := CloseOnSignal(o, sig) 65 | c.Close() 66 | } 67 | 68 | func TestSubOutlet(t *testing.T) { 69 | goroutines := resources.NewGoroutinesChecker() 70 | defer goroutines.Check(t) 71 | 72 | o := &dummyOutletter{c: make(chan struct{})} 73 | so := SubOutlet(o) 74 | so.Close() 75 | assert.False(t, o.closed) 76 | } 77 | 78 | func TestCloseOnSignalSubOutlet(t *testing.T) { 79 | goroutines := resources.NewGoroutinesChecker() 80 | defer goroutines.Check(t) 81 | 82 | o := &dummyOutletter{c: make(chan struct{})} 83 | c := CloseOnSignal(SubOutlet(o), make(chan struct{})) 84 | o.Close() 85 | c.Close() 86 | } 87 | -------------------------------------------------------------------------------- /input/k8s/pods_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package k8s 19 | 20 | import ( 21 | "context" 22 | "testing" 23 | "time" 24 | 25 | "github.com/stretchr/testify/assert" 26 | v1 "k8s.io/api/core/v1" 27 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | k8sfake "k8s.io/client-go/kubernetes/fake" 29 | 30 | "github.com/elastic/assetbeat/input/testutil" 31 | "github.com/elastic/elastic-agent-libs/logp" 32 | ) 33 | 34 | func TestGetPodWatcher(t *testing.T) { 35 | client := k8sfake.NewSimpleClientset() 36 | log := logp.NewLogger("mylogger") 37 | _, err := getPodWatcher(context.Background(), log, client, time.Second*60) 38 | if err != nil { 39 | t.Fatalf("error initiating Pod watcher") 40 | } 41 | assert.NoError(t, err) 42 | } 43 | 44 | func TestPublishK8sPods(t *testing.T) { 45 | client := k8sfake.NewSimpleClientset() 46 | log := logp.NewLogger("mylogger") 47 | podWatcher, err := getPodWatcher(context.Background(), log, client, time.Second*60) 48 | if err != nil { 49 | t.Fatalf("error initiating Pod watcher") 50 | } 51 | input := &v1.Pod{ 52 | ObjectMeta: metav1.ObjectMeta{ 53 | Name: "mypod", 54 | UID: "a375d24b-fa20-4ea6-a0ee-1d38671d2c09", 55 | Namespace: "default", 56 | Labels: map[string]string{ 57 | "foo": "bar", 58 | }, 59 | Annotations: map[string]string{ 60 | "app": "production", 61 | }, 62 | }, 63 | TypeMeta: metav1.TypeMeta{ 64 | Kind: "Pod", 65 | APIVersion: "v1", 66 | }, 67 | Spec: v1.PodSpec{ 68 | NodeName: "testnode", 69 | }, 70 | Status: v1.PodStatus{PodIP: "127.0.0.5"}, 71 | } 72 | _ = podWatcher.Store().Add(input) 73 | publisher := testutil.NewInMemoryPublisher() 74 | publishK8sPods(context.Background(), log, publisher, podWatcher, nil) 75 | 76 | assert.Equal(t, 1, len(publisher.Events)) 77 | } 78 | -------------------------------------------------------------------------------- /input/aws/asset_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package aws 19 | 20 | import ( 21 | "testing" 22 | 23 | "github.com/stretchr/testify/assert" 24 | 25 | "github.com/elastic/assetbeat/input/internal" 26 | "github.com/elastic/assetbeat/input/testutil" 27 | "github.com/elastic/beats/v7/libbeat/beat" 28 | "github.com/elastic/elastic-agent-libs/mapstr" 29 | ) 30 | 31 | func TestWithAssetTags(t *testing.T) { 32 | for _, tt := range []struct { 33 | name string 34 | 35 | opts []internal.AssetOption 36 | expectedEvent beat.Event 37 | }{ 38 | { 39 | name: "with valid tags", 40 | opts: []internal.AssetOption{ 41 | internal.WithAssetCloudProvider("aws"), 42 | WithAssetTags(mapstr.M{"tag1": "a", "tag2": "b"}), 43 | }, 44 | expectedEvent: beat.Event{Fields: mapstr.M{ 45 | "cloud.provider": "aws", 46 | "asset.metadata.tags.tag1": "a", 47 | "asset.metadata.tags.tag2": "b", 48 | }, Meta: mapstr.M{"index": internal.GetDefaultIndexName()}}, 49 | }, 50 | { 51 | name: "with valid tags and metadata", 52 | opts: []internal.AssetOption{ 53 | internal.WithAssetCloudProvider("aws"), 54 | internal.WithAssetMetadata(mapstr.M{"foo": "bar"}), 55 | WithAssetTags(mapstr.M{"tag1": "a", "tag2": "b"}), 56 | }, 57 | expectedEvent: beat.Event{Fields: mapstr.M{ 58 | "cloud.provider": "aws", 59 | "asset.metadata.foo": "bar", 60 | "asset.metadata.tags.tag1": "a", 61 | "asset.metadata.tags.tag2": "b", 62 | }, Meta: mapstr.M{"index": internal.GetDefaultIndexName()}}, 63 | }, 64 | } { 65 | t.Run(tt.name, func(t *testing.T) { 66 | publisher := testutil.NewInMemoryPublisher() 67 | 68 | internal.Publish(publisher, nil, tt.opts...) 69 | 70 | assert.Equal(t, 1, len(publisher.Events)) 71 | assert.Equal(t, tt.expectedEvent, publisher.Events[0]) 72 | }) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /input/gcp/asset_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package gcp 19 | 20 | import ( 21 | "testing" 22 | 23 | "github.com/stretchr/testify/assert" 24 | 25 | "github.com/elastic/assetbeat/input/internal" 26 | "github.com/elastic/assetbeat/input/testutil" 27 | "github.com/elastic/beats/v7/libbeat/beat" 28 | "github.com/elastic/elastic-agent-libs/mapstr" 29 | ) 30 | 31 | func TestWithAssetLabels(t *testing.T) { 32 | for _, tt := range []struct { 33 | name string 34 | 35 | opts []internal.AssetOption 36 | expectedEvent beat.Event 37 | }{ 38 | { 39 | name: "with valid labels", 40 | opts: []internal.AssetOption{ 41 | internal.WithAssetCloudProvider("gcp"), 42 | WithAssetLabels(mapstr.M{"label1": "a", "label2": "b"}), 43 | }, 44 | expectedEvent: beat.Event{Fields: mapstr.M{ 45 | "cloud.provider": "gcp", 46 | "asset.metadata.labels.label1": "a", 47 | "asset.metadata.labels.label2": "b", 48 | }, Meta: mapstr.M{"index": internal.GetDefaultIndexName()}}, 49 | }, 50 | { 51 | name: "with valid labels and metadata", 52 | opts: []internal.AssetOption{ 53 | internal.WithAssetCloudProvider("gcp"), 54 | internal.WithAssetMetadata(mapstr.M{"foo": "bar"}), 55 | WithAssetLabels(mapstr.M{"label1": "a", "label2": "b"}), 56 | }, 57 | expectedEvent: beat.Event{Fields: mapstr.M{ 58 | "cloud.provider": "gcp", 59 | "asset.metadata.foo": "bar", 60 | "asset.metadata.labels.label1": "a", 61 | "asset.metadata.labels.label2": "b", 62 | }, Meta: mapstr.M{"index": internal.GetDefaultIndexName()}}, 63 | }, 64 | } { 65 | t.Run(tt.name, func(t *testing.T) { 66 | publisher := testutil.NewInMemoryPublisher() 67 | 68 | internal.Publish(publisher, nil, tt.opts...) 69 | assert.Equal(t, 1, len(publisher.Events)) 70 | assert.Equal(t, tt.expectedEvent, publisher.Events[0]) 71 | }) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /internal/dev-tools/build_platforms.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package dev_tools 19 | 20 | import ( 21 | "fmt" 22 | "golang.org/x/exp/slices" 23 | "os" 24 | "strings" 25 | ) 26 | 27 | type Platform struct { 28 | GOOS string 29 | GOARCH string 30 | } 31 | 32 | var supportedPlatforms = []string{"linux/amd64", "linux/arm64"} 33 | 34 | // GetPlatforms return the list of Platform to use for cross-builds. 35 | // By default, it returns the list of supported platforms. It can be overridden by setting the PLATFORMS 36 | // environment variable. 37 | func GetPlatforms() []Platform { 38 | var platformsList []Platform 39 | platforms, ok := os.LookupEnv("PLATFORMS") 40 | if ok { 41 | platformsList = getPlatformsList(platforms) 42 | } else { 43 | fmt.Println("PLATFORMS env variable not defined.") 44 | for _, platform := range supportedPlatforms { 45 | platformsList = append(platformsList, newPlatform(platform)) 46 | } 47 | } 48 | fmt.Printf("Platforms: %+v\n", platformsList) 49 | return platformsList 50 | } 51 | 52 | // getPlatformsList returns a list of Platform from a space-delimited string of GOOS/GOARCH pairs. 53 | // If the Platform is not supported, it is discarded from the returned list 54 | func getPlatformsList(platforms string) []Platform { 55 | var platformsList []Platform 56 | inputPlatformsList := strings.Split(platforms, " ") 57 | for _, platform := range inputPlatformsList { 58 | if slices.Contains(supportedPlatforms, platform) { 59 | platformsList = append(platformsList, newPlatform(platform)) 60 | } else { 61 | fmt.Printf("Unsupported platform %s. Skipping...", platform) 62 | } 63 | } 64 | return platformsList 65 | } 66 | 67 | // newPlatform returns a new Platform from a GOOS/GOARCH string 68 | func newPlatform(p string) Platform { 69 | platformSplit := strings.Split(p, "/") 70 | return Platform{ 71 | GOOS: platformSplit[0], 72 | GOARCH: platformSplit[1], 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /internal/dev-tools/notice.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package dev_tools 19 | 20 | import ( 21 | "fmt" 22 | "github.com/elastic/assetbeat/version" 23 | "github.com/elastic/elastic-agent-libs/dev-tools/mage" 24 | "github.com/elastic/elastic-agent-libs/dev-tools/mage/gotool" 25 | "github.com/magefile/mage/mg" 26 | "github.com/magefile/mage/sh" 27 | "os" 28 | ) 29 | 30 | func GenerateNotice(overrides, rules, noticeTemplate string) error { 31 | mg.Deps(mage.InstallGoNoticeGen) 32 | depsFile := generateDepsFile() 33 | defer os.Remove(depsFile) 34 | 35 | generator := gotool.NoticeGenerator 36 | return generator( 37 | generator.Dependencies(depsFile), 38 | generator.IncludeIndirect(), 39 | generator.Overrides(overrides), 40 | generator.Rules(rules), 41 | generator.NoticeTemplate(noticeTemplate), 42 | generator.NoticeOutput("NOTICE.txt"), 43 | ) 44 | } 45 | 46 | func GenerateDependencyReport(overrides, rules, dependencyReportTemplate string, isSnapshot bool) error { 47 | mg.Deps(mage.InstallGoNoticeGen) 48 | depsFile := generateDepsFile() 49 | defer os.Remove(depsFile) 50 | 51 | if err := sh.RunV("mkdir", "-p", defaultPackageFolder); err != nil { 52 | return err 53 | } 54 | 55 | generator := gotool.NoticeGenerator 56 | dependencyReportFilename := fmt.Sprintf("dependencies-%s", version.GetVersion()) 57 | if isSnapshot { 58 | dependencyReportFilename = dependencyReportFilename + "-SNAPSHOT" 59 | } 60 | return generator( 61 | generator.Dependencies(depsFile), 62 | generator.IncludeIndirect(), 63 | generator.Overrides(overrides), 64 | generator.Rules(rules), 65 | generator.NoticeTemplate(dependencyReportTemplate), 66 | generator.NoticeOutput(fmt.Sprintf("%s/%s.csv", defaultPackageFolder, dependencyReportFilename)), 67 | ) 68 | } 69 | 70 | func generateDepsFile() string { 71 | 72 | out, _ := gotool.ListDepsForNotice() 73 | depsFile, _ := os.CreateTemp("", "depsout") 74 | _, _ = depsFile.Write([]byte(out)) 75 | depsFile.Close() 76 | 77 | return depsFile.Name() 78 | } 79 | -------------------------------------------------------------------------------- /.mergify.yml: -------------------------------------------------------------------------------- 1 | queue_rules: 2 | - name: default 3 | conditions: 4 | - check-success=buildkite/assetbeat 5 | - check-success=CLA 6 | pull_request_rules: 7 | - name: self-assign PRs 8 | conditions: 9 | - -merged 10 | - -closed 11 | - "#assignee=0" 12 | actions: 13 | assign: 14 | add_users: 15 | - "{{ author }}" 16 | - name: ask to resolve conflict 17 | conditions: 18 | - conflict 19 | actions: 20 | comment: 21 | message: | 22 | This pull request is now in conflicts. Could you fix it @{{author}}? 🙏 23 | To fixup this pull request, you can check out it locally. See documentation: https://help.github.com/articles/checking-out-pull-requests-locally/ 24 | ``` 25 | git fetch upstream 26 | git checkout -b {{head}} upstream/{{head}} 27 | git merge upstream/{{base}} 28 | git push upstream {{head}} 29 | ``` 30 | - name: notify the backport policy 31 | conditions: 32 | - -label~=^backport 33 | - base=main 34 | actions: 35 | comment: 36 | message: | 37 | This pull request does not have a backport label. Could you fix it @{{author}}? 🙏 38 | To fixup this pull request, you need to add the backport labels for the needed 39 | branches, such as: 40 | * `backport-8./d` is the label to automatically backport to the `8./d` branch. `/d` is the digit. 41 | 42 | **NOTE**: `backport-skip` has been added to this pull request. 43 | label: 44 | add: 45 | - backport-skip 46 | - name: remove backport-skip label 47 | conditions: 48 | - label~=^backport-\d 49 | actions: 50 | label: 51 | remove: 52 | - backport-skip 53 | - name: notify the backport has not been merged yet 54 | conditions: 55 | - -merged 56 | - -closed 57 | - author=mergify[bot] 58 | - "#check-success>0" 59 | - schedule=Mon-Mon 06:00-10:00[Europe/Paris] 60 | actions: 61 | comment: 62 | message: | 63 | This pull request has not been merged yet. Could you please review and merge it @{{ assignee | join(', @') }}? 🙏 64 | - name: squash and merge backport PRs after CI passes 65 | conditions: 66 | - label=backport 67 | - author=mergify[bot] 68 | - -conflict 69 | actions: 70 | queue: 71 | method: squash 72 | name: default 73 | 74 | - name: backport patches to 8.11 branch 75 | conditions: 76 | - merged 77 | - label=backport-v8.11.0 78 | actions: 79 | backport: 80 | assignees: 81 | - "{{ author }}" 82 | branches: 83 | - "8.11" 84 | labels: 85 | - "backport" 86 | title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}" 87 | -------------------------------------------------------------------------------- /input/k8s/containers_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package k8s 19 | 20 | import ( 21 | "context" 22 | "testing" 23 | "time" 24 | 25 | "github.com/stretchr/testify/assert" 26 | v1 "k8s.io/api/core/v1" 27 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | k8sfake "k8s.io/client-go/kubernetes/fake" 29 | 30 | "github.com/elastic/assetbeat/input/testutil" 31 | "github.com/elastic/elastic-agent-libs/logp" 32 | ) 33 | 34 | func TestPublishK8sContainers(t *testing.T) { 35 | client := k8sfake.NewSimpleClientset() 36 | log := logp.NewLogger("mylogger") 37 | podWatcher, err := getPodWatcher(context.Background(), log, client, time.Second*60) 38 | if err != nil { 39 | t.Fatalf("error initiating Pod watcher") 40 | } 41 | containers := []v1.Container{ 42 | { 43 | Name: "nginx", 44 | Image: "nginx:1.120", 45 | Ports: []v1.ContainerPort{ 46 | { 47 | Name: "http", 48 | Protocol: v1.ProtocolTCP, 49 | ContainerPort: 80, 50 | }, 51 | }, 52 | }, 53 | } 54 | containerStatuses := []v1.ContainerStatus{ 55 | { 56 | Name: "nginx", 57 | Ready: true, 58 | ContainerID: "crio://asdfghdeadbeef", 59 | State: v1.ContainerState{ 60 | Running: &v1.ContainerStateRunning{}, 61 | }, 62 | }, 63 | } 64 | input := &v1.Pod{ 65 | ObjectMeta: metav1.ObjectMeta{ 66 | Name: "testpod", 67 | UID: "a375d24b-fa20-4ea6-a0ee-1d38671d2c09", 68 | Namespace: "testns", 69 | Labels: map[string]string{ 70 | "foo": "bar", 71 | "with-dash": "dash-value", 72 | "with/slash": "some/path", 73 | }, 74 | Annotations: map[string]string{ 75 | "app": "production", 76 | }, 77 | }, 78 | TypeMeta: metav1.TypeMeta{ 79 | Kind: "Pod", 80 | APIVersion: "v1", 81 | }, 82 | Spec: v1.PodSpec{ 83 | NodeName: "testnode", 84 | Containers: containers, 85 | }, 86 | Status: v1.PodStatus{ 87 | PodIP: "127.0.0.5", 88 | ContainerStatuses: containerStatuses, 89 | }, 90 | } 91 | _ = podWatcher.Store().Add(input) 92 | publisher := testutil.NewInMemoryPublisher() 93 | publishK8sContainers(context.Background(), log, publisher, podWatcher) 94 | 95 | assert.Equal(t, 1, len(publisher.Events)) 96 | } 97 | -------------------------------------------------------------------------------- /input/k8s/containers.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package k8s 19 | 20 | import ( 21 | "context" 22 | "fmt" 23 | 24 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 25 | 26 | "github.com/elastic/assetbeat/input/internal" 27 | stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" 28 | kube "github.com/elastic/elastic-agent-autodiscover/kubernetes" 29 | "github.com/elastic/elastic-agent-libs/logp" 30 | ) 31 | 32 | // publishK8sPods publishes the pod assets stored in pod watcher cache 33 | func publishK8sContainers(ctx context.Context, log *logp.Logger, publisher stateless.Publisher, podWatcher kube.Watcher) { 34 | log.Info("Publishing container assets\n") 35 | assetType := "k8s.container" 36 | assetKind := "container" 37 | for _, obj := range podWatcher.Store().List() { 38 | o, ok := obj.(*kube.Pod) 39 | if ok { 40 | log.Debugf("Publish Pod: %+v", o.Name) 41 | parentId := string(o.UID) 42 | parentEan := fmt.Sprintf("%s:%s", "container_group", parentId) 43 | assetParents := []string{parentEan} 44 | namespace := o.Namespace 45 | 46 | containers := kube.GetContainersInPod(o) 47 | for _, c := range containers { 48 | // If it doesn't have an ID, container doesn't exist in 49 | // the runtime 50 | if c.ID == "" { 51 | continue 52 | } 53 | assetId := c.ID 54 | assetName := c.Spec.Name 55 | cPhase := c.Status.State 56 | state := "" 57 | assetStartTime := metav1.Time{} 58 | if cPhase.Waiting != nil { 59 | state = "Waiting" 60 | } else if cPhase.Running != nil { 61 | state = "Running" 62 | assetStartTime = cPhase.Running.StartedAt 63 | } else if cPhase.Terminated != nil { 64 | state = "Terminated" 65 | assetStartTime = cPhase.Terminated.StartedAt 66 | } 67 | 68 | internal.Publish(publisher, nil, 69 | internal.WithAssetKindAndID(assetKind, assetId), 70 | internal.WithAssetType(assetType), 71 | internal.WithAssetName(assetName), 72 | internal.WithAssetParents(assetParents), 73 | internal.WithContainerData(assetName, assetId, namespace, state, &assetStartTime), 74 | ) 75 | } 76 | } else { 77 | log.Error("Publishing pod assets failed. Type assertion of pod object failed") 78 | } 79 | 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /internal/dev-tools/tar.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package dev_tools 19 | 20 | import ( 21 | "archive/tar" 22 | "compress/gzip" 23 | "fmt" 24 | "io" 25 | "os" 26 | "path/filepath" 27 | "strings" 28 | ) 29 | 30 | // CreateTarball creates a tar.gz compressed archive from a list of files. 31 | func CreateTarball(baseFolderName string, outputTarballFilePath string, filePaths []string) error { 32 | fmt.Printf("Creating tarball... Filepath: %s\n", outputTarballFilePath) 33 | file, err := os.Create(outputTarballFilePath) 34 | if err != nil { 35 | return fmt.Errorf("could not create tarball file '%s', got error '%s'", outputTarballFilePath, err) 36 | } 37 | defer file.Close() 38 | 39 | gzipWriter := gzip.NewWriter(file) 40 | defer gzipWriter.Close() 41 | 42 | tarWriter := tar.NewWriter(gzipWriter) 43 | defer tarWriter.Close() 44 | 45 | for _, filePath := range filePaths { 46 | err := addFileToTarWriter(baseFolderName, filePath, tarWriter) 47 | if err != nil { 48 | return fmt.Errorf("could not add file '%s', to tarball, got error '%s'", filePath, err) 49 | } 50 | } 51 | 52 | return nil 53 | } 54 | 55 | func addFileToTarWriter(baseFolderName string, filePath string, tarWriter *tar.Writer) error { 56 | file, err := os.Open(filePath) 57 | if err != nil { 58 | return fmt.Errorf("could not open file '%s', got error '%s'", filePath, err) 59 | } 60 | defer file.Close() 61 | 62 | stat, err := file.Stat() 63 | if err != nil { 64 | return fmt.Errorf("could not get stat for file '%s', got error '%s'", filePath, err) 65 | } 66 | 67 | headerName := filepath.Base(filePath) 68 | if strings.Contains(headerName, "assetbeat-") { 69 | //This makes sure that platform details are removed from the packaged assetbeat binary filename 70 | headerName = "assetbeat" 71 | } 72 | header := &tar.Header{ 73 | Name: baseFolderName + "/" + headerName, 74 | Size: stat.Size(), 75 | Mode: int64(stat.Mode()), 76 | ModTime: stat.ModTime(), 77 | } 78 | 79 | err = tarWriter.WriteHeader(header) 80 | if err != nil { 81 | return fmt.Errorf("could not write header for file '%s', got error '%s'", filePath, err) 82 | } 83 | 84 | _, err = io.Copy(tarWriter, file) 85 | if err != nil { 86 | return fmt.Errorf("could not copy the file '%s' data to the tarball, got error '%s'", filePath, err) 87 | } 88 | 89 | return nil 90 | } 91 | -------------------------------------------------------------------------------- /input/gcp/gcp_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package gcp 19 | 20 | import ( 21 | "context" 22 | "sync" 23 | "testing" 24 | "time" 25 | 26 | "github.com/stretchr/testify/assert" 27 | "google.golang.org/api/option" 28 | 29 | "github.com/elastic/assetbeat/input/testutil" 30 | v2 "github.com/elastic/beats/v7/filebeat/input/v2" 31 | "github.com/elastic/elastic-agent-libs/logp" 32 | ) 33 | 34 | func TestPlugin(t *testing.T) { 35 | p := Plugin() 36 | assert.Equal(t, "assets_gcp", p.Name) 37 | assert.NotNil(t, p.Manager) 38 | } 39 | 40 | func TestAssetsGCP_Run(t *testing.T) { 41 | publisher := testutil.NewInMemoryPublisher() 42 | 43 | ctx, cancel := context.WithCancel(context.Background()) 44 | inputCtx := v2.Context{ 45 | Logger: logp.NewLogger("test"), 46 | Cancelation: ctx, 47 | } 48 | 49 | input, err := newAssetsGCP(defaultConfig()) 50 | assert.NoError(t, err) 51 | 52 | var wg sync.WaitGroup 53 | wg.Add(1) 54 | go func() { 55 | defer wg.Done() 56 | err = input.Run(inputCtx, publisher) 57 | assert.NoError(t, err) 58 | }() 59 | 60 | time.Sleep(time.Millisecond) 61 | cancel() 62 | timeout := time.After(time.Second) 63 | closeCh := make(chan struct{}) 64 | go func() { 65 | defer close(closeCh) 66 | wg.Wait() 67 | }() 68 | select { 69 | case <-timeout: 70 | t.Fatal("Test timed out") 71 | case <-closeCh: 72 | // Waitgroup finished in time, nothing to do 73 | } 74 | } 75 | 76 | func TestAssetsGCP_CollectAll(t *testing.T) { 77 | publisher := testutil.NewInMemoryPublisher() 78 | 79 | ctx := context.Background() 80 | logger := logp.NewLogger("test") 81 | 82 | input, err := newAssetsGCP(defaultConfig()) 83 | assert.NoError(t, err) 84 | 85 | err = input.collectAll(ctx, logger, publisher) 86 | assert.NoError(t, err) 87 | } 88 | 89 | func TestBuildClientOptions(t *testing.T) { 90 | for _, tt := range []struct { 91 | name string 92 | 93 | cfg config 94 | expectedOpts []option.ClientOption 95 | }{ 96 | { 97 | name: "with an empty config", 98 | }, 99 | { 100 | name: "with a credentials file path", 101 | 102 | cfg: config{ 103 | CredsFilePath: "/tmp/file_path", 104 | }, 105 | expectedOpts: []option.ClientOption{ 106 | option.WithCredentialsFile("/tmp/file_path"), 107 | }, 108 | }, 109 | } { 110 | t.Run(tt.name, func(t *testing.T) { 111 | opts := buildClientOptions(tt.cfg) 112 | assert.Equal(t, tt.expectedOpts, opts) 113 | }) 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /input/azure/README.md: -------------------------------------------------------------------------------- 1 | # Azure Assets Input 2 | 3 | ## What does it do? 4 | 5 | The Azure Assets Input collects data about Azure resources and their relationships to each other. 6 | Information about the following resources is currently collected: 7 | 8 | - Azure VM instances 9 | 10 | ## Configuration 11 | 12 | ```yaml 13 | assetbeat.inputs: 14 | - type: assets_azure 15 | regions: 16 | - 17 | subscription_id: 18 | client_id: 19 | client_secret: 20 | tenant_id: 21 | ``` 22 | 23 | The Azure Assets Input supports the following configuration options plus the [Common options](../README.md#Common options). 24 | 25 | * `regions`: The list of Azure regions to collect data from. 26 | * `subscription_id`: The unique identifier for the azure subscription 27 | * `client_id`: The unique identifier for the application (also known as Application Id) 28 | * `client_secret`: The client/application secret/key 29 | * `tenant_id`: The unique identifier of the Azure Active Directory instance 30 | 31 | **_Note_:** `client_id`, `client_secret` and `tenant_id` can be omitted if: 32 | * The environment variables `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET` and `AZURE_TENANT_ID` are set. 33 | * `az login` was ran on the host where `assetbeat` is running. 34 | 35 | **_Note_:** if `subscription_id` is omitted, the input will collect data from all the subscriptions you have access to. 36 | 37 | **_Note_:** if no region is provided under `regions` is omitted, the input will collect data from all the regions. 38 | 39 | 40 | ## Asset schema 41 | 42 | ### VM instances 43 | 44 | #### Exported fields 45 | 46 | | Field | Description | Example | 47 | |-------------------------------|-----------------------------------|-----------------------------------------------| 48 | | asset.type | The type of asset | `"azure.vm.instance"` | 49 | | asset.kind | The kind of asset | `"host` | 50 | | asset.name | The name of the Azure instance | `"my_instance"` | 51 | | asset.id | The VM id of the Azure instance | `"00830b08-f63d-495b-9b04-989f83c50111"` | 52 | | asset.ean | The EAN of this specific resource | `"host:00830b08-f63d-495b-9b04-989f83c50111"` | 53 | | asset.metadata.resource_group | The Azure resource group | `TESTVM` | 54 | | asset.metadata.state | The status of the VM instance | `"VM running"` | 55 | 56 | #### Example 57 | 58 | ```json 59 | { 60 | "@timestamp": "2023-09-13T14:42:51.494Z", 61 | "asset.metadata.resource_group": "TESTVM", 62 | "host": { 63 | "name": "host" 64 | }, 65 | "cloud.region": "westeurope", 66 | "cloud.provider": "azure", 67 | "agent": { 68 | "ephemeral_id": "a80c69df-22dd-4f97-bfd2-14572af2b9d4", 69 | "id": "9a7ef1a9-0cce-4857-90f9-699bc14d8df3", 70 | "name": "host", 71 | "type": "assetbeat", 72 | "version": "8.9.0" 73 | }, 74 | "input": { 75 | "type": "assets_azure" 76 | }, 77 | "cloud.account.id": "12cabcb4-86e8-404f-a3d2-111111111111", 78 | "asset.kind": "host", 79 | "asset.id": "00830b08-f63d-495b-9b04-989f83c50111", 80 | "asset.ean": "host:00830b08-f63d-495b-9b04-989f83c50111", 81 | "asset.metadata.state": "VM running", 82 | "asset.type": "azure.vm.instance", 83 | "asset.name": "my_instance", 84 | "ecs": { 85 | "version": "8.0.0" 86 | } 87 | } 88 | ``` -------------------------------------------------------------------------------- /channel/util.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package channel 19 | 20 | import ( 21 | "sync" 22 | 23 | "github.com/elastic/beats/v7/libbeat/beat" 24 | ) 25 | 26 | type subOutlet struct { 27 | done chan struct{} 28 | ch chan beat.Event 29 | res chan bool 30 | mutex sync.Mutex 31 | closeOnce sync.Once 32 | } 33 | 34 | // SubOutlet create a sub-outlet, which can be closed individually, without closing the 35 | // underlying outlet. 36 | func SubOutlet(out Outleter) Outleter { 37 | s := &subOutlet{ 38 | done: make(chan struct{}), 39 | ch: make(chan beat.Event), 40 | res: make(chan bool, 1), 41 | } 42 | 43 | go func() { 44 | for event := range s.ch { 45 | s.res <- out.OnEvent(event) 46 | } 47 | }() 48 | 49 | return s 50 | } 51 | 52 | func (o *subOutlet) Close() error { 53 | o.closeOnce.Do(func() { 54 | // Signal OnEvent() to terminate 55 | close(o.done) 56 | // This mutex prevents the event channel to be closed if OnEvent is 57 | // still running. 58 | o.mutex.Lock() 59 | defer o.mutex.Unlock() 60 | close(o.ch) 61 | }) 62 | return nil 63 | } 64 | 65 | func (o *subOutlet) Done() <-chan struct{} { 66 | return o.done 67 | } 68 | 69 | func (o *subOutlet) OnEvent(event beat.Event) bool { 70 | o.mutex.Lock() 71 | defer o.mutex.Unlock() 72 | select { 73 | case <-o.done: 74 | return false 75 | default: 76 | } 77 | 78 | select { 79 | case <-o.done: 80 | return false 81 | 82 | case o.ch <- event: 83 | select { 84 | case <-o.done: 85 | 86 | // Note: log harvester specific (leaky abstractions). 87 | // The close at this point in time indicates an event 88 | // already send to the publisher worker, forwarding events 89 | // to the publisher pipeline. The harvester insists on updating the state 90 | // (by pushing another state update to the publisher pipeline) on shutdown 91 | // and requires most recent state update in the harvester (who can only 92 | // update state on 'true' response). 93 | // The state update will appear after the current event in the publisher pipeline. 94 | // That is, by returning true here, the final state update will 95 | // be presented to the registrar, after the last event being processed. 96 | // Once all messages are in the publisher pipeline, in correct order, 97 | // it depends on registrar/publisher pipeline if state is finally updated 98 | // in the registrar. 99 | return true 100 | 101 | case ret := <-o.res: 102 | return ret 103 | } 104 | } 105 | } 106 | 107 | // CloseOnSignal closes the outlet, once the signal triggers. 108 | func CloseOnSignal(outlet Outleter, sig <-chan struct{}) Outleter { 109 | if sig != nil { 110 | go func() { 111 | select { 112 | case <-outlet.Done(): 113 | return 114 | case <-sig: 115 | outlet.Close() 116 | } 117 | }() 118 | } 119 | return outlet 120 | } 121 | -------------------------------------------------------------------------------- /internal/dev-tools/build.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package dev_tools 19 | 20 | import ( 21 | "fmt" 22 | "github.com/elastic/assetbeat/version" 23 | "github.com/magefile/mage/sh" 24 | "path/filepath" 25 | "strings" 26 | ) 27 | 28 | const assetbeatModulePath = "github.com/elastic/assetbeat" 29 | 30 | var qualifierVarPath = assetbeatModulePath + "/version.buildQualifier" 31 | var defaultCrossBuildFolder = filepath.Join("build", "golang-crossbuild") 32 | 33 | type BuildArgs struct { 34 | name string //name of the binary 35 | targetFolder string 36 | flags []string 37 | env map[string]string 38 | ldflags []string 39 | } 40 | 41 | // DefaultBuildArgs returns the default BuildArgs for use in builds. 42 | func DefaultBuildArgs() BuildArgs { 43 | 44 | args := BuildArgs{ 45 | name: "assetbeat", 46 | targetFolder: "", 47 | // -trimpath -> remove all file system paths from the resulting executable. 48 | // E.g a stack trace for /home/me/stuff/src/github.com/me/something.go:9 would be shown as github.com/me/something.go:9 49 | flags: []string{"-trimpath"}, 50 | // -ldflags=-s -w -> removes debug symbols from the resulting executable, reducing its size. 51 | ldflags: []string{"-s", "-w"}, 52 | env: map[string]string{}, 53 | } 54 | 55 | if version.HasQualifier { 56 | args.ldflags = append(args.ldflags, fmt.Sprintf("-X %s=%s", qualifierVarPath, version.Qualifier)) 57 | } 58 | 59 | return args 60 | } 61 | 62 | // DefaultCrossBuildArgs returns the default BuildArgs for cross-builds of a specific Platform. 63 | func DefaultCrossBuildArgs(platform Platform) BuildArgs { 64 | args := DefaultBuildArgs() 65 | args.targetFolder = defaultCrossBuildFolder 66 | args.name = strings.Join([]string{"assetbeat", platform.GOOS, platform.GOARCH}, "-") 67 | 68 | args.env = map[string]string{ 69 | "GOOS": platform.GOOS, 70 | "GOARCH": platform.GOARCH, 71 | "CGO_ENABLED": "0", 72 | } 73 | return args 74 | } 75 | 76 | // Build builds assetbeat using the defined BuildArgs and returns the executable file path. 77 | func Build(args BuildArgs) (string, error) { 78 | 79 | if err := sh.RunV("go", "mod", "download"); err != nil { 80 | return "", err 81 | } 82 | 83 | if len(args.targetFolder) > 0 { 84 | if err := sh.RunV("mkdir", "-p", args.targetFolder); err != nil { 85 | return "", err 86 | } 87 | } 88 | 89 | executablePath := filepath.Join(args.targetFolder, args.name) 90 | buildArgs := []string{"build"} 91 | buildArgs = append(buildArgs, "-o", executablePath) 92 | buildArgs = append(buildArgs, args.flags...) 93 | ldflags := strings.Join(args.ldflags, " ") 94 | buildArgs = append(buildArgs, "-ldflags", ldflags) 95 | fmt.Printf("%v+\n", buildArgs) 96 | err := sh.RunWithV(args.env, "go", buildArgs...) 97 | if err != nil { 98 | return "", nil 99 | } 100 | return executablePath, nil 101 | } 102 | -------------------------------------------------------------------------------- /input/README.md: -------------------------------------------------------------------------------- 1 | ## Intro 2 | 3 | All the inputs in this folder collect "Assets". Assets are defined as elements within your infrastructure, such as containers, machines, pods, clusters, etc. 4 | 5 | ## Supported Asset Inputs 6 | 7 | assetbeat supports the following asset input types at the moment: 8 | 9 | - [assets_aws](aws/README.md) 10 | - [assets_gcp](gcp/README.md) 11 | - [assets_k8s](k8s/README.md) 12 | 13 | 14 | ## Index name 15 | 16 | Each Asset input publishes documents to the same index, `assets-raw-default` 17 | 18 | ## Common configuration options 19 | 20 | The following configuration options are supported by all Asset inputs. 21 | 22 | * `period`: How often data should be collected. 23 | * `asset_types`: The list of specific asset types to collect data about. 24 | 25 | ### Type specific options 26 | 27 | - [assets_aws](aws/README.md#Configuration) 28 | - [assets_gcp](gcp/README.md#Configuration) 29 | - [assets_k8s](k8s/README.md#Configuration) 30 | 31 | ## Asset Inputs Relationships 32 | 33 | Certain assets types collected by the different inputs can be connected with each other 34 | with parent/children hierarchy. 35 | 36 | ## Asset identifier 37 | 38 | Each asset is identified by its Elastic Asset Name (EAN), which is an URN-style identifier with the following pattern, 39 | 40 | `{asset.kind}:{asset.id}` (e.g. `host:i-123456`). 41 | 42 | assetbeat publishes this field under `asset.ean`. 43 | 44 | ### GKE clusters and nodes 45 | In case `assets_k8s` input is collecting Kubernetes nodes assets and those nodes belong to a GKE cluster, the following field mapping can be used to link the Kubernetes nodes with their cluster. 46 | 47 | | assets_k8s (k8s.node) | assets_gcp (k8s.cluster) | Notes/Description | 48 | |-----------------------|--------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 49 | | cloud.instance.id | asset.children | For each GKE cluster, the field `asset.children` contains the EANs of the GCP instances linked. You can extract an instance ID from each EAN and map it to the field `cloud.instance.id`, which assetbeat publishes for GKE nodes. | 50 | | asset.parents | asset.ean | The `asset.parents` of k8s.node asset type contains the EAN of the kubernetes cluster it belongs to. | 51 | 52 | ### EKS clusters and nodes 53 | 54 | In case `assets_k8s` input is collecting Kubernetes nodes assets and those nodes belong to an EKS cluster, the following field mapping can be used to link the Kubernetes nodes with their cluster. 55 | 56 | | assets_k8s (k8s.node) | assets_aws (k8s.cluster) | Notes/Description | 57 | |-----------------------|--------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 58 | | cloud.instance.id | asset.children | For each EKS cluster, the field `asset.children` contains the EANs of the EC2 instances linked. You can extract an instance ID from each EAN and map it to the field `cloud.instance.id`, which assetbeat publishes for EKS nodes. | 59 | 60 | **_Note_:** The above mapping is not currently available for EKS Fargate clusters. 61 | -------------------------------------------------------------------------------- /internal/dev-tools/package.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package dev_tools 19 | 20 | import ( 21 | "fmt" 22 | "github.com/elastic/assetbeat/version" 23 | "github.com/magefile/mage/sh" 24 | "path/filepath" 25 | "strings" 26 | ) 27 | 28 | var defaultPackageFolder = filepath.Join("build", "distributions") 29 | 30 | type PackageSpec struct { 31 | Os string 32 | Arch string 33 | PackageType string 34 | ExecutablePath string 35 | IsSnapshot bool 36 | ExtraFilesList []string 37 | } 38 | 39 | var packageArchOverrides = map[string]string{ 40 | "amd64": "x86_64", 41 | } 42 | 43 | func GetPackageArch(goarch string) string { 44 | arch, overrideExists := packageArchOverrides[goarch] 45 | if overrideExists { 46 | return arch 47 | } 48 | return goarch 49 | } 50 | 51 | // GetDefaultExtraFiles returns the default list of files to include in an assetbeat package, 52 | // in addition to assetbeat's executable 53 | func GetDefaultExtraFiles() []string { 54 | return []string{"assetbeat.yml", "assetbeat.spec.yml"} 55 | } 56 | 57 | // CreatePackage assetbeat for distribution. It generates packages based on the provided PackageSpec/ 58 | func CreatePackage(spec PackageSpec) error { 59 | switch spec.PackageType { 60 | case "docker": 61 | return packageDocker(spec) 62 | case "tar.gz": 63 | return packageTar(spec) 64 | default: 65 | return fmt.Errorf("unsupported package type %s", spec.PackageType) 66 | } 67 | } 68 | 69 | func packageDocker(spec PackageSpec) error { 70 | filePath := fmt.Sprintf("build/package/assetbeat/assetbeat-%s-%s.docker/docker-build", spec.Os, spec.Arch) 71 | dockerfile := filePath + "/Dockerfile" 72 | executable := filePath + "/assetbeat" 73 | 74 | fmt.Printf("Creating folder %s\n", filePath) 75 | if err := sh.RunV("mkdir", "-p", filePath); err != nil { 76 | return err 77 | } 78 | 79 | fmt.Println("Copying Executable") 80 | if err := sh.RunV("cp", spec.ExecutablePath, executable); err != nil { 81 | return err 82 | } 83 | 84 | fmt.Println("Copying Dockerfile") 85 | return sh.RunV("cp", "Dockerfile.reference", dockerfile) 86 | } 87 | 88 | func packageTar(spec PackageSpec) error { 89 | filesPathList := []string{spec.ExecutablePath} 90 | filesPathList = append(filesPathList, spec.ExtraFilesList...) 91 | 92 | if err := sh.RunV("mkdir", "-p", defaultPackageFolder); err != nil { 93 | return err 94 | } 95 | 96 | basePackageName := getPackageBaseName(spec) 97 | tarFilePath := filepath.Join(defaultPackageFolder, basePackageName+".tar.gz") 98 | err := CreateTarball(basePackageName, tarFilePath, filesPathList) 99 | if err != nil { 100 | return err 101 | } 102 | return CreateSHA512File(tarFilePath) 103 | } 104 | 105 | func getPackageBaseName(spec PackageSpec) string { 106 | tarFileNameElements := []string{"assetbeat", version.GetVersion()} 107 | if spec.IsSnapshot { 108 | tarFileNameElements = append(tarFileNameElements, "SNAPSHOT") 109 | } 110 | tarFileNameElements = append(tarFileNameElements, []string{spec.Os, spec.Arch}...) 111 | 112 | return strings.Join(tarFileNameElements, "-") 113 | } 114 | -------------------------------------------------------------------------------- /beater/channels.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package beater 19 | 20 | import ( 21 | "sync" 22 | 23 | "github.com/elastic/beats/v7/libbeat/beat" 24 | "github.com/elastic/beats/v7/libbeat/publisher/pipetool" 25 | "github.com/elastic/elastic-agent-libs/monitoring" 26 | ) 27 | 28 | type eventCounter struct { 29 | added *monitoring.Uint 30 | done *monitoring.Uint 31 | count *monitoring.Int 32 | wg sync.WaitGroup 33 | } 34 | 35 | // countingClient adds and subtracts from a counter when events have been 36 | // published, dropped or ACKed. The countingClient can be used to keep track of 37 | // inflight events for a beat.Client instance. The counter is updated after the 38 | // client has been disconnected from the publisher pipeline via 'Closed'. 39 | type countingClient struct { 40 | counter *eventCounter 41 | client beat.Client 42 | } 43 | 44 | type countingEventer struct { 45 | wgEvents *eventCounter 46 | } 47 | 48 | type combinedEventer struct { 49 | a, b beat.ClientEventer 50 | } 51 | 52 | func (c *eventCounter) Add(delta int) { 53 | c.count.Add(int64(delta)) 54 | c.added.Add(uint64(delta)) 55 | c.wg.Add(delta) 56 | } 57 | 58 | func (c *eventCounter) Done() { 59 | c.count.Dec() 60 | c.done.Inc() 61 | c.wg.Done() 62 | } 63 | 64 | func (c *eventCounter) Wait() { 65 | c.wg.Wait() 66 | } 67 | 68 | // withPipelineEventCounter adds a counter to the pipeline that keeps track of 69 | // all events published, dropped and ACKed by any active client. 70 | // The type accepted by counter is compatible with sync.WaitGroup. 71 | func withPipelineEventCounter(pipeline beat.PipelineConnector, counter *eventCounter) beat.PipelineConnector { 72 | counterListener := &countingEventer{counter} 73 | 74 | pipeline = pipetool.WithClientConfigEdit(pipeline, func(config beat.ClientConfig) (beat.ClientConfig, error) { 75 | if evts := config.Events; evts != nil { 76 | config.Events = &combinedEventer{evts, counterListener} 77 | } else { 78 | config.Events = counterListener 79 | } 80 | return config, nil 81 | }) 82 | 83 | pipeline = pipetool.WithClientWrapper(pipeline, func(client beat.Client) beat.Client { 84 | return &countingClient{ 85 | counter: counter, 86 | client: client, 87 | } 88 | }) 89 | return pipeline 90 | } 91 | 92 | func (c *countingClient) Publish(event beat.Event) { 93 | c.counter.Add(1) 94 | c.client.Publish(event) 95 | } 96 | 97 | func (c *countingClient) PublishAll(events []beat.Event) { 98 | c.counter.Add(len(events)) 99 | c.client.PublishAll(events) 100 | } 101 | 102 | func (c *countingClient) Close() error { 103 | return c.client.Close() 104 | } 105 | 106 | func (*countingEventer) Closing() {} 107 | func (*countingEventer) Closed() {} 108 | func (*countingEventer) Published() {} 109 | 110 | func (c *countingEventer) FilteredOut(_ beat.Event) {} 111 | func (c *countingEventer) DroppedOnPublish(_ beat.Event) { 112 | c.wgEvents.Done() 113 | } 114 | 115 | func (c *combinedEventer) Closing() { 116 | c.a.Closing() 117 | c.b.Closing() 118 | } 119 | 120 | func (c *combinedEventer) Closed() { 121 | c.a.Closed() 122 | c.b.Closed() 123 | } 124 | 125 | func (c *combinedEventer) Published() { 126 | c.a.Published() 127 | c.b.Published() 128 | } 129 | 130 | func (c *combinedEventer) FilteredOut(event beat.Event) { 131 | c.a.FilteredOut(event) 132 | c.b.FilteredOut(event) 133 | } 134 | 135 | func (c *combinedEventer) DroppedOnPublish(event beat.Event) { 136 | c.a.DroppedOnPublish(event) 137 | c.b.DroppedOnPublish(event) 138 | } 139 | -------------------------------------------------------------------------------- /input/aws/ec2.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package aws 19 | 20 | import ( 21 | "context" 22 | "fmt" 23 | 24 | "github.com/elastic/assetbeat/input/internal" 25 | "github.com/elastic/assetbeat/util" 26 | stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" 27 | 28 | "github.com/elastic/elastic-agent-libs/logp" 29 | "github.com/elastic/elastic-agent-libs/mapstr" 30 | 31 | "github.com/aws/aws-sdk-go-v2/service/ec2" 32 | "github.com/aws/aws-sdk-go-v2/service/ec2/types" 33 | ) 34 | 35 | type EC2Instance struct { 36 | InstanceID string 37 | InstanceName string 38 | OwnerID string 39 | SubnetID string 40 | Tags []types.Tag 41 | Metadata mapstr.M 42 | } 43 | 44 | func collectEC2Assets(ctx context.Context, client ec2.DescribeInstancesAPIClient, region string, log *logp.Logger, publisher stateless.Publisher) error { 45 | instances, err := describeEC2Instances(ctx, client) 46 | if err != nil { 47 | return err 48 | } 49 | 50 | for _, instance := range instances { 51 | var parents []string 52 | if instance.SubnetID != "" { 53 | parents = []string{"network:" + instance.SubnetID} 54 | } 55 | assetType := "aws.ec2.instance" 56 | assetKind := "host" 57 | options := []internal.AssetOption{ 58 | internal.WithAssetCloudProvider("aws"), 59 | internal.WithAssetRegion(region), 60 | internal.WithAssetAccountID(instance.OwnerID), 61 | internal.WithAssetKindAndID(assetKind, instance.InstanceID), 62 | internal.WithAssetType(assetType), 63 | WithAssetTags(flattenEC2Tags(instance.Tags)), 64 | internal.WithAssetMetadata(instance.Metadata), 65 | } 66 | if parents != nil { 67 | options = append(options, internal.WithAssetParents(parents)) 68 | } 69 | if instance.InstanceName != "" { 70 | options = append(options, internal.WithAssetName(instance.InstanceName)) 71 | } 72 | internal.Publish(publisher, nil, 73 | options..., 74 | ) 75 | } 76 | 77 | return nil 78 | } 79 | 80 | func describeEC2Instances(ctx context.Context, client ec2.DescribeInstancesAPIClient) ([]EC2Instance, error) { 81 | instances := make([]EC2Instance, 0, 100) 82 | paginator := ec2.NewDescribeInstancesPaginator(client, &ec2.DescribeInstancesInput{}) 83 | for paginator.HasMorePages() { 84 | resp, err := paginator.NextPage(ctx) 85 | if err != nil { 86 | return nil, fmt.Errorf("error describing EC2 instances: %w", err) 87 | } 88 | for _, reservation := range resp.Reservations { 89 | instances = append(instances, util.Map(func(i types.Instance) EC2Instance { 90 | inst := EC2Instance{ 91 | InstanceID: *i.InstanceId, 92 | InstanceName: getEC2AssetNameFromTags(i.Tags), 93 | OwnerID: *reservation.OwnerId, 94 | Tags: i.Tags, 95 | Metadata: mapstr.M{ 96 | "state": string(i.State.Name), 97 | }, 98 | } 99 | if i.SubnetId != nil { 100 | inst.SubnetID = *i.SubnetId 101 | } 102 | return inst 103 | }, reservation.Instances)...) 104 | } 105 | } 106 | return instances, nil 107 | } 108 | 109 | // flattenEC2Tags converts the EC2 tag format to a simple `map[string]string` 110 | func flattenEC2Tags(tags []types.Tag) mapstr.M { 111 | out := mapstr.M{} 112 | for _, t := range tags { 113 | //Name is already published as asset.name 114 | if *t.Key != "Name" { 115 | out[*t.Key] = *t.Value 116 | } 117 | } 118 | return out 119 | } 120 | 121 | func getEC2AssetNameFromTags(tags []types.Tag) string { 122 | for _, tag := range tags { 123 | if *tag.Key == "Name" { 124 | return *tag.Value 125 | } 126 | } 127 | return "" 128 | } 129 | -------------------------------------------------------------------------------- /input/k8s/pods.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package k8s 19 | 20 | import ( 21 | "context" 22 | "fmt" 23 | "time" 24 | 25 | "github.com/elastic/assetbeat/input/internal" 26 | stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" 27 | kube "github.com/elastic/elastic-agent-autodiscover/kubernetes" 28 | "github.com/elastic/elastic-agent-libs/logp" 29 | 30 | kuberntescli "k8s.io/client-go/kubernetes" 31 | ) 32 | 33 | type pod struct { 34 | watcher kube.Watcher 35 | client kuberntescli.Interface 36 | logger *logp.Logger 37 | ctx context.Context 38 | } 39 | 40 | // getPodWatcher initiates and returns a watcher of kubernetes pods 41 | func getPodWatcher(ctx context.Context, log *logp.Logger, client kuberntescli.Interface, timeout time.Duration) (kube.Watcher, error) { 42 | watcher, err := kube.NewNamedWatcher("pod", client, &kube.Pod{}, kube.WatchOptions{ 43 | SyncTimeout: timeout, 44 | Node: "", 45 | Namespace: "", 46 | HonorReSyncs: true, 47 | }, nil) 48 | 49 | if err != nil { 50 | log.Errorf("could not create kubernetes watcher %v", err) 51 | return nil, err 52 | } 53 | 54 | p := &pod{ 55 | watcher: watcher, 56 | client: client, 57 | logger: log, 58 | ctx: ctx, 59 | } 60 | 61 | watcher.AddEventHandler(p) 62 | 63 | return watcher, nil 64 | } 65 | 66 | // Start starts the eventer 67 | func (p *pod) Start() error { 68 | return p.watcher.Start() 69 | } 70 | 71 | // Stop stops the eventer 72 | func (p *pod) Stop() { 73 | p.watcher.Stop() 74 | } 75 | 76 | // OnUpdate handles events for pods that have been updated. 77 | func (p *pod) OnUpdate(obj interface{}) { 78 | o := obj.(*kube.Pod) 79 | p.logger.Debugf("Watcher Pod update: %+v", o.Name) 80 | } 81 | 82 | // OnDelete stops pod objects that are deleted. 83 | func (p *pod) OnDelete(obj interface{}) { 84 | o := obj.(*kube.Pod) 85 | p.logger.Debugf("Watcher Pod delete: %+v", o.Name) 86 | } 87 | 88 | // OnAdd ensures processing of pod objects that are newly added. 89 | func (p *pod) OnAdd(obj interface{}) { 90 | o := obj.(*kube.Pod) 91 | p.logger.Debugf("Watcher Pod add: %+v", o.Name) 92 | } 93 | 94 | // publishK8sPods publishes the pod assets stored in pod watcher cache 95 | func publishK8sPods(ctx context.Context, log *logp.Logger, publisher stateless.Publisher, podWatcher, nodeWatcher kube.Watcher) { 96 | 97 | log.Info("Publishing pod assets\n") 98 | assetType := "k8s.pod" 99 | assetKind := "container_group" 100 | for _, obj := range podWatcher.Store().List() { 101 | o, ok := obj.(*kube.Pod) 102 | if ok { 103 | log.Debugf("Publish Pod: %+v", o.Name) 104 | assetName := o.Name 105 | assetId := string(o.UID) 106 | assetStartTime := o.Status.StartTime 107 | namespace := o.Namespace 108 | nodeName := o.Spec.NodeName 109 | 110 | assetParents := []string{} 111 | if nodeWatcher != nil { 112 | nodeId, err := getNodeIdFromName(nodeName, nodeWatcher) 113 | if err == nil { 114 | nodeAssetName := fmt.Sprintf("%s:%s", "host", nodeId) 115 | assetParents = append(assetParents, nodeAssetName) 116 | } else { 117 | log.Errorf("pod asset parents not collected: %w", err) 118 | } 119 | } 120 | 121 | internal.Publish(publisher, nil, 122 | internal.WithAssetKindAndID(assetKind, assetId), 123 | internal.WithAssetType(assetType), 124 | internal.WithAssetParents(assetParents), 125 | internal.WithAssetName(assetName), 126 | internal.WithPodData(assetName, assetId, namespace, assetStartTime), 127 | ) 128 | } else { 129 | log.Error("Publishing pod assets failed. Type assertion of pod object failed") 130 | } 131 | 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /deploy/assetbeat-kubernetes-manifest.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: assetbeatconfig 6 | namespace: kube-system 7 | labels: 8 | k8s-app: assetbeat 9 | data: 10 | assetbeat.yml: |- 11 | assetbeat.inputs: 12 | - type: assets_k8s 13 | period: 600s 14 | kube_config: "" 15 | asset_types: ["k8s.node", "k8s.pod", "k8s.container"] 16 | 17 | output.elasticsearch: 18 | hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] 19 | username: ${ELASTICSEARCH_USERNAME} 20 | password: ${ELASTICSEARCH_PASSWORD} 21 | ssl.verification_mode: "none" 22 | 23 | logging.level: info 24 | logging.to_files: false 25 | logging.to_stderr: true 26 | logging.selectors: ["*"] 27 | --- 28 | apiVersion: apps/v1 29 | kind: Deployment 30 | metadata: 31 | name: assetbeat 32 | namespace: kube-system 33 | labels: 34 | k8s-app: assetbeat 35 | spec: 36 | replicas: 1 37 | selector: 38 | matchLabels: 39 | k8s-app: assetbeat 40 | template: 41 | metadata: 42 | labels: 43 | k8s-app: assetbeat 44 | spec: 45 | serviceAccountName: assetbeat 46 | hostNetwork: true 47 | dnsPolicy: ClusterFirstWithHostNet 48 | containers: 49 | - name: assetbeat 50 | image: docker.elastic.co/observability/assetbeat:latest 51 | env: 52 | # The basic authentication username used to connect to Elasticsearch 53 | # This user needs the privileges required to publish events to Elasticsearch. 54 | - name: ELASTICSEARCH_USERNAME 55 | value: "elastic" 56 | # The basic authentication password used to connect to Elasticsearch 57 | - name: ELASTICSEARCH_PASSWORD 58 | value: "changeme" 59 | # The Elasticsearch host to communicate with 60 | - name: ELASTICSEARCH_HOST 61 | value: "elasticsearch" 62 | # The Elasticsearch port to communicate with 63 | - name: ELASTICSEARCH_PORT 64 | value: "9200" 65 | volumeMounts: 66 | - name: config 67 | mountPath: /usr/share/assetbeat/assetbeat.yml 68 | readOnly: true 69 | subPath: assetbeat.yml 70 | - name: etc-mid 71 | mountPath: /etc/machine-id 72 | 73 | volumes: 74 | - name: config 75 | configMap: 76 | defaultMode: 0640 77 | name: assetbeatconfig 78 | 79 | # Mount /etc/machine-id from the host to determine host ID; needed for 'hostdata' input 80 | - name: etc-mid 81 | hostPath: 82 | path: /etc/machine-id 83 | type: File 84 | --- 85 | apiVersion: v1 86 | kind: ServiceAccount 87 | metadata: 88 | name: assetbeat 89 | namespace: kube-system 90 | labels: 91 | k8s-app: assetbeat 92 | --- 93 | apiVersion: rbac.authorization.k8s.io/v1 94 | kind: ClusterRole 95 | metadata: 96 | name: assetbeat 97 | labels: 98 | k8s-app: assetbeat 99 | rules: 100 | - apiGroups: [""] 101 | resources: 102 | - nodes 103 | - namespaces 104 | - events 105 | - pods 106 | - services 107 | - configmaps 108 | verbs: ["get", "list", "watch"] 109 | - apiGroups: ["extensions"] 110 | resources: 111 | - replicasets 112 | verbs: ["get", "list", "watch"] 113 | - apiGroups: ["apps"] 114 | resources: 115 | - statefulsets 116 | - deployments 117 | - replicasets 118 | - daemonsets 119 | verbs: ["get", "list", "watch"] 120 | - apiGroups: 121 | - "" 122 | resources: 123 | - nodes/stats 124 | verbs: 125 | - get 126 | - apiGroups: [ "batch" ] 127 | resources: 128 | - jobs 129 | - cronjobs 130 | verbs: [ "get", "list", "watch" ] 131 | # Needed for apiserver 132 | - nonResourceURLs: 133 | - "/metrics" 134 | verbs: 135 | - get 136 | --- 137 | apiVersion: rbac.authorization.k8s.io/v1 138 | kind: ClusterRoleBinding 139 | metadata: 140 | name: assetbeat 141 | subjects: 142 | - kind: ServiceAccount 143 | name: assetbeat 144 | namespace: kube-system 145 | roleRef: 146 | kind: ClusterRole 147 | name: assetbeat 148 | apiGroup: rbac.authorization.k8s.io 149 | --- -------------------------------------------------------------------------------- /input/aws/aws_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package aws 19 | 20 | import ( 21 | "context" 22 | "sync" 23 | "testing" 24 | "time" 25 | 26 | "github.com/elastic/assetbeat/input/testutil" 27 | v2 "github.com/elastic/beats/v7/filebeat/input/v2" 28 | "github.com/elastic/elastic-agent-libs/logp" 29 | 30 | "github.com/elastic/assetbeat/input/internal" 31 | 32 | "github.com/aws/aws-sdk-go-v2/aws" 33 | "github.com/stretchr/testify/assert" 34 | ) 35 | 36 | func TestGetAWSConfigForRegion(t *testing.T) { 37 | for _, tt := range []struct { 38 | name string 39 | 40 | env map[string]string 41 | inputCfg config 42 | region string 43 | expectedCreds aws.Credentials 44 | }{ 45 | { 46 | name: "with explicit creds", 47 | 48 | inputCfg: config{ 49 | BaseConfig: internal.BaseConfig{ 50 | Period: time.Second * 600, 51 | AssetTypes: []string{}, 52 | }, 53 | Regions: []string{"eu-west-2", "eu-west-1"}, 54 | AccessKeyId: "accesskey123", 55 | SecretAccessKey: "secretkey123", 56 | SessionToken: "token123", 57 | }, 58 | region: "eu-west-2", 59 | 60 | expectedCreds: aws.Credentials{ 61 | AccessKeyID: "accesskey123", 62 | SecretAccessKey: "secretkey123", 63 | SessionToken: "token123", 64 | Source: "assetbeat configuration", 65 | }, 66 | }, 67 | { 68 | name: "with environment variable creds", 69 | 70 | env: map[string]string{ 71 | "AWS_ACCESS_KEY": "EXAMPLE_ACCESS_KEY", 72 | "AWS_SECRET_ACCESS_KEY": "EXAMPLE_SECRET_KEY", 73 | }, 74 | inputCfg: config{ 75 | BaseConfig: internal.BaseConfig{ 76 | Period: time.Second * 600, 77 | AssetTypes: []string{}, 78 | }, 79 | Regions: []string{"eu-west-2", "eu-west-1"}, 80 | AccessKeyId: "", 81 | SecretAccessKey: "", 82 | SessionToken: "", 83 | }, 84 | region: "eu-west-2", 85 | 86 | expectedCreds: aws.Credentials{ 87 | AccessKeyID: "EXAMPLE_ACCESS_KEY", 88 | SecretAccessKey: "EXAMPLE_SECRET_KEY", 89 | SessionToken: "", 90 | Source: "EnvConfigCredentials", 91 | }, 92 | }, 93 | } { 94 | t.Run(tt.name, func(t *testing.T) { 95 | ctx := context.Background() 96 | 97 | for k, v := range tt.env { 98 | t.Setenv(k, v) 99 | } 100 | 101 | awsCfg, err := getAWSConfigForRegion(ctx, tt.inputCfg, tt.region) 102 | assert.NoError(t, err) 103 | 104 | retrievedAWSCreds, err := awsCfg.Credentials.Retrieve(context.Background()) 105 | assert.NoError(t, err) 106 | assert.Equal(t, tt.expectedCreds, retrievedAWSCreds) 107 | }) 108 | } 109 | } 110 | 111 | func TestPlugin(t *testing.T) { 112 | p := Plugin() 113 | assert.Equal(t, "assets_aws", p.Name) 114 | assert.NotNil(t, p.Manager) 115 | } 116 | 117 | func TestAssetsAWS_Run(t *testing.T) { 118 | publisher := testutil.NewInMemoryPublisher() 119 | 120 | ctx, cancel := context.WithCancel(context.Background()) 121 | inputCtx := v2.Context{ 122 | Logger: logp.NewLogger("test"), 123 | Cancelation: ctx, 124 | } 125 | 126 | input, err := newAssetsAWS(defaultConfig()) 127 | assert.NoError(t, err) 128 | 129 | var wg sync.WaitGroup 130 | wg.Add(1) 131 | go func() { 132 | defer wg.Done() 133 | err = input.Run(inputCtx, publisher) 134 | assert.NoError(t, err) 135 | }() 136 | 137 | time.Sleep(time.Millisecond) 138 | cancel() 139 | timeout := time.After(time.Second) 140 | closeCh := make(chan struct{}) 141 | go func() { 142 | defer close(closeCh) 143 | wg.Wait() 144 | }() 145 | select { 146 | case <-timeout: 147 | t.Fatal("Test timed out") 148 | case <-closeCh: 149 | // Waitgroup finished in time, nothing to do 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /input/aws/vpc.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package aws 19 | 20 | import ( 21 | "context" 22 | "fmt" 23 | 24 | "github.com/elastic/assetbeat/input/internal" 25 | stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" 26 | 27 | "github.com/elastic/elastic-agent-libs/logp" 28 | "github.com/elastic/elastic-agent-libs/mapstr" 29 | 30 | "github.com/aws/aws-sdk-go-v2/service/ec2" 31 | "github.com/aws/aws-sdk-go-v2/service/ec2/types" 32 | ) 33 | 34 | func collectVPCAssets(ctx context.Context, client ec2.DescribeVpcsAPIClient, region string, log *logp.Logger, publisher stateless.Publisher) error { 35 | vpcs, err := describeVPCs(ctx, client) 36 | if err != nil { 37 | return err 38 | } 39 | 40 | assetType := "aws.vpc" 41 | assetKind := "network" 42 | for _, vpc := range vpcs { 43 | options := []internal.AssetOption{ 44 | internal.WithAssetCloudProvider("aws"), 45 | internal.WithAssetRegion(region), 46 | internal.WithAssetAccountID(*vpc.OwnerId), 47 | internal.WithAssetName(getEC2AssetNameFromTags(vpc.Tags)), 48 | internal.WithAssetKindAndID(assetKind, *vpc.VpcId), 49 | internal.WithAssetType(assetType), 50 | WithAssetTags(flattenEC2Tags(vpc.Tags)), 51 | internal.WithAssetMetadata(mapstr.M{ 52 | "isDefault": vpc.IsDefault, 53 | }), 54 | } 55 | vpcName := getEC2AssetNameFromTags(vpc.Tags) 56 | if vpcName != "" { 57 | options = append(options, internal.WithAssetName(vpcName)) 58 | } 59 | internal.Publish(publisher, nil, 60 | options..., 61 | ) 62 | } 63 | 64 | return nil 65 | } 66 | 67 | func collectSubnetAssets(ctx context.Context, client ec2.DescribeSubnetsAPIClient, region string, log *logp.Logger, publisher stateless.Publisher) error { 68 | subnets, err := describeSubnets(ctx, client) 69 | if err != nil { 70 | return err 71 | } 72 | 73 | assetType := "aws.subnet" 74 | assetKind := "network" 75 | for _, subnet := range subnets { 76 | var parents []string 77 | if subnet.VpcId != nil { 78 | parents = []string{"network:" + *subnet.VpcId} 79 | } 80 | 81 | options := []internal.AssetOption{ 82 | internal.WithAssetCloudProvider("aws"), 83 | internal.WithAssetRegion(region), 84 | internal.WithAssetAccountID(*subnet.OwnerId), 85 | internal.WithAssetKindAndID(assetKind, *subnet.SubnetId), 86 | internal.WithAssetType(assetType), 87 | internal.WithAssetParents(parents), 88 | WithAssetTags(flattenEC2Tags(subnet.Tags)), 89 | internal.WithAssetMetadata(mapstr.M{ 90 | "state": string(subnet.State), 91 | }), 92 | } 93 | subnetName := getEC2AssetNameFromTags(subnet.Tags) 94 | if subnetName != "" { 95 | options = append(options, internal.WithAssetName(subnetName)) 96 | } 97 | internal.Publish(publisher, nil, 98 | options..., 99 | ) 100 | } 101 | 102 | return nil 103 | } 104 | 105 | func describeVPCs(ctx context.Context, client ec2.DescribeVpcsAPIClient) ([]types.Vpc, error) { 106 | vpcs := make([]types.Vpc, 0, 100) 107 | paginator := ec2.NewDescribeVpcsPaginator(client, &ec2.DescribeVpcsInput{}) 108 | for paginator.HasMorePages() { 109 | resp, err := paginator.NextPage(ctx) 110 | if err != nil { 111 | return nil, fmt.Errorf("error describing VPCs: %w", err) 112 | } 113 | 114 | vpcs = append(vpcs, resp.Vpcs...) 115 | } 116 | 117 | return vpcs, nil 118 | } 119 | 120 | func describeSubnets(ctx context.Context, client ec2.DescribeSubnetsAPIClient) ([]types.Subnet, error) { 121 | subnets := make([]types.Subnet, 0, 100) 122 | paginator := ec2.NewDescribeSubnetsPaginator(client, &ec2.DescribeSubnetsInput{}) 123 | for paginator.HasMorePages() { 124 | resp, err := paginator.NextPage(ctx) 125 | if err != nil { 126 | return nil, fmt.Errorf("error describing subnets: %w", err) 127 | } 128 | 129 | subnets = append(subnets, resp.Subnets...) 130 | } 131 | 132 | return subnets, nil 133 | } 134 | -------------------------------------------------------------------------------- /input/k8s/nodes_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package k8s 19 | 20 | import ( 21 | "context" 22 | "fmt" 23 | "testing" 24 | "time" 25 | 26 | "github.com/stretchr/testify/assert" 27 | v1 "k8s.io/api/core/v1" 28 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 29 | k8sfake "k8s.io/client-go/kubernetes/fake" 30 | 31 | "github.com/elastic/assetbeat/input/testutil" 32 | "github.com/elastic/elastic-agent-autodiscover/kubernetes" 33 | "github.com/elastic/elastic-agent-libs/logp" 34 | ) 35 | 36 | func TestGetNodeWatcher(t *testing.T) { 37 | client := k8sfake.NewSimpleClientset() 38 | log := logp.NewLogger("mylogger") 39 | _, err := getNodeWatcher(context.Background(), log, client, time.Second*60) 40 | if err != nil { 41 | t.Fatalf("error initiating Node watcher") 42 | } 43 | assert.NoError(t, err) 44 | } 45 | 46 | func TestGetNodeIdFromName(t *testing.T) { 47 | for _, tt := range []struct { 48 | name string 49 | nodeName string 50 | input kubernetes.Resource 51 | output error 52 | }{ 53 | { 54 | name: "node not found", 55 | nodeName: "node2", 56 | input: &v1.Node{ 57 | ObjectMeta: metav1.ObjectMeta{ 58 | Name: "node1", 59 | UID: "60988eed-1885-4b63-9fa4-780206969deb", 60 | Labels: map[string]string{ 61 | "foo": "bar", 62 | }, 63 | Annotations: map[string]string{ 64 | "key1": "value1", 65 | "key2": "value2", 66 | }, 67 | }, 68 | TypeMeta: metav1.TypeMeta{ 69 | Kind: "Node", 70 | APIVersion: "v1", 71 | }, 72 | Status: v1.NodeStatus{ 73 | Addresses: []v1.NodeAddress{{Type: v1.NodeHostName, Address: "node1"}}, 74 | }, 75 | }, 76 | output: fmt.Errorf("node with name %s does not exist in cache", "node2"), 77 | }, 78 | { 79 | name: "node found", 80 | nodeName: "node2", 81 | input: &v1.Node{ 82 | ObjectMeta: metav1.ObjectMeta{ 83 | Name: "node2", 84 | UID: "60988eed-1885-4b63-9fa4-780206969deb", 85 | Labels: map[string]string{ 86 | "foo": "bar", 87 | }, 88 | Annotations: map[string]string{ 89 | "key1": "value1", 90 | "key2": "value2", 91 | }, 92 | }, 93 | TypeMeta: metav1.TypeMeta{ 94 | Kind: "Node", 95 | APIVersion: "v1", 96 | }, 97 | Status: v1.NodeStatus{ 98 | Addresses: []v1.NodeAddress{{Type: v1.NodeHostName, Address: "node2"}}, 99 | }, 100 | }, 101 | output: nil, 102 | }, 103 | } { 104 | t.Run(tt.name, func(t *testing.T) { 105 | client := k8sfake.NewSimpleClientset() 106 | log := logp.NewLogger("mylogger") 107 | nodeWatcher, _ := getNodeWatcher(context.Background(), log, client, time.Second*60) 108 | _ = nodeWatcher.Store().Add(tt.input) 109 | _, err := getNodeIdFromName(tt.nodeName, nodeWatcher) 110 | assert.Equal(t, err, tt.output) 111 | }) 112 | } 113 | } 114 | 115 | func TestPublishK8sNodes(t *testing.T) { 116 | client := k8sfake.NewSimpleClientset() 117 | log := logp.NewLogger("mylogger") 118 | nodeWatcher, err := getNodeWatcher(context.Background(), log, client, time.Second*60) 119 | if err != nil { 120 | t.Fatalf("error initiating Node watcher") 121 | } 122 | input := &v1.Node{ 123 | ObjectMeta: metav1.ObjectMeta{ 124 | Name: "nodeName", 125 | UID: "60988eed-1885-4b63-9fa4-780206969deb", 126 | Labels: map[string]string{ 127 | "foo": "bar", 128 | }, 129 | Annotations: map[string]string{ 130 | "key": "value", 131 | }, 132 | }, 133 | TypeMeta: metav1.TypeMeta{ 134 | Kind: "Node", 135 | APIVersion: "v1", 136 | }, 137 | Status: v1.NodeStatus{ 138 | Addresses: []v1.NodeAddress{{Type: v1.NodeHostName, Address: "node1"}}, 139 | }, 140 | } 141 | _ = nodeWatcher.Store().Add(input) 142 | publisher := testutil.NewInMemoryPublisher() 143 | publishK8sNodes(context.Background(), log, publisher, nodeWatcher, false) 144 | 145 | assert.Equal(t, 1, len(publisher.Events)) 146 | } 147 | -------------------------------------------------------------------------------- /input/gcp/compute.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package gcp 19 | 20 | import ( 21 | "context" 22 | "strconv" 23 | 24 | "github.com/googleapis/gax-go/v2" 25 | 26 | compute "cloud.google.com/go/compute/apiv1" 27 | "cloud.google.com/go/compute/apiv1/computepb" 28 | "google.golang.org/api/iterator" 29 | 30 | "github.com/elastic/assetbeat/input/internal" 31 | stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" 32 | "github.com/elastic/elastic-agent-libs/logp" 33 | "github.com/elastic/elastic-agent-libs/mapstr" 34 | "github.com/elastic/go-freelru" 35 | ) 36 | 37 | type AggregatedInstanceIterator interface { 38 | Next() (compute.InstancesScopedListPair, error) 39 | } 40 | 41 | type listInstanceAPIClient struct { 42 | AggregatedList func(ctx context.Context, req *computepb.AggregatedListInstancesRequest, opts ...gax.CallOption) AggregatedInstanceIterator 43 | } 44 | 45 | type computeInstance struct { 46 | ID string 47 | Region string 48 | Account string 49 | VPCs []string 50 | Labels map[string]string 51 | Metadata mapstr.M 52 | RawMd *computepb.Metadata 53 | Name string 54 | } 55 | 56 | func collectComputeAssets(ctx context.Context, cfg config, subnetAssetCache *freelru.LRU[string, *subnet], computeAssetCache *freelru.LRU[string, *computeInstance], client listInstanceAPIClient, publisher stateless.Publisher, log *logp.Logger) error { 57 | 58 | instances, err := getAllComputeInstances(ctx, cfg, subnetAssetCache, computeAssetCache, client) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | assetType := "gcp.compute.instance" 64 | assetKind := "host" 65 | log.Debug("Publishing GCP compute instances") 66 | 67 | for _, instance := range instances { 68 | var parents []string 69 | for _, vpc := range instance.VPCs { 70 | if len(vpc) > 0 { 71 | parents = append(parents, "network:"+vpc) 72 | } 73 | } 74 | options := []internal.AssetOption{ 75 | internal.WithAssetCloudProvider("gcp"), 76 | internal.WithAssetRegion(instance.Region), 77 | internal.WithAssetAccountID(instance.Account), 78 | internal.WithAssetKindAndID(assetKind, instance.ID), 79 | internal.WithAssetType(assetType), 80 | internal.WithAssetParents(parents), 81 | WithAssetLabels(internal.ToMapstr(instance.Labels)), 82 | internal.WithAssetMetadata(instance.Metadata), 83 | } 84 | 85 | if instance.Name != "" { 86 | options = append(options, internal.WithAssetName(instance.Name)) 87 | } 88 | internal.Publish(publisher, nil, options...) 89 | } 90 | 91 | return nil 92 | } 93 | 94 | func getAllComputeInstances(ctx context.Context, cfg config, subnetAssetCache *freelru.LRU[string, *subnet], computeAssetCache *freelru.LRU[string, *computeInstance], client listInstanceAPIClient) ([]computeInstance, error) { 95 | var instances []computeInstance 96 | 97 | for _, p := range cfg.Projects { 98 | req := &computepb.AggregatedListInstancesRequest{ 99 | Project: p, 100 | } 101 | it := client.AggregatedList(ctx, req) 102 | 103 | for { 104 | instanceScopedPair, err := it.Next() 105 | if err == iterator.Done { 106 | break 107 | } 108 | if err != nil { 109 | return nil, err 110 | } 111 | zone := instanceScopedPair.Key 112 | if wantZone(zone, cfg.Regions) { 113 | for _, i := range instanceScopedPair.Value.Instances { 114 | var subnets []string 115 | for _, ni := range i.NetworkInterfaces { 116 | subnets = append(subnets, getSubnetIdFromLink(*ni.Subnetwork, subnetAssetCache)) 117 | } 118 | cI := computeInstance{ 119 | ID: strconv.FormatUint(*i.Id, 10), 120 | Region: getRegionFromZoneURL(zone), 121 | Account: p, 122 | VPCs: subnets, 123 | Labels: i.Labels, 124 | Metadata: mapstr.M{ 125 | "state": *i.Status, 126 | }, 127 | RawMd: i.GetMetadata(), 128 | Name: i.GetName(), 129 | } 130 | selfLink := *i.SelfLink 131 | computeAssetCache.AddWithExpire(selfLink, &cI, cfg.Period*2) 132 | instances = append(instances, cI) 133 | } 134 | } 135 | 136 | } 137 | } 138 | 139 | return instances, nil 140 | } 141 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package config 19 | 20 | import ( 21 | "fmt" 22 | "log" 23 | "os" 24 | "path/filepath" 25 | "sort" 26 | "time" 27 | 28 | "github.com/elastic/beats/v7/libbeat/autodiscover" 29 | "github.com/elastic/beats/v7/libbeat/cfgfile" 30 | conf "github.com/elastic/elastic-agent-libs/config" 31 | "github.com/elastic/elastic-agent-libs/logp" 32 | "github.com/elastic/elastic-agent-libs/paths" 33 | ) 34 | 35 | // Defaults for config variables which are not set 36 | const ( 37 | DefaultType = "log" 38 | ) 39 | 40 | type Config struct { 41 | Inputs []*conf.C `config:"inputs"` 42 | ConfigDir string `config:"config_dir"` 43 | ShutdownTimeout time.Duration `config:"shutdown_timeout"` 44 | ConfigInput *conf.C `config:"config.inputs"` 45 | Autodiscover *autodiscover.Config `config:"autodiscover"` 46 | } 47 | 48 | var DefaultConfig = Config{ 49 | ShutdownTimeout: 0, 50 | } 51 | 52 | // getConfigFiles returns list of config files. 53 | // In case path is a file, it will be directly returned. 54 | // In case it is a directory, it will fetch all .yml files inside this directory 55 | func getConfigFiles(path string) (configFiles []string, err error) { 56 | // Check if path is valid file or dir 57 | stat, err := os.Stat(path) 58 | if err != nil { 59 | return nil, err 60 | } 61 | 62 | // Create empty slice for config file list 63 | configFiles = make([]string, 0) 64 | 65 | if stat.IsDir() { 66 | files, err := filepath.Glob(path + "/*.yml") 67 | if err != nil { 68 | return nil, err 69 | } 70 | 71 | configFiles = append(configFiles, files...) 72 | 73 | } else { 74 | // Only 1 config file 75 | configFiles = append(configFiles, path) 76 | } 77 | 78 | return configFiles, nil 79 | } 80 | 81 | // mergeConfigFiles reads in all config files given by list configFiles and merges them into config 82 | func mergeConfigFiles(configFiles []string, config *Config) error { 83 | for _, file := range configFiles { 84 | logp.Info("Additional configs loaded from: %s", file) 85 | 86 | tmpConfig := struct { 87 | Assetbeat Config 88 | }{} 89 | //nolint:staticcheck // Let's keep the logic here 90 | err := cfgfile.Read(&tmpConfig, file) 91 | if err != nil { 92 | return fmt.Errorf("failed to read %s: %w", file, err) 93 | } 94 | 95 | config.Inputs = append(config.Inputs, tmpConfig.Assetbeat.Inputs...) 96 | } 97 | 98 | return nil 99 | } 100 | 101 | // Fetches and merges all config files given by configDir. All are put into one config object 102 | func (config *Config) FetchConfigs() error { 103 | configDir := config.ConfigDir 104 | 105 | // If option not set, do nothing 106 | if configDir == "" { 107 | return nil 108 | } 109 | 110 | // If configDir is relative, consider it relative to the config path 111 | configDir = paths.Resolve(paths.Config, configDir) 112 | 113 | // Check if optional configDir is set to fetch additional config files 114 | logp.Info("Additional config files are fetched from: %s", configDir) 115 | 116 | configFiles, err := getConfigFiles(configDir) 117 | if err != nil { 118 | log.Fatal("Could not use config_dir of: ", configDir, err) 119 | return err 120 | } 121 | 122 | err = mergeConfigFiles(configFiles, config) 123 | if err != nil { 124 | log.Fatal("Error merging config files: ", err) 125 | return err 126 | } 127 | 128 | return nil 129 | } 130 | 131 | // ListEnabledInputs returns a list of enabled inputs sorted by alphabetical order. 132 | func (config *Config) ListEnabledInputs() []string { 133 | t := struct { 134 | Type string `config:"type"` 135 | }{} 136 | var inputs []string 137 | for _, input := range config.Inputs { 138 | if input.Enabled() { 139 | _ = input.Unpack(&t) 140 | inputs = append(inputs, t.Type) 141 | } 142 | } 143 | sort.Strings(inputs) 144 | return inputs 145 | } 146 | 147 | // IsInputEnabled returns true if the plugin name is enabled. 148 | func (config *Config) IsInputEnabled(name string) bool { 149 | enabledInputs := config.ListEnabledInputs() 150 | for _, input := range enabledInputs { 151 | if name == input { 152 | return true 153 | } 154 | } 155 | return false 156 | } 157 | -------------------------------------------------------------------------------- /input/hostdata/hostdata_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package hostdata 19 | 20 | import ( 21 | "context" 22 | "github.com/elastic/assetbeat/input/internal" 23 | "testing" 24 | 25 | "github.com/elastic/go-sysinfo" 26 | 27 | "github.com/elastic/beats/v7/libbeat/beat" 28 | "github.com/elastic/elastic-agent-system-metrics/metric/system/host" 29 | 30 | "github.com/elastic/assetbeat/input/testutil" 31 | "github.com/elastic/elastic-agent-libs/logp" 32 | 33 | "github.com/stretchr/testify/assert" 34 | 35 | conf "github.com/elastic/elastic-agent-libs/config" 36 | ) 37 | 38 | type fakeCloudMetadataProcessor struct{} 39 | 40 | func (p fakeCloudMetadataProcessor) Run(event *beat.Event) (*beat.Event, error) { 41 | _, err := event.PutValue("cloud.instance.id", "i-12342") 42 | if err != nil { 43 | return nil, err 44 | } 45 | return event, err 46 | } 47 | 48 | func (p fakeCloudMetadataProcessor) String() string { 49 | return "fake_add_cloud_metadata" 50 | } 51 | 52 | func TestHostdata_configurationAndInitialization(t *testing.T) { 53 | input, err := configure(conf.NewConfig()) 54 | assert.Nil(t, err) 55 | 56 | hostdata := input.(*hostdata) 57 | assert.Equal(t, defaultCollectionPeriod, hostdata.config.Period) 58 | 59 | assert.NotEmpty(t, hostdata.hostInfo) 60 | hostname, _ := hostdata.hostInfo.GetValue("host.hostname") 61 | assert.NotEmpty(t, hostname) 62 | } 63 | 64 | func TestHostdata_reportHostDataAssets(t *testing.T) { 65 | input, _ := configure(conf.NewConfig()) 66 | 67 | publisher := testutil.NewInMemoryPublisher() 68 | input.(*hostdata).reportHostDataAssets(context.Background(), logp.NewLogger("test"), publisher) 69 | assert.NotEmpty(t, publisher.Events) 70 | event := publisher.Events[0] 71 | 72 | // check that the base fields are populated 73 | hostID, _ := event.Fields.GetValue("host.id") 74 | assetID, _ := event.Fields.GetValue("asset.id") 75 | assetType, _ := event.Fields.GetValue("asset.type") 76 | assetKind, _ := event.Fields.GetValue("asset.kind") 77 | destinationDatastream, _ := event.Meta.GetValue("index") 78 | 79 | //assert.NotEmpty(t, hostID) 80 | assert.Equal(t, hostID, assetID) 81 | assert.Equal(t, "host", assetType) 82 | assert.Equal(t, "host", assetKind) 83 | assert.Equal(t, internal.GetDefaultIndexName(), destinationDatastream) 84 | 85 | // check that the networking fields are populated 86 | // (and that the stored host data has not been modified) 87 | ips, _ := event.Fields.GetValue("host.ip") 88 | assert.NotEmpty(t, ips) 89 | 90 | _, err := input.(*hostdata).hostInfo.GetValue("host.ip") 91 | assert.Error(t, err) 92 | } 93 | 94 | func TestHostdata_reportHostDataAssetsWithCloudMeta(t *testing.T) { 95 | input, _ := configure(conf.NewConfig()) 96 | hostDataProvider, _ := sysinfo.Host() 97 | 98 | hd := hostdata{ 99 | config: defaultConfig(), 100 | hostInfo: host.MapHostInfo(hostDataProvider.Info()), 101 | addCloudMetadataProcessor: fakeCloudMetadataProcessor{}, 102 | } 103 | publisher := testutil.NewInMemoryPublisher() 104 | hd.reportHostDataAssets(context.Background(), logp.NewLogger("test"), publisher) 105 | assert.NotEmpty(t, publisher.Events) 106 | event := publisher.Events[0] 107 | 108 | // check that the base fields are populated 109 | hostID, _ := event.Fields.GetValue("host.id") 110 | assetID, _ := event.Fields.GetValue("asset.id") 111 | assetType, _ := event.Fields.GetValue("asset.type") 112 | assetKind, _ := event.Fields.GetValue("asset.kind") 113 | destinationDatastream, _ := event.Meta.GetValue("index") 114 | cloudID, _ := event.Fields.GetValue("cloud.instance.id") 115 | 116 | assert.NotEmpty(t, hostID) 117 | assert.NotEmpty(t, cloudID) 118 | assert.Equal(t, cloudID, assetID) 119 | assert.Equal(t, hostID, assetID) 120 | assert.Equal(t, "host", assetType) 121 | assert.Equal(t, "host", assetKind) 122 | assert.Equal(t, internal.GetDefaultIndexName(), destinationDatastream) 123 | 124 | // check that the networking fields are populated 125 | // (and that the stored host data has not been modified) 126 | ips, _ := event.Fields.GetValue("host.ip") 127 | assert.NotEmpty(t, ips) 128 | 129 | _, err := input.(*hostdata).hostInfo.GetValue("host.ip") 130 | assert.Error(t, err) 131 | } 132 | -------------------------------------------------------------------------------- /beater/crawler.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package beater 19 | 20 | import ( 21 | "fmt" 22 | "sync" 23 | 24 | "github.com/elastic/beats/v7/filebeat/input" 25 | 26 | "github.com/mitchellh/hashstructure" 27 | 28 | "github.com/elastic/beats/v7/libbeat/beat" 29 | "github.com/elastic/beats/v7/libbeat/cfgfile" 30 | conf "github.com/elastic/elastic-agent-libs/config" 31 | "github.com/elastic/elastic-agent-libs/logp" 32 | ) 33 | 34 | type crawler struct { 35 | log *logp.Logger 36 | inputs map[uint64]cfgfile.Runner 37 | inputConfigs []*conf.C 38 | wg sync.WaitGroup 39 | inputsFactory cfgfile.RunnerFactory 40 | inputReloader *cfgfile.Reloader 41 | once bool 42 | beatDone chan struct{} 43 | } 44 | 45 | func newCrawler( 46 | inputFactory, module cfgfile.RunnerFactory, 47 | inputConfigs []*conf.C, 48 | beatDone chan struct{}, 49 | once bool, 50 | ) (*crawler, error) { 51 | return &crawler{ 52 | log: logp.NewLogger("crawler"), 53 | inputs: map[uint64]cfgfile.Runner{}, 54 | inputsFactory: inputFactory, 55 | inputConfigs: inputConfigs, 56 | once: once, 57 | beatDone: beatDone, 58 | }, nil 59 | } 60 | 61 | // Start starts the crawler with all inputs 62 | func (c *crawler) Start( 63 | pipeline beat.PipelineConnector, 64 | configInputs *conf.C, 65 | ) error { 66 | log := c.log 67 | 68 | log.Infof("Loading Inputs: %d", len(c.inputConfigs)) 69 | 70 | // Prospect the globs/paths given on the command line and launch harvesters 71 | for _, inputConfig := range c.inputConfigs { 72 | err := c.startInput(pipeline, inputConfig) 73 | if err != nil { 74 | return fmt.Errorf("starting input failed: %w", err) 75 | } 76 | } 77 | 78 | if configInputs.Enabled() { 79 | c.inputReloader = cfgfile.NewReloader(pipeline, configInputs) 80 | if err := c.inputReloader.Check(c.inputsFactory); err != nil { 81 | return fmt.Errorf("creating input reloader failed: %w", err) 82 | } 83 | } 84 | 85 | if c.inputReloader != nil { 86 | go func() { 87 | c.inputReloader.Run(c.inputsFactory) 88 | }() 89 | } 90 | 91 | log.Infof("Loading and starting Inputs completed. Enabled inputs: %d", len(c.inputs)) 92 | 93 | return nil 94 | } 95 | 96 | func (c *crawler) startInput( 97 | pipeline beat.PipelineConnector, 98 | config *conf.C, 99 | ) error { 100 | // TODO: Either use debug or remove it after https://github.com/elastic/beats/pull/30534 101 | // is fixed. 102 | c.log.Infof("starting input, keys present on the config: %v", 103 | config.FlattenedKeys()) 104 | 105 | if !config.Enabled() { 106 | c.log.Infof("input disabled, skipping it") 107 | return nil 108 | } 109 | 110 | var h map[string]interface{} 111 | err := config.Unpack(&h) 112 | if err != nil { 113 | return fmt.Errorf("could not unpack config: %w", err) 114 | } 115 | id, err := hashstructure.Hash(h, nil) 116 | if err != nil { 117 | return fmt.Errorf("can not compute id from configuration: %w", err) 118 | } 119 | if _, ok := c.inputs[id]; ok { 120 | return fmt.Errorf("input with same ID already exists: %d", id) 121 | } 122 | 123 | runner, err := c.inputsFactory.Create(pipeline, config) 124 | if err != nil { 125 | return fmt.Errorf("error while initializing input: %w", err) 126 | } 127 | if assetbeat, ok := runner.(*input.Runner); ok { 128 | assetbeat.Once = c.once 129 | } 130 | 131 | c.inputs[id] = runner 132 | 133 | c.log.Infof("Starting input (ID: %d)", id) 134 | runner.Start() 135 | 136 | return nil 137 | } 138 | 139 | func (c *crawler) Stop() { 140 | logp.Info("Stopping Crawler") 141 | 142 | asyncWaitStop := func(stop func()) { 143 | c.wg.Add(1) 144 | go func() { 145 | defer c.wg.Done() 146 | stop() 147 | }() 148 | } 149 | 150 | logp.Info("Stopping %d inputs", len(c.inputs)) 151 | // Stop inputs in parallel 152 | for id, p := range c.inputs { 153 | id, p := id, p 154 | asyncWaitStop(func() { 155 | c.log.Infof("Stopping input: %d", id) 156 | p.Stop() 157 | }) 158 | } 159 | 160 | if c.inputReloader != nil { 161 | asyncWaitStop(c.inputReloader.Stop) 162 | } 163 | 164 | c.WaitForCompletion() 165 | 166 | logp.Info("Crawler stopped") 167 | } 168 | 169 | func (c *crawler) WaitForCompletion() { 170 | c.wg.Wait() 171 | } 172 | -------------------------------------------------------------------------------- /input/gcp/util.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package gcp 19 | 20 | import ( 21 | "strings" 22 | "time" 23 | 24 | "cloud.google.com/go/compute/apiv1/computepb" 25 | "cloud.google.com/go/container/apiv1/containerpb" 26 | "google.golang.org/protobuf/proto" 27 | 28 | "github.com/cespare/xxhash" 29 | 30 | "github.com/elastic/go-freelru" 31 | ) 32 | 33 | func getResourceNameFromURL(res string) string { 34 | s := strings.Split(res, "/") 35 | return s[len(s)-1] 36 | } 37 | 38 | func getRegionFromZoneURL(zone string) string { 39 | z := getResourceNameFromURL(zone) 40 | r := strings.Split(z, "-") 41 | return strings.Join(r[:len(r)-1], "-") 42 | } 43 | 44 | func getVpcIdFromLink(selfLink string, vpcAssetCache *freelru.LRU[string, *vpc]) string { 45 | v, ok := vpcAssetCache.Get(selfLink) 46 | if ok { 47 | return v.ID 48 | } 49 | return "" 50 | } 51 | 52 | func getSubnetIdFromLink(selfLink string, subnetAssetCache *freelru.LRU[string, *subnet]) string { 53 | v, ok := subnetAssetCache.Get(selfLink) 54 | if ok { 55 | return v.ID 56 | } 57 | return "" 58 | } 59 | 60 | func getNetSelfLinkFromNetConfig(networkConfig *containerpb.NetworkConfig) string { 61 | network := networkConfig.Network 62 | if len(network) > 0 { 63 | return "https://www.googleapis.com/compute/v1/" + network 64 | } 65 | 66 | return "" 67 | } 68 | 69 | func hashStringXXHASH(s string) uint32 { 70 | return uint32(xxhash.Sum64String(s)) 71 | } 72 | 73 | // region is in the form of regions/us-west2 74 | func wantRegion(region string, confRegions []string) bool { 75 | if len(confRegions) == 0 { 76 | return true 77 | } 78 | ss := strings.Split(region, "/") 79 | subnetsRegion := ss[len(ss)-1] 80 | for _, region := range confRegions { 81 | if region == subnetsRegion { 82 | return true 83 | } 84 | } 85 | 86 | return false 87 | } 88 | 89 | func wantZone(zone string, confRegions []string) bool { 90 | if len(confRegions) == 0 { 91 | return true 92 | } 93 | 94 | region := getRegionFromZoneURL(zone) 95 | for _, z := range confRegions { 96 | if z == region { 97 | return true 98 | } 99 | } 100 | 101 | return false 102 | } 103 | 104 | func getTestVpcCache() *freelru.LRU[string, *vpc] { 105 | vpcAssetsCache, _ := freelru.New[string, *vpc](8192, hashStringXXHASH) 106 | nv := vpc{ 107 | ID: "1", 108 | } 109 | selfLink := "https://www.googleapis.com/compute/v1/projects/my_project/global/networks/my_network" 110 | vpcAssetsCache.AddWithExpire(selfLink, &nv, 60*time.Second) 111 | return vpcAssetsCache 112 | } 113 | 114 | func getTestSubnetCache() *freelru.LRU[string, *subnet] { 115 | subnetAssetsCache, _ := freelru.New[string, *subnet](8192, hashStringXXHASH) 116 | sb := subnet{ 117 | ID: "2", 118 | } 119 | selfLink := "https://www.googleapis.com/compute/v1/projects/elastic-observability/regions/us-central1/subnetworks/my_subnet" 120 | subnetAssetsCache.AddWithExpire(selfLink, &sb, 60*time.Second) 121 | return subnetAssetsCache 122 | } 123 | 124 | func getTestComputeCache() *freelru.LRU[string, *computeInstance] { 125 | computeAssetsCache, _ := freelru.New[string, *computeInstance](8192, hashStringXXHASH) 126 | cI := computeInstance{ 127 | ID: "123", 128 | Region: "europe-west1", 129 | RawMd: &computepb.Metadata{ 130 | Items: []*computepb.Items{ 131 | { 132 | Key: proto.String("kube-labels"), 133 | Value: proto.String("cloud.google.com/gke-nodepool=mynodepool"), 134 | }, 135 | }, 136 | }, 137 | } 138 | selfLink := "https://www.googleapis.com/compute/v1/projects/elastic-observability/zones/europe-west1-d/instances/my-instance-1" 139 | computeAssetsCache.AddWithExpire(selfLink, &cI, 60*time.Second) 140 | return computeAssetsCache 141 | } 142 | 143 | func getComputeCache() *freelru.LRU[string, *computeInstance] { 144 | computeAssetsCache, _ := freelru.New[string, *computeInstance](8192, hashStringXXHASH) 145 | return computeAssetsCache 146 | } 147 | 148 | func getSubnetCache() *freelru.LRU[string, *subnet] { 149 | computeAssetsCache, _ := freelru.New[string, *subnet](8192, hashStringXXHASH) 150 | return computeAssetsCache 151 | } 152 | 153 | func getVpcCache() *freelru.LRU[string, *vpc] { 154 | computeAssetsCache, _ := freelru.New[string, *vpc](8192, hashStringXXHASH) 155 | return computeAssetsCache 156 | } 157 | -------------------------------------------------------------------------------- /input/internal/publish.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package internal 19 | 20 | import ( 21 | "fmt" 22 | 23 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 24 | 25 | stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" 26 | "github.com/elastic/beats/v7/libbeat/beat" 27 | "github.com/elastic/elastic-agent-libs/mapstr" 28 | ) 29 | 30 | // Assets data is published to indexes following the same name pattern used in Agent 31 | // type-dataset-namespace, and has its own index type. 32 | const indexType = "assets" 33 | const indexDefaultNamespace = "default" 34 | const indexDefaultDataset = "raw" 35 | 36 | func GetDefaultIndexName() string { 37 | return fmt.Sprintf("%s-%s-%s", indexType, indexDefaultDataset, indexDefaultNamespace) 38 | } 39 | 40 | func NewEvent() *beat.Event { 41 | return &beat.Event{ 42 | Fields: mapstr.M{}, 43 | Meta: mapstr.M{ 44 | "index": GetDefaultIndexName(), 45 | }} 46 | } 47 | 48 | type AssetOption func(beat.Event) beat.Event 49 | 50 | // Publish emits a `beat.Event` to the specified publisher, with the provided parameters 51 | func Publish(publisher stateless.Publisher, baseEvent *beat.Event, opts ...AssetOption) { 52 | var event beat.Event 53 | if baseEvent == nil { 54 | event = *NewEvent() 55 | } else { 56 | event = *baseEvent 57 | } 58 | 59 | for _, o := range opts { 60 | event = o(event) 61 | } 62 | publisher.Publish(event) 63 | } 64 | 65 | func WithAssetCloudProvider(value string) AssetOption { 66 | return func(e beat.Event) beat.Event { 67 | e.Fields["cloud.provider"] = value 68 | return e 69 | } 70 | } 71 | 72 | func WithAssetName(value string) AssetOption { 73 | return func(e beat.Event) beat.Event { 74 | e.Fields["asset.name"] = value 75 | return e 76 | } 77 | } 78 | 79 | func WithAssetRegion(value string) AssetOption { 80 | return func(e beat.Event) beat.Event { 81 | e.Fields["cloud.region"] = value 82 | return e 83 | } 84 | } 85 | 86 | func WithAssetAccountID(value string) AssetOption { 87 | return func(e beat.Event) beat.Event { 88 | e.Fields["cloud.account.id"] = value 89 | return e 90 | } 91 | } 92 | 93 | func WithAssetKindAndID(k, id string) AssetOption { 94 | return func(e beat.Event) beat.Event { 95 | e.Fields["asset.kind"] = k 96 | e.Fields["asset.id"] = id 97 | e.Fields["asset.ean"] = fmt.Sprintf("%s:%s", k, id) 98 | return e 99 | } 100 | } 101 | func WithAssetType(value string) AssetOption { 102 | return func(e beat.Event) beat.Event { 103 | e.Fields["asset.type"] = value 104 | return e 105 | } 106 | } 107 | 108 | func WithAssetParents(value []string) AssetOption { 109 | return func(e beat.Event) beat.Event { 110 | e.Fields["asset.parents"] = value 111 | return e 112 | } 113 | } 114 | 115 | func WithAssetChildren(value []string) AssetOption { 116 | return func(e beat.Event) beat.Event { 117 | e.Fields["asset.children"] = value 118 | return e 119 | } 120 | } 121 | 122 | func WithAssetMetadata(value mapstr.M) AssetOption { 123 | return func(e beat.Event) beat.Event { 124 | flattenedValue := value.Flatten() 125 | for k, v := range flattenedValue { 126 | e.Fields["asset.metadata."+k] = v 127 | } 128 | return e 129 | } 130 | } 131 | 132 | func WithNodeData(name string, startTime *metav1.Time) AssetOption { 133 | return func(e beat.Event) beat.Event { 134 | e.Fields["kubernetes.node.name"] = name 135 | e.Fields["kubernetes.node.start_time"] = startTime 136 | return e 137 | } 138 | } 139 | 140 | func WithPodData(name, uid, namespace string, startTime *metav1.Time) AssetOption { 141 | return func(e beat.Event) beat.Event { 142 | e.Fields["kubernetes.pod.name"] = name 143 | e.Fields["kubernetes.pod.uid"] = uid 144 | e.Fields["kubernetes.pod.start_time"] = startTime 145 | e.Fields["kubernetes.namespace"] = namespace 146 | return e 147 | } 148 | } 149 | 150 | func WithContainerData(name, uid, namespace, state string, startTime *metav1.Time) AssetOption { 151 | return func(e beat.Event) beat.Event { 152 | e.Fields["kubernetes.container.name"] = name 153 | e.Fields["kubernetes.container.uid"] = uid 154 | e.Fields["kubernetes.container.start_time"] = startTime 155 | e.Fields["kubernetes.container.state"] = state 156 | e.Fields["kubernetes.namespace"] = namespace 157 | return e 158 | } 159 | } 160 | 161 | func ToMapstr(input map[string]string) mapstr.M { 162 | out := mapstr.M{} 163 | for k, v := range input { 164 | out[k] = v 165 | } 166 | return out 167 | } 168 | -------------------------------------------------------------------------------- /config/config_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | //go:build !integration 19 | 20 | package config 21 | 22 | import ( 23 | "github.com/stretchr/testify/assert" 24 | "path/filepath" 25 | "testing" 26 | 27 | conf "github.com/elastic/elastic-agent-libs/config" 28 | ) 29 | 30 | func TestGetConfigFiles_File(t *testing.T) { 31 | absPath, err := filepath.Abs("../tests/files/") 32 | 33 | assert.NotNil(t, absPath) 34 | assert.NoError(t, err) 35 | 36 | files, err := getConfigFiles(absPath + "/config.yml") 37 | 38 | assert.NoError(t, err) 39 | assert.Equal(t, 1, len(files)) 40 | 41 | assert.Equal(t, absPath+"/config.yml", files[0]) 42 | } 43 | 44 | func TestGetConfigFiles_Dir(t *testing.T) { 45 | absPath, err := filepath.Abs("../tests/files/") 46 | 47 | assert.NotNil(t, absPath) 48 | assert.NoError(t, err) 49 | 50 | files, err := getConfigFiles(absPath) 51 | 52 | assert.NoError(t, err) 53 | assert.Equal(t, 2, len(files)) 54 | 55 | assert.Equal(t, filepath.Join(absPath, "/config.yml"), files[0]) 56 | assert.Equal(t, filepath.Join(absPath, "/config2.yml"), files[1]) 57 | } 58 | 59 | func TestGetConfigFiles_EmptyDir(t *testing.T) { 60 | absPath, err := filepath.Abs("../tests") 61 | 62 | assert.NotNil(t, absPath) 63 | assert.NoError(t, err) 64 | 65 | files, err := getConfigFiles(absPath) 66 | 67 | assert.NoError(t, err) 68 | assert.Equal(t, 0, len(files)) 69 | } 70 | 71 | func TestGetConfigFiles_Invalid(t *testing.T) { 72 | absPath, err := filepath.Abs("../tests/files/") 73 | 74 | assert.NotNil(t, absPath) 75 | assert.NoError(t, err) 76 | 77 | // Invalid directory 78 | files, err := getConfigFiles(absPath + "/qwerwer") 79 | 80 | assert.Error(t, err) 81 | assert.Nil(t, files) 82 | } 83 | 84 | func TestMergeConfigFiles(t *testing.T) { 85 | absPath, err := filepath.Abs("../tests/files/") 86 | 87 | assert.NotNil(t, absPath) 88 | assert.NoError(t, err) 89 | 90 | files, err := getConfigFiles(absPath) 91 | assert.NoError(t, err) 92 | assert.Equal(t, 2, len(files)) 93 | 94 | config := &Config{} 95 | err = mergeConfigFiles(files, config) 96 | assert.NoError(t, err) 97 | 98 | assert.Equal(t, 2, len(config.Inputs)) 99 | } 100 | 101 | func TestEnabledInputs(t *testing.T) { 102 | stdinEnabled, err := conf.NewConfigFrom(map[string]interface{}{ 103 | "type": "stdin", 104 | "enabled": true, 105 | }) 106 | if !assert.NoError(t, err) { 107 | return 108 | } 109 | 110 | udpDisabled, err := conf.NewConfigFrom(map[string]interface{}{ 111 | "type": "udp", 112 | "enabled": false, 113 | }) 114 | if !assert.NoError(t, err) { 115 | return 116 | } 117 | 118 | logDisabled, err := conf.NewConfigFrom(map[string]interface{}{ 119 | "type": "log", 120 | "enabled": false, 121 | }) 122 | if !assert.NoError(t, err) { 123 | return 124 | } 125 | 126 | t.Run("ListEnabledInputs", func(t *testing.T) { 127 | tests := []struct { 128 | name string 129 | config *Config 130 | expected []string 131 | }{ 132 | { 133 | name: "all inputs disabled", 134 | config: &Config{Inputs: []*conf.C{udpDisabled, logDisabled}}, 135 | expected: []string{}, 136 | }, 137 | { 138 | name: "all inputs enabled", 139 | config: &Config{Inputs: []*conf.C{stdinEnabled}}, 140 | expected: []string{"stdin"}, 141 | }, 142 | { 143 | name: "disabled and enabled inputs", 144 | config: &Config{Inputs: []*conf.C{stdinEnabled, udpDisabled, logDisabled}}, 145 | expected: []string{"stdin"}, 146 | }, 147 | } 148 | 149 | for _, test := range tests { 150 | t.Run(test.name, func(t *testing.T) { 151 | assert.ElementsMatch(t, test.expected, test.config.ListEnabledInputs()) 152 | }) 153 | } 154 | }) 155 | 156 | t.Run("IsInputEnabled", func(t *testing.T) { 157 | config := &Config{Inputs: []*conf.C{stdinEnabled, udpDisabled, logDisabled}} 158 | 159 | tests := []struct { 160 | name string 161 | input string 162 | expected bool 163 | config *Config 164 | }{ 165 | {name: "input exists and enabled", input: "stdin", expected: true, config: config}, 166 | {name: "input exists and disabled", input: "udp", expected: false, config: config}, 167 | {name: "input doesn't exist", input: "redis", expected: false, config: config}, 168 | {name: "no inputs are enabled", input: "redis", expected: false, config: &Config{}}, 169 | } 170 | 171 | for _, test := range tests { 172 | t.Run(test.name, func(t *testing.T) { 173 | assert.Equal(t, test.expected, config.IsInputEnabled(test.input)) 174 | }) 175 | } 176 | }) 177 | } 178 | -------------------------------------------------------------------------------- /input/hostdata/hostdata.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package hostdata 19 | 20 | import ( 21 | "context" 22 | "fmt" 23 | "github.com/elastic/go-sysinfo" 24 | "time" 25 | 26 | "github.com/elastic/beats/v7/libbeat/processors/add_cloud_metadata" 27 | 28 | "github.com/elastic/beats/v7/libbeat/beat" 29 | "github.com/elastic/beats/v7/libbeat/processors/util" 30 | 31 | "github.com/elastic/assetbeat/input/internal" 32 | 33 | input "github.com/elastic/beats/v7/filebeat/input/v2" 34 | stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" 35 | "github.com/elastic/beats/v7/libbeat/feature" 36 | conf "github.com/elastic/elastic-agent-libs/config" 37 | "github.com/elastic/elastic-agent-libs/logp" 38 | "github.com/elastic/elastic-agent-libs/mapstr" 39 | "github.com/elastic/elastic-agent-system-metrics/metric/system/host" 40 | "github.com/elastic/go-concert/ctxtool" 41 | ) 42 | 43 | const defaultCollectionPeriod = time.Minute 44 | 45 | func Plugin() input.Plugin { 46 | return input.Plugin{ 47 | Name: "hostdata", 48 | Stability: feature.Stable, 49 | Deprecated: false, 50 | Info: "hostdata", 51 | Manager: stateless.NewInputManager(configure), 52 | } 53 | } 54 | 55 | type config struct { 56 | internal.BaseConfig `config:",inline"` 57 | } 58 | 59 | type hostdata struct { 60 | config config 61 | hostInfo mapstr.M 62 | addCloudMetadataProcessor beat.Processor 63 | } 64 | 65 | func defaultConfig() config { 66 | return config{ 67 | BaseConfig: internal.BaseConfig{ 68 | Period: defaultCollectionPeriod, 69 | }, 70 | } 71 | } 72 | 73 | func configure(inputCfg *conf.C) (stateless.Input, error) { 74 | cfg := defaultConfig() 75 | if err := inputCfg.Unpack(&cfg); err != nil { 76 | return nil, fmt.Errorf("error unpacking config: %w", err) 77 | } 78 | 79 | return newHostdata(cfg) 80 | } 81 | 82 | func newHostdata(cfg config) (*hostdata, error) { 83 | hostDataProvider, err := sysinfo.Host() 84 | if err != nil { 85 | return nil, fmt.Errorf("error getting host data: %w", err) 86 | } 87 | cloudMetadataProcessor, err := add_cloud_metadata.New(conf.NewConfig()) 88 | if err != nil { 89 | return nil, fmt.Errorf("error creating cloud metadata processor: %w", err) 90 | } 91 | return &hostdata{ 92 | config: cfg, 93 | hostInfo: host.MapHostInfo(hostDataProvider.Info()), 94 | addCloudMetadataProcessor: cloudMetadataProcessor, 95 | }, nil 96 | } 97 | 98 | func (h *hostdata) Name() string { return "hostdata" } 99 | 100 | func (h *hostdata) Test(_ input.TestContext) error { 101 | return nil 102 | } 103 | 104 | func (h *hostdata) Run(inputCtx input.Context, publisher stateless.Publisher) error { 105 | ctx := ctxtool.FromCanceller(inputCtx.Cancelation) 106 | logger := inputCtx.Logger 107 | 108 | logger.Info("hostdata asset collector run started") 109 | defer logger.Info("hostdata asset collector run stopped") 110 | 111 | ticker := time.NewTicker(h.config.Period) 112 | select { 113 | case <-ctx.Done(): 114 | return nil 115 | default: 116 | h.reportHostDataAssets(ctx, logger, publisher) 117 | } 118 | for { 119 | select { 120 | case <-ctx.Done(): 121 | return nil 122 | case <-ticker.C: 123 | h.reportHostDataAssets(ctx, logger, publisher) 124 | } 125 | } 126 | } 127 | 128 | func (h *hostdata) reportHostDataAssets(_ context.Context, logger *logp.Logger, publisher stateless.Publisher) { 129 | logger.Debug("collecting hostdata asset information") 130 | 131 | hostData := h.hostInfo.Clone() 132 | ipList, hwList, err := util.GetNetInfo() 133 | if err != nil { 134 | logger.Errorf("error when getting network information: %w", err) 135 | } 136 | 137 | if len(ipList) > 0 { 138 | _, _ = hostData.Put("host.ip", ipList) 139 | } 140 | if len(hwList) > 0 { 141 | _, _ = hostData.Put("host.mac", hwList) 142 | } 143 | 144 | event := internal.NewEvent() 145 | event.Fields = hostData 146 | // add cloud metadata 147 | event, err = h.addCloudMetadataProcessor.Run(event) 148 | if err != nil { 149 | logger.Error("error collecting cloud metadata: %w", err) 150 | return 151 | } 152 | 153 | cloudID, err := event.GetValue("cloud.instance.id") 154 | if err == nil { 155 | _, _ = event.PutValue("host.id", cloudID) 156 | } 157 | 158 | hostID, err := hostData.GetValue("host.id") 159 | if err != nil { 160 | logger.Error("no host ID in collected hostdata") 161 | return 162 | } 163 | assetKind := "host" 164 | assetType := "host" 165 | internal.Publish(publisher, event, 166 | internal.WithAssetKindAndID(assetKind, hostID.(string)), 167 | internal.WithAssetType(assetType), 168 | ) 169 | } 170 | -------------------------------------------------------------------------------- /input/gcp/util_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package gcp 19 | 20 | import ( 21 | "testing" 22 | 23 | "cloud.google.com/go/container/apiv1/containerpb" 24 | 25 | "github.com/stretchr/testify/assert" 26 | ) 27 | 28 | func TestGetResourceNameFromURL(t *testing.T) { 29 | for _, tt := range []struct { 30 | name string 31 | 32 | URL string 33 | expectedName string 34 | }{ 35 | { 36 | name: "with an empty value", 37 | 38 | URL: "", 39 | expectedName: "", 40 | }, 41 | { 42 | name: "with a valid URL", 43 | 44 | URL: "https://www.googleapis.com/compute/v1/projects/my_project/global/networks/my_network", 45 | expectedName: "my_network", 46 | }, 47 | } { 48 | t.Run(tt.name, func(t *testing.T) { 49 | assert.Equal(t, tt.expectedName, getResourceNameFromURL(tt.URL)) 50 | }) 51 | } 52 | } 53 | 54 | func TestGetRegionFromZoneURL(t *testing.T) { 55 | for _, tt := range []struct { 56 | name string 57 | 58 | URL string 59 | expectedName string 60 | }{ 61 | { 62 | name: "with an empty value", 63 | 64 | URL: "", 65 | expectedName: "", 66 | }, 67 | { 68 | name: "with a valid URL", 69 | 70 | URL: "https://www.googleapis.com/compute/v1/projects/my_project/zones/europe-west1-d", 71 | expectedName: "europe-west1", 72 | }, 73 | } { 74 | t.Run(tt.name, func(t *testing.T) { 75 | assert.Equal(t, tt.expectedName, getRegionFromZoneURL(tt.URL)) 76 | }) 77 | } 78 | } 79 | 80 | func TestGetVpcIdFromLink(t *testing.T) { 81 | vpcAssetsCache := getTestVpcCache() 82 | for _, tt := range []struct { 83 | name string 84 | 85 | selfLink string 86 | expectedId string 87 | }{ 88 | { 89 | name: "with a non existing selfLink", 90 | 91 | selfLink: "https://www.googleapis.com/compute/v1/projects/my_project/global/networks/test", 92 | expectedId: "", 93 | }, 94 | { 95 | name: "with an existing selfLink", 96 | 97 | selfLink: "https://www.googleapis.com/compute/v1/projects/my_project/global/networks/my_network", 98 | expectedId: "1", 99 | }, 100 | } { 101 | t.Run(tt.name, func(t *testing.T) { 102 | assert.Equal(t, tt.expectedId, getVpcIdFromLink(tt.selfLink, vpcAssetsCache)) 103 | }) 104 | } 105 | } 106 | 107 | func TestGetNetSelfLinkFromNetConfig(t *testing.T) { 108 | 109 | for _, tt := range []struct { 110 | name string 111 | 112 | networkConfig *containerpb.NetworkConfig 113 | expectedSelfLink string 114 | }{ 115 | { 116 | name: "with a non existing network", 117 | 118 | networkConfig: &containerpb.NetworkConfig{ 119 | Network: "", 120 | }, 121 | expectedSelfLink: "", 122 | }, 123 | { 124 | name: "with an existing network", 125 | 126 | networkConfig: &containerpb.NetworkConfig{ 127 | Network: "projects/my_project/global/networks/my_network", 128 | }, 129 | expectedSelfLink: "https://www.googleapis.com/compute/v1/projects/my_project/global/networks/my_network", 130 | }, 131 | } { 132 | t.Run(tt.name, func(t *testing.T) { 133 | assert.Equal(t, tt.expectedSelfLink, getNetSelfLinkFromNetConfig(tt.networkConfig)) 134 | }) 135 | } 136 | } 137 | 138 | func TestWantRegion(t *testing.T) { 139 | 140 | for _, tt := range []struct { 141 | name string 142 | 143 | confRegions []string 144 | region string 145 | expectedWanted bool 146 | }{ 147 | { 148 | name: "wanted", 149 | 150 | confRegions: []string{"us-east-1", "us-west-1"}, 151 | region: "us-east-1", 152 | expectedWanted: true, 153 | }, 154 | { 155 | name: "not wanted", 156 | 157 | confRegions: []string{"us-east-1", "us-west-1"}, 158 | region: "europe-east-1", 159 | expectedWanted: false, 160 | }, 161 | { 162 | name: "no regions in configurations", 163 | 164 | confRegions: []string{}, 165 | region: "us-east-1", 166 | expectedWanted: true, 167 | }, 168 | } { 169 | t.Run(tt.name, func(t *testing.T) { 170 | assert.Equal(t, tt.expectedWanted, wantRegion(tt.region, tt.confRegions)) 171 | }) 172 | } 173 | } 174 | func TestWantZone(t *testing.T) { 175 | 176 | for _, tt := range []struct { 177 | name string 178 | 179 | confRegions []string 180 | zone string 181 | expectedWanted bool 182 | }{ 183 | { 184 | name: "wanted", 185 | 186 | confRegions: []string{"us-east1", "us-west1"}, 187 | zone: "zone/us-east1-c", 188 | expectedWanted: true, 189 | }, 190 | { 191 | name: "not wanted", 192 | 193 | confRegions: []string{"us-east1", "us-west1"}, 194 | zone: "zone/europe-east1-b", 195 | expectedWanted: false, 196 | }, 197 | { 198 | name: "no regions in configurations", 199 | 200 | confRegions: []string{}, 201 | zone: "zone/us-east1-c", 202 | expectedWanted: true, 203 | }, 204 | } { 205 | t.Run(tt.name, func(t *testing.T) { 206 | assert.Equal(t, tt.expectedWanted, wantZone(tt.zone, tt.confRegions)) 207 | }) 208 | } 209 | } 210 | -------------------------------------------------------------------------------- /input/aws/ec2_test.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package aws 19 | 20 | import ( 21 | "context" 22 | "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" 23 | "github.com/elastic/assetbeat/input/internal" 24 | "testing" 25 | 26 | "github.com/aws/aws-sdk-go-v2/service/ec2" 27 | "github.com/aws/aws-sdk-go-v2/service/ec2/types" 28 | "github.com/stretchr/testify/assert" 29 | 30 | "github.com/elastic/assetbeat/input/testutil" 31 | "github.com/elastic/beats/v7/libbeat/beat" 32 | "github.com/elastic/elastic-agent-libs/logp" 33 | "github.com/elastic/elastic-agent-libs/mapstr" 34 | ) 35 | 36 | var instanceID_1 = "i-1111111" 37 | var instanceName_1 = "test-instance-1" 38 | var ownerID_1 = "11111111111111" 39 | var tag_1_k = "mykey" 40 | var tag_1_v = "myvalue" 41 | var subnetID1 = "mysubnetid1" 42 | var instanceID_2 = "i-2222222" 43 | var instanceName_2 = "test-instance-2" 44 | 45 | type mockDescribeInstancesAPI func(ctx context.Context, params *ec2.DescribeInstancesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) 46 | 47 | func (m mockDescribeInstancesAPI) DescribeInstances(ctx context.Context, params *ec2.DescribeInstancesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) { 48 | return m(ctx, params, optFns...) 49 | } 50 | 51 | func TestAssetsAWS_collectEC2Assets(t *testing.T) { 52 | for _, tt := range []struct { 53 | name string 54 | region string 55 | client func(t *testing.T) ec2.DescribeInstancesAPIClient 56 | expectedEvents []beat.Event 57 | }{{ 58 | name: "Test with multiple EC2 instances returned", 59 | region: "eu-west-1", 60 | client: func(t *testing.T) ec2.DescribeInstancesAPIClient { 61 | return mockDescribeInstancesAPI(func(ctx context.Context, params *ec2.DescribeInstancesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) { 62 | t.Helper() 63 | return &ec2.DescribeInstancesOutput{ 64 | NextToken: nil, 65 | Reservations: []types.Reservation{ 66 | { 67 | OwnerId: &ownerID_1, 68 | Instances: []types.Instance{ 69 | { 70 | InstanceId: &instanceID_1, 71 | State: &types.InstanceState{Name: "running"}, 72 | Tags: []types.Tag{ 73 | { 74 | Key: &tag_1_k, 75 | Value: &tag_1_v, 76 | }, 77 | { 78 | Key: to.Ptr("Name"), 79 | Value: &instanceName_1, 80 | }, 81 | }, 82 | SubnetId: &subnetID1, 83 | }, 84 | { 85 | InstanceId: &instanceID_2, 86 | State: &types.InstanceState{Name: "stopped"}, 87 | SubnetId: &subnetID1, 88 | Tags: []types.Tag{ 89 | { 90 | Key: to.Ptr("Name"), 91 | Value: &instanceName_2, 92 | }, 93 | }, 94 | }, 95 | }, 96 | }, 97 | }, 98 | }, nil 99 | }) 100 | }, 101 | expectedEvents: []beat.Event{ 102 | { 103 | Fields: mapstr.M{ 104 | "asset.ean": "host:" + instanceID_1, 105 | "asset.id": instanceID_1, 106 | "asset.metadata.state": "running", 107 | "asset.type": "aws.ec2.instance", 108 | "asset.kind": "host", 109 | "asset.name": instanceName_1, 110 | "asset.parents": []string{ 111 | "network:" + subnetID1, 112 | }, 113 | "asset.metadata.tags." + tag_1_k: tag_1_v, 114 | "cloud.account.id": "11111111111111", 115 | "cloud.provider": "aws", 116 | "cloud.region": "eu-west-1", 117 | }, 118 | Meta: mapstr.M{ 119 | "index": internal.GetDefaultIndexName(), 120 | }, 121 | }, 122 | { 123 | Fields: mapstr.M{ 124 | "asset.ean": "host:" + instanceID_2, 125 | "asset.id": instanceID_2, 126 | "asset.metadata.state": "stopped", 127 | "asset.type": "aws.ec2.instance", 128 | "asset.kind": "host", 129 | "asset.name": instanceName_2, 130 | "asset.parents": []string{ 131 | "network:" + subnetID1, 132 | }, 133 | "cloud.account.id": "11111111111111", 134 | "cloud.provider": "aws", 135 | "cloud.region": "eu-west-1", 136 | }, 137 | Meta: mapstr.M{ 138 | "index": internal.GetDefaultIndexName(), 139 | }, 140 | }, 141 | }, 142 | }, 143 | } { 144 | t.Run(tt.name, func(t *testing.T) { 145 | publisher := testutil.NewInMemoryPublisher() 146 | 147 | ctx := context.Background() 148 | logger := logp.NewLogger("test") 149 | 150 | err := collectEC2Assets(ctx, tt.client(t), tt.region, logger, publisher) 151 | assert.NoError(t, err) 152 | assert.Equal(t, tt.expectedEvents, publisher.Events) 153 | }) 154 | 155 | } 156 | } 157 | 158 | func TestAssetsAWS_flattenEC2Tags(t *testing.T) { 159 | tag1, tag2, a, b := "tag1", "tag2", "a", "b" 160 | tags := []types.Tag{{Key: &tag1, Value: &a}, {Key: &tag2, Value: &b}} 161 | flat := flattenEC2Tags(tags) 162 | expected := mapstr.M{"tag1": "a", "tag2": "b"} 163 | assert.Equal(t, expected, flat) 164 | } 165 | -------------------------------------------------------------------------------- /input/k8s/util.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package k8s 19 | 20 | import ( 21 | "context" 22 | "encoding/json" 23 | "fmt" 24 | "io" 25 | "net" 26 | "net/http" 27 | "strings" 28 | "time" 29 | 30 | "github.com/pkg/errors" 31 | 32 | "github.com/elastic/elastic-agent-autodiscover/kubernetes" 33 | "github.com/elastic/elastic-agent-libs/logp" 34 | ) 35 | 36 | type httpResponse interface { 37 | FetchResponse(context.Context, string, map[string]string) ([]byte, error) 38 | } 39 | 40 | // httpFetcher struct to implement httpResponse interface 41 | type httpFetcher struct { 42 | httpClient http.Client 43 | } 44 | 45 | // newhttpFetcher returns a new httpFetcher 46 | func newhttpFetcher() httpFetcher { 47 | client := http.Client{ 48 | Timeout: time.Second * 10, 49 | Transport: &http.Transport{ 50 | DisableKeepAlives: true, 51 | DialContext: (&net.Dialer{ 52 | Timeout: time.Second * 10, 53 | KeepAlive: 0, 54 | }).DialContext, 55 | }, 56 | } 57 | return httpFetcher{httpClient: client} 58 | } 59 | 60 | // getInstanceId returns the cloud instance id in case 61 | // the node runs in one of [aws, gcp] csp. 62 | // In case of aws the instance id is retrieved from providerId 63 | // which is in the form of aws:///region/instanceId for not fargate nodes. 64 | // In case of gcp it is retrieved by the annotation container.googleapis.com/instance_id 65 | // In all other cases empty string is returned 66 | func getInstanceId(node *kubernetes.Node) string { 67 | providerId := node.Spec.ProviderID 68 | 69 | switch csp := getCspFromProviderId(providerId); csp { 70 | case "aws": 71 | slice := strings.Split(providerId, "/") 72 | // in case of fargate the slice length will be 6 73 | if len(slice) == 5 { 74 | return slice[4] 75 | } 76 | case "gcp": 77 | annotations := node.GetAnnotations() 78 | return annotations["container.googleapis.com/instance_id"] 79 | case "azure": 80 | return node.Status.NodeInfo.SystemUUID 81 | default: 82 | return "" 83 | } 84 | return "" 85 | } 86 | 87 | // getNodeState returns the state of kubernets node 88 | // the state can be Ready, MemoryPressure, DiskPressure, PIDPressure, 89 | // NetworkUnavailable or Unknown 90 | func getNodeState(node *kubernetes.Node) string { 91 | conditions := node.Status.Conditions 92 | status := "Unknown" 93 | for _, condition := range conditions { 94 | if condition.Status == "True" { 95 | status = string(condition.Type) 96 | } 97 | } 98 | return status 99 | } 100 | 101 | // getCspFromProviderId return the cps for a given providerId string. 102 | // In case of aws providerId is in the form of aws:///region/instanceId 103 | // In case of gcp providerId is in the form of gce://project/region/nodeName 104 | // In case of azure providerId is in the form of azure:///subscriptions/subscriptionId 105 | func getCspFromProviderId(providerId string) string { 106 | if strings.HasPrefix(providerId, "aws") { 107 | return "aws" 108 | } 109 | if strings.HasPrefix(providerId, "gce") { 110 | return "gcp" 111 | } 112 | if strings.HasPrefix(providerId, "azure") { 113 | return "azure" 114 | } 115 | return "" 116 | } 117 | 118 | // getGKEClusterUid gets the GKE cluster uid from metadata endpoint 119 | func getGKEClusterUid(ctx context.Context, log *logp.Logger, hF httpResponse) (string, error) { 120 | url := fmt.Sprintf("http://%s%s", metadataHost, gceMetadataURI) 121 | gceHeaders := map[string]string{"Metadata-Flavor": "Google"} 122 | 123 | response, err := hF.FetchResponse(ctx, url, gceHeaders) 124 | if err != nil { 125 | return "", err 126 | } 127 | var gcpMetadataRes = map[string]interface{}{} 128 | if err = json.Unmarshal(response, &gcpMetadataRes); err != nil { 129 | return "", err 130 | } 131 | if instance, ok := gcpMetadataRes["instance"].(map[string]interface{}); ok { 132 | if attributes, ok := instance["attributes"].(map[string]interface{}); ok { 133 | if clusterUid, ok := attributes["cluster-uid"].(string); ok { 134 | log.Debugf("Cloud uid is %s ", clusterUid) 135 | return clusterUid, nil 136 | } 137 | } 138 | } 139 | return "", errors.Errorf("Cluster uid not found in metadata") 140 | } 141 | 142 | // FetchResponse returns http response of a provided URL 143 | func (c httpFetcher) FetchResponse(ctx context.Context, url string, headers map[string]string) ([]byte, error) { 144 | var response []byte 145 | req, err := http.NewRequest("GET", url, nil) 146 | if err != nil { 147 | return response, errors.Wrapf(err, "failed to create http request for gcp") 148 | } 149 | for k, v := range headers { 150 | req.Header.Add(k, v) 151 | } 152 | req = req.WithContext(ctx) 153 | 154 | rsp, err := c.httpClient.Do(req) 155 | if err != nil { 156 | return response, errors.Wrapf(err, "failed requesting gcp metadata") 157 | } 158 | defer rsp.Body.Close() 159 | 160 | if rsp.StatusCode != http.StatusOK { 161 | return response, errors.Errorf("failed with http status code %v", rsp.StatusCode) 162 | } 163 | 164 | response, err = io.ReadAll(rsp.Body) 165 | if err != nil { 166 | return response, errors.Wrapf(err, "failed requesting gcp metadata") 167 | } 168 | return response, nil 169 | } 170 | -------------------------------------------------------------------------------- /input/k8s/nodes.go: -------------------------------------------------------------------------------- 1 | // Licensed to Elasticsearch B.V. under one or more contributor 2 | // license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright 4 | // ownership. Elasticsearch B.V. licenses this file to you under 5 | // the Apache License, Version 2.0 (the "License"); you may 6 | // not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | package k8s 19 | 20 | import ( 21 | "context" 22 | "fmt" 23 | "time" 24 | 25 | "github.com/elastic/assetbeat/input/internal" 26 | stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" 27 | kube "github.com/elastic/elastic-agent-autodiscover/kubernetes" 28 | "github.com/elastic/elastic-agent-libs/mapstr" 29 | 30 | "github.com/elastic/elastic-agent-libs/logp" 31 | 32 | kuberntescli "k8s.io/client-go/kubernetes" 33 | ) 34 | 35 | type node struct { 36 | watcher kube.Watcher 37 | client kuberntescli.Interface 38 | logger *logp.Logger 39 | ctx context.Context 40 | } 41 | 42 | const ( 43 | // metadataHost is the IP that each of the GCP uses for metadata service. 44 | metadataHost = "169.254.169.254" 45 | gceMetadataURI = "/computeMetadata/v1/?recursive=true&alt=json" 46 | ) 47 | 48 | // getNodeWatcher initiates and returns a watcher of kubernetes nodes 49 | func getNodeWatcher(ctx context.Context, log *logp.Logger, client kuberntescli.Interface, timeout time.Duration) (kube.Watcher, error) { 50 | watcher, err := kube.NewNamedWatcher("node", client, &kube.Node{}, kube.WatchOptions{ 51 | SyncTimeout: timeout, 52 | Node: "", 53 | Namespace: "", 54 | HonorReSyncs: true, 55 | }, nil) 56 | 57 | if err != nil { 58 | log.Errorf("could not create kubernetes watcher %v", err) 59 | return nil, err 60 | } 61 | 62 | n := &node{ 63 | watcher: watcher, 64 | client: client, 65 | logger: log, 66 | ctx: ctx, 67 | } 68 | 69 | watcher.AddEventHandler(n) 70 | 71 | return watcher, nil 72 | } 73 | 74 | // Start starts the eventer 75 | func (n *node) Start() error { 76 | return n.watcher.Start() 77 | } 78 | 79 | // Stop stops the eventer 80 | func (n *node) Stop() { 81 | n.watcher.Stop() 82 | } 83 | 84 | // OnUpdate handles events for pods that have been updated. 85 | func (n *node) OnUpdate(obj interface{}) { 86 | o := obj.(*kube.Node) 87 | n.logger.Debugf("Watcher Node update: %+v", o.Name) 88 | } 89 | 90 | // OnDelete stops pod objects that are deleted. 91 | func (n *node) OnDelete(obj interface{}) { 92 | o := obj.(*kube.Node) 93 | n.logger.Debugf("Watcher Node delete: %+v", o.Name) 94 | } 95 | 96 | // OnAdd ensures processing of node objects that are newly added. 97 | func (n *node) OnAdd(obj interface{}) { 98 | o := obj.(*kube.Node) 99 | n.logger.Debugf("Watcher Node add: %+v", o.Name) 100 | } 101 | 102 | // getNodeIdFromName returns kubernetes node id from a provided node name 103 | func getNodeIdFromName(nodeName string, nodeWatcher kube.Watcher) (string, error) { 104 | node, exists, err := nodeWatcher.Store().GetByKey(nodeName) 105 | if err != nil { 106 | return "", err 107 | } 108 | if !exists { 109 | return "", fmt.Errorf("node with name %s does not exist in cache", nodeName) 110 | } 111 | n := node.(*kube.Node) 112 | return string(n.ObjectMeta.UID), nil 113 | } 114 | 115 | // publishK8sNodes publishes the node assets stored in node watcher cache 116 | func publishK8sNodes(ctx context.Context, log *logp.Logger, publisher stateless.Publisher, watcher kube.Watcher, isInCluster bool) { 117 | log.Info("Publishing nodes assets\n") 118 | assetType := "k8s.node" 119 | assetKind := "host" 120 | var assetParents []string 121 | 122 | // Get the first stored node to extract the cluster uid in case of GCP. 123 | // Only in case of running InCluster 124 | log.Debugf("Is in cluster is %s", isInCluster) 125 | if isInCluster { 126 | if len(watcher.Store().List()) > 0 { 127 | if n1, ok := watcher.Store().List()[0].(*kube.Node); ok { 128 | if getCspFromProviderId(n1.Spec.ProviderID) == "gcp" { 129 | clusterUid, err := getGKEClusterUid(ctx, log, newhttpFetcher()) 130 | if err != nil { 131 | log.Debugf("Unable to fetch cluster uid from metadata: %+v \n", err) 132 | } 133 | assetParents = append(assetParents, fmt.Sprintf("%s:%s", "cluster", clusterUid)) 134 | } 135 | } 136 | } 137 | } 138 | 139 | for _, obj := range watcher.Store().List() { 140 | o, ok := obj.(*kube.Node) 141 | if ok { 142 | log.Debugf("Publish Node: %+v", o.Name) 143 | metadata := mapstr.M{ 144 | "state": getNodeState(o), 145 | } 146 | log.Debug("Node status: ", metadata["state"]) 147 | instanceId := getInstanceId(o) 148 | log.Debug("Node instance id: ", instanceId) 149 | assetId := string(o.ObjectMeta.UID) 150 | assetStartTime := o.ObjectMeta.CreationTimestamp 151 | options := []internal.AssetOption{ 152 | internal.WithAssetKindAndID(assetKind, assetId), 153 | internal.WithAssetType(assetType), 154 | internal.WithAssetName(o.Name), 155 | internal.WithAssetMetadata(metadata), 156 | internal.WithNodeData(o.Name, &assetStartTime), 157 | } 158 | if instanceId != "" { 159 | options = append(options, internal.WithCloudInstanceId(instanceId)) 160 | } 161 | if assetParents != nil { 162 | options = append(options, internal.WithAssetParents(assetParents)) 163 | } 164 | internal.Publish(publisher, nil, options...) 165 | 166 | } else { 167 | log.Error("Publishing nodes assets failed. Type assertion of node object failed") 168 | } 169 | } 170 | } 171 | --------------------------------------------------------------------------------