├── .github ├── CODEOWNERS ├── dependabot.yml └── workflows │ ├── ci-docs.yml │ └── ci.yml ├── .gitignore ├── .go-version ├── CONTRIBUTING.md ├── LICENSE.md ├── Makefile ├── README.md ├── assets └── templates │ ├── aws.billing │ └── schema-b │ │ ├── configs.yml │ │ ├── fields.yml │ │ └── gotext.tpl │ ├── aws.ec2_logs │ └── schema-b │ │ ├── configs.yml │ │ ├── fields.yml │ │ └── gotext.tpl │ ├── aws.ec2_metrics │ └── schema-b │ │ ├── configs.yml │ │ ├── fields.yml │ │ └── gotext.tpl │ ├── aws.sqs │ └── schema-b │ │ ├── configs.yml │ │ ├── fields.yml │ │ ├── gotext.tpl │ │ └── placeholder.tpl │ ├── aws.vpcflow │ └── schema-a │ │ ├── configs.yml │ │ ├── fields.yml │ │ ├── gotext.tpl │ │ └── placeholder.tpl │ ├── kubernetes.container │ └── schema-b │ │ ├── configs.yml │ │ ├── fields.yml │ │ ├── gotext.tpl │ │ └── gotext_multiline.tpl │ └── kubernetes.pod │ └── schema-b │ ├── configs.yml │ ├── fields.yml │ ├── gotext.tpl │ └── gotext_multiline.tpl ├── cmd ├── generate.go ├── generate_common.go ├── generate_with_template.go ├── local-template.go ├── root.go ├── root_test.go ├── version.go └── version_test.go ├── docs ├── cardinality.md ├── cli-help.md ├── data-schemas.md ├── dimensionality.md ├── fields-configuration.md ├── glossary.md ├── go-text-template-helpers.md ├── performances.md ├── usage.md └── writing-templates.md ├── go.mod ├── go.sum ├── internal ├── corpus │ ├── generator.go │ └── generator_test.go ├── settings │ ├── settings.go │ ├── xdg.go │ └── xdg_test.go └── version │ ├── version.go │ └── version_test.go ├── main.go └── pkg └── genlib ├── config ├── config.go └── config_test.go ├── fields ├── cache.go ├── fields.go ├── load.go ├── version.go └── yaml.go ├── generator.go ├── generator_interface.go ├── generator_test.go ├── generator_with_custom_template.go ├── generator_with_custom_template_test.go ├── generator_with_text_template.go └── generator_with_text_template_test.go /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @elastic/obs-infraobs-integrations 2 | 3 | /assets/templates/ec2* @elastic/obs-ds-hosted-services 4 | /assets/templates/kubernetes* @elastic/obs-cloudnative-monitoring -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "gomod" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "daily" 12 | 13 | - package-ecosystem: "github-actions" 14 | directories: 15 | - "/" 16 | - "/.github/actions/*" 17 | schedule: 18 | interval: "weekly" 19 | day: "sunday" 20 | time: "22:00" 21 | groups: 22 | github-actions: 23 | patterns: 24 | - "*" 25 | -------------------------------------------------------------------------------- /.github/workflows/ci-docs.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # This workflow sets the ci / test status check to success in case it's a docs only PR and ci.yml is not triggered 3 | # https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks 4 | name: ci # The name must be the same as in ci.yml 5 | 6 | on: 7 | pull_request: 8 | paths-ignore: # This expression needs to match the paths ignored on ci.yml. 9 | - '**' 10 | - '!**/*.md' 11 | - '!**/*.asciidoc' 12 | 13 | permissions: 14 | contents: read 15 | 16 | jobs: 17 | test: 18 | runs-on: ubuntu-latest 19 | steps: 20 | - run: 'echo "No build required"' 21 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: ci 3 | 4 | on: 5 | workflow_dispatch: 6 | push: 7 | branches: 8 | - main 9 | paths-ignore: 10 | - '**.md' 11 | - '**.asciidoc' 12 | pull_request: 13 | paths-ignore: 14 | - '**.md' 15 | - '**.asciidoc' 16 | 17 | permissions: 18 | contents: read 19 | 20 | ## Concurrency only allowed in the main branch. 21 | ## So old builds running for old commits within the same Pull Request are cancelled 22 | concurrency: 23 | group: ${{ github.workflow }}-${{ github.ref }} 24 | cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} 25 | 26 | jobs: 27 | test: 28 | runs-on: ubuntu-latest 29 | steps: 30 | - uses: actions/checkout@v4 31 | 32 | - uses: actions/setup-go@v5 33 | with: 34 | go-version-file: .go-version 35 | cache: true 36 | cache-dependency-path: '**/go.sum' 37 | 38 | - name: Lint 39 | run: |- 40 | go mod tidy && git diff --exit-code 41 | gofmt -l . | read && echo "Code differs from gofmt's style. Run 'gofmt -w .'" 1>&2 && exit 1 || true 42 | go vet 43 | 44 | - name: Build 45 | run: go build 46 | 47 | - name: Test 48 | run: go test -v ./... 49 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | elastic-integration-corpus-generator-tool 8 | 9 | # Test binary, built with `go test -c` 10 | *.test 11 | 12 | # Output of the go coverage tool, specifically when used with LiteIDE 13 | *.out 14 | 15 | # Dependency directories (remove the comment below to include it) 16 | # vendor/ 17 | 18 | # OSX 19 | .DS_Store 20 | -------------------------------------------------------------------------------- /.go-version: -------------------------------------------------------------------------------- 1 | 1.19.1 2 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Requirements 2 | 3 | `make` CLI should be installed and available. 4 | 5 | `git` CLI should be installed and available. 6 | 7 | # Building the CLI 8 | 9 | `$ make build` 10 | 11 | # Running tests 12 | 13 | `$ make test` 14 | 15 | # Adding License header 16 | 17 | All files in this repository that not resides in the `assets` folder should have the License header. To do this, use: 18 | 19 | `$ make licenser` 20 | 21 | # This CLI and genlib 22 | 23 | TODO 24 | 25 | # Release process 26 | 27 | See [#77](https://github.com/elastic/elastic-integration-corpus-generator-tool/issues/77). 28 | 29 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Elastic License 2.0 2 | 3 | URL: https://www.elastic.co/licensing/elastic-license 4 | 5 | ## Acceptance 6 | 7 | By using the software, you agree to all of the terms and conditions below. 8 | 9 | ## Copyright License 10 | 11 | The licensor grants you a non-exclusive, royalty-free, worldwide, 12 | non-sublicensable, non-transferable license to use, copy, distribute, make 13 | available, and prepare derivative works of the software, in each case subject to 14 | the limitations and conditions below. 15 | 16 | ## Limitations 17 | 18 | You may not provide the software to third parties as a hosted or managed 19 | service, where the service provides users with access to any substantial set of 20 | the features or functionality of the software. 21 | 22 | You may not move, change, disable, or circumvent the license key functionality 23 | in the software, and you may not remove or obscure any functionality in the 24 | software that is protected by the license key. 25 | 26 | You may not alter, remove, or obscure any licensing, copyright, or other notices 27 | of the licensor in the software. Any use of the licensor’s trademarks is subject 28 | to applicable law. 29 | 30 | ## Patents 31 | 32 | The licensor grants you a license, under any patent claims the licensor can 33 | license, or becomes able to license, to make, have made, use, sell, offer for 34 | sale, import and have imported the software, in each case subject to the 35 | limitations and conditions in this license. This license does not cover any 36 | patent claims that you cause to be infringed by modifications or additions to 37 | the software. If you or your company make any written claim that the software 38 | infringes or contributes to infringement of any patent, your patent license for 39 | the software granted under these terms ends immediately. If your company makes 40 | such a claim, your patent license ends immediately for work on behalf of your 41 | company. 42 | 43 | ## Notices 44 | 45 | You must ensure that anyone who gets a copy of any part of the software from you 46 | also gets a copy of these terms. 47 | 48 | If you modify the software, you must include in any modified copies of the 49 | software prominent notices stating that you have modified the software. 50 | 51 | ## No Other Rights 52 | 53 | These terms do not imply any licenses other than those expressly granted in 54 | these terms. 55 | 56 | ## Termination 57 | 58 | If you use the software in violation of these terms, such use is not licensed, 59 | and your licenses will automatically terminate. If the licensor provides you 60 | with a notice of your violation, and you cease all violation of this license no 61 | later than 30 days after you receive that notice, your licenses will be 62 | reinstated retroactively. However, if you violate these terms after such 63 | reinstatement, any additional violation of these terms will cause your licenses 64 | to terminate automatically and permanently. 65 | 66 | ## No Liability 67 | 68 | *As far as the law allows, the software comes as is, without any warranty or 69 | condition, and the licensor will not be liable to you for any damages arising 70 | out of these terms or the use or nature of the software, under any kind of 71 | legal claim.* 72 | 73 | ## Definitions 74 | 75 | The **licensor** is the entity offering these terms, and the **software** is the 76 | software the licensor makes available under these terms, including any portion 77 | of it. 78 | 79 | **you** refers to the individual or entity agreeing to these terms. 80 | 81 | **your company** is any legal entity, sole proprietorship, or other kind of 82 | organization that you work for, plus all organizations that have control over, 83 | are under the control of, or are under common control with that 84 | organization. **control** means ownership of substantially all the assets of an 85 | entity, or the power to direct its management and policies by vote, contract, or 86 | otherwise. Control can be direct or indirect. 87 | 88 | **your licenses** are all the licenses granted to you for the software under 89 | these terms. 90 | 91 | **use** means anything you do with the software requiring one of your licenses. 92 | 93 | **trademark** means trademarks, service marks, and similar rights. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | MODULE = github.com/elastic/elastic-integration-corpus-generator-tool 2 | VERSION_IMPORT_PATH = $(MODULE)/internal/version 3 | VERSION_COMMIT_HASH = `git describe --always --long --dirty` 4 | SOURCE_DATE_EPOCH = `git log -1 --pretty=%ct` # https://reproducible-builds.org/docs/source-date-epoch/ 5 | VERSION_TAG = `(git describe --exact-match --tags 2>/dev/null || echo '') | tr -d '\n'` 6 | VERSION_LDFLAGS = -X $(VERSION_IMPORT_PATH).CommitHash=$(VERSION_COMMIT_HASH) -X $(VERSION_IMPORT_PATH).SourceDateEpoch=$(SOURCE_DATE_EPOCH) -X $(VERSION_IMPORT_PATH).Tag=$(VERSION_TAG) 7 | 8 | .PHONY: build 9 | 10 | build: 11 | go build -ldflags "$(VERSION_LDFLAGS)" -o elastic-integration-corpus-generator-tool 12 | 13 | licenser: 14 | go run github.com/elastic/go-licenser -license Elasticv2 15 | 16 | test: 17 | go test -v ./... -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # elastic-integration-corpus-generator-tool 2 | Command line tool used for generating events corpus dynamically given a specific integration. 3 | 4 | Want to contribute? See [CONTRIBUTING.md](./CONTRIBUTING.md). 5 | 6 | Want to generate data? TODO. 7 | 8 | Want to create a template? See [writing templates](./docs/writing-templates.md). 9 | 10 | Do you care about performances? See [performances](./docs/performances.md). 11 | 12 | Want help on the CLI? See [CLI help](./docs/cli-help.md). 13 | 14 | Want to explore use cases? See [usage](./docs/usage.md). 15 | 16 | # Maintainers 17 | 18 | [Observability Integrations Team](https://github.com/orgs/elastic/teams/obs-infraobs-integrations) 19 | 20 | # License 21 | 22 | Elastic License 2.0 23 | 24 | -------------------------------------------------------------------------------- /assets/templates/aws.billing/schema-b/configs.yml: -------------------------------------------------------------------------------- 1 | fields: 2 | - name: cloud.region 3 | enum: ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ap-south-1", "ap-northeast-3", "ap-northeast-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ca-central-1", "eu-central-1", "eu-west-1", "eu-west-2", "eu-west-3", "eu-north-1", "sa-east-1", "af-south-1", "ap-east-1", "ap-south-2", "ap-southeast-3", "eu-south-2", "eu-central-2", "me-south-1", "me-central-1"] 4 | cardinality: 25 5 | - name: cloud.account.id 6 | value: "123456789" 7 | - name: cloud.account.name 8 | value: sample-account 9 | - name: aws.billing.currency 10 | value: "USD" 11 | - name: aws.billing.ServiceName 12 | # NOTE: When empty the data refers to estimated charged for the entire account. We cannot reproduce the content (as it's a sum of previous data) but we want to provide the case. 13 | enum: ["", "AWSCloudTrail", "AWSCodeArtifact", "AWSConfig", "AWSCostExplorer", "AWSDataTransfer", "AWSELB", "AWSLambda", "AWSMarketplace", "AWSQueueService", "AWSSecretsManager", "AWSServiceCatalog", "AWSSystemsManager", "AWSXRay", "AmazonApiGateway", "AmazonCloudWatch", "AmazonCognito", "AmazonDynamoDB", "AmazonEC2", "AmazonECR", "AmazonEKS", "AmazonKinesis", "AmazonKinesisFirehose", "AmazonRDS", "AmazonRedshift", "AmazonRoute53", "AmazonS3", "AmazonSNS", "AmazonVPC", "awskms"] 14 | - name: agent.id 15 | value: "12f376ef-5186-4e8b-a175-70f1140a8f30" 16 | - name: agent.ephemeral_id 17 | value: "5fd278ce-2a12-4a09-a125-0c5b39aa69e3" 18 | - name: agent.name 19 | value: "host.local" 20 | - name: metricset.period 21 | value: 86400 22 | - name: aws.billing.group_definition.key 23 | # NOTE: repeated values are needed to produce 10% cases with "" value 24 | enum: ["", "AZ", "INSTANCE_TYPE", "SERVICE", "LINKED_ACCOUNT", "AZ", "INSTANCE_TYPE", "SERVICE", "LINKED_ACCOUNT"] 25 | 26 | - name: event.duration 27 | range: 28 | min: 1 29 | max: 1000 30 | - name: aws.billing.EstimatedCharges 31 | cardinality: 25 32 | fuzziness: 0.2 33 | - name: aws.billing.AmortizedCost.amount 34 | cardinality: 25 35 | fuzziness: 0.2 36 | - name: aws.billing.BlendedCost.amount 37 | cardinality: 25 38 | fuzziness: 0.2 39 | - name: aws.billing.NormalizedUsageAmount.amount 40 | cardinality: 25 41 | fuzziness: 0.2 42 | - name: aws.billing.UnblendedCost.amount 43 | cardinality: 25 44 | fuzziness: 0.2 45 | - name: aws.billing.UsageQuantity.amount 46 | cardinality: 25 47 | fuzziness: 0.2 48 | - name: aws.billing.group_definition.type 49 | value: "DIMENSION" 50 | - name: aws.billing.group_by.INSTANCE_TYPE 51 | enum: ["NoInstanceType", "a1.large", "c5.2xlarge", "c5.xlarge", "c6i.2xlarge", "db.r6g.2xlarge", "db.t2.micro", "dc2.large", "m5.large", "t1.micro", "t2.medium", "t2.micro", "t2.small", "t2.xlarge", "t3.2xlarge", "t3.medium", "t3.xlarge","t3.xlarge"] 52 | - name: aws.billing.group_by.SERVICE 53 | enum: ["Amazon Simple Storage Service", "Amazon Elastic Compute Cloud - Compute", "EC2 - Other", "Amazon Kinesis", "Amazon Relational Database Service", "Amazon Elastic Load Balancing", "AmazonCloudWatch", "AWS CloudTrail", "AWS Config", "AWS Key Management Service", "AWS Lambda", "AWS Secrets Manager", "AWS Service Catalog", "Amazon API Gateway", "Amazon DynamoDB", "Amazon EC2 Container Registry (ECR)", "Amazon Elastic Container Service for Kubernetes", "Amazon Kinesis Firehose", "Amazon Redshift", "Amazon Simple Notification Service", "Amazon Simple Queue Service", "Amazon Virtual Private Cloud"] 54 | 55 | -------------------------------------------------------------------------------- /assets/templates/aws.billing/schema-b/fields.yml: -------------------------------------------------------------------------------- 1 | - name: timestamp 2 | type: date 3 | - name: cloud.region 4 | type: keyword 5 | - name: cloud.account.id 6 | type: keyword 7 | - name: cloud.account.name 8 | type: keyword 9 | - name: event.duration 10 | type: long 11 | - name: metricset.period 12 | type: long 13 | - name: aws.billing.currency 14 | type: keyword 15 | - name: aws.billing.EstimatedCharges 16 | type: float 17 | # positive 18 | - name: aws.billing.ServiceName 19 | type: keyword 20 | - name: aws.billing.AmortizedCost.amount 21 | type: float 22 | # positive 23 | - name: aws.billing.BlendedCost.amount 24 | type: float 25 | # positive 26 | - name: aws.billing.NormalizedUsageAmount.amount 27 | type: integer 28 | # positive 29 | - name: aws.billing.UnblendedCost.amount 30 | type: float 31 | # positive 32 | - name: aws.billing.UsageQuantity.amount 33 | type: integer 34 | # positive 35 | - name: agent.id 36 | type: keyword 37 | - name: agent.name 38 | type: keyword 39 | - name: agent.ephemeral_id 40 | type: keyword 41 | example: 12f376ef-5186-4e8b-a175-70f1140a8f30 42 | - name: aws.billing.group_definition.key 43 | type: keyword 44 | - name: aws.billing.start_date 45 | type: date 46 | - name: aws.billing.group_definition.type 47 | type: keyword 48 | - name: aws.billing.group_by.INSTANCE_TYPE 49 | type: keyword 50 | - name: aws.billing.group_by.SERVICE 51 | type: keyword 52 | -------------------------------------------------------------------------------- /assets/templates/aws.billing/schema-b/gotext.tpl: -------------------------------------------------------------------------------- 1 | {{- $currency := generate "aws.billing.currency" }} 2 | {{- $groupBy := generate "aws.billing.group_definition.key" }} 3 | {{- $period := generate "metricset.period" }} 4 | {{- $cloudId := generate "cloud.account.id" }} 5 | {{- $cloudRegion := generate "cloud.region" }} 6 | {{- $timestamp := generate "timestamp" }} 7 | { 8 | "@timestamp": "{{$timestamp.Format "2006-01-02T15:04:05.999999Z07:00"}}", 9 | "cloud": { 10 | "provider": "aws", 11 | "region": "{{$cloudRegion}}", 12 | "account": { 13 | "id": "{{$cloudId}}", 14 | "name": "{{generate "cloud.account.name"}}" 15 | } 16 | }, 17 | "event": { 18 | "dataset": "aws.billing", 19 | "module": "aws", 20 | "duration": {{generate "event.duration"}} 21 | }, 22 | "metricset": { 23 | "name": "billing", 24 | "period": {{$period}} 25 | }, 26 | "ecs": { 27 | "version": "8.2.0" 28 | }, 29 | "aws": { 30 | "billing": { 31 | {{- if eq $groupBy "" }} 32 | "Currency": "{{$currency}}", 33 | "EstimatedCharges": {{generate "aws.billing.EstimatedCharges"}}, 34 | "ServiceName": "{{generate "aws.billing.ServiceName"}}" 35 | {{- else }} 36 | {{- $sd := generate "aws.billing.start_date" }} 37 | "start_date": "{{ $sd.Format "2006-01-02T15:04:05.999999Z07:00" }}", 38 | "end_date": "{{ $sd | date_modify (print "+" $period "s") | date "2006-01-02T15:04:05.999999Z07:00" }}", 39 | "AmortizedCost": { 40 | "amount": {{printf "%.2f" (generate "aws.billing.AmortizedCost.amount")}}, 41 | "unit": "{{$currency}}" 42 | }, 43 | "BlendedCost": { 44 | "amount": {{printf "%.2f" (generate "aws.billing.BlendedCost.amount")}}, 45 | "unit": "{{$currency}}" 46 | }, 47 | "NormalizedUsageAmount": { 48 | "amount": {{generate "aws.billing.NormalizedUsageAmount.amount"}}, 49 | "unit": "N/A" 50 | }, 51 | "UnblendedCost": { 52 | "amount": {{printf "%.2f" (generate "aws.billing.UnblendedCost.amount")}}, 53 | "unit": "{{$currency}}" 54 | }, 55 | "UsageQuantity": { 56 | "amount": {{generate "aws.billing.UsageQuantity.amount"}}, 57 | "unit": "N/A" 58 | }, 59 | "group_definition": { 60 | "key": "{{$groupBy}}", 61 | "type": "{{generate "aws.billing.group_definition.type"}}" 62 | }, 63 | "group_by": { 64 | {{- if eq $groupBy "AZ"}} 65 | "AZ": "{{awsAZFromRegion $cloudRegion}}" 66 | {{- else if eq $groupBy "INSTANCE_TYPE"}} 67 | "INSTANCE_TYPE": "{{generate "aws.billing.group_by.INSTANCE_TYPE"}}" 68 | {{- else if eq $groupBy "SERVICE"}} 69 | "SERVICE": "{{generate "aws.billing.group_by.SERVICE"}}" 70 | {{- else if eq $groupBy "LINKED_ACCOUNT"}} 71 | "LINKED_ACCOUNT": "{{$cloudId}}" 72 | {{- end}} 73 | } 74 | {{- end}} 75 | } 76 | }, 77 | "service": { 78 | "type": "aws" 79 | }, 80 | "agent": { 81 | "id": "{{generate "agent.id"}}", 82 | "name": "{{generate "agent.name"}}", 83 | "type": "metricbeat", 84 | "version": "8.0.0", 85 | "ephemeral_id": "{{generate "agent.ephemeral_id"}}" 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /assets/templates/aws.ec2_logs/schema-b/configs.yml: -------------------------------------------------------------------------------- 1 | fields: 2 | - name: process.name 3 | enum: ["journal", "kernel", "systemd"] 4 | - name: agent.id 5 | value: "da6cb4c8-c84c-4c5f-97c7-f8586a098af4" 6 | - name: cloud.region 7 | enum: ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ap-south-1", "ap-northeast-3", "ap-northeast-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ca-central-1", "eu-central-1", "eu-west-1", "eu-west-2", "eu-west-3", "eu-north-1", "sa-east-1", "af-south-1", "ap-east-1", "ap-south-2", "ap-southeast-3", "eu-south-2", "eu-central-2", "me-south-1", "me-central-1"] 8 | cardinality: 100 9 | - name: aws.cloudwatch.log_stream 10 | cardinality: 100 11 | - name: host.name 12 | cardinality: 100 13 | - name: aws.ec2.ip_address 14 | cardinality: 100 15 | -------------------------------------------------------------------------------- /assets/templates/aws.ec2_logs/schema-b/fields.yml: -------------------------------------------------------------------------------- 1 | - name: ts 2 | type: date 3 | - name: process.name 4 | type: keyword 5 | - name: event.ingested 6 | type: date 7 | - name: aws.ec2.ip_address 8 | type: ip 9 | - name: message 10 | type: text 11 | - name: aws.cloudwatch.log_stream 12 | type: keyword 13 | - name: host.name 14 | type: keyword 15 | - name: agent.id 16 | type: keyword 17 | - name: event.id 18 | type: keyword 19 | - name: cloud.region 20 | type: keyword 21 | -------------------------------------------------------------------------------- /assets/templates/aws.ec2_logs/schema-b/gotext.tpl: -------------------------------------------------------------------------------- 1 | {{- $ts := generate "ts" }} 2 | {{- $ip := generate "aws.ec2.ip_address" }} 3 | {{- $pname := generate "process.name" }} 4 | {{- $logstream := generate "aws.cloudwatch.log_stream" }} 5 | {{- $hostname := generate "host.name" }} 6 | {{- $agentId := generate "agent.id" }} 7 | { 8 | "@timestamp": "{{ $ts.Format "2006-01-02T15:04:05.999999Z07:00" }}", 9 | "aws.cloudwatch": { 10 | "log_stream": "{{$logstream}}", 11 | "ingestion_time": "{{ $ts | date "2006-01-02T15:04:05.000Z" }}", 12 | "log_group": "/var/log/messages" 13 | }, 14 | "cloud": { 15 | "region": "{{ generate "cloud.region" }}" 16 | }, 17 | "log.file.path": "/var/log/messages/{{$logstream}}", 18 | "input": { 19 | "type": "aws-cloudwatch" 20 | }, 21 | "data_stream": { 22 | "namespace": "default", 23 | "type": "logs", 24 | "dataset": "generic" 25 | }, 26 | "process": { 27 | "name": "{{ $pname }}" 28 | }, 29 | "message": "{{$ts | date "2006-01-02T15:04:05.000Z"}} {{$ts | date "Jan"}} {{$ts | date "02"}} {{$ts | date "15:04:05"}} {{printf "ip-%s" ($ip | splitList "." | join "-")}} {{$pname}}: {{generate "message"}}", 30 | "event": { 31 | "id": "{{ generate "event.id" }}", 32 | "ingested": "{{ generate "event.ingested" | date "2006-01-02T15:04:05.000000000Z" }}", 33 | "dataset": "generic" 34 | }, 35 | "host": { 36 | "name": "{{$hostname}}" 37 | }, 38 | "agent": { 39 | "id": "{{$agentId}}", 40 | "name": "{{$hostname}}", 41 | "type": "filebeat", 42 | "version": "8.8.0", 43 | "ephemeral_id": "{{$agentId}}" 44 | }, 45 | "tags": [ 46 | "preserve_original_event" 47 | ] 48 | } 49 | -------------------------------------------------------------------------------- /assets/templates/aws.ec2_metrics/schema-b/configs.yml: -------------------------------------------------------------------------------- 1 | fields: 2 | - name: dimensionType 3 | # no dimension: 2.5%, AutoScalingGroupName: 10%, ImageId: 5%, InstanceType: 2.5%, InstanceId: 80% 4 | enum: ["", "AutoScalingGroupName", "AutoScalingGroupName", "AutoScalingGroupName", "AutoScalingGroupName", "ImageId", "ImageId", "InstanceType", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId", "InstanceId"] 5 | cardinality: 600 6 | # we want every single different "dimension identifier", regardless of its type, to have always the same generated fixed "metadata" once the cardinality kicks in 7 | # for this we must take the ordered highest enum length appending one by one the ones that does not have a 0 module between each others. 8 | # we start from the first two, multiple between their values and exclude from the order list the ones that have a 0 module on the result of the multiplication. 9 | # we end up with the list of enum lengths whose value, multiplied, define the least common multiple: this is the value we must use for the cardinality of all fields. 10 | # in this case the remaining enum are two: `dimensionType` (40) and `region` (15), resulting in cardinality `600` 11 | - name: Region 12 | enum: ["ap-south-1", "eu-north-1", "eu-west-3", "eu-west-2", "eu-west-1", "ap-northeast-3", "ap-northeast-2", "ap-northeast-1", "ap-southeast-1", "ap-southeast-2", "eu-central-1", "us-east-1", "us-east-2", "us-west-1", "us-west-2"] 13 | cardinality: 600 14 | - name: AutoScalingGroupName 15 | cardinality: 600 16 | - name: ImageId 17 | cardinality: 600 18 | - name: InstanceId 19 | cardinality: 600 20 | - name: instanceTypeIdx 21 | # we generate and index for the instance type enums, so that all the information related to a given type are properly matched 22 | range: 23 | min: 0 24 | max: 19 25 | cardinality: 600 26 | - name: InstanceType 27 | value: ["a1.medium", "c3.2xlarge", "c4.4xlarge", "c5.9xlarge", "c5a.12xlarge", "c5ad.16xlarge", "c5d.24xlarge", "c6a.32xlarge", "g5.48xlarge", "d2.2xlarge", "d3.xlarge", "t2.medium", "t2.micro", "t2.nano", "t2.small", "t3.large", "t3.medium", "t3.micro", "t3.nano", "t3.small"] 28 | - name: instanceCoreCount 29 | # they map instance types 30 | value: ["1", "4", "8", "18", "24", "32", "48", "64", "96", "4", "2", "2", "1", "1", "1", "1", "1", "1", "1", "1"] 31 | - name: instanceThreadPerCore 32 | # they map instance types 33 | value: ["1", "2", "2", " 2", " 2", " 2", " 2", " 2", " 2", "2", "2", "1", "1", "1", "1", "2", "2", "2", "2", "2"] 34 | - name: instanceImageId 35 | cardinality: 600 36 | - name: instanceMonitoringState 37 | # enable: 10%, disabled: 90% 38 | enum: ["enabled", "disabled", "disabled", "disabled", "disabled", "disabled", "disabled", "disabled", "disabled", "disabled"] 39 | cardinality: 600 40 | - name: instancePrivateIP 41 | cardinality: 600 42 | - name: instancePrivateDnsEmpty 43 | # without private dns entry: 10%, with private dns entry: 90% 44 | enum: ["empty", "fromPrivateIP", "fromPrivateIP", "fromPrivateIP", "fromPrivateIP", "fromPrivateIP", "fromPrivateIP", "fromPrivateIP", "fromPrivateIP", "fromPrivateIP"] 45 | cardinality: 600 46 | - name: instancePublicIP 47 | cardinality: 600 48 | - name: instancePublicDnsEmpty 49 | # without public dns entry: 20%, with public dns entry: 80% 50 | enum: ["empty", "fromPublicIP", "fromPublicIP", "fromPublicIP", "fromPublicIP"] 51 | cardinality: 600 52 | - name: instanceStateName 53 | # terminated: 10%, running: 90% 54 | enum: ["terminated", "running", "running", "running", "running", "running", "running", "running", "running", "running"] 55 | cardinality: 600 56 | - name: cloudInstanceName 57 | cardinality: 600 58 | - name: StatusCheckFailed_InstanceAvg 59 | range: 60 | min: 0 61 | max: 10 62 | fuzziness: 0.05 63 | - name: StatusCheckFailed_SystemAvg 64 | range: 65 | min: 0 66 | max: 10 67 | fuzziness: 0.05 68 | - name: StatusCheckFailedAvg 69 | range: 70 | min: 0 71 | max: 10 72 | fuzziness: 0.05 73 | - name: CPUUtilizationAvg 74 | range: 75 | min: 0 76 | max: 100 77 | fuzziness: 0.05 78 | - name: NetworkPacketsInSum 79 | range: 80 | min: 0 81 | max: 1500000 82 | fuzziness: 0.05 83 | - name: NetworkPacketsOutSum 84 | range: 85 | min: 0 86 | max: 1500000 87 | fuzziness: 0.05 88 | - name: CPUCreditBalanceAvg 89 | range: 90 | min: 0 91 | max: 5000 92 | fuzziness: 0.05 93 | - name: CPUSurplusCreditBalanceAvg 94 | range: 95 | min: 0 96 | max: 5000 97 | fuzziness: 0.05 98 | - name: CPUSurplusCreditsChargedAvg 99 | range: 100 | min: 0 101 | max: 5000 102 | fuzziness: 0.05 103 | - name: CPUCreditUsageAvg 104 | range: 105 | min: 0 106 | max: 10 107 | fuzziness: 0.05 108 | - name: DiskReadBytesSum 109 | range: 110 | min: 0 111 | max: 1500000 112 | fuzziness: 0.05 113 | - name: DiskReadOpsSum 114 | range: 115 | min: 0 116 | max: 1000 117 | fuzziness: 0.05 118 | - name: DiskWriteBytesSum 119 | range: 120 | min: 0 121 | max: 1500000000 122 | fuzziness: 0.05 123 | - name: DiskWriteOpsSum 124 | range: 125 | min: 0 126 | max: 1000 127 | fuzziness: 0.05 128 | - name: EventDuration 129 | range: 130 | min: 1 131 | max: 1000 132 | - name: partOfAutoScalingGroup 133 | # we dived this value by 20 in the template, giving 20% chance to be part of an autoscaling group: in this case we append the related aws.tags 134 | range: 135 | min: 1 136 | max: 100 137 | -------------------------------------------------------------------------------- /assets/templates/aws.ec2_metrics/schema-b/fields.yml: -------------------------------------------------------------------------------- 1 | - name: dimensionType 2 | type: keyword 3 | - name: Region 4 | type: keyword 5 | - name: AutoScalingGroupName 6 | type: keyword 7 | example: eks-standard-workers-22c2aa67-05ef-ad12-6406-b992651f6024 8 | - name: ImageId 9 | type: keyword 10 | example: ami-099ccc441b2ef41ec 11 | - name: InstanceId 12 | type: keyword 13 | example: i-0af20a3fedc456530 14 | - name: InstanceType 15 | type: keyword 16 | - name: instanceTypeIdx 17 | type: long 18 | - name: instanceCoreCount 19 | type: keyword 20 | - name: instanceThreadPerCore 21 | type: keyword 22 | - name: instanceImageId 23 | type: keyword 24 | example: ami-099ccc441b2ef41ec 25 | - name: instanceMonitoringState 26 | type: keyword 27 | - name: instancePrivateIP 28 | type: ip 29 | - name: instancePrivateDnsEmpty 30 | type: keyword 31 | - name: instancePublicIP 32 | type: ip 33 | - name: instancePublicDnsEmpty 34 | type: keyword 35 | - name: instanceStateName 36 | type: keyword 37 | - name: cloudInstanceName 38 | type: keyword 39 | example: an-instance-name 40 | - name: StatusCheckFailed_InstanceAvg 41 | type: double 42 | - name: StatusCheckFailed_SystemAvg 43 | type: double 44 | - name: StatusCheckFailedAvg 45 | type: double 46 | - name: CPUUtilizationAvg 47 | type: double 48 | - name: NetworkPacketsInSum 49 | type: double 50 | - name: NetworkPacketsOutSum 51 | type: double 52 | - name: CPUCreditBalanceAvg 53 | type: double 54 | - name: CPUSurplusCreditBalanceAvg 55 | type: double 56 | - name: CPUSurplusCreditsChargedAvg 57 | type: double 58 | - name: CPUCreditUsageAvg 59 | type: double 60 | - name: DiskReadBytesSum 61 | type: double 62 | - name: DiskReadOpsSum 63 | type: double 64 | - name: DiskWriteBytesSum 65 | type: double 66 | - name: DiskWriteOpsSum 67 | type: double 68 | - name: EventDuration 69 | type: long 70 | - name: EventIngested 71 | type: date 72 | - name: partOfAutoScalingGroup 73 | type: long 74 | -------------------------------------------------------------------------------- /assets/templates/aws.ec2_metrics/schema-b/gotext.tpl: -------------------------------------------------------------------------------- 1 | {{- /* metadata */ -}} 2 | {{- $Region := generate "Region" }} 3 | {{- $eventIngested := generate "EventIngested" }} 4 | {{- $eventDuration := generate "EventDuration" }} 5 | {{- /* availability zone */ -}} 6 | {{- $AvailabilityZone := awsAZFromRegion $Region }} 7 | {{- /* dimensions */ -}} 8 | {{- $AutoScalingGroupName := generate "AutoScalingGroupName" }} 9 | {{- $ImageId := generate "ImageId" }} 10 | {{- $InstanceId := generate "InstanceId" }} 11 | {{- $instanceTypeIdx := generate "instanceTypeIdx" }} 12 | {{- $InstanceTypeValues := generate "InstanceType" }} 13 | {{- $InstanceType := index $InstanceTypeValues $instanceTypeIdx }} 14 | {{- /* metrics */ -}} 15 | {{- $StatusCheckFailed_InstanceAvg := generate "StatusCheckFailed_InstanceAvg" }} 16 | {{- $StatusCheckFailed_SystemAvg := generate "StatusCheckFailed_SystemAvg" }} 17 | {{- $StatusCheckFailedAvg := generate "StatusCheckFailedAvg" }} 18 | {{- $CPUUtilizationAvg := generate "CPUUtilizationAvg" }} 19 | {{- $NetworkPacketsInSum := generate "NetworkPacketsInSum" }} 20 | {{- $NetworkPacketsOutSum := generate "NetworkPacketsOutSum" }} 21 | {{- $NetworkInSum := mul $NetworkPacketsInSum 15 }} 22 | {{- $NetworkOutSum := mul $NetworkPacketsOutSum 15 }} 23 | {{- $CPUCreditBalanceAvg := generate "CPUCreditBalanceAvg" }} 24 | {{- $CPUSurplusCreditBalanceAvg := generate "CPUSurplusCreditBalanceAvg" }} 25 | {{- $CPUSurplusCreditsChargedAvg := generate "CPUSurplusCreditsChargedAvg" }} 26 | {{- $CPUCreditUsageAvg := generate "CPUCreditUsageAvg" }} 27 | {{- $DiskReadBytesSum := generate "DiskReadBytesSum" }} 28 | {{- $DiskReadOpsSum := generate "DiskReadOpsSum" }} 29 | {{- $DiskWriteBytesSum := generate "DiskWriteBytesSum" }} 30 | {{- $DiskWriteOpsSum := generate "DiskWriteOpsSum" }} 31 | {{- /* instance data */ -}} 32 | {{- $instanceCoreCountValues := generate "instanceCoreCount" }} 33 | {{- $instanceCoreCount := index $instanceCoreCountValues $instanceTypeIdx }} 34 | {{- $instanceThreadPerCoreValues := generate "instanceThreadPerCore" }} 35 | {{- $instanceThreadPerCore := index $instanceThreadPerCoreValues $instanceTypeIdx }} 36 | {{- $instanceImageId := generate "instanceImageId" }} 37 | {{- $instanceMonitoringState := generate "instanceMonitoringState" }} 38 | {{- $instancePrivateIP := generate "instancePrivateIP" }} 39 | {{- $instancePrivateDnsEmpty := generate "instancePrivateDnsEmpty" }} 40 | {{- $instancePublicIP := generate "instancePublicIP" }} 41 | {{- $instancePublicDnsEmpty := generate "instancePublicDnsEmpty" }} 42 | {{- $instanceStateName := generate "instanceStateName" }} 43 | {{- $instanceStateCode := 16 }} 44 | {{- if eq $instanceStateName "running" }} 45 | {{- $instanceStateCode = 48 }} 46 | {{- end}} 47 | {{- $cloudInstanceName := generate "cloudInstanceName" }} 48 | {{- /* rate period */ -}} 49 | {{- $period := 60. }} 50 | {{- if eq $instanceMonitoringState "disabled" }} 51 | {{- $period = 300. }} 52 | {{- end}} 53 | {{- /* ip */ -}} 54 | {{- $instancePrivateDns := "" }} 55 | {{- if eq $instancePrivateDnsEmpty "fromPrivateIP" }} 56 | {{- $instancePrivateDnsPrefix := $instancePrivateIP | replace "." "-" }} 57 | {{- $instancePrivateDns = printf "%s.%s.compute.internal" $instancePrivateDnsPrefix $Region }} 58 | {{- end}} 59 | {{- $instancePublicDns := "" }} 60 | {{- if eq $instancePublicDnsEmpty "fromPublicIP" }} 61 | {{- $instancePublicDnsPrefix := $instancePublicIP | replace "." "-" }} 62 | {{- $instancePublicDns = printf "e2-%s.compute-1.amazonaws.com" $instancePublicDnsPrefix }} 63 | {{- end}} 64 | {{- /* tags */ -}} 65 | {{- $partOfAutoScalingGroup := generate "partOfAutoScalingGroup" | mod 20 }}{{- /* 5% chance the instance is part of an autoscaling group */ -}} 66 | {{- /* events */ -}} 67 | { 68 | "@timestamp": "{{ $eventIngested.Format "2006-01-02T15:04:05.999999Z07:00" }}", 69 | "ecs.version": "8.0.0", 70 | "agent": { 71 | "name": "docker-fleet-agent", 72 | "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", 73 | "type": "metricbeat", 74 | "ephemeral_id": "cdaaaabb-be7e-432f-816b-bda019fd7c15", 75 | "version": "8.3.2" 76 | }, 77 | "elastic_agent": { 78 | "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", 79 | "version": "8.3.2", 80 | "snapshot": false 81 | }, 82 | "cloud": { 83 | "provider": "aws", 84 | "region": "{{ $Region }}", 85 | "account": { 86 | "name": "elastic-beats", 87 | "id": "000000000000" 88 | } 89 | }, 90 | "ecs": { 91 | "version": "8.0.0" 92 | }, 93 | "service": { 94 | "type": "aws" 95 | }, 96 | "data_stream": { 97 | "namespace": "default", 98 | "type": "metrics", 99 | "dataset": "aws.ec2_metrics" 100 | }, 101 | "metricset": { 102 | "period": 3600000, 103 | "name": "cloudwatch" 104 | }, 105 | "event": { 106 | "duration": {{ $eventDuration }}, 107 | "agent_id_status": "verified", 108 | "ingested": "{{ $eventIngested.Format "2006-01-02T15:04:05.999999Z07:00" }}", 109 | "module": "aws", 110 | "dataset": "aws.ec2_metrics" 111 | }, 112 | "aws": { 113 | "cloudwatch": { 114 | "namespace": "AWS/EC2" 115 | } 116 | }, 117 | {{ $dimensionType := generate "dimensionType" }} 118 | {{ if eq $dimensionType "AutoScalingGroupName" }} 119 | "aws.dimensions.AutoScalingGroupName": "{{ $AutoScalingGroupName }}", 120 | "aws.ec2.metrics.CPUCreditBalance.avg": {{ $CPUCreditBalanceAvg }}, 121 | "aws.ec2.metrics.CPUCreditUsage.avg": {{ $CPUCreditUsageAvg }}, 122 | "aws.ec2.metrics.CPUSurplusCreditBalance.avg": {{ $CPUSurplusCreditBalanceAvg }}, 123 | "aws.ec2.metrics.CPUSurplusCreditsCharged.avg": {{ $CPUSurplusCreditsChargedAvg }}, 124 | "aws.ec2.metrics.CPUUtilization.avg": {{ $CPUUtilizationAvg }}, 125 | "aws.ec2.metrics.NetworkIn.sum": {{ $NetworkInSum }}, 126 | "aws.ec2.metrics.NetworkOut.sum": {{ $NetworkOutSum }}, 127 | "aws.ec2.metrics.NetworkPacketsIn.sum": {{ $NetworkPacketsInSum }}, 128 | "aws.ec2.metrics.NetworkPacketsOut.sum": {{ $NetworkPacketsOutSum }}, 129 | "aws.ec2.metrics.StatusCheckFailed_Instance.avg": {{ $StatusCheckFailed_InstanceAvg }}, 130 | "aws.ec2.metrics.StatusCheckFailed_System.avg": {{ $StatusCheckFailed_SystemAvg }}, 131 | "aws.ec2.metrics.StatusCheckFailed.avg": {{ $StatusCheckFailedAvg }} 132 | {{ else if eq $dimensionType "ImageId" }} 133 | "aws.dimensions.ImageId": "{{ $ImageId }}", 134 | "aws.ec2.metrics.CPUUtilization.avg": {{ $CPUUtilizationAvg }}, 135 | "aws.ec2.metrics.DiskReadBytes.sum": {{ $DiskReadBytesSum }}, 136 | "aws.ec2.metrics.DiskReadOps.sum": {{ $DiskReadOpsSum }}, 137 | "aws.ec2.metrics.DiskWriteBytes.sum": {{ $DiskWriteBytesSum }}, 138 | "aws.ec2.metrics.DiskWriteOps.sum": {{ $DiskWriteOpsSum }}, 139 | "aws.ec2.metrics.NetworkIn.sum": {{ $NetworkInSum }}, 140 | "aws.ec2.metrics.NetworkOut.sum": {{ $NetworkOutSum }} 141 | {{ else if eq $dimensionType "InstanceId" }} 142 | "aws.dimensions.InstanceId": "{{ $InstanceId }}", 143 | "aws.ec2.instance.core.count": {{ $instanceCoreCount }}, 144 | "aws.ec2.instance.image.id": "{{ $instanceImageId }}", 145 | "aws.ec2.instance.monitoring.state": "{{ $instanceMonitoringState }}", 146 | "aws.ec2.instance.private.dns_name": "{{ $instancePrivateDns }}", 147 | "aws.ec2.instance.private.ip": "{{ $instancePrivateIP }}", 148 | "aws.ec2.instance.public.dns_name": "{{ $instancePublicDns }}", 149 | "aws.ec2.instance.public.ip": "{{ $instancePublicIP }}", 150 | "aws.ec2.instance.state.code": {{ $instanceStateCode }}, 151 | "aws.ec2.instance.state.name": "{{ $instanceStateName }}", 152 | "aws.ec2.instance.threads_per_core": {{ $instanceThreadPerCore }}, 153 | "aws.ec2.metrics.CPUCreditBalance.avg": {{ $CPUCreditBalanceAvg }}, 154 | "aws.ec2.metrics.CPUCreditUsage.avg": {{ $CPUCreditUsageAvg }}, 155 | "aws.ec2.metrics.CPUSurplusCreditBalance.avg": {{ $CPUSurplusCreditBalanceAvg }}, 156 | "aws.ec2.metrics.CPUSurplusCreditsCharged.avg": {{ $CPUSurplusCreditsChargedAvg }}, 157 | "aws.ec2.metrics.CPUUtilization.avg": {{ $CPUUtilizationAvg }}, 158 | "aws.ec2.metrics.DiskReadBytes.rate": {{ divf $DiskReadBytesSum $period }}, 159 | "aws.ec2.metrics.DiskReadBytes.sum": {{ $DiskReadBytesSum }}, 160 | "aws.ec2.metrics.DiskReadOps.rate": {{ divf $DiskReadOpsSum $period }}, 161 | "aws.ec2.metrics.DiskReadOps.sum": {{ $DiskReadOpsSum }}, 162 | "aws.ec2.metrics.DiskWriteBytes.rate": {{ divf $DiskWriteBytesSum $period }}, 163 | "aws.ec2.metrics.DiskWriteBytes.sum": {{ $DiskWriteBytesSum }}, 164 | "aws.ec2.metrics.DiskWriteOps.rate": {{ divf $DiskWriteOpsSum $period }}, 165 | "aws.ec2.metrics.DiskWriteOps.sum": {{ $DiskWriteOpsSum }}, 166 | "aws.ec2.metrics.NetworkIn.rate": {{ divf $NetworkInSum $period }}, 167 | "aws.ec2.metrics.NetworkIn.sum": {{ $NetworkInSum }}, 168 | "aws.ec2.metrics.NetworkOut.rate": {{ divf $NetworkOutSum $period }}, 169 | "aws.ec2.metrics.NetworkOut.sum": {{ $NetworkOutSum }}, 170 | "aws.ec2.metrics.NetworkPacketsIn.rate": {{ divf $NetworkPacketsInSum $period }}, 171 | "aws.ec2.metrics.NetworkPacketsIn.sum": {{ $NetworkPacketsInSum }}, 172 | "aws.ec2.metrics.NetworkPacketsOut.rate": {{ divf $NetworkPacketsOutSum $period }}, 173 | "aws.ec2.metrics.NetworkPacketsOut.sum": {{ $NetworkPacketsOutSum }}, 174 | "aws.ec2.metrics.StatusCheckFailed_Instance.avg": {{ $StatusCheckFailed_InstanceAvg }}, 175 | "aws.ec2.metrics.StatusCheckFailed_System.avg": {{ $StatusCheckFailed_SystemAvg }}, 176 | "aws.ec2.metrics.StatusCheckFailed.avg": {{ $StatusCheckFailedAvg }}, 177 | "aws.tags.Name": "{{ $cloudInstanceName }}", 178 | {{ if eq $partOfAutoScalingGroup 0 }} 179 | "aws.tags.aws:autoscaling:groupName": "{{ $AutoScalingGroupName }}", 180 | {{ end }} 181 | "cloud.availability_zone": "{{ $AvailabilityZone }}", 182 | "cloud.instance.id": "{{ $InstanceId }}", 183 | "cloud.instance.name": "{{ $cloudInstanceName }}", 184 | "cloud.machine.type": "{{ $InstanceType }}", 185 | "host.cpu.usage": {{ $CPUUtilizationAvg }}, 186 | "host.disk.read.bytes": {{ $DiskReadBytesSum }}, 187 | "host.disk.write.bytes": {{ $DiskWriteBytesSum }}, 188 | "host.id": "{{ $InstanceId }}", 189 | "host.name": "{{ $cloudInstanceName }}", 190 | "host.network.egress.bytes": {{ $NetworkOutSum }}, 191 | "host.network.egress.packets": {{ $NetworkPacketsOutSum }}, 192 | "host.network.ingress.bytes": {{ $NetworkInSum }}, 193 | "host.network.ingress.packets": {{ $NetworkPacketsInSum }} 194 | {{ else if eq $dimensionType "InstanceType" }} 195 | "aws.dimensions.InstanceType": "{{ $InstanceType }}", 196 | "aws.ec2.metrics.CPUUtilization.avg": {{ $CPUUtilizationAvg }}, 197 | "aws.ec2.metrics.DiskReadBytes.sum": {{ $DiskReadBytesSum }}, 198 | "aws.ec2.metrics.DiskReadOps.sum": {{ $DiskReadOpsSum }}, 199 | "aws.ec2.metrics.DiskWriteBytes.sum": {{ $DiskWriteBytesSum }}, 200 | "aws.ec2.metrics.DiskWriteOps.sum": {{ $DiskWriteOpsSum }}, 201 | "aws.ec2.metrics.NetworkIn.sum": {{ $NetworkInSum }}, 202 | "aws.ec2.metrics.NetworkOut.sum": {{ $NetworkOutSum }} 203 | {{ else }} 204 | "aws.ec2.metrics.CPUUtilization.avg": {{ $CPUUtilizationAvg }}, 205 | "aws.ec2.metrics.DiskReadBytes.sum": {{ $DiskReadBytesSum }}, 206 | "aws.ec2.metrics.DiskReadOps.sum": {{ $DiskReadOpsSum }}, 207 | "aws.ec2.metrics.DiskWriteBytes.sum": {{ $DiskWriteBytesSum }}, 208 | "aws.ec2.metrics.DiskWriteOps.sum": {{ $DiskWriteOpsSum }}, 209 | "aws.ec2.metrics.NetworkIn.sum": {{ $NetworkInSum }}, 210 | "aws.ec2.metrics.NetworkOut.sum": {{ $NetworkOutSum }} 211 | {{ end }} 212 | } -------------------------------------------------------------------------------- /assets/templates/aws.sqs/schema-b/configs.yml: -------------------------------------------------------------------------------- 1 | fields: 2 | - name: Region 3 | enum: ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ap-south-1", "ap-northeast-3", "ap-northeast-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ca-central-1", "eu-central-1", "eu-west-1", "eu-west-2", "eu-west-3", "eu-north-1", "sa-east-1", "af-south-1", "ap-east-1", "ap-south-2", "ap-southeast-3", "eu-south-2", "eu-central-2", "me-south-1", "me-central-1"] 4 | cardinality: 25 5 | - name: Visible 6 | range: 7 | min: 0 8 | max: 1000 9 | fuzziness: 0.05 10 | - name: Deleted 11 | range: 12 | min: 0 13 | max: 1000 14 | fuzziness: 0.05 15 | - name: NotVisible 16 | range: 17 | min: 0 18 | max: 1000 19 | fuzziness: 0.05 20 | - name: Delayed 21 | range: 22 | min: 0 23 | max: 1000 24 | fuzziness: 0.05 25 | - name: Received 26 | range: 27 | min: 0 28 | max: 1000 29 | fuzziness: 0.05 30 | - name: Sent 31 | range: 32 | min: 0 33 | max: 1000 34 | fuzziness: 0.05 35 | - name: EmptyReceives 36 | range: 37 | min: 0 38 | max: 10 39 | fuzziness: 0.1 40 | - name: SentMessageSize 41 | range: 42 | min: 1024 43 | max: 262144 44 | fuzziness: 0.5 45 | - name: OldestMessageAge 46 | range: 47 | min: 1 48 | max: 7200 49 | fuzziness: 0.05 50 | - name: QueueName 51 | cardinality: 200 52 | - name: EventDuration 53 | range: 54 | min: 1 55 | max: 1000 56 | - name: TagsCreatedBy 57 | cardinality: 200 58 | -------------------------------------------------------------------------------- /assets/templates/aws.sqs/schema-b/fields.yml: -------------------------------------------------------------------------------- 1 | - name: Region 2 | type: keyword 3 | - name: Visible 4 | type: double 5 | - name: Deleted 6 | type: double 7 | - name: NotVisible 8 | type: double 9 | - name: Delayed 10 | type: double 11 | - name: Received 12 | type: double 13 | - name: Sent 14 | type: double 15 | - name: EmptyReceives 16 | type: double 17 | - name: SentMessageSize 18 | type: double 19 | - name: OldestMessageAge 20 | type: double 21 | - name: QueueName 22 | type: keyword 23 | example: a-queue-name 24 | - name: EventDuration 25 | type: long 26 | - name: EventIngested 27 | type: date 28 | - name: TagsCreatedBy 29 | type: keyword 30 | example: First Last 31 | -------------------------------------------------------------------------------- /assets/templates/aws.sqs/schema-b/gotext.tpl: -------------------------------------------------------------------------------- 1 | {{ $queueName := generate "QueueName" }}{{ $eventIngested := generate "EventIngested" }}{ "@timestamp": "{{ $eventIngested.Format "2006-01-02T15:04:05.999999Z07:00" }}", "agent": { "name": "docker-fleet-agent", "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", "type": "metricbeat", "ephemeral_id": "cdaaaabb-be7e-432f-816b-bda019fd7c15", "version": "8.3.2" }, "elastic_agent": { "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", "version": "8.3.2", "snapshot": false }, "cloud": { "provider": "aws", "region": "{{ generate "Region" }}", "account": { "name": "elastic-beats", "id": "000000000000" } }, "ecs": { "version": "8.0.0" }, "service": { "type": "aws" }, "data_stream": { "namespace": "default", "type": "metrics", "dataset": "aws.sqs" }, "metricset": { "period": 300000, "name": "cloudwatch" }, "event": { "duration": {{ generate "EventDuration" }}, "agent_id_status": "verified", "ingested": "{{ $eventIngested.Format "2006-01-02T15:04:05.999999Z07:00" }}", "module": "aws", "dataset": "aws.sqs" }, "aws": { "cloudwatch": { "namespace": "AWS/SQS" }, "dimensions": { "QueueName": "{{ $queueName }}" }, "sqs": { "queue": { "name": "{{ $queueName }}" }, "metrics": { "ApproximateAgeOfOldestMessage": { "avg": {{ generate "OldestMessageAge" }} }, "ApproximateNumberOfMessagesDelayed": { "avg": {{ generate "Delayed" }} }, "ApproximateNumberOfMessagesNotVisible": { "avg": {{ generate "NotVisible" }} }, "ApproximateNumberOfMessagesVisible": { "avg": {{ generate "Visible" }} }, "NumberOfMessagesDeleted": { "avg": {{ generate "Deleted" }} }, "NumberOfMessagesReceived": { "avg": {{ generate "Received" }} }, "NumberOfMessagesSent": { "avg": {{ generate "Sent" }} }, "NumberOfEmptyReceives": { "avg": {{ generate "EmptyReceives" }} }, "SentMessageSize": { "avg": {{ generate "SentMessageSize" }} } } }, "tags": { "createdBy": "{{ generate "TagsCreatedBy" }}" } } } -------------------------------------------------------------------------------- /assets/templates/aws.sqs/schema-b/placeholder.tpl: -------------------------------------------------------------------------------- 1 | { "@timestamp": "{{ .EventIngested }}", "agent": { "name": "docker-fleet-agent", "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", "type": "metricbeat", "ephemeral_id": "cdaaaabb-be7e-432f-816b-bda019fd7c15", "version": "8.3.2" }, "elastic_agent": { "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", "version": "8.3.2", "snapshot": false }, "cloud": { "provider": "aws", "region": "{{ .Region }}", "account": { "name": "elastic-beats", "id": "000000000000" } }, "ecs": { "version": "8.0.0" }, "service": { "type": "aws" }, "data_stream": { "namespace": "default", "type": "metrics", "dataset": "aws.sqs" }, "metricset": { "period": 300000, "name": "cloudwatch" }, "event": { "duration": {{ .EventDuration }}, "agent_id_status": "verified", "ingested": "{{ .EventIngested }}", "module": "aws", "dataset": "aws.sqs" }, "aws": { "cloudwatch": { "namespace": "AWS/SQS" }, "dimensions": { "QueueName": "{{ .QueueName }}" }, "sqs": { "queue": { "name": "{{ .QueueName }}" }, "metrics": { "ApproximateAgeOfOldestMessage": { "avg": {{ .OldestMessageAge }} }, "ApproximateNumberOfMessagesDelayed": { "avg": {{ .Delayed }} }, "ApproximateNumberOfMessagesNotVisible": { "avg": {{ .NotVisible }} }, "ApproximateNumberOfMessagesVisible": { "avg": {{ .Visible }} }, "NumberOfMessagesDeleted": { "avg": {{ .Deleted }} }, "NumberOfMessagesReceived": { "avg": {{ .Received }} }, "NumberOfMessagesSent": { "avg": {{ .Sent }} }, "NumberOfEmptyReceives": { "avg": {{ .EmptyReceives }} }, "SentMessageSize": { "avg": {{ .SentMessageSize }} } } }, "tags": { "createdBy": "{{ .TagsCreatedBy }}" } } } -------------------------------------------------------------------------------- /assets/templates/aws.vpcflow/schema-a/configs.yml: -------------------------------------------------------------------------------- 1 | fields: 2 | - name: Version 3 | value: 2 4 | - name: AccountID 5 | value: 627286350134 6 | - name: InterfaceID 7 | cardinality: 100 8 | - name: SrcAddr 9 | cardinality: 1000 10 | - name: DstAddr 11 | cardinality: 10 12 | - name: SrcPort 13 | range: 14 | min: 0 15 | max: 65535 16 | - name: DstPort 17 | range: 18 | min: 0 19 | max: 65535 20 | cardinality: 10 21 | - name: Protocol 22 | range: 23 | min: 1 24 | max: 256 25 | - name: Packets 26 | range: 27 | min: 1 28 | max: 1048576 29 | - name: Bytes 30 | range: 31 | min: 1 32 | max: 15728640 33 | - name: StartOffset 34 | range: 35 | min: 1 36 | max: 60 37 | - name: Action 38 | enum: ["ACCEPT", "REJECT"] 39 | - name: LogStatus 40 | enum: ["OK", "SKIPDATA"] -------------------------------------------------------------------------------- /assets/templates/aws.vpcflow/schema-a/fields.yml: -------------------------------------------------------------------------------- 1 | - name: Version 2 | type: long 3 | - name: AccountID 4 | type: long 5 | - name: InterfaceID 6 | type: keyword 7 | example: eni-1235b8ca123456789 8 | - name: SrcAddr 9 | type: ip 10 | - name: DstAddr 11 | type: ip 12 | - name: SrcPort 13 | type: long 14 | - name: DstPort 15 | type: long 16 | - name: Protocol 17 | type: long 18 | - name: Packets 19 | type: long 20 | - name: Bytes 21 | type: long 22 | - name: End 23 | type: date 24 | - name: Start 25 | type: date 26 | - name: StartOffset 27 | type: long 28 | - name: Action 29 | type: keyword 30 | - name: LogStatus 31 | type: keyword 32 | -------------------------------------------------------------------------------- /assets/templates/aws.vpcflow/schema-a/gotext.tpl: -------------------------------------------------------------------------------- 1 | {{- $startOffset := generate "StartOffset" }} 2 | {{- $end := generate "End" }} 3 | {{- $start := $end | dateModify (mul -1 $startOffset | int64 | duration) }} 4 | {{generate "Version"}} {{generate "AccountID"}} {{generate "InterfaceID"}} {{generate "SrcAddr"}} {{generate "DstAddr"}} {{generate "SrcPort"}} {{generate "DstPort"}} {{generate "Protocol"}}{{ $packets := generate "Packets" }} {{ $packets }} {{mul $packets 15 }} {{$start.Format "2006-01-02T15:04:05.999999Z07:00" }} {{$end.Format "2006-01-02T15:04:05.999999Z07:00"}} {{generate "Action"}}{{ if eq $packets 0 }} NODATA {{ else }} {{generate "LogStatus"}} {{ end }} 5 | -------------------------------------------------------------------------------- /assets/templates/aws.vpcflow/schema-a/placeholder.tpl: -------------------------------------------------------------------------------- 1 | {{.Version}} {{.AccountID}} {{.InterfaceID}} {{.SrcAddr}} {{.DstAddr}} {{.SrcPort}} {{.DstPort}} {{.Protocol}} {{.Packets}} {{.Bytes}} {{.Start}} {{.End}} {{.Action}} {{.LogStatus}} -------------------------------------------------------------------------------- /assets/templates/kubernetes.container/schema-b/configs.yml: -------------------------------------------------------------------------------- 1 | fields: 2 | - name: cloud.availabilit_zone 3 | value: "europe-west1-d" 4 | - name: agent.id 5 | value: "12f376ef-5186-4e8b-a175-70f1140a8f30" 6 | - name: agent.name 7 | value: "kubernetes-scale-123456" 8 | - name: agent.ephemeral_id 9 | value: "f94220b0-2ca6-4809-8656-eb478a66c541" 10 | - name: agent.version 11 | value: "8.7.0" 12 | - name: agent.snasphost 13 | value: false 14 | - name: metricset.period 15 | value: 10000 16 | - name: event.duration 17 | range: 18 | min: 1 19 | max: 4000000 20 | - name: faults 21 | range: 22 | min: 0 23 | max: 500000 24 | - name: kubernetes.container.rootfs.inodes.used 25 | range: 26 | min: 1 27 | max: 100000 28 | - name: Bytes 29 | range: 30 | min: 1 31 | max: 3000000 32 | - name: rangeofid 33 | range: 34 | min: 0 35 | max: 10000 36 | - name: PercentageMemory 37 | range: 38 | min: 0.0 39 | max: 1.0 40 | - name: PercentageCPU 41 | range: 42 | min: 0.0 43 | max: 1.0 44 | - name: usage.* 45 | object_keys: 46 | - nanoseconds 47 | - nanocores 48 | - name: usage.nanoseconds 49 | range: 50 | min: 100000 51 | max: 9000000000 52 | cardinality: 10000 53 | - name: usage.nanocores 54 | range: 55 | min: 100000 56 | max: 9000000 57 | cardinality: 10000 58 | - name: container.name 59 | enum: ["web", "default-http-backend", "dnsmasq", "csi-driver", "web", "web", "web", "prometheus", "konnectivity-agent", "sidecar", "kubedns", "metrics-server-nanny", "web", "web", "fluentbit", "autoscaler", "gke-metrics-agent", "elastic-agent", "web", "kube-state-metrics", "metrics-server", "fluentbit", "elastic-agent", "web", "prometheus-to-sd-exporter"] 60 | - name: timedate 61 | # Add more dates below in case you want your data to be spread.Those are fixed because this is required in the nightly tests of Rally 62 | enum: ["2023-05-15" , "2023-05-16"] 63 | - name: timehour 64 | # Repeat or remove hours below to make data appear in specific hours. Below default enumaeration makes sure that data are spread throughout the 24 hours 65 | enum: ["00", "01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23" ] -------------------------------------------------------------------------------- /assets/templates/kubernetes.container/schema-b/fields.yml: -------------------------------------------------------------------------------- 1 | - name: timestamp 2 | type: date 3 | - name: timedate 4 | type: keyword 5 | - name: timehour 6 | type: keyword 7 | - name: agent.id 8 | type: keyword 9 | - name: agent.name 10 | type: keyword 11 | - name: agent.version 12 | type: keyword 13 | - name: agent.ephemeral_id 14 | type: keyword 15 | - name: event.duration 16 | type: long 17 | - name: event.ingested 18 | type: date 19 | - name: metricset.period 20 | type: long 21 | - name: kubernetes.container.rootfs.inodes.used 22 | type: integer 23 | - name: faults 24 | type: integer 25 | - name: Bytes 26 | type: long 27 | - name: Ip 28 | type: ip 29 | - name: rangeofid 30 | type: integer 31 | - name: PercentageMemory 32 | type: double 33 | - name: PercentageCPU 34 | type: double 35 | - name: usage 36 | type: group 37 | fields: 38 | - name: nanoseconds 39 | type: integer 40 | - name: nanocores 41 | type: integer 42 | - name: agent.snapshot 43 | type: boolean 44 | - name: container.name 45 | type: keyword -------------------------------------------------------------------------------- /assets/templates/kubernetes.container/schema-b/gotext.tpl: -------------------------------------------------------------------------------- 1 | {{- $period := generate "metricset.period" -}} 2 | {{- $timestamp := generate "timestamp" -}} 3 | {{- $fulltimestamp := $timestamp.Format "2006-01-02T15:04:05.999999Z07:00" -}} 4 | {{- $resttime := split ":" $fulltimestamp -}} 5 | {{- $picktimedate := generate "timedate" -}} 6 | {{- $timehour := generate "timehour" -}} 7 | {{- $agentId := generate "agent.id" -}} 8 | {{- $agentVersion := generate "agent.version" -}} 9 | {{- $agentName := generate "agent.name" -}} 10 | {{- $agentEphemeralid := generate "agent.ephemeral_id" -}} 11 | {{- $rangeofid := generate "rangeofid" -}} 12 | {{- $nodeid := div $rangeofid 110 -}} 13 | {{- $faults := generate "faults" -}} 14 | {{- $pctmem := generate "PercentageMemory" }} 15 | {{- $pctcpu := generate "PercentageCPU" }} 16 | {{- $usage_nanoseconds := generate "usage.nanoseconds" | mul 1000-}} 17 | {{- $usage_nanocores := generate "usage.nanocores" | mul 1000 -}} 18 | {{- $name := generate "container.name" -}} 19 | { "@timestamp": "{{$picktimedate}}T{{$timehour}}:{{ $resttime._1 }}:{{ $resttime._2 }}:{{ $resttime._3}}", "container":{ "memory":{ "usage": {{$pctmem}} }, "name":"{{ $name }}", "runtime":"containerd", "cpu":{ "usage": {{$pctcpu}} }, "id":"container-{{ $rangeofid }}" }, "kubernetes": { "container":{ "start_time":"{{$picktimedate}}T{{$timehour}}:{{ $resttime._1 }}:{{ $resttime._2 }}:{{ $resttime._3}}", "memory":{ "rss":{ "bytes": {{generate "Bytes"}} }, "majorpagefaults": {{ $faults }}, "usage":{ "node":{ "pct": {{$pctmem}} }, "bytes": {{generate "Bytes"}}, "limit":{ "pct": {{$pctmem}} } }, "available":{ "bytes": {{generate "Bytes"}} }, "workingset":{ "bytes": {{generate "Bytes"}}, "limit":{ "pct": {{$pctmem}} } }, "pagefaults": "{{ $faults }}" }, "rootfs":{ "inodes":{ "used": {{ generate "kubernetes.container.rootfs.inodes.used" }} }, "available":{ "bytes": {{generate "Bytes"}} }, "used":{ "bytes": {{generate "Bytes"}} }, "capacity":{ "bytes": {{generate "Bytes"}} } }, "name":"{{ $name }}", "cpu":{ "usage":{ "core":{ "ns": {{$usage_nanoseconds}} }, "node":{ "pct": {{$pctcpu}} }, "nanocores":{{$usage_nanocores}}, "limit":{ "pct": {{$pctcpu}} } } }, "logs":{ "inodes":{ "count": {{ generate "kubernetes.container.rootfs.inodes.used" }}, "used":5, "free": {{ generate "kubernetes.container.rootfs.inodes.used" }} }, "available":{ "bytes": {{generate "Bytes"}} }, "used":{ "bytes": {{generate "Bytes"}} }, "capacity":{ "bytes": {{generate "Bytes"}} } } }, "node":{ "uid": "host-{{ $nodeid }}" , "hostname":"host-{{ $nodeid }}", "name":"host-{{ $nodeid }}", "labels":{ "cloud_google_com/machine-family":"e2", "cloud_google_com/gke-nodepool":"kubernetes-scale-nl", "kubernetes_io/hostname":"host-{{ $nodeid }}", "cloud_google_com/gke-os-distribution":"cos", "topology_kubernetes_io/zone":"europe-west1-d", "topology_gke_io/zone":"europe-west1-d", "topology_kubernetes_io/region":"europe-west1", "kubernetes_io/arch":"amd64", "cloud_google_com/gke-cpu-scaling-level":"4", "env":"kubernetes-scale", "failure-domain_beta_kubernetes_io/region":"europe-west1", "cloud_google_com/gke-max-pods-per-node":"110", "cloud_google_com/gke-container-runtime":"containerd", "beta_kubernetes_io/instance-type":"e2-standard-4", "failure-domain_beta_kubernetes_io/zone":"europe-west1-d", "node_kubernetes_io/instance-type":"e2-standard-4", "beta_kubernetes_io/os":"linux", "cloud_google_com/gke-boot-disk":"pd-balanced", "kubernetes_io/os":"linux", "cloud_google_com/private-node":"false", "cloud_google_com/gke-logging-variant":"DEFAULT", "beta_kubernetes_io/arch":"amd64" } }, "pod":{ "uid": "demo-pod-{{ $rangeofid }}", "ip":"{{generate "Ip"}}", "name":"demo-pod-{{ $rangeofid }}", "namespace":"demo-{{ $rangeofid }}", "namespace_uid":"demo-{{ $rangeofid }}", "replicaset":{ "name":"demo-deployment-{{ $rangeofid }}" }, "namespace_labels":{ "kubernetes_io/metadata_name":"demo-{{ $rangeofid }}" }, "labels":{ "app":"demo", "pod-template-hash":"{{ $rangeofid }}", "app-2":"demo-2", "app-1":"demo-1" }, "deployment":{ "name":"demo-deployment-{{ $rangeofid }}" } } }, "cloud": { "provider": "gcp", "availability_zone": "europe-west1-d", "instance":{ "name": "{{ $agentName }}" , "id": "{{ $agentId }}" }, "machine":{ "type":"e2-standard-4" }, "service":{ "name":"GCE" }, "project":{ "id":"elastic-obs-integrations-dev" }, "account":{ "id":"elastic-obs-integrations-dev" } }, "orchestrator":{ "cluster":{ "name":"kubernetes-scale", "url":"https://{{ generate "Ip" }}" } }, "service":{ "address": "https://{{ $agentName }}:10250/stats/summary", "type":"kubernetes" }, "data_stream":{ "namespace":"default", "type":"metrics", "dataset":"kubernetes.container" }, "ecs": { "version": "8.2.0" }, "agent": { "id": "{{ $agentId}}", "name": "{{ $agentName }}" , "type": "metricbeat", "version": "{{ $agentVersion }}", "ephemeral_id": "{{ $agentEphemeralid }}" }, "elastic_agent": { "id": "{{ $agentId }}" , "version": "{{ $agentVersion }}", "snapshot": {{ generate "agent.snapshot" }} }, "metricset":{ "period": "{{ $period }}" , "name":"pod" }, "event":{ "duration": "{{generate "event.duration"}}", "agent_id_status": "verified", "ingested": "{{$picktimedate}}T{{$timehour}}:{{ $resttime._1 }}:{{ $resttime._2 }}:{{ $resttime._3}}", "module":"kubernetes", "dataset":"kubernetes.container" }, "host":{ "hostname":"host-{{ $nodeid }}", "os":{ "kernel":"5.10.161+", "codename":"focal", "name":"Ubuntu", "type":"linux", "family":"debian", "version":"20.04.5 LTS (Focal Fossa)", "platform":"ubuntu" }, "containerized":false, "name": "host-{{ $nodeid }}", "id": "host-{{ $nodeid }}", "architecture":"x86_64" } } -------------------------------------------------------------------------------- /assets/templates/kubernetes.container/schema-b/gotext_multiline.tpl: -------------------------------------------------------------------------------- 1 | {{- $period := generate "metricset.period" }} 2 | {{- $agentId := generate "agent.id" }} 3 | {{- $agentVersion := generate "agent.version" }} 4 | {{- $agentName := generate "agent.name" }} 5 | {{- $agentEphemeralid := generate "agent.ephemeral_id" }} 6 | {{- $timestamp := generate "timestamp" }} 7 | {{- $fulltimestamp := $timestamp.Format "2006-01-02T15:04:05.999999Z07:00" }} 8 | {{- $resttime := split ":" $fulltimestamp }} 9 | {{- $picktimedate := generate "timedate" }} 10 | {{- $timehour := generate "timehour" }} 11 | {{- $faults := generate "faults" }} 12 | {{- $pctmem := generate "PercentageMemory" }} 13 | {{- $pctcpu := generate "PercentageCPU" }} 14 | {{- $usage_nanoseconds := generate "usage.nanoseconds" | mul 1000 -}} 15 | {{- $usage_nanocores := generate "usage.nanocores" | mul 1000 -}} 16 | {{- $rangeofid := generate "rangeofid" -}} 17 | {{- $nodeid := div $rangeofid 110 -}} 18 | {{- $name := generate "container.name" }} 19 | { "@timestamp": "{{$picktimedate}}T{{$timehour}}:{{ $resttime._1 }}:{{ $resttime._2 }}:{{ $resttime._3}}", 20 | "container":{ 21 | "memory":{ 22 | "usage": {{$pctmem}} 23 | }, 24 | "name":"{{ $name }}", 25 | "runtime":"containerd", 26 | "cpu":{ 27 | "usage": {{$pctcpu}} 28 | }, 29 | "id":"container-{{ $rangeofid }}" 30 | }, 31 | "kubernetes": { 32 | "container":{ 33 | "start_time":"{{$picktimedate}}T{{$timehour}}:{{ $resttime._1 }}:{{ $resttime._2 }}:{{ $resttime._3}}", 34 | "memory":{ 35 | "rss":{ 36 | "bytes": {{generate "Bytes"}} 37 | }, 38 | "majorpagefaults": {{ $faults }}, 39 | "usage":{ 40 | "node":{ 41 | "pct": {{$pctmem}} 42 | }, 43 | "bytes": {{generate "Bytes"}}, 44 | "limit":{ 45 | "pct": {{$pctmem}} 46 | } 47 | }, 48 | "available":{ 49 | "bytes": {{generate "Bytes"}} 50 | }, 51 | "workingset":{ 52 | "bytes": {{generate "Bytes"}}, 53 | "limit":{ 54 | "pct": {{$pctmem}} 55 | } 56 | }, 57 | "pagefaults": "{{ $faults }}" 58 | }, 59 | "rootfs":{ 60 | "inodes":{ 61 | "used": {{ generate "kubernetes.container.rootfs.inodes.used" }} 62 | }, 63 | "available":{ 64 | "bytes": {{generate "Bytes"}} 65 | }, 66 | "used":{ 67 | "bytes": {{generate "Bytes"}} 68 | }, 69 | "capacity":{ 70 | "bytes": {{generate "Bytes"}} 71 | } 72 | }, 73 | "name":"{{ $name }}", 74 | "cpu":{ 75 | "usage":{ 76 | "core":{ 77 | "ns": {{$usage_nanoseconds}} 78 | }, 79 | "node":{ 80 | "pct": {{$pctcpu}} 81 | }, 82 | "nanocores":{{$usage_nanocores}}, 83 | "limit":{ 84 | "pct": {{$pctcpu}} 85 | } 86 | } 87 | }, 88 | "logs":{ 89 | "inodes":{ 90 | "count": {{ generate "kubernetes.container.rootfs.inodes.used" }}, 91 | "used":5, 92 | "free": {{ generate "kubernetes.container.rootfs.inodes.used" }} 93 | }, 94 | "available":{ 95 | "bytes": {{generate "Bytes"}} 96 | }, 97 | "used":{ 98 | "bytes": {{generate "Bytes"}} 99 | }, 100 | "capacity":{ 101 | "bytes": {{generate "Bytes"}} 102 | } 103 | } 104 | }, 105 | "node":{ 106 | "uid": "host-{{ $nodeid }}" , 107 | "hostname":"host-{{ $nodeid }}", 108 | "name":host-{{ $nodeid }}", 109 | "labels":{ 110 | "cloud_google_com/machine-family":"e2", 111 | "cloud_google_com/gke-nodepool":"kubernetes-scale-nl", 112 | "kubernetes_io/hostname":"host-{{ $nodeid }}", 113 | "cloud_google_com/gke-os-distribution":"cos", 114 | "topology_kubernetes_io/zone":"europe-west1-d", 115 | "topology_gke_io/zone":"europe-west1-d", 116 | "topology_kubernetes_io/region":"europe-west1", 117 | "kubernetes_io/arch":"amd64", 118 | "cloud_google_com/gke-cpu-scaling-level":"4", 119 | "env":"kubernetes-scale", 120 | "failure-domain_beta_kubernetes_io/region":"europe-west1", 121 | "cloud_google_com/gke-max-pods-per-node":"110", 122 | "cloud_google_com/gke-container-runtime":"containerd", 123 | "beta_kubernetes_io/instance-type":"e2-standard-4", 124 | "failure-domain_beta_kubernetes_io/zone":"europe-west1-d", 125 | "node_kubernetes_io/instance-type":"e2-standard-4", 126 | "beta_kubernetes_io/os":"linux", 127 | "cloud_google_com/gke-boot-disk":"pd-balanced", 128 | "kubernetes_io/os":"linux", 129 | "cloud_google_com/private-node":"false", 130 | "cloud_google_com/gke-logging-variant":"DEFAULT", 131 | "beta_kubernetes_io/arch":"amd64" 132 | } 133 | }, 134 | "pod":{ 135 | "uid": "demo-pod-{{ $rangeofid }}", 136 | "ip":"{{generate "Ip"}}", 137 | "name":"demo-pod-{{ $rangeofid }}", 138 | "namespace":"demo-{{ $rangeofid }}", 139 | "namespace_uid":"demo-{{ $rangeofid }}", 140 | "replicaset":{ 141 | "name":"demo-deployment-{{ $rangeofid }}" 142 | }, 143 | "namespace_labels":{ 144 | "kubernetes_io/metadata_name":"demo-{{ $rangeofid }}" 145 | }, 146 | "labels":{ 147 | "app":"demo", 148 | "pod-template-hash":"{{ $rangeofid }}", 149 | "app-2":"demo-2", 150 | "app-1":"demo-1" 151 | }, 152 | "deployment":{ 153 | "name":"demo-deployment-{{ $rangeofid }}" 154 | } 155 | }, 156 | "cloud": { 157 | "provider": "gcp", 158 | "availability_zone": "europe-west1-d", 159 | "instance":{ 160 | "name": "{{ $agentName }}" , 161 | "id": "{{ $agentId }}" 162 | }, 163 | "machine":{ 164 | "type":"e2-standard-4" 165 | }, 166 | "service":{ 167 | "name":"GCE" 168 | }, 169 | "project":{ 170 | "id":"elastic-obs-integrations-dev" 171 | }, 172 | "account":{ 173 | "id":"elastic-obs-integrations-dev" 174 | } 175 | }, 176 | "orchestrator":{ 177 | "cluster":{ 178 | "name":"kubernetes-scale", 179 | "url":"https://{{ generate "Ip" }}" 180 | } 181 | }, 182 | "service":{ 183 | "address": "https://{{ $agentName }}:10250/stats/summary", 184 | "type":"kubernetes" 185 | }, 186 | "data_stream":{ 187 | "namespace":"default", 188 | "type":"metrics", 189 | "dataset":"kubernetes.container" 190 | }, 191 | "ecs": { 192 | "version": "8.2.0" 193 | }, 194 | "agent": { 195 | "id": "{{ $agentId}}", 196 | "name": "{{ $agentName }}" , 197 | "type": "metricbeat", 198 | "version": "{{ $agentVersion }}", 199 | "ephemeral_id": "{{ $agentEphemeralid }}" 200 | }, 201 | "elastic_agent": { 202 | "id": "{{ $agentId }}" , 203 | "version": "{{ $agentVersion }}", 204 | "snapshot": {{ generate "agent.snapshot" }} 205 | }, 206 | "metricset":{ 207 | "period": "{{ $period }}" , 208 | "name":"pod" 209 | }, 210 | "event":{ 211 | "duration": "{{generate "event.duration"}}", 212 | "agent_id_status": "verified", 213 | "ingested": "{{$picktimedate}}T{{$timehour}}:{{ $resttime._1 }}:{{ $resttime._2 }}:{{ $resttime._3}}", 214 | "module":"kubernetes", 215 | "dataset":"kubernetes.container" 216 | }, 217 | "host":{ 218 | "hostname":"host-{{ $nodeid }}", 219 | "os":{ 220 | "kernel":"5.10.161+", 221 | "codename":"focal", 222 | "name":"Ubuntu", 223 | "type":"linux", 224 | "family":"debian", 225 | "version":"20.04.5 LTS (Focal Fossa)", 226 | "platform":"ubuntu" 227 | }, 228 | "containerized":false, 229 | "name": "host-{{ $nodeid }}", 230 | "id": "host-{{ $nodeid }}", 231 | "architecture":"x86_64" 232 | } 233 | } 234 | -------------------------------------------------------------------------------- /assets/templates/kubernetes.pod/schema-b/configs.yml: -------------------------------------------------------------------------------- 1 | fields: 2 | - name: cloud.availabilit_zone 3 | value: "europe-west1-d" 4 | - name: agent.id 5 | value: "12f376ef-5186-4e8b-a175-70f1140a8f30" 6 | - name: agent.name 7 | value: "kubernetes-scale-123456" 8 | - name: agent.ephemeral_id 9 | value: "f94220b0-2ca6-4809-8656-eb478a66c541" 10 | - name: agent.version 11 | value: "8.7.0" 12 | - name: agent.snasphost 13 | value: false 14 | - name: metricset.period 15 | value: 10000 16 | - name: event.duration 17 | range: 18 | min: 1 19 | max: 4000000 20 | - name: container.network.ingress.bytes 21 | range: 22 | min: 0 23 | max: 15000 24 | - name: container.network.egress.bytes 25 | range: 26 | min: 0 27 | max: 10000 28 | - name: Bytes 29 | range: 30 | min: 1 31 | max: 3000000 32 | - name: rangeofid 33 | range: 34 | min: 9000 35 | max: 10000 36 | - name: PercentageMemory 37 | range: 38 | min: 0.0 39 | max: 1.0 40 | fuzziness: 0.005 41 | - name: PercentageCPU 42 | range: 43 | min: 0.0 44 | max: 1.0 45 | fuzziness: 0.005 46 | - name: Nanocores 47 | range: 48 | min: 100000 49 | max: 9000000 50 | cardinality: 10000 51 | - name: timedate 52 | # Add more dates below in case you want your data to be spread. Those are fixed because this is required in the nightly tests of Rally 53 | enum: ["2023-05-15", "2023-05-16"] 54 | - name: timehour 55 | # Repeat or remove hours below to make data appear in specific hours. Below default enumaeration makes sure that data are spread throughout the 24 hours 56 | enum: ["00", "01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23" ] 57 | -------------------------------------------------------------------------------- /assets/templates/kubernetes.pod/schema-b/fields.yml: -------------------------------------------------------------------------------- 1 | - name: timestamp 2 | type: date 3 | - name: timedate 4 | type: keyword 5 | - name: timehour 6 | type: keyword 7 | - name: agent.id 8 | type: keyword 9 | - name: agent.name 10 | type: keyword 11 | - name: agent.version 12 | type: keyword 13 | - name: agent.ephemeral_id 14 | type: keyword 15 | - name: container.network.ingress.bytes 16 | type: integer 17 | - name: container.network.egress.bytes 18 | type: integer 19 | - name: event.duration 20 | type: long 21 | - name: event.ingested 22 | type: date 23 | - name: metricset.period 24 | type: long 25 | - name: Offset 26 | type: integer 27 | - name: Bytes 28 | type: long 29 | - name: rangeofid 30 | type: integer 31 | - name: Ip 32 | type: ip 33 | - name: PercentageMemory 34 | type: double 35 | - name: PercentageCPU 36 | type: double 37 | - name: Nanocores 38 | type: integer 39 | - name: agent.snapshot 40 | type: boolean -------------------------------------------------------------------------------- /assets/templates/kubernetes.pod/schema-b/gotext.tpl: -------------------------------------------------------------------------------- 1 | {{- $period := generate "metricset.period" -}} 2 | {{- $timestamp := generate "timestamp" -}} 3 | {{- $fulltimestamp := $timestamp.Format "2006-01-02T15:04:05.999999Z07:00" -}} 4 | {{- $resttime := split ":" $fulltimestamp -}} 5 | {{- $picktimedate := generate "timedate" -}} 6 | {{- $timehour := generate "timehour" -}} 7 | {{- $agentId := generate "agent.id" -}} 8 | {{- $agentVersion := generate "agent.version" -}} 9 | {{- $agentName := generate "agent.name" -}} 10 | {{- $agentEphemeralid := generate "agent.ephemeral_id" -}} 11 | {{- $rxbytes := generate "container.network.ingress.bytes" -}} 12 | {{- $txbytes := generate "container.network.egress.bytes" -}} 13 | {{- $rangeofid := generate "rangeofid" -}} 14 | {{- $nodeid := div $rangeofid 110 -}} 15 | {{- $pctmem := generate "PercentageMemory" }} 16 | {{- $pctcpu := generate "PercentageCPU" }} 17 | {{- $nanocores := generate "Nanocores" | mul 1000-}} 18 | {"@timestamp":"{{$picktimedate}}T{{$timehour}}:{{ $resttime._1 }}:{{ $resttime._2 }}:{{ $resttime._3}}", "container":{ "network":{ "ingress":{ "bytes": {{ $rxbytes }} }, "egress":{ "bytes": {{ $txbytes }} } } }, "kubernetes": { "node":{ "uid": "host-{{ $nodeid }}" , "hostname":"host-{{ $nodeid }}", "name":"host-{{ $nodeid }}", "labels":{ "cloud_google_com/machine-family":"e2", "cloud_google_com/gke-nodepool":"kubernetes-scale-nl", "kubernetes_io/hostname":"host-{{ $nodeid }}", "cloud_google_com/gke-os-distribution":"cos", "topology_kubernetes_io/zone":"europe-west1-d", "topology_gke_io/zone":"europe-west1-d", "topology_kubernetes_io/region":"europe-west1", "kubernetes_io/arch":"amd64", "cloud_google_com/gke-cpu-scaling-level":"4", "env":"kubernetes-scale", "failure-domain_beta_kubernetes_io/region":"europe-west1", "cloud_google_com/gke-max-pods-per-node":"110", "cloud_google_com/gke-container-runtime":"containerd", "beta_kubernetes_io/instance-type":"e2-standard-4", "failure-domain_beta_kubernetes_io/zone":"europe-west1-d", "node_kubernetes_io/instance-type":"e2-standard-4", "beta_kubernetes_io/os":"linux", "cloud_google_com/gke-boot-disk":"pd-balanced", "kubernetes_io/os":"linux", "cloud_google_com/private-node":"false", "cloud_google_com/gke-logging-variant":"DEFAULT", "beta_kubernetes_io/arch":"amd64" } }, "pod":{ "uid": "demo-pod-{{ $rangeofid }}", "start_time": "{{$picktimedate}}T{{$timehour}}:{{ $resttime._1 }}:{{ $resttime._2 }}:{{ $resttime._3}}", "memory":{ "rss":{ "bytes":"{{generate "Bytes"}}" }, "major_page_faults":0, "usage":{ "node":{ "pct": {{$pctmem}} }, "bytes": "{{generate "Bytes"}}", "limit":{ "pct":{{$pctmem}} } }, "available":{ "bytes":0 }, "page_faults":1386, "working_set":{ "bytes": "{{generate "Bytes"}}", "limit":{ "pct": {{$pctmem}} } } }, "ip":"{{generate "Ip"}}", "name":"demo-pod-{{ $rangeofid }}", "cpu":{ "usage":{ "node":{ "pct":{{$pctcpu}} }, "nanocores": {{$nanocores}}, "limit":{ "pct":{{$pctcpu}} } } }, "network":{ "tx":{ "bytes": {{ $txbytes }}, "errors":0 }, "rx":{ "bytes": {{ $rxbytes }}, "errors":0 } } }, "namespace":"demo-{{ $rangeofid }}", "namespace_uid":"demo-{{ $rangeofid }}", "replicaset":{ "name":"demo-deployment-{{ $rangeofid }}" }, "namespace_labels":{ "kubernetes_io/metadata_name":"demo-{{ $rangeofid }}" }, "labels":{ "app":"demo", "pod-template-hash":"{{ $rangeofid }}", "app-2":"demo-2", "app-1":"demo-1" }, "deployment":{ "name":"demo-deployment-{{ $rangeofid }}" } }, "cloud": { "provider": "gcp", "availability_zone": "europe-west1-d", "instance":{ "name": "{{ $agentName }}" , "id": "{{ $agentId }}" }, "machine":{ "type":"e2-standard-4" }, "service":{ "name":"GCE" }, "project":{ "id":"elastic-obs-integrations-dev" }, "account":{ "id":"elastic-obs-integrations-dev" } }, "orchestrator":{ "cluster":{ "name":"kubernetes-scale", "url":"https://{{generate "Ip"}}" } }, "service":{ "address": "https://{{ $agentName }}:10250/stats/summary", "type":"kubernetes" }, "data_stream":{ "namespace":"default", "type":"metrics", "dataset":"kubernetes.pod" }, "ecs": { "version": "8.2.0" }, "agent": { "id": "{{ $agentId}}", "name": "{{ $agentName }}" , "type": "metricbeat", "version": "{{ $agentVersion }}", "ephemeral_id": "{{ $agentEphemeralid }}" }, "elastic_agent": { "id": "{{ $agentId }}" , "version": "{{ $agentVersion }}", "snapshot": {{ generate "agent.snapshot" }} }, "metricset":{ "period": "{{ $period }}" , "name":"pod" }, "event":{ "duration": "{{generate "event.duration"}}", "agent_id_status": "verified", "ingested": "{{$picktimedate}}T{{$timehour}}:{{ $resttime._1 }}:{{ $resttime._2 }}:{{ $resttime._3}}", "module":"kubernetes", "dataset":"kubernetes.pod" }, "host":{ "hostname":"host-{{ $nodeid }}", "os":{ "kernel":"5.10.161+", "codename":"focal", "name":"Ubuntu", "type":"linux", "family":"debian", "version":"20.04.5 LTS (Focal Fossa)", "platform":"ubuntu" }, "containerized":false, "name": "host-{{ $nodeid }}", "id": "host-{{ $nodeid }}", "architecture":"x86_64"}} -------------------------------------------------------------------------------- /assets/templates/kubernetes.pod/schema-b/gotext_multiline.tpl: -------------------------------------------------------------------------------- 1 | {{- $period := generate "metricset.period" }} 2 | {{- $timestamp := generate "timestamp" }} 3 | {{- $fulltimestamp := $timestamp.Format "2006-01-02T15:04:05.999999Z07:00" }} 4 | {{- $resttime := split ":" $fulltimestamp }} 5 | {{- $picktimedate := generate "timedate" }} 6 | {{- $timehour := generate "timehour" }} 7 | {{- $agentId := generate "agent.id" }} 8 | {{- $agentVersion := generate "agent.version" }} 9 | {{- $agentName := generate "agent.name" }} 10 | {{- $agentEphemeralid := generate "agent.ephemeral_id" }} 11 | {{- $rxbytes := generate "container.network.ingress.bytes" }} 12 | {{- $txbytes := generate "container.network.egress.bytes" }} 13 | {{- $rangeofid := generate "rangeofid" }} 14 | {{- $nodeid := div $rangeofid 110 -}} 15 | {{- $pctmem := generate "PercentageMemory" }} 16 | {{- $pctcpu := generate "PercentageCPU" }} 17 | {{- $nanocores := generate "Nanocores" | mul 1000 -}} 18 | { "@timestamp": "{{$picktimedate}}T{{$timehour}}:{{ $resttime._1 }}:{{ $resttime._2 }}:{{ $resttime._3}}", 19 | "container":{ 20 | "network":{ 21 | "ingress":{ 22 | "bytes": {{ $rxbytes }} 23 | }, 24 | "egress":{ 25 | "bytes": {{ $txbytes }} 26 | } 27 | } 28 | }, 29 | "kubernetes": { 30 | "node":{ 31 | "uid":"host-{{ $nodeid }}" , 32 | "hostname":"host-{{ $nodeid }}", 33 | "name":"host-{{ $nodeid }}", 34 | "labels":{ 35 | "cloud_google_com/machine-family":"e2", 36 | "cloud_google_com/gke-nodepool":"kubernetes-scale-nl", 37 | "kubernetes_io/hostname":"host-{{ $nodeid }}", 38 | "cloud_google_com/gke-os-distribution":"cos", 39 | "topology_kubernetes_io/zone":"europe-west1-d", 40 | "topology_gke_io/zone":"europe-west1-d", 41 | "topology_kubernetes_io/region":"europe-west1", 42 | "kubernetes_io/arch":"amd64", 43 | "cloud_google_com/gke-cpu-scaling-level":"4", 44 | "env":"kubernetes-scale", 45 | "failure-domain_beta_kubernetes_io/region":"europe-west1", 46 | "cloud_google_com/gke-max-pods-per-node":"110", 47 | "cloud_google_com/gke-container-runtime":"containerd", 48 | "beta_kubernetes_io/instance-type":"e2-standard-4", 49 | "failure-domain_beta_kubernetes_io/zone":"europe-west1-d", 50 | "node_kubernetes_io/instance-type":"e2-standard-4", 51 | "beta_kubernetes_io/os":"linux", 52 | "cloud_google_com/gke-boot-disk":"pd-balanced", 53 | "kubernetes_io/os":"linux", 54 | "cloud_google_com/private-node":"false", 55 | "cloud_google_com/gke-logging-variant":"DEFAULT", 56 | "beta_kubernetes_io/arch":"amd64" 57 | } 58 | }, 59 | "pod":{ 60 | "uid": "demo-pod-{{ $rangeofid }}", 61 | "start_time": "{{$picktimedate}}T{{$timehour}}:{{ $resttime._1 }}:{{ $resttime._2 }}:{{ $resttime._3}}", 62 | "memory":{ 63 | "rss":{ 64 | "bytes":"{{generate "Bytes"}}" 65 | }, 66 | "major_page_faults":0, 67 | "usage":{ 68 | "node":{ 69 | "pct": {{$pctmem}} 70 | }, 71 | "bytes": "{{generate "Bytes"}}", 72 | "limit":{ 73 | "pct":{{$pctmem}} 74 | } 75 | }, 76 | "available":{ 77 | "bytes":0 78 | }, 79 | "page_faults":1386, 80 | "working_set":{ 81 | "bytes": "{{generate "Bytes"}}", 82 | "limit":{ 83 | "pct": {{$pctmem}} 84 | } 85 | } 86 | }, 87 | "ip":"{{generate "Ip"}}", 88 | "name":"demo-pod-{{ $rangeofid }}", 89 | "cpu":{ 90 | "usage":{ 91 | "node":{ 92 | "pct":{{$pctcpu}} 93 | }, 94 | "nanocores":{{$nanocores}}, 95 | "limit":{ 96 | "pct":{{$pctcpu}} 97 | } 98 | } 99 | }, 100 | "network":{ 101 | "tx":{ 102 | "bytes": {{ $txbytes }}, 103 | "errors":0 104 | }, 105 | "rx":{ 106 | "bytes": {{ $rxbytes }}, 107 | "errors":0 108 | } 109 | } 110 | }, 111 | "namespace":"demo-{{ $rangeofid }}", 112 | "namespace_uid":"demo-{{ $rangeofid }}", 113 | "replicaset":{ 114 | "name":"demo-deployment-{{ $rangeofid }}" 115 | }, 116 | "namespace_labels":{ 117 | "kubernetes_io/metadata_name":"demo-{{ $rangeofid }}" 118 | }, 119 | "labels":{ 120 | "app":"demo", 121 | "pod-template-hash":"{{ $rangeofid }}", 122 | "app-2":"demo-2", 123 | "app-1":"demo-1" 124 | }, 125 | "deployment":{ 126 | "name":"demo-deployment-{{ $rangeofid }}" 127 | } 128 | }, 129 | "cloud": { 130 | "provider": "gcp", 131 | "availability_zone": "europe-west1-d", 132 | "instance":{ 133 | "name": "{{ $agentName }}" , 134 | "id": "{{ $agentId }}" 135 | }, 136 | "machine":{ 137 | "type":"e2-standard-4" 138 | }, 139 | "service":{ 140 | "name":"GCE" 141 | }, 142 | "project":{ 143 | "id":"elastic-obs-integrations-dev" 144 | }, 145 | "account":{ 146 | "id":"elastic-obs-integrations-dev" 147 | } 148 | }, 149 | "orchestrator":{ 150 | "cluster":{ 151 | "name":"kubernetes-scale", 152 | "url":"https://{{generate "Ip"}}" 153 | } 154 | }, 155 | "service":{ 156 | "address": "https://{{ $agentName }}:10250/stats/summary", 157 | "type":"kubernetes" 158 | }, 159 | "data_stream":{ 160 | "namespace":"default", 161 | "type":"metrics", 162 | "dataset":"kubernetes.pod" 163 | }, 164 | "ecs": { 165 | "version": "8.2.0" 166 | }, 167 | "agent": { 168 | "id": "{{ $agentId}}", 169 | "name": "{{ $agentName }}" , 170 | "type": "metricbeat", 171 | "version": "{{ $agentVersion }}", 172 | "ephemeral_id": "{{ $agentEphemeralid }}" 173 | }, 174 | "elastic_agent": { 175 | "id": "{{ $agentId }}" , 176 | "version": "{{ $agentVersion }}", 177 | "snapshot": {{ generate "agent.snapshot" }} 178 | }, 179 | "metricset":{ 180 | "period": "{{ $period }}" , 181 | "name":"pod" 182 | }, 183 | "event":{ 184 | "duration": "{{generate "event.duration"}}", 185 | "agent_id_status": "verified", 186 | "ingested": "{{$picktimedate}}T{{$timehour}}:{{ $resttime._1 }}:{{ $resttime._2 }}:{{ $resttime._3}}", 187 | "module":"kubernetes", 188 | "dataset":"kubernetes.pod" 189 | }, 190 | "host":{ 191 | "hostname":"host-{{ $nodeid }}", 192 | "os":{ 193 | "kernel":"5.10.161+", 194 | "codename":"focal", 195 | "name":"Ubuntu", 196 | "type":"linux", 197 | "family":"debian", 198 | "version":"20.04.5 LTS (Focal Fossa)", 199 | "platform":"ubuntu" 200 | }, 201 | "containerized":false, 202 | "name": "host-{{ $nodeid }}", 203 | "id": "host-{{ $nodeid }}", 204 | "architecture":"x86_64" 205 | } 206 | } -------------------------------------------------------------------------------- /cmd/generate.go: -------------------------------------------------------------------------------- 1 | // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 2 | // or more contributor license agreements. Licensed under the Elastic License 2.0; 3 | // you may not use this file except in compliance with the Elastic License 2.0. 4 | 5 | package cmd 6 | 7 | import ( 8 | "errors" 9 | "fmt" 10 | "github.com/elastic/elastic-integration-corpus-generator-tool/internal/corpus" 11 | "github.com/elastic/elastic-integration-corpus-generator-tool/pkg/genlib/config" 12 | "github.com/spf13/afero" 13 | "github.com/spf13/cobra" 14 | "github.com/spf13/viper" 15 | "go.uber.org/multierr" 16 | ) 17 | 18 | var integrationPackage string 19 | var dataStream string 20 | var packageVersion string 21 | 22 | func GenerateCmd() *cobra.Command { 23 | generateCmd := &cobra.Command{ 24 | Use: "generate integration data_stream version", 25 | Short: "Generate a corpus", 26 | Long: "Generate a bulk request corpus for a given integration data stream downloaded from a package registry", 27 | Args: func(cmd *cobra.Command, args []string) error { 28 | var errs []error 29 | if len(args) != 3 { 30 | return errors.New("you must pass the integration package the data stream and the package vesion") 31 | } 32 | 33 | if packageRegistryBaseURL == "" { 34 | errs = append(errs, errors.New("you must provide a not empty --package-registry-base-url flag value")) 35 | } 36 | 37 | integrationPackage = args[0] 38 | if integrationPackage == "" { 39 | errs = append(errs, errors.New("you must provide a not empty integration argument")) 40 | } 41 | 42 | dataStream = args[1] 43 | if dataStream == "" { 44 | errs = append(errs, errors.New("you must provide a not empty data stream argument")) 45 | } 46 | 47 | packageVersion = args[2] 48 | if packageVersion == "" { 49 | errs = append(errs, errors.New("you must provide a not empty package version argument")) 50 | } 51 | 52 | if len(errs) > 0 { 53 | return multierr.Combine(errs...) 54 | } 55 | 56 | return nil 57 | }, 58 | RunE: func(cmd *cobra.Command, args []string) error { 59 | fs := afero.NewOsFs() 60 | location := viper.GetString("corpora_location") 61 | 62 | cfg, err := config.LoadConfig(fs, configFile) 63 | if err != nil { 64 | return err 65 | } 66 | 67 | fc, err := corpus.NewGenerator(cfg, fs, location) 68 | if err != nil { 69 | return err 70 | } 71 | 72 | timeNow, err := getTimeNowFromFlag(timeNowAsString) 73 | if err != nil { 74 | return err 75 | } 76 | 77 | payloadFilename, err := fc.Generate(packageRegistryBaseURL, integrationPackage, dataStream, packageVersion, totEvents, timeNow, randSeed) 78 | if err != nil { 79 | return err 80 | } 81 | 82 | fmt.Println("File generated:", payloadFilename) 83 | 84 | return nil 85 | }, 86 | } 87 | 88 | generateCmd.Flags().StringVarP(&packageRegistryBaseURL, "package-registry-base-url", "r", "https://epr.elastic.co/", "base url of the package registry with schema") 89 | generateCmd.Flags().StringVarP(&configFile, "config-file", "c", "", "path to config file for generator settings") 90 | generateCmd.Flags().Uint64VarP(&totEvents, "tot-events", "t", 1, "total events of the corpus to generate") 91 | generateCmd.Flags().StringVarP(&timeNowAsString, "now", "n", "", "time to use for generation based on now (`date` type)") 92 | generateCmd.Flags().Int64VarP(&randSeed, "seed", "s", 1, "seed to set as source of rand") 93 | 94 | return generateCmd 95 | } 96 | -------------------------------------------------------------------------------- /cmd/generate_common.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/elastic/elastic-integration-corpus-generator-tool/pkg/genlib" 8 | ) 9 | 10 | var packageRegistryBaseURL string 11 | var configFile string 12 | var totEvents uint64 13 | var timeNowAsString string 14 | var randSeed int64 15 | 16 | func getTimeNowFromFlag(timeNowAsString string) (time.Time, error) { 17 | if len(timeNowAsString) > 0 { 18 | if timeNow, err := time.Parse(genlib.FieldTypeTimeLayout, timeNowAsString); err != nil { 19 | return timeNow, fmt.Errorf("wrong --now flag: %s (%w)", timeNowAsString, err) 20 | } else { 21 | return timeNow, nil 22 | } 23 | } 24 | 25 | return time.Now(), nil 26 | } 27 | -------------------------------------------------------------------------------- /cmd/generate_with_template.go: -------------------------------------------------------------------------------- 1 | // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 2 | // or more contributor license agreements. Licensed under the Elastic License 2.0; 3 | // you may not use this file except in compliance with the Elastic License 2.0. 4 | 5 | package cmd 6 | 7 | import ( 8 | "errors" 9 | "fmt" 10 | "github.com/elastic/elastic-integration-corpus-generator-tool/internal/corpus" 11 | "github.com/elastic/elastic-integration-corpus-generator-tool/pkg/genlib/config" 12 | "github.com/spf13/afero" 13 | "github.com/spf13/cobra" 14 | "github.com/spf13/viper" 15 | "go.uber.org/multierr" 16 | ) 17 | 18 | var templateType string 19 | 20 | var templatePath string 21 | var fieldsDefinitionPath string 22 | 23 | func GenerateWithTemplateCmd() *cobra.Command { 24 | generateWithTemplateCmd := &cobra.Command{ 25 | Use: "generate-with-template template-path fields-definition-path", 26 | Short: "Generate a corpus", 27 | Long: "Generate a bulk request corpus given a template path and a fields definition path", 28 | Args: func(cmd *cobra.Command, args []string) error { 29 | var errs []error 30 | if len(args) != 2 { 31 | return errors.New("you must pass the template path and the fields definition path") 32 | } 33 | 34 | templatePath = args[0] 35 | if templatePath == "" { 36 | errs = append(errs, errors.New("you must provide a not empty template path argument")) 37 | } 38 | 39 | fieldsDefinitionPath = args[1] 40 | if fieldsDefinitionPath == "" { 41 | errs = append(errs, errors.New("you must provide a not empty fields definition path argument")) 42 | } 43 | 44 | if len(errs) > 0 { 45 | return multierr.Combine(errs...) 46 | } 47 | 48 | return nil 49 | }, 50 | RunE: func(cmd *cobra.Command, args []string) error { 51 | fs := afero.NewOsFs() 52 | location := viper.GetString("corpora_location") 53 | 54 | cfg, err := config.LoadConfig(fs, configFile) 55 | if err != nil { 56 | return err 57 | } 58 | 59 | fc, err := corpus.NewGeneratorWithTemplate(cfg, fs, location, templateType) 60 | if err != nil { 61 | return err 62 | } 63 | 64 | timeNow, err := getTimeNowFromFlag(timeNowAsString) 65 | if err != nil { 66 | return err 67 | } 68 | 69 | payloadFilename, err := fc.GenerateWithTemplate(templatePath, fieldsDefinitionPath, totEvents, timeNow, randSeed) 70 | if err != nil { 71 | return err 72 | } 73 | 74 | fmt.Println("File generated:", payloadFilename) 75 | 76 | return nil 77 | }, 78 | } 79 | 80 | generateWithTemplateCmd.Flags().StringVarP(&configFile, "config-file", "c", "", "path to config file for generator settings") 81 | generateWithTemplateCmd.Flags().StringVarP(&templateType, "template-type", "y", "placeholder", "either 'placeholder' or 'gotext'") 82 | generateWithTemplateCmd.Flags().Uint64VarP(&totEvents, "tot-events", "t", 1, "total events of the corpus to generate") 83 | generateWithTemplateCmd.Flags().StringVarP(&timeNowAsString, "now", "n", "", "time to use for generation based on now (`date` type)") 84 | generateWithTemplateCmd.Flags().Int64VarP(&randSeed, "seed", "s", 1, "seed to set as source of rand") 85 | 86 | return generateWithTemplateCmd 87 | } 88 | -------------------------------------------------------------------------------- /cmd/local-template.go: -------------------------------------------------------------------------------- 1 | // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 2 | // or more contributor license agreements. Licensed under the Elastic License 2.0; 3 | // you may not use this file except in compliance with the Elastic License 2.0. 4 | 5 | package cmd 6 | 7 | import ( 8 | "errors" 9 | "fmt" 10 | "log" 11 | "os" 12 | "path/filepath" 13 | 14 | "github.com/elastic/elastic-integration-corpus-generator-tool/internal/corpus" 15 | "github.com/elastic/elastic-integration-corpus-generator-tool/pkg/genlib/config" 16 | "github.com/spf13/afero" 17 | "github.com/spf13/cobra" 18 | "github.com/spf13/viper" 19 | "go.uber.org/multierr" 20 | ) 21 | 22 | var flagSchema string 23 | 24 | func TemplateCmd() *cobra.Command { 25 | command := &cobra.Command{ 26 | Use: "local-template package dataset", 27 | Example: "local-template aws billing", 28 | Short: "Generate a corpus from a local template", 29 | Long: "Generate a bulk request corpus for the specified package dataset in the assets/templates folder", 30 | Args: func(cmd *cobra.Command, args []string) error { 31 | if len(args) != 2 { 32 | return errors.New("package and dataset arguments are required") 33 | } 34 | 35 | datasetFolder := filepath.Join("assets", "templates", fmt.Sprintf("%s.%s", args[0], args[1])) 36 | if _, err := os.Stat(datasetFolder); errors.Is(err, os.ErrNotExist) { 37 | return errors.New(fmt.Sprintf("dataset folder %s does not exists", datasetFolder)) 38 | } 39 | 40 | return nil 41 | }, 42 | RunE: func(cmd *cobra.Command, args []string) error { 43 | fs := afero.NewOsFs() 44 | location := viper.GetString("corpora_location") 45 | 46 | cfg, err := config.LoadConfig(fs, configFile) 47 | if err != nil { 48 | return err 49 | } 50 | 51 | var errs []error 52 | datasetFolder := fmt.Sprintf("%s.%s", args[0], args[1]) 53 | schema := fmt.Sprintf("schema-%s", flagSchema) 54 | datasetFolderPath := filepath.Join("assets", "templates", datasetFolder, schema) 55 | 56 | templateFile := fmt.Sprintf("%s.tpl", templateType) 57 | templatePath := filepath.Join(datasetFolderPath, templateFile) 58 | if _, err := os.Stat(templatePath); errors.Is(err, os.ErrNotExist) { 59 | errs = append(errs, errors.New(fmt.Sprintf("template file %s does not exist", templatePath))) 60 | } 61 | 62 | fieldsDefinitionFile := "fields.yml" 63 | fieldsDefinitionPath := filepath.Join(datasetFolderPath, fieldsDefinitionFile) 64 | if _, err := os.Stat(templatePath); errors.Is(err, os.ErrNotExist) { 65 | errs = append(errs, errors.New(fmt.Sprintf("fields definition file %s does not exist", fieldsDefinitionPath))) 66 | } 67 | 68 | fieldsConfigFile := "configs.yml" 69 | fieldsConfigFilePath := filepath.Join(datasetFolderPath, fieldsConfigFile) 70 | if _, err := os.Stat(fieldsConfigFilePath); errors.Is(err, os.ErrNotExist) { 71 | log.Printf("fields config file %s does not exist", fieldsConfigFilePath) 72 | } 73 | 74 | if len(errs) > 0 { 75 | return multierr.Combine(errs...) 76 | } 77 | 78 | fc, err := corpus.NewGeneratorWithTemplate(cfg, afero.NewOsFs(), location, templateType) 79 | if err != nil { 80 | return err 81 | } 82 | 83 | timeNow, err := getTimeNowFromFlag(timeNowAsString) 84 | if err != nil { 85 | return err 86 | } 87 | 88 | payloadFilename, err := fc.GenerateWithTemplate(templatePath, fieldsDefinitionPath, totEvents, timeNow, randSeed) 89 | if err != nil { 90 | return err 91 | } 92 | 93 | fmt.Println("File generated:", payloadFilename) 94 | 95 | return nil 96 | }, 97 | } 98 | 99 | command.Flags().StringVarP(&configFile, "config-file", "c", "", "path to config file for generator settings") 100 | command.Flags().StringVarP(&templateType, "engine", "e", "gotext", "either 'placeholder' or 'gotext'") 101 | command.Flags().Uint64VarP(&totEvents, "tot-events", "t", 1, "total events of the corpus to generate") 102 | command.Flags().StringVarP(&flagSchema, "schema", "", "b", "schema to generate data for; valid values: a, b") 103 | command.Flags().StringVarP(&timeNowAsString, "now", "n", "", "time to use for generation based on now (`date` type)") 104 | 105 | command.Flags().Int64VarP(&randSeed, "seed", "s", 1, "seed to set as source of rand") 106 | return command 107 | } 108 | -------------------------------------------------------------------------------- /cmd/root.go: -------------------------------------------------------------------------------- 1 | // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 2 | // or more contributor license agreements. Licensed under the Elastic License 2.0; 3 | // you may not use this file except in compliance with the Elastic License 2.0. 4 | 5 | package cmd 6 | 7 | import ( 8 | "github.com/spf13/cobra" 9 | ) 10 | 11 | // RootCmd creates and returns root cmd for elastic-integration-corpus-generator-tool. 12 | func RootCmd() *cobra.Command { 13 | 14 | rootCmd := &cobra.Command{ 15 | Use: "elastic-integration-corpus-generator-tool", 16 | Long: "elastic-integration-corpus-generator-tool - Command line tool used for generating events corpus dynamically given a specific integration.", 17 | SilenceUsage: true, 18 | } 19 | 20 | return rootCmd 21 | } 22 | -------------------------------------------------------------------------------- /cmd/root_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 2 | // or more contributor license agreements. Licensed under the Elastic License 2.0; 3 | // you may not use this file except in compliance with the Elastic License 2.0. 4 | 5 | package cmd 6 | 7 | import ( 8 | "testing" 9 | ) 10 | 11 | func TestRootCmd(t *testing.T) { 12 | cmd := RootCmd() 13 | 14 | err := cmd.Execute() 15 | if err != nil { 16 | t.Fatal(err) 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /cmd/version.go: -------------------------------------------------------------------------------- 1 | // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 2 | // or more contributor license agreements. Licensed under the Elastic License 2.0; 3 | // you may not use this file except in compliance with the Elastic License 2.0. 4 | 5 | package cmd 6 | 7 | import ( 8 | "fmt" 9 | "strings" 10 | 11 | "github.com/spf13/cobra" 12 | 13 | "github.com/elastic/elastic-integration-corpus-generator-tool/internal/version" 14 | ) 15 | 16 | const versionLongDescription = `Use this command to print the version of elastic-integration-corpus-generator-tool that you have installed. This is especially useful when reporting bugs.` 17 | 18 | func VersionCmd() *cobra.Command { 19 | cmd := &cobra.Command{ 20 | Use: "version", 21 | Short: "Show application version", 22 | Long: versionLongDescription, 23 | RunE: func(cmd *cobra.Command, args []string) error { 24 | var sb strings.Builder 25 | sb.WriteString("elastic-integration-corpus-generator-tool ") 26 | if version.Tag != "" { 27 | sb.WriteString(version.Tag) 28 | sb.WriteString(" ") 29 | } else { 30 | sb.WriteString("devel ") 31 | } 32 | sb.WriteString(fmt.Sprintf("version-hash %s ", version.CommitHash)) 33 | sb.WriteString(fmt.Sprintf("(source date: %s)", version.SourceTimeFormatted())) 34 | 35 | // NOTE: allow replacing stdout for testing 36 | fmt.Fprint(cmd.OutOrStdout(), sb.String()) 37 | 38 | return nil 39 | }, 40 | } 41 | 42 | return cmd 43 | } 44 | -------------------------------------------------------------------------------- /cmd/version_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 2 | // or more contributor license agreements. Licensed under the Elastic License 2.0; 3 | // you may not use this file except in compliance with the Elastic License 2.0. 4 | 5 | package cmd_test 6 | 7 | import ( 8 | "bytes" 9 | "testing" 10 | 11 | "github.com/elastic/elastic-integration-corpus-generator-tool/cmd" 12 | "github.com/elastic/elastic-integration-corpus-generator-tool/internal/version" 13 | "github.com/stretchr/testify/require" 14 | ) 15 | 16 | func TestVersionCmd_default(t *testing.T) { 17 | cmd := cmd.VersionCmd() 18 | 19 | b := new(bytes.Buffer) 20 | cmd.SetOut(b) 21 | 22 | err := cmd.Execute() 23 | require.Nil(t, err) 24 | 25 | const expected = "elastic-integration-corpus-generator-tool devel version-hash undefined (source date: unknown)" 26 | require.Equal(t, expected, b.String()) 27 | } 28 | 29 | func TestVersionCmd_withValues(t *testing.T) { 30 | cmd := cmd.VersionCmd() 31 | 32 | b := new(bytes.Buffer) 33 | cmd.SetOut(b) 34 | 35 | version.Tag = "v0.1.0" 36 | version.SourceDateEpoch = "1648570012" 37 | version.CommitHash = "5561aef" 38 | 39 | err := cmd.Execute() 40 | require.Nil(t, err) 41 | 42 | const expected = "elastic-integration-corpus-generator-tool v0.1.0 version-hash 5561aef (source date: 2022-03-29T16:06:52Z)" 43 | require.Equal(t, expected, b.String()) 44 | } 45 | -------------------------------------------------------------------------------- /docs/cardinality.md: -------------------------------------------------------------------------------- 1 | # Cardinality 2 | 3 | The cardinality of a field refers to the number of distinct values that it can have. 4 | 5 | For example: a `boolean` will have cardinality of 2, an `integer` 4294967295, a `version` field may have a cardinality of some dozens. 6 | 7 | Low cardinality fields, like `boolean` or `version` in the example above do not pose a particular issue when observing your system. 8 | This is the opposite for high cardinality fields, which due to their size create challenges in indexing, searching and visualising them. 9 | 10 | We refer to **high cardinality** when fields cardinality is in the order of hundreds of thousands or millions. 11 | 12 | Example of these values may be: request IDs, trace IDs, value of tags attached to compute instances (es a tag with 20 distinct values in a 5000 instances fleet). 13 | 14 | These fields are extremely valuable as they allow to fine grain your search. From a testing point of view they allow to stress test a system. 15 | 16 | ## Fields generation configuration 17 | 18 | For these reasons one of the goals for this tool is to be able to generate high cardinality fields. An additional complexity we face is to generate plausible cardinality. 19 | 20 | Let's make an exmaple: we manage a fleet of 1000 Kubernetes nodes. Each node hosts 100 pods. Pods in Kubernetes are within a namespace. Let's say we want to test the use case of few namespaces with thousands of pods (i.e. 1:1000 ration). This is a valid scenario, but we may be interested in another use case: namespaces containing very few pods (i.e. 1:10 ratio). 21 | 22 | To support generating dataset for both uses cases, is possible to specify a `cardinality` parameter in the field generation configuration file to tweak generated data. 23 | See [Fields generation configuration](./fields-configuration.md). 24 | 25 | -------------------------------------------------------------------------------- /docs/cli-help.md: -------------------------------------------------------------------------------- 1 | # Generic help 2 | 3 | Run `elastic-corpus-generator-tool --help`. 4 | 5 | If you are using the source code, replace executable name with `go run main.go`. 6 | 7 | # Specific command help 8 | 9 | Run `elastic-corpus-generator-tool --help`. 10 | 11 | If you are using the source code, replace executable name with `go run main.go`. 12 | 13 | # HELP! 14 | 15 | Didn't you find what you where searching for? Open an issue or a discussion in this repo. 16 | -------------------------------------------------------------------------------- /docs/data-schemas.md: -------------------------------------------------------------------------------- 1 | # Data schemas 2 | 3 | In order to understand how generating sample data for Elastic integrations works you need to understand what kind of data you may be interested in. 4 | 5 | ## What data schemas are there? 6 | 7 | When collecting data with Elastic Agent and shipping to Elasticsearch, there are 4 different data schemas. This is important as the data schemas look different and we must align during generation on what data schemas we are talking about. In the diagram below, the schemas A, B, C and D are shown: 8 | 9 | ```mermaid 10 | flowchart LR 11 | D[Data/Endpoint] 12 | EA[Elastic Agent] 13 | ES[Elasticearch] 14 | IP[Ingest Pipeline] 15 | D -->|Schema A| EA -->|Schema B| IP -->|Schema C| ES -->|Schema D| Query; 16 | ``` 17 | 18 | * **Schema A**: This is the schema the Elastic Agent collects. It could be a line in a log file, response of an http request, syslog event etc. The Elastic Agent input knows how to handle this structure. 19 | * **Schema B**: This is the schema the Elastic Agent ships to Elasticsearch (to the ingest pipeline). This is a JSON document which contains all the processing the Elastic Agent did on schema A. For example the content of the log line is the in the message field and meta information around the host was added to the event. 20 | * **Schema C**: In case an ingest pipeline exists, the ingest pipeline converts schema B to schema C. This can be taking apart a log message with grok or enriching data with geoip. If there is no ingest pipeline, schema B and C are equal. 21 | * **Schema D**: This is the schema users write queries on in Elasticsearch. Schema C can be different from D in the scenario of runtime fields, otherwise C and D are equal. 22 | 23 | Schema C is the one that is defined in integration packages in the `fields.yml` files for each data stream. 24 | 25 | ## What data schemas are we interested in? 26 | 27 | Schema A, as it removes the need for a working infrastructure to pull data from, simplifying testing setups. 28 | 29 | Schema B, as it removes the need for a running Elastic Agent to process data from source, allowing to test ingest pipelines. Another goal for Schema B generation is to generate [rally tracks](https://github.com/elastic/rally-tracks) that can be used for performance testing. 30 | -------------------------------------------------------------------------------- /docs/dimensionality.md: -------------------------------------------------------------------------------- 1 | # Dimensionality 2 | 3 | The dimensionality of a dataset is the number of fields that it has. **High dimensionality** datasets have many attributes. 4 | 5 | Dimensionality plays a role in `array` and `object` type fields. 6 | 7 | Dimensionality is valuable to test storage consumption, in particular for metrics, where the data point itself is small compared to the metadata enriching it. 8 | 9 | At the moment there is limited support for configuring dimensionality. When a field of type `object` is using `object_keys` is possible to configure the specific keys within the object itself. 10 | Note that the `name: object_keys.*` configurations are not mandatory and if missing the field will be treated as having an empty config. 11 | 12 | All unconfigured keys will be randomly generated. 13 | 14 | For example: 15 | 16 | ``` 17 | - name: object_field 18 | object_keys: 19 | - a_key 20 | - another_key 21 | - name: object_keys.a_key 22 | enum: ["a_value", "another_value"] 23 | - name: object_keys.another_key 24 | cardinality: 2 25 | ``` 26 | -------------------------------------------------------------------------------- /docs/fields-configuration.md: -------------------------------------------------------------------------------- 1 | # Fields generation configuration 2 | 3 | Fields generation configuration are applied to fields defined in Fields definition file to tweak how the data is generated. 4 | 5 | They must be added to a file named `configs.yml` in the assets template folder of a data stream. 6 | 7 | ## Config entries definition 8 | 9 | The config file is a yaml file consisting of root level `fields` object that's an array of config entry. 10 | 11 | For each config entry the following fields are available: 12 | - `name` *mandatory*: dotted path field, matching an entry in [Fields definition](./glossary.md#fields-definition) 13 | - `fuzziness` *optional (`long` and `double` type only)*: when generating data you could want generated values to change in a known interval. Fuzziness allow to specify the maximum delta a generated value can have from the previous value (for the same field), as a delta percentage that will be applied below and above the previous value; value must be between 0.0 and 1.0, where 0 is 0% and 1 is 100%. When not specified there is no constraint on the generated values, boundaries will be defined by the underlying field type. For example, `fuzziness: 0.1`, assuming a `double` field type and with first value generated `10.`, will generate the second value in the range between `9.` and `11.`. Assuming the second value generated will be `10.5`, the third one will be generated in the range between `9.45` and `11.55`, and so on. 14 | - `range` *optional (`long` and `double` type only)*: value will be generated between `min` and `max`. If `fuzziness` is defined, the value will be generated within a delta defined by `fuzziness` from the previous value. In any case (`fuzziness` or not) the value would not escape the `min`/`max` bounds. 15 | - `range` *optional (`date` type only)*: value will be generated between `from` and `to`. Only one between `from` and `to` can be set, in this case the dates will be generated between `from`/`to` and `time.Now()`. Progressive order of the generated dates is always assured regardless the interval involving `from`, `to` and `time.Now()` is positive or negative. If both at least one of `from` or `to` and `period` settings are defined an error will be returned and the generator will stop. The format of the date must be parsable by the following golang date format: `2006-01-02T15:04:05.999999999-07:00`. 16 | - `cardinality` *optional*: exact number of different values to generate for the field; note that this setting may not be respected if not enough events are generated. For example, `cardinality: 1000` with `100` generated events would produce `100` different values, not `1000`. Similarly, the setting may not be respected if other settings prevents it. For example, `cardinality: 10` with an `enum` list of only 5 strings would produce `5` different values, not `10`. Or `cardinality: 10` for a `long` with `range.min: 1` and `range.max: 5` would produce `5` different values, not `10`. 17 | - `counter` *optional (`long` and `double` type only)*: if set to `true` values will be generated only ever-increasing. If `fuzziness` is not defined, the positive delta from the previous value will be totally random and unbounded. For example, assuming `counter: true`, assuming a `int` field type and with first value generated `10.`, will generate the second value with any random value greater than `10`, like `11` or `987615243`. If `fuzziness` is defined, the value will be generated within a positive delta defined by `fuzziness` from the previous value. For example, `fuzziness: 0.1`, assuming `counter: true` , assuming a `double` field type and with first value generated `10.`, will generate the second value in the range between `10.` and `11.`. Assuming the second value generated will be `10.5`, the third one will be generated in the range between `10.5` and `11.55`, and so on. If both `counter: true` and at least one of `range.min` or `range.max` settings are defined an error will be returned and the generator will stop. 18 | - `counter_reset` *optional (only applicable when `counter: true`)*: configures how and when the counter should reset. It has the following sub-fields: 19 | - `strategy` *mandatory*: defines the reset strategy. Possible values are: 20 | - `"random"`: resets the counter at random intervals. 21 | - `"probabilistic"`: resets the counter based on a probability. 22 | - `"after_n"`: resets the counter after a specific number of iterations. 23 | - `probability` *required when strategy is "probabilistic"*: an integer between 1 and 100 representing the percentage chance of reset for each generated value. 24 | - `reset_after_n` *required when strategy is "after_n"*: an integer specifying the number of values to generate before resetting the counter. 25 | 26 | Note: The `counter_reset` configuration is only applicable when `counter` is set to `true`. 27 | - `period` *optional (`date` type only)*: values will be evenly generated between `time.Now()` and `time.Now().Add(period)`, where period is expressed as `time.Duration`. It accepts also a negative duration: in this case values will be evenly generated between `time.Now().Add(period)` and `time.Now()`. If both `period` and at least one of `range.from` or `range.to` settings are defined an error will be returned and the generator will stop. 28 | - `object_keys` *optional (`object` type only)*: list of field names to generate in a object field type; if not specified a random number of field names will be generated in the object filed type 29 | - `value` *optional*: hardcoded value to set for the field (any `cardinality` will be ignored) 30 | - `enum` *optional (`keyword` type only)*: list of strings to randomly chose from a value to set for the field (any `cardinality` will be applied limited to the size of the `enum` values) 31 | 32 | If you have an `object` type field that you defined one or multiple `object_keys` for, you can reference them as a root level field with their own customisation. Beware that if a `cardinality` is set for the `object` type field, cardinality will be ignored for the children `object_keys` fields. 33 | 34 | ## Example configuration 35 | 36 | ```yaml 37 | fields: 38 | - name: timestamp 39 | period: "1h" 40 | - name: lastSnapshot 41 | range: 42 | from: "2023-11-23T11:29:48-00:00" 43 | to: "2023-12-13T01:39:58-00:00" 44 | - name: aws.dynamodb.metrics.AccountMaxReads.max 45 | fuzziness: 0.1 46 | range: 47 | min: 0 48 | max: 100 49 | - name: aws.dynamodb.metrics.AccountMaxTableLevelReads.max 50 | fuzziness: 0.05 51 | range: 52 | min: 0 53 | max: 50 54 | cardinality: 20 55 | - name: aws.dynamodb.metrics.AccountProvisionedReadCapacityUtilization.avg 56 | fuzziness: 0.1 57 | - name: aws.cloudwatch.namespace 58 | cardinality: 1000 59 | - name: aws.dimensions.* 60 | object_keys: 61 | - TableName 62 | - Operation 63 | - name: data_stream.type 64 | value: metrics 65 | - name: data_stream.dataset 66 | value: aws.dynamodb 67 | - name: data_stream.namespace 68 | value: default 69 | - name: aws.dimensions.TableName 70 | enum: ["table1", "table2"] 71 | - name: aws.dimensions.Operation 72 | cardinality: 2 73 | ``` 74 | 75 | Related [fields definition](./writing-templates.md#fieldsyml---fields-definition) 76 | ```yaml 77 | - name: timestamp 78 | type: date 79 | - name: lastSnapshot 80 | type: date 81 | - name: data_stream.type 82 | type: constant_keyword 83 | - name: data_stream.dataset 84 | type: constant_keyword 85 | - name: data_stream.namespace 86 | type: constant_keyword 87 | - name: aws 88 | type: group 89 | fields: 90 | - name: dimensions 91 | type: group 92 | fields: 93 | - name: Operation 94 | type: keyword 95 | - name: TableName 96 | type: keyword 97 | - name: dynamodb 98 | type: group 99 | fields: 100 | - name: metrics 101 | type: group 102 | fields: 103 | - name: AccountProvisionedReadCapacityUtilization.avg 104 | type: double 105 | - name: AccountMaxReads.max 106 | type: long 107 | - name: AccountMaxTableLevelReads.max 108 | type: long 109 | - name: cloudwatch 110 | type: group 111 | fields: 112 | - name: namespace 113 | type: keyword 114 | ``` 115 | -------------------------------------------------------------------------------- /docs/glossary.md: -------------------------------------------------------------------------------- 1 | # Corpus 2 | 3 | # Data schemas 4 | 5 | Different data structures we observe in the end to end data collection flow for Elastic (Beats & Agent). See [Data Schemas](./data-schemas.md). 6 | 7 | In the context of this tool it also refers to folder names containing template information for a specific data schema. 8 | 9 | # Dataset 10 | 11 | A dataset is a component of a [Data Stream](https://www.elastic.co/guide/en/fleet/master/data-streams.html). Is defined by an _integration_ and describes the ingested data and its structure. 12 | 13 | In the context of this tool `dataset` refers to the name of the dataset you can generate data for. Data structure and definitions are part of the integration package the dataset belongs to. 14 | 15 | # Fields definition 16 | 17 | Datasets define data structure. Within them there are fields definition that describe the fields a dataset provides. 18 | 19 | In the context of this tool we can refer to: 20 | - the fields definition within a dataset in an integration package 21 | - a file named `fields.yml` that contains fields definition that is not (yet) part of a package 22 | 23 | # Fields generation configuration 24 | 25 | Complete randomness in generated data may not always be advisable. There may be relationships or constraints that must be expressed to create corpus that have life-like characteristics. Through the Fields generation configuration file, named `configs.yml`, is possible to specify these constraints. 26 | 27 | See [Fields generation configuration](./fields-configuration.md). 28 | 29 | # Integration 30 | 31 | An [Elastic integration package](https://www.elastic.co/guide/en/integrations-developer/current/what-is-an-integration.html). 32 | 33 | # Template 34 | 35 | A file containing a template for a specific template engine. 36 | -------------------------------------------------------------------------------- /docs/go-text-template-helpers.md: -------------------------------------------------------------------------------- 1 | This file documents helper functions available when using the Go `text/template` template engine. 2 | 3 | Within this template is possible to use **all** helpers from [`mastermings/sprig`](https://masterminds.github.io/sprig/). 4 | 5 | 6 | 7 | # `awsAZFromRegion` 8 | 9 | This helper accepts a string representing an AWS region (es. `us-east-1`) and returns a valid Availability Zone from that AWS region. 10 | 11 | _NOTE_: Not all regions are supported at the moment (but can be added at need). For supported regions look [here](https://github.com/elastic/elastic-integration-corpus-generator-tool/blob/2c64e07461467aef4faacd5eb41efc3b0399c270/pkg/genlib/generator_with_text_template.go#L28-L30) 12 | 13 | **Example**: 14 | 15 | ```text 16 | {{ awsAZFromRegion "us-east-1" }} 17 | ``` 18 | ```text 19 | us-east-1a 20 | ``` 21 | -------------------------------------------------------------------------------- /docs/performances.md: -------------------------------------------------------------------------------- 1 | # Performances 2 | 3 | Performances while generating data are a key element, as they: 4 | - allow generating big quantities of data 5 | - allow generating data on demand 6 | - allow re-generating data when needed 7 | 8 | Achieving maximum performances has also some disadvantages, mainly around feature-completeness. 9 | 10 | This tool aims to support uses cases where is needed to trade features for performances and use cases where this isn't needed, but without compromising performances too much (max 10x less performant). 11 | 12 | This tool comes with a benchmark testsuite that can be run to evaluate new features performance impact. 13 | 14 | ## Benchmarks 15 | 16 | In PR #40, where multiple templates support has been added, we run two different benchmark for each template engine: 17 | 18 | JSONContent: producing Schema C data for "endpoint process 8.2.0" integration 19 | VPCFlowLogs: producing Schema A data for aws vpc flow logs 20 | (beware the memory benchmark for Hero are misleading since they "happens" in a forked process) 21 | 22 | We tested 3 different engines: 23 | - `legacy`: the only generator available before #40 24 | - `CustomTemplate`: what we refer now as `placeholder` template engine 25 | - `TextTemplate`: what we refer now as `gotext` template engine 26 | 27 | ``` 28 | _GeneratorLegacyJSONContent: the original generator, generating from fields definitions for endpoint package v8.2.0 data stream "process" 29 | _GeneratorCustomTemplateJSONContent-16: placeholder template with arbitrary JSON content 30 | _GeneratorTextTemplateJSONContent-16: Go text/template with arbitraty JSON content 31 | _GeneratorCustomTemplateVPCFlowLogs-16: placeholder template generating Schema A data for AWS VPCFlowLogs 32 | _GeneratorTextTemplateVPCFlowLogs-16: Go text/template generating Schema A data for AWS VPCFlowLogs 33 | 34 | Tests have been executed on a 16 Cores machine. 35 | 36 | name time/op 37 | _GeneratorLegacyJSONContent-16 47.7µs ± 0% 38 | _GeneratorCustomTemplateJSONContent-16 30.0µs ± 0% 39 | _GeneratorTextTemplateJSONContent-16 281µs ± 0% 40 | _GeneratorCustomTemplateVPCFlowLogs-16 1.09µs ± 0% 41 | _GeneratorTextTemplateVPCFlowLogs-16 12.8µs ± 0% 42 | 43 | name alloc/op 44 | _GeneratorLegacyJSONContent-16 3.82kB ± 0% 45 | _GeneratorCustomTemplateJSONContent-16 432B ± 0% 46 | _GeneratorTextTemplateJSONContent-16 48.3kB ± 0% 47 | _GeneratorCustomTemplateVPCFlowLogs-16 64.0B ± 0% 48 | _GeneratorTextTemplateVPCFlowLogs-16 2.32kB ± 0% 49 | 50 | name allocs/op 51 | _GeneratorLegacyJSONContent-16 22.0 ± 0% 52 | _GeneratorCustomTemplateJSONContent-16 14.0 ± 0% 53 | _GeneratorTextTemplateJSONContent-16 2.23k ± 0% 54 | _GeneratorCustomTemplateVPCFlowLogs-16 2.00 ± 0% 55 | _GeneratorTextTemplateVPCFlowLogs-16 95.0 ± 0% 56 | 57 | ``` 58 | 59 | If you are curious how those benchmarks translate to time needed for generating dataset, we ran some test runs monitoring the execution times. 60 | We generated directly from the built binaries 20GB of "aws dynamodb 1.28.3" Schema C data. 61 | 62 | ``` 63 | $ time ./gen-legacy generate aws dynamodb 1.28.3 -t 20GB 64 | File generated: [...]/elastic-integration-corpus-generator-tool/corpora/1671594228-aws-dynamodb-1.28.3.ndjson 65 | 66 | real 1m44.869s 67 | user 1m6.599s 68 | sys 0m37.354s 69 | 70 | 71 | $ time ./gen-with-custom_template generate aws dynamodb 1.28.3 -t 20GB 72 | File generated: [...]/elastic-integration-corpus-generator-tool/corpora/1671611719-aws-dynamodb-1.28.3.ndjson 73 | 74 | real 1m34.968s 75 | user 0m55.029s 76 | sys 0m37.175s 77 | 78 | 79 | $ time ./gen-with-text_template generate aws dynamodb 1.28.3 -t 20GB 80 | File generated: [...]/elastic-integration-corpus-generator-tool/corpora/1671612518-aws-dynamodb-1.28.3.ndjson 81 | 82 | real 6m50.022s 83 | user 6m10.642s 84 | sys 0m50.909s 85 | ``` 86 | -------------------------------------------------------------------------------- /docs/usage.md: -------------------------------------------------------------------------------- 1 | This file collects some common use cases for this tool. 2 | 3 | # Generate schema-c data from integration package fields 4 | 5 | To do this, use the `generate` command. This command targets a specific dataset within an integration package at a specific version. 6 | 7 | You can pass a local Fields generation configuration file. 8 | 9 | `go run main.go generate --tot-events ` 10 | 11 | `package`, `dataset` and `version` are mandatory. `--tot-events` is not mandatory and in case it is not provided a single event will be generated. You can generate an infinite number of events expressly passing to the flag the value of `0`. `--now` is not mandatory and in case it is provided must be a string parsable according the following `time.Parse()` layout: `2006-01-02T15:04:05.999999Z07:00`. The value provided will be used as base `time.Now()` for `date` type fields (see [Fields generation configuration](./fields-configuration.md#config-entries-definition)) 12 | 13 | **Example**: 14 | 15 | ```shell 16 | $ go run main.go generate aws dynamodb 1.14.0 -t 1000 --config-file config.yml 17 | File generated: /path/to/corpora/1649330390-aws-dynamodb-1.14.0.ndjson 18 | ``` 19 | 20 | # Generate schema-b data from a template 21 | 22 | To do this, use the `generate-with-template` command. This command targets a specific template, fields definition and fields generation configuration. 23 | 24 | A body of templates, fields definition and fields generation configuration are already available in the `assets/templates` folder. You can rely on them or write your own ones. Please consider opening a PR adding your custom templates, fields definition and fields configuration to the existing body, if they belong to an integration, so that we can enrich the catalogue for everyone. 25 | within the `assets/templates` folder. 26 | 27 | You can pass a local Fields generation configuration file. 28 | 29 | `go run main.go generate-with-template --tot-events ` 30 | 31 | `template-path` and `fields-definition-path` are mandatory. `--tot-events` is not mandatory and in case it is not provided a single event will be generated. You can generate an infinite number of events expressly passing to the flag the value of `0`. `--now` is not mandatory and in case it is provided must be a string parsable according the following `time.Parse()` layout: `2006-01-02T15:04:05.999999Z07:00`. The value provided will be used as base `time.Now()` for `date` type fields (see [Fields generation configuration](./fields-configuration.md#config-entries-definition)) 32 | 33 | **Example**: 34 | 35 | ```shell 36 | $ go run main.go generate-with-template ./assets/templates/aws.vpcflow/schema-a/gotext.tpl ./assets/templates/aws.vpcflow/schema-a/fields.yml -t 1000 --config-file ./assets/templates/aws.vpcflow/schema-a/configs.yml -y gotext 37 | File generated: /path/to/corpora/1684304483-gotext.tpl 38 | ``` 39 | 40 | -------------------------------------------------------------------------------- /docs/writing-templates.md: -------------------------------------------------------------------------------- 1 | # Writing templates 2 | 3 | Templates are, as implied from their name, files that define the format of the generated data by this tool. 4 | 5 | The tool is designed to leverage any template in the compatible format. This file will guide you through writing template files for any occurrence. 6 | 7 | There are 2 supported template types: `placeholder` (fast) and `gotext` (flexible). We suggest to use `gotext` unless performances are **critical** to your workflow. 8 | 9 | ## How does templating works? 10 | 11 | At its core, this tool works by loading fields definition from a file and a template. The template is rendered using information from: 12 | - the fields definition file 13 | - the (optional) fields generation configuration 14 | 15 | This operation has some caveats: 16 | - generating data is a computing and memory intensive task; you can expect to face computing pressure more easily while memory usage depends on the choosen template type and the template itself; performances of this tool is taken in great consideration, for more information see [performances.md](./performances.md); 17 | - the use case is generating fake but plausible data, a challenge for a random data generator, as it implies: 18 | - in observability dataset usually exhibit high value cardinality; generating high cardinality dataset is a goal, for more information see [cardinality.md](./cardinality.md); 19 | - on the opposite side, a data generator may produce data with high similarity, which will not exhibit a behaviour similar to real data; while we aim to increase data dissimilarity, is not a goal to have complete real like data generation; 20 | - another characteristic of datasets in observability is high field dimensionality; generating high dimensionality dataset is a goal, for more information see [dimensionality.md](./dimensionality.md); 21 | - data generation may not be a real time operation executed together/just before using the generated data; we aim to provide corpus of generated data that can be replayed as if they were being emitted in real time with the goal of allowing creation of big and complex datasets. 22 | 23 | ## Folder structure 24 | 25 | All templates are contained in `/assets/templates` folder. Within it there should be a folder for each Integration data stream templates are available from. 26 | 27 | The folder MUST be named as: `.`. For example: `apache.access`, `apache.error`, `aws.billing`, `aws.vpcflow`. For example the final folder where to place the files listed below will be `/assets/templates/apache.error/schema-a`. 28 | 29 | As this tool aims to support different [schemas](./data-schemas.md) for generated data, files related to a schema should be put within a folder named after the schema they relate to: `schema-a`, `schema-b`, `schema-c`, `schema-d`. 30 | 31 | Within these folder, these files should be added: 32 | - (_mandatory_) `fields.yml`: a field definition file 33 | - (_optional_) `configs.yml`: a field generation configuration file 34 | - (_mandatory_) `gotext.tpl`: a `gotext` template file 35 | - (_optional_) `placeholder.tpl`: a `placeholder` template file 36 | 37 | ## `fields.yml` - Fields definition 38 | 39 | A `YAML` file containing field mapping definitions. Ideally this file is extracted from Integration packages, but there is no automation for doing so at the moment. 40 | 41 | ## `configs.yml` - Fields generation configuration 42 | 43 | A `YAML` file containing configurations for field mappings defined in `fields.yml`. Details on configurations are in [Fields generation configuration](./fields-configuration.md). 44 | 45 | This file allows to tweak the randomness of the generated data. 46 | 47 | ## Template types 48 | 49 | This tool supports multiple templates types so to adapt to different scenarios. Rendering a template requires using a template engine and they greatly differ for performances and capabilities. 50 | 51 | 2 template engines are currently supported to optimise for 2 use cases: 52 | - `placeholder` engine: uncompromised performances, is ok to lose features to gain performances; 53 | - `gotext` engine: performant (but less) and feature rich, to aid development. 54 | 55 | ### placeholder 56 | 57 | This template type is the most performant in terms of throughput: use this type **only** if data generation speed is relevant for you and you can trade off on the provided randomness and customisation given by the fields and config definitions. 58 | The format of the template is similar to the one in Go `text/template` package, only supporting the feature subset of placeholder replacement. 59 | Here's a template sample: 60 | ```text 61 | {{ .Field1 }}-{{ .Field2 }} ({{ .Field3 }}) 62 | ``` 63 | 64 | ### gotext 65 | 66 | This template type is less performant in terms of throughput than `placeholder` (our benchmarks shows from 3x to 9x slower according to the scenario), it uses the go text/template package with a few added functions: prefer this type as it supports data generation customisation that cannot be achieved only by the fields and config definitions. 67 | 68 | #### "generate" function 69 | The template provides a function named "generate" that accept the name of a field from the fields definition file as parameter: the output of this function is the random generated value for the field, respecting its definition and config: 70 | ```text 71 | {{generate "Field1"}} 72 | ``` 73 | 74 | This is equivalent to the following when using the `placeholder` template type: 75 | ```text 76 | {{ .Field1 }} 77 | ``` 78 | 79 | #### Helpers 80 | 81 | This template type supports other [helper functions](./go-text-template-helpers.md). 82 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/elastic/elastic-integration-corpus-generator-tool 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/Masterminds/sprig/v3 v3.2.3 7 | github.com/OpenPeeDeeP/xdg v1.0.0 8 | github.com/Pallinder/go-randomdata v1.2.0 9 | github.com/elastic/go-ucfg v0.8.8 10 | github.com/lithammer/shortuuid/v3 v3.0.7 11 | github.com/spf13/afero v1.11.0 12 | github.com/spf13/cobra v1.8.0 13 | github.com/spf13/viper v1.18.2 14 | github.com/stretchr/testify v1.9.0 15 | go.uber.org/multierr v1.11.0 16 | golang.org/x/mod v0.16.0 17 | golang.org/x/sync v0.7.0 18 | ) 19 | 20 | require ( 21 | github.com/Masterminds/goutils v1.1.1 // indirect 22 | github.com/Masterminds/semver/v3 v3.2.0 // indirect 23 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 24 | github.com/fsnotify/fsnotify v1.7.0 // indirect 25 | github.com/google/uuid v1.4.0 // indirect 26 | github.com/hashicorp/hcl v1.0.0 // indirect 27 | github.com/huandu/xstrings v1.3.3 // indirect 28 | github.com/imdario/mergo v0.3.11 // indirect 29 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 30 | github.com/magiconair/properties v1.8.7 // indirect 31 | github.com/mitchellh/copystructure v1.0.0 // indirect 32 | github.com/mitchellh/mapstructure v1.5.0 // indirect 33 | github.com/mitchellh/reflectwalk v1.0.0 // indirect 34 | github.com/pelletier/go-toml/v2 v2.1.0 // indirect 35 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 36 | github.com/sagikazarmark/locafero v0.4.0 // indirect 37 | github.com/sagikazarmark/slog-shim v0.1.0 // indirect 38 | github.com/shopspring/decimal v1.2.0 // indirect 39 | github.com/sourcegraph/conc v0.3.0 // indirect 40 | github.com/spf13/cast v1.6.0 // indirect 41 | github.com/spf13/pflag v1.0.5 // indirect 42 | github.com/subosito/gotenv v1.6.0 // indirect 43 | golang.org/x/crypto v0.16.0 // indirect 44 | golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect 45 | golang.org/x/sys v0.15.0 // indirect 46 | golang.org/x/text v0.14.0 // indirect 47 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 48 | gopkg.in/ini.v1 v1.67.0 // indirect 49 | gopkg.in/yaml.v2 v2.4.0 // indirect 50 | gopkg.in/yaml.v3 v3.0.1 // indirect 51 | ) 52 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= 2 | github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= 3 | github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= 4 | github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= 5 | github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= 6 | github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= 7 | github.com/OpenPeeDeeP/xdg v1.0.0 h1:UDLmNjCGFZZCaVMB74DqYEtXkHxnTxcr4FeJVF9uCn8= 8 | github.com/OpenPeeDeeP/xdg v1.0.0/go.mod h1:tMoSueLQlMf0TCldjrJLNIjAc5qAOIcHt5REi88/Ygo= 9 | github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg= 10 | github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y= 11 | github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= 12 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 13 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 14 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 15 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 16 | github.com/elastic/go-ucfg v0.8.8 h1:54KIF/2zFKfl0MzsSOCGOsZ3O2bnjFQJ0nDJcLhviyk= 17 | github.com/elastic/go-ucfg v0.8.8/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA= 18 | github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= 19 | github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= 20 | github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= 21 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 22 | github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 23 | github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 24 | github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= 25 | github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 26 | github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= 27 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 28 | github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= 29 | github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= 30 | github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= 31 | github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= 32 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 33 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 34 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 35 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 36 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 37 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 38 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 39 | github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8= 40 | github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts= 41 | github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= 42 | github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= 43 | github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= 44 | github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= 45 | github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= 46 | github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= 47 | github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= 48 | github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= 49 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= 50 | github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= 51 | github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= 52 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 53 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 54 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 55 | github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= 56 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 57 | github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= 58 | github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= 59 | github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= 60 | github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= 61 | github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= 62 | github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= 63 | github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= 64 | github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= 65 | github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= 66 | github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= 67 | github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= 68 | github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= 69 | github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= 70 | github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= 71 | github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= 72 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 73 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 74 | github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= 75 | github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= 76 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 77 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 78 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 79 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 80 | github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= 81 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 82 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 83 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 84 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 85 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 86 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 87 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 88 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 89 | github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= 90 | github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= 91 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 92 | go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= 93 | go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= 94 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 95 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 96 | golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= 97 | golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= 98 | golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= 99 | golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= 100 | golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= 101 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 102 | golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= 103 | golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= 104 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 105 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 106 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 107 | golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= 108 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 109 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 110 | golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= 111 | golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 112 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 113 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 114 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 115 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 116 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 117 | golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 118 | golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= 119 | golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 120 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 121 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 122 | golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= 123 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 124 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 125 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 126 | golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 127 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= 128 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 129 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 130 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 131 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 132 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 133 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 134 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 135 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 136 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 137 | gopkg.in/hjson/hjson-go.v3 v3.0.1/go.mod h1:X6zrTSVeImfwfZLfgQdInl9mWjqPqgH90jom9nym/lw= 138 | gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= 139 | gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 140 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 141 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 142 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 143 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 144 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 145 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 146 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 147 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 148 | -------------------------------------------------------------------------------- /internal/corpus/generator.go: -------------------------------------------------------------------------------- 1 | // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 2 | // or more contributor license agreements. Licensed under the Elastic License 2.0; 3 | // you may not use this file except in compliance with the Elastic License 2.0. 4 | 5 | package corpus 6 | 7 | import ( 8 | "bytes" 9 | "context" 10 | "errors" 11 | "fmt" 12 | "io" 13 | "os" 14 | "path" 15 | "strings" 16 | "time" 17 | 18 | "github.com/elastic/elastic-integration-corpus-generator-tool/pkg/genlib" 19 | "github.com/elastic/elastic-integration-corpus-generator-tool/pkg/genlib/config" 20 | "github.com/elastic/elastic-integration-corpus-generator-tool/pkg/genlib/fields" 21 | "github.com/spf13/afero" 22 | ) 23 | 24 | const ( 25 | templateTypeCustom = iota 26 | templateTypeGoText 27 | ) 28 | 29 | var ErrNotValidTemplate = errors.New("please, pass --template-type as one of 'placeholder' or 'gotext'") 30 | 31 | type Config = config.Config 32 | type Fields = fields.Fields 33 | 34 | // timestamp represent a function providing a timestamp. 35 | // It's used to allow replacing the value with a known one during testing. 36 | type timestamp func() int64 37 | 38 | func NewGenerator(config Config, fs afero.Fs, location string) (GeneratorCorpus, error) { 39 | return GeneratorCorpus{ 40 | config: config, 41 | fs: fs, 42 | templateType: templateTypeCustom, 43 | location: location, 44 | timestamp: time.Now().Unix, 45 | }, nil 46 | } 47 | 48 | func NewGeneratorWithTemplate(config Config, fs afero.Fs, location, templateType string) (GeneratorCorpus, error) { 49 | 50 | var templateTypeValue int 51 | if templateType == "placeholder" { 52 | templateTypeValue = templateTypeCustom 53 | } else if templateType == "gotext" { 54 | templateTypeValue = templateTypeGoText 55 | } else { 56 | return GeneratorCorpus{}, ErrNotValidTemplate 57 | } 58 | 59 | return GeneratorCorpus{ 60 | config: config, 61 | fs: fs, 62 | templateType: templateTypeValue, 63 | location: location, 64 | timestamp: time.Now().Unix, 65 | }, nil 66 | } 67 | 68 | // TestNewGenerator sets up a GeneratorCorpus configured to be used in testing. 69 | func TestNewGenerator() GeneratorCorpus { 70 | f, _ := NewGenerator(Config{}, afero.NewMemMapFs(), "testdata") 71 | f.timestamp = func() int64 { return 1647345675 } 72 | return f 73 | } 74 | 75 | type GeneratorCorpus struct { 76 | config Config 77 | fs afero.Fs 78 | location string 79 | templateType int 80 | // timestamp allow overriding value in tests 81 | timestamp timestamp 82 | } 83 | 84 | func (gc GeneratorCorpus) Location() string { 85 | return gc.location 86 | } 87 | 88 | // bulkPayloadFilename computes the bulkPayloadFilename for the corpus to be generated. 89 | // To provide unique names the provided slug is prepended with current timestamp. 90 | func (gc GeneratorCorpus) bulkPayloadFilename(integrationPackage, dataStream, packageVersion string) string { 91 | slug := integrationPackage + "-" + dataStream + "-" + packageVersion 92 | filename := fmt.Sprintf("%d-%s.ndjson", gc.timestamp(), sanitizeFilename(slug)) 93 | return filename 94 | } 95 | 96 | // bulkPayloadFilenameWithTemplate computes the bulkPayloadFilename for the corpus to be generated. 97 | // To provide unique names the provided slug is prepended with current timestamp. 98 | func (gc GeneratorCorpus) bulkPayloadFilenameWithTemplate(templatePath string) string { 99 | slug := path.Base(templatePath) 100 | ext := path.Ext(templatePath) 101 | slug = slug[0 : len(slug)-len(ext)] 102 | filename := fmt.Sprintf("%d-%s%s", gc.timestamp(), sanitizeFilename(slug), sanitizeFilename(ext)) 103 | return filename 104 | } 105 | 106 | var corpusLocPerm = os.FileMode(0770) 107 | var corpusPerm = os.FileMode(0660) 108 | 109 | func (gc GeneratorCorpus) eventsPayloadFromFields(template []byte, fields Fields, totEvents uint64, timeNow time.Time, randSeed int64, createPayload []byte, f afero.File) error { 110 | genlib.InitGeneratorTimeNow(timeNow) 111 | genlib.InitGeneratorRandSeed(randSeed) 112 | 113 | var evgen genlib.Generator 114 | var err error 115 | if len(template) == 0 { 116 | evgen, err = genlib.NewGenerator(gc.config, fields, totEvents) 117 | } else { 118 | if gc.templateType == templateTypeCustom { 119 | evgen, err = genlib.NewGeneratorWithCustomTemplate(template, gc.config, fields, totEvents) 120 | } else if gc.templateType == templateTypeGoText { 121 | evgen, err = genlib.NewGeneratorWithTextTemplate(template, gc.config, fields, totEvents) 122 | } else { 123 | return ErrNotValidTemplate 124 | } 125 | 126 | } 127 | 128 | if err != nil { 129 | return err 130 | } 131 | 132 | var buf *bytes.Buffer 133 | if len(template) == 0 { 134 | buf = bytes.NewBuffer(createPayload) 135 | } else { 136 | buf = bytes.NewBufferString("") 137 | } 138 | 139 | defer func() { 140 | _ = evgen.Close() 141 | }() 142 | 143 | for { 144 | buf.Truncate(len(createPayload)) 145 | err := evgen.Emit(buf) 146 | if err == nil { 147 | buf.WriteByte('\n') 148 | 149 | if _, err = f.Write(buf.Bytes()); err != nil { 150 | return err 151 | } 152 | } 153 | 154 | if err == io.EOF { 155 | return nil 156 | } 157 | 158 | if err != nil { 159 | return err 160 | } 161 | } 162 | } 163 | 164 | // Generate generates a bulk request corpus and persist it to file. 165 | func (gc GeneratorCorpus) Generate(packageRegistryBaseURL, integrationPackage, dataStream, packageVersion string, totEvents uint64, timeNow time.Time, randSeed int64) (string, error) { 166 | if err := gc.fs.MkdirAll(gc.location, corpusLocPerm); err != nil { 167 | return "", fmt.Errorf("cannot generate corpus location folder: %v", err) 168 | } 169 | 170 | payloadFilename := path.Join(gc.location, gc.bulkPayloadFilename(integrationPackage, dataStream, packageVersion)) 171 | f, err := gc.fs.OpenFile(payloadFilename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, corpusPerm) 172 | if err != nil { 173 | return "", err 174 | } 175 | 176 | ctx := context.Background() 177 | flds, dataStreamType, err := fields.LoadFields(ctx, packageRegistryBaseURL, integrationPackage, dataStream, packageVersion) 178 | if err != nil { 179 | return "", err 180 | } 181 | 182 | createPayload := []byte(`{ "create" : { "_index": "` + dataStreamType + `-` + integrationPackage + `.` + dataStream + `-default" } }` + "\n") 183 | 184 | err = gc.eventsPayloadFromFields(nil, flds, totEvents, timeNow, randSeed, createPayload, f) 185 | if err != nil { 186 | return "", err 187 | } 188 | 189 | if err := f.Close(); err != nil { 190 | return "", err 191 | } 192 | 193 | return payloadFilename, err 194 | } 195 | 196 | // GenerateWithTemplate generates a template based corpus and persist it to file. 197 | func (gc GeneratorCorpus) GenerateWithTemplate(templatePath, fieldsDefinitionPath string, totEvents uint64, timeNow time.Time, randSeed int64) (string, error) { 198 | if err := gc.fs.MkdirAll(gc.location, corpusLocPerm); err != nil { 199 | return "", fmt.Errorf("cannot generate corpus location folder: %v", err) 200 | } 201 | 202 | payloadFilename := path.Join(gc.location, gc.bulkPayloadFilenameWithTemplate(templatePath)) 203 | f, err := gc.fs.OpenFile(payloadFilename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, corpusPerm) 204 | if err != nil { 205 | return "", err 206 | } 207 | 208 | template, err := os.ReadFile(templatePath) 209 | if err != nil { 210 | return "", err 211 | } 212 | 213 | if len(template) == 0 { 214 | return "", errors.New("you must provide a non empty template content") 215 | } 216 | 217 | ctx := context.Background() 218 | flds, err := fields.LoadFieldsWithTemplate(ctx, fieldsDefinitionPath) 219 | if err != nil { 220 | return "", err 221 | } 222 | 223 | err = gc.eventsPayloadFromFields(template, flds, totEvents, timeNow, randSeed, nil, f) 224 | if err != nil { 225 | return "", err 226 | } 227 | 228 | if err := f.Close(); err != nil { 229 | return "", err 230 | } 231 | 232 | return payloadFilename, err 233 | } 234 | 235 | // sanitizeFilename takes care of removing dangerous elements from a string so it can be safely 236 | // used as a bulkPayloadFilename. 237 | // NOTE: does not prevent command injection or ensure complete escaping of input 238 | func sanitizeFilename(s string) string { 239 | s = strings.Replace(s, " ", "-", -1) 240 | s = strings.Replace(s, ":", "-", -1) 241 | s = strings.Replace(s, "/", "-", -1) 242 | s = strings.Replace(s, "\\", "-", -1) 243 | return s 244 | } 245 | -------------------------------------------------------------------------------- /internal/corpus/generator_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 2 | // or more contributor license agreements. Licensed under the Elastic License 2.0; 3 | // you may not use this file except in compliance with the Elastic License 2.0. 4 | 5 | package corpus 6 | 7 | import ( 8 | "reflect" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestFilename(t *testing.T) { 15 | fc := TestNewGenerator() 16 | 17 | expected := "1647345675-integration-data_stream-0.0.1.ndjson" 18 | got := fc.bulkPayloadFilename("integration", "data_stream", "0.0.1") 19 | assert.Equal(t, expected, got) 20 | } 21 | 22 | func TestSanitizeFilename(t *testing.T) { 23 | type test struct { 24 | input string 25 | want string 26 | } 27 | 28 | tests := []test{ 29 | {input: "foo bar", want: "foo-bar"}, 30 | {input: "foo bar foobar", want: "foo-bar-foobar"}, 31 | {input: "foo/bar", want: "foo-bar"}, 32 | {input: "foo\\bar", want: "foo-bar"}, 33 | {input: "foo bar/foobar\\", want: "foo-bar-foobar-"}, 34 | } 35 | 36 | for _, tc := range tests { 37 | got := sanitizeFilename(tc.input) 38 | if !reflect.DeepEqual(tc.want, got) { 39 | t.Fatalf("expected: %v, got: %v", tc.want, got) 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /internal/settings/settings.go: -------------------------------------------------------------------------------- 1 | // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 2 | // or more contributor license agreements. Licensed under the Elastic License 2.0; 3 | // you may not use this file except in compliance with the Elastic License 2.0. 4 | 5 | package settings 6 | 7 | import ( 8 | "github.com/OpenPeeDeeP/xdg" 9 | "github.com/spf13/viper" 10 | "os" 11 | "path" 12 | ) 13 | 14 | // Init initalize settings and default values 15 | func Init() { 16 | viper.AutomaticEnv() 17 | // NOTE: err value is ignored as it only checks for missing argument 18 | _ = viper.BindEnv("ELASTIC_INTEGRATION_CORPUS") 19 | 20 | setDefaults() 21 | setConstants() 22 | } 23 | 24 | func setDefaults() { 25 | viper.SetDefault("cache_dir", xdg.CacheHome()) 26 | viper.SetDefault("config_dir", xdg.ConfigHome()) 27 | viper.SetDefault("data_dir", xdg.DataHome()) 28 | 29 | // fragment_root supports env var expansion 30 | viper.SetDefault("corpora_root", path.Join(viper.GetString("data_dir"), "elastic-integration-corpus-generator-tool")) 31 | viper.SetDefault("corpora_path", "corpora") 32 | viper.SetDefault("corpora_location", path.Join( 33 | os.ExpandEnv(viper.GetString("corpora_root")), 34 | viper.GetString("corpora_path"))) 35 | } 36 | 37 | func setConstants() { 38 | // viper.Set() 39 | } 40 | -------------------------------------------------------------------------------- /internal/settings/xdg.go: -------------------------------------------------------------------------------- 1 | // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 2 | // or more contributor license agreements. Licensed under the Elastic License 2.0; 3 | // you may not use this file except in compliance with the Elastic License 2.0. 4 | 5 | package settings 6 | 7 | import ( 8 | "github.com/spf13/viper" 9 | ) 10 | 11 | func CacheDir() string { 12 | return viper.GetString("cache_dir") 13 | } 14 | 15 | func ConfigDir() string { 16 | return viper.GetString("config_dir") 17 | } 18 | 19 | func DataDir() string { 20 | return viper.GetString("data_dir") 21 | } 22 | -------------------------------------------------------------------------------- /internal/settings/xdg_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 2 | // or more contributor license agreements. Licensed under the Elastic License 2.0; 3 | // you may not use this file except in compliance with the Elastic License 2.0. 4 | 5 | package settings_test 6 | 7 | import ( 8 | "os" 9 | "testing" 10 | 11 | "github.com/OpenPeeDeeP/xdg" 12 | "github.com/spf13/viper" 13 | "github.com/stretchr/testify/assert" 14 | 15 | "github.com/elastic/elastic-integration-corpus-generator-tool/internal/settings" 16 | ) 17 | 18 | func TestCacheDir(t *testing.T) { 19 | settings.Init() 20 | 21 | expected := xdg.CacheHome() 22 | got := settings.CacheDir() 23 | 24 | assert.Equal(t, expected, got) 25 | } 26 | 27 | func TestCacheDir_customValue(t *testing.T) { 28 | settings.Init() 29 | 30 | expected := "foobar" 31 | viper.Set("cache_dir", expected) 32 | got := settings.CacheDir() 33 | 34 | assert.Equal(t, expected, got) 35 | } 36 | 37 | func TestCacheDir_valueFromEnv(t *testing.T) { 38 | settings.Init() 39 | 40 | expected := "foobar" 41 | os.Setenv("ELASTIC_INTEGRATION_CORPUS_CACHE_DIR", expected) 42 | got := settings.CacheDir() 43 | 44 | assert.Equal(t, expected, got) 45 | } 46 | 47 | func TestConfigDir(t *testing.T) { 48 | settings.Init() 49 | 50 | expected := xdg.ConfigHome() 51 | got := settings.ConfigDir() 52 | 53 | assert.Equal(t, expected, got) 54 | } 55 | 56 | func TestConfigDir_customValue(t *testing.T) { 57 | settings.Init() 58 | 59 | expected := "foobar" 60 | viper.Set("config_dir", expected) 61 | got := settings.ConfigDir() 62 | 63 | assert.Equal(t, expected, got) 64 | } 65 | 66 | func TestConfigDir_valueFromEnv(t *testing.T) { 67 | settings.Init() 68 | 69 | expected := "foobar" 70 | os.Setenv("ELASTIC_INTEGRATION_CORPUS_CONFIG_DIR", expected) 71 | got := settings.ConfigDir() 72 | 73 | assert.Equal(t, expected, got) 74 | } 75 | 76 | func TestDataDir(t *testing.T) { 77 | settings.Init() 78 | 79 | expected := xdg.DataHome() 80 | got := settings.DataDir() 81 | 82 | assert.Equal(t, expected, got) 83 | } 84 | 85 | func TestDataDir_customValue(t *testing.T) { 86 | settings.Init() 87 | 88 | expected := "foobar" 89 | viper.Set("data_dir", expected) 90 | got := settings.DataDir() 91 | 92 | assert.Equal(t, expected, got) 93 | } 94 | 95 | func TestDataDir_valueFromEnv(t *testing.T) { 96 | settings.Init() 97 | 98 | expected := "foobar" 99 | os.Setenv("ELASTIC_INTEGRATION_CORPUS_DATA_DIR", expected) 100 | got := settings.DataDir() 101 | 102 | assert.Equal(t, expected, got) 103 | } 104 | -------------------------------------------------------------------------------- /internal/version/version.go: -------------------------------------------------------------------------------- 1 | // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 2 | // or more contributor license agreements. Licensed under the Elastic License 2.0; 3 | // you may not use this file except in compliance with the Elastic License 2.0. 4 | 5 | package version 6 | 7 | import ( 8 | "strconv" 9 | "time" 10 | ) 11 | 12 | var ( 13 | // CommitHash is the Git hash of the branch, used for version purposes (set externally with ldflags). 14 | CommitHash = "undefined" 15 | // SourceDateEpoch is the build time of the binary (set externally with ldflags). 16 | // https://reproducible-builds.org/docs/source-date-epoch/ 17 | SourceDateEpoch string 18 | // Tag describes the semver version of the application (set externally with ldflags). 19 | Tag string 20 | ) 21 | 22 | // SourceTimeFormatted method returns the source last changed time in UTC preserving the RFC3339 format. 23 | func SourceTimeFormatted() string { 24 | if SourceDateEpoch == "" { 25 | return "unknown" 26 | } 27 | 28 | seconds, err := strconv.ParseInt(SourceDateEpoch, 10, 64) 29 | if err != nil { 30 | return "invalid" 31 | } 32 | 33 | // NOTE: time is returned in UTC to avoid timezone difference issues 34 | return time.Unix(seconds, 0).UTC().Format(time.RFC3339) 35 | } 36 | -------------------------------------------------------------------------------- /internal/version/version_test.go: -------------------------------------------------------------------------------- 1 | // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 2 | // or more contributor license agreements. Licensed under the Elastic License 2.0; 3 | // you may not use this file except in compliance with the Elastic License 2.0. 4 | 5 | package version_test 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/elastic/elastic-integration-corpus-generator-tool/internal/version" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestCommitHashDefault(t *testing.T) { 15 | require.Equal(t, "undefined", version.CommitHash) 16 | } 17 | 18 | func TestSourceTimeFormattedDefault(t *testing.T) { 19 | // NOTE: this test is order sensitive, as it tests the default value 20 | v := version.SourceTimeFormatted() 21 | require.Equal(t, "unknown", v) 22 | } 23 | 24 | func TestSourceTimeFormatted_invalid(t *testing.T) { 25 | version.SourceDateEpoch = "foobar" 26 | v := version.SourceTimeFormatted() 27 | require.Equal(t, "invalid", v) 28 | // NOTE: reset value to default to avoid test order issues 29 | version.SourceDateEpoch = "" 30 | } 31 | 32 | func TestSourceTimeFormatted_valid(t *testing.T) { 33 | version.SourceDateEpoch = "1648570012" 34 | v := version.SourceTimeFormatted() 35 | require.Equal(t, "2022-03-29T16:06:52Z", v) 36 | // NOTE: reset value to default to avoid test order issues 37 | version.SourceDateEpoch = "" 38 | } 39 | 40 | func TestTagDefault(t *testing.T) { 41 | require.Empty(t, version.Tag) 42 | } 43 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 2 | // or more contributor license agreements. Licensed under the Elastic License 2.0; 3 | // you may not use this file except in compliance with the Elastic License 2.0. 4 | 5 | package main 6 | 7 | import ( 8 | "os" 9 | 10 | "github.com/elastic/elastic-integration-corpus-generator-tool/cmd" 11 | "github.com/elastic/elastic-integration-corpus-generator-tool/internal/settings" 12 | ) 13 | 14 | func main() { 15 | settings.Init() 16 | 17 | rootCmd := cmd.RootCmd() 18 | rootCmd.AddCommand(cmd.GenerateCmd()) 19 | rootCmd.AddCommand(cmd.GenerateWithTemplateCmd()) 20 | rootCmd.AddCommand(cmd.TemplateCmd()) 21 | rootCmd.AddCommand(cmd.VersionCmd()) 22 | 23 | err := rootCmd.Execute() 24 | if err != nil { 25 | os.Exit(1) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /pkg/genlib/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "errors" 5 | "time" 6 | 7 | "math" 8 | "os" 9 | 10 | "github.com/elastic/go-ucfg/yaml" 11 | "github.com/spf13/afero" 12 | ) 13 | 14 | var rangeBoundNotSet = errors.New("range bound not set") 15 | var rangeTimeNotSet = errors.New("range time not set") 16 | var rangeInvalidConfig = errors.New("range defining both `period` and `from`/`to`") 17 | var counterInvalidConfig = errors.New("both `range` and `counter` defined") 18 | 19 | type TimeRange struct { 20 | time.Time 21 | } 22 | 23 | func (ct *TimeRange) Unpack(t string) error { 24 | var err error 25 | ct.Time, err = time.Parse("2006-01-02T15:04:05.999999999-07:00", t) 26 | return err 27 | } 28 | 29 | type Range struct { 30 | // NOTE: we want to distinguish when Min/Max/From/To are explicitly set to zero value or are not set at all. We use a pointer, such that when not set will be `nil`. 31 | Min *float64 `config:"min"` 32 | Max *float64 `config:"max"` 33 | From *TimeRange `config:"from"` 34 | To *TimeRange `config:"to"` 35 | } 36 | 37 | type Config struct { 38 | m map[string]ConfigField 39 | } 40 | 41 | type ConfigField struct { 42 | Name string `config:"name"` 43 | Fuzziness float64 `config:"fuzziness"` 44 | Range Range `config:"range"` 45 | Cardinality int `config:"cardinality"` 46 | Period time.Duration `config:"period"` 47 | Enum []string `config:"enum"` 48 | ObjectKeys []string `config:"object_keys"` 49 | Value any `config:"value"` 50 | Counter bool `config:"counter"` 51 | CounterReset *CounterReset `config:"counter_reset"` 52 | } 53 | 54 | const ( 55 | CounterResetStrategyRandom string = "random" 56 | CounterResetStrategyProbabilistic string = "probabilistic" 57 | CounterResetStrategyAfterN string = "after_n" 58 | ) 59 | 60 | type CounterReset struct { 61 | Strategy string `config:"strategy"` 62 | Probability *uint64 `config:"probability"` 63 | ResetAfterN *uint64 `config:"reset_after_n"` 64 | } 65 | 66 | func (cf ConfigField) ValidateCounterResetStrategy() error { 67 | if cf.Counter && cf.CounterReset != nil && 68 | cf.CounterReset.Strategy != CounterResetStrategyRandom && 69 | cf.CounterReset.Strategy != CounterResetStrategyProbabilistic && 70 | cf.CounterReset.Strategy != CounterResetStrategyAfterN { 71 | return errors.New("counter_reset strategy must be one of 'random', 'probabilistic', 'after_n'") 72 | } 73 | 74 | return nil 75 | } 76 | 77 | func (cf ConfigField) ValidateCounterResetAfterN() error { 78 | if cf.Counter && cf.CounterReset != nil && cf.CounterReset.Strategy == CounterResetStrategyAfterN && cf.CounterReset.ResetAfterN == nil { 79 | return errors.New("counter_reset after_n requires 'reset_after_n' value to be set") 80 | } 81 | 82 | return nil 83 | } 84 | 85 | func (cf ConfigField) ValidateCounterResetProbabilistic() error { 86 | if cf.Counter && cf.CounterReset != nil && cf.CounterReset.Strategy == CounterResetStrategyProbabilistic && cf.CounterReset.Probability == nil { 87 | return errors.New("counter_reset probabilistic requires 'probability' value to be set") 88 | } 89 | 90 | return nil 91 | } 92 | 93 | func (cf ConfigField) ValidForDateField() error { 94 | if cf.Period.Abs() > 0 && (cf.Range.From != nil || cf.Range.To != nil) { 95 | return rangeInvalidConfig 96 | } 97 | 98 | return nil 99 | } 100 | 101 | func (cf ConfigField) ValidCounter() error { 102 | if cf.Counter && (cf.Range.Min != nil || cf.Range.Max != nil) { 103 | return counterInvalidConfig 104 | } 105 | 106 | return nil 107 | } 108 | 109 | func (r Range) FromAsTime() (time.Time, error) { 110 | if r.From == nil { 111 | return time.Time{}, rangeTimeNotSet 112 | } 113 | 114 | return r.From.Time, nil 115 | } 116 | 117 | func (r Range) ToAsTime() (time.Time, error) { 118 | if r.To == nil { 119 | return time.Time{}, rangeTimeNotSet 120 | } 121 | 122 | return r.To.Time, nil 123 | } 124 | 125 | func (r Range) MinAsInt64() (int64, error) { 126 | if r.Min == nil { 127 | return 0, rangeBoundNotSet 128 | } 129 | 130 | return int64(*r.Min), nil 131 | } 132 | 133 | func (r Range) MaxAsInt64() (int64, error) { 134 | if r.Max == nil { 135 | return math.MaxInt64, rangeBoundNotSet 136 | } 137 | 138 | return int64(*r.Max), nil 139 | } 140 | 141 | func (r Range) MinAsFloat64() (float64, error) { 142 | if r.Min == nil { 143 | return 0, rangeBoundNotSet 144 | } 145 | 146 | return *r.Min, nil 147 | } 148 | 149 | func (r Range) MaxAsFloat64() (float64, error) { 150 | if r.Max == nil { 151 | return math.MaxFloat64, rangeBoundNotSet 152 | } 153 | 154 | return *r.Max, nil 155 | } 156 | 157 | type ConfigFile struct { 158 | Fields []ConfigField `config:"fields"` 159 | } 160 | 161 | func LoadConfig(fs afero.Fs, configFile string) (Config, error) { 162 | if len(configFile) == 0 { 163 | return Config{}, nil 164 | } 165 | 166 | configFile = os.ExpandEnv(configFile) 167 | if _, err := fs.Stat(configFile); err != nil { 168 | return Config{}, err 169 | } 170 | 171 | data, err := afero.ReadFile(fs, configFile) 172 | if err != nil { 173 | return Config{}, err 174 | } 175 | 176 | return LoadConfigFromYaml(data) 177 | } 178 | 179 | func LoadConfigFromYaml(c []byte) (Config, error) { 180 | 181 | cfg, err := yaml.NewConfig(c) 182 | if err != nil { 183 | return Config{}, err 184 | } 185 | 186 | var cfgfile ConfigFile 187 | err = cfg.Unpack(&cfgfile) 188 | if err != nil { 189 | return Config{}, err 190 | } 191 | 192 | outCfg := Config{ 193 | m: make(map[string]ConfigField), 194 | } 195 | 196 | for _, c := range cfgfile.Fields { 197 | outCfg.m[c.Name] = c 198 | } 199 | 200 | return outCfg, nil 201 | } 202 | 203 | func (c Config) GetField(fieldName string) (ConfigField, bool) { 204 | v, ok := c.m[fieldName] 205 | return v, ok 206 | } 207 | 208 | func (c Config) SetField(fieldName string, configField ConfigField) { 209 | configField.Name = fieldName 210 | c.m[fieldName] = configField 211 | } 212 | -------------------------------------------------------------------------------- /pkg/genlib/config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/elastic/go-ucfg/yaml" 5 | "github.com/spf13/afero" 6 | "github.com/stretchr/testify/assert" 7 | "math" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | const sampleConfigFile = `--- 13 | fields: 14 | - name: field 15 | value: foobar 16 | ` 17 | 18 | func TestLoadConfig(t *testing.T) { 19 | fs := afero.NewMemMapFs() 20 | configFile := "/cfg.yml" 21 | 22 | data := []byte(sampleConfigFile) 23 | afero.WriteFile(fs, configFile, data, 0666) 24 | 25 | cfg, err := LoadConfig(fs, configFile) 26 | assert.Nil(t, err) 27 | 28 | f, ok := cfg.GetField("field") 29 | assert.True(t, ok) 30 | assert.Equal(t, "field", f.Name) 31 | assert.Equal(t, "foobar", f.Value.(string)) 32 | } 33 | 34 | func TestSetField(t *testing.T) { 35 | fs := afero.NewMemMapFs() 36 | configFile := "/cfg.yml" 37 | 38 | data := []byte(sampleConfigFile) 39 | afero.WriteFile(fs, configFile, data, 0666) 40 | 41 | cfg, err := LoadConfig(fs, configFile) 42 | assert.Nil(t, err) 43 | 44 | cfg.SetField("field", ConfigField{Value: "foobaz"}) 45 | f, ok := cfg.GetField("field") 46 | assert.True(t, ok) 47 | assert.Equal(t, "field", f.Name) 48 | assert.Equal(t, "foobaz", f.Value.(string)) 49 | } 50 | 51 | func TestIsValidForDateField(t *testing.T) { 52 | testCases := []struct { 53 | scenario string 54 | config string 55 | hasError bool 56 | }{ 57 | { 58 | scenario: "no range", 59 | config: "name: field", 60 | hasError: false, 61 | }, 62 | { 63 | scenario: "zero period", 64 | config: "name: field\nrange:\n period: 0", 65 | hasError: false, 66 | }, 67 | { 68 | scenario: "positive period", 69 | config: "name: field\nrange:\n period: 1", 70 | hasError: false, 71 | }, 72 | { 73 | scenario: "negative period", 74 | config: "name: field\nrange:\n period: -1", 75 | hasError: false, 76 | }, 77 | { 78 | scenario: "form only", 79 | config: "name: field\nrange:\n from: \"2006-01-02T15:04:05+07:00\"", 80 | hasError: false, 81 | }, 82 | { 83 | scenario: "form and zero period", 84 | config: "name: field\nrange:\n period: 0\n from: \"2006-01-02T15:04:05+07:00\"", 85 | hasError: false, 86 | }, 87 | { 88 | scenario: "form and positive period", 89 | config: "name: field\nrange:\n period: 1\n from: \"2006-01-02T15:04:05+07:00\"", 90 | hasError: true, 91 | }, 92 | { 93 | scenario: "form and negative period", 94 | config: "name: field\nrange:\n period: -1\n from: \"2006-01-02T15:04:05+07:00\"", 95 | hasError: true, 96 | }, 97 | { 98 | scenario: "to only", 99 | config: "name: field\nrange:\n to: \"2006-01-02T15:04:05-07:00\"", 100 | hasError: false, 101 | }, 102 | { 103 | scenario: "to and zero period", 104 | config: "name: field\nrange:\n period: 0\n to: \"2006-01-02T15:04:05-07:00\"", 105 | hasError: false, 106 | }, 107 | { 108 | scenario: "to and positive period", 109 | config: "name: field\nrange:\n period: 1\n to: \"2006-01-02T15:04:05-07:00\"", 110 | hasError: true, 111 | }, 112 | { 113 | scenario: "to and negative period", 114 | config: "name: field\nrange:\n period: -1\n to: \"2006-01-02T15:04:05-07:00\"", 115 | hasError: true, 116 | }, 117 | { 118 | scenario: "from and to only", 119 | config: "name: field\nrange:\n from: \"2006-01-02T15:04:05-07:00\"\n to: \"2006-01-02T15:04:05+07:00\"", 120 | hasError: false, 121 | }, 122 | { 123 | scenario: "from and to and zero period", 124 | config: "name: field\nrange:\n period: 0\n from: \"2006-01-02T15:04:05-07:00\"\n\n to: \"2006-01-02T15:04:05+07:00\"", 125 | hasError: false, 126 | }, 127 | { 128 | scenario: "from and to and positive period", 129 | config: "name: field\nrange:\n period: 1\n from: \"2006-01-02T15:04:05-07:00\"\n to: \"2006-01-02T15:04:05+07:00\"", 130 | hasError: true, 131 | }, 132 | { 133 | scenario: "from and to and negative period", 134 | config: "name: field\nrange:\n period: -1\n from: \"2006-01-02T15:04:05-07:00\"\n to: \"2006-01-02T15:04:05+07:00\"", 135 | hasError: true, 136 | }, 137 | } 138 | for _, testCase := range testCases { 139 | t.Run(testCase.scenario, func(t *testing.T) { 140 | cfg, err := yaml.NewConfig([]byte(testCase.config)) 141 | if err != nil { 142 | t.Fatal(err) 143 | } 144 | 145 | var config ConfigField 146 | err = cfg.Unpack(&config) 147 | if err != nil { 148 | t.Fatal(err) 149 | } 150 | 151 | err = config.ValidForDateField() 152 | if testCase.hasError && err == nil { 153 | 154 | } 155 | 156 | if !testCase.hasError && err != nil { 157 | 158 | } 159 | }) 160 | } 161 | } 162 | 163 | func TestIsValidCounter(t *testing.T) { 164 | testCases := []struct { 165 | scenario string 166 | config string 167 | hasError bool 168 | }{ 169 | { 170 | scenario: "no range, no counter", 171 | config: "name: field", 172 | hasError: false, 173 | }, 174 | { 175 | scenario: "range with both min and max, no counter", 176 | config: "name: field\nrange:\n min: 1\n max: 10", 177 | hasError: false, 178 | }, 179 | { 180 | scenario: "range with min and no max, no counter", 181 | config: "name: field\nrange:\n min: 1", 182 | hasError: false, 183 | }, 184 | { 185 | scenario: "range with max and no min, no counter", 186 | config: "name: field\nrange:\n max: 10", 187 | hasError: false, 188 | }, 189 | { 190 | scenario: "with counter, no range", 191 | config: "name: field\ncounter: true", 192 | hasError: false, 193 | }, 194 | { 195 | scenario: "range with both min and max, with counter", 196 | config: "name: field\nrange:\n min: 1\n max: 10\ncounter: true", 197 | hasError: true, 198 | }, 199 | { 200 | scenario: "range with min and no max, with counter", 201 | config: "name: field\nrange:\n min: 1\ncounter: true", 202 | hasError: true, 203 | }, 204 | { 205 | scenario: "range with max and no min, with counter", 206 | config: "name: field\nrange:\n max: 10\ncounter: true", 207 | hasError: true, 208 | }, 209 | { 210 | scenario: "range with both min and max, with counter:false", 211 | config: "name: field\nrange:\n min: 1\n max: 10\ncounter: false", 212 | hasError: false, 213 | }, 214 | { 215 | scenario: "range with min and no max, with counter:false", 216 | config: "name: field\nrange:\n min: 1\ncounter: false", 217 | hasError: false, 218 | }, 219 | { 220 | scenario: "range with max and no min, with counter:false", 221 | config: "name: field\nrange:\n max: 10\ncounter: false", 222 | hasError: false, 223 | }, 224 | } 225 | for _, testCase := range testCases { 226 | t.Run(testCase.scenario, func(t *testing.T) { 227 | cfg, err := yaml.NewConfig([]byte(testCase.config)) 228 | if err != nil { 229 | t.Fatal(err) 230 | } 231 | 232 | var config ConfigField 233 | err = cfg.Unpack(&config) 234 | if err != nil { 235 | t.Fatal(err) 236 | } 237 | 238 | err = config.ValidCounter() 239 | if testCase.hasError && err == nil { 240 | 241 | } 242 | 243 | if !testCase.hasError && err != nil { 244 | 245 | } 246 | }) 247 | } 248 | } 249 | 250 | func TestRange_MaxAsFloat64(t *testing.T) { 251 | testCases := []struct { 252 | scenario string 253 | rangeYaml string 254 | expected float64 255 | hasError bool 256 | }{ 257 | { 258 | scenario: "max nil", 259 | rangeYaml: "min: 10", 260 | expected: math.MaxFloat64, 261 | hasError: true, 262 | }, 263 | { 264 | scenario: "float64", 265 | rangeYaml: "max: 10.", 266 | expected: 10, 267 | }, 268 | { 269 | scenario: "uint64", 270 | rangeYaml: "max: 10", 271 | expected: 10, 272 | }, 273 | { 274 | scenario: "int64", 275 | rangeYaml: "max: -10", 276 | expected: -10, 277 | }, 278 | } 279 | 280 | for _, testCase := range testCases { 281 | t.Run(testCase.scenario, func(t *testing.T) { 282 | cfg, err := yaml.NewConfig([]byte(testCase.rangeYaml)) 283 | if err != nil { 284 | t.Fatal(err) 285 | } 286 | 287 | var rangeCfg Range 288 | err = cfg.Unpack(&rangeCfg) 289 | if err != nil { 290 | t.Fatal(err) 291 | } 292 | 293 | v, err := rangeCfg.MaxAsFloat64() 294 | if testCase.hasError && err == nil { 295 | t.Fatal("expected error but got nil") 296 | } 297 | if !testCase.hasError && err != nil { 298 | t.Fatal("expected no error but got one") 299 | } 300 | if testCase.expected != v { 301 | t.Fatalf("expected %v, got %v", testCase.expected, v) 302 | } 303 | }) 304 | } 305 | } 306 | 307 | func TestRange_MaxAsInt64(t *testing.T) { 308 | testCases := []struct { 309 | scenario string 310 | rangeYaml string 311 | expected int64 312 | hasError bool 313 | }{ 314 | { 315 | scenario: "max nil", 316 | rangeYaml: "min: 10", 317 | expected: math.MaxInt64, 318 | hasError: true, 319 | }, 320 | { 321 | scenario: "float64", 322 | rangeYaml: "max: 10.", 323 | expected: 10, 324 | }, 325 | { 326 | scenario: "uint64", 327 | rangeYaml: "max: 10", 328 | expected: 10, 329 | }, 330 | { 331 | scenario: "int64", 332 | rangeYaml: "max: -10", 333 | expected: -10, 334 | }, 335 | } 336 | 337 | for _, testCase := range testCases { 338 | t.Run(testCase.scenario, func(t *testing.T) { 339 | cfg, err := yaml.NewConfig([]byte(testCase.rangeYaml)) 340 | if err != nil { 341 | t.Fatal(err) 342 | } 343 | 344 | var rangeCfg Range 345 | err = cfg.Unpack(&rangeCfg) 346 | if err != nil { 347 | t.Fatal(err) 348 | } 349 | 350 | v, err := rangeCfg.MaxAsInt64() 351 | if testCase.hasError && err == nil { 352 | t.Fatal("expected error but got nil") 353 | } 354 | if !testCase.hasError && err != nil { 355 | t.Fatal("expected no error but got one") 356 | } 357 | if testCase.expected != v { 358 | t.Fatalf("expected %v, got %v", testCase.expected, v) 359 | } 360 | }) 361 | } 362 | } 363 | 364 | func TestRange_MinAsFloat64(t *testing.T) { 365 | testCases := []struct { 366 | scenario string 367 | rangeYaml string 368 | expected float64 369 | hasError bool 370 | }{ 371 | { 372 | scenario: "min nil", 373 | rangeYaml: "max: 10", 374 | expected: 0, 375 | hasError: true, 376 | }, 377 | { 378 | scenario: "float64", 379 | rangeYaml: "min: 10.", 380 | expected: 10, 381 | }, 382 | { 383 | scenario: "uint64", 384 | rangeYaml: "min: 10", 385 | expected: 10, 386 | }, 387 | { 388 | scenario: "int64", 389 | rangeYaml: "min: -10", 390 | expected: -10, 391 | }, 392 | } 393 | 394 | for _, testCase := range testCases { 395 | t.Run(testCase.scenario, func(t *testing.T) { 396 | cfg, err := yaml.NewConfig([]byte(testCase.rangeYaml)) 397 | if err != nil { 398 | t.Fatal(err) 399 | } 400 | 401 | var rangeCfg Range 402 | err = cfg.Unpack(&rangeCfg) 403 | if err != nil { 404 | t.Fatal(err) 405 | } 406 | 407 | v, err := rangeCfg.MinAsFloat64() 408 | if testCase.hasError && err == nil { 409 | t.Fatal("expected error but got nil") 410 | } 411 | if !testCase.hasError && err != nil { 412 | t.Fatal("expected no error but got one") 413 | } 414 | if testCase.expected != v { 415 | t.Fatalf("expected %v, got %v", testCase.expected, v) 416 | } 417 | }) 418 | } 419 | } 420 | 421 | func TestRange_MinAsInt64(t *testing.T) { 422 | testCases := []struct { 423 | scenario string 424 | rangeYaml string 425 | expected int64 426 | hasError bool 427 | }{ 428 | { 429 | scenario: "min nil", 430 | rangeYaml: "max: 10", 431 | expected: 0, 432 | hasError: true, 433 | }, 434 | { 435 | scenario: "float64", 436 | rangeYaml: "min: 10.", 437 | expected: 10, 438 | }, 439 | { 440 | scenario: "uint64", 441 | rangeYaml: "min: 10", 442 | expected: 10, 443 | }, 444 | { 445 | scenario: "int64", 446 | rangeYaml: "min: -10", 447 | expected: -10, 448 | }, 449 | } 450 | 451 | for _, testCase := range testCases { 452 | t.Run(testCase.scenario, func(t *testing.T) { 453 | cfg, err := yaml.NewConfig([]byte(testCase.rangeYaml)) 454 | if err != nil { 455 | t.Fatal(err) 456 | } 457 | 458 | var rangeCfg Range 459 | err = cfg.Unpack(&rangeCfg) 460 | if err != nil { 461 | t.Fatal(err) 462 | } 463 | 464 | v, err := rangeCfg.MinAsInt64() 465 | if testCase.hasError && err == nil { 466 | t.Fatal("expected error but got nil") 467 | } 468 | if !testCase.hasError && err != nil { 469 | t.Fatal("expected no error but got one") 470 | } 471 | if testCase.expected != v { 472 | t.Fatalf("expected %v, got %v", testCase.expected, v) 473 | } 474 | }) 475 | } 476 | } 477 | 478 | func TestRange_FromAsTime(t *testing.T) { 479 | from, err := time.Parse("2006-01-02T15:04:05.999999999-07:00", "2023-11-23T08:35:38+00:00") 480 | if err != nil { 481 | t.Fatal(err) 482 | } 483 | 484 | testCases := []struct { 485 | scenario string 486 | rangeYaml string 487 | expected time.Time 488 | hasError bool 489 | }{ 490 | { 491 | scenario: "from nil", 492 | rangeYaml: "to: 2023-11-23T08:35:38+00:00", 493 | expected: time.Time{}, 494 | hasError: true, 495 | }, 496 | { 497 | scenario: "from not nil", 498 | rangeYaml: "from: 2023-11-23T08:35:38-00:00", 499 | expected: from, 500 | hasError: false, 501 | }, 502 | } 503 | 504 | for _, testCase := range testCases { 505 | t.Run(testCase.scenario, func(t *testing.T) { 506 | cfg, err := yaml.NewConfig([]byte(testCase.rangeYaml)) 507 | if err != nil { 508 | t.Fatal(err) 509 | } 510 | 511 | var rangeCfg Range 512 | err = cfg.Unpack(&rangeCfg) 513 | if err != nil { 514 | t.Fatal(err) 515 | } 516 | 517 | v, err := rangeCfg.FromAsTime() 518 | if testCase.hasError && err == nil { 519 | t.Fatal("expected error but got nil") 520 | } 521 | if !testCase.hasError && err != nil { 522 | t.Fatal("expected no error but got one") 523 | } 524 | if testCase.expected != v { 525 | t.Fatalf("expected %v, got %v", testCase.expected, v) 526 | } 527 | }) 528 | } 529 | } 530 | 531 | func TestRange_ToAsTime(t *testing.T) { 532 | to, err := time.Parse("2006-01-02T15:04:05.999999999-07:00", "2023-11-23T08:35:38-00:00") 533 | if err != nil { 534 | t.Fatal(err) 535 | } 536 | 537 | testCases := []struct { 538 | scenario string 539 | rangeYaml string 540 | expected time.Time 541 | hasError bool 542 | }{ 543 | { 544 | scenario: "to nil", 545 | rangeYaml: "from: 2023-11-23T08:35:38+00:00", 546 | expected: time.Time{}, 547 | hasError: true, 548 | }, 549 | { 550 | scenario: "to not nil", 551 | rangeYaml: "to: 2023-11-23T08:35:38-00:00", 552 | expected: to, 553 | hasError: false, 554 | }, 555 | } 556 | 557 | for _, testCase := range testCases { 558 | t.Run(testCase.scenario, func(t *testing.T) { 559 | cfg, err := yaml.NewConfig([]byte(testCase.rangeYaml)) 560 | if err != nil { 561 | t.Fatal(err) 562 | } 563 | 564 | var rangeCfg Range 565 | err = cfg.Unpack(&rangeCfg) 566 | if err != nil { 567 | t.Fatal(err) 568 | } 569 | 570 | v, err := rangeCfg.ToAsTime() 571 | if testCase.hasError && err == nil { 572 | t.Fatal("expected error but got nil") 573 | } 574 | if !testCase.hasError && err != nil { 575 | t.Fatal("expected no error but got one") 576 | } 577 | if testCase.expected != v { 578 | t.Fatalf("expected %v, got %v", testCase.expected, v) 579 | } 580 | }) 581 | } 582 | } 583 | 584 | func TestPeriod(t *testing.T) { 585 | testCases := []struct { 586 | scenario string 587 | periodYaml string 588 | expected time.Duration 589 | hasField bool 590 | }{ 591 | { 592 | scenario: "time duration as number", 593 | periodYaml: "- name: testField\n period: 10", 594 | expected: 10 * time.Second, 595 | hasField: true, 596 | }, 597 | { 598 | scenario: "empty period", 599 | hasField: false, 600 | }, 601 | { 602 | scenario: "1h", 603 | periodYaml: "- name: testField\n period: 1h", 604 | expected: 3600 * time.Second, 605 | hasField: true, 606 | }, 607 | { 608 | scenario: "-1h", 609 | periodYaml: "- name: testField\n period: -1h", 610 | expected: -3600 * time.Second, 611 | hasField: true, 612 | }, 613 | } 614 | 615 | for _, testCase := range testCases { 616 | t.Run(testCase.scenario, func(t *testing.T) { 617 | cfg, err := yaml.NewConfig([]byte(testCase.periodYaml)) 618 | if err != nil { 619 | t.Fatal(err) 620 | } 621 | 622 | var periodCfg []ConfigField 623 | err = cfg.Unpack(&periodCfg) 624 | if err != nil { 625 | t.Fatal(err) 626 | } 627 | 628 | ok := len(periodCfg) == 1 629 | if !testCase.hasField && ok { 630 | t.Fatalf("expected missing field but got: %v", periodCfg[0]) 631 | } 632 | if testCase.hasField && !ok { 633 | t.Fatal("expected field but missing") 634 | } 635 | 636 | if ok && testCase.expected != periodCfg[0].Period { 637 | t.Fatalf("expected %v, got %v", testCase.expected, periodCfg[0].Period) 638 | } 639 | }) 640 | } 641 | } 642 | -------------------------------------------------------------------------------- /pkg/genlib/fields/cache.go: -------------------------------------------------------------------------------- 1 | package fields 2 | 3 | import ( 4 | "context" 5 | "golang.org/x/sync/semaphore" 6 | "sync" 7 | ) 8 | 9 | const ( 10 | ProductionBaseURL = "https://epr.elastic.co/" 11 | maxParallel = 4 12 | ) 13 | 14 | type tuple struct { 15 | integration string 16 | stream string 17 | version string 18 | } 19 | 20 | type Manifest struct { 21 | Title string `config:"title"` 22 | Type string `config:"type"` 23 | DataSet string `config:"dataset"` 24 | } 25 | 26 | type CacheOption func(*Cache) 27 | 28 | func WithBaseUrl(url string) CacheOption { 29 | return func(c *Cache) { 30 | c.baseUrl = url 31 | } 32 | } 33 | 34 | type Cache struct { 35 | mut sync.RWMutex 36 | sema *semaphore.Weighted 37 | baseUrl string 38 | fields map[tuple]Fields 39 | manifest map[tuple]Manifest 40 | } 41 | 42 | func NewCache(opts ...CacheOption) *Cache { 43 | c := &Cache{ 44 | baseUrl: ProductionBaseURL, 45 | sema: semaphore.NewWeighted(maxParallel), 46 | fields: make(map[tuple]Fields), 47 | manifest: make(map[tuple]Manifest), 48 | } 49 | 50 | for _, opt := range opts { 51 | opt(c) 52 | } 53 | 54 | return c 55 | } 56 | 57 | func (f *Cache) LoadFields(ctx context.Context, integration, stream, version string) (Fields, error) { 58 | var err error 59 | 60 | t := tuple{ 61 | integration: integration, 62 | stream: stream, 63 | version: version, 64 | } 65 | 66 | f.mut.RLock() 67 | flds, ok := f.fields[t] 68 | f.mut.RUnlock() 69 | 70 | if ok { 71 | return flds, nil 72 | } 73 | 74 | // Limit the number of parallel outbound transactions 75 | if err = f.sema.Acquire(ctx, 1); err != nil { 76 | return nil, err 77 | } 78 | 79 | defer f.sema.Release(1) 80 | 81 | // Check again after aquiring semaphore; fields may have been retrieved by another thread 82 | f.mut.RLock() 83 | flds, ok = f.fields[t] 84 | f.mut.RUnlock() 85 | 86 | if !ok { 87 | 88 | if flds, _, err = LoadFields(ctx, f.baseUrl, integration, stream, version); err != nil { 89 | return nil, err 90 | } else { 91 | f.mut.Lock() 92 | f.fields[t] = flds 93 | f.mut.Unlock() 94 | } 95 | } 96 | 97 | return flds, nil 98 | } 99 | -------------------------------------------------------------------------------- /pkg/genlib/fields/fields.go: -------------------------------------------------------------------------------- 1 | package fields 2 | 3 | import ( 4 | "regexp" 5 | "sort" 6 | "strings" 7 | ) 8 | 9 | type Fields []Field 10 | 11 | func (f Fields) Len() int { return len(f) } 12 | func (f Fields) Less(i, j int) bool { return f[i].Name < f[j].Name } 13 | func (f Fields) Swap(i, j int) { f[i], f[j] = f[j], f[i] } 14 | 15 | type Field struct { 16 | Name string 17 | Type string 18 | ObjectType string 19 | Example string 20 | Value string 21 | } 22 | 23 | func (fields Fields) merge(fieldsToMerge ...Field) Fields { 24 | merged := false 25 | for _, field := range fieldsToMerge { 26 | for _, currentField := range fields { 27 | if currentField.Name != field.Name { 28 | continue 29 | } 30 | 31 | if currentField.Example > field.Example { 32 | field.Example = currentField.Example 33 | } 34 | 35 | if currentField.Value > field.Value { 36 | field.Value = currentField.Value 37 | } 38 | 39 | merged = true 40 | break 41 | } 42 | 43 | if !merged { 44 | fields = append(fields, field) 45 | } 46 | } 47 | 48 | return fields 49 | } 50 | 51 | func normaliseFields(fields Fields) (Fields, error) { 52 | sort.Sort(fields) 53 | normalisedFields := make(Fields, 0, len(fields)) 54 | for _, field := range fields { 55 | if !strings.Contains(field.Name, "*") { 56 | normalisedFields = append(normalisedFields, field) 57 | continue 58 | } 59 | 60 | normalizationPattern := strings.NewReplacer(".", "\\.", "*", ".+").Replace(field.Name) 61 | re, err := regexp.Compile(normalizationPattern) 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | hasMatch := false 67 | for _, otherField := range fields { 68 | if otherField.Name == field.Name { 69 | continue 70 | } 71 | 72 | if re.MatchString(otherField.Name) { 73 | hasMatch = true 74 | break 75 | } 76 | } 77 | 78 | if !hasMatch { 79 | normalisedFields = append(normalisedFields, field) 80 | } 81 | } 82 | 83 | sort.Sort(normalisedFields) 84 | return normalisedFields, nil 85 | } 86 | -------------------------------------------------------------------------------- /pkg/genlib/fields/load.go: -------------------------------------------------------------------------------- 1 | package fields 2 | 3 | import ( 4 | "archive/zip" 5 | "bytes" 6 | "context" 7 | "encoding/json" 8 | "errors" 9 | "fmt" 10 | "github.com/elastic/go-ucfg/yaml" 11 | "io" 12 | "io/ioutil" 13 | "net/http" 14 | "net/url" 15 | "os" 16 | "path" 17 | "path/filepath" 18 | "strings" 19 | ) 20 | 21 | var ErrNotFound = errors.New("not found") 22 | 23 | const ( 24 | fieldsSlug = "fields" 25 | packageSlug = "package" 26 | dataStreamSlug = "data_stream" 27 | searchSlug = "search" 28 | kibanaVersionSlug = "kibana.version" 29 | manifestSlug = "manifest.yml" 30 | ) 31 | 32 | type yamlManifest struct { 33 | Type string `config:"type"` 34 | } 35 | 36 | func LoadFields(ctx context.Context, baseURL, integration, dataStream, version string) (Fields, string, error) { 37 | 38 | fieldsContent, dataStreamType, err := getFieldsFilesAndDataStreamType(ctx, baseURL, integration, dataStream, version) 39 | if err != nil { 40 | return nil, dataStreamType, err 41 | } 42 | 43 | if len(fieldsContent) == 0 { 44 | return nil, dataStreamType, ErrNotFound 45 | } 46 | 47 | fieldsFromYaml, err := loadFieldsFromYaml(fieldsContent) 48 | if err != nil { 49 | return nil, dataStreamType, err 50 | } 51 | 52 | fields := collectFields(fieldsFromYaml, "") 53 | 54 | fields, err = normaliseFields(fields) 55 | return fields, dataStreamType, err 56 | } 57 | 58 | func LoadFieldsWithTemplateFromString(ctx context.Context, fieldsContent string) (Fields, error) { 59 | if len(fieldsContent) == 0 { 60 | return nil, ErrNotFound 61 | } 62 | 63 | fieldsYaml := []byte("- key: key\n fields:\n") 64 | for _, line := range strings.Split(fieldsContent, "\n") { 65 | fieldsYaml = append(fieldsYaml, []byte(` `+line+"\n")...) 66 | } 67 | 68 | fieldsFromYaml, err := loadFieldsFromYaml(fieldsYaml) 69 | if err != nil { 70 | return nil, err 71 | } 72 | 73 | fields := collectFields(fieldsFromYaml, "") 74 | 75 | return normaliseFields(fields) 76 | } 77 | 78 | func LoadFieldsWithTemplate(ctx context.Context, fieldYamlPath string) (Fields, error) { 79 | fieldsFileContent, err := os.ReadFile(fieldYamlPath) 80 | if err != nil { 81 | return nil, err 82 | } 83 | 84 | var fieldsContent string 85 | 86 | key := strings.TrimSuffix(filepath.Base(fieldYamlPath), filepath.Ext(fieldYamlPath)) 87 | keyEntry := fmt.Sprintf("- key: %s\n fields:\n", key) 88 | for _, line := range strings.Split(string(fieldsFileContent), "\n") { 89 | keyEntry += ` ` + line + "\n" 90 | } 91 | 92 | fieldsContent += keyEntry 93 | if len(fieldsContent) == 0 { 94 | return nil, ErrNotFound 95 | } 96 | 97 | fieldsFromYaml, err := loadFieldsFromYaml([]byte(fieldsContent)) 98 | if err != nil { 99 | return nil, err 100 | } 101 | 102 | fields := collectFields(fieldsFromYaml, "") 103 | 104 | return normaliseFields(fields) 105 | } 106 | 107 | func makePackageURL(baseURL, integration, version string) (*url.URL, error) { 108 | 109 | u, err := url.Parse(baseURL) 110 | if err != nil { 111 | return nil, err 112 | } 113 | 114 | u.Path = path.Join(u.Path, packageSlug, integration, version) 115 | return u, nil 116 | } 117 | 118 | func makeDownloadURL(baseURL, donwloadPath string) (*url.URL, error) { 119 | 120 | u, err := url.Parse(baseURL) 121 | if err != nil { 122 | return nil, err 123 | } 124 | 125 | u.Path = path.Join(u.Path, donwloadPath) 126 | return u, nil 127 | } 128 | 129 | func getFieldsFilesAndDataStreamType(ctx context.Context, baseURL, integration, dataStream, version string) ([]byte, string, error) { 130 | packageURL, err := makePackageURL(baseURL, integration, version) 131 | if err != nil { 132 | return nil, "", err 133 | } 134 | 135 | r, err := getFromURL(ctx, packageURL.String()) 136 | if err != nil { 137 | return nil, "", err 138 | } 139 | 140 | var downloadPayload struct { 141 | Download string `json:"download"` 142 | } 143 | 144 | body, err := ioutil.ReadAll(r) 145 | if err = json.Unmarshal(body, &downloadPayload); err != nil { 146 | return nil, "", err 147 | } 148 | 149 | downloadURL, err := makeDownloadURL(baseURL, downloadPayload.Download) 150 | r, err = getFromURL(ctx, downloadURL.String()) 151 | defer func(r io.ReadCloser) { 152 | if r != nil { 153 | _ = r.Close() 154 | } 155 | }(r) 156 | 157 | if err != nil { 158 | return nil, "", err 159 | } 160 | 161 | zipContent, err := ioutil.ReadAll(r) 162 | if err != nil { 163 | return nil, "", err 164 | } 165 | 166 | archive, err := zip.NewReader(bytes.NewReader(zipContent), int64(len(zipContent))) 167 | if err != nil { 168 | return nil, "", err 169 | } 170 | 171 | prefixFieldsPath := path.Join(fmt.Sprintf("%s-%s", integration, version), dataStreamSlug, dataStream, fieldsSlug) 172 | manifestPath := path.Join(fmt.Sprintf("%s-%s", integration, version), dataStreamSlug, dataStream, manifestSlug) 173 | 174 | var dataStreamType string 175 | var fieldsContent string 176 | for _, z := range archive.File { 177 | if z.FileInfo().IsDir() { 178 | continue 179 | } 180 | 181 | if !strings.HasPrefix(z.Name, prefixFieldsPath) && !strings.HasPrefix(z.Name, manifestPath) { 182 | continue 183 | } 184 | 185 | fieldsFileName := z.Name 186 | zr, err := z.Open() 187 | if err != nil { 188 | if zr != nil { 189 | _ = zr.Close() 190 | } 191 | return nil, "", err 192 | } 193 | 194 | fieldsFileContent, err := ioutil.ReadAll(zr) 195 | if err != nil { 196 | if zr != nil { 197 | _ = zr.Close() 198 | } 199 | return nil, "", err 200 | } 201 | 202 | _ = zr.Close() 203 | 204 | if strings.HasPrefix(z.Name, prefixFieldsPath) { 205 | key := strings.TrimSuffix(filepath.Base(fieldsFileName), filepath.Ext(fieldsFileName)) 206 | keyEntry := fmt.Sprintf("- key: %s\n fields:\n", key) 207 | for _, line := range strings.Split(string(fieldsFileContent), "\n") { 208 | keyEntry += ` ` + line + "\n" 209 | } 210 | 211 | fieldsContent += keyEntry 212 | } 213 | 214 | if strings.HasPrefix(z.Name, manifestPath) { 215 | var manifest yamlManifest 216 | 217 | cfg, err := yaml.NewConfig(fieldsFileContent) 218 | if err != nil { 219 | return nil, "", err 220 | } 221 | err = cfg.Unpack(&manifest) 222 | if err != nil { 223 | return nil, "", err 224 | } 225 | 226 | dataStreamType = manifest.Type 227 | } 228 | } 229 | 230 | return []byte(fieldsContent), dataStreamType, nil 231 | } 232 | 233 | func getFromURL(ctx context.Context, srcURL string) (io.ReadCloser, error) { 234 | 235 | req, err := http.NewRequestWithContext(ctx, "GET", srcURL, nil) 236 | 237 | if err != nil { 238 | return nil, err 239 | } 240 | 241 | client := &http.Client{} 242 | resp, err := client.Do(req) 243 | if err != nil { 244 | if resp.Body != nil { 245 | _ = resp.Body.Close() 246 | } 247 | return nil, err 248 | } 249 | 250 | if resp.StatusCode != http.StatusOK { 251 | if resp.Body != nil { 252 | _ = resp.Body.Close() 253 | } 254 | return nil, ErrNotFound 255 | } 256 | 257 | return resp.Body, nil 258 | } 259 | -------------------------------------------------------------------------------- /pkg/genlib/fields/version.go: -------------------------------------------------------------------------------- 1 | package fields 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "io/ioutil" 8 | "net/url" 9 | "path" 10 | "strings" 11 | 12 | "golang.org/x/mod/semver" 13 | ) 14 | 15 | func MapVersion(ctx context.Context, baseUrl, integration, kibanaVersion string) (string, error) { 16 | searchUrl, err := makeSearchURL(baseUrl, integration, kibanaVersion) 17 | if err != nil { 18 | return "", err 19 | } 20 | 21 | r, err := getFromURL(ctx, searchUrl.String()) 22 | if err != nil { 23 | return "", err 24 | } 25 | 26 | var payload []struct { 27 | Version string `json:"version"` 28 | } 29 | 30 | body, err := ioutil.ReadAll(r) 31 | if err != nil { 32 | _ = r.Close() 33 | return "", err 34 | } 35 | 36 | if err = json.Unmarshal(body, &payload); err != nil { 37 | return "", err 38 | } 39 | 40 | if len(payload) == 0 { 41 | return "", errors.New("empty payload") 42 | } 43 | 44 | version := payload[0].Version 45 | 46 | // semver is picky, requires the prefix 47 | if !strings.HasPrefix(version, "v") { 48 | version = "v" + version 49 | } 50 | 51 | if !semver.IsValid(version) { 52 | return "", errors.New("invalid version") 53 | } 54 | 55 | return payload[0].Version, nil 56 | } 57 | 58 | func makeSearchURL(baseURL, integration, kibanaVersion string) (*url.URL, error) { 59 | 60 | u, err := url.Parse(baseURL) 61 | if err != nil { 62 | return nil, err 63 | } 64 | 65 | u.Path = path.Join(u.Path, searchSlug) 66 | 67 | q := u.Query() 68 | q.Set(kibanaVersionSlug, kibanaVersion) 69 | q.Set(packageSlug, integration) 70 | u.RawQuery = q.Encode() 71 | 72 | return u, nil 73 | } 74 | -------------------------------------------------------------------------------- /pkg/genlib/fields/yaml.go: -------------------------------------------------------------------------------- 1 | package fields 2 | 3 | import ( 4 | "github.com/elastic/go-ucfg/yaml" 5 | ) 6 | 7 | type yamlFields []yamlField 8 | 9 | type yamlField struct { 10 | Name string `config:"name"` 11 | Type string `config:"type"` 12 | ObjectType string `config:"object_type"` 13 | Value string `config:"value"` 14 | Example string `config:"example"` 15 | Fields yamlFields `config:"fields"` 16 | } 17 | 18 | func loadFieldsFromYaml(f []byte) (yamlFields, error) { 19 | var keys []yamlField 20 | 21 | cfg, err := yaml.NewConfig(f) 22 | if err != nil { 23 | return nil, err 24 | } 25 | err = cfg.Unpack(&keys) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | fields := yamlFields{} 31 | for _, key := range keys { 32 | fields = append(fields, key.Fields...) 33 | } 34 | return fields, nil 35 | } 36 | 37 | func collectFields(fieldsFromYaml yamlFields, namePrefix string) Fields { 38 | fields := make(Fields, 0, len(fieldsFromYaml)) 39 | for _, fieldFromYaml := range fieldsFromYaml { 40 | field := Field{ 41 | Type: fieldFromYaml.Type, 42 | ObjectType: fieldFromYaml.ObjectType, 43 | Example: fieldFromYaml.Example, 44 | Value: fieldFromYaml.Value, 45 | } 46 | 47 | if len(namePrefix) == 0 { 48 | field.Name = fieldFromYaml.Name 49 | } else { 50 | field.Name = namePrefix + "." + fieldFromYaml.Name 51 | } 52 | 53 | if len(fieldFromYaml.Fields) == 0 { 54 | // There are examples of fields of type "group" with no subfields; ignore these. 55 | if field.Type != "group" { 56 | fields = fields.merge(field) 57 | } 58 | } else { 59 | subFields := collectFields(fieldFromYaml.Fields, field.Name) 60 | fields = fields.merge(subFields...) 61 | } 62 | } 63 | 64 | return fields 65 | } 66 | -------------------------------------------------------------------------------- /pkg/genlib/generator.go: -------------------------------------------------------------------------------- 1 | // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 2 | // or more contributor license agreements. Licensed under the Elastic License 2.0; 3 | // you may not use this file except in compliance with the Elastic License 2.0. 4 | 5 | package genlib 6 | 7 | import ( 8 | "bytes" 9 | "fmt" 10 | "github.com/Pallinder/go-randomdata" 11 | "github.com/lithammer/shortuuid/v3" 12 | "math/rand" 13 | "strings" 14 | "time" 15 | ) 16 | 17 | var customRand *rand.Rand 18 | 19 | const ( 20 | textTemplateEngine = iota 21 | customTemplateEngine 22 | ) 23 | 24 | func fieldValueWrapByType(field Field) string { 25 | if len(field.Value) > 0 { 26 | return "" 27 | } 28 | 29 | switch field.Type { 30 | case FieldTypeDate, FieldTypeIP: 31 | return "\"" 32 | case FieldTypeDouble, FieldTypeFloat, FieldTypeHalfFloat, FieldTypeScaledFloat: 33 | return "" 34 | case FieldTypeInteger, FieldTypeLong, FieldTypeUnsignedLong: 35 | return "" 36 | case FieldTypeConstantKeyword: 37 | return "\"" 38 | case FieldTypeKeyword: 39 | return "\"" 40 | case FieldTypeBool: 41 | return "" 42 | case FieldTypeObject, FieldTypeNested, FieldTypeFlattened: 43 | if len(field.ObjectType) > 0 { 44 | field.Type = field.ObjectType 45 | } else { 46 | field.Type = FieldTypeKeyword 47 | } 48 | return fieldValueWrapByType(field) 49 | case FieldTypeGeoPoint: 50 | return "\"" 51 | default: 52 | return "\"" 53 | } 54 | } 55 | 56 | func generateCustomTemplateFromField(cfg Config, fields Fields) ([]byte, []Field) { 57 | return generateTemplateFromField(cfg, fields, customTemplateEngine) 58 | } 59 | 60 | func generateTextTemplateFromField(cfg Config, fields Fields) ([]byte, []Field) { 61 | return generateTemplateFromField(cfg, fields, textTemplateEngine) 62 | } 63 | 64 | func generateTemplateFromField(cfg Config, fields Fields, templateEngine int) ([]byte, []Field) { 65 | if len(fields) == 0 { 66 | return nil, nil 67 | } 68 | 69 | dupes := make(map[string]struct{}) 70 | objectKeysField := make([]Field, 0, len(fields)) 71 | 72 | templatePrefix := "{ " 73 | templateBuffer := bytes.NewBufferString(templatePrefix) 74 | for i, field := range fields { 75 | fieldWrap := fieldValueWrapByType(field) 76 | if fieldCfg, ok := cfg.GetField(field.Name); ok { 77 | if fieldCfg.Value != nil { 78 | fieldWrap = "" 79 | } 80 | } 81 | 82 | fieldTrailer := []byte(",") 83 | if i == len(fields)-1 { 84 | fieldTrailer = []byte(" }") 85 | } 86 | 87 | if strings.HasSuffix(field.Name, ".*") || field.Type == FieldTypeObject || field.Type == FieldTypeNested || field.Type == FieldTypeFlattened { 88 | // This is a special case. We are randomly generating keys on the fly 89 | // Will set the json field name as "field.Name.N" 90 | N := 5 91 | for ii := 0; ii < N; ii++ { 92 | // Fire or skip 93 | if rand.Int()%2 == 0 { 94 | continue 95 | } 96 | 97 | if string(fieldTrailer) == "}" && ii < N-1 { 98 | fieldTrailer = []byte(",") 99 | } 100 | 101 | var try int 102 | const maxTries = 10 103 | rNoun := randomdata.Noun() 104 | _, ok := dupes[rNoun] 105 | for ; ok && try < maxTries; try++ { 106 | rNoun = randomdata.Noun() 107 | _, ok = dupes[rNoun] 108 | } 109 | 110 | // If all else fails, use a shortuuid. 111 | // Try to avoid this as it is alloc expensive 112 | if try >= maxTries { 113 | rNoun = shortuuid.New() 114 | } 115 | 116 | dupes[rNoun] = struct{}{} 117 | var fieldTemplate string 118 | 119 | fieldNameRoot := replacer.Replace(field.Name) 120 | fieldVariableName := fieldNormalizerRegex.ReplaceAllString(fmt.Sprintf("%s%s", fieldNameRoot, rNoun), "") 121 | fieldVariableName += "Var" 122 | if field.Type == FieldTypeDate { 123 | if templateEngine == textTemplateEngine { 124 | fieldTemplate = fmt.Sprintf(`{{ $%s := generate "%s.%s" }}"%s.%s": %s{{$%s.Format "2006-01-02T15:04:05.999999999Z07:00"}}%s%s`, fieldVariableName, fieldNameRoot, rNoun, fieldNameRoot, rNoun, fieldWrap, fieldVariableName, fieldWrap, fieldTrailer) 125 | } else if templateEngine == customTemplateEngine { 126 | fieldTemplate = fmt.Sprintf(`"%s.%s": %s{{.%s.%s}}%s%s`, fieldNameRoot, rNoun, fieldWrap, fieldNameRoot, rNoun, fieldWrap, fieldTrailer) 127 | } 128 | } else { 129 | if templateEngine == textTemplateEngine { 130 | fieldTemplate = fmt.Sprintf(`"%s.%s": %s{{generate "%s.%s"}}%s%s`, fieldNameRoot, rNoun, fieldWrap, fieldNameRoot, rNoun, fieldWrap, fieldTrailer) 131 | } else if templateEngine == customTemplateEngine { 132 | fieldTemplate = fmt.Sprintf(`"%s.%s": %s{{.%s.%s}}%s%s`, fieldNameRoot, rNoun, fieldWrap, fieldNameRoot, rNoun, fieldWrap, fieldTrailer) 133 | } 134 | } 135 | 136 | originalFieldName := field.Name 137 | field.Name = fieldNameRoot + "." + rNoun 138 | objectKeysField = append(objectKeysField, field) 139 | field.Name = originalFieldName 140 | 141 | templateBuffer.WriteString(fieldTemplate) 142 | } 143 | } else { 144 | var fieldTemplate string 145 | fieldVariableName := fieldNormalizerRegex.ReplaceAllString(field.Name, "") 146 | fieldVariableName += "Var" 147 | if field.Type == FieldTypeDate { 148 | if templateEngine == textTemplateEngine { 149 | fieldTemplate = fmt.Sprintf(`{{ $%s := generate "%s" }}"%s": %s{{$%s.Format "2006-01-02T15:04:05.999999999Z07:00"}}%s%s`, fieldVariableName, field.Name, field.Name, fieldWrap, fieldVariableName, fieldWrap, fieldTrailer) 150 | } else if templateEngine == customTemplateEngine { 151 | fieldTemplate = fmt.Sprintf(`"%s": %s{{.%s}}%s%s`, field.Name, fieldWrap, field.Name, fieldWrap, fieldTrailer) 152 | } 153 | } else { 154 | if templateEngine == textTemplateEngine { 155 | fieldTemplate = fmt.Sprintf(`"%s": %s{{generate "%s"}}%s%s`, field.Name, fieldWrap, field.Name, fieldWrap, fieldTrailer) 156 | } else if templateEngine == customTemplateEngine { 157 | fieldTemplate = fmt.Sprintf(`"%s": %s{{.%s}}%s%s`, field.Name, fieldWrap, field.Name, fieldWrap, fieldTrailer) 158 | } 159 | } 160 | 161 | templateBuffer.WriteString(fieldTemplate) 162 | } 163 | } 164 | 165 | return templateBuffer.Bytes(), objectKeysField 166 | } 167 | 168 | func NewGenerator(cfg Config, flds Fields, totEvents uint64) (Generator, error) { 169 | template, objectKeysField := generateCustomTemplateFromField(cfg, flds) 170 | flds = append(flds, objectKeysField...) 171 | 172 | return NewGeneratorWithCustomTemplate(template, cfg, flds, totEvents) 173 | } 174 | 175 | // InitGeneratorTimeNow sets base timeNow for `date` field 176 | func InitGeneratorTimeNow(timeNow time.Time) { 177 | // set timeNowToBind to --now flag (already parsed or now) 178 | timeNowToBind = timeNow 179 | } 180 | 181 | // InitGeneratorRandSeed sets rand seed 182 | func InitGeneratorRandSeed(randSeed int64) { 183 | // set rand and randomdata seed to --seed flag (custom or 1) 184 | customRand = rand.New(rand.NewSource(randSeed)) 185 | randomdata.CustomRand(customRand) 186 | } 187 | -------------------------------------------------------------------------------- /pkg/genlib/generator_test.go: -------------------------------------------------------------------------------- 1 | package genlib 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "github.com/elastic/elastic-integration-corpus-generator-tool/pkg/genlib/config" 7 | "github.com/elastic/elastic-integration-corpus-generator-tool/pkg/genlib/fields" 8 | "log" 9 | "math/rand" 10 | "os" 11 | "testing" 12 | "time" 13 | ) 14 | 15 | func TestMain(m *testing.M) { 16 | timeNow := time.Now() 17 | randSeed := rand.Int63() 18 | 19 | log.Printf("rand seed generator initialised with value `%d`\n", randSeed) 20 | log.Printf("time now generator initialised with value `%s`\n", timeNow.UTC().Format(time.RFC3339Nano)) 21 | 22 | InitGeneratorRandSeed(randSeed) 23 | InitGeneratorTimeNow(timeNow) 24 | 25 | os.Exit(m.Run()) 26 | } 27 | 28 | func Benchmark_GeneratorCustomTemplateJSONContent(b *testing.B) { 29 | ctx := context.Background() 30 | flds, _, err := fields.LoadFields(ctx, fields.ProductionBaseURL, "endpoint", "process", "8.2.0") 31 | 32 | template, objectKeysField := generateCustomTemplateFromField(Config{}, flds) 33 | flds = append(flds, objectKeysField...) 34 | g, err := NewGeneratorWithCustomTemplate(template, Config{}, flds, uint64(b.N)) 35 | defer func() { 36 | _ = g.Close() 37 | }() 38 | 39 | if err != nil { 40 | b.Fatal(err) 41 | } 42 | 43 | var buf bytes.Buffer 44 | 45 | b.ResetTimer() 46 | for i := 0; i < b.N; i++ { 47 | err := g.Emit(&buf) 48 | if err != nil { 49 | b.Fatal(err) 50 | } 51 | buf.Reset() 52 | } 53 | } 54 | 55 | func Benchmark_GeneratorTextTemplateJSONContent(b *testing.B) { 56 | ctx := context.Background() 57 | flds, _, err := fields.LoadFields(ctx, fields.ProductionBaseURL, "endpoint", "process", "8.2.0") 58 | 59 | template, objectKeysField := generateTextTemplateFromField(Config{}, flds) 60 | flds = append(flds, objectKeysField...) 61 | 62 | g, err := NewGeneratorWithTextTemplate(template, Config{}, flds, uint64(b.N)) 63 | defer func() { 64 | _ = g.Close() 65 | }() 66 | 67 | if err != nil { 68 | b.Fatal(err) 69 | } 70 | 71 | var buf bytes.Buffer 72 | 73 | b.ResetTimer() 74 | for i := 0; i < b.N; i++ { 75 | err := g.Emit(&buf) 76 | if err != nil { 77 | b.Fatal(err) 78 | } 79 | buf.Reset() 80 | } 81 | } 82 | 83 | func Benchmark_GeneratorCustomTemplateVPCFlowLogs(b *testing.B) { 84 | flds := Fields{ 85 | { 86 | Name: "Version", 87 | Type: FieldTypeLong, 88 | }, 89 | { 90 | Name: "AccountID", 91 | Type: FieldTypeLong, 92 | }, 93 | { 94 | Name: "InterfaceID", 95 | Type: FieldTypeKeyword, 96 | Example: "eni-1235b8ca123456789", 97 | }, 98 | { 99 | Name: "SrcAddr", 100 | Type: FieldTypeIP, 101 | }, 102 | { 103 | Name: "DstAddr", 104 | Type: FieldTypeIP, 105 | }, 106 | { 107 | Name: "SrcPort", 108 | Type: FieldTypeLong, 109 | }, 110 | { 111 | Name: "DstPort", 112 | Type: FieldTypeLong, 113 | }, 114 | { 115 | Name: "Protocol", 116 | Type: FieldTypeLong, 117 | }, 118 | { 119 | Name: "Packets", 120 | Type: FieldTypeLong, 121 | }, 122 | { 123 | Name: "Bytes", 124 | Type: FieldTypeLong, 125 | }, 126 | { 127 | Name: "Start", 128 | Type: FieldTypeDate, 129 | }, 130 | { 131 | Name: "End", 132 | Type: FieldTypeDate, 133 | }, 134 | { 135 | Name: "Action", 136 | Type: FieldTypeKeyword, 137 | }, 138 | { 139 | Name: "LogStatus", 140 | Type: FieldTypeKeyword, 141 | }, 142 | } 143 | 144 | configYaml := `- name: Version 145 | value: 2 146 | - name: AccountID 147 | value: 627286350134 148 | - name: InterfaceID 149 | cardinality: 100 150 | - name: SrcAddr 151 | cardinality: 1000 152 | - name: DstAddr 153 | cardinality: 10 154 | - name: SrcPort 155 | range: 156 | min: 0 157 | max: 65535 158 | - name: DstPort 159 | range: 160 | min: 0 161 | max: 65535 162 | cardinality: 10 163 | - name: Protocol 164 | range: 165 | min: 1 166 | max: 256 167 | - name: Packets 168 | range: 169 | min: 1 170 | max: 1048576 171 | - name: Bytes 172 | range: 173 | min: 1 174 | max: 15728640 175 | - name: Action 176 | enum: ["ACCEPT", "REJECT"] 177 | - name: LogStatus 178 | enum: ["NODATA", "OK", "SKIPDATA"] 179 | ` 180 | cfg, err := config.LoadConfigFromYaml([]byte(configYaml)) 181 | 182 | if err != nil { 183 | b.Fatal(err) 184 | } 185 | 186 | template := []byte(`{{.Version}} {{.AccountID}} {{.InterfaceID}} {{.SrcAddr}} {{.DstAddr}} {{.SrcPort}} {{.DstPort}} {{.Protocol}} {{.Packets}} {{.Bytes}} {{.Start}} {{.End}} {{.Action}} {{.LogStatus}}`) 187 | g, err := NewGeneratorWithCustomTemplate(template, cfg, flds, uint64(b.N)) 188 | defer func() { 189 | _ = g.Close() 190 | }() 191 | 192 | if err != nil { 193 | b.Fatal(err) 194 | } 195 | 196 | var buf bytes.Buffer 197 | 198 | b.ResetTimer() 199 | for i := 0; i < b.N; i++ { 200 | err := g.Emit(&buf) 201 | if err != nil { 202 | b.Fatal(err) 203 | } 204 | buf.Reset() 205 | } 206 | } 207 | 208 | func Benchmark_GeneratorTextTemplateVPCFlowLogs(b *testing.B) { 209 | flds := Fields{ 210 | { 211 | Name: "Version", 212 | Type: FieldTypeLong, 213 | }, 214 | { 215 | Name: "AccountID", 216 | Type: FieldTypeLong, 217 | }, 218 | { 219 | Name: "InterfaceID", 220 | Type: FieldTypeKeyword, 221 | Example: "eni-1235b8ca123456789", 222 | }, 223 | { 224 | Name: "SrcAddr", 225 | Type: FieldTypeIP, 226 | }, 227 | { 228 | Name: "DstAddr", 229 | Type: FieldTypeIP, 230 | }, 231 | { 232 | Name: "SrcPort", 233 | Type: FieldTypeLong, 234 | }, 235 | { 236 | Name: "DstPort", 237 | Type: FieldTypeLong, 238 | }, 239 | { 240 | Name: "Protocol", 241 | Type: FieldTypeLong, 242 | }, 243 | { 244 | Name: "Packets", 245 | Type: FieldTypeLong, 246 | }, 247 | { 248 | Name: "Bytes", 249 | Type: FieldTypeLong, 250 | }, 251 | { 252 | Name: "Start", 253 | Type: FieldTypeDate, 254 | }, 255 | { 256 | Name: "End", 257 | Type: FieldTypeDate, 258 | }, 259 | { 260 | Name: "Action", 261 | Type: FieldTypeKeyword, 262 | }, 263 | { 264 | Name: "LogStatus", 265 | Type: FieldTypeKeyword, 266 | }, 267 | } 268 | 269 | configYaml := `- name: Version 270 | value: 2 271 | - name: AccountID 272 | value: 627286350134 273 | - name: InterfaceID 274 | cardinality: 100 275 | - name: SrcAddr 276 | cardinality: 1000 277 | - name: DstAddr 278 | cardinality: 10 279 | - name: SrcPort 280 | range: 281 | min: 0 282 | max: 65535 283 | - name: DstPort 284 | range: 285 | min: 0 286 | max: 65535 287 | cardinality: 10 288 | - name: Protocol 289 | range: 290 | min: 1 291 | max: 256 292 | - name: Packets 293 | range: 294 | min: 1 295 | max: 1048576 296 | - name: Bytes 297 | range: 298 | min: 1 299 | max: 15728640 300 | - name: Action 301 | enum: ["ACCEPT", "REJECT"] 302 | - name: LogStatus 303 | enum: ["OK", "SKIPDATA"] 304 | ` 305 | cfg, err := config.LoadConfigFromYaml([]byte(configYaml)) 306 | 307 | if err != nil { 308 | b.Fatal(err) 309 | } 310 | 311 | template := []byte(`{{generate "Version"}} {{generate "AccountID"}} {{generate "InterfaceID"}} {{generate "SrcAddr"}} {{generate "DstAddr"}} {{generate "SrcPort"}} {{generate "DstPort"}} {{generate "Protocol"}}{{ $packets := generate "Packets" }} {{ $packets }} {{generate "Bytes"}} {{$start := generate "Start" }}{{$start.Format "22006-01-02T15:04:05.999999999Z07:00" }} {{$end := generate "End" }}{{$end.Format "2006-01-02T15:04:05.999999Z07:00"}} {{generate "Action"}}{{ if eq $packets 0 }} NODATA {{ else }} {{generate "LogStatus"}} {{ end }}`) 312 | g, err := NewGeneratorWithTextTemplate(template, cfg, flds, uint64(b.N)) 313 | defer func() { 314 | _ = g.Close() 315 | }() 316 | 317 | if err != nil { 318 | b.Fatal(err) 319 | } 320 | 321 | var buf bytes.Buffer 322 | 323 | b.ResetTimer() 324 | for i := 0; i < b.N; i++ { 325 | err := g.Emit(&buf) 326 | if err != nil { 327 | b.Fatal(err) 328 | } 329 | buf.Reset() 330 | } 331 | } 332 | -------------------------------------------------------------------------------- /pkg/genlib/generator_with_custom_template.go: -------------------------------------------------------------------------------- 1 | // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 2 | // or more contributor license agreements. Licensed under the Elastic License 2.0; 3 | // you may not use this file except in compliance with the Elastic License 2.0. 4 | 5 | package genlib 6 | 7 | import ( 8 | "bytes" 9 | "io" 10 | "regexp" 11 | ) 12 | 13 | type emitter struct { 14 | fieldName string 15 | fieldType string 16 | emitFunc emitFNotReturn 17 | prefix []byte 18 | } 19 | 20 | // GeneratorWithCustomTemplate is resolved at construction to a slice of emit functions 21 | type GeneratorWithCustomTemplate struct { 22 | totEvents uint64 23 | emitters []emitter 24 | trailingTemplate []byte 25 | state *genState 26 | } 27 | 28 | func parseCustomTemplate(template []byte) ([]string, map[string][]byte, []byte) { 29 | if len(template) == 0 { 30 | return nil, nil, nil 31 | } 32 | 33 | tokenizer := regexp.MustCompile(`([^{]*)({{\.[^}]+}})*`) 34 | allIndexes := tokenizer.FindAllSubmatchIndex(template, -1) 35 | 36 | orderedFields := make([]string, 0, len(allIndexes)) 37 | templateFieldsMap := make(map[string][]byte, len(allIndexes)) 38 | 39 | var fieldPrefixBuffer []byte 40 | var fieldPrefixPreviousN int 41 | var trimTrailingTemplateN int 42 | 43 | for i, loc := range allIndexes { 44 | var fieldName []byte 45 | var fieldPrefix []byte 46 | 47 | if loc[4] > -1 && loc[5] > -1 { 48 | fieldName = template[loc[4]+3 : loc[5]-2] 49 | } 50 | 51 | if loc[2] > -1 && loc[3] > -1 { 52 | fieldPrefix = template[loc[2]:loc[3]] 53 | } 54 | 55 | if len(fieldName) == 0 { 56 | if template[fieldPrefixPreviousN] == byte(123) { 57 | fieldPrefixBuffer = append(fieldPrefixBuffer, byte(123)) 58 | } else { 59 | if i == len(allIndexes)-1 { 60 | fieldPrefixBuffer = template[trimTrailingTemplateN:] 61 | } else { 62 | fieldPrefixBuffer = append(fieldPrefixBuffer, fieldPrefix...) 63 | fieldPrefixBufferIdx := bytes.Index(template[trimTrailingTemplateN:], fieldPrefixBuffer) 64 | if fieldPrefixBufferIdx > 0 { 65 | trimTrailingTemplateN += fieldPrefixBufferIdx 66 | } 67 | 68 | } 69 | } 70 | } else { 71 | fieldPrefixBuffer = append(fieldPrefixBuffer, fieldPrefix...) 72 | trimTrailingTemplateN = loc[5] 73 | templateFieldsMap[string(fieldName)] = fieldPrefixBuffer 74 | orderedFields = append(orderedFields, string(fieldName)) 75 | fieldPrefixBuffer = nil 76 | } 77 | 78 | fieldPrefixPreviousN = loc[2] 79 | } 80 | 81 | return orderedFields, templateFieldsMap, fieldPrefixBuffer 82 | 83 | } 84 | 85 | func NewGeneratorWithCustomTemplate(template []byte, cfg Config, fields Fields, totEvents uint64) (*GeneratorWithCustomTemplate, error) { 86 | // Parse the template and extract relevant information 87 | orderedFields, templateFieldsMap, trailingTemplate := parseCustomTemplate(template) 88 | 89 | // Preprocess the fields, generating appropriate emit functions 90 | state := newGenState() 91 | fieldMap := make(map[string]any) 92 | fieldTypes := make(map[string]string) 93 | for _, field := range fields { 94 | if err := bindField(cfg, field, fieldMap, false); err != nil { 95 | return nil, err 96 | } 97 | 98 | fieldTypes[field.Name] = field.Type 99 | state.prevCacheForDup[field.Name] = make(map[any]struct{}) 100 | state.prevCacheCardinality[field.Name] = make([]any, 0) 101 | } 102 | 103 | // Roll into slice of emit functions 104 | emitters := make([]emitter, 0, len(fieldMap)) 105 | for _, fieldName := range orderedFields { 106 | emitters = append(emitters, emitter{ 107 | fieldName: fieldName, 108 | emitFunc: fieldMap[fieldName].(emitFNotReturn), 109 | fieldType: fieldTypes[fieldName], 110 | prefix: templateFieldsMap[fieldName], 111 | }) 112 | } 113 | 114 | state.totEvents = totEvents 115 | 116 | return &GeneratorWithCustomTemplate{emitters: emitters, trailingTemplate: trailingTemplate, totEvents: totEvents, state: state}, nil 117 | } 118 | 119 | func (gen *GeneratorWithCustomTemplate) Close() error { 120 | return nil 121 | } 122 | 123 | func (gen *GeneratorWithCustomTemplate) Emit(buf *bytes.Buffer) error { 124 | if err := gen.emit(buf); err != nil { 125 | return err 126 | } 127 | 128 | gen.state.counter += 1 129 | 130 | return nil 131 | } 132 | 133 | func (gen *GeneratorWithCustomTemplate) emit(buf *bytes.Buffer) error { 134 | if gen.totEvents == 0 || gen.state.counter < gen.totEvents { 135 | for _, e := range gen.emitters { 136 | buf.Write(e.prefix) 137 | if err := e.emitFunc(gen.state, buf); err != nil { 138 | return err 139 | } 140 | } 141 | 142 | buf.Write(gen.trailingTemplate) 143 | } else { 144 | return io.EOF 145 | } 146 | 147 | return nil 148 | } 149 | -------------------------------------------------------------------------------- /pkg/genlib/generator_with_text_template.go: -------------------------------------------------------------------------------- 1 | // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 2 | // or more contributor license agreements. Licensed under the Elastic License 2.0; 3 | // you may not use this file except in compliance with the Elastic License 2.0. 4 | 5 | package genlib 6 | 7 | import ( 8 | "bytes" 9 | "errors" 10 | "github.com/Masterminds/sprig/v3" 11 | "io" 12 | "math/rand" 13 | "text/template" 14 | ) 15 | 16 | var generateOnFieldNotInFieldsYaml = errors.New("generate called on a field not present in fields yaml definition") 17 | 18 | // GeneratorWithTextTemplate 19 | type GeneratorWithTextTemplate struct { 20 | tpl *template.Template 21 | state *genState 22 | errChan chan error 23 | totEvents uint64 24 | } 25 | 26 | // awsAZs list all possible AZs for a specific AWS region 27 | // NOTE: this list is not comprehensive 28 | // missing regions: af-south-1, ap-south-2, ap-southeast-3, ap-southeast-4, eu-central-2, eu-south-1, eu-south-2, me-central-1 29 | var awsAZs map[string][]string = map[string][]string{ 30 | "ap-east-1": {"ap-east-1a", "ap-east-1b", "ap-east-1c"}, 31 | "ap-northeast-1": {"ap-northeast-1a", "ap-northeast-1c", "ap-northeast-1d"}, 32 | "ap-northeast-2": {"ap-northeast-2a", "ap-northeast-2b", "ap-northeast-2c", "ap-northeast-2d"}, 33 | "ap-northeast-3": {"ap-northeast-3a", "ap-northeast-3b", "ap-northeast-3c"}, 34 | "ap-south-1": {"ap-south-1a", "ap-south-1b", "ap-south-1c"}, 35 | "ap-southeast-1": {"ap-southeast-1a", "ap-southeast-1b", "ap-southeast-1c"}, 36 | "ap-southeast-2": {"ap-southeast-2a", "ap-southeast-2b", "ap-southeast-2c"}, 37 | "ca-central-1": {"ca-central-1a", "ca-central-1b", "ca-central-1d"}, 38 | "eu-central-1": {"eu-central-1a", "eu-central-1b", "eu-central-1c"}, 39 | "eu-north-1": {"eu-north-1a", "eu-north-1b", "eu-north-1c"}, 40 | "eu-west-1": {"eu-west-1a", "eu-west-1b", "eu-west-1c"}, 41 | "eu-west-2": {"eu-west-2a", "eu-west-2b", "eu-west-2c"}, 42 | "eu-west-3": {"eu-west-3a", "eu-west-3b", "eu-west-3c"}, 43 | "me-south-1": {"me-south-1a", "me-south-1b", "me-south-1c"}, 44 | "sa-east-1": {"sa-east-1a", "sa-east-1b", "sa-east-1c"}, 45 | "us-east-1": {"us-east-1a", "us-east-1b", "us-east-1c", "us-east-1d", "us-east-1e", "us-east-1f"}, 46 | "us-east-2": {"us-east-2a", "us-east-2b", "us-east-2c"}, 47 | "us-west-1": {"us-west-1a", "us-west-1b"}, 48 | "us-west-2": {"us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"}, 49 | } 50 | 51 | func NewGeneratorWithTextTemplate(tpl []byte, cfg Config, fields Fields, totEvents uint64) (*GeneratorWithTextTemplate, error) { 52 | // Preprocess the fields, generating appropriate bound function 53 | state := newGenState() 54 | fieldMap := make(map[string]any) 55 | for _, field := range fields { 56 | if err := bindField(cfg, field, fieldMap, true); err != nil { 57 | return nil, err 58 | } 59 | 60 | state.prevCacheForDup[field.Name] = make(map[any]struct{}) 61 | state.prevCacheCardinality[field.Name] = make([]any, 0) 62 | } 63 | 64 | errChan := make(chan error) 65 | 66 | templateFns := sprig.TxtFuncMap() 67 | 68 | templateFns["awsAZFromRegion"] = func(region string) string { 69 | azs, ok := awsAZs[region] 70 | if !ok { 71 | return "NoAZ" 72 | } 73 | 74 | return azs[rand.Intn(len(azs))] 75 | } 76 | 77 | templateFns["generate"] = func(field string) any { 78 | bindF, ok := fieldMap[field].(emitF) 79 | if !ok { 80 | close(errChan) 81 | return nil 82 | } 83 | 84 | return bindF(state) 85 | } 86 | 87 | t := template.New("generator") 88 | t = t.Option("missingkey=error") 89 | 90 | parsedTpl, err := t.Funcs(templateFns).Parse(string(tpl)) 91 | if err != nil { 92 | return nil, err 93 | } 94 | 95 | state.totEvents = totEvents 96 | 97 | return &GeneratorWithTextTemplate{tpl: parsedTpl, totEvents: totEvents, state: state, errChan: errChan}, nil 98 | } 99 | 100 | func (gen *GeneratorWithTextTemplate) Close() error { 101 | return nil 102 | } 103 | 104 | func (gen *GeneratorWithTextTemplate) Emit(buf *bytes.Buffer) error { 105 | if err := gen.emit(buf); err != nil { 106 | return err 107 | } 108 | 109 | gen.state.counter += 1 110 | return nil 111 | } 112 | 113 | func (gen *GeneratorWithTextTemplate) emit(buf *bytes.Buffer) error { 114 | if gen.totEvents == 0 || gen.state.counter < gen.totEvents { 115 | select { 116 | case <-gen.errChan: 117 | return generateOnFieldNotInFieldsYaml 118 | default: 119 | err := gen.tpl.Execute(buf, nil) 120 | if err != nil { 121 | return err 122 | } 123 | } 124 | } else { 125 | return io.EOF 126 | } 127 | 128 | return nil 129 | } 130 | --------------------------------------------------------------------------------