├── .github └── workflows │ ├── docker.yml │ ├── go.yml │ └── reviewdog.yml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── README.md ├── go.mod ├── go.sum └── main.go /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | name: docker 2 | 3 | on: 4 | push: 5 | branches: 6 | - "**" 7 | tags: 8 | - "v*.*.*" 9 | 10 | jobs: 11 | main: 12 | runs-on: "ubuntu-latest" 13 | steps: 14 | - name: Checkout 15 | uses: actions/checkout@v2 16 | - name: Docker meta 17 | id: docker_meta 18 | uses: crazy-max/ghaction-docker-meta@v1 19 | with: 20 | images: kscarlett/nginx-log-generator,ghcr.io/kscarlett/nginx-log-generator 21 | tag-sha: true 22 | - name: Set up QEMU 23 | uses: docker/setup-qemu-action@v1 24 | - name: Set up Docker Buildx 25 | uses: docker/setup-buildx-action@v1 26 | - name: Cache Docker layers 27 | uses: actions/cache@v2 28 | with: 29 | path: /tmp/.buildx-cache 30 | key: ${{ runner.os }}-buildx-${{ github.sha }} 31 | restore-keys: | 32 | ${{ runner.os }}-buildx- 33 | - name: Login to DockerHub 34 | uses: docker/login-action@v1 35 | with: 36 | username: ${{ secrets.DOCKERHUB_USERNAME }} 37 | password: ${{ secrets.DOCKERHUB_TOKEN }} 38 | - name: Login to GitHub Container Registry 39 | uses: docker/login-action@v1 40 | with: 41 | registry: ghcr.io 42 | username: ${{ github.repository_owner }} 43 | password: ${{ secrets.CR_PAT }} 44 | - name: Build and push 45 | id: docker_build 46 | uses: docker/build-push-action@v2 47 | with: 48 | context: . 49 | file: ./Dockerfile 50 | platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64 51 | push: ${{ github.event_name != 'pull_request' }} 52 | tags: ${{ steps.docker_meta.outputs.tags }} 53 | labels: ${{ steps.docker_meta.outputs.labels }} 54 | # Disabled because this keeps giving a 403 error. 55 | # - name: Docker Hub Description 56 | # uses: peter-evans/dockerhub-description@v2 57 | # with: 58 | # username: ${{ secrets.DOCKERHUB_USERNAME }} 59 | # password: ${{ secrets.DOCKERHUB_PASSWORD }} 60 | # repository: peterevans/dockerhub-description 61 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: go test 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | test: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: Install Go 10 | uses: actions/setup-go@v2 11 | with: 12 | go-version: 1.15.x 13 | - name: Checkout code 14 | uses: actions/checkout@v2 15 | - uses: actions/cache@v2 16 | with: 17 | path: ~/go/pkg/mod 18 | key: go-${{ hashFiles('**/go.sum') }} 19 | restore-keys: | 20 | go- 21 | - name: Test 22 | run: go test ./... 23 | -------------------------------------------------------------------------------- /.github/workflows/reviewdog.yml: -------------------------------------------------------------------------------- 1 | name: reviewdog 2 | 3 | on: [pull_request] 4 | 5 | jobs: 6 | hadolint: 7 | name: runner / hadolint 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Check out code 11 | uses: actions/checkout@v1 12 | - name: hadolint 13 | uses: reviewdog/action-hadolint@v1 14 | with: 15 | github_token: ${{ secrets.github_token }} 16 | hadolint_ignore: DL3006 # No need to explicitly tag the build image 17 | tool_name: hadolint 18 | golangci-lint: 19 | name: runner / golangci-lint 20 | runs-on: ubuntu-latest 21 | steps: 22 | - name: Check out code into the Go module directory 23 | uses: actions/checkout@v2 24 | - name: golangci-lint 25 | uses: reviewdog/action-golangci-lint@v1 26 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # CHANGELOG 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | ## [v1.1.0] - 2021-09-18 6 | 7 | You are now able to configure the tool better with a bunch of environment variables. See [README.md](README.md#configuration) for an up-to-date list of variables with their defaults. 8 | 9 | ## [v1.0.1] - 2021-02-12 10 | 11 | Minor release to fix automated deployments to Docker Hub - see https://github.com/peter-evans/dockerhub-description/issues/10 -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | I'm more than happy to receive contributions from others for this tool. If you have made any changes that you think others may benefit from, feel free to open a pull request. If you're not sure whether it may be useful or don't know where to start, you can open an issue for discussion. 4 | 5 | ## Pull Requests 6 | 7 | To make everything go as smoothly as possible, please open a pull request to the `dev` branch. 8 | 9 | If you make any changes that require documentation, please update the README accordingly. -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:alpine AS builder 2 | 3 | WORKDIR /workspace 4 | 5 | # Copy in just the go.mod and go.sum files, and download the dependencies. By 6 | # doing this before copying in the other dependencies, the Docker build cache 7 | # can skip these steps so long as neither of these two files change. 8 | COPY go.mod go.sum ./ 9 | RUN go mod download 10 | 11 | # Copy the rest 12 | COPY . . 13 | 14 | # Build the Go app with CGO_ENABLED=0 so we use the pure-Go implementations for 15 | # things like DNS resolution (so we don't build a binary that depends on system 16 | # libraries) 17 | RUN CGO_ENABLED=0 go build -o /app 18 | 19 | # Create the 'nobody' user and group files that will be used in the running container to 20 | # run the process an unprivileged user. 21 | RUN mkdir /user && \ 22 | echo 'nobody:x:65534:65534:nobody:/:' > /user/passwd && \ 23 | echo 'nobody:x:65534:' > /user/group 24 | 25 | # The final stage 26 | FROM scratch 27 | 28 | # Copy the binary from the builder stage 29 | COPY --from=builder /app /app 30 | 31 | # Copy the /etc/passwd file we created in the builder stage. This creates a new 32 | # non-root user as a security best practice. 33 | COPY --from=builder /user/group /user/passwd /etc/ 34 | 35 | # Run as the new non-root by default 36 | USER nobody:nobody 37 | 38 | # Run the binary 39 | ENTRYPOINT [ "/app" ] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Kellen Scarlett 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Nginx Log Generator 2 | 3 | A tiny Go utility to generate a large amount realistic-looking Nginx logs quickly. It was written to aid in testing logging pipelines and other such tools, and demoing them in Kubernetes. 4 | 5 | Most of the heavy lifting is done by the amazing [gofakeit](https://github.com/brianvoe/gofakeit) library, with some extra work to skew the results towards typical values. 6 | 7 | ## Usage 8 | 9 | The most important step is to set the desired rate in the `RATE` environment variable. The simplest way to do this is the following: 10 | 11 | ```sh 12 | $ # Will generate 10 entries per second 13 | $ RATE=10 ./nginx-log-generator 14 | ``` 15 | 16 | The reason this is an environment variable is so it's easier to run via Docker as well: 17 | 18 | ```sh 19 | $ docker pull kscarlett/nginx-log-generator 20 | $ docker run -e "RATE=10" kscarlett/nginx-log-generator 21 | ``` 22 | 23 | ### Configuration 24 | 25 | The following environment variables can be set to modify the output. 26 | 27 | | Name | Default | Notes | 28 | | ----------------- | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | 29 | | RATE | 1 | Logs to output per second. | 30 | | IPV4_PERCENT | 100 | Percentage of IP addresses that will be IPv4. Change to 0 to only get IPv6. | 31 | | STATUS_OK_PERCENT | 80 | _Roughly_ the percentage of `200` status codes. The rest will be randomised and may contain `200` as well. | 32 | | PATH_MIN | 1 | Minimum elements to put in the request path. | 33 | | PATH_MAX | 5 | Maximum elements to put in the request path. | 34 | | GET_PERCENT | 60 | Percentage of requests that will be `GET` requests. If the total adds up to less than 100%, the rest will be made up of random HTTP methods. | 35 | | POST_PERCENT | 30 | Percentage of requests that will be `POST` requests. If the total adds up to less than 100%, the rest will be made up of random HTTP methods. | 36 | | PUT_PERCENT | 0 | Percentage of requests that will be `PUT` requests. If the total adds up to less than 100%, the rest will be made up of random HTTP methods. | 37 | | PATCH_PERCENT | 0 | Percentage of requests that will be `PATCH` requests. If the total adds up to less than 100%, the rest will be made up of random HTTP methods. | 38 | | DELETE_PERCENT | 0 | Percentage of requests that will be `DELETE` requests. If the total adds up to less than 100%, the rest will be made up of random HTTP methods. | 39 | 40 | ## Note 41 | 42 | This is a tool I made in no time at all, because I needed a tool that did exactly this right that second. The code quality isn't optimal and it can probably be optimised. I will be coming back to it at some other time. 43 | 44 | ## License 45 | 46 | This tool is released under the [MIT License](LICENSE). -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/kscarlett/nginx-log-generator 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/brianvoe/gofakeit/v6 v6.8.0 7 | github.com/caarlos0/env/v6 v6.7.1 8 | ) 9 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/brianvoe/gofakeit/v6 v6.8.0 h1:qAAlgNUqnRc/CU04WwBPspE5cAghce2YHRYihIKv2gI= 2 | github.com/brianvoe/gofakeit/v6 v6.8.0/go.mod h1:palrJUk4Fyw38zIFB/uBZqsgzW5VsNllhHKKwAebzew= 3 | github.com/caarlos0/env/v6 v6.7.1 h1:2r2GyonA8aJX6lDEhwFfpxwAX8Z3mvbE1X6vhaSzEyU= 4 | github.com/caarlos0/env/v6 v6.7.1/go.mod h1:FE0jGiAnQqtv2TenJ4KTa8+/T2Ss8kdS5s1VEjasoN0= 5 | github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= 6 | github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= 7 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "time" 7 | 8 | "github.com/brianvoe/gofakeit/v6" 9 | "github.com/caarlos0/env/v6" 10 | ) 11 | 12 | type config struct { 13 | Rate float32 `env:"RATE" envDefault:"1"` 14 | IPv4Percent int `env:"IPV4_PERCENT" envDefault:"100"` 15 | StatusOkPercent int `env:"STATUS_OK_PERCENT" envDefault:"80"` 16 | PathMinLength int `env:"PATH_MIN" envDefault:"1"` 17 | PathMaxLength int `env:"PATH_MAX" envDefault:"5"` 18 | PercentageGet int `env:"GET_PERCENT" envDefault:"60"` 19 | PercentagePost int `env:"POST_PERCENT" envDefault:"30"` 20 | PercentagePut int `env:"PUT_PERCENT" envDefault:"0"` 21 | PercentagePatch int `env:"PATCH_PERCENT" envDefault:"0"` 22 | PercentageDelete int `env:"DELETE_PERCENT" envDefault:"0"` 23 | } 24 | 25 | func main() { 26 | cfg := config{} 27 | if err := env.Parse(&cfg); err != nil { 28 | panic(err) 29 | } 30 | checkMinMax(&cfg.PathMinLength, &cfg.PathMaxLength) 31 | 32 | ticker := time.NewTicker(time.Second / time.Duration(cfg.Rate)) 33 | 34 | gofakeit.Seed(time.Now().UnixNano()) 35 | 36 | var ip, httpMethod, path, httpVersion, referrer, userAgent string 37 | var statusCode, bodyBytesSent int 38 | var timeLocal time.Time 39 | 40 | httpVersion = "HTTP/1.1" 41 | referrer = "-" 42 | 43 | for range ticker.C { 44 | timeLocal = time.Now() 45 | 46 | ip = weightedIPVersion(cfg.IPv4Percent) 47 | httpMethod = weightedHTTPMethod(cfg.PercentageGet, cfg.PercentagePost, cfg.PercentagePut, cfg.PercentagePatch, cfg.PercentageDelete) 48 | path = randomPath(cfg.PathMinLength, cfg.PathMaxLength) 49 | statusCode = weightedStatusCode(cfg.StatusOkPercent) 50 | bodyBytesSent = realisticBytesSent(statusCode) 51 | userAgent = gofakeit.UserAgent() 52 | 53 | fmt.Printf("%s - - [%s] \"%s %s %s\" %v %v \"%s\" \"%s\"\n", ip, timeLocal.Format("02/Jan/2006:15:04:05 -0700"), httpMethod, path, httpVersion, statusCode, bodyBytesSent, referrer, userAgent) 54 | } 55 | } 56 | 57 | func realisticBytesSent(statusCode int) int { 58 | if statusCode != 200 { 59 | return gofakeit.Number(30, 120) 60 | } 61 | 62 | return gofakeit.Number(800, 3100) 63 | } 64 | 65 | func weightedStatusCode(percentageOk int) int { 66 | roll := gofakeit.Number(0, 100) 67 | if roll <= percentageOk { 68 | return 200 69 | } 70 | 71 | return gofakeit.HTTPStatusCodeSimple() 72 | } 73 | 74 | func weightedHTTPMethod(percentageGet, percentagePost, percentagePut, percentagePatch, percentageDelete int) string { 75 | if percentageGet+percentagePost >= 100 { 76 | panic("HTTP method percentages add up to more than 100%") 77 | } 78 | 79 | roll := gofakeit.Number(0, 100) 80 | if roll <= percentageGet { 81 | return "GET" 82 | } else if roll <= percentagePost { 83 | return "POST" 84 | } else if roll <= percentagePut { 85 | return "PUT" 86 | } else if roll <= percentagePatch { 87 | return "PATCH" 88 | } else if roll <= percentageDelete { 89 | return "DELETE" 90 | } 91 | 92 | return gofakeit.HTTPMethod() 93 | } 94 | 95 | func weightedIPVersion(percentageIPv4 int) string { 96 | roll := gofakeit.Number(0, 100) 97 | if roll <= percentageIPv4 { 98 | return gofakeit.IPv4Address() 99 | } else { 100 | return gofakeit.IPv6Address() 101 | } 102 | } 103 | 104 | func randomPath(min, max int) string { 105 | var path strings.Builder 106 | length := gofakeit.Number(min, max) 107 | 108 | path.WriteString("/") 109 | 110 | for i := 0; i < length; i++ { 111 | if i > 0 { 112 | path.WriteString(gofakeit.RandomString([]string{"-", "-", "_", "%20", "/", "/", "/"})) 113 | } 114 | path.WriteString(gofakeit.BuzzWord()) 115 | } 116 | 117 | path.WriteString(gofakeit.RandomString([]string{".hmtl", ".php", ".htm", ".jpg", ".png", ".gif", ".svg", ".css", ".js"})) 118 | 119 | result := path.String() 120 | return strings.Replace(result, " ", "%20", -1) 121 | } 122 | 123 | func checkMinMax(min, max *int) { 124 | if *min < 1 { 125 | *min = 1 126 | } 127 | if *max < 1 { 128 | *max = 1 129 | } 130 | if *min > *max { 131 | *min, *max = *max, *min 132 | } 133 | } 134 | --------------------------------------------------------------------------------