├── .github └── workflows │ ├── build_upload_all.yml │ ├── ci_integration_go.yml │ ├── launch_debugger.yml │ └── zulip_notifier.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── config ├── .env ├── Dockerfile └── docker-compose.yml ├── control ├── Dockerfile ├── Makefile ├── control.go ├── go.mod └── go.sum ├── vault ├── Dockerfile ├── Makefile ├── go.mod ├── go.sum └── vault.go └── workload ├── Dockerfile ├── Makefile └── workload.sh /.github/workflows/build_upload_all.yml: -------------------------------------------------------------------------------- 1 | name: Build & Upload All 2 | 3 | on: [workflow_dispatch] 4 | 5 | jobs: 6 | antithesis: 7 | runs-on: ubuntu-latest 8 | 9 | permissions: 10 | contents: read 11 | packages: write 12 | 13 | steps: 14 | - uses: actions/checkout@v4 15 | 16 | - name: Set up Docker Buildx 17 | uses: docker/setup-buildx-action@v3 18 | 19 | - name: Login to Docker Hub 20 | uses: docker/login-action@v3 21 | with: 22 | # These secrets would need to be populated in your repo and named 23 | # this way if you want to copy-and-paste this configuration. 24 | username: ${{ secrets.DOCKER_USERNAME }} 25 | password: ${{ secrets.DOCKER_PAT }} 26 | 27 | # There is a lot of copy-and-paste in these sections. 28 | # This could be streamlined by using a matrix configuration, which would 29 | # parameterize the extract and build stages. However, it would run both the 30 | # setup and teardown sections N times, once for each of the N containers we 31 | # are going to build and deploy. 32 | - name: Extract metadata (tags) for Docker workload 33 | id: meta-workload 34 | uses: docker/metadata-action@v5 35 | with: 36 | images: antithesishq/demo-workload 37 | tags: | 38 | type=sha 39 | antithesis 40 | - name: Extract metadata (tags) for Docker Go config 41 | id: meta-go-config 42 | uses: docker/metadata-action@v5 43 | with: 44 | images: antithesishq/demo-go-config 45 | tags: | 46 | type=sha 47 | antithesis 48 | - name: Extract metadata (tags) for Docker Go control 49 | id: meta-go-control 50 | uses: docker/metadata-action@v5 51 | with: 52 | images: antithesishq/demo-go-control 53 | tags: | 54 | type=sha 55 | antithesis 56 | - name: Extract metadata (tags) for Docker Go vault 57 | id: meta-go-vault 58 | uses: docker/metadata-action@v5 59 | with: 60 | images: antithesishq/demo-go-vault 61 | tags: | 62 | type=sha 63 | antithesis 64 | 65 | # Now build and push each of the containers. 66 | - name: Build and push workload 67 | uses: docker/build-push-action@v5 68 | with: 69 | context: ./workload 70 | file: ./workload/Dockerfile.workload 71 | push: true 72 | tags: ${{ steps.meta-workload.outputs.tags }} 73 | labels: ${{ steps.meta-workload.outputs.labels }} 74 | - name: Build and push Go config 75 | uses: docker/build-push-action@v5 76 | with: 77 | context: ./go 78 | file: ./go/Dockerfile.config 79 | push: true 80 | tags: ${{ steps.meta-go-config.outputs.tags }} 81 | labels: ${{ steps.meta-go-config.outputs.labels }} 82 | - name: Build and push Go control 83 | uses: docker/build-push-action@v5 84 | with: 85 | context: ./go 86 | file: ./go/Dockerfile.control 87 | push: true 88 | tags: ${{ steps.meta-go-control.outputs.tags }} 89 | labels: ${{ steps.meta-go-control.outputs.labels }} 90 | - name: Build and push Go vault 91 | uses: docker/build-push-action@v5 92 | with: 93 | context: ./go 94 | file: ./go/Dockerfile.vault 95 | push: true 96 | tags: ${{ steps.meta-go-vault.outputs.tags }} 97 | labels: ${{ steps.meta-go-vault.outputs.labels }} 98 | -------------------------------------------------------------------------------- /.github/workflows/ci_integration_go.yml: -------------------------------------------------------------------------------- 1 | name: Continous Integration (Go Services) 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | test: 7 | description: 'Test name' 8 | required: false 9 | default: '' 10 | type: string 11 | pull_request: 12 | schedule: 13 | - cron: "5 1 * * *" 14 | 15 | jobs: 16 | antithesis: 17 | runs-on: ubuntu-latest 18 | 19 | permissions: 20 | contents: read 21 | packages: write 22 | 23 | steps: 24 | - uses: actions/checkout@v4 25 | 26 | - name: Set up Docker Buildx 27 | uses: docker/setup-buildx-action@v3 28 | 29 | - name: Login to Docker Hub 30 | uses: docker/login-action@v3 31 | with: 32 | # These secrets would need to be populated in your repo and named 33 | # this way if you want to copy-and-paste this configuration. 34 | username: ${{ secrets.DOCKER_USERNAME }} 35 | password: ${{ secrets.DOCKER_PAT }} 36 | 37 | # Extract the metadata (tags) for all the containers 38 | - name: Extract metadata (tags) for Docker workload 39 | id: meta-workload 40 | uses: docker/metadata-action@v5 41 | with: 42 | images: antithesishq/demo-workload 43 | tags: | 44 | type=sha 45 | antithesis 46 | - name: Extract metadata (tags) for Docker Go config 47 | id: meta-go-config 48 | uses: docker/metadata-action@v5 49 | with: 50 | images: antithesishq/demo-go-config 51 | tags: | 52 | type=sha 53 | antithesis 54 | - name: Extract metadata (tags) for Docker Go control 55 | id: meta-go-control 56 | uses: docker/metadata-action@v5 57 | with: 58 | images: antithesishq/demo-go-control 59 | tags: | 60 | type=sha 61 | antithesis 62 | - name: Extract metadata (tags) for Docker Go vault 63 | id: meta-go-vault 64 | uses: docker/metadata-action@v5 65 | with: 66 | images: antithesishq/demo-go-vault 67 | tags: | 68 | type=sha 69 | antithesis 70 | 71 | # Now build and push each of the containers. 72 | - name: Build and push workload 73 | id: build-push-go-workload 74 | uses: docker/build-push-action@v5 75 | with: 76 | context: ./workload 77 | file: ./workload/Dockerfile 78 | push: true 79 | tags: ${{ steps.meta-workload.outputs.tags }} 80 | labels: ${{ steps.meta-workload.outputs.labels }} 81 | - name: Build and push Go config 82 | id: build-push-go-config 83 | uses: docker/build-push-action@v5 84 | with: 85 | context: ./config 86 | file: ./config/Dockerfile 87 | push: true 88 | tags: ${{ steps.meta-go-config.outputs.tags }} 89 | labels: ${{ steps.meta-go-config.outputs.labels }} 90 | - name: Build and push Go control 91 | id: build-push-go-control 92 | uses: docker/build-push-action@v5 93 | with: 94 | context: ./control 95 | file: ./control/Dockerfile 96 | push: true 97 | tags: ${{ steps.meta-go-control.outputs.tags }} 98 | labels: ${{ steps.meta-go-control.outputs.labels }} 99 | - name: Build and push Go vault 100 | id: build-push-go-vault 101 | uses: docker/build-push-action@v5 102 | with: 103 | context: ./vault 104 | file: ./vault/Dockerfile 105 | push: true 106 | tags: ${{ steps.meta-go-vault.outputs.tags }} 107 | labels: ${{ steps.meta-go-vault.outputs.labels }} 108 | 109 | # Run Antithesis Tests 110 | - name: Run Antithesis Tests 111 | uses: antithesishq/antithesis-trigger-action@main 112 | with: 113 | notebook_name: demo_go 114 | tenant: demo 115 | username: ${{ secrets.ANTITHESIS_USERNAME }} 116 | password: ${{ secrets.ANTITHESIS_PASSWORD }} 117 | github_token: ${{ secrets.GH_PAT }} 118 | config_image: demo-go-config@${{ steps.build-push-go-config.outputs.digest }} 119 | images: docker.io/antithesishq/demo-workload@${{ steps.build-push-go-workload.outputs.digest }};docker.io/antithesishq/demo-go-control@${{ steps.build-push-go-control.outputs.digest }};demo-go-vault@${{ steps.build-push-go-vault.outputs.digest }}; 120 | description: "The CI run for ref - ${{ github.ref_name }} commit # ${{ github.sha }}" 121 | test_name: ${{ inputs.test }} 122 | additional_parameters: |- 123 | custom.repo_name=${{github.repository}} 124 | custom.action_name = ${{github.action}} 125 | custom.actor = ${{ github.actor }} 126 | 127 | -------------------------------------------------------------------------------- /.github/workflows/launch_debugger.yml: -------------------------------------------------------------------------------- 1 | name: Launch Debugger 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | session: 7 | description: 'Session id' 8 | required: true 9 | default: '' 10 | type: string 11 | input_hash: 12 | description: 'Input hash' 13 | required: true 14 | default: '' 15 | type: number 16 | vtime: 17 | description: 'vtime' 18 | required: true 19 | default: '' 20 | type: number 21 | email: 22 | description: 'email' 23 | required: true 24 | default: '' 25 | type: string 26 | 27 | jobs: 28 | antithesis: 29 | runs-on: ubuntu-latest 30 | 31 | permissions: 32 | contents: read 33 | packages: write 34 | 35 | steps: 36 | - uses: actions/checkout@v4 37 | 38 | # Start Antithesis Debugging 39 | - name: Start Antithesis Debugging 40 | uses: antithesishq/antithesis-trigger-action@main 41 | with: 42 | notebook_name: launch_debugging 43 | tenant: demo 44 | username: ${{ secrets.ANTITHESIS_USERNAME }} 45 | password: ${{ secrets.ANTITHESIS_PASSWORD }} 46 | github_token: ${{ secrets.GH_PAT }} 47 | email_recipients: ${{ inputs.email }} 48 | additional_parameters: |- 49 | antithesis.debugging.session_id=${{ inputs.session }} 50 | antithesis.debugging.input_hash = ${{ inputs.input_hash }} 51 | antithesis.debugging.vtime = ${{ inputs.vtime }} 52 | -------------------------------------------------------------------------------- /.github/workflows/zulip_notifier.yml: -------------------------------------------------------------------------------- 1 | name: Zulip Notification Bot 2 | 3 | on: 4 | push: 5 | 6 | permissions: 7 | contents: read 8 | 9 | jobs: 10 | post-to-zulip: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v3 15 | - uses: actions/github-script@v7 16 | id: generate-msg 17 | with: 18 | script: | 19 | let message = `- **${context.actor}** \`${context.ref}\` | ${context.sha.substring(0,7)} | [${context.payload.head_commit.message?.split('\n')[0]}](${context.payload.compare})` 20 | let topic = context.repo.repo 21 | core.setOutput("topic", topic); 22 | core.setOutput("msg", message); 23 | 24 | - name: Send a stream message 25 | uses: zulip/github-actions-zulip/send-message@v1 26 | with: 27 | api-key: ${{ secrets.ZULIP_API_KEY }} 28 | email: ${{ secrets.ZULIP_BOT_EMAIL }} 29 | organization-url: ${{ secrets.ZULIP_ORG_URL }} 30 | to: "Commits" 31 | type: "stream" 32 | topic: ${{ steps.generate-msg.outputs.topic }} 33 | content: ${{ steps.generate-msg.outputs.msg }} 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Golang ignore 2 | # Binaries for programs and plugins 3 | *.exe 4 | *.exe~ 5 | *.dll 6 | *.so 7 | *.dylib 8 | 9 | # Test binary, built with `go test -c` 10 | *.test 11 | 12 | # Our binaries 13 | # control 14 | # vault 15 | 16 | # Output of the go coverage tool, specifically when used with LiteIDE 17 | *.out 18 | 19 | # Dependency directories (remove the comment below to include it) 20 | # vendor/ 21 | 22 | # Go workspace file 23 | go.work 24 | 25 | ## Python ignore 26 | # Byte-compiled / optimized / DLL files 27 | __pycache__/ 28 | *.py[cod] 29 | *$py.class 30 | 31 | # C extensions 32 | *.so 33 | 34 | # Distribution / packaging 35 | .Python 36 | build/ 37 | develop-eggs/ 38 | dist/ 39 | downloads/ 40 | eggs/ 41 | .eggs/ 42 | lib/ 43 | lib64/ 44 | parts/ 45 | sdist/ 46 | var/ 47 | wheels/ 48 | share/python-wheels/ 49 | *.egg-info/ 50 | .installed.cfg 51 | *.egg 52 | MANIFEST 53 | 54 | # PyInstaller 55 | # Usually these files are written by a python script from a template 56 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 57 | *.manifest 58 | *.spec 59 | 60 | # Installer logs 61 | pip-log.txt 62 | pip-delete-this-directory.txt 63 | 64 | # Unit test / coverage reports 65 | htmlcov/ 66 | .tox/ 67 | .nox/ 68 | .coverage 69 | .coverage.* 70 | .cache 71 | nosetests.xml 72 | coverage.xml 73 | *.cover 74 | *.py,cover 75 | .hypothesis/ 76 | .pytest_cache/ 77 | cover/ 78 | 79 | # Translations 80 | *.mo 81 | *.pot 82 | 83 | # Django stuff: 84 | *.log 85 | local_settings.py 86 | db.sqlite3 87 | db.sqlite3-journal 88 | 89 | # Flask stuff: 90 | instance/ 91 | .webassets-cache 92 | 93 | # Scrapy stuff: 94 | .scrapy 95 | 96 | # Sphinx documentation 97 | docs/_build/ 98 | 99 | # PyBuilder 100 | .pybuilder/ 101 | target/ 102 | 103 | # Jupyter Notebook 104 | .ipynb_checkpoints 105 | 106 | # IPython 107 | profile_default/ 108 | ipython_config.py 109 | 110 | # pyenv 111 | # For a library or package, you might want to ignore these files since the code is 112 | # intended to run in multiple environments; otherwise, check them in: 113 | # .python-version 114 | 115 | # pipenv 116 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 117 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 118 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 119 | # install all needed dependencies. 120 | #Pipfile.lock 121 | 122 | # poetry 123 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 124 | # This is especially recommended for binary packages to ensure reproducibility, and is more 125 | # commonly ignored for libraries. 126 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 127 | #poetry.lock 128 | 129 | # pdm 130 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 131 | #pdm.lock 132 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 133 | # in version control. 134 | # https://pdm.fming.dev/#use-with-ide 135 | .pdm.toml 136 | 137 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 138 | __pypackages__/ 139 | 140 | # Celery stuff 141 | celerybeat-schedule 142 | celerybeat.pid 143 | 144 | # SageMath parsed files 145 | *.sage.py 146 | 147 | # Environments 148 | .venv 149 | env/ 150 | venv/ 151 | ENV/ 152 | env.bak/ 153 | venv.bak/ 154 | 155 | # Spyder project settings 156 | .spyderproject 157 | .spyproject 158 | 159 | # Rope project settings 160 | .ropeproject 161 | 162 | # mkdocs documentation 163 | /site 164 | 165 | # mypy 166 | .mypy_cache/ 167 | .dmypy.json 168 | dmypy.json 169 | 170 | # Pyre type checker 171 | .pyre/ 172 | 173 | # pytype static type analyzer 174 | .pytype/ 175 | 176 | # Cython debug symbols 177 | cython_debug/ 178 | 179 | # Random but common clutter 180 | .DS_Store 181 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 antithesishq 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Top-level makefile. Go into each subdirectory and run make 2 | # SUBDIRS := $(wildcard system_under_test/*/.) 3 | 4 | SUBDIRS := ./control/ ./vault/ ./workload/ 5 | 6 | all: $(SUBDIRS) 7 | $(SUBDIRS): 8 | $(MAKE) -C $@ 9 | 10 | .PHONY: all $(SUBDIRS) 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Motivation 2 | 3 | Glitch Grid is a toy distributed system for demonstrating how to 4 | integrate with the [Antithesis Platform](https://antithesis.com/). This project demonstrates: 5 | 6 | * Use of [the Antithesis SDK]((https://antithesis.com/docs/using_antithesis/sdk/overview.html#)) 7 | to [define properties](https://antithesis.com/docs/using_antithesis/properties/) 8 | * Instrumenting a Go project for [coverage information](https://antithesis.com/docs/instrumentation/overview.html) 9 | * Triggering tests and receiving results using [Github Actions](https://antithesis.com/docs/using_antithesis/ci.html) 10 | 11 | Why demonstrate with a toy project? Most software has bugs, but for demonstration purposes 12 | we wanted obvious, low-context bugs that project maintainers would not fix. Welcome, GlitchGrid! 13 | 14 | Please refer to the [Antithesis Documentation](https://antithesis.com/docs/) for more information about 15 | how to get started with Antithesis, best practices, etc. 16 | 17 | ### SDK Use 18 | 19 | This project demonstrates how to [use the Antithesis SDK](https://github.com/search?q=repo%3Aantithesishq%2Fglitch-grid+%28Always+OR+Sometimes%29&type=code) to add assertions about your software. 20 | This includes conventional assertions, and also [Sometimes Assertions](https://antithesis.com/docs/best_practices/sometimes_assertions.html) which can help you 21 | assess the quality of your testing or check for unreachable code. 22 | 23 | When software starts up in the Antithesis platform, there is usually setup work during which 24 | injecting faults is not productive. Because of this, Antithesis waits to start injecting 25 | faults until the software under test indicates that it is booted and ready. The SDK 26 | [lifecycle functions](https://antithesis.com/docs/using_antithesis/sdk/overview.html#) 27 | [are used](https://github.com/search?q=repo%3Aantithesishq%2Fglitch-grid+SetupComplete&type=code) 28 | to coordinate with the simulation. 29 | 30 | ### Architecture 31 | 32 | This is a distributed system which will remember the most recent positive 33 | integer sent to it by a client. Clients talk to a mostly-stateless **control** server, 34 | which in turn will write to and read from one or more **vault** servers. Operations 35 | are only successful if *more than 50%* of vaults report success. The number stored in 36 | the system *should* only increase in value; vaults will log an error if the number 37 | decreases, but will not block the update. Both the control server and all vaults are 38 | multi-threaded. 39 | 40 | There are two types of error states a client might see from the system: 41 | * on both reads and writes, the system may report it is in an *inconsistent* state if 42 | it cannot get a successful response from a majority of the vaults; 43 | * on reads, a client might find the system in an *incorrect* state if the value the 44 | system returns does not match the most recent value written by the client. 45 | 46 | The control server can detect if the system is in an *inconsistent* state, but only the 47 | client can detect if the system is in an *incorrect* state. 48 | 49 | The language-agnostic workload will attempt to write increasingly-large numbers to the 50 | system, and occasionally pause to confirm that the value it reads from the system 51 | matches the most recent value it wrote to the system. At the end of the test, it will 52 | pause for a few seconds before performing one final read to confirm that the system 53 | completed in a correct and consistent state. 54 | 55 | ### Workload 56 | 57 | The test workload is written in bash and uses `curl` to perform reads and writes. The workload 58 | can be configured using environment variables, controlling everything from startup times to the 59 | number of values written to the system during the test to (approximately) how often it will stop 60 | writing values and check if the system is in the correct state. 61 | 62 | ### Go 63 | 64 | The Go implementation can be found in the `go/` subdirectory. It requires Go version 1.20 or higher. 65 | -------------------------------------------------------------------------------- /config/.env: -------------------------------------------------------------------------------- 1 | IMAGE_TAG=antithesis 2 | -------------------------------------------------------------------------------- /config/Dockerfile: -------------------------------------------------------------------------------- 1 | # The config image is only used to bring up the containers 2 | # of the system under test and the workload generator. 3 | FROM scratch 4 | 5 | COPY docker-compose.yml .env / 6 | -------------------------------------------------------------------------------- /config/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | 3 | services: 4 | # Define three vaults which can provide some degree of replication/redundancy. 5 | vault1: 6 | command: "--port 8001 --logtostderr --stderrthreshold=INFO" 7 | image: demo-go-vault:${IMAGE_TAG} 8 | build: ../vault/ 9 | container_name: vault1 10 | ports: 11 | - "8001:8001" 12 | networks: 13 | go-demo: 14 | ipv4_address: 10.0.1.121 15 | 16 | vault2: 17 | command: "--port 8002 --logtostderr --stderrthreshold=INFO" 18 | image: demo-go-vault:${IMAGE_TAG} 19 | build: ../vault/ 20 | container_name: vault2 21 | ports: 22 | - "8002:8002" 23 | networks: 24 | go-demo: 25 | ipv4_address: 10.0.1.122 26 | 27 | vault3: 28 | command: "--port 8003 --logtostderr --stderrthreshold=INFO" 29 | image: demo-go-vault:${IMAGE_TAG} 30 | build: ../vault/ 31 | container_name: vault3 32 | ports: 33 | - "8003:8003" 34 | networks: 35 | go-demo: 36 | ipv4_address: 10.0.1.123 37 | 38 | control: 39 | # The address of the various vaults we've defined. 40 | # Define some extra logging for the controller. 41 | command: "--vaults 10.0.1.121:8001,10.0.1.122:8002,10.0.1.123:8003 --logtostderr --stderrthreshold=INFO" 42 | image: demo-go-control:${IMAGE_TAG} 43 | build: ../control/ 44 | container_name: control 45 | ports: 46 | - "8000:8000" 47 | networks: 48 | go-demo: 49 | ipv4_address: 10.0.1.120 50 | 51 | workload: 52 | # The address of the controller 53 | command: "10.0.1.120:8000" 54 | image: demo-workload:${IMAGE_TAG} 55 | build: ../workload/ 56 | container_name: workload 57 | environment: 58 | - START_DELAY=15 59 | - FINAL_DELAY=5 60 | - NUM_STEPS=100 61 | - MAX_STEP_SIZE=3 62 | - CHECK_WHEN_MULTIPLE_OF=10 63 | networks: 64 | go-demo: 65 | ipv4_address: 10.0.1.110 66 | 67 | networks: 68 | go-demo: 69 | driver: bridge 70 | ipam: 71 | config: 72 | - subnet: 10.0.1.0/24 73 | -------------------------------------------------------------------------------- /control/Dockerfile: -------------------------------------------------------------------------------- 1 | # Stage 1: Get Golang image 2 | FROM docker.io/library/golang:1.20-bookworm AS builder 3 | LABEL maintainer="Antithesis " 4 | 5 | # Add source code 6 | RUN mkdir -p /go/src/antithesis/control 7 | COPY go.sum go.mod *.go /go/src/antithesis/control/ 8 | 9 | # Download and install antithesis-go-instrumentor 10 | # Installs into $GOPATH/bin => /go/bin 11 | RUN cd /go/src/antithesis/control && \ 12 | go install github.com/antithesishq/antithesis-sdk-go/tools/antithesis-go-instrumentor@v0.3.6 && \ 13 | go mod tidy 14 | 15 | # Create the destination output directory for the instrumented code. 16 | RUN mkdir -p /go/src/antithesis/control-instrumented 17 | 18 | # Perform instrumentation 19 | RUN /go/bin/antithesis-go-instrumentor \ 20 | /go/src/antithesis/control \ 21 | /go/src/antithesis/control-instrumented 22 | 23 | # Build control binary 24 | RUN cd /go/src/antithesis/control-instrumented/customer && \ 25 | cat *_antithesis_catalog.go && \ 26 | go build -o control *.go 27 | 28 | # Stage 2: lightweight "release" 29 | FROM docker.io/library/debian:bookworm-slim 30 | LABEL maintainer="Antithesis " 31 | 32 | # Copy the instrumented binary, and symbols from the build image. 33 | COPY --from=builder \ 34 | /go/src/antithesis/control-instrumented/customer/control /bin/ 35 | RUN mkdir -p /symbols 36 | COPY --from=builder /go/src/antithesis/control-instrumented/symbols /symbols/ 37 | 38 | ENTRYPOINT [ "/bin/control" ] 39 | -------------------------------------------------------------------------------- /control/Makefile: -------------------------------------------------------------------------------- 1 | ifeq ($(shell command -v podman 2> /dev/null),) 2 | CMD=docker 3 | else 4 | CMD=podman 5 | endif 6 | 7 | GIT_HASH ?= $(shell git log --format="%h" -n 1) 8 | LANGUAGE = go 9 | _BUILD_ARGS_TAG ?= ${GIT_HASH} 10 | _BUILD_ARGS_RELEASE_TAG ?= latest 11 | _BUILD_ARGS_DOCKERFILE ?= Dockerfile 12 | _BUILD_ARGS_APPLICATION ?= __does_not_exist__ 13 | 14 | all: build_control 15 | 16 | .PHONY: all 17 | 18 | _builder: 19 | $(CMD) build --tag ${LANGUAGE}-demo-${_BUILD_ARGS_APPLICATION}:${_BUILD_ARGS_TAG} -f ${_BUILD_ARGS_DOCKERFILE} . 20 | 21 | _pusher: 22 | $(CMD) push ${LANGUAGE}-demo-${_BUILD_ARGS_APPLICATION}:${_BUILD_ARGS_TAG} 23 | 24 | _releaser: 25 | $(CMD) pull ${LANGUAGE}-demo-${_BUILD_ARGS_APPLICATION}:${_BUILD_ARGS_TAG} 26 | $(CMD) tag ${LANGUAGE}-demo-${_BUILD_ARGS_APPLICATION}:${_BUILD_ARGS_TAG} ${APPLICATION_NAME}:latest 27 | $(CMD) push ${LANGUAGE}-demo-${_BUILD_ARGS_APPLICATION}:${_BUILD_ARGS_RELEASE_TAG} 28 | 29 | build_%: 30 | $(MAKE) _builder \ 31 | -e _BUILD_ARGS_TAG="$*-${GIT_HASH}" \ 32 | -e _BUILD_ARGS_DOCKERFILE="Dockerfile" \ 33 | -e _BUILD_ARGS_APPLICATION="$*" 34 | 35 | push_%: 36 | $(MAKE) _pusher -e _BUILD_ARGS_TAG="$*-${GIT_HASH}" 37 | 38 | release_%: 39 | $(MAKE) _releaser \ 40 | -e _BUILD_ARGS_TAG="$*-${GIT_HASH}" \ 41 | -e _BUILD_ARGS_RELEASE_TAG="$*-latest" 42 | -------------------------------------------------------------------------------- /control/control.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "flag" 7 | "fmt" 8 | "io" 9 | "net/http" 10 | "os" 11 | "strconv" 12 | "strings" 13 | "sync" 14 | "time" 15 | 16 | "github.com/antithesishq/antithesis-sdk-go/assert" 17 | "github.com/antithesishq/antithesis-sdk-go/lifecycle" 18 | "github.com/golang/glog" 19 | ) 20 | 21 | type Details map[string]any 22 | 23 | // A control server which maintains a list of vaults which will store the data. 24 | type ControlServer struct { 25 | mux *http.ServeMux 26 | Vaults []string 27 | minValue int 28 | lock sync.RWMutex 29 | } 30 | 31 | //go:generate antithesis-go-generator -v antithesis.com/go/glitch-grid 32 | 33 | // Create and return a new Control server instance. 34 | // Provide a comma-separated list of vaults with which we will communicate. 35 | func NewControlServer(vaults string) *ControlServer { 36 | assert.Always(true, "Instantiates a Control Server", nil) 37 | s := new(ControlServer) 38 | s.mux = http.NewServeMux() 39 | s.Vaults = strings.Split(vaults, ",") 40 | s.minValue = 0 41 | s.lock = sync.RWMutex{} 42 | s.mux.HandleFunc("/", s.handle) 43 | // Set the default timeout for all HTTP operations to be one second. 44 | http.DefaultClient.Timeout = time.Second 45 | glog.Infof("Defined %d vaults", len(s.Vaults)) 46 | if len(s.Vaults) == 23456789 { 47 | assert.Unreachable("We have 23456789 vaults should be unreachable", Details{"numVaults": len(s.Vaults)}) 48 | 49 | assert.Always(true, "This line should never execute, but since this is an always assert, it will fail in Antithesis.", nil) 50 | assert.Reachable("This line should never execute, but since this is a reachable assert, it will fail in Antithesis.", Details{"numVaults": len(s.Vaults)}) 51 | } 52 | assert.Reachable("Always returns a ControlServer when requested", Details{"vaults": vaults, "numVaults": len(s.Vaults)}) 53 | return s 54 | } 55 | 56 | // Handle GET and POST requests to the root path. 57 | func (s *ControlServer) handle(w http.ResponseWriter, r *http.Request) { 58 | 59 | lifecycle.SendEvent("handle_event", Details{"message":"Handle is called.", "method": r.Method}) 60 | 61 | if r.URL.Path != "/" { 62 | assert.AlwaysOrUnreachable(true, "Control service: received a non-root request paths & handled that correctly.", Details{"path": r.URL.Path}) 63 | // We only support operations on the root path. 64 | http.NotFound(w, r) 65 | return 66 | } 67 | if r.Method == http.MethodGet { 68 | s.get(w, r) 69 | } else if r.Method == http.MethodPost { 70 | s.post(w, r) 71 | } else { 72 | assert.AlwaysOrUnreachable(true, "Control service: received a http method that is not a GET or a POST & handled that correctly.", Details{"method": r.Method}) 73 | // Do not support PATCH, DELETE, etc, operations. 74 | http.NotFound(w, r) 75 | } 76 | } 77 | 78 | // Get the current value of the counter. 79 | // Poll all our backend servers and see if we have majority consensus. 80 | // Sends a 200 and the value to the client if we have a consensus, 500 otherwise. 81 | func (s *ControlServer) get(w http.ResponseWriter, r *http.Request) { 82 | assert.Always(true, "Control service: received a request to retrieve the counter's value", nil) 83 | result := s.getValueFromVaults() 84 | var statusCode int 85 | var body string 86 | if result >= 0 { 87 | assert.AlwaysOrUnreachable(true, "Counter's value retrieved", Details{"counter": body, "status": statusCode}) 88 | statusCode = http.StatusOK 89 | body = fmt.Sprintf("%d", result) 90 | } else { 91 | assert.Unreachable("Counter should never be unavailable", Details{"result": result}) 92 | statusCode = http.StatusInternalServerError 93 | body = "-1" 94 | } 95 | 96 | expected_status := (statusCode == http.StatusOK) || (statusCode == http.StatusInternalServerError) 97 | assert.AlwaysOrUnreachable(expected_status, "HTTP return status is expected", Details{"status": statusCode}) 98 | assert.Always(statusCode != http.StatusInternalServerError, "The server never return a 500 HTTP response code", Details{"status": statusCode}) 99 | w.WriteHeader(statusCode) 100 | w.Write([]byte(body)) 101 | } 102 | 103 | // Get the consensus value stored across our vaults. 104 | // Talk to each vault and get the value stored in said vault. If a majority of the vaults have the same 105 | // value, then we have consensus and can return that value. If there is no consensus, return -1. 106 | func (s *ControlServer) getValueFromVaults() int { 107 | var wg sync.WaitGroup 108 | m := sync.RWMutex{} 109 | // Map from a value to the number of vaults which currently have that value. 110 | counts := map[int]int{} 111 | // Loop over all the vault addresses, and execute each one in a separate goroutine. 112 | // Use a WaitGroup to keep track of the pending functions, and a ReadWrite lock to 113 | // protect access to the counts tracker. 114 | for _, vault := range s.Vaults { 115 | wg.Add(1) 116 | go func(m *sync.RWMutex, vault string, counts map[int]int) { 117 | defer wg.Done() 118 | getValueFromVault(m, vault, counts) 119 | }(&m, vault, counts) 120 | } 121 | wg.Wait() 122 | glog.Infof("Counts data: %v", counts) 123 | if len(counts) == 0 { 124 | glog.Error("Could not reach any vaults to get counts data") 125 | return -1 126 | } 127 | // Iterate over the map of values to the count of vaults with that value. 128 | // If any count represents a majority, then by default it will have the maximum 129 | // number of vaults associated with it. Otherwise, just keep track the maximum 130 | // number of counts associated with any value. 131 | // E.g., if we have seven vaults, and: 132 | // - vaults (A, C, G) have value "1"; 133 | // - vaults (B, D, E) have value "2"; and 134 | // - vault F has value "4" 135 | // then the maximum number of vaults with the same value is three (the first two groups), 136 | // but is not enough to achieve consensus. 137 | maxVal := 0 138 | for v, c := range counts { 139 | if c > maxVal { 140 | maxVal = c 141 | } 142 | if s.hasMajority(c) { 143 | // We have consensus. Return the value. 144 | return v 145 | } 146 | } 147 | // We do not have consensus, but we do know how popular the most common value(s) is/are. 148 | glog.Warningf("No majority; only have %d/%d with a consensus value", maxVal, len(s.Vaults)) 149 | return -1 150 | } 151 | 152 | // Get the value stored in a single vault. 153 | // If we are able to fetch a valid integer from the vault, update the counts map with that 154 | // information in a thread-safe way. Otherwise, return without updating (but log the issue). 155 | func getValueFromVault(m *sync.RWMutex, vault string, counts map[int]int) { 156 | url := fmt.Sprintf("http://%s/", vault) 157 | var resp *http.Response 158 | var err error 159 | if resp, err = http.Get(url); err != nil { 160 | // This could include a timeout. 161 | glog.Warningf("Error getting value from vault %s: %v\n", url, err) 162 | return 163 | } 164 | if resp.StatusCode != http.StatusOK { 165 | // Vault was not happy. 166 | glog.Warningf("Error getting value from vault %s: invalid status code %v\n", url, resp.StatusCode) 167 | return 168 | } 169 | body, readError := io.ReadAll(resp.Body) 170 | if readError != nil { 171 | // Vault was supposedly-happy but did not return a value. 172 | glog.Warningf("Error getting value from vault %s: error reading from body: %v\n", url, readError) 173 | return 174 | } 175 | v, e := strconv.Atoi(string(body)) 176 | if e != nil { 177 | // Vault returned a value, but it was not a valid integer. 178 | glog.Warningf("Error getting value from vault %s: invalid body response: %v (%v)\n", url, body, e) 179 | return 180 | } 181 | // If we've gotten here, then we received a valid integer back from the vault. 182 | // Start the map manipulation operation critical section. 183 | m.Lock() 184 | count, ok := counts[v] 185 | if !ok { 186 | // This value is not (yet) in the map. IOW, there are currently 0 vaults storing that value. 187 | count = 0 188 | } 189 | counts[v] = count + 1 190 | m.Unlock() 191 | // End of the map manipulation critical section. 192 | glog.V(1).Infof("Get vault %s Value %d", url, v) 193 | } 194 | 195 | // TODO: Call this when we detect that a vault is in a bad state. 196 | func healFailingVault(vault string) { 197 | assert.Sometimes(true, "Control service: invoked heal function on unhealthy vault", Details{"vault": vault}) 198 | 199 | // Code to heal a failing vault 200 | } 201 | 202 | // Update the value in storage to what is provided in the body. 203 | // Contact each vault and store that value in the vault. 204 | func (s *ControlServer) post(w http.ResponseWriter, r *http.Request) { 205 | body, err := io.ReadAll(r.Body) 206 | if err != nil { 207 | // We did not get a valid body from the client. Tell them so. 208 | glog.Warningf("Could not read body: %v\n", err) 209 | w.WriteHeader(http.StatusBadRequest) 210 | w.Write([]byte("Invalid or missing POST body")) 211 | return 212 | } 213 | n, e := strconv.Atoi(string(body)) 214 | if n < 0 || e != nil { 215 | // We got a body, but it is not a valid integer (or not valid for us). 216 | w.WriteHeader(http.StatusBadRequest) 217 | w.Write([]byte("Invalid or missing POST body")) 218 | return 219 | } 220 | // Check to make sure that this value is larger than the one we've previously committed 221 | s.lock.RLock() 222 | if n < s.minValue { 223 | msg := fmt.Sprintf("Client would make value decrease from %d to %d", s.minValue, n) 224 | s.lock.RUnlock() 225 | glog.Warning(msg) 226 | w.WriteHeader(http.StatusBadRequest) 227 | w.Write([]byte(msg)) 228 | return 229 | } 230 | s.lock.RUnlock() 231 | // Send the update to the vaults, keeping track of how many vaults actually responded to us. 232 | // Technically this is a set(), but because Go doesn't have sets, this is a map of vaults to 233 | // booleans, where the value stored in the map doesn't really matter. The presence of ANY 234 | // value is enough to show that we got a successful response from the vault. 235 | resp := make(map[string]bool) 236 | assert.AlwaysOrUnreachable( 237 | len(s.Vaults) > 0, 238 | "Control service: there are vaults to update", 239 | Details{"numVaults": len(s.Vaults)}, 240 | ) 241 | s.postValueToVaults(body, resp) 242 | // If the number of responses represents a majority of the vaults, then we can claim success 243 | // in storing this value in our system. Otherwise it represents a server failure. 244 | if s.hasMajority(len(resp)) { 245 | w.WriteHeader(http.StatusOK) 246 | // Set the min value here to prevent us from going backwards. 247 | s.lock.Lock() 248 | assert.AlwaysOrUnreachable( 249 | n > s.minValue, 250 | "Control service: unnecessary update attempted", 251 | Details{"minValue": s.minValue, "requestedValue": n}, 252 | ) 253 | s.minValue = n 254 | s.lock.Unlock() 255 | } else { 256 | w.WriteHeader(http.StatusInternalServerError) 257 | } 258 | // In addition to the status code, unconditionally return a message of how many vaults we updated. 259 | w.Write([]byte(fmt.Sprintf("Sent updates to %d/%d vaults", len(resp), len(s.Vaults)))) 260 | } 261 | 262 | // Actually send the POST commands to the vaults. 263 | func (s *ControlServer) postValueToVaults(body []byte, resp map[string]bool) { 264 | // Use a WaitGroup so we can run the requests in parallel goroutine threads. 265 | var wg sync.WaitGroup 266 | // We will need to synchronize access to the response map. 267 | m := sync.RWMutex{} 268 | // For each vault, send a POST message containing the same body we received from the client. 269 | for _, vault := range s.Vaults { 270 | wg.Add(1) 271 | go func(m *sync.RWMutex, vault string, body []byte, resp map[string]bool) { 272 | defer wg.Done() 273 | glog.V(1).Infof("Setting vault %s value to %s", vault, string(body)) 274 | url := fmt.Sprintf("http://%s/", vault) 275 | r, err := http.Post(url, "text/plain", bytes.NewBuffer(body)) 276 | 277 | // No error was provided by http.Post() 278 | if err == nil { 279 | if r != nil { 280 | if r.StatusCode == http.StatusOK { 281 | m.Lock() 282 | resp[url] = true 283 | m.Unlock() 284 | } else { 285 | assert.AlwaysOrUnreachable( 286 | true, 287 | "HTTP Status might not be OK when http.Post() reports no error has occurred", 288 | Details{"statusCode": r.StatusCode}, 289 | ) 290 | // This could include a failure to connect or a timeout during the update. 291 | glog.Warningf("Error setting vault %s value to %s: %v", vault, string(body), err) 292 | } 293 | } else { 294 | assert.Unreachable("There is no error reported by http.Post(), and HTTP Status is not available", nil) 295 | } 296 | } 297 | 298 | // An error was provided by http.Post() 299 | if err != nil { 300 | errText := fmt.Sprintf("%v", err) 301 | if r != nil { 302 | assert.AlwaysOrUnreachable( 303 | r.StatusCode != http.StatusOK, 304 | "HTTP Status is never OK when receiving a Post error", 305 | Details{"err": errText, "httpStatus": r.StatusCode}, 306 | ) 307 | } else { 308 | assert.AlwaysOrUnreachable( 309 | true, 310 | "HTTP Status may not be available when http.Post() returns an error", 311 | Details{"err": errText}, 312 | ) 313 | } 314 | // This could include a failure to connect or a timeout during the update. 315 | glog.Warningf("Error setting vault %s value to %s: %v", vault, string(body), err) 316 | } 317 | }(&m, vault, body, resp) 318 | } 319 | // Wait for all the connections to complete/timeout/fail. 320 | wg.Wait() 321 | } 322 | 323 | // Check if this number represents a majority of the vaults, where majority has to be >50%. 324 | func (s *ControlServer) hasMajority(count int) bool { 325 | assert.Always(true, "Control service: determine if there is a majority", nil) 326 | assert.Always(count > 0, "Control service: majority is always expected to be positive", Details{"count": count}) 327 | assert.Always(len(s.Vaults) > 0, "Control service: there are vaults known to the service", nil) 328 | numVaults := len(s.Vaults) 329 | // By default this division will do the equivalent of math.Floor() 330 | numForMajority := (numVaults / 2) + 1 331 | haveEnoughVaults := (count >= numForMajority) 332 | // We expect both conditions below to be sometimes true 333 | assert.Sometimes(haveEnoughVaults, "Control service: there is a majority of vaults", Details{"count": count, "majorityNeeded": numForMajority}) 334 | assert.Sometimes(!haveEnoughVaults, "Control service: there is not a majority of vaults", Details{"count": count, "majorityNeeded": numForMajority}) 335 | // We expect numForMajority to be less than 99 336 | if numForMajority < 99 { 337 | assert.Unreachable("Control Service: expected failure as we expect the numForMajority to be less than 99 sometimes", Details{"majorityNeeded": numForMajority}) 338 | } 339 | return haveEnoughVaults 340 | } 341 | 342 | func main() { 343 | fmt.Print("Control Server booting...\n") 344 | assert.Always(true, "Control service: service started", nil) 345 | portPtr := flag.Int("port", 8000, "Port on which to listen for requests") 346 | vaultsPtr := flag.String("vaults", "", "Comma-separated list of vaults") 347 | flag.Parse() 348 | s := NewControlServer(*vaultsPtr) 349 | lifecycle.SetupComplete(Details{"port": *portPtr, "vaults": *vaultsPtr}) 350 | assert.Always(true, "Control service: setup complete", nil) 351 | err := http.ListenAndServe(fmt.Sprintf(":%d", *portPtr), s.mux) 352 | if errors.Is(err, http.ErrServerClosed) { 353 | assert.Unreachable("Control service: closed unexpectedly", Details{"error": err}) 354 | fmt.Printf("server closed\n") 355 | } else if err != nil { 356 | assert.Unreachable("Control service: did not start", Details{"error": err}) 357 | fmt.Printf("error starting server: %s\n", err) 358 | os.Exit(1) 359 | } 360 | } 361 | -------------------------------------------------------------------------------- /control/go.mod: -------------------------------------------------------------------------------- 1 | module antithesis.com/glitch-grid-control 2 | 3 | go 1.20 4 | 5 | require github.com/golang/glog v1.2.0 6 | 7 | require github.com/antithesishq/antithesis-sdk-go v0.3.6 8 | -------------------------------------------------------------------------------- /control/go.sum: -------------------------------------------------------------------------------- 1 | github.com/antithesishq/antithesis-sdk-go v0.3.6 h1:29gIXzrMlUrtkGUD2/jwp3yMAlE0+CUIdXBJRtsjBNE= 2 | github.com/antithesishq/antithesis-sdk-go v0.3.6/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E= 3 | github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= 4 | github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= 5 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 6 | -------------------------------------------------------------------------------- /vault/Dockerfile: -------------------------------------------------------------------------------- 1 | # Stage 1: Get Golang image 2 | FROM docker.io/library/golang:1.20-bookworm AS builder 3 | LABEL maintainer="Antithesis " 4 | 5 | # Add source code 6 | RUN mkdir -p /go/src/antithesis/vault 7 | COPY go.sum go.mod *.go /go/src/antithesis/vault/ 8 | 9 | # Download and install antithesis-go-instrumentor 10 | # Installs into $GOPATH/bin => /go/bin 11 | RUN cd /go/src/antithesis/vault && \ 12 | go install github.com/antithesishq/antithesis-sdk-go/tools/antithesis-go-instrumentor@v0.3.6 && \ 13 | go mod tidy 14 | 15 | # Create the destination output directory for the instrumented code. 16 | RUN mkdir -p /go/src/antithesis/vault-instrumented 17 | 18 | # Perform instrumentation 19 | RUN /go/bin/antithesis-go-instrumentor \ 20 | /go/src/antithesis/vault \ 21 | /go/src/antithesis/vault-instrumented 22 | 23 | # Build vault binary 24 | RUN cd /go/src/antithesis/vault-instrumented/customer && \ 25 | go build -o vault *.go 26 | 27 | # Stage 2: lightweight "release" 28 | FROM docker.io/library/debian:bookworm-slim 29 | LABEL maintainer="Antithesis " 30 | 31 | # Copy the instrumented binary, and symbols from the build image. 32 | COPY --from=builder \ 33 | /go/src/antithesis/vault-instrumented/customer/vault /bin/ 34 | RUN mkdir -p /symbols 35 | COPY --from=builder /go/src/antithesis/vault-instrumented/symbols /symbols/ 36 | 37 | ENTRYPOINT [ "/bin/vault" ] 38 | -------------------------------------------------------------------------------- /vault/Makefile: -------------------------------------------------------------------------------- 1 | ifeq ($(shell command -v podman 2> /dev/null),) 2 | CMD=docker 3 | else 4 | CMD=podman 5 | endif 6 | 7 | GIT_HASH ?= $(shell git log --format="%h" -n 1) 8 | LANGUAGE = go 9 | _BUILD_ARGS_TAG ?= ${GIT_HASH} 10 | _BUILD_ARGS_RELEASE_TAG ?= latest 11 | _BUILD_ARGS_DOCKERFILE ?= Dockerfile 12 | _BUILD_ARGS_APPLICATION ?= __does_not_exist__ 13 | 14 | all: build_vault 15 | 16 | .PHONY: all 17 | 18 | _builder: 19 | $(CMD) build --tag ${LANGUAGE}-demo-${_BUILD_ARGS_APPLICATION}:${_BUILD_ARGS_TAG} -f ${_BUILD_ARGS_DOCKERFILE} . 20 | 21 | _pusher: 22 | $(CMD) push ${LANGUAGE}-demo-${_BUILD_ARGS_APPLICATION}:${_BUILD_ARGS_TAG} 23 | 24 | _releaser: 25 | $(CMD) pull ${LANGUAGE}-demo-${_BUILD_ARGS_APPLICATION}:${_BUILD_ARGS_TAG} 26 | $(CMD) tag ${LANGUAGE}-demo-${_BUILD_ARGS_APPLICATION}:${_BUILD_ARGS_TAG} ${APPLICATION_NAME}:latest 27 | $(CMD) push ${LANGUAGE}-demo-${_BUILD_ARGS_APPLICATION}:${_BUILD_ARGS_RELEASE_TAG} 28 | 29 | build_%: 30 | $(MAKE) _builder \ 31 | -e _BUILD_ARGS_TAG="$*-${GIT_HASH}" \ 32 | -e _BUILD_ARGS_DOCKERFILE="Dockerfile" \ 33 | -e _BUILD_ARGS_APPLICATION="$*" 34 | 35 | push_%: 36 | $(MAKE) _pusher -e _BUILD_ARGS_TAG="$*-${GIT_HASH}" 37 | 38 | release_%: 39 | $(MAKE) _releaser \ 40 | -e _BUILD_ARGS_TAG="$*-${GIT_HASH}" \ 41 | -e _BUILD_ARGS_RELEASE_TAG="$*-latest" 42 | -------------------------------------------------------------------------------- /vault/go.mod: -------------------------------------------------------------------------------- 1 | module antithesis.com/glitch-grid-vault 2 | 3 | go 1.20 4 | 5 | require github.com/golang/glog v1.2.0 6 | -------------------------------------------------------------------------------- /vault/go.sum: -------------------------------------------------------------------------------- 1 | github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= 2 | github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= 3 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 4 | -------------------------------------------------------------------------------- /vault/vault.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "os" 10 | "strconv" 11 | "time" 12 | 13 | "github.com/golang/glog" 14 | ) 15 | 16 | // A vault server which maintains a list of vaults which will store the data (value). 17 | // We only store positive values. 18 | type VaultServer struct { 19 | mux *http.ServeMux 20 | port int 21 | value int 22 | } 23 | 24 | // Create and return a new Vault server instance. 25 | // Provide the port on which we will listen. 26 | // We store the port of the vault and not the controller, since the port is how we will 27 | // distinguish the vaults in the logs when run via `docker-compose up` 28 | func NewVaultServer(port int) *VaultServer { 29 | s := new(VaultServer) 30 | s.mux = http.NewServeMux() 31 | s.value = 0 32 | s.port = port 33 | s.mux.HandleFunc("/", s.handle) 34 | http.DefaultClient.Timeout = time.Second 35 | return s 36 | } 37 | 38 | // Handle GET and POST requests to the root path. 39 | func (s *VaultServer) handle(w http.ResponseWriter, r *http.Request) { 40 | if r.URL.Path != "/" { 41 | // We only support operations on the root path. 42 | http.NotFound(w, r) 43 | return 44 | } 45 | if r.Method == http.MethodGet { 46 | s.get(w, r) 47 | } else if r.Method == http.MethodPost { 48 | s.post(w, r) 49 | } else { 50 | // Do not support PATCH, DELETE, etc, operations. 51 | http.NotFound(w, r) 52 | } 53 | } 54 | 55 | // Return the value stored in the vault. This should always be a success. 56 | func (s *VaultServer) get(w http.ResponseWriter, r *http.Request) { 57 | w.WriteHeader(http.StatusOK) 58 | w.Write([]byte(fmt.Sprintf("%d", s.value))) 59 | } 60 | 61 | // Update the value stored in the vault. 62 | // Logs a warning if the value decreases for whatever reason (but still update it). 63 | func (s *VaultServer) post(w http.ResponseWriter, r *http.Request) { 64 | body, err := io.ReadAll(r.Body) 65 | if err != nil { 66 | // Make sure we actually get a valid body from the client. 67 | glog.Warningf("Could not read body: %v\n", err) 68 | w.WriteHeader(http.StatusBadRequest) 69 | w.Write([]byte("Invalid or missing POST body")) 70 | return 71 | } 72 | v := string(body) 73 | n, e := strconv.Atoi(v) 74 | if n >= 0 && e == nil { 75 | // We only store positive values. 76 | if n < s.value { 77 | glog.Warningf("THIS SHOULD NEVER HAPPEN: Counter value regressed from %d to %d", s.value, n) 78 | } 79 | s.value = n 80 | glog.Infof("Set Vault :%d Counter %d", s.port, s.value) 81 | w.WriteHeader(http.StatusOK) 82 | w.Write(body) 83 | } else { 84 | // Either the body was not a valid integer, or it was negative. 85 | w.WriteHeader(http.StatusBadRequest) 86 | w.Write([]byte("Invalid or missing POST body")) 87 | } 88 | } 89 | 90 | func main() { 91 | portPtr := flag.Int("port", 8001, "Port on which to listen for requests") 92 | flag.Parse() 93 | s := NewVaultServer(*portPtr) 94 | err := http.ListenAndServe(fmt.Sprintf(":%d", s.port), s.mux) 95 | if errors.Is(err, http.ErrServerClosed) { 96 | glog.Info("server closed") 97 | } else if err != nil { 98 | glog.Errorf("error starting server: %s", err) 99 | os.Exit(1) 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /workload/Dockerfile: -------------------------------------------------------------------------------- 1 | # Start from a thin linux distribution. 2 | FROM docker.io/library/alpine:latest 3 | LABEL maintainer="Antithesis " 4 | 5 | # Make sure to update the current packages, then grab the two programs 6 | # we'll need to have in order for the workload driver to run. 7 | RUN apk update 8 | RUN apk upgrade 9 | RUN apk add bash curl 10 | 11 | # Add source code 12 | ADD workload.sh . 13 | 14 | # Define the entrypoint. 15 | # Make sure to use the array version of this specification, so other command-line 16 | # arguments and parameters can be specified in the Dockerfile. 17 | ENTRYPOINT [ "/bin/bash", "./workload.sh" ] -------------------------------------------------------------------------------- /workload/Makefile: -------------------------------------------------------------------------------- 1 | ifeq ($(shell command -v podman 2> /dev/null),) 2 | CMD=docker 3 | else 4 | CMD=podman 5 | endif 6 | 7 | GIT_HASH ?= $(shell git log --format="%h" -n 1) 8 | _BUILD_ARGS_TAG ?= ${GIT_HASH} 9 | _BUILD_ARGS_RELEASE_TAG ?= latest 10 | _BUILD_ARGS_DOCKERFILE ?= Dockerfile 11 | _BUILD_ARGS_APPLICATION ?= __does_not_exist__ 12 | 13 | all: build_workload 14 | 15 | .PHONY: all 16 | 17 | _builder: 18 | $(CMD) build --tag demo-${_BUILD_ARGS_APPLICATION}:${_BUILD_ARGS_TAG} -f ${_BUILD_ARGS_DOCKERFILE} . 19 | 20 | _pusher: 21 | $(CMD) push demo-${_BUILD_ARGS_APPLICATION}:${_BUILD_ARGS_TAG} 22 | 23 | _releaser: 24 | $(CMD) pull demo-${_BUILD_ARGS_APPLICATION}:${_BUILD_ARGS_TAG} 25 | $(CMD) tag demo-${_BUILD_ARGS_APPLICATION}:${_BUILD_ARGS_TAG} ${APPLICATION_NAME}:latest 26 | $(CMD) push demo-${_BUILD_ARGS_APPLICATION}:${_BUILD_ARGS_RELEASE_TAG} 27 | 28 | build_%: 29 | $(MAKE) _builder \ 30 | -e _BUILD_ARGS_TAG="$*-${GIT_HASH}" \ 31 | -e _BUILD_ARGS_DOCKERFILE="Dockerfile" \ 32 | -e _BUILD_ARGS_APPLICATION="$*" 33 | 34 | push_%: 35 | $(MAKE) _pusher -e _BUILD_ARGS_TAG="$*-${GIT_HASH}" 36 | 37 | release_%: 38 | $(MAKE) _releaser \ 39 | -e _BUILD_ARGS_TAG="$*-${GIT_HASH}" \ 40 | -e _BUILD_ARGS_RELEASE_TAG="$*-latest" 41 | -------------------------------------------------------------------------------- /workload/workload.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # How long to wait for everything to come up before starting the test? 4 | START_DELAY=${START_DELAY:-10} 5 | # How long to wait after the final write to check if the system gets into a consistent state? 6 | FINAL_DELAY=${FINAL_DELAY:-5} 7 | # How many steps (i.e., numbers) should we try and write to the system during this test? 8 | NUM_STEPS=${NUM_STEPS:-100} 9 | # What is the maximum amount the counter should increment per step? 10 | MAX_STEP_SIZE=${MAX_STEP_SIZE:-5} 11 | # Check the consistency of the system when the counter is a multiple of what number? 12 | CHECK_WHEN_MULTIPLE_OF=${CHECK_WHEN_MULTIPLE_OF:-5} 13 | 14 | CONTROL_SERVER=$1 15 | if [[ -z "${CONTROL_SERVER}" ]] 16 | then 17 | echo "Usage: $0 " 18 | exit 1 19 | fi 20 | 21 | # Wait a bit for the other services to come up 22 | echo -n "Waiting ${START_DELAY} seconds for the storage system to become available ... " 23 | sleep ${START_DELAY} 24 | echo "done." 25 | 26 | # Emit a message we can look for to see that the test is starting. 27 | # This is useful for us to know when we've pivoted from setup to actual test so we don't 28 | # fuzz/test the setup phase. 29 | echo {\"antithesis_setup\": {\"status\": \"complete\"}} > "$ANTITHESIS_OUTPUT_DIR/sdk.json" 30 | echo "=== START TEST ===" 31 | 32 | # Go through and store numbers in our system. When our current number is a multiple of a certain value, 33 | # do a read to check that the storage system is consistent and fully caught-up. 34 | n=0 35 | for c in $(seq 1 ${NUM_STEPS} ) 36 | do 37 | n=$(( $n + ($RANDOM % $MAX_STEP_SIZE) + 1 )) 38 | outfile="/tmp/${n}.out" 39 | http_code=$(curl -s -o "${outfile}" -w "%{http_code}" -X POST "http://${CONTROL_SERVER}" -d "${n}") 40 | echo "HttpStatus=${http_code} Message=$(cat "${outfile}" 2> /dev/null)" 41 | rm -f "${outfile}" 42 | if [[ $(( $n % $CHECK_WHEN_MULTIPLE_OF )) -eq 0 ]] 43 | then 44 | outfile="/tmp/${n}.out" 45 | http_code=$(curl -s -o "${outfile}" -w "%{http_code}" "http://${CONTROL_SERVER}") 46 | actual=$(cat "${outfile}" 2> /dev/null) 47 | actual=${actual//[!0-9]/} 48 | echo "HttpStatus=${http_code} Expected=${n} Actual=${actual}" 49 | if [[ "${http_code}" != "200" ]] 50 | then 51 | echo "ERROR: Storage vaults are in an inconsistent state!" 52 | elif [[ "${n}" != "${actual}" ]] 53 | then 54 | echo "ERROR: Storage vaults returned unexpected value!" 55 | fi 56 | rm -f "${outfile}" 57 | sleep 1 58 | fi 59 | done 60 | 61 | # Now give the system a few seconds to settle, and do one final check for consistency. 62 | echo -n "Waiting ${FINAL_DELAY} seconds for any straggling writes ... " 63 | sleep ${FINAL_DELAY} 64 | echo "done." 65 | outfile="/tmp/${n}.out" 66 | http_code=$(curl -s -o "${outfile}" -w "%{http_code}" "http://${CONTROL_SERVER}") 67 | actual=$(cat "${outfile}" 2> /dev/null) 68 | actual=${actual//[!0-9]/} 69 | echo "HttpStatus=${http_code} Expected=${n} Actual=${actual}" 70 | if [[ "${http_code}" != "200" ]] 71 | then 72 | echo "ERROR: Storage vaults finished in an inconsistent state!" 73 | elif [[ "${n}" != "${actual}" ]] 74 | then 75 | echo "ERROR: Storage vaults finished with unexpected value!" 76 | else 77 | echo "Test completed with storage in a consistent and correct state." 78 | fi 79 | rm -f "${outfile}" 80 | 81 | # Tag the end of the test. 82 | echo "==== END TEST ====" 83 | --------------------------------------------------------------------------------