├── .github ├── configs │ ├── functions-under-test.txt │ ├── linkcheck.json │ ├── spellcheck.yml │ └── wordlist.txt ├── dependabot.yml └── workflows │ ├── base_image_ci.yml_ │ ├── build_gem5.yml │ ├── build_kernel.yml │ ├── build_runner.yml │ ├── create_base_disk.yml │ ├── create_base_disk_2.yml_ │ ├── create_base_disk_self_hosted.yml │ ├── create_base_disk_self_hosted_arm.yml │ ├── delete_old.yml │ ├── enable-automerge.yaml │ ├── function_ci.yml │ ├── linters.yml │ ├── release.yml │ ├── test_client.yml │ ├── test_download.yml │ └── test_fn_install.yml ├── .gitignore ├── LICENSE ├── README.md ├── analysis ├── analyze_results.ipynb ├── check_simulations.py └── gem5_utils.py ├── configs ├── disk-image-configs │ ├── autoinstall-amd64.yaml │ ├── autoinstall-arm64.yaml │ ├── client-amd64 │ ├── client-arm64 │ ├── finalize.sh │ ├── gem5init │ ├── init_disk_image.sh │ ├── m5.amd64 │ └── m5.arm64 └── linux-configs │ ├── check-config.sh │ ├── kernel_m5.patch │ ├── v5.15.59-amd64.config │ ├── v5.15.59-arm64.config │ ├── v5.4.84-amd64.config │ └── v5.4.84-arm64.config ├── docs ├── .gitignore ├── Gemfile ├── README.md ├── _config.yml ├── analysis │ ├── analysis.md │ └── basic_analysis.md ├── figures │ ├── basic_analysis.png │ ├── conventional-system.jpg │ ├── conventional-workloads.jpg │ ├── serverless-stack.jpg │ ├── serverless-system.jpg │ ├── serverless-workloads.jpg │ ├── simple_system.jpg │ ├── skylake-system.jpg │ ├── trigger-points.jpg │ ├── two-core.jpg │ ├── two-machine.jpg │ ├── vSwarm-u-hdr.png │ └── vswarm-u-design.png ├── index.md ├── methodology.md ├── misc │ └── resources.md ├── quick_start.md ├── setup │ ├── disk-image.md │ ├── kernel.md │ ├── resources.md │ └── setup.md ├── simulation │ ├── arm_simulation.md │ ├── basics.md │ ├── functions.md │ ├── instrumentation.md │ ├── manual-install.md │ ├── simple_component.md │ ├── simulations.md │ └── systems.md └── test │ └── function_test.md ├── gem5utils ├── __init__.py ├── scripts │ └── update_upstream.sh └── systems │ ├── __init__.py │ ├── drive │ ├── __init__.py │ ├── system.py │ └── x86.py │ ├── simple │ ├── __init__.py │ ├── caches.py │ ├── system.py │ └── x86.py │ └── skylake │ ├── __init__.py │ ├── caches.py │ ├── core.py │ ├── system.py │ └── x86.py ├── resources ├── Makefile ├── artifacts.py ├── artifacts.sh ├── release.json └── upload_assets.py ├── runner ├── Dockerfile ├── Makefile ├── create-build-runner.yaml ├── create-runner.yaml ├── delete-runners.yaml ├── easy-create.sh ├── setup-build-host.yaml └── setup-host.yaml ├── scripts ├── cpi.sh ├── run_function.sh └── run_qemu.sh ├── setup ├── build_gem5.sh ├── build_kernel.sh ├── create_disk_image.sh ├── disk.Makefile ├── disk_arm.Makefile ├── gem5.Makefile ├── kernel.Makefile ├── requirements.txt └── setup_host.sh ├── simulation ├── Makefile ├── arm.Makefile ├── functions │ ├── all_vswarm_functions.list │ ├── all_vswarm_functions.yaml │ ├── functions.list │ └── functions.yaml ├── install_functions.sh └── wkdir-tmpl │ ├── run_sim.arm.tmpl.py │ ├── run_sim.tmpl.py │ ├── run_sim_test.tmpl.sh │ ├── run_sim_two_machine.tmpl.py │ ├── setup_all_functions.tmpl.sh │ ├── setup_function.tmpl.sh │ ├── sim_all_functions.tmpl.sh │ ├── sim_function.tmpl.sh │ ├── vswarm_simple.tmpl.py │ └── vswarm_simple_arm.tmpl.py ├── test ├── Makefile ├── run_emu_test.template.sh └── run_sim_test.py └── tools └── client ├── Makefile ├── go.mod ├── go.sum ├── m5 ├── amd64 │ ├── m5op.S │ └── m5op_addr.S ├── arm64 │ ├── m5op.S │ └── m5op_addr.S ├── m5_mmap.c ├── m5_mmap.h ├── m5ops.go ├── m5ops.h └── m5ops_generic.h ├── main.go └── version.go /.github/configs/functions-under-test.txt: -------------------------------------------------------------------------------- 1 | fibonacci-go 2 | fibonacci-nodejs 3 | fibonacci-python 4 | aes-go 5 | aes-nodejs 6 | aes-python 7 | 8 | -------------------------------------------------------------------------------- /.github/configs/linkcheck.json: -------------------------------------------------------------------------------- 1 | { 2 | "ignorePatterns": [ 3 | { 4 | "pattern": "^http://localhost:*" 5 | }, 6 | { 7 | "pattern": "^http://127.0.0.1:*" 8 | } 9 | ], 10 | "replacementPatterns": [ 11 | { 12 | "pattern": "^/", 13 | "replacement": "/github/workspace/" 14 | } 15 | ], 16 | "httpHeaders": [ 17 | { 18 | "urls": ["https://docs.github.com/"], 19 | "headers": { 20 | "Accept-Encoding": "zstd, br, gzip, deflate" 21 | } 22 | } 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /.github/configs/spellcheck.yml: -------------------------------------------------------------------------------- 1 | matrix: 2 | - name: Markdown 3 | aspell: 4 | lang: en 5 | dictionary: 6 | wordlists: 7 | - ./.github/configs/wordlist.txt 8 | encoding: utf-8 9 | pipeline: 10 | - pyspelling.filters.markdown: 11 | - pyspelling.filters.html: 12 | comments: false 13 | ignores: 14 | - code 15 | - pre 16 | sources: 17 | - '**/*.md' 18 | default_encoding: utf-8 -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # Use https://dependabot.com/docs/config-file/validator/ to check for errors. 2 | version: 2 3 | updates: 4 | 5 | - package-ecosystem: "gomod" 6 | directory: "./tools/client" 7 | schedule: 8 | interval: "weekly" 9 | ignore: 10 | - dependency-name: "*" 11 | update-types: [ "version-update:semver-patch" ] 12 | groups: 13 | otel: 14 | patterns: 15 | - "go.opentelemetry.io/otel*" 16 | 17 | # Enable version updates for Actions 18 | - package-ecosystem: "github-actions" 19 | # Look for `.github/workflows` in the `root` directory 20 | directory: "/" 21 | schedule: 22 | interval: "weekly" 23 | -------------------------------------------------------------------------------- /.github/workflows/build_gem5.yml: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2022 David Schall and EASE Lab 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | name: Build Gem5 24 | 25 | env: 26 | WORKDIR: setup/ 27 | MAKEFILE: setup/gem5.Makefile 28 | 29 | on: 30 | # Allows you to run this workflow manually from the Actions tab 31 | workflow_dispatch: 32 | schedule: 33 | - cron: "0 7 * * 1" 34 | 35 | push: 36 | branches: [main] 37 | paths: 38 | - "setup/gem5.Makefile" 39 | 40 | pull_request: 41 | branches: [main] 42 | paths: 43 | - "setup/gem5.Makefile" 44 | 45 | 46 | jobs: 47 | build-x86-64: 48 | name: Build Gem5 for x86 49 | # Building the kernel works also on the github runners. 50 | # However, they only offer building on one core therefor it take a while 51 | # Using self hosted is faster 52 | runs-on: ubuntu-24.04 53 | # runs-on: [self-hosted, gem5-build] 54 | strategy: 55 | fail-fast: true 56 | 57 | steps: 58 | - name: Check out code 59 | uses: actions/checkout@v4 60 | 61 | - name: Install dependencies 62 | shell: bash 63 | run: | 64 | make -f ${{ env.MAKEFILE }} dep_install 65 | 66 | - name: Build gem5 67 | shell: bash 68 | run: | 69 | make -f ${{ env.MAKEFILE }} all 70 | -------------------------------------------------------------------------------- /.github/workflows/build_kernel.yml: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2022 David Schall and EASE Lab 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | name: Build Linux Kernel 24 | 25 | on: 26 | # Allows you to run this workflow manually from the Actions tab 27 | workflow_dispatch: 28 | schedule: 29 | - cron: "0 7 * * 1" 30 | 31 | push: 32 | branches: [main] 33 | paths: 34 | - "setup/kernel.Makefile" 35 | - "configs/linux-configs/**" 36 | 37 | pull_request: 38 | branches: [main] 39 | paths: 40 | - "setup/kernel.Makefile" 41 | - "configs/linux-configs/**" 42 | 43 | env: 44 | WORKDIR: setup/ 45 | MAKEFILE: setup/kernel.Makefile 46 | KERNEL: kernel 47 | 48 | jobs: 49 | build: 50 | name: Build kernel 51 | # Building the kernel works also on the github runners. 52 | # However, they only offer building on one core therefor it take a while 53 | # Using self hosted is faster 54 | runs-on: ubuntu-24.04 55 | # runs-on: [self-hosted, gem5-build] 56 | strategy: 57 | fail-fast: true 58 | matrix: 59 | arch: [ amd64, arm64 ] 60 | version: [ focal, jammy ] 61 | include: 62 | - version: focal 63 | kversion: v5.4.84 64 | - version: jammy 65 | kversion: v5.15.59 66 | 67 | env: 68 | ARCH: ${{ matrix.arch }} 69 | KVERSION: ${{ matrix.kversion }} 70 | 71 | steps: 72 | - name: Check out code 73 | uses: actions/checkout@v4 74 | 75 | - name: Install dependencies 76 | shell: bash 77 | run: | 78 | make -f ${{ env.MAKEFILE }} dep_install 79 | 80 | - name: Configure and build the Linux kernel 81 | shell: bash 82 | run: | 83 | make -f ${{ env.MAKEFILE }} build 84 | 85 | - name: Save the outup binary 86 | shell: bash 87 | env: 88 | OUTPUT: ${{ env.KERNEL }} 89 | run: | 90 | make -f ${{ env.MAKEFILE }} save_output 91 | 92 | - name: Upload Kernel Artifact 93 | uses: actions/upload-artifact@v4 94 | with: 95 | name: vmlinux-${{ matrix.version }}-${{ matrix.arch }} 96 | path: ${{ env.KERNEL }} -------------------------------------------------------------------------------- /.github/workflows/build_runner.yml: -------------------------------------------------------------------------------- 1 | name: Runner container pipeline 2 | 3 | on: 4 | schedule: 5 | - cron: "0 9 1 * 1" 6 | workflow_dispatch: 7 | 8 | env: 9 | PLATFORMS: linux/amd64,linux/arm64 10 | 11 | jobs: 12 | build-and-push: 13 | name: Build and push all images 14 | runs-on: ubuntu-20.04 15 | strategy: 16 | fail-fast: false 17 | matrix: 18 | runner: [gem5-test-runner] 19 | 20 | 21 | steps: 22 | - name: Check out code into the Go module directory 23 | uses: actions/checkout@v4 24 | with: 25 | lfs: "true" 26 | 27 | - name: Login to Docker Hub 28 | uses: docker/login-action@v3 29 | with: 30 | username: ${{ secrets.DOCKER_HUB_USERNAME }} 31 | password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} 32 | 33 | - name: Set up QEMU 34 | uses: docker/setup-qemu-action@v3 35 | 36 | - name: Set up Docker Buildx 37 | id: buildx 38 | uses: docker/setup-buildx-action@v3 39 | 40 | 41 | - name: Build and push 42 | uses: docker/build-push-action@v6 43 | with: 44 | push: true 45 | file: runner/Dockerfile 46 | platforms: ${{ env.PLATFORMS }} 47 | target: ${{ matrix.runner }} 48 | tags: vhiveease/${{ matrix.runner }}:latest 49 | context: . 50 | 51 | -------------------------------------------------------------------------------- /.github/workflows/create_base_disk_2.yml_: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2022 David Schall and EASE Lab 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | name: Create base disk image 24 | 25 | env: 26 | WORKDIR: setup/ 27 | MAKEFILE: setup/disk.Makefile 28 | BUILD_DIR: wkdir/ 29 | DISK_PRE_TEST: disk.img 30 | 31 | on: 32 | # Allows you to run this workflow manually from the Actions tab 33 | workflow_dispatch: 34 | schedule: 35 | - cron: "0 7 * * 1" 36 | 37 | push: 38 | branches: [main] 39 | paths: 40 | - "setup/disk.Makefile" 41 | - "configs/disk-image-configs/**" 42 | 43 | 44 | jobs: 45 | build-x86-64: 46 | name: Build disk image for x86 47 | runs-on: ubuntu-20.04 48 | strategy: 49 | fail-fast: true 50 | 51 | steps: 52 | - name: Check out code 53 | uses: actions/checkout@v2 54 | 55 | - name: Install dependencies 56 | shell: bash 57 | run: | 58 | make -f ${{ env.MAKEFILE }} dep_install 59 | 60 | - name: Download the iso file 61 | shell: bash 62 | run: | 63 | make -f ${{ env.MAKEFILE }} download 64 | 65 | - name: Install ubuntu on the disk 66 | shell: bash 67 | run: | 68 | make -f ${{ env.MAKEFILE }} install_no_kvm 69 | 70 | - name: Save the image as base image 71 | shell: bash 72 | env: 73 | OUTPUT: ${{ env.DISK_PRE_TEST }} 74 | run: | 75 | make -f ${{ env.MAKEFILE }} save_output 76 | 77 | - name: Clean everything 78 | if: ${{ always() }} 79 | run: | 80 | make -f ${{ env.MAKEFILE }} clean 81 | 82 | - name: Upload Base Image Artifact 83 | uses: actions/upload-artifact@v3 84 | with: 85 | name: base-disk-pre-test 86 | path: ${{ env.DISK_PRE_TEST }} 87 | -------------------------------------------------------------------------------- /.github/workflows/create_base_disk_self_hosted_arm.yml: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2022 David Schall and EASE Lab 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | name: Create ARM base disk image on self-hosted runners 24 | 25 | env: 26 | ARTIFACTS_DIR: artifacts/ 27 | KERNEL: artifacts/kernel 28 | DISK_PRE_TEST: artifacts/disk-image.qcow2 29 | DISK_FINAL: artifacts/disk.qcow2 30 | CHECKSUMS: artifacts/checksums 31 | CLIENT: artifacts/test-client 32 | 33 | on: 34 | # Allows you to run this workflow manually from the Actions tab 35 | workflow_dispatch: 36 | 37 | push: 38 | branches: [main] 39 | paths: 40 | - "test/**" 41 | - "setup/disk.Makefile" 42 | - "configs/disk-image-configs/**" 43 | 44 | pull_request: 45 | branches: [main] 46 | paths: 47 | - "test/**" 48 | - "setup/disk.Makefile" 49 | - "configs/disk-image-configs/**" 50 | 51 | jobs: 52 | 53 | ######################################### 54 | ### Install the OS on the new disk image 55 | 56 | install-base-image: 57 | name: Build Base image with Ubuntu 58 | strategy: 59 | fail-fast: false 60 | matrix: 61 | ubuntu-version: [ focal, jammy ] 62 | # arch: [ amd64, arm64 ] 63 | include: 64 | # - arch: amd64 65 | # makefile: setup/disk.Makefile 66 | # runner-arch: X64 67 | - arch: arm64 68 | makefile: setup/disk_arm.Makefile 69 | runner-arch: ARM64 70 | runs-on: [self-hosted, vswarm-u-build, ARM64 ] 71 | env: 72 | WORKDIR: setup/ 73 | MAKEFILE: ${{ matrix.makefile }} 74 | TEST_MAKEFILE: test/Makefile 75 | BUILD_DIR: wkdir/ 76 | UBUNTU_VERSION: ${{ matrix.ubuntu-version }} 77 | ARCH: ${{ matrix.arch }} 78 | 79 | steps: 80 | - name: Check out code 81 | uses: actions/checkout@v4 82 | 83 | 84 | ### Get Kernel and test-client ################# 85 | - name: Download latest kernel 86 | uses: dawidd6/action-download-artifact@v9 87 | with: 88 | workflow: build_kernel.yml 89 | name: vmlinux-${{ matrix.ubuntu-version }}-${{ matrix.arch }} 90 | path: tmp 91 | 92 | - name: Download latest test client 93 | uses: dawidd6/action-download-artifact@v9 94 | with: 95 | workflow: test_client.yml 96 | name: test-client-${{ matrix.arch }} 97 | path: tmp 98 | 99 | - name: Copy artifacts 100 | run: | 101 | mkdir -p ${{ env.ARTIFACTS_DIR }} 102 | cp tmp/kernel ${{ env.KERNEL }} 103 | cp tmp/test-client ${{ env.CLIENT }} 104 | 105 | 106 | 107 | ### Build the disk ############################### 108 | - name: Install dependencies 109 | shell: bash 110 | run: | 111 | make -f ${{ env.MAKEFILE }} dep_install 112 | 113 | - name: Download the iso file 114 | shell: bash 115 | run: | 116 | make -f ${{ env.MAKEFILE }} download 117 | 118 | - name: Settup the working directory 119 | shell: bash 120 | run: | 121 | make -f ${{ env.MAKEFILE }} build 122 | 123 | - name: Install ubuntu on the disk 124 | shell: bash 125 | run: | 126 | make -f ${{ env.MAKEFILE }} install_kvm 127 | 128 | - name: Finalize Disk image 129 | shell: bash 130 | env: 131 | KERNEL_CUSTOM: ${{ env.KERNEL }} 132 | run: | 133 | make -f ${{ env.MAKEFILE }} install_finalize_kvm 134 | 135 | - name: Save the image as base image 136 | shell: bash 137 | env: 138 | OUTPUT: ${{ env.DISK_PRE_TEST }} 139 | run: | 140 | make -f ${{ env.MAKEFILE }} save_output 141 | 142 | - name: Clean everything 143 | if: ${{ always() }} 144 | run: | 145 | make -f ${{ env.MAKEFILE }} clean 146 | 147 | 148 | 149 | ## Upload the artifact ############################### 150 | - name: Compress and split 151 | run: | 152 | mkdir temp 153 | cp ${{ env.DISK_PRE_TEST }} temp/ 154 | ./resources/artifacts.sh compress-split temp/* 155 | rm temp/disk-image.qcow2 156 | 157 | - name: Upload Base Image Artifact 158 | uses: actions/upload-artifact@v4 159 | with: 160 | name: disk-image-${{ matrix.ubuntu-version }}-${{ matrix.arch }} 161 | path: temp 162 | 163 | - name: Create checksums of the artifacts 164 | shell: bash 165 | run: | 166 | shasum ${{ env.DISK_PRE_TEST }} > ${{ env.CHECKSUMS }} 167 | 168 | - name: Upload artifact 169 | uses: actions/upload-artifact@v4 170 | with: 171 | name: checksum-${{ matrix.ubuntu-version }}-${{ matrix.arch }} 172 | path: ${{ env.CHECKSUMS }} 173 | 174 | 175 | 176 | -------------------------------------------------------------------------------- /.github/workflows/delete_old.yml: -------------------------------------------------------------------------------- 1 | name: Delete old workflow runs 2 | on: 3 | workflow_dispatch: 4 | # inputs: 5 | # days: 6 | # description: 'Number of days.' 7 | # required: true 8 | # default: 30 9 | # minimum_runs: 10 | # description: 'The minimum runs to keep for each workflow.' 11 | # required: true 12 | # default: 6 13 | # delete_workflow_pattern: 14 | # description: 'The name of the workflow. if not set then it will target all workflows.' 15 | # required: false 16 | 17 | schedule: 18 | - cron: '0 0 1 * *' 19 | # Run monthly, at 00:00 on the 1st day of month. 20 | 21 | jobs: 22 | del_runs: 23 | runs-on: ubuntu-latest 24 | 25 | steps: 26 | 27 | # - name: Delete workflow runs manually 28 | # uses: Mattraks/delete-workflow-runs@v2 29 | # with: 30 | # token: ${{ github.token }} 31 | # repository: ${{ github.repository }} 32 | # retain_days: ${{ github.event.inputs.days }} 33 | # keep_minimum_runs: ${{ github.event.inputs.minimum_runs }} 34 | # delete_workflow_pattern: ${{ github.event.inputs.delete_workflow_pattern }} 35 | 36 | - name: Delete workflow runs monthly 37 | uses: Mattraks/delete-workflow-runs@v2 38 | with: 39 | token: ${{ github.token }} 40 | repository: ${{ github.repository }} 41 | retain_days: 30 42 | keep_minimum_runs: 6 43 | 44 | -------------------------------------------------------------------------------- /.github/workflows/enable-automerge.yaml: -------------------------------------------------------------------------------- 1 | name: Enable automerge on dependabot PRs 2 | 3 | on: 4 | # See note below about using pull_request_target 5 | pull_request_target: 6 | 7 | jobs: 8 | automerge: 9 | name: Enable automerge on dependabot PRs 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Enable automerge on dependabot PRs 13 | uses: daneden/enable-automerge-action@v1 14 | with: 15 | # A personal access token that you have generated and saved in the 16 | # repo or org’s encrypted secrets 17 | github-token: ${{ secrets.ENABLE_AUTOMERGE_TOKEN }} 18 | 19 | # The name of the PR author to enable automerge for 20 | # Defaults to dependabot[bot] 21 | allowed-author: "dependabot[bot]" 22 | 23 | # Allowed values: MERGE | SQUASH | REBASE 24 | # Defaults to MERGE 25 | merge-method: REBASE 26 | -------------------------------------------------------------------------------- /.github/workflows/linters.yml: -------------------------------------------------------------------------------- 1 | name: Linters 2 | on: 3 | push: 4 | branches: [main] 5 | pull_request: 6 | branches: [main] 7 | 8 | jobs: 9 | build: 10 | name: Spellcheck 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v4 14 | - uses: rojopolis/spellcheck-github-actions@0.47.0 15 | name: Spellcheck 16 | with: 17 | config_path: .github/configs/spellcheck.yml 18 | markdown-link-check: 19 | name: LinkCheck 20 | runs-on: ubuntu-latest 21 | steps: 22 | - uses: actions/checkout@v4 23 | - uses: gaurav-nelson/github-action-markdown-link-check@v1 24 | with: 25 | use-quiet-mode: "yes" 26 | config-file: ".github/configs/linkcheck.json" 27 | -------------------------------------------------------------------------------- /.github/workflows/test_client.yml: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2022 EASE lab, University of Edinburgh 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | # 23 | # Authors: David Schall 24 | 25 | 26 | name: Build Test client 27 | 28 | on: 29 | workflow_dispatch: 30 | schedule: 31 | - cron: "0 7 * * 1" 32 | 33 | push: 34 | branches: [main] 35 | paths: 36 | - "tools/**" 37 | 38 | pull_request: 39 | branches: [main] 40 | paths: 41 | - "tools/**" 42 | 43 | env: 44 | GOOS: linux 45 | GO111MODULE: on 46 | 47 | 48 | jobs: 49 | build: 50 | name: Build 51 | runs-on: ubuntu-22.04 52 | strategy: 53 | matrix: 54 | arch: [ amd64, arm64 ] 55 | env: 56 | ARCH: ${{ matrix.arch }} 57 | steps: 58 | - uses: actions/checkout@v4 59 | - uses: actions/setup-go@v5 60 | with: 61 | go-version: '1.21' 62 | 63 | - name: Install cross compiler 64 | working-directory: tools/client 65 | run: make dep_install 66 | 67 | - name: Build client 68 | working-directory: tools/client 69 | run: | 70 | go mod tidy 71 | make all 72 | 73 | - name: Save the client 74 | working-directory: tools/client 75 | env: 76 | OUTPUT: test-client 77 | run: make save_output 78 | 79 | - name: Upload the artifact 80 | uses: actions/upload-artifact@v4 81 | with: 82 | name: test-client-${{ matrix.arch }} 83 | path: tools/client/test-client 84 | -------------------------------------------------------------------------------- /.github/workflows/test_download.yml: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2022 David Schall and EASE Lab 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | name: Test Resource download script 24 | 25 | on: 26 | # Allows you to run this workflow manually from the Actions tab 27 | workflow_dispatch: 28 | schedule: 29 | - cron: "0 7 * * 1" 30 | 31 | push: 32 | branches: [main] 33 | paths: 34 | - "resources/**" 35 | 36 | pull_request: 37 | branches: [main] 38 | paths: 39 | - "resources/**" 40 | 41 | jobs: 42 | build: 43 | name: Test download 44 | # Building the kernel works also on the github runners. 45 | runs-on: ubuntu-latest 46 | strategy: 47 | fail-fast: false 48 | matrix: 49 | arch: [ amd64, arm64 ] 50 | os-version: [ focal, jammy ] 51 | env: 52 | OUTPUT: tmp_dir/ 53 | VERSION: latest 54 | 55 | steps: 56 | - name: Check out code 57 | uses: actions/checkout@v4 58 | 59 | - name: Set up Python version 60 | uses: actions/setup-python@v5 61 | with: 62 | python-version: '3.9' 63 | cache: 'pip' # caching pip dependencies 64 | 65 | - name: Set up python dependencies 66 | run: pip install -r ./setup/requirements.txt 67 | 68 | # - name: Set up python dependencies 69 | # run: | 70 | # python -m pip install --upgrade pip 71 | # python -m pip install tqdm 72 | 73 | - name: Make test dir 74 | shell: bash 75 | run: | 76 | mkdir $OUTPUT 77 | 78 | - name: Download 79 | shell: bash 80 | run: | 81 | python ./resources/artifacts.py \ 82 | --arch ${{ matrix.arch }} \ 83 | --os-version ${{ matrix.os-version }} \ 84 | --version $VERSION \ 85 | --output $OUTPUT 86 | 87 | - name: Test 88 | shell: bash 89 | run: | 90 | ls -la $OUTPUT -------------------------------------------------------------------------------- /.github/workflows/test_fn_install.yml: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2022 David Schall and EASE Lab 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | name: Test function installation 24 | 25 | on: 26 | # Allows you to run this workflow manually from the Actions tab 27 | workflow_dispatch: 28 | schedule: 29 | - cron: "0 7 * * 1" 30 | 31 | push: 32 | branches: [main] 33 | 34 | pull_request: 35 | branches: [main] 36 | 37 | jobs: 38 | build: 39 | name: Test download 40 | # Building the kernel works also on the github runners. 41 | runs-on: ${{ matrix.host-os }} 42 | strategy: 43 | fail-fast: false 44 | matrix: 45 | arch: [ amd64 ] 46 | os-version: [ focal ] 47 | host-os: [ ubuntu-22.04, ubuntu-24.04 ] 48 | env: 49 | WORKING_DIR: wkdir/ 50 | RESOURCES: resources/ 51 | VERSION: latest 52 | 53 | steps: 54 | - name: Check out code 55 | uses: actions/checkout@v4 56 | 57 | ### Setup working environment #### 58 | - name: Set up Python version 59 | uses: actions/setup-python@v5 60 | with: 61 | python-version: '3.9' 62 | cache: 'pip' # caching pip dependencies 63 | 64 | - name: Set up python dependencies 65 | run: pip install -r ./setup/requirements.txt 66 | 67 | - name: Set Qemu dependencies 68 | run: | 69 | make -f ./setup/disk.Makefile dep_install 70 | 71 | - name: Download Artifacts 72 | shell: bash 73 | run: | 74 | python ./resources/artifacts.py \ 75 | --version $VERSION 76 | 77 | 78 | 79 | 80 | ## Test the function installation as described 81 | ## in the quick start guide 82 | 83 | - name: Build the working directory 84 | shell: bash 85 | run: | 86 | make -f ./simulation/Makefile build-wkdir 87 | 88 | - name: Pull, and test the function containers 89 | shell: bash 90 | timeout-minutes: 15 91 | env: 92 | CPU: max 93 | run: | 94 | make -f ./simulation/Makefile install_functions 95 | 96 | - name: Verify a successful installation 97 | shell: bash 98 | run: | 99 | make -f ./simulation/Makefile install_check 100 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | *.smod 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.exe 31 | *.out 32 | *.app 33 | 34 | 35 | ### Go ### 36 | # If you prefer the allow list template instead of the deny list, see community template: 37 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 38 | # 39 | # Binaries for programs and plugins 40 | *.exe 41 | *.exe~ 42 | *.dll 43 | *.so 44 | *.dylib 45 | 46 | # Test binary, built with `go test -c` 47 | *.test 48 | 49 | # Output of the go coverage tool, specifically when used with LiteIDE 50 | *.out 51 | 52 | # Dependency directories (remove the comment below to include it) 53 | # vendor/ 54 | 55 | # Go workspace file 56 | go.work 57 | 58 | 59 | 60 | ### Python ### 61 | # Byte-compiled / optimized / DLL files 62 | __pycache__/ 63 | *.py[cod] 64 | *$py.class 65 | 66 | # C extensions 67 | *.so 68 | 69 | # Distribution / packaging 70 | .Python 71 | build/ 72 | develop-eggs/ 73 | dist/ 74 | downloads/ 75 | eggs/ 76 | .eggs/ 77 | lib/ 78 | lib64/ 79 | parts/ 80 | sdist/ 81 | var/ 82 | wheels/ 83 | share/python-wheels/ 84 | *.egg-info/ 85 | .installed.cfg 86 | *.egg 87 | MANIFEST 88 | 89 | # PyInstaller 90 | # Usually these files are written by a python script from a template 91 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 92 | *.manifest 93 | *.spec 94 | 95 | # Installer logs 96 | pip-log.txt 97 | pip-delete-this-directory.txt 98 | 99 | # Unit test / coverage reports 100 | htmlcov/ 101 | .tox/ 102 | .nox/ 103 | .coverage 104 | .coverage.* 105 | .cache 106 | nosetests.xml 107 | coverage.xml 108 | *.cover 109 | *.py,cover 110 | .hypothesis/ 111 | .pytest_cache/ 112 | cover/ 113 | 114 | # Translations 115 | *.mo 116 | *.pot 117 | 118 | # Django stuff: 119 | *.log 120 | local_settings.py 121 | db.sqlite3 122 | db.sqlite3-journal 123 | 124 | # Flask stuff: 125 | instance/ 126 | .webassets-cache 127 | 128 | # Scrapy stuff: 129 | .scrapy 130 | 131 | # Sphinx documentation 132 | docs/_build/ 133 | 134 | # PyBuilder 135 | .pybuilder/ 136 | target/ 137 | 138 | # Jupyter Notebook 139 | .ipynb_checkpoints 140 | 141 | # IPython 142 | profile_default/ 143 | ipython_config.py 144 | 145 | # pyenv 146 | # For a library or package, you might want to ignore these files since the code is 147 | # intended to run in multiple environments; otherwise, check them in: 148 | # .python-version 149 | 150 | # pipenv 151 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 152 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 153 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 154 | # install all needed dependencies. 155 | #Pipfile.lock 156 | 157 | # poetry 158 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 159 | # This is especially recommended for binary packages to ensure reproducibility, and is more 160 | # commonly ignored for libraries. 161 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 162 | #poetry.lock 163 | 164 | # pdm 165 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 166 | #pdm.lock 167 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 168 | # in version control. 169 | # https://pdm.fming.dev/#use-with-ide 170 | .pdm.toml 171 | 172 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 173 | __pypackages__/ 174 | 175 | # Celery stuff 176 | celerybeat-schedule 177 | celerybeat.pid 178 | 179 | # SageMath parsed files 180 | *.sage.py 181 | 182 | # Environments 183 | .env 184 | .venv 185 | env/ 186 | venv/ 187 | ENV/ 188 | env.bak/ 189 | venv.bak/ 190 | 191 | # Spyder project settings 192 | .spyderproject 193 | .spyproject 194 | 195 | # Rope project settings 196 | .ropeproject 197 | 198 | # mkdocs documentation 199 | /site 200 | 201 | # mypy 202 | .mypy_cache/ 203 | .dmypy.json 204 | dmypy.json 205 | 206 | # Pyre type checker 207 | .pyre/ 208 | 209 | # pytype static type analyzer 210 | .pytype/ 211 | 212 | # Cython debug symbols 213 | cython_debug/ 214 | 215 | # PyCharm 216 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 217 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 218 | # and can be added to the global gitignore or merged into this file. For a more nuclear 219 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 220 | #.idea/ 221 | 222 | 223 | ## Other build and working directories ----- 224 | 225 | **/linux 226 | **/workload 227 | **/wkdir* 228 | !**/wkdir-tmpl 229 | **/results 230 | 231 | **/tmp 232 | **/iso 233 | **/mnt 234 | 235 | ## Resources folder 236 | resources/* 237 | !resources/Makefile 238 | !resources/artifacts* 239 | !resources/release* 240 | !resources/**.py 241 | 242 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Edinburgh Architecture and Systems (EASE) Lab @ University of Edinburgh 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /analysis/check_simulations.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from itertools import count 3 | import os 4 | import re 5 | import argparse 6 | 7 | ROOT = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../") 8 | 9 | 10 | def parse_arguments(): 11 | parser = argparse.ArgumentParser(description="") 12 | parser.add_argument("dir", type = str, help = "Results directory") 13 | return parser.parse_args() 14 | 15 | 16 | def prRed(skk): print("\033[91m {}\033[00m" .format(skk)) 17 | def prGreen(skk): print("\033[92m {}\033[00m" .format(skk)) 18 | 19 | 20 | 21 | args = parse_arguments() 22 | subdirs = [s for s in os.listdir(args.dir) if os.path.isdir(args.dir + s)] 23 | subdirs.sort() 24 | 25 | rerun_cmds = [] 26 | results = [] 27 | 28 | 29 | for subdir in subdirs[:]: 30 | filename = "{}/{}/gem5.log".format(args.dir,subdir) 31 | cmd = "" 32 | success = False 33 | 34 | if os.path.exists(filename): 35 | with open(filename) as f: 36 | start_tick = 0 37 | count = 0 38 | for line in f: 39 | if line[:14] == "command line: ": 40 | cmd = line[14:].strip() 41 | # if "'fwait'" in line: 42 | # print("{} stopped with 'fwait'".format(subdir)) 43 | if "Simulation done" in line: 44 | success = True 45 | break 46 | if "End invokation:" in line: 47 | count += 1 48 | 49 | if success: 50 | if count < 20: 51 | status = "\033[93m WARN\033[00m" 52 | else: 53 | status = "\033[92m succeed\033[00m" 54 | 55 | else: 56 | status= "\033[91m fail\033[00m" 57 | print(f"{subdir:>25} > {count} : {status}") 58 | rerun_cmds += [(success, f"{cmd} > {filename} 2>&1 &")] 59 | 60 | total = len(rerun_cmds) -1 61 | with open("rerun.sh", "w") as f: 62 | for success,cmd in rerun_cmds: 63 | comment = "# " if success else "" 64 | f.write(f"{comment} sudo {cmd} \n") 65 | 66 | -------------------------------------------------------------------------------- /analysis/gem5_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: UTF8 -*- 3 | import csv 4 | import json 5 | 6 | import collections 7 | from numpy import dtype 8 | from objectpath import * 9 | from os import path, listdir, stat 10 | from pyparsing import Word, Optional, ParseException, printables, nums, restOfLine 11 | import matplotlib.pyplot as plt 12 | import pandas as pd 13 | import pickle 14 | import seaborn as sns 15 | 16 | 17 | class ExperimentResults: 18 | def __init__(self, configs, stats, props): 19 | self.configs = configs 20 | self.stats = stats 21 | self.props = props 22 | 23 | 24 | class ExperimentConfigs: 25 | def __init__(self, raw_configs): 26 | self.raw_configs = raw_configs 27 | 28 | def __getitem__(self, index): 29 | return self.raw_configs.execute('$.' + index) 30 | 31 | 32 | class ExperimentStats: 33 | def __init__(self, raw_stats): 34 | self.raw_stats = raw_stats 35 | 36 | def __getitem__(self, index): 37 | return self.raw_stats[index] if index in self.raw_stats else None 38 | 39 | 40 | def read_configs(result_dir, config_json_file_name): 41 | try: 42 | with open(path.join(result_dir, config_json_file_name)) as config_json_file: 43 | # configs = Tree(json.load(config_json_file)) 44 | configs = json.load(config_json_file) 45 | except Exception as e: 46 | print(e) 47 | return None 48 | else: 49 | return configs 50 | 51 | 52 | def read_stats(result_dir, stats_file_name): 53 | stat_rule = Word(printables) + Word('nan.%' + nums) + Optional(restOfLine) 54 | 55 | stats = [] 56 | 57 | try: 58 | with open(path.join(result_dir, stats_file_name)) as stats_file: 59 | i = 0 60 | for stat_line in stats_file: 61 | if len(stats) <= i: 62 | stats.append(collections.OrderedDict()) 63 | 64 | try: 65 | stat = stat_rule.parseString(stat_line) 66 | key = stat[0] 67 | value = stat[1] 68 | 69 | stats[i][key] = value 70 | except ParseException as e: 71 | # print(e) 72 | pass 73 | 74 | if 'End Simulation Statistics' in stat_line: 75 | i += 1 76 | except Exception as e: 77 | print(e) 78 | return None 79 | else: 80 | return stats 81 | 82 | def find_stats(result_dir, stats_file_name="stats.txt"): 83 | ## In case this is the first time we read values from this directory 84 | # Or if the stats file has changed in the meantime parse the file to be more faster 85 | filename = result_dir + "/" + stats_file_name[:-5] 86 | if not path.exists(filename) or path.getmtime(result_dir + "/" + stats_file_name) > path.getmtime(filename): 87 | tmp = read_stats(result_dir, stats_file_name) 88 | with open(filename, 'wb') as f: 89 | pickle.dump(tmp,f) 90 | 91 | with open(filename, 'rb') as f: 92 | stats = pickle.load(f) 93 | return stats 94 | return None 95 | 96 | 97 | def find_stats_group(result_dir): 98 | ## With this function we want to get the values from an entire folder full of results 99 | subdirs = [s for s in listdir(result_dir) if path.isdir(result_dir + s)] 100 | subdirs.sort() 101 | stats_group = {} 102 | 103 | for subdir in subdirs: 104 | stats = find_stats(path.join(result_dir,subdir)) 105 | if stats: 106 | stats_group[subdir] = stats 107 | 108 | return stats_group 109 | 110 | 111 | 112 | def parse_result(result_dir, config_json_file_name='config.json', stats_file_name='stats.txt', **props): 113 | return ExperimentResults(ExperimentConfigs(read_configs(result_dir, config_json_file_name)), 114 | [ExperimentStats(stat) for stat in read_stats(result_dir, stats_file_name)], props) 115 | 116 | 117 | def to_csv(output_file_name, results, fields): 118 | with open(output_file_name, 'w') as output_file: 119 | writer = csv.writer(output_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL) 120 | 121 | writer.writerow([field[0] for field in fields]) 122 | 123 | for result in results: 124 | writer.writerow([field[1](result) for field in fields]) 125 | 126 | def to_pandas(results, fields): 127 | columns = [field[0] for field in fields] 128 | dtype = {s:t for s,t in zip([field[0] for field in fields],[field[2] for field in fields])} 129 | data = [] 130 | for result in results: 131 | try: 132 | data += [[int(field[1](result)) for field in fields]] 133 | except: 134 | data += [[field[1](result) for field in fields]] 135 | return pd.DataFrame(data=data, columns=columns).astype(dtype) 136 | 137 | 138 | def generate_plot(csv_file_name, plot_file_name, x, y, hue, y_title, xticklabels_rotation=90): 139 | sns.set(font_scale=1.5) 140 | 141 | sns.set_style("white", {"legend.frameon": True}) 142 | 143 | df = pd.read_csv(csv_file_name) 144 | 145 | ax = sns.barplot(data=df, x=x, y=y, hue=hue, palette=sns.color_palette("Paired")) 146 | ax.set_xlabel('') 147 | ax.set_ylabel(y_title) 148 | 149 | labels = ax.get_xticklabels() 150 | ax.set_xticklabels(labels, rotation=xticklabels_rotation) 151 | 152 | fig = ax.get_figure() 153 | 154 | if hue: 155 | legend = ax.legend(bbox_to_anchor=(1.05, 1), loc='upper left', borderaxespad=0.) 156 | legend.set_label('') 157 | 158 | fig.savefig(plot_file_name, bbox_extra_artists=(legend,), bbox_inches='tight') 159 | fig.savefig(plot_file_name + '.jpg', bbox_extra_artists=(legend,), bbox_inches='tight') 160 | else: 161 | fig.tight_layout() 162 | 163 | fig.savefig(plot_file_name) 164 | fig.savefig(plot_file_name + '.jpg') 165 | 166 | plt.clf() 167 | plt.close('all') 168 | -------------------------------------------------------------------------------- /configs/disk-image-configs/autoinstall-amd64.yaml: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | autoinstall: 3 | version: 1 4 | apt: 5 | geoip: true 6 | preserve_sources_list: false 7 | primary: 8 | - arches: [amd64, i386] 9 | uri: http://us.archive.ubuntu.com/ubuntu 10 | - arches: [default] 11 | uri: http://ports.ubuntu.com/ubuntu-ports 12 | 13 | # The passwords are all 'root' 14 | user-data: 15 | disable_root: false 16 | chpasswd: 17 | list: | 18 | root:$6$FhcddHFVZ7ABA4Gi$9l4yURWASWe8xEa1jzI0bacVLvhe3Yn4/G3AnU11K3X0yu/mICVRxfo6tZTB2noKljlIRzjkVZPocdf63MtzC0 19 | identity: 20 | hostname: ubuntu-server 21 | username: ubuntu # root does not work 22 | password: "$6$FhcddHFVZ7ABA4Gi$9l4yURWASWe8xEa1jzI0bacVLvhe3Yn4/G3AnU11K3X0yu/mICVRxfo6tZTB2noKljlIRzjkVZPocdf63MtzC0" # root 23 | 24 | # kernel: 25 | # kernel: 26 | # package: linux-generic 27 | keyboard: 28 | layout: us 29 | toggle: null 30 | variant: '' 31 | locale: en_US.UTF-8 32 | network: 33 | network: 34 | ethernets: 35 | ens3: 36 | dhcp4: true 37 | version: 2 38 | ssh: 39 | allow-pw: true 40 | authorized-keys: [] 41 | install-server: true 42 | 43 | updates: security 44 | storage: 45 | layout: 46 | name: "direct" 47 | refresh-installer: 48 | update: true 49 | 50 | ## The following late commands are executed as soon as ubuntu is fully installed 51 | ## We enable some useful features to use vSwarm in the gem5 simulator 52 | 53 | late-commands: 54 | 55 | # Instead of listing all install commands here we 56 | # put them in a separate script. Here we only download and execute this script 57 | - curtin in-target --target=/target -- wget -P /root/ http://_gateway:3003/init_disk_image.sh 58 | - curtin in-target --target=/target -- chmod +x /root/init_disk_image.sh 59 | - curtin in-target --target=/target -- bash /root/init_disk_image.sh 60 | 61 | runcmd: 62 | - [ systemctl, restart, gem5.service ] 63 | - [ sh, -xc, "echo $(date) ': hello world!'" ] -------------------------------------------------------------------------------- /configs/disk-image-configs/autoinstall-arm64.yaml: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | autoinstall: 3 | version: 1 4 | apt: 5 | geoip: true 6 | preserve_sources_list: false 7 | primary: 8 | # - arches: [amd64, arm64, default] 9 | # uri: http://us.archive.ubuntu.com/ubuntu 10 | - arches: [default] 11 | uri: http://us.ports.ubuntu.com/ubuntu-ports 12 | 13 | # The passwords are all 'root' 14 | user-data: 15 | disable_root: false 16 | chpasswd: 17 | list: | 18 | root:$6$FhcddHFVZ7ABA4Gi$9l4yURWASWe8xEa1jzI0bacVLvhe3Yn4/G3AnU11K3X0yu/mICVRxfo6tZTB2noKljlIRzjkVZPocdf63MtzC0 19 | identity: 20 | hostname: ubuntu-server 21 | username: ubuntu # root does not work 22 | password: "$6$FhcddHFVZ7ABA4Gi$9l4yURWASWe8xEa1jzI0bacVLvhe3Yn4/G3AnU11K3X0yu/mICVRxfo6tZTB2noKljlIRzjkVZPocdf63MtzC0" # root 23 | 24 | # kernel: 25 | # kernel: 26 | # package: linux-generic 27 | keyboard: 28 | layout: us 29 | toggle: null 30 | variant: '' 31 | locale: en_US.UTF-8 32 | network: 33 | network: 34 | ethernets: 35 | enp0s1: 36 | dhcp4: true 37 | version: 2 38 | ssh: 39 | allow-pw: true 40 | authorized-keys: [] 41 | install-server: true 42 | 43 | updates: security 44 | storage: 45 | layout: 46 | name: "direct" 47 | refresh-installer: 48 | update: true 49 | 50 | ## The following late commands are executed as soon as ubuntu is fully installed 51 | ## We enable some useful features to use vSwarm in the gem5 simulator 52 | 53 | late-commands: 54 | # Instead of listing all install commands here we 55 | # put them in a separate script. Here we only download and execute this script 56 | - curtin in-target --target=/target -- wget -P /root/ http://_gateway:3003/init_disk_image.sh 57 | - curtin in-target --target=/target -- chmod +x /root/init_disk_image.sh 58 | - curtin in-target --target=/target -- bash /root/init_disk_image.sh -------------------------------------------------------------------------------- /configs/disk-image-configs/client-amd64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vhive-serverless/vSwarm-u/b262a4bde890380d4f6e7b0e8ea8667b028051ea/configs/disk-image-configs/client-amd64 -------------------------------------------------------------------------------- /configs/disk-image-configs/client-arm64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vhive-serverless/vSwarm-u/b262a4bde890380d4f6e7b0e8ea8667b028051ea/configs/disk-image-configs/client-arm64 -------------------------------------------------------------------------------- /configs/disk-image-configs/finalize.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | ## Wait until cloud-init is finish 6 | 7 | sleep 15; 8 | 9 | while [ ! -f /var/lib/cloud/instance/boot-finished ]; 10 | do echo wait; sleep 5; 11 | done 12 | 13 | # Prevent start by creating an empty file 14 | touch /etc/cloud/cloud-init.disabled 15 | 16 | # Or Uninstall the package and delete the folders 17 | 18 | # sudo dpkg-reconfigure cloud-init 19 | # sudo apt-get purge cloud-init 20 | # sudo rm -rf /etc/cloud/ && sudo rm -rf /var/lib/cloud/ 21 | 22 | sudo update-alternatives --set iptables /usr/sbin/iptables-legacy 23 | sudo update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy 24 | 25 | ## Disable other unnecessary services 26 | systemctl disable systemd-modules-load.service 27 | systemctl disable multipathd.service 28 | systemctl disable apt-daily-upgrade.timer 29 | 30 | apt update && apt upgrade -y 31 | 32 | shutdown -h now -------------------------------------------------------------------------------- /configs/disk-image-configs/gem5init: -------------------------------------------------------------------------------- 1 | #!/bin/bash - 2 | 3 | 4 | if [ $(uname -i) == "aarch64" ]; 5 | then 6 | echo "On Arm machine" 7 | M5_ADDR="--addr=0x10010000" 8 | else 9 | echo "On x86 machine" 10 | M5_ADDR=0xffff0000 11 | M5_ADDR="" 12 | fi 13 | 14 | 15 | ###################################################### 16 | # Qemu run service 17 | # ==== 18 | # 19 | # Try to read in a run script from the host system. 20 | # For qemu we will try to get the file from an 21 | # HTTP server running on the host port 3003. 22 | # 23 | # 24 | printf "Try QEMU read \n" 25 | 26 | curl "http://10.0.2.2:3003/run.sh" -f -o /tmp/script 27 | 28 | if [ 0 -eq $? ]; 29 | then 30 | echo "Found file server in qemu." 31 | echo "Run script found... run it." 32 | chmod 755 /tmp/script 33 | 34 | # If there is a script, execute the script and then shutdown the machine 35 | /tmp/script || true 36 | echo "Done running script, exiting." 37 | rm -f /tmp/script 38 | 39 | exit 0 40 | fi 41 | 42 | 43 | 44 | 45 | ###################################################### 46 | # gem5 init service 47 | # ==== 48 | printf "Starting gem5 init... trying to read run script file via readfile.\n" 49 | # 50 | # Try to read in a run script from the host system. 51 | # For gem5 use the special magic instruction `m5 readfile` 52 | # 53 | 54 | if m5 $M5_ADDR readfile > /tmp/script; then 55 | 56 | echo "Run script found... run it." 57 | chmod 755 /tmp/script 58 | /tmp/script || true 59 | echo "Done running script, exiting." 60 | rm -f /tmp/script 61 | /sbin/m5 $M5_ADDR exit 62 | fi 63 | 64 | 65 | rm -f /tmp/script 66 | 67 | echo "All done, exiting." 68 | exit 0 69 | -------------------------------------------------------------------------------- /configs/disk-image-configs/init_disk_image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # MIT License 4 | # 5 | # Copyright (c) 2022 David Schall and EASE lab 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | # Install dependencies 25 | 26 | set -e 27 | 28 | # Allow root to login with ssh 29 | echo 'ubuntu ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/ubuntu 30 | sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config 31 | systemctl restart sshd 32 | 33 | # Do a full upgrade 34 | apt update 35 | # apt full-upgrade -y 36 | apt install -y net-tools 37 | # apt full-upgrade -y 38 | # sudo apt-get -y install \ 39 | # gcc \ 40 | # g++ \ 41 | # make 42 | 43 | 44 | if [ $(uname -i) == "aarch64" ]; 45 | then 46 | ARCH=arm64 47 | else 48 | ARCH=amd64 49 | fi 50 | 51 | ##### GEM5 specific setup ##### 52 | ## Prepare gem5 utility tool 53 | wget http://_gateway:3003/m5.${ARCH} 54 | mv m5.${ARCH} /sbin/m5 55 | chmod +x /sbin/m5 56 | 57 | ## Create and enable the gem5 init service 58 | wget -P /sbin/ http://_gateway:3003/gem5init 59 | chmod +x /sbin/gem5init 60 | 61 | cat > /lib/systemd/system/gem5.service <> /etc/profile" 111 | 112 | ## Get the client binary 113 | wget -P /root/ http://_gateway:3003/client-${ARCH} 114 | chmod +x /root/client-${ARCH} 115 | 116 | # Enable gem5 service 117 | systemctl daemon-reload 118 | systemctl enable gem5.service -------------------------------------------------------------------------------- /configs/disk-image-configs/m5.amd64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vhive-serverless/vSwarm-u/b262a4bde890380d4f6e7b0e8ea8667b028051ea/configs/disk-image-configs/m5.amd64 -------------------------------------------------------------------------------- /configs/disk-image-configs/m5.arm64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vhive-serverless/vSwarm-u/b262a4bde890380d4f6e7b0e8ea8667b028051ea/configs/disk-image-configs/m5.arm64 -------------------------------------------------------------------------------- /configs/linux-configs/kernel_m5.patch: -------------------------------------------------------------------------------- 1 | diff --git a/kernel/Makefile b/kernel/Makefile 2 | index f2cc0d118a0b..767b5911eb4d 100644 3 | --- a/kernel/Makefile 4 | +++ b/kernel/Makefile 5 | @@ -12,6 +12,9 @@ obj-y = fork.o exec_domain.o panic.o \ 6 | notifier.o ksysfs.o cred.o reboot.o \ 7 | async.o range.o smpboot.o ucount.o 8 | 9 | +# gem5 hooks for streamline support 10 | +obj-y += m5struct.o 11 | + 12 | obj-$(CONFIG_MODULES) += kmod.o 13 | obj-$(CONFIG_MULTIUSER) += groups.o 14 | 15 | diff --git a/kernel/m5struct.c b/kernel/m5struct.c 16 | new file mode 100644 17 | index 000000000000..abce546008a9 18 | --- /dev/null 19 | +++ b/kernel/m5struct.c 20 | @@ -0,0 +1,47 @@ 21 | +/* 22 | + * Copyright (c) 2005 23 | + * The Regents of The University of Michigan 24 | + * All Rights Reserved 25 | + * 26 | + * This code is part of the M5 simulator, developed by Nathan Binkert, 27 | + * Erik Hallnor, Steve Raasch, and Steve Reinhardt, with contributions 28 | + * from Ron Dreslinski, Dave Greene, Lisa Hsu, Kevin Lim, Ali Saidi, 29 | + * and Andrew Schultz. 30 | + * 31 | + * Permission is granted to use, copy, create derivative works and 32 | + * redistribute this software and such derivative works for any 33 | + * purpose, so long as the copyright notice above, this grant of 34 | + * permission, and the disclaimer below appear in all copies made; and 35 | + * so long as the name of The University of Michigan is not used in 36 | + * any advertising or publicity pertaining to the use or distribution 37 | + * of this software without specific, written prior authorization. 38 | + * 39 | + * THIS SOFTWARE IS PROVIDED AS IS, WITHOUT REPRESENTATION FROM THE 40 | + * UNIVERSITY OF MICHIGAN AS TO ITS FITNESS FOR ANY PURPOSE, AND 41 | + * WITHOUT WARRANTY BY THE UNIVERSITY OF MICHIGAN OF ANY KIND, EITHER 42 | + * EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION THE IMPLIED 43 | + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 44 | + * PURPOSE. THE REGENTS OF THE UNIVERSITY OF MICHIGAN SHALL NOT BE 45 | + * LIABLE FOR ANY DAMAGES, INCLUDING DIRECT, SPECIAL, INDIRECT, 46 | + * INCIDENTAL, OR CONSEQUENTIAL DAMAGES, WITH RESPECT TO ANY CLAIM 47 | + * ARISING OUT OF OR IN CONNECTION WITH THE USE OF THE SOFTWARE, EVEN 48 | + * IF IT HAS BEEN OR IS HEREAFTER ADVISED OF THE POSSIBILITY OF SUCH 49 | + * DAMAGES. 50 | + */ 51 | + 52 | +#include 53 | +#include 54 | + 55 | +/* 56 | + * Structure size information 57 | + */ 58 | +const int thread_info_size = sizeof(struct thread_info); 59 | +// const int thread_info_task = offsetof(struct thread_info, task); 60 | + 61 | +const int task_struct_size = sizeof(struct task_struct); 62 | +const int task_struct_pid = offsetof(struct task_struct, pid); 63 | +const int task_struct_start_time = offsetof(struct task_struct, start_time); 64 | +const int task_struct_tgid = offsetof(struct task_struct, tgid); 65 | +const int task_struct_comm = offsetof(struct task_struct, comm); 66 | +const int task_struct_comm_size = TASK_COMM_LEN; 67 | +const int task_struct_mm = offsetof(struct task_struct, mm); 68 | \ No newline at end of file 69 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | _site 2 | .sass-cache 3 | .jekyll-metadata 4 | .DS_Store 5 | .jekyll-cache/ -------------------------------------------------------------------------------- /docs/Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | # Define theme 4 | gem "just-the-docs" 5 | 6 | gem "webrick", "~> 1.7" 7 | 8 | # If you want to use GitHub Pages, remove the "gem "jekyll"" above and 9 | # uncomment the line below. To upgrade, run `bundle update github-pages`. 10 | gem "github-pages", "~> 227", group: :jekyll_plugins 11 | # If you have any plugins, put them here! 12 | group :jekyll_plugins do 13 | gem "jekyll-feed", "~> 0.15" 14 | end 15 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # vSwarm-u Website 2 | 3 | ## Theme template 4 | 5 | For the documentation we use the [Just-the-docs](https://just-the-docs.github.io/just-the-docs/) theme. Refer to it for more details how to customize the documentation. 6 | 7 | ## Serve the website locally 8 | 9 | In order to create new features on the website its usually more convenient to do this locally. I.e. by using the "livereload" feature one can see changes immediately. For more details see [here](https://jekyllrb.com/docs/installation/). 10 | 11 | ### Install dependencies 12 | ``` 13 | sudo apt-get install ruby-full build-essential zlib1g-dev 14 | # Install jekyll 15 | gem install jekyll bundler 16 | # Install the gems (in ./docs/ folder) 17 | bundle install 18 | ``` 19 | 20 | ### Serving 21 | After setting up jekyll locally we are now ready to serve the website by the following command in the `docs/ directory of this repo. 22 | ``` 23 | bundle exec jekyll serve 24 | ``` 25 | Jekyll will build the website and serve it on your localhost. For the address check the command output. Usually this is (http://127.0.0.1:4000/vSwarm-u/). Now make the changes you want and then re-execute the command above. Alternatively add `--livereload` to the command. Now whenever you save a file the rebuild and serving update will happen automatically. 26 | 27 | 28 | ## Generating Documentation (Markdown) files from Jupyter Notebook 29 | We can generate documentation from jupyter notebooks if we already have a nice one. 30 | To do this `jupyter` and the `nbconvert` module need to be installed: 31 | ```bash 32 | sudo apt install jupyter-core 33 | python -m pip install nbconvert -U 34 | ``` 35 | Export a notebook with: 36 | ``` 37 | jupyter nbconvert --to markdown --output ./docs/ 38 | ``` 39 | Afterwards go to output file and integrate it with the remaining documentation. 40 | -------------------------------------------------------------------------------- /docs/_config.yml: -------------------------------------------------------------------------------- 1 | title: "vSwarm-μ" 2 | remote_theme: just-the-docs/just-the-docs 3 | 4 | domain: vhive-serverless.github.io # if you want to force HTTPS, specify the domain without the http at the start, e.g. example.com 5 | url: "https://vhive-serverless.github.io" # the base hostname and protocol for your site, e.g. http://example.com 6 | baseurl: "/vSwarm-u" 7 | 8 | # Aux links for the upper right navigation 9 | aux_links: 10 | "GitHub repo": 11 | - "https://github.com/vhive-serverless/vSwarm-u" 12 | # Makes Aux links open in a new tab. Default is false 13 | aux_links_new_tab: false 14 | 15 | 16 | 17 | permalink: pretty 18 | exclude: ["node_modules/", "*.gemspec", "*.gem", "Gemfile", "Gemfile.lock", "package.json", "package-lock.json", "script/", "LICENSE.txt", "lib/", "bin/", "README.md", "Rakefile" 19 | , "docs/tests/" 20 | ] 21 | 22 | 23 | # Set a path/url to a logo that will be displayed instead of the title 24 | #logo: "/figures/vSwarm-u-small-04.png" 25 | 26 | # Enable or disable the site search 27 | # Supports true (default) or false 28 | search_enabled: true 29 | search: 30 | # Split pages into sections that can be searched individually 31 | # Supports 1 - 6, default: 2 32 | heading_level: 2 33 | # Maximum amount of previews per search result 34 | # Default: 3 35 | previews: 2 36 | # Maximum amount of words to display before a matched word in the preview 37 | # Default: 5 38 | preview_words_before: 3 39 | # Maximum amount of words to display after a matched word in the preview 40 | # Default: 10 41 | preview_words_after: 3 42 | # Set the search token separator 43 | # Default: /[\s\-/]+/ 44 | # Example: enable support for hyphenated search words 45 | tokenizer_separator: /[\s/]+/ 46 | # Display the relative url in search results 47 | # Supports true (default) or false 48 | rel_url: true 49 | # Enable or disable the search button that appears in the bottom right corner of every page 50 | # Supports true or false (default) 51 | button: false 52 | 53 | # Enable or disable heading anchors 54 | heading_anchors: true 55 | 56 | 57 | # Sort order for navigation links 58 | # nav_sort: case_insensitive # default, equivalent to nil 59 | nav_sort: case_sensitive # Capital letters sorted before lowercase 60 | 61 | # Footer content 62 | # appears at the bottom of every page's main content 63 | 64 | # Back to top link 65 | back_to_top: true 66 | back_to_top_text: "Back to top" 67 | 68 | footer_content: "Copyright © 2022 EASE-lab. Distributed by an MIT license." 69 | 70 | # Footer last edited timestamp 71 | last_edit_timestamp: true # show or hide edit time - page must have `last_modified_date` defined in the frontmatter 72 | last_edit_time_format: "%b %e %Y at %I:%M %p" # uses ruby's time format: https://ruby-doc.org/stdlib-2.7.0/libdoc/time/rdoc/Time.html 73 | 74 | 75 | 76 | # Footer "Edit this page on GitHub" link text 77 | gh_edit_link: true # show or hide edit this page link 78 | gh_edit_link_text: "Edit this page on GitHub" 79 | gh_edit_repository: "https://github.com/vhive-serverless/vSwarm-u" # the github URL for your repo 80 | gh_edit_branch: "main" # the branch that your docs is served from 81 | gh_edit_source: docs # the source that your files originate from 82 | gh_edit_view_mode: "tree" # "tree" or "edit" if you want the user to jump into the editor immediately 83 | 84 | # Google Analytics Tracking (optional) 85 | # e.g, UA-1234567-89 86 | # ga_tracking: UA-2709176-10 87 | # ga_tracking_anonymize_ip: true # Use GDPR compliant Google Analytics settings (true/nil by default) 88 | 89 | plugins: 90 | - jekyll-seo-tag 91 | 92 | # Color scheme currently only supports "dark", "light"/nil (default), or a custom scheme that you define 93 | color_scheme: nil 94 | 95 | kramdown: 96 | syntax_highlighter_opts: 97 | block: 98 | line_numbers: false 99 | 100 | compress_html: 101 | clippings: all 102 | comments: all 103 | endings: all 104 | startings: [] 105 | blanklines: false 106 | profile: false 107 | # ignore: 108 | # envs: all -------------------------------------------------------------------------------- /docs/analysis/analysis.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Analysis 4 | nav_order: 15 5 | has_children: true 6 | permalink: /docs/analysis 7 | --- 8 | 9 | # Analyze simulation results 10 | 11 | We provide some helper functionality to analyze the raw output of the simulations. 12 | {: .fs-6 .fw-300 } 13 | 14 | -------------------------------------------------------------------------------- /docs/analysis/basic_analysis.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Analyze gem5 Results 4 | parent: Analysis 5 | nav_order: 2 6 | --- 7 | 8 | # Analyze the Result of gem5 Experiments 9 | 10 | In order to make it more convenient to process results from gem5 experiments the `analysis` folder contain some a python helper library which is based on [here](https://github.com/mcai/gem5_utils). 11 | 12 | In the following we describe on a high level how to use this library. 13 | > *Note that there is a analogue `jupyter notebook` to this documentation in `analysis/analyze_results.ipynb`* 14 | This notebook aims to get you started with processing the results from the gem5 simulations. 15 | 16 | ## Parse raw Results 17 | First define the results directory and the functions you have benchmarked and for which you want to analyze the results 18 | ```python 19 | import gem5_utils as gu 20 | import pandas as pd 21 | # Define results path and benchmarks 22 | results_path = "../wkdir/results/" 23 | benchmarks = [ 24 | "fibonacci-go", 25 | "fibonacci-nodejs", 26 | "fibonacci-python", 27 | ] 28 | ## Read raw data 29 | raw_data = [] 30 | for benchmark in benchmarks: 31 | dir = "{}/{}".format(results_path, benchmark) 32 | raw_data.append(gu.parse_result(dir,benchmark=benchmark)) 33 | 34 | ``` 35 | 36 | ### Structure of the raw data: 37 | The `parse_results` function in the `gem5utils` module will read the three folders of the fibonacci functions, parse the `stats.txt` and `config.json` files and will put everything together in a list of `ExperimentResults`. 38 | 39 | ```python 40 | class ExperimentResults: 41 | def __init__(self, configs, stats, props): 42 | self.configs = configs 43 | self.stats = stats 44 | self.props = props 45 | ``` 46 | The each `ExperimentResults` has three subclasses. ExperimentConfigs containers the configuration parsed from the config.json, the ExperimentStats from the `stats.txt` file and an additional properties `dict` for maintaining useful information's like the benchmarks name or other things you can specify. You can populate properties while parsing with the `**props` argument. 47 | 48 | For more details refer to the `analysis/gem5utils.py` file. 49 | 50 | ### Extract statistics from raw data 51 | The raw data are nice to have but not very useful. For further processing we want to extract some interesting once. 52 | To extract statistics we need to define a list in the form of `(,,)`. Here an example where we want to extract instructions cycles and IPC/CPI from the raw. 53 | 54 | 55 | ```python 56 | dump_number = 0 57 | stats_per_core_model = { 58 | "simple" : [ 59 | ('Benchmark', lambda r: r.props['benchmark'], str), 60 | ('Cycles', lambda r: r.stats[dump_number]['system.cpu1.numCycles'], int), 61 | ('Instructions', lambda r: r.stats[dump_number]['system.cpu1.exec_context.thread_0.numInsts'], int), 62 | ('IPC', lambda r: r.stats[dump_number]['system.cpu1.ipc'], float), 63 | ('CPI', lambda r: r.stats[dump_number]['system.cpu1.cpi'], float), 64 | ], 65 | "detailed" : [ 66 | ('Benchmark', lambda r: r.props['benchmark'], str), 67 | ('Cycles', lambda r: r.stats[dump_number]['system.cpu1.numCycles'], int), 68 | ('Instructions', lambda r: r.stats[dump_number]['system.cpu1.numInsts'], int), 69 | ('IPC', lambda r: r.stats[dump_number]['system.cpu1.ipc'], float), 70 | ('CPI', lambda r: r.stats[dump_number]['system.cpu1.cpi'], float), 71 | ] 72 | } 73 | 74 | stats=stats_per_core_model["detailed"] 75 | ``` 76 | 77 | > Note that the stats are different depending on the core model that was used for simulation. 78 | > - `simple` is for the AtomicSimpleCPU and the TimingSimpleCPU in gem5 79 | > - `detailed` for the detailed OoO core model. 80 | 81 | >Note further that the stats.txt contains a list of dumps. The `dump_number` specify which dump to use. 82 | 83 | Once the statistics are defined they can be extracted from the raw data using the `to_pandas(,)` function. 84 | 85 | 86 | ```python 87 | # Extract the statistics from the raw data and convert the data types 88 | df = gu.to_pandas(raw_data, stats) 89 | # If needed write the data as csv file 90 | df.to_csv(results_path + '/results.csv') 91 | df 92 | ``` 93 | 94 | |Benchmark | Cycles | Instructions | IPC | CPI| 95 | |:--|--:|--:|--:|--:| 96 | |fibonacci-go | 5039711 |4903235 |0.352214|2.839184| 97 | |fibonacci-nodejs |6510927|5996027|0.253948|3.937809| 98 | |fibonacci-python |10533559|12510532|0.330886|3.022189| 99 | 100 | 101 | ### Plot results 102 | Pandas data frames have nice abilities to [plot](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.plot.html) data. Plotting graphs is out of scope of this documentation and we refer to other documentations. 103 | 104 | Here just an example to plot the stats as bar plot 105 | ```python 106 | df.plot.bar(x='Benchmark',rot=0,subplots=True, figsize=(8,8), grid=True) 107 | ``` 108 | ![png](./../figures/basic_analysis.png) 109 | 110 | -------------------------------------------------------------------------------- /docs/figures/basic_analysis.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vhive-serverless/vSwarm-u/b262a4bde890380d4f6e7b0e8ea8667b028051ea/docs/figures/basic_analysis.png -------------------------------------------------------------------------------- /docs/figures/conventional-system.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vhive-serverless/vSwarm-u/b262a4bde890380d4f6e7b0e8ea8667b028051ea/docs/figures/conventional-system.jpg -------------------------------------------------------------------------------- /docs/figures/conventional-workloads.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vhive-serverless/vSwarm-u/b262a4bde890380d4f6e7b0e8ea8667b028051ea/docs/figures/conventional-workloads.jpg -------------------------------------------------------------------------------- /docs/figures/serverless-stack.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vhive-serverless/vSwarm-u/b262a4bde890380d4f6e7b0e8ea8667b028051ea/docs/figures/serverless-stack.jpg -------------------------------------------------------------------------------- /docs/figures/serverless-system.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vhive-serverless/vSwarm-u/b262a4bde890380d4f6e7b0e8ea8667b028051ea/docs/figures/serverless-system.jpg -------------------------------------------------------------------------------- /docs/figures/serverless-workloads.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vhive-serverless/vSwarm-u/b262a4bde890380d4f6e7b0e8ea8667b028051ea/docs/figures/serverless-workloads.jpg -------------------------------------------------------------------------------- /docs/figures/simple_system.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vhive-serverless/vSwarm-u/b262a4bde890380d4f6e7b0e8ea8667b028051ea/docs/figures/simple_system.jpg -------------------------------------------------------------------------------- /docs/figures/skylake-system.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vhive-serverless/vSwarm-u/b262a4bde890380d4f6e7b0e8ea8667b028051ea/docs/figures/skylake-system.jpg -------------------------------------------------------------------------------- /docs/figures/trigger-points.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vhive-serverless/vSwarm-u/b262a4bde890380d4f6e7b0e8ea8667b028051ea/docs/figures/trigger-points.jpg -------------------------------------------------------------------------------- /docs/figures/two-core.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vhive-serverless/vSwarm-u/b262a4bde890380d4f6e7b0e8ea8667b028051ea/docs/figures/two-core.jpg -------------------------------------------------------------------------------- /docs/figures/two-machine.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vhive-serverless/vSwarm-u/b262a4bde890380d4f6e7b0e8ea8667b028051ea/docs/figures/two-machine.jpg -------------------------------------------------------------------------------- /docs/figures/vSwarm-u-hdr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vhive-serverless/vSwarm-u/b262a4bde890380d4f6e7b0e8ea8667b028051ea/docs/figures/vSwarm-u-hdr.png -------------------------------------------------------------------------------- /docs/figures/vswarm-u-design.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vhive-serverless/vSwarm-u/b262a4bde890380d4f6e7b0e8ea8667b028051ea/docs/figures/vswarm-u-design.png -------------------------------------------------------------------------------- /docs/methodology.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Events 4 | nav_order: 50 5 | has_children: false 6 | --- 7 | 8 | # What is vSwarm-μ? 9 | 10 | [Presentation](https://youtu.be/4q69hhO64Cc?t=6363) of vSwarm-μ at the [ISCA 2022: GEM5 USERS’ WORKSHOP](https://www.gem5.org/events/isca-2022), New York City, New York, USA 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /docs/misc/resources.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Miscellaneous 4 | parent: main 5 | nav_order: 1 6 | --- 7 | 8 | 9 | #### Build disk image with self-hosted runner 10 | 11 | 1. Start two machines in cloudlab, one based on arm (m400) and one x86 (x170) machine. Use `vSwarm-u-focal` profiles. 12 | 2. From the client machine start the self hosted runners using the `create-build-runner.yaml` in the `runner` folder. 13 | 14 | The `runner` folder contains two scripts to start a test and a build runner. It requires ansible to be installed. 15 | 16 | ``` 17 | cd `runner` 18 | GH_ACCESS_TOKEN= ./easy-create.sh 2 [identity file] 19 | ``` 20 | 21 | 3. Once the runners are registered in Github trigger the workflows: 22 | - __create-build-runner.yaml__ and __create_base_disk_self_hosted_arm.yml__. 23 | 24 | 25 | 26 | #### Make release 27 | 1. Update the `VERSION` in `resources/Makefile` and commit. 28 | 2. Make sure all artifact workflows completed successfully. (kernel, disk-image and test-client) 29 | 3. Release with: 30 | 31 | ``` 32 | make -f resources/Makefile release 33 | ``` 34 | The current commit will be tagged and pushed. This will trigger the __release.yaml__ workflow which will gather all artifacts and add them to a newly created release. 35 | 36 | > You can also trigger the release workflow manually. Note that this will automatically create a tag like: `refs/tags/refs/heads/xxxxxx` with the branch name. If you now push changes it might fail. For making further changes you need to delete the remote tag: like `git push origin :refs/tags/refs/heads/release-v2` 37 | https://devconnected.com/how-to-delete-local-and-remote-tags-on-git/ -------------------------------------------------------------------------------- /docs/quick_start.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Quick Start 4 | nav_order: 2 5 | has_children: false 6 | --- 7 | 8 | # vSwarm-μ: Quickstart with Serverless and gem5 9 | 10 | This guide describes how to quickly setup your machine and run the first cycle accurate simulations of serverless functions using gem5. 11 | 12 | ## Setup Machine 13 | 14 | Before experimenting with gem5 and serverless functions several things need to be installed downloaded. 15 | For a quick start use the `setup/setup_host.sh` script to install everything. 16 | Brew yourself a coffee ☕️😉 in the meantime. It can take its time to to set everything up for you. Four your notice the script will: 17 | 18 | 1. **Install all dependencies** *(~5 min)* 19 | 20 | 2. **Download and build gem5** *(~15 min - 70 min)* 21 | 22 | *Note the compile time can vary a lot depending on your machine. The stated compile times are with a 10 core, 20 hyper threads machine using 64GiB memory and a two core machine using 7GB memory respectively.* 23 | 24 | 3. **Download latest release artifacts** *(~3 min)* 25 | 26 | With vSwarm-&mu, we distribute a compiled kernel, client, and ready-to-use and tested disk image. You can find the latest release of those resources on the [release page](https://github.com/vhive-serverless/vSwarm-u/releases) of the repo. Refer to the [resources](./setup/resources.md#download-resources-artifacts) section in the documentation for more details how to download and use them. 27 | 28 | 29 | ## Perform simulations 30 | Once the setup has been completed, you are ready to perform the first simulations. 31 | Here we just show the basic steps to be done. Please refer to [simulation](./simulation/basics.md#first-simulation-with-gem5-and-serverless) section for more details and customization. 32 | 33 | 34 | #### 1. Setup working directory 35 | 36 | The first step is to create and setup a initial working directory. Use the `Makefile` in `simulation/` with: 37 | ```bash 38 | make -f simulation/Makefile build-wkdir 39 | ``` 40 | to create the directory `wkdir/` and copy all necessary things into this directory. 41 | 42 | #### 2. Install functions on disk 43 | 44 | The disk image we provide does not contain any function image so we need to pull an image from a registry. Use the `install_functions` and `install_check` recipe to pull the images and verify the that the installation was successful. By default we install the three functions `fibonacci-go`, `fibonacci-nodejs`, `fibonacci-python`. Refer to [here](./simulation/basics.md) to find out how to install others. 45 | ```bash 46 | ## Pull, and test function containers on empty disk image 47 | make -f simulation/Makefile install_functions 48 | ## Verify installation 49 | make -f simulation/Makefile install_check 50 | ``` 51 | The installation will happen using the qemu simulator and will take a few minutes to complete. *(~ 2min)* 52 | 53 | #### 3. Boot System with the Simulator (kvm) 54 | 55 | Now everything is setup to switch to the simulator. At this point the "machine" we want to simulate is in power off state and we need to boot. 56 | For booting and starting the container the kvm core is used. Once booted and the function is started a checkpoint is taken. 57 | 58 | For convenience we generated a script with all the parameters set for gem5 in you working directory. Use it to simulate one of the functions. 59 | ```bash 60 | cd wkdir 61 | ./setup_function.sh 62 | ``` 63 | Use for example `fibonacci-go` and `results/` as parameters. 64 | Once started you can attach to the simulator with the [m5 terminal](https://www.gem5.org/documentation/general_docs/fullsystem/m5term) `$GEM5_DIR/util/term/m5term localhost 3456`. The simulator will boot linux, start the container and invoke the function and take a checkpoint. After 20 invocations the simulator will exit. Check [here](./simulation/basics.md#simulations) for more details. 65 | 66 | 67 | #### 4. Simulate function 68 | 69 | Finally the simulator is in a state to do the exciting stuff. With the checkpoint that was generated during the previous step we can start detailed simulations from a deterministic state of the our system. Your working dir contains again a script with all the parameters set to start from the checkpoint with the detailed core. 70 | ```bash 71 | cd wkdir 72 | ./sim_function.sh 73 | ``` 74 | This time the simulator will not use the kvm but an simple core to perform some warming of the caches. Then it switches to the most detailed core in gem5, the O3CPU and simulate 10 further very detailed invocations. 75 | 76 | #### 5. Analyze Results 77 | After the simulation you can find results and statistics in the `results/` folder. 78 | 79 | Yippee 😀 You are done. Your first simulation of a serverless function with gem5. Please refer to our [simulation basics](./simulation/basics.md) page to find out more how everything works. 80 | -------------------------------------------------------------------------------- /docs/setup/disk-image.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Create Basic Disk Image 4 | parent: Setup 5 | nav_order: 3 6 | --- 7 | # Disk Image 8 | 9 | As we will run linux in gem5 we need to provide it with a disk with a root filesystem installed on it. There are several ways to create disk images for gem5 all described [here](https://www.gem5.org/documentation/general_docs/fullsystem/disks). 10 | 11 | We recommend to follow the instructions in option 3) and use `qemu` to create your first disk images. Using `qemu` enables you to configure your disk image with your serverless function play around a bit and test all your setup from the software side before switching to the actual simulation. 12 | 13 | > *Note: Setting up a disk image from scratch can be tricky to get it right and also installation costs a lot of time. To get around this we distribute ready to use disk-images with every vSwarm-u release. Refer to the [download](./resources.md) section in the setup guide.* 14 | 15 | 16 | ## Install OS and packages 17 | 18 | We compiled a `Makefile` which will use the [autoinstall](https://ubuntu.com/server/docs/install/autoinstall) feature in ubuntu to bake a new disk image with all tools required for your experiments 19 | 20 | The installer will: 21 | 1. Create a empty disk image with 22 | 1. Install Ubuntu as operating system 23 | 2. Create users **root** (password _root_) and **ubuntu** (password _root_) with privileged permissions to all resources. 24 | 3. Setup ssh-login for both users. 25 | 4. Install the `m5` binary and gem5 init service. 26 | 5. Install the docker engine, docker-compose and the golang tools. 27 | 28 | 29 | ```bash 30 | # Install dependencies 31 | make -f setup/disk.Makefile dep_install 32 | # Download the installation medium for Ubuntu server 33 | make -f setup/disk.Makefile download 34 | # Run the installation 35 | make -f setup/disk.Makefile install 36 | # Save the disk into $RESOURCES/base-disk-image.img 37 | make -f setup/disk.Makefile save 38 | ``` 39 | 40 | > Note: The installation will take a while therefore we recommend to make a backup of this disk image once the installation has completed. 41 | 42 | ## Test Installation 43 | 44 | 45 | 46 | 47 | ## Use the disk image 48 | Once you have baked your basic disk image you can test and play around with it using the `qemu` emulator. To start the emulator use the command: 49 | ```bash 50 | sudo qemu-system-x86_64 \ 51 | -nographic \ 52 | -cpu host -enable-kvm \ 53 | -smp \ 54 | -m \ 55 | -drive file=,format=raw \ 56 | -kernel \ 57 | -append 'console=ttyS0 root=/dev/hda2' 58 | ``` 59 | 60 | 61 | 62 | 63 | ## Gem5 binary 64 | The `m5` binary is a useful tool to execute magic instructions from the running system. After installation you can use this tool in scripts or even in the command line to for example take a snapshot or exit the simulation. Type `m5 -h` for available subcommands. More information about the `m5` binary you will find [here](https://www.gem5.org/documentation/general_docs/m5ops/). 65 | 66 | > Note: Using the `gem5.Makefile` will already build the m5 utility tool for you. 67 | 68 | ## Gem5 init service. 69 | The gem5 init service is a neat way to automatically start execution a workload as soon as linux is fully booted. This service is very general in that it uses the `m5` tool retrieve any script you can specify in your gem5-config file and execute it. Just check the `--run-script` argument in the `gem5-config/run.py` file to see how such a script is send to the simulator and the `config/gem5init` file to find out how the linux obtains and execute it. 70 | 71 | 72 | ## Useful Links 73 | - [Create disk image for gem5](http://www.lowepower.com/jason/setting-up-gem5-full-system.html) 74 | - [About the m5 binary](https://www.gem5.org/documentation/general_docs/m5ops/) 75 | - [How to build a kernel in general](https://kernelnewbies.org/KernelBuild) 76 | - [Ubuntu auto install](https://ubuntu.com/server/docs/install/autoinstall-quickstart) 77 | - [Cloud-init data sources](https://cloudinit.readthedocs.io/en/latest/topics/datasources/nocloud.html#datasource-nocloud) -------------------------------------------------------------------------------- /docs/setup/kernel.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Build Linux Kernel 4 | parent: Setup 5 | nav_order: 2 6 | --- 7 | 8 | # Build Linux Kernel 9 | 10 | Gem5 requires a binary that is executed on the simulated hardware system. In our case this is the linux kernel. To build a custom kernel configured for gem5 you can follow the setup steps [here](https://gem5.googlesource.com/public/gem5-resources/+/refs/heads/stable/src/linux-kernel/). From this site you also get kernel configs with all required modules enabled that gem5 can execute it. 11 | 12 | However, since we want to not only run bare linux but containerized workloads we need further additional modules enabled. The `configs/linux-configs/` folder contains a configuration for a linux kernel version `5.4.84` which can run container workloads in gem5. 13 | We put together a `Makefile` that will use this config to build a fresh kernel for you: 14 | 15 | ```bash 16 | # Install all build tools 17 | make -f setup/kernel.Makefile dep_install 18 | # Build the kernel 19 | make -f setup/kernel.Makefile build 20 | # Save the kernel binary `vmlinux` into $RESOURCES/vmlinux 21 | make -f setup/kernel.Makefile save 22 | # Or save the kernel binary to wherever you want 23 | OUTPUT= make -f setup/kernel.Makefile save_output 24 | ``` 25 | 26 | 27 | ## Customize linux kernel 28 | If you need to enable additional modules you may want to customize the kernel config. For this the easiest way is to start from one of the existing pre-configurations. For this overwrite the good `.config` file in the linux repo with the config you want and start the configuration process as usual with `make oldconfig`. 29 | > Note: Gem5 cannot load modules at runtime therefore all modules need to be build into the binary. 30 | 31 | ## Check kernel config for container workloads 32 | To find out if your kernel is ready to run container workloads the developers from the moby project provide a neat [script](https://github.com/moby/moby/raw/master/contrib/check-config.sh). 33 | This [blog post](https://blog.hypriot.com/post/verify-kernel-container-compatibility/) explains nicely how to use this script in detail to check your kernel config for compatibility with containers. 34 | 35 | In the linux-config folder you can find a copy of this script. Execute the following command to check your kernel config: 36 | ```bash 37 | ./configs/linux-configs/check-config.sh 38 | ``` -------------------------------------------------------------------------------- /docs/setup/resources.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Resources 4 | parent: Setup 5 | nav_order: 1 6 | --- 7 | 8 | 9 | # Required Resources 10 | 11 | In order to run serverless simulations with gem5 you will need we will need a few more ingredients apart of gem5 and vSwarm-u. First of all a compiled linux kernel configured for gem5 and containers. Secondly a disk image with a root file system and all components installed to run serverless workloads. Finally a client we will use to drive the functions during simulation. 12 | 13 | Setting up and building all those resources from scratch can be tricky to get it right and is also quite time consuming therefore we distribute together with vSwarm-u an up to date and pre-configured disk image as well as a compiled kernel and test client. For quickly ramping up your system follow the easy steps [here](#download-resources-artifacts) to download those resources. For more advanced use and customization find out all the details how we build and configure the resources yourself in related documentation ([kernel](./kernel.md), [disk-image](./disk-image.md)) 14 | 15 | 16 | ## Environmental Variables 17 | We use the `RESOURCES` environmental variable in all our scripts to define where they can find the resources: kernel, base disk image and the client. Furthermore we `GEM5_DIR` to define where gem5 will be downloaded and installed. If you want to specify your own paths set and expose this variables before running one of the scripts. 18 | ```bash 19 | export RESOURCES= 20 | echo 'export RESOURCES=${RESOURCES}' >> ${HOME}/.bashrc 21 | ``` 22 | The default paths are `resources/` and `${RESOURCES}/gem5` for `RESOURCES` and `GEM5_DIR` respectively. 23 | 24 | 25 | ## Download Resources Artifacts 26 | 27 | To download resource artifact use the `artifacts.py` script in `resources/` with: 28 | ```bash 29 | ./resources/artifacts.py 30 | ``` 31 | The script will fetch the latest kernel, disk-image and client from github. In case you want another version use the `--version` argument. 32 | Note that the size of the disk image is a few GiB. Furthermore, Github has a limit of 2GiB per asset. We compress and split the disk image. But no worries the script will do everything for you ;). Downloading merging and decompression. *Usually it took about three minutes. (2.5min for download and 30s for decompressing)* 33 | 34 | By default the resources will be stored in the `resources/` 35 | 36 | ### Released Artifacts 37 | In addition to the stable disk images Ubuntu 20.04 and Kernel 5.4.84 for x86/amd64 architecture we distribute also images for Ubuntu 22.04 as well as kernels and disk images for arm based architectures. 38 | To download another artifact then the default set the arguments `--arch ` and `--os-version ` when downloading with the `./resources/artifacts.py` script. 39 | 40 | > {: .warning } 41 | > So far only all resources work with qemu but there is NO support for gem5 simulation yet. All but the stable artifacts we therefore distribute for experimentation reasons. We try to add support in the future and welcome any help. For further information about missing support ask David: [GitHub](https://github.com/dhschall). 42 | 43 | 44 | #### Kernel 45 | 46 | | Version | Architecture | State | Qemu support | gem5 support | 47 | |:---|---|---|---|---| 48 | | v5.4.84 | x86/amd64 | stable | ✓ | ✓ | 49 | | v5.15.59 | x86/amd64 | experimental | ✓ | ✕ | 50 | | v5.4.84 | arm64 | experimental | ✓ | ✓ | 51 | | v5.15.59 | arm64 | experimental | ✓ | ✕ | 52 | 53 | #### Disk Image 54 | 55 | | Version | Architecture | State | Qemu support | gem5 support | 56 | |---|---|---|---|---| 57 | | Ubuntu 20.04 (focal) | x86/amd64 | stable | ✓ | ✓ | 58 | | Ubuntu 22.04 (jammy) | x86/amd64 | experimental | ✓ | ✕ | 59 | | Ubuntu 20.04 (focal) | arm64 | experimental | ✓ | ✓ | 60 | | Ubuntu 22.04 (jammy) | arm64 | experimental | ✓ | ✕ | 61 | 62 | #### Test-client 63 | 64 | | Architecture | State | Qemu support | gem5 support | 65 | |---|---|---|---| 66 | | x86/amd64 | stable | ✓ | ✓ | 67 | | arm64 | stable | ✓ | ✓ | 68 | 69 | 70 | 71 | 72 | ### Disk format qcow2 and raw 73 | We distribute the disk image in qemu's [`qcow2` compressed format](https://qemu.readthedocs.io/en/latest/system/images.html#disk-image-file-formats). This has the advantage that the disk image is smaller. However gem5 cannot use qcow2 but only raw. Therefore, after downloading and decompression you will need to convert the disk image before using it with gem5. The scripts does all the work from you in the back however you can use `qemu-img` to convert the disk yourself: 74 | ```bash 75 | # Convert disk image from qcow2 -> raw format 76 | qemu-img convert 77 | ``` 78 | 79 | 80 | -------------------------------------------------------------------------------- /docs/setup/setup.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Setup 4 | nav_order: 3 5 | has_children: true 6 | permalink: /docs/setup 7 | --- 8 | 9 | # Setup Infrastructure 10 | 11 | To perform simulations of serverless functions with gem5 the infrastructure need to be setup in advance. 12 | {: .fs-6 .fw-300 } -------------------------------------------------------------------------------- /docs/simulation/arm_simulation.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Simple Simulation on Arm machines 4 | parent: Simulation 5 | nav_order: 1 6 | --- 7 | 8 | # Simulation Methodology 9 | {: .no_toc } 10 | 11 |
12 | 13 | Table of contents 14 | 15 | {: .text-delta } 16 | 1. TOC 17 | {:toc} 18 |
19 | 20 | 25 | 26 | 27 | --- 28 | 29 | We also support running serverless functions on an Arm machine. 30 | > Note: Since we require kvm acceleration to boot the kernel and the function, we only support running the simulation on an host machine with an Arm CPU. 31 | 32 | ## Preliminary 33 | 34 | The setup steps are the same as the x86 simulation. The `setup/setup_host.sh` setup script should work for Arm machines as well. 35 | Also preparation of the working directory, installation of the functions on the disks are the same as for x86 machines. The only difference is that you need to use the `arm.Makefile` instead of the `Makefile` in the `simulation/` directory. 36 | 37 | In the following we will list the commands needed to setup and perform the simulation on an Arm machine. For details refer to the [x86 simulation](.simulation.md). 38 | 39 | ### Build initial working directory for Arm 40 | 41 | 42 | ```bash 43 | make -f simulation/arm.Makefile build-wkdir 44 | ``` 45 | Will create the new folder `wkdir/` and copy the necessary files to this directory.: 46 | 47 | 1. Kernel and base disk image. 48 | 2. Templates of `function.yaml` and `function.list` to define your functions. 49 | 5. A basic gem5 config script. 50 | 51 | In the following the files and their usage will be described in more detail. 52 | > Note: In case you want to don't want to use the default location you can use the `WORKING_DIR` environmental variable to change the location. 53 | 54 | 55 | ## Install Function images on disk 56 | 57 | Use the command to install all functions defined in the `functions.list/yaml` files on the disk.: 58 | ```bash 59 | make -f simulation/arm.Makefile install_functions 60 | ``` 61 | 62 | ### Verify successful installation 63 | The installation will generate a log file of the installation process. Use the recipe `install_check` to verify that everything was installed successfully. 64 | 65 | 66 | 67 | ## Simulations 68 | To perform simulations on an Arm machine you can use the `vswarm_simple_arm.py` script in the working directory. The script leverages the gem5 component library and is the same as `vswarm_simple.py` but for Arm instead of x86. 69 | 70 | To boot the kernel and the function and perform functional warming of the container - for JIT compiled functions - you can use the following command: 71 | ```bash 72 | cd wkdir 73 | ## Perform setup 74 | /build/ARM/gem5.opt vswarm_simple_arm.py --kernel kernel --disk disk.img --mode=setup --atomic-warming=50 --num-invocations=20 75 | ``` 76 | 77 | Once the checkpoint is created the simulation can be performed with: 78 | ```bash 79 | ## Perform evaluation 80 | /build/ARM/gem5.opt vswarm_simple_arm.py --kernel kernel --disk disk.img --mode=evaluation --atomic-warming=50 --num-invocations=20 81 | ``` 82 | 83 | The script was tested with ATOMIC,TIMING and O3 core which can be configured commenting the corresponding lines in the script 84 | ``` 85 | eval_core = CPUTypes. 86 | ``` -------------------------------------------------------------------------------- /docs/simulation/functions.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Function benchmarks from vSwarm 4 | parent: Simulation 5 | nav_order: 40 6 | --- 7 | 8 | # Function Benchmarks from vSwarm 9 | 10 | Currently we support and tested 21 functions from our benchmark suite vSwarm 11 | {: .fs-6 .fw-300 } 12 | 13 | 14 | 15 | ## Standalone kernels 16 | The benchmark suite contains three functions that implement the same functionality in different runtime. With this the difference in the runtimes can be explored. The runtimes golang, nodejs, python are most common used for serverless and are representatives for a compiled, JIT-compiled and interpreted language. Details about the functions can be found in [vSwarm](https://github.com/vhive-serverless/vSwarm/tree/main/benchmarks) 17 | 18 | 19 | | Benchmark | Name | Languages implemented | gem5 support | 20 | |----------------|------|------------------------|--------------| 21 | | AES | `aes-` | python, golang, nodejs | ✓ | 22 | | Authentication | `auth-` | python, golang, nodejs | ✓ | 23 | | Fibonacci | `fibonacci-` | python, golang, nodejs | ✓ | 24 | > `` stands for runtime: python, nodejs, go) 25 | {: .fs-4 .fw-300 } 26 | 27 | 28 | ## Hotel Reservation app 29 | Functions from DeathStarBenchs hotel reservation app. Description of the benchmarks can be found in [vSwarm](https://github.com/vhive-serverless/vSwarm/tree/main/benchmarks/hotel-app) 30 | 31 | | Benchmark | Dependent on | Knative infra | Tracing | Gem5 support | Runtimes | Languages implemented | 32 | |---|---|---|---|---|---|---| 33 | | Geo | Serving | database | ✓ | ✓ | docker, knative | golang | 34 | | Profile | Serving | database, memcached |✓ | ✓ | docker, knative | golang | 35 | | Rate | Serving | database, memcached | ✓ | ✓ | docker, knative | golang | 36 | | Recommendation | Serving | database | ✓ | ✓ | docker, knative | golang | 37 | | Reservation | Serving | database, memcached | ✓ | ✓ | docker, knative | golang | 38 | | User | Serving | database | ✓ | ✓ | docker, knative | golang | 39 | | Search | Serving | Geo, Profile, Rate | ✓ | ✕ | docker, knative | golang | 40 | 41 | ## Online Shop 42 | Function kernels from online shop example from google. Description of the benchmarks can be found in [vSwarm](https://github.com/vhive-serverless/vSwarm/tree/main/benchmarks/online-shop) 43 | 44 | | Benchmark | Language | gem5 support | 45 | |---|---|---| 46 | | cartservice | C# | ✕ | 47 | | productcatalogservice | Go | ✓ | 48 | | currencyservice| Node.js | ✓ | 49 | | paymentservice | Node.js |✓ | 50 | | shippingservice | Go | ✓ | 51 | | emailservice | Python | ✓ | 52 | | checkoutservice | Go | ✕ | 53 | | recommendationservice | Python | ✓ | 54 | | adservice | Java | ✕ | 55 | -------------------------------------------------------------------------------- /docs/simulation/instrumentation.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Workload Instrumentation 4 | parent: Simulation 5 | nav_order: 10 6 | --- 7 | 8 | # Workload Instrumentation 9 | {: .no_toc } 10 | 11 |
12 | 13 | Table of contents 14 | 15 | {: .text-delta } 16 | 1. TOC 17 | {:toc} 18 |
19 | 20 | --- 21 | 22 | In contrast to conventional workloads serverless function run for short amount of time. Furthermore we might want to run several invocations to compare different inputs. The challenge is that now the intervals where to do measurements need to defined more precise. 23 | 24 |
25 | 26 |
Figure 1: Overall simulation flow and several intervals to be measured.
27 |
28 | 29 | Figure 1 show the the overall simulation and different intervals that are interesting to do measurements. The following describe how different points in this flow are instrumented. 30 | 31 | ## M5 fail codes 32 | For driving the simulator `m5 fail ` codes are executed at various points during the simulation. Using `m5 fail` instead of `m5 exit` has the advantage to send also a code which can encode more information. Refer to the templates in your initial working directory. 33 | 34 | 35 | ## Client side instrumentation 36 | Using the `m5` binary is fine for rough trigger points. However, it is not very precise as some OS interaction is involved when executing a command. (Need to load the binary, allocate memory,...) 37 | 38 | In order to get around this issue and get more precise trigger points the repo includes a instrumented to client in `tools/client/`. Under to hood it takes advantage of the [`m5ops`]() magic instruction. Refer to the gem5 documentation to for more information how magic instructions work. 39 | 40 | For convenience we wrap gem5's magic instructions in a go package to be found in the m5 subfolder of the client. 41 | 42 | To run the client with instrumentation set the flag `-m5ops` when running the client. 43 | Note that m5 magic instruction take advantage of `mmap` which is only available with root permissions. This is not an issue issue when running the client on the disk image as we have root permissions there. However, it will raise a warning when running the client on your local machine. 44 | 45 | 46 | ## Server side instrumentation 47 | For the server side we instrumented the linux scheduler by hooking up a `FuncEvent` on the `__schedule` symbol. With compiling also the `thread_info` into the kernel we can get the previous and next PID for every scheduling event. 48 | This enables us to react on certain scheduling events. 49 | 50 | However, this is still work in progress and will be rolled out soon. -------------------------------------------------------------------------------- /docs/simulation/manual-install.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Install Function manually 4 | parent: Simulation 5 | nav_order: 50 6 | --- 7 | 8 | 9 | ## Install Function manually 10 | 1. Make a copy of the disk image in your working directory 11 | 12 | 2. Start the emulator with the following command: 13 | ```bash 14 | sudo qemu-system-x86_64 \ 15 | -nographic \ 16 | -cpu host -enable-kvm \ 17 | -smp 4 -m 8GB \ 18 | -device e1000,netdev=net0 \ 19 | -netdev type=user,id=net0,hostfwd=tcp:127.0.0.1:5555-:22 \ 20 | -drive format=raw,file= \ 21 | -kernel \ 22 | -append 'earlyprintk=ttyS0 console=ttyS0 lpj=7999923 root=/dev/hda2' 23 | ``` 24 | Qemu will boot linux from the disk image you specified. Once booted login as `root` (password `root`). 25 | 26 | Now you are able to pull function images onto base disk image. We will use the the `vhiveease/fibonacci-go` function image as example: 27 | ```bash 28 | # Pull your containerized function image 29 | docker pull vhiveease/fibonacci-go 30 | ``` 31 | Pull all images you want to benchmark. 32 | 33 | Once all images are installed you can shutdown qemu with `shutdown -h now`. However, to make sure that the disk image is properly installed and works from a software perspective the `test-client` we put onto the image to perform a quick test. 34 | > Note you can find the source code of the client together with your hands out material. 35 | 36 | ```bash 37 | # 1. Start your function container 38 | # -d detaches the process and we can continue in the same console. 39 | # -p must be set to export the ports 40 | docker run -d --name mycontainer -p 50051:50051 vhiveease/fibonacci-go 41 | 42 | # run the client with the port you export in docker as well as the number of invocations you want to run. 43 | # -addr is the address and port we where exporting with the docker command 44 | # -n is the number of invocations the client should perform 45 | ./client -addr localhost:50051 -n 100 46 | ``` 47 | The client should print its progress after every 10 invocations. 48 | Now the disk image is ready for the Gem5 simulator. Stop the container and shutdown qemu. 49 | ``` 50 | docker stop mycontainer && docker rm mycontainer 51 | shutdown -h now 52 | ``` 53 | > Note: Qemu breaks the line wrapping you might want to reset the console by executing `reset`. 54 | -------------------------------------------------------------------------------- /docs/simulation/simple_component.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Simple Simulation with gem5 Component Library 4 | parent: Simulation 5 | nav_order: 1 6 | --- 7 | 8 | # Simulation Methodology 9 | {: .no_toc } 10 | 11 |
12 | 13 | Table of contents 14 | 15 | {: .text-delta } 16 | 1. TOC 17 | {:toc} 18 |
19 | 20 | 25 | 26 | 27 | --- 28 | 29 | Gem5 offers a wide range of pre-build configurations, aka. standard library, for CPUs, boards, caches, etc. To simplify the configuration process we provide a configuration script that is mostly constructed out the component models provided by gem5. This script is called `vswarm_simple.py` and will be copied into your working directory. 30 | 31 | After installing the functions on the disk you can perform the *setup* step with the following command: 32 | 33 | ```bash 34 | /build//gem5.opt vswarm_simple.py --kernel kernel --disk disk.img --mode setup --atomic-warming=50 --num-invocations=20 35 | ``` 36 | 37 | This will boot the kernel, function, invoke the function for 50 times and then create a checkpoint. The simulation will exit or can be killed otherwise 38 | 39 | Once the checkpoint is created the simulation can be performed with: 40 | 41 | ```bash 42 | /build//gem5.opt vswarm_simple.py --kernel kernel --disk disk.img --mode evaluation --atomic-warming=50 --num-invocations=20 43 | ``` 44 | 45 | The script was tested with ATOMIC,TIMING and O3 core which can be configured commenting the corresponding lines in the script 46 | ``` 47 | eval_core = CPUTypes. 48 | ``` -------------------------------------------------------------------------------- /docs/simulation/simulations.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Simulation 4 | nav_order: 10 5 | has_children: true 6 | permalink: /docs/simulation 7 | --- 8 | 9 | # Simulate Serverless Functions with gem5 10 | 11 | To perform simulations of serverless functions with gem5 the infrastructure need to be setup in advance. 12 | {: .fs-6 .fw-300 } -------------------------------------------------------------------------------- /docs/simulation/systems.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Systems 4 | parent: Simulation 5 | nav_order: 4 6 | --- 7 | 8 | # Simulated Systems Models 9 | 10 | At the moment we provide two systems setups ready to use. A simple and a complex model. The complex system is base on Intel Skylake parameters and was adopted from [here](https://github.com/darchr/gem5-skylake-config) 11 | 12 | In the following the two systems and the main parameters. For all details refer to the configs in `gem5utils/systems/`. 13 | 14 | ### Simple System 15 | 16 | | | Parameters | 17 | |---|---| 18 | | | **Core:** TimingSimpleCPU
**L1-I/D:** 32KB
**LLC:** 128KB
**Memory:** 2GB 19 | 20 | ### Complex Skylake System 21 | 22 | The complex model is the Intel Skylake CPU and its parameters are taken from the reference setup in [here](https://github.com/darchr/gem5-skylake-config). 23 | 24 | | | Parameters | 25 | |---|---| 26 | | | **Core:** Intel Skylake
**L1-I/D:** 32KB
**L2: 1MB**
**LLC:** 8MB
**Memory:** 2GB, DDR4, 2400, 16x4. 27 | 28 | 29 | ## Two Machine Model 30 | For a truly isolated server-client communication we also provide a two machine setup. In order to use it build our initial working directory with `make -f simulation/Makefile build-two-machine` setup. The only difference is that it will use a different template to create the `run_sim.py` config file 31 | 32 | | | Parameters | 33 | |---|---| 34 | | | **Detailed Node:** Skylake system model

**Driving Node:** TimingSimpleCPU, 8MB LLC, 2GB SimpleMemory | 35 | 36 | In the two machine setup the client is started on the driving node and the function on the detailed node. To communicate both nodes are connected via gem5 EthernetLink model. To make it work two different workflows need to be defined one for the client and one for the server respectively. 37 | 38 | | Client workflow | Server workflow | 39 | |---|---| 40 | | 1. Booting linux
2. Spinning up the container
3. Pin the container to core 1
4. Enable network interface and assign ip | 1. Booting linux
2. Wait until server ip is reachable
3. Run the invoker to warm the function container.
5. Reset the gem5 stats
6. Invoker the function
7. Dump stats and exit | 41 | 42 | > Note: We enable the network interface not before the function is started. This allows the synchronization between server and client. 43 | -------------------------------------------------------------------------------- /docs/test/function_test.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Integration test 4 | nav_order: 20 5 | has_children: false 6 | permalink: /docs/test 7 | --- 8 | 9 | 10 | # Function Integration test 11 | {: .no_toc } 12 | 13 |
14 | 15 | Table of contents 16 | 17 | {: .text-delta } 18 | 1. TOC 19 | {:toc} 20 |
21 | 22 | --- 23 | 24 | 25 | The folder `test/` implement the functionality to test function images on their compatibility with the base image and vSwarm-μ. The folder contains tests for the qemu emulator and the gem5 simulator. The tests are fully integrated with github action workflows and aim as continuous integration tests. 26 | 27 | 28 | ## Test setup 29 | To order to perform tests the host system need to be setup correctly with all requirements. Refer to the [setup](./../setup/setup.md) to get help with setting things up. 30 | 31 | The environmental variable `RESOURCES` is used to let the tests find all all dependent resources like base-image, kernel and gem5. required base files. After building the resources place them into the default `resources` folder or setting the path of this variable accordingly. 32 | 33 | Furthermore. for emulator tests [qemu](https://www.qemu.org/docs/master/) need to be installed as well as the python uploadserver. 34 | > Use `make -f test/Makefile dep_check_qemu` to check if all requirements are fulfilled. 35 | 36 | ## Test flow 37 | The test is composed of the following steps 38 | 1. Build test setup 39 | The first step is to build a new working directory where we perform the test. A new disk image must be created from the `base-disk-image.img` from the resource folder. 40 | Furthermore, the run script template for the test run must be adjusted to 41 | the **function under test (FUT)**. For this rename or copy `run_test.template.sh` to your working directory `run.sh` and replace `<__IMAGE_NAME__>` with the image name of the FUT. 42 | 43 | Build the test setup with the `IMAGE_NAME= make build` command. 44 | 45 | 2. Test run 46 | The test run is fully automated will basically perform the following steps: 47 | 1. Pull the FUT image. 48 | 1. Start the FUT container. 49 | 1. Run a test client locally that issues request to the FUT container. 50 | 1. Stop the FUT container 51 | 1. Check if all previous commands have been executed properly 52 | 1. Upload the log file. 53 | 1. Shutdown the machine. 54 | 55 | 3. Result check 56 | To let the host system know if the test has completed successfully qemu uploads back the log file as `results.log`. The final step is now to parse this file for failures. 57 | 58 | 59 | ## Workflow automation 60 | 61 | The underlying concept of the automation is the gem5.service. After boot this service will try to fetch a run script from a predefined resource depending on the CPU model. 62 | 63 | ### Gem5 simulator 64 | In case `gem5.service` encounters the `M5 Simulator` as cpu model it will use the magic instruction `m5 readfile`. This instruction is the counter part of the `.readfile` parameter in the `m5.System` object. 65 | 66 | Example how to set the script in the gem5 config 67 | ```python 68 | system = System() 69 | system.readfile = <"path/to/your/run/script"> 70 | ``` 71 | 72 | 73 | ### Qemu Emulator 74 | When the gem5.service encounters another cpu type it knows that not a simulation but emulation is running. In here we do not have the magic instruction functionality. Instead we use the network interface and qemu's gateway connection to the host system. For accessing the host network from within qemu via the gateway the network address `10.0.2.2` or the variable `_gateway` is used. See [networking](https://wiki.qemu.org/Documentation/Networking) for more details. 75 | 76 | Using the gateway the `gem5.service` will try to download the run script (`run.sh`) from a http server at port `3003`. 77 | 78 | > The exact address is: `http://_gateway:3003/run.sh` 79 | 80 | In case such file is found it will be automatically executed by the service with `root` privileges. If none such server or file is found the service terminate. 81 | 82 | With this entry point (`run.sh`) any arbitrary complex workflow can be implemented and automatically executed by the emulator. Only this script need to be served by the host system at port 3003. One easiest way to do this is to use the [python http server](https://docs.python.org/3/library/http.server.html). 83 | I.e. `python -m http.server 3003` will serve files from the current directory at port 3003. Use `-d` to defined another directory to be served. 84 | The folder can contain more files but the run script that can be downloaded if necessary with `curl` or `wget`. 85 | 86 | In order to get feedback from the emulator one can use python's [upload server](https://pypi.org/project/uploadserver/) to not just download but also upload files. 87 | ```python 88 | python3 -m uploadserver -d tmp/ 3003 89 | ``` 90 | Will serve the directory `tmp` at port 3003 for GET and PUT request. The emulator can now not just download more files that remain in this folder. I.e. `curl "http://10.0.2.2:3003/config.json" -f -o config.json` will download config.json from the host and save it as config.json at the guest. But also files can be uploaded from the guest to this tmp folder in the host. I.e. `curl "http://10.0.2.2:3003/upload" -F 'files=@results.log'` will upload results.log. 91 | 92 | 93 | 94 | -------------------------------------------------------------------------------- /gem5utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vhive-serverless/vSwarm-u/b262a4bde890380d4f6e7b0e8ea8667b028051ea/gem5utils/__init__.py -------------------------------------------------------------------------------- /gem5utils/scripts/update_upstream.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # MIT License 4 | # 5 | # Copyright (c) 2022 David Schall and EASE lab 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | # Install dependencies 25 | 26 | set -e -x 27 | 28 | pushd $GEM5_DIR 29 | 30 | ## Add upstream 31 | git remote add upstream https://gem5.googlesource.com/public/gem5 32 | ## Update remote 33 | git fetch upstream 34 | 35 | ## Rebase main 36 | git checkout origin/main 37 | ## 38 | git rebase upstream/master 39 | 40 | ## Push the updates 41 | git push -------------------------------------------------------------------------------- /gem5utils/systems/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vhive-serverless/vSwarm-u/b262a4bde890380d4f6e7b0e8ea8667b028051ea/gem5utils/systems/__init__.py -------------------------------------------------------------------------------- /gem5utils/systems/drive/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vhive-serverless/vSwarm-u/b262a4bde890380d4f6e7b0e8ea8667b028051ea/gem5utils/systems/drive/__init__.py -------------------------------------------------------------------------------- /gem5utils/systems/simple/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vhive-serverless/vSwarm-u/b262a4bde890380d4f6e7b0e8ea8667b028051ea/gem5utils/systems/simple/__init__.py -------------------------------------------------------------------------------- /gem5utils/systems/simple/caches.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2015 Jason Power 3 | # Copyright (c) 2022 EASE lab, University of Edinburgh 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are 8 | # met: redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer; 10 | # redistributions in binary form must reproduce the above copyright 11 | # notice, this list of conditions and the following disclaimer in the 12 | # documentation and/or other materials provided with the distribution; 13 | # neither the name of the copyright holders nor the names of its 14 | # contributors may be used to endorse or promote products derived from 15 | # this software without specific prior written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | # 29 | # Authors: Jason Power, David Schall 30 | 31 | """ Simple caches with default values 32 | 33 | This file contains L1 I/D and L2 caches to be used in the simple 34 | gem5 configuration script. 35 | 36 | """ 37 | 38 | from m5.objects import Cache 39 | 40 | # Some specific options for caches 41 | # For all options see src/mem/cache/Cache.py 42 | 43 | class L1Cache(Cache): 44 | """Simple L1 Cache with default values""" 45 | 46 | # Default parameters for both L1 I and D caches 47 | assoc = 8 48 | tag_latency = 1 49 | data_latency = 1 50 | response_latency = 2 51 | mshrs = 4 52 | tgts_per_mshr = 16 53 | write_buffers = 32 54 | 55 | def connectCPU(self, cpu): 56 | """Connect this cache's port to a CPU-side port 57 | This must be defined in a subclass""" 58 | raise NotImplementedError 59 | 60 | def connectBus(self, bus): 61 | """Connect this cache to a memory-side bus""" 62 | self.mem_side = bus.cpu_side_ports 63 | 64 | class L1ICache(L1Cache): 65 | """Simple L1 instruction cache with default values""" 66 | 67 | # Set the default size 68 | size = '16kB' 69 | is_read_only=True 70 | 71 | def connectCPU(self, cpu): 72 | """Connect this cache's port to a CPU icache port""" 73 | self.cpu_side = cpu.icache_port 74 | 75 | class L1DCache(L1Cache): 76 | """Simple L1 data cache with default values""" 77 | 78 | # Set the default size 79 | size = '16kB' 80 | 81 | def connectCPU(self, cpu): 82 | """Connect this cache's port to a CPU dcache port""" 83 | self.cpu_side = cpu.dcache_port 84 | 85 | class LLCCache(Cache): 86 | """Simple LLC Cache with default values""" 87 | 88 | # Default parameters 89 | size = '128kB' 90 | assoc = 8 91 | tag_latency = 12 92 | data_latency = 12 93 | response_latency = 12 94 | mshrs = 20 95 | tgts_per_mshr = 12 96 | write_buffers = 56 97 | 98 | def connectCPUSideBus(self, bus): 99 | """"Connect this cache to a cpu-side bus""" 100 | self.cpu_side = bus.mem_side_ports 101 | 102 | def connectMemSideBus(self, bus): 103 | """"Connect this cache to a memory-side bus""" 104 | self.mem_side = bus.cpu_side_ports 105 | -------------------------------------------------------------------------------- /gem5utils/systems/skylake/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vhive-serverless/vSwarm-u/b262a4bde890380d4f6e7b0e8ea8667b028051ea/gem5utils/systems/skylake/__init__.py -------------------------------------------------------------------------------- /resources/release.json: -------------------------------------------------------------------------------- 1 | { 2 | "tag_name": "v0.0.6", 3 | "id": 68788024, 4 | "kernel": "https://github.com/vhive-serverless/vSwarm-u/releases/download/v0.0.6/vmlinux-amd64", 5 | "disk-image": [ 6 | "https://github.com/vhive-serverless/vSwarm-u/releases/download/v0.0.6/test-disk-image-amd64.tar.gz.aa", 7 | "https://github.com/vhive-serverless/vSwarm-u/releases/download/v0.0.6/test-disk-image-amd64.tar.gz.ab", 8 | "https://github.com/vhive-serverless/vSwarm-u/releases/download/v0.0.6/test-disk-image-amd64.tar.gz.ac", 9 | "https://github.com/vhive-serverless/vSwarm-u/releases/download/v0.0.6/test-disk-image-amd64.tar.gz.ad", 10 | "https://github.com/vhive-serverless/vSwarm-u/releases/download/v0.0.6/test-disk-image-amd64.tar.gz.ae" 11 | ], 12 | "client": "https://github.com/vhive-serverless/vSwarm-u/releases/download/v0.0.6/test-client-amd64" 13 | } -------------------------------------------------------------------------------- /runner/Dockerfile: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2022 David Schall and EASE lab 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | 24 | 25 | # DIR=/tmp/_work-${REPO} 26 | # mkdir -p $DIR 27 | 28 | 29 | # # hadolint ignore=DL3007 30 | # FROM myoung34/github-runner-base:latest 31 | # LABEL maintainer="myoung34@my.apsu.edu" 32 | 33 | # ENV AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache 34 | # RUN mkdir -p /opt/hostedtoolcache 35 | 36 | # ARG GH_RUNNER_VERSION="2.289.1" 37 | # ARG TARGETPLATFORM 38 | 39 | # SHELL ["/bin/bash", "-o", "pipefail", "-c"] 40 | 41 | # WORKDIR /actions-runner 42 | # COPY install_actions.sh /actions-runner 43 | 44 | # # TODO: remove this terrible sed once 45 | # # https://github.com/actions/runner/pull/1585 is merged or similar 46 | # RUN chmod +x /actions-runner/install_actions.sh \ 47 | # && sed -i.bak 's/.\/bin\/installdependencies.sh/wget https:\/\/raw.githubusercontent.com\/myoung34\/runner\/main\/src\/Misc\/layoutbin\/installdependencies.sh -O .\/bin\/installdependencies.sh; bash .\/bin\/installdependencies.sh/g' /actions-runner/install_actions.sh \ 48 | # && /actions-runner/install_actions.sh ${GH_RUNNER_VERSION} ${TARGETPLATFORM} \ 49 | # && rm /actions-runner/install_actions.sh 50 | 51 | # COPY token.sh entrypoint.sh ephemeral-runner.sh / 52 | # RUN chmod +x /token.sh /entrypoint.sh /ephemeral-runner.sh 53 | 54 | # ENTRYPOINT ["/entrypoint.sh"] 55 | # CMD ["./bin/Runner.Listener", "run", "--startuptype", "service"] 56 | 57 | 58 | #---------- Build runner -----------# 59 | ARG OS=ubuntu-focal 60 | FROM myoung34/github-runner:$OS as gem5-build-runner 61 | 62 | ARG ORG=vhive-serverless \ 63 | REPO=vSwarm-u \ 64 | OS_LABEL=ubuntu-20.04 65 | 66 | ENV REPO_URL=https://github.com/$ORG/$REPO \ 67 | RUNNER_NAME="gem5-build-runner" \ 68 | ACCESS_TOKEN="${GH_ACCESS_TOKEN}" \ 69 | RUNNER_WORKDIR=_work-$REPO \ 70 | RUNNER_GROUP="default" \ 71 | LABELS="gem5-build" \ 72 | GIT_SSL_NO_VERIFY=1 73 | 74 | # Install dependencies 75 | RUN apt-get update \ 76 | && apt-get install -y \ 77 | build-essential git m4 scons zlib1g zlib1g-dev \ 78 | libprotobuf-dev protobuf-compiler libprotoc-dev libgoogle-perftools-dev \ 79 | python3-dev python3-six python-is-python3 libboost-all-dev pkg-config \ 80 | qemu-kvm bridge-utils \ 81 | && apt-get install -y \ 82 | python3-pip \ 83 | curl wget \ 84 | lsof \ 85 | qemu-kvm bridge-utils \ 86 | && rm -rf /var/lib/apt/lists/* 87 | 88 | ENTRYPOINT ["/entrypoint.sh"] 89 | CMD ["./bin/Runner.Listener", "run", "--startuptype", "service"] 90 | 91 | 92 | 93 | 94 | #---------- Qemu test runner -----------# 95 | # First stage (Builder): 96 | ARG OS=ubuntu-focal 97 | FROM myoung34/github-runner:$OS as qemu-test-runner 98 | 99 | ARG ORG=vhive-serverless \ 100 | REPO=vSwarm-u \ 101 | OS_LABEL=ubuntu-20.04 102 | 103 | ENV REPO_URL=https://github.com/$ORG/$REPO \ 104 | RUNNER_NAME="qemu-test-runner" \ 105 | ACCESS_TOKEN="${GH_ACCESS_TOKEN}" \ 106 | RUNNER_WORKDIR=_work-$REPO \ 107 | RUNNER_GROUP="default" \ 108 | LABELS="qemu-test" \ 109 | RESOURCES=/tmp/resources 110 | 111 | # Install dependencies 112 | RUN apt-get update \ 113 | && apt-get install -y \ 114 | python3-pip \ 115 | curl \ 116 | lsof \ 117 | qemu-kvm bridge-utils \ 118 | && rm -rf /var/lib/apt/lists/* 119 | 120 | 121 | # Install the upload server 122 | RUN python3 -m pip install --user uploadserver 123 | 124 | 125 | ENTRYPOINT ["/entrypoint.sh"] 126 | CMD ["./bin/Runner.Listener", "run", "--startuptype", "service"] 127 | 128 | 129 | 130 | 131 | 132 | #---------- gem5 test runner -----------# 133 | ARG OS=ubuntu-focal 134 | FROM myoung34/github-runner:$OS as gem5-test-runner 135 | 136 | ARG ORG=vhive-serverless \ 137 | REPO=vSwarm-u \ 138 | OS_LABEL=ubuntu-20.04 139 | 140 | ENV REPO_URL=https://github.com/$ORG/$REPO \ 141 | RUNNER_NAME="gem5-test-runner" \ 142 | ACCESS_TOKEN="${GH_ACCESS_TOKEN}" \ 143 | RUNNER_WORKDIR=_work-$REPO \ 144 | RUNNER_GROUP="default" \ 145 | LABELS="gem5-test" \ 146 | RESOURCES=/tmp/resources 147 | 148 | # Install dependencies 149 | RUN apt-get update \ 150 | && apt-get install -y \ 151 | build-essential git m4 scons zlib1g zlib1g-dev \ 152 | libprotobuf-dev protobuf-compiler libprotoc-dev libgoogle-perftools-dev \ 153 | python3-dev python3-six python-is-python3 libboost-all-dev pkg-config \ 154 | && apt-get install -y \ 155 | python3-pip \ 156 | curl \ 157 | lsof \ 158 | qemu-kvm bridge-utils \ 159 | && rm -rf /var/lib/apt/lists/* 160 | 161 | ENTRYPOINT ["/entrypoint.sh"] 162 | CMD ["./bin/Runner.Listener", "run", "--startuptype", "service"] 163 | -------------------------------------------------------------------------------- /runner/Makefile: -------------------------------------------------------------------------------- 1 | # MIT License 2 | 3 | # Copyright (c) 2022 EASE lab 4 | 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | 24 | RESOURCES ?=$(ROOT)/resources/ 25 | ACCESS_TOKEN :=${GH_ACCESS_TOKEN} 26 | 27 | RUNNERS := gem5-test-runner 28 | 29 | 30 | #### Build the runner images 31 | %-runner: Dockerfile 32 | DOCKER_BUILDKIT=1 docker build \ 33 | --tag vhiveease/$@ \ 34 | --target $@ \ 35 | -f Dockerfile \ 36 | . 37 | 38 | build: $(RUNNERS) 39 | 40 | start: start-gem5-test-runner 41 | stop: stop-gem5-test-runner 42 | 43 | 44 | 45 | ## Push and pull 46 | # push-%: 47 | # docker push docker.io/vhiveease/$(subst push-,,$@):latest 48 | 49 | # push: $(addprefix push-, $(RUNNERS)) 50 | 51 | # pull-%: 52 | # docker pull docker.io/vhiveease/$(subst pull-,,$@):latest 53 | 54 | # pull: $(addprefix pull-, $(RUNNERS)) 55 | 56 | pull: 57 | docker pull vhiveease/$(RUNNERS) 58 | 59 | 60 | #@$(eval runner_name := "$(runner)-$(shell echo $$RANDOM | md5sum | head -c 10; echo;)") 61 | 62 | ##### Start and stop runners #### 63 | # Test runners 64 | start-test-runners: start-gem5-test-runner 65 | stop-test-runners: stop-gem5-test-runner 66 | 67 | start-%-test-runner: 68 | @$(eval runner := $(subst start-,,$@)) 69 | @$(eval runner_name := "$(runner)-$(shell date +%s%N | cut -b10-19)") 70 | echo $(runner_name) 71 | docker run --name $(runner_name) -d \ 72 | --privileged --cap-add=ALL \ 73 | -e RESOURCES=/tmp/resources/ \ 74 | -e RUNNER_NAME=$(runner_name) \ 75 | -e ACCESS_TOKEN=${ACCESS_TOKEN} \ 76 | -v /var/run/docker.socker.sock:/var/run/docker.sock \ 77 | -v /tmp/_work-vSwarm-u:/tmp/_work-vSwarm-u \ 78 | -v $(RESOURCES):/tmp/resources/ \ 79 | vhiveease/$(runner) 80 | 81 | 82 | # Build runners 83 | start-build-runners: start-gem5-build-runner 84 | stop-build-runners: stop-gem5-build-runner 85 | 86 | start-gem5-build-runner: 87 | @$(eval runner := gem5-build-runner) 88 | @$(eval runner_name := "$(runner)-$(shell date +%s%N | cut -b10-19)") 89 | echo $(runner_name) 90 | docker run --name $(runner_name) -d \ 91 | --privileged --cap-add=ALL \ 92 | -e RUNNER_NAME=$(runner_name) \ 93 | -e ACCESS_TOKEN=${ACCESS_TOKEN} \ 94 | -v /var/run/docker.socker.sock:/var/run/docker.sock \ 95 | -v /tmp/_work-vSwarm-u:/tmp/_work-vSwarm-u \ 96 | vhiveease/$(runner) 97 | 98 | 99 | stop-%-runner: 100 | $(eval runner := $(subst stop-,,$@)) 101 | $(eval runners := $(shell docker container ls -a -q --filter "ancestor=vhiveease/$(runner)")) 102 | echo "Stop $(runners) $$(expr length "$(runners)")" 103 | if [ $$(expr length "$(runners)") -gt 0 ]; then \ 104 | docker stop $(runners); \ 105 | docker rm $(runners); \ 106 | fi; 107 | 108 | # stop-all: 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | -------------------------------------------------------------------------------- /runner/create-build-runner.yaml: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2022 David Schall and EASE lab 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | 24 | # Execute this playbook using 25 | # ansible-playbook -u YOUR_SSH_USERNAME -i REMOTE_HOSTNAME, setup-host.yaml 26 | 27 | --- 28 | - name: Setup vSwarm-u runner host 29 | hosts: all 30 | vars: 31 | RESOURCES: /resources/ 32 | tasks: 33 | 34 | - name: Run the github runner 35 | become: yes 36 | shell: | 37 | GH_ACCESS_TOKEN={{ lookup('env', 'GH_ACCESS_TOKEN') }} 38 | TMP=$(date +%s%N | cut -b10-19) 39 | docker run --rm -d --name github-runner-${TMP} \ 40 | -e REPO_URL="https://github.com/vhive-serverless/vSwarm-u" \ 41 | -e ACCESS_TOKEN="${GH_ACCESS_TOKEN}" \ 42 | -e RUNNER_NAME="vswarm-u-build-runner-${TMP}" \ 43 | -e RUNNER_WORKDIR="/tmp/_work" \ 44 | -e RUNNER_GROUP=default \ 45 | -e LABELS="vswarm-u-build" \ 46 | --privileged --cap-add=ALL \ 47 | -v /tmp/github-runner-your-repo:/tmp/github-runner-your-repo \ 48 | myoung34/github-runner:latest 49 | -------------------------------------------------------------------------------- /runner/create-runner.yaml: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2021 Mert Bora Alper and EASE lab 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | # Execute this playbook using 24 | # GH_ACCESS_TOKEN=... ansible-playbook -u YOUR_SSH_USERNAME -i REMOTE_HOSTNAME, create-runner.yaml 25 | # 26 | --- 27 | - name: Create GitHub Runners for gem5 CI 28 | hosts: all 29 | tasks: 30 | 31 | # - name: Generate a random runner name 32 | # shell: 'printf "$(shuf -n1 /usr/share/dict/predicates)-$(shuf -n1 /usr/share/dict/objects)"' 33 | # register: runner_name 34 | 35 | # - name: Print the generated name 36 | # ansible.builtin.debug: 37 | # var: runner_name.stdout 38 | 39 | - name: Start the gem5 runners 40 | shell: | 41 | GH_ACCESS_TOKEN={{ lookup('env', 'GH_ACCESS_TOKEN') }} \ 42 | RESOURCES=/wkdir/vSwarm-u/resources/ \ 43 | sudo make -f /wkdir/vSwarm-u/runner/Makefile start -------------------------------------------------------------------------------- /runner/delete-runners.yaml: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2021 Mert Bora Alper and EASE lab 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | # Execute this playbook using 24 | # GH_ACCESS_TOKEN=... ansible-playbook -u YOUR_SSH_USERNAME -i REMOTE_HOSTNAME, delete-runners.yaml 25 | 26 | --- 27 | - name: Delete All GitHub Runners 28 | hosts: all 29 | tasks: 30 | 31 | - name: Stop the gem5 runners 32 | shell: | 33 | GH_ACCESS_TOKEN={{ lookup('env', 'GH_ACCESS_TOKEN') }} sudo make -f /wkdir/vSwarm-u/runner/Makefile stop -------------------------------------------------------------------------------- /runner/easy-create.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # MIT License 4 | # 5 | # Copyright (c) 2022 David Schall and EASE lab 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | 25 | # Execute this script (on the local) as 26 | # GH_ACCESS_TOKEN=... ./easy-create.sh [USER] [PRIVATE KEY] 27 | # 28 | 29 | set -x 30 | set -e 31 | 32 | PWD="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 33 | 34 | REMOTE=$1 35 | COUNT=$2 36 | AS_USER=${3:-`whoami`} 37 | KEY=${4:-~/.ssh/id_rsa} 38 | 39 | # ansible-playbook -v --private-key ${KEY} -u $AS_USER -i ${REMOTE}, ${PWD}/setup-host.yaml 40 | ansible-playbook -v --private-key ${KEY} -u $AS_USER -i ${REMOTE}, ${PWD}/setup-build-host.yaml 41 | 42 | 43 | # ansible-playbook -u $AS_USER --private-key ${KEY} -i ${REMOTE}, ${PWD}/delete-runners.yaml 44 | 45 | for i in $(seq ${COUNT}); do 46 | # GH_ACCESS_TOKEN=${GH_ACCESS_TOKEN} ansible-playbook --private-key ${KEY} -v -u $AS_USER -i ${REMOTE}, ${PWD}/create-runner.yaml 47 | GH_ACCESS_TOKEN=${GH_ACCESS_TOKEN} ansible-playbook --private-key ${KEY} -v -u $AS_USER -i ${REMOTE}, ${PWD}/create-build-runner.yaml 48 | done 49 | -------------------------------------------------------------------------------- /runner/setup-build-host.yaml: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2022 David Schall and EASE lab 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | 24 | # Execute this playbook using 25 | # ansible-playbook -u YOUR_SSH_USERNAME -i REMOTE_HOSTNAME, setup-host.yaml 26 | 27 | --- 28 | - name: Setup vSwarm-u runner host 29 | hosts: all 30 | vars: 31 | RESOURCES: /resources/ 32 | tasks: 33 | 34 | - name: Remove any old Docker installations 35 | become: yes 36 | apt: 37 | state: absent 38 | purge: yes 39 | autoremove: yes 40 | name: 41 | - docker 42 | - docker-engine 43 | - docker.io 44 | - containerd 45 | 46 | - name: Remove Docker directories 47 | become: yes 48 | file: 49 | state: absent 50 | path: "{{ item }}" 51 | loop: 52 | - /var/lib/docker 53 | - /var/lib/containerd 54 | 55 | - name: Install GPG 56 | become: yes 57 | apt: 58 | state: latest 59 | update_cache: yes 60 | name: 61 | - gpg 62 | - dirmngr 63 | - gpg-agent 64 | 65 | 66 | - name: Get dependencies and docker repositories 67 | become: yes 68 | shell: | 69 | sudo apt-get update 70 | sudo apt-get install ca-certificates curl 71 | sudo install -m 0755 -d /etc/apt/keyrings 72 | sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc 73 | sudo chmod a+r /etc/apt/keyrings/docker.asc 74 | echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null 75 | 76 | 77 | 78 | - name: Upgrade packages 79 | become: yes 80 | apt: 81 | update_cache: yes 82 | upgrade: safe 83 | 84 | - name: Install required packages 85 | become: yes 86 | apt: 87 | state: latest 88 | name: 89 | - docker-ce 90 | - docker-ce-cli 91 | - containerd.io 92 | - docker-buildx-plugin 93 | - docker-compose-plugin 94 | 95 | 96 | - name: Create Working directory 97 | become: yes 98 | ansible.builtin.file: 99 | path: /wkdir/ 100 | state: directory 101 | mode: '0777' 102 | 103 | - name: Restart docker service 104 | become: yes 105 | shell: | 106 | sudo systemctl restart docker 107 | 108 | 109 | -------------------------------------------------------------------------------- /runner/setup-host.yaml: -------------------------------------------------------------------------------- 1 | # MIT License 2 | # 3 | # Copyright (c) 2022 David Schall and EASE lab 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | 24 | # Execute this playbook using 25 | # ansible-playbook -u YOUR_SSH_USERNAME -i REMOTE_HOSTNAME, setup-host.yaml 26 | 27 | --- 28 | - name: Setup vSwarm-u runner host 29 | hosts: all 30 | vars: 31 | RESOURCES: /resources/ 32 | tasks: 33 | 34 | - name: Remove any old Docker installations 35 | become: yes 36 | apt: 37 | state: absent 38 | purge: yes 39 | autoremove: yes 40 | name: 41 | - docker 42 | - docker-engine 43 | - docker.io 44 | - containerd 45 | 46 | - name: Remove Docker directories 47 | become: yes 48 | file: 49 | state: absent 50 | path: "{{ item }}" 51 | loop: 52 | - /var/lib/docker 53 | - /var/lib/containerd 54 | 55 | - name: Install GPG 56 | become: yes 57 | apt: 58 | state: latest 59 | update_cache: yes 60 | name: 61 | - gpg 62 | - dirmngr 63 | - gpg-agent 64 | 65 | 66 | 67 | 68 | - name: Create Working directory 69 | become: yes 70 | ansible.builtin.file: 71 | path: /wkdir/ 72 | state: directory 73 | mode: '0777' 74 | 75 | # - name: Create The RESOURCES directory 76 | # become: yes 77 | # ansible.builtin.file: 78 | # path: "{{ RESOURCES }}" 79 | # state: directory 80 | # mode: '0777' 81 | 82 | # - name: Export the RESOURCE path 83 | # become: yes 84 | # lineinfile: 85 | # path: "/etc/environment" 86 | # line: 'RESOURCES={{ RESOURCES }}' 87 | # insertafter: 'EOF' 88 | # state: present 89 | 90 | # - name: Source the bashrc file 91 | # shell: source {{ lookup('env', 'HOME') }}/.bashrc 92 | 93 | # - name: print env 94 | # # shell: echo {{ lookup('env', 'RESOURCES') }} 95 | # shell: env 96 | 97 | # - name: print env 98 | # shell: cat /etc/environment 99 | 100 | 101 | ## Clone vSwarm-u 102 | - name: Clone a github repository 103 | shell: git clone https://github.com/vhive-serverless/vSwarm-u.git /wkdir/vSwarm-u 104 | 105 | 106 | ## Put all remaining installation into the setup script. 107 | ## This makes it easier to do the setup manually 108 | - name: Run the Host setup script 109 | shell: "/wkdir/vSwarm-u/setup/setup_host.sh" 110 | 111 | - name: Build the gem5 runners 112 | shell: | 113 | sudo make -f /wkdir/vSwarm-u/runner/Makefile pull 114 | 115 | # - name: Add cron entry to reboot at midnight 116 | # become: yes 117 | # become_method: sudo 118 | # ansible.builtin.cron: 119 | # user: root 120 | # name: "reboot" 121 | # minute: "0" 122 | # hour: "0" 123 | # job: "/sbin/shutdown --reboot now" 124 | -------------------------------------------------------------------------------- /scripts/cpi.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | inst=$(awk '/system.cpu1.exec_context.thread_0.numInsts/ {print $2}' $1) 4 | cycles=$(awk '/system.cpu1.numCycles/ {print $2}' $1) 5 | 6 | echo $inst | awk '{printf"Insts:\t%i\n", $1}' 7 | echo $cycles | awk '{printf"Cycles:\t%i\n", $1}' 8 | 9 | # Also do the math: 10 | echo $cycles $inst | awk '{ tmp=$1/$2 ; printf"CPI:\t%0.4f\n", tmp }' 11 | echo $inst $cycles | awk '{ tmp=$1/$2 ; printf"IPC:\t%0.4f\n", tmp }' 12 | 13 | -------------------------------------------------------------------------------- /scripts/run_function.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ## Define the image name of your function. 4 | IMAGE_NAME= 5 | 6 | ## Spin up Container 7 | echo "Start the container..." 8 | docker run -d --name mycontainer -p 50051:50051 $IMAGE_NAME 9 | 10 | echo "Pin to core 1" 11 | docker update mycontainer --cpuset-cpus 1 12 | 13 | sleep 5 14 | 15 | # Booting Linux and starting the invoker 16 | # is completed. Break out of the simulation and switch CPU 17 | echo "Execute m5 exit to indicate boot complete" 18 | m5 exit 19 | 20 | ## Now start the invoker 21 | # Modify the invoker parameters depending on your need. 22 | 23 | /root/client -addr localhost:50051 -n 20 24 | 25 | ## Call exit again to end the simulation 26 | m5 exit -------------------------------------------------------------------------------- /scripts/run_qemu.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # MIT License 4 | # 5 | # Copyright (c) 2022 David Schall and EASE lab 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | # Install dependencies 25 | 26 | set -e -x 27 | 28 | 29 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 30 | ROOT="$( cd $DIR && cd .. && pwd)" 31 | 32 | 33 | DISK_IMG=${1:-$ROOT/workload/disk-image.img} 34 | KERNEL=${2:-$ROOT/workload/vmlinux} 35 | RAM=8G 36 | CPUS=4 37 | 38 | sudo qemu-system-x86_64 \ 39 | -nographic \ 40 | -cpu host -enable-kvm \ 41 | -smp ${CPUS} \ 42 | -m ${RAM} \ 43 | -device e1000,netdev=net0 \ 44 | -netdev type=user,id=net0,hostfwd=tcp:127.0.0.1:5555-:22 \ 45 | -drive format=raw,file=$DISK_IMG \ 46 | -kernel $KERNEL \ 47 | -append 'earlyprintk=ttyS0 console=ttyS0 lpj=7999923 root=/dev/hda2' 48 | 49 | -------------------------------------------------------------------------------- /setup/build_gem5.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # MIT License 4 | # 5 | # Copyright (c) 2022 David Schall and EASE lab 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | # Install dependencies 25 | 26 | set -e 27 | 28 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 29 | ROOT="$( cd $DIR && cd .. && pwd)" 30 | 31 | 32 | ARCH=X86 33 | GEM5_DIR=${1:-$ROOT/gem5/} 34 | 35 | RED='\033[0;31m' 36 | GREEN='\033[0;32m' 37 | NC='\033[0m' # No Color 38 | 39 | printf " 40 | ============================= 41 | ${GREEN} Build gem5 ${NC} 42 | --- 43 | Architecture: $ARCH 44 | Output: $GEM5_DIR 45 | =============================\n\n" 46 | 47 | 48 | 49 | # Setup on Ubuntu 20.04 50 | sudo apt update 51 | sudo apt install -y build-essential git m4 scons zlib1g zlib1g-dev \ 52 | libprotobuf-dev protobuf-compiler libprotoc-dev libgoogle-perftools-dev \ 53 | python3-dev python3-six python-is-python3 libboost-all-dev pkg-config 54 | 55 | 56 | git clone https://gem5.googlesource.com/public/gem5 $GEM5_DIR 57 | 58 | pushd $GEM5_DIR 59 | 60 | ## Build gem5 61 | scons build/$ARCH/gem5.opt -j $(nproc) --install-hooks 62 | popd 63 | 64 | ### 65 | ## Build the terminal 66 | pushd $GEM5_DIR/util/term 67 | make 68 | popd 69 | 70 | ## the path for x86 is slightly different 71 | if [ $ARCH == X86 ]; then ARCH=x86; fi 72 | 73 | ## Build m5 tools 74 | pushd $GEM5_DIR/util/m5 75 | scons build/$ARCH/out/m5 76 | popd 77 | 78 | # Export absolute path 79 | export GEM5_DIR=$GEM5_DIR 80 | 81 | sudo sh -c "echo 'export GEM5_DIR=${GEM5_DIR}' >> /etc/profile" 82 | sh -c "echo 'export GEM5_DIR=${GEM5_DIR}' >> $HOME/.bashrc" 83 | -------------------------------------------------------------------------------- /setup/build_kernel.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # MIT License 4 | # 5 | # Copyright (c) 2022 David Schall and EASE lab 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | # Install dependencies 25 | 26 | set -e 27 | 28 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 29 | ROOT="$( cd $DIR && cd .. && pwd)" 30 | 31 | 32 | ### Build kernel for gem5 supporting running docker images 33 | KVERSION=v5.4.84 34 | OUTDIR=${1:-$ROOT/workload/} 35 | 36 | printf " 37 | ============================= 38 | \033[0;32m Build Linux kernel for gem5 \033[0m 39 | --- 40 | KVERSION: $KVERSION 41 | OUTDIR: $OUTDIR 42 | =============================\n\n" 43 | 44 | ## Install dependencies 45 | sudo apt-get install -y \ 46 | git build-essential ncurses-dev xz-utils libssl-dev bc \ 47 | flex libelf-dev bison 48 | 49 | # Get sources 50 | git clone https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git linux 51 | pushd linux 52 | git checkout ${KVERSION} 53 | 54 | # Apply the configuration 55 | cp ${ROOT}/configs/linux-configs/${KVERSION}.config .config 56 | 57 | ## build kernel 58 | make -j $(nproc) 59 | 60 | ## place in ouput folder 61 | mkdir -p $OUTDIR 62 | cp vmlinux $OUTDIR 63 | popd 64 | 65 | ## Clean up the linux sources 66 | # rm -rf ../linux 67 | 68 | -------------------------------------------------------------------------------- /setup/create_disk_image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # MIT License 4 | # 5 | # Copyright (c) 2022 David Schall and EASE lab 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | # Install dependencies 25 | 26 | set -e -x 27 | 28 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 29 | ROOT="$( cd $DIR && cd .. && pwd)" 30 | 31 | ## User specific inputs 32 | OUTDIR=${1:-../workload/} 33 | 34 | INSTALL_ISO=ubuntu-20.04.3-live-server-amd64.iso 35 | DISK_IMG=base-disk-image.img 36 | DISK_SIZE=4G 37 | RAM=8G 38 | CPUS=4 39 | 40 | printf " 41 | ============================= 42 | \033[0;32m Build base disk image for gem5 \033[0m 43 | --- 44 | INSTALL_ISO: $INSTALL_ISO 45 | Disk size: $DISK_SIZE 46 | Output: $OUTDIR/$DISK_IMG 47 | =============================\n\n" 48 | 49 | 50 | ## Install dependencies 51 | sudo apt-get update 52 | sudo apt-get install -y qemu-kvm libvirt-daemon-system libvirt-clients bridge-utils wget 53 | 54 | ## Install dependencies 55 | sudo apt-get install -y qemu-kvm libvirt-daemon-system libvirt-clients bridge-utils 56 | 57 | if [ $(uname -i) != "x86_64" ] ; 58 | then 59 | echo "Target platform is not x86" 60 | exit 61 | fi 62 | 63 | 64 | ## Get and mount Ubuntu 20.04 65 | if [ -f "$INSTALL_ISO" ]; then 66 | echo "$INSTALL_ISO exists." 67 | else 68 | echo "$INSTALL_ISO does not exist. Download it..." 69 | wget https://releases.ubuntu.com/20.04.3/$INSTALL_ISO 70 | echo "f8e3086f3cea0fb3fefb29937ab5ed9d19e767079633960ccb50e76153effc98 *${INSTALL_ISO}" | shasum -a 256 --check 71 | fi 72 | 73 | mkdir -p iso 74 | sudo mount -r $INSTALL_ISO iso 75 | 76 | ## Serve all files for the setup over a http server. 77 | mkdir -p tmp/ 78 | cp -r ${ROOT}/configs/disk-image-configs/* tmp/ 79 | 80 | ## Rename the autoconfig to user-data 81 | mv tmp/autoinstall.yaml tmp/user-data 82 | touch tmp/meta-data 83 | touch tmp/vendor-data 84 | 85 | python3 -m http.server -d tmp 3003 & 86 | 87 | SERVER_PID=$! 88 | 89 | 90 | #### Create the disk image 91 | qemu-img create disk.img $DISK_SIZE 92 | 93 | 94 | ## Do the actual installation 95 | # The command will boot from the iso file. 96 | # Then it will listen to port 3003 to retive the autoconfig files. 97 | # If such provided the install process will automatically done for you. 98 | sudo qemu-system-x86_64 \ 99 | -nographic \ 100 | -cpu host -enable-kvm \ 101 | -smp ${CPUS} \ 102 | -m ${RAM} \ 103 | -no-reboot \ 104 | -drive file=disk.img,format=raw \ 105 | -cdrom $INSTALL_ISO \ 106 | -kernel iso/casper/vmlinuz \ 107 | -initrd iso/casper/initrd \ 108 | -append 'autoinstall ds=nocloud-net;s=http://_gateway:3003/' 109 | 110 | ## To get the full output of the install process add 111 | ## 'earlyprintk=ttyS0 console=ttyS0 lpj=7999923' to the 112 | ## kernel commands (-append flag) 113 | 114 | echo "Ubuntu installed on disk image" 115 | 116 | ## Cleanup everything 117 | kill "$SERVER_PID" 118 | 119 | sudo umount iso 120 | rm -r iso *.iso tmp 121 | 122 | 123 | # Move the final ready to use disk image to the workload folder 124 | mkdir -p $OUTDIR 125 | mv disk.img $OUTDIR/$DISK_IMG 126 | 127 | 128 | -------------------------------------------------------------------------------- /setup/gem5.Makefile: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # MIT License 4 | # 5 | # Copyright (c) 2022 David Schall and EASE lab 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | 25 | 26 | mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST))) 27 | ROOT := $(abspath $(dir $(mkfile_path))/../) 28 | 29 | ## User specific inputs 30 | RESOURCES ?=$(ROOT)/resources/ 31 | ARCH := ALL 32 | VERSION := v24.0.0.0 33 | 34 | GEM5_DIR ?= $(RESOURCES)/gem5/ 35 | GEM5 := $(GEM5_DIR)/build/$(ARCH)/gem5.opt 36 | 37 | 38 | .PONY: all gem5 term m5 39 | 40 | all: gem5 term m5 41 | 42 | 43 | ## Dependencies ------------------------------------------------- 44 | ## Install all dependencies to build gem5 45 | ## 46 | dep_install: 47 | sudo apt-get update \ 48 | && sudo apt-get install -y \ 49 | build-essential git m4 scons zlib1g zlib1g-dev \ 50 | libprotobuf-dev protobuf-compiler libprotoc-dev libgoogle-perftools-dev \ 51 | python3-dev libboost-all-dev pkg-config python3-tk libunwind-dev 52 | 53 | 54 | ## Clone repo -- 55 | $(GEM5_DIR): 56 | git clone https://github.com/gem5/gem5.git $@ 57 | cd $@; git checkout $(VERSION) 58 | 59 | 60 | ## Build 61 | gem5: $(GEM5_DIR) 62 | @$(call print_config) 63 | cd $(GEM5_DIR); \ 64 | git checkout $(VERSION); \ 65 | scons build/$(ARCH)/gem5.opt -j $$(nproc) --install-hooks 66 | 67 | 68 | ## Build the terminal 69 | term: $(GEM5_DIR) 70 | @printf "$\n{GREEN}Build gem5 terminal${NC}\n" 71 | cd $(GEM5_DIR)/util/term; \ 72 | make 73 | 74 | 75 | _ARCH=x86 76 | ifeq ($(ARCH), X86) 77 | _ARCH =x86 78 | endif 79 | 80 | ## Build m5 tools 81 | m5: $(GEM5_DIR) 82 | @printf "\n${GREEN}Build gem5's m5 binaries${NC}\n" 83 | # $(eval $(if [ $(ARCH) == X86 ]; then _ARCH :=x86; fi)) 84 | cd $(GEM5_DIR)/util/m5; \ 85 | scons build/$(_ARCH)/out/m5 86 | 87 | 88 | RED=\033[0;31m 89 | GREEN=\033[0;32m 90 | NC=\033[0m # No Color 91 | 92 | define print_config 93 | printf "\n=============================\n"; \ 94 | printf "${GREEN} Build gem5 ${NC}\n"; \ 95 | printf " ---\n"; \ 96 | printf "Architecture: $(ARCH)\n"; \ 97 | printf "version: $(VERSION)\n"; \ 98 | printf "Output: $(GEM5_DIR)\n"; \ 99 | printf "=============================\n\n"; 100 | endef -------------------------------------------------------------------------------- /setup/kernel.Makefile: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # MIT License 4 | # 5 | # Copyright (c) 2022 EASE lab, University of Edinburgh 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | # 25 | # Authors: David Schall 26 | 27 | 28 | mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST))) 29 | ROOT := $(abspath $(dir $(mkfile_path))/../) 30 | 31 | ## User specific inputs 32 | RESOURCES ?=$(ROOT)/resources/ 33 | 34 | LINUX_DIR := linux/ 35 | KERNEL_OUT ?= $(RESOURCES)/vmlinux 36 | OUTPUT ?= 37 | 38 | KVERSION ?= v5.4.84 39 | ARCH ?= amd64 40 | KERNEL_CONFIG_GEM5 := $(ROOT)/configs/linux-configs/$(KVERSION)-$(ARCH).config 41 | KERNEL_CONFIG := $(LINUX_DIR)/.config 42 | KERNEL_PATCH_GEM5 := $(ROOT)/configs/linux-configs/kernel_m5.patch 43 | 44 | 45 | BUILD_OBJ := $(LINUX_DIR)/vmlinux 46 | 47 | .PONY: all config 48 | 49 | all: build save 50 | 51 | ## Dependencies ------------------------------------------------- 52 | ## Install all dependencies to build linux kernel 53 | ## 54 | dep_install: 55 | sudo apt-get update \ 56 | && sudo apt-get install -y \ 57 | git build-essential ncurses-dev xz-utils libssl-dev bc \ 58 | flex libelf-dev bison \ 59 | gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu device-tree-compiler 60 | 61 | ## Get sources -- 62 | $(LINUX_DIR): 63 | git clone https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git $@ 64 | 65 | ## Make config 66 | config: $(LINUX_DIR) 67 | cd $(LINUX_DIR); \ 68 | git checkout $(KVERSION) 69 | cp $(KERNEL_CONFIG_GEM5) $(KERNEL_CONFIG) 70 | 71 | 72 | ## Add patches to the kernel 73 | patch: $(LINUX_DIR) 74 | -cd $(LINUX_DIR); \ 75 | git apply --ignore-space-change $(KERNEL_PATCH_GEM5) 76 | 77 | 78 | ## Build 79 | build-amd64: $(LINUX_DIR) config patch 80 | @$(call print_config) 81 | cd $(LINUX_DIR); \ 82 | make ARCH=x86_64 -j $$(nproc) 83 | 84 | build-arm64: $(LINUX_DIR) config patch 85 | @$(call print_config) 86 | cd $(LINUX_DIR); \ 87 | make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- -j $$(nproc) 88 | 89 | 90 | ifeq ($(ARCH), arm64) 91 | build: build-arm64 92 | else 93 | build: build-amd64 94 | endif 95 | 96 | 97 | 98 | save: build 99 | cp $(BUILD_OBJ) $(KERNEL_OUT) 100 | 101 | save_output: 102 | cp $(BUILD_OBJ) $(OUTPUT) 103 | 104 | 105 | clean: 106 | rm -rf $(LINUX_DIR) 107 | 108 | clean_all: clean 109 | rm $(KERNEL) 110 | 111 | RED=\033[0;31m 112 | GREEN=\033[0;32m 113 | NC=\033[0m # No Color 114 | 115 | define print_config 116 | printf "\n=============================\n"; \ 117 | printf "${GREEN} Build Linux kernel for gem5 ${NC}\n"; \ 118 | printf " ---\n"; \ 119 | printf "kernel version: $(KVERSION)\n"; \ 120 | printf "Output: $(KERNEL)\n"; \ 121 | printf "=============================\n\n"; 122 | endef -------------------------------------------------------------------------------- /setup/requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib 2 | numpy 3 | brewer2mpl 4 | pydot 5 | objectpath 6 | seaborn 7 | docker-compose 8 | uploadserver 9 | niet 10 | tqdm 11 | requests 12 | #mlpd3 13 | -------------------------------------------------------------------------------- /setup/setup_host.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # MIT License 4 | # 5 | # Copyright (c) 2022 David Schall and EASE lab 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | # Install dependencies 25 | 26 | set -e 27 | 28 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 29 | ROOT="$( cd $DIR && cd .. && pwd)" 30 | 31 | if [ $(uname -i) == "aarch64" ]; 32 | then 33 | ARCH=arm64 34 | else 35 | ARCH=amd64 36 | fi 37 | 38 | START=$(date +%s.%N) 39 | 40 | ## Set the path to the directory where 41 | ## all resources should be placed 42 | ## - gem5 43 | ## - kernel 44 | ## - base disk image 45 | 46 | if [[ -z "${RESOURCES}" ]]; then 47 | echo "Warning: RESOURCES environmental variable is not set." 48 | echo "Warning: Will be set to the default `resources/` in this repo" 49 | RESOURCES=$ROOT/resources/ 50 | export RESOURCES=$RESOURCES 51 | sudo sh -c "echo 'export RESOURCES=${RESOURCES}' >> /etc/profile" 52 | sudo sh -c "echo 'export RESOURCES=${RESOURCES}' >> ${HOME}/.bashrc" 53 | fi 54 | 55 | if [[ -z "${GEM5_DIR}" ]]; then 56 | echo "Warning: GEM5_DIR environmental variable is not set." 57 | echo "Warning: Will be set to the default `resources/` in this repo" 58 | GEM5_DIR=$ROOT/resources/gem5/ 59 | export GEM5_DIR=$GEM5_DIR 60 | sudo sh -c "echo 'export GEM5_DIR=${GEM5_DIR}' >> /etc/profile" 61 | sudo sh -c "echo 'export GEM5_DIR=${GEM5_DIR}' >> ${HOME}/.bashrc" 62 | fi 63 | 64 | echo "Install all resources to: ${RESOURCES}" 65 | 66 | sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test 67 | sudo apt-get update >> /dev/null 68 | sudo apt-get -y upgrade 69 | 70 | END=$(date +%s.%N) 71 | echo "Took now:" $(echo "$END - $START" | bc) 72 | 73 | # Install docker 74 | sudo apt-get install -y \ 75 | apt-transport-https \ 76 | ca-certificates \ 77 | curl \ 78 | gnupg-agent \ 79 | software-properties-common \ 80 | && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - 81 | 82 | sudo add-apt-repository \ 83 | "deb [arch=${ARCH}] https://download.docker.com/linux/ubuntu \ 84 | $(lsb_release -cs) \ 85 | stable" \ 86 | && sudo apt-get update >> /dev/null \ 87 | && sudo apt-get install -y docker-ce docker-ce-cli containerd.io 88 | 89 | ## Install also docker compose 90 | sudo apt install -y python3-pip 91 | sudo pip3 install -r ${ROOT}/setup/requirements.txt 92 | 93 | ## Install qemu 94 | make -f ${ROOT}/setup/disk.Makefile dep_install 95 | 96 | 97 | # Install golang 98 | GO_VERSION=1.21.6 99 | GO_BUILD="go${GO_VERSION}.linux-${ARCH}" 100 | 101 | wget --continue https://golang.org/dl/${GO_BUILD}.tar.gz 102 | 103 | sudo rm -rf /usr/local/go 104 | sudo tar -C /usr/local -xzf ${GO_BUILD}.tar.gz 105 | rm ${GO_BUILD}.tar.gz 106 | 107 | export PATH=$PATH:/usr/local/go/bin 108 | sudo sh -c "echo 'export PATH=\$PATH:/usr/local/go/bin' >> /etc/profile" 109 | sudo sh -c "echo 'export PATH=\$PATH:/usr/local/go/bin' >> ${HOME}/.bashrc" 110 | 111 | source ${HOME}/.bashrc 112 | echo "Installed: $(go version)" 113 | 114 | END=$(date +%s.%N) 115 | printf "\nInstalling dependencies completed successfully after: $(echo "$END - $START" | bc) sec.\n" 116 | 117 | 118 | 119 | ## Now prepare the base setup to run the functions. 120 | START=$(date +%s.%N) 121 | echo "Build gem5..." 122 | make -f ${ROOT}/setup/gem5.Makefile dep_install 123 | make -f ${ROOT}/setup/gem5.Makefile all 124 | 125 | END=$(date +%s.%N) 126 | printf "\nBuilding gem5 completed after: $(echo "$END - $START" | bc) sec.\n" 127 | 128 | 129 | ## Download the most recent release artifacts 130 | START=$(date +%s.%N) 131 | echo "Download Release artifacts from: " 132 | 133 | ${ROOT}/resources/artifacts.py --output ${ROOT}/resources/ 134 | 135 | END=$(date +%s.%N) 136 | printf "\nDownload artifacts complete. Took: $(echo "$END - $START" | bc) sec.\n" 137 | -------------------------------------------------------------------------------- /simulation/functions/all_vswarm_functions.list: -------------------------------------------------------------------------------- 1 | fibonacci-go 2 | fibonacci-nodejs 3 | fibonacci-python 4 | aes-go 5 | aes-nodejs 6 | aes-python 7 | auth-go 8 | auth-nodejs 9 | auth-python 10 | geo 11 | profile 12 | rate 13 | recommendation 14 | reservation 15 | user 16 | productcatalogservice 17 | currencyservice 18 | paymentservice 19 | shippingservice 20 | emailservice 21 | recommendationservice 22 | -------------------------------------------------------------------------------- /simulation/functions/functions.list: -------------------------------------------------------------------------------- 1 | fibonacci-go 2 | fibonacci-nodejs 3 | fibonacci-python 4 | 5 | -------------------------------------------------------------------------------- /simulation/functions/functions.yaml: -------------------------------------------------------------------------------- 1 | # MIT License 2 | 3 | # Copyright (c) 2022 EASE lab 4 | 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # fibonacciORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | name: vswarmu 24 | services: 25 | ## Fibonacci --------------------------------- 26 | fibonacci-go: 27 | image: vhiveease/fibonacci-go:latest 28 | container_name: function 29 | entrypoint: 30 | - /app/server 31 | - -addr=0.0.0.0:50051 32 | ports: 33 | - target: 50051 34 | published: 50000 35 | 36 | fibonacci-nodejs: 37 | image: vhiveease/fibonacci-nodejs:latest 38 | container_name: function 39 | entrypoint: 40 | - node 41 | - server.js 42 | - --addr=0.0.0.0 43 | - --port=50051 44 | ports: 45 | - target: 50051 46 | published: 50000 47 | 48 | fibonacci-python: 49 | image: vhiveease/fibonacci-python:latest 50 | container_name: function 51 | entrypoint: 52 | - python 53 | - /app/server.py 54 | - --addr=0.0.0.0 55 | - --port=50051 56 | ports: 57 | - target: 50051 58 | published: 50000 59 | 60 | 61 | ## AES --------------------------------- 62 | aes-go: 63 | image: vhiveease/aes-go:latest 64 | container_name: function 65 | entrypoint: 66 | - /app/server 67 | - -addr=0.0.0.0:50051 68 | ports: 69 | - target: 50051 70 | published: 50000 71 | 72 | aes-nodejs: 73 | image: vhiveease/aes-nodejs:latest 74 | container_name: function 75 | entrypoint: 76 | - node 77 | - server.js 78 | - --addr=0.0.0.0 79 | - --port=50051 80 | ports: 81 | - target: 50051 82 | published: 50000 83 | 84 | aes-python: 85 | image: vhiveease/aes-python:latest 86 | container_name: function 87 | entrypoint: 88 | - python 89 | - /app/server.py 90 | - --addr=0.0.0.0 91 | - --port=50051 92 | ports: 93 | - target: 50051 94 | published: 50000 95 | 96 | 97 | ## Authentication ----------------------- 98 | auth-go: 99 | image: vhiveease/auth-go:latest 100 | container_name: function 101 | entrypoint: 102 | - /app/server 103 | - -addr=0.0.0.0:50051 104 | ports: 105 | - target: 50051 106 | published: 50000 107 | 108 | auth-nodejs: 109 | image: vhiveease/auth-nodejs:latest 110 | container_name: function 111 | entrypoint: 112 | - node 113 | - server.js 114 | - --addr=0.0.0.0 115 | - --port=50051 116 | ports: 117 | - target: 50051 118 | published: 50000 119 | 120 | auth-python: 121 | image: vhiveease/auth-python:latest 122 | container_name: function 123 | entrypoint: 124 | - python 125 | - /app/server.py 126 | - --addr=0.0.0.0 127 | - --port=50051 128 | ports: 129 | - target: 50051 130 | published: 50000 -------------------------------------------------------------------------------- /simulation/install_functions.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | 5 | LOGFILE=/root/install.log 6 | 7 | function start_logging { 8 | ## Log everything to a empty log file 9 | [ -e $LOGFILE ] && rm -- $LOGFILE 10 | touch $LOGFILE 11 | exec 3>&1 4>&2 12 | trap 'exec 2>&4 1>&3' 0 1 2 3 13 | exec 1> $LOGFILE 2>&1 14 | } 15 | function end_logging { 16 | # Restore file descriptors 17 | exec 2>&4 1>&3 18 | } 19 | 20 | 21 | function pull_test_function { 22 | FUNCTION_NAME=$1 23 | 24 | echo "Install and test: ${FUNCTION_NAME} " 25 | 26 | ## Make sure no other containers are running 27 | docker-compose -f /root/functions.yaml down --remove-orphans 28 | 29 | ## Pull the image from regestry 30 | docker-compose -f /root/functions.yaml pull ${FUNCTION_NAME} 31 | docker-compose -f /root/functions.yaml up -d --remove-orphans ${FUNCTION_NAME} 32 | 33 | sleep 5 34 | 35 | ## Now start the invoker 36 | /root/test-client \ 37 | -function-name ${FUNCTION_NAME} \ 38 | -url localhost \ 39 | -port 50000 \ 40 | -n 5 -input 1 41 | 42 | ## Stop container 43 | docker-compose -f /root/functions.yaml down 44 | CONTAINERS="$(docker ps -a -q)" 45 | if [ $(expr length "$CONTAINERS") -gt 0 ]; 46 | then 47 | docker stop $CONTAINERS 48 | docker rm $CONTAINERS 49 | fi 50 | } 51 | 52 | 53 | start_logging 54 | { 55 | # set -e 56 | ## Download the test client. 57 | curl "http://10.0.2.2:3003/test-client" -f -o /root/test-client 58 | chmod 755 /root/test-client 59 | 60 | ## Download the function yaml and list. 61 | curl "http://10.0.2.2:3003/functions.yaml" -f -o /root/functions.yaml 62 | curl "http://10.0.2.2:3003/functions.list" -f -o /root/functions.list 63 | 64 | 65 | ## List all functions can be commented out 66 | FUNCTIONS=$(cat /root/functions.list | sed '/^\s*#/d;/^\s*$/d') 67 | 68 | for f in $FUNCTIONS 69 | do 70 | pull_test_function $f 71 | done 72 | 73 | ## Catch for failiure ---------- 74 | } || { 75 | echo "\033[0;31m----------------" 76 | echo "FAIL" 77 | echo "----------------\033[0m" 78 | cat /root/results.log 79 | } 80 | # set +e 81 | end_logging 82 | 83 | ## Upload the log file. 84 | curl "http://10.0.2.2:3003/upload" -F "files=@${LOGFILE}" 85 | 86 | shutdown -h now 87 | -------------------------------------------------------------------------------- /simulation/wkdir-tmpl/run_sim_test.tmpl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # MIT License 4 | # 5 | # Copyright (c) 2022 David Schall and EASE lab 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | 25 | ## Define the image name of your function. 26 | FUNCTION_NAME=<__FUNCTION_NAME__> 27 | 28 | # We use the fail code magic instruction to indicate the 29 | # python script where in test process we are. 30 | 31 | m5 fail 1 ## 1: BOOTING complete 32 | 33 | 34 | ## Spin up Container 35 | echo "Start the container..." 36 | docker-compose -f functions.yaml up -d ${FUNCTION_NAME} &&DOCKER_START_RES=$? 37 | m5 fail 2 ## 2: Started container 38 | 39 | echo "Pin to core 1" 40 | docker update ${FUNCTION_NAME} --cpuset-cpus 1 41 | 42 | sleep 5 43 | m5 fail 3 ## 3: Pinned container 44 | 45 | 46 | ## Now start the invoker 47 | # Modify the invoker parameters depending on your need. 48 | # /root/test-client -addr localhost:50051 -n 20 && INVOKER_RES=$? 49 | /root/test-client \ 50 | -function-name ${FUNCTION_NAME} \ 51 | -url localhost \ 52 | -port 50000 \ 53 | -n 5 -input 1 \ 54 | && INVOKER_RES=$? 55 | 56 | m5 fail 4 ## 4: Invoking done 57 | 58 | ## Stop container 59 | docker-compose -f functions.yaml down && DOCKER_STOP_RES=$? 60 | m5 fail 5 ## 5: Container stop 61 | 62 | 63 | if [ $INVOKER_RES ] && [ $DOCKER_START_RES ] && [ $DOCKER_STOP_RES ] ; then 64 | echo "SUCCESS: All commands completed successfully" 65 | else 66 | echo "FAIL: Commands failed" 67 | fi 68 | 69 | ## M5 fail -1 will exit the simulations 70 | m5 fail -1 ## 5: Test done 71 | -------------------------------------------------------------------------------- /simulation/wkdir-tmpl/setup_all_functions.tmpl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # MIT License 4 | # 5 | # Copyright (c) 2022 David Schall and EASE lab 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | 25 | # Execute this script using 26 | # ./setup_all_functions.sh 27 | 28 | set -x 29 | 30 | GEM5=<__GEM5__> 31 | KERNEL=kernel 32 | DISK_IMAGE=disk.img 33 | GEM5_CONFIG=<__GEM5_CONFIG__> 34 | 35 | 36 | 37 | ################################################################################ 38 | sudo chown $USER /dev/kvm 39 | 40 | 41 | ## Parse the functions.list file 42 | ## Functions can be commented out with '#' 43 | FUNCTIONS=$(cat functions.list | sed '/^\s*#/d;/^\s*$/d') 44 | 45 | 46 | # Define the output file of your run 47 | RESULTS_DIR=../results/setup 48 | 49 | 50 | for fn in $FUNCTIONS ; 51 | do 52 | OUTDIR=$RESULTS_DIR/${fn}/ 53 | 54 | ## Create output directory 55 | mkdir -p $OUTDIR 56 | 57 | sudo $GEM5 \ 58 | --outdir=$OUTDIR \ 59 | $GEM5_CONFIG \ 60 | --kernel $KERNEL \ 61 | --disk $DISK_IMAGE \ 62 | --function ${fn} \ 63 | --mode=setup \ 64 | > $OUTDIR/gem5.log 2>&1 \ 65 | & 66 | 67 | done 68 | -------------------------------------------------------------------------------- /simulation/wkdir-tmpl/setup_function.tmpl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # MIT License 4 | # 5 | # Copyright (c) 2022 David Schall and EASE lab 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | 25 | # Execute this script using 26 | # ./sim_function.sh 27 | 28 | 29 | set -x 30 | 31 | GEM5=<__GEM5__> 32 | KERNEL=kernel 33 | DISK_IMAGE=disk.img 34 | GEM5_CONFIG=<__GEM5_CONFIG__> 35 | 36 | 37 | 38 | ################################################################################ 39 | 40 | sudo chown $USER /dev/kvm 41 | 42 | # Define the results directory and the function for your run. 43 | FN=${1:-fibonacci-go} 44 | RESULTS_DIR=${2:-results} 45 | 46 | 47 | OUTDIR=$RESULTS_DIR/${FN}/ 48 | ## Create output directory 49 | mkdir -p $OUTDIR 50 | 51 | sudo $GEM5 \ 52 | --outdir=$OUTDIR \ 53 | $GEM5_CONFIG \ 54 | --kernel $KERNEL \ 55 | --disk $DISK_IMAGE \ 56 | --function ${FN} \ 57 | --mode=setup 58 | -------------------------------------------------------------------------------- /simulation/wkdir-tmpl/sim_all_functions.tmpl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # MIT License 4 | # 5 | # Copyright (c) 2022 David Schall and EASE lab 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | 25 | # Execute this script using 26 | # ./sim_all_functions.sh 27 | 28 | set -x 29 | 30 | GEM5=<__GEM5__> 31 | KERNEL=kernel 32 | DISK_IMAGE=disk.img 33 | GEM5_CONFIG=<__GEM5_CONFIG__> 34 | 35 | 36 | 37 | ################################################################################ 38 | sudo chown $USER /dev/kvm 39 | 40 | 41 | ## Parse the functions.list file 42 | ## Functions can be commented out with '#' 43 | FUNCTIONS=$(cat functions.list | sed '/^\s*#/d;/^\s*$/d') 44 | 45 | 46 | # Define the output file of your run 47 | RESULTS_DIR=${1:-results} 48 | 49 | 50 | for fn in $FUNCTIONS ; 51 | do 52 | OUTDIR=$RESULTS_DIR/${fn}/ 53 | 54 | ## Create output directory 55 | mkdir -p $OUTDIR 56 | 57 | sudo $GEM5 \ 58 | --outdir=$OUTDIR \ 59 | $GEM5_CONFIG \ 60 | --kernel $KERNEL \ 61 | --disk $DISK_IMAGE \ 62 | --function ${fn} \ 63 | --mode=evaluation \ 64 | --atomic-warming 10 \ 65 | --num-invocation 10 \ 66 | > $OUTDIR/gem5.log 2>&1 \ 67 | & 68 | 69 | done 70 | -------------------------------------------------------------------------------- /simulation/wkdir-tmpl/sim_function.tmpl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # MIT License 4 | # 5 | # Copyright (c) 2022 David Schall and EASE lab 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | 25 | # Execute this script using 26 | # ./sim_function.sh 27 | 28 | 29 | set -x 30 | 31 | GEM5=<__GEM5__> 32 | KERNEL=kernel 33 | DISK_IMAGE=disk.img 34 | GEM5_CONFIG=<__GEM5_CONFIG__> 35 | 36 | 37 | 38 | ################################################################################ 39 | 40 | sudo chown $USER /dev/kvm 41 | 42 | # Define the results directory and the function for your run. 43 | FN=${1:-fibonacci-go} 44 | RESULTS_DIR=${2:-results} 45 | 46 | 47 | OUTDIR=$RESULTS_DIR/${FN}/ 48 | ## Create output directory 49 | mkdir -p $OUTDIR 50 | 51 | sudo $GEM5 \ 52 | --outdir=$OUTDIR \ 53 | $GEM5_CONFIG \ 54 | --kernel $KERNEL \ 55 | --disk $DISK_IMAGE \ 56 | --function ${FN} \ 57 | --mode=evaluation \ 58 | --atomic-warming 10 \ 59 | --num-invocation 10 60 | -------------------------------------------------------------------------------- /test/run_emu_test.template.sh: -------------------------------------------------------------------------------- 1 | 2 | #!/bin/bash 3 | 4 | # MIT License 5 | # 6 | # Copyright (c) 2022 EASE lab, University of Edinburgh 7 | # 8 | # Permission is hereby granted, free of charge, to any person obtaining a copy 9 | # of this software and associated documentation files (the "Software"), to deal 10 | # in the Software without restriction, including without limitation the rights 11 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | # copies of the Software, and to permit persons to whom the Software is 13 | # furnished to do so, subject to the following conditions: 14 | # 15 | # The above copyright notice and this permission notice shall be included in all 16 | # copies or substantial portions of the Software. 17 | # 18 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | # SOFTWARE. 25 | # 26 | # Authors: David Schall 27 | 28 | 29 | LOGFILE=/root/test.log 30 | 31 | function start_logging { 32 | ## Log everything to a empty log file 33 | [ -e $LOGFILE ] && rm -- $LOGFILE 34 | touch $LOGFILE 35 | exec 3>&1 4>&2 36 | trap 'exec 2>&4 1>&3' 0 1 2 3 37 | exec 1> $LOGFILE 2>&1 38 | } 39 | function end_logging { 40 | # Restore file descriptors 41 | exec 2>&4 1>&3 42 | } 43 | 44 | 45 | 46 | 47 | function pull_test_function { 48 | FUNCTION_NAME=$1 49 | 50 | echo "Test: ${FUNCTION_NAME} " 51 | 52 | ## Pull the image from regestry 53 | docker compose -f /root/functions.yaml pull ${FUNCTION_NAME} 54 | docker compose -f /root/functions.yaml up -d --remove-orphans ${FUNCTION_NAME} && DOCKER_START_RES=$? 55 | 56 | sleep 5 57 | 58 | ## Now start the invoker 59 | # Modify the invoker parameters depending on your need. 60 | # /root/test-client -addr localhost:50051 -n 20 && INVOKER_RES=$? 61 | /root/test-client \ 62 | -function-name ${FUNCTION_NAME} \ 63 | -function-method 0 \ 64 | -url localhost \ 65 | -port 50000 \ 66 | -n 20 -input 1 \ 67 | && INVOKER_RES=$? 68 | 69 | ## Stop container 70 | docker compose -f /root/functions.yaml down && DOCKER_STOP_RES=$? 71 | CONTAINERS="$(docker ps -a -q)" 72 | if [ $(expr length "$CONTAINERS") -gt 0 ]; 73 | then 74 | docker stop $CONTAINERS 75 | docker rm $CONTAINERS 76 | fi 77 | 78 | if [ $INVOKER_RES ] && [ $DOCKER_START_RES ] && [ $DOCKER_STOP_RES ] ; then 79 | echo "SUCCESS: All commands completed successfully" 80 | else 81 | echo "FAIL: Commands failed" 82 | fi 83 | } 84 | 85 | 86 | start_logging 87 | { 88 | # set -e 89 | ## Download the test client. 90 | curl "http://10.0.2.2:3003/test-client" -f -o /root/test-client 91 | chmod 755 /root/test-client 92 | 93 | ## Download the function yaml and list. 94 | curl "http://10.0.2.2:3003/functions.yaml" -f -o /root/functions.yaml 95 | curl "http://10.0.2.2:3003/functions.list" -f -o /root/functions.list 96 | 97 | 98 | # docker-compose -f functions.yaml pull 99 | 100 | ## List all functions, can be commented out 101 | FUNCTIONS=$(cat /root/functions.list | sed '/^\s*#/d;/^\s*$/d') 102 | 103 | for f in $FUNCTIONS 104 | do 105 | pull_test_function $f 106 | done 107 | 108 | ## Catch for failiure ---------- 109 | } || { 110 | echo "\033[0;31m----------------" 111 | echo "FAIL" 112 | echo "----------------\033[0m" 113 | cat ${LOGFILE} 114 | } 115 | # set +e 116 | end_logging 117 | 118 | ## Upload the log file. 119 | curl "http://10.0.2.2:3003/upload" -F "files=@${LOGFILE}" 120 | 121 | shutdown -h now 122 | -------------------------------------------------------------------------------- /tools/client/Makefile: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # MIT License 4 | # 5 | # Copyright (c) 2022 EASE lab, University of Edinburgh 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | # 25 | # Authors: David Schall 26 | 27 | MKFILE := $(abspath $(lastword $(MAKEFILE_LIST))) 28 | ROOT := $(abspath $(dir $(MKFILE))/../../) 29 | OUTPUT ?= test-client 30 | ARCH ?= amd64 31 | 32 | # Go parameters 33 | GOCMD=$(shell which go) 34 | GOCMD=/usr/local/go/bin/go 35 | 36 | .PHONY: m5ops 37 | .PHONY: clean 38 | 39 | SRC=m5ops.go 40 | BIN=client 41 | 42 | test: build 43 | $(GOCMD) test 44 | 45 | 46 | dep_install: 47 | sudo apt-get update \ 48 | && sudo apt-get install -y \ 49 | gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu device-tree-compiler 50 | 51 | 52 | ## Configure the m5 address depending on the architecture 53 | ifeq ($(ARCH), arm64) 54 | M5OP_ADDR=0x10010000 55 | else 56 | M5OP_ADDR=0xffff0000 57 | endif 58 | 59 | ## Assembly files 60 | M5OP_S := $(DIR)/m5op.S 61 | $(M5OP_S): 62 | cp $(DIR)/$(ARCH)/* $(DIR) 63 | 64 | 65 | ## m5 ops library ------------------------- 66 | CC = gcc 67 | AR = ar 68 | RANLIB = ranlib 69 | ifeq ($(ARCH), arm64) 70 | ifeq ($(shell uname -i),x86_64) 71 | CC = aarch64-linux-gnu-gcc 72 | AR = aarch64-linux-gnu-ar 73 | RANLIB = aarch64-linux-gnu-ranlib 74 | endif 75 | endif 76 | 77 | CFLAGS = -g -fPIC 78 | DIR = m5 79 | 80 | SFILES :=$(DIR)/m5op.S $(DIR)/m5op_addr.S 81 | OBJFILES := $(DIR)/m5_mmap.o $(SFILES:.S=.o) 82 | 83 | 84 | %.o : %.S 85 | $(CC) -o $@ $< -c -g -fPIC 86 | 87 | $(DIR)/m5_mmap.o : $(DIR)/m5_mmap.c 88 | sed -i 's/__M5_ADDR__/$(M5OP_ADDR)/g' $< 89 | $(CC) -o $@ $< -c -g -fPIC 90 | 91 | m5ops: $(OBJFILES) 92 | $(AR) rc -o $(DIR)/lib$@.a $^ 93 | $(RANLIB) $(DIR)/lib$@.a 94 | 95 | m5lib: $(M5OP_S) m5ops 96 | ## ---------------------------------------- 97 | 98 | 99 | 100 | _CGO_LDFLAGS="-g -O2 -no-pie" 101 | 102 | $(BIN): main.go version.go m5lib 103 | CC=$(CC) CGO_LDFLAGS=$(_CGO_LDFLAGS) CGO_ENABLED=1 GOOS=linux GOARCH=$(ARCH) go build -o $@ -v . 104 | 105 | all: $(BIN) 106 | 107 | libclean: 108 | rm $(DIR)/*.o 109 | rm $(DIR)/*.a 110 | rm $(DIR)/*.S 111 | 112 | clean: libclean 113 | go clean 114 | 115 | 116 | save_output: all 117 | cp $(BIN) $(OUTPUT) 118 | 119 | -------------------------------------------------------------------------------- /tools/client/go.mod: -------------------------------------------------------------------------------- 1 | module client 2 | 3 | go 1.21 4 | toolchain go1.23.7 5 | 6 | require ( 7 | github.com/sirupsen/logrus v1.9.3 8 | github.com/vhive-serverless/vSwarm-proto v0.5.4 9 | ) 10 | 11 | require ( 12 | github.com/go-logr/logr v1.4.1 // indirect 13 | github.com/go-logr/stdr v1.2.2 // indirect 14 | github.com/openzipkin/zipkin-go v0.4.2 // indirect 15 | github.com/vhive-serverless/vSwarm/utils/tracing/go v0.0.0-20240130084512-c5ae2f26d045 // indirect 16 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect 17 | go.opentelemetry.io/otel v1.22.0 // indirect 18 | go.opentelemetry.io/otel/exporters/zipkin v1.22.0 // indirect 19 | go.opentelemetry.io/otel/metric v1.22.0 // indirect 20 | go.opentelemetry.io/otel/sdk v1.22.0 // indirect 21 | go.opentelemetry.io/otel/trace v1.22.0 // indirect 22 | golang.org/x/net v0.36.0 // indirect 23 | golang.org/x/sys v0.30.0 // indirect 24 | golang.org/x/text v0.22.0 // indirect 25 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect 26 | google.golang.org/grpc v1.64.1 // indirect 27 | google.golang.org/protobuf v1.34.2 // indirect 28 | ) 29 | -------------------------------------------------------------------------------- /tools/client/m5/amd64/m5op.S: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2003-2006 The Regents of The University of Michigan 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are 7 | * met: redistributions of source code must retain the above copyright 8 | * notice, this list of conditions and the following disclaimer; 9 | * redistributions in binary form must reproduce the above copyright 10 | * notice, this list of conditions and the following disclaimer in the 11 | * documentation and/or other materials provided with the distribution; 12 | * neither the name of the copyright holders nor the names of its 13 | * contributors may be used to endorse or promote products derived from 14 | * this software without specific prior written permission. 15 | * 16 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | */ 28 | 29 | ; #include 30 | #include "m5ops_generic.h" 31 | 32 | .macro m5op_func, name, func 33 | .globl \name 34 | .func \name 35 | \name: 36 | .byte 0x0F, 0x04 37 | .word \func 38 | ret 39 | .endfunc 40 | .endm 41 | 42 | .text 43 | 44 | #define M5OP(name, func) m5op_func name, func; 45 | M5OP_FOREACH 46 | #undef M5OP 47 | -------------------------------------------------------------------------------- /tools/client/m5/amd64/m5op_addr.S: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2003-2006 The Regents of The University of Michigan 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are 7 | * met: redistributions of source code must retain the above copyright 8 | * notice, this list of conditions and the following disclaimer; 9 | * redistributions in binary form must reproduce the above copyright 10 | * notice, this list of conditions and the following disclaimer in the 11 | * documentation and/or other materials provided with the distribution; 12 | * neither the name of the copyright holders nor the names of its 13 | * contributors may be used to endorse or promote products derived from 14 | * this software without specific prior written permission. 15 | * 16 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | */ 28 | 29 | ; #include 30 | #include "m5ops_generic.h" 31 | 32 | /* 33 | * Note: The ABI for pseudo ops using the M5OP_ADDR is defined in 34 | * src/arch/x86/pseudo_inst_abi.hh. If the ABI is changed below, it's likely 35 | * that the ABI in the arch directory will also need to be updated. 36 | */ 37 | 38 | .macro m5op_func, name, func 39 | .globl \name 40 | .func \name 41 | \name: 42 | #if defined(M5OP_PIC) 43 | mov m5_mem@gotpcrel(%rip), %r11 44 | mov (%r11), %r11 45 | #else 46 | mov m5_mem, %r11 47 | #endif 48 | mov $\func, %rax 49 | shl $8, %rax 50 | mov 0(%r11, %rax, 1), %rax 51 | ret 52 | .endfunc 53 | .endm 54 | 55 | .text 56 | #define M5OP(name, func) m5op_func M5OP_MERGE_TOKENS(name, _addr), func; 57 | M5OP_FOREACH 58 | #undef M5OP 59 | -------------------------------------------------------------------------------- /tools/client/m5/arm64/m5op.S: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2013, 2016-2017 ARM Limited 3 | * All rights reserved 4 | * 5 | * The license below extends only to copyright in the software and shall 6 | * not be construed as granting a license to any other intellectual 7 | * property including but not limited to intellectual property relating 8 | * to a hardware implementation of the functionality of the software 9 | * licensed hereunder. You may use the software subject to the license 10 | * terms below provided that you ensure that this notice is replicated 11 | * unmodified and in its entirety in all distributions of the software, 12 | * modified or unmodified, in source code or in binary form. 13 | * 14 | * Copyright (c) 2003-2006 The Regents of The University of Michigan 15 | * All rights reserved. 16 | * 17 | * Redistribution and use in source and binary forms, with or without 18 | * modification, are permitted provided that the following conditions are 19 | * met: redistributions of source code must retain the above copyright 20 | * notice, this list of conditions and the following disclaimer; 21 | * redistributions in binary form must reproduce the above copyright 22 | * notice, this list of conditions and the following disclaimer in the 23 | * documentation and/or other materials provided with the distribution; 24 | * neither the name of the copyright holders nor the names of its 25 | * contributors may be used to endorse or promote products derived from 26 | * this software without specific prior written permission. 27 | * 28 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 29 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 30 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 31 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 32 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 33 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 34 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 35 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 36 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 37 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 38 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 39 | */ 40 | 41 | ; #include 42 | #include "m5ops_generic.h" 43 | 44 | .macro m5op_func, name, func 45 | .globl \name 46 | \name: 47 | .long 0xff000110 | (\func << 16) 48 | ret 49 | .endm 50 | 51 | .text 52 | #define M5OP(name, func) m5op_func name, func; 53 | M5OP_FOREACH 54 | #undef M5OP 55 | -------------------------------------------------------------------------------- /tools/client/m5/arm64/m5op_addr.S: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2013, 2016-2017 ARM Limited 3 | * All rights reserved 4 | * 5 | * The license below extends only to copyright in the software and shall 6 | * not be construed as granting a license to any other intellectual 7 | * property including but not limited to intellectual property relating 8 | * to a hardware implementation of the functionality of the software 9 | * licensed hereunder. You may use the software subject to the license 10 | * terms below provided that you ensure that this notice is replicated 11 | * unmodified and in its entirety in all distributions of the software, 12 | * modified or unmodified, in source code or in binary form. 13 | * 14 | * Copyright (c) 2003-2006 The Regents of The University of Michigan 15 | * All rights reserved. 16 | * 17 | * Redistribution and use in source and binary forms, with or without 18 | * modification, are permitted provided that the following conditions are 19 | * met: redistributions of source code must retain the above copyright 20 | * notice, this list of conditions and the following disclaimer; 21 | * redistributions in binary form must reproduce the above copyright 22 | * notice, this list of conditions and the following disclaimer in the 23 | * documentation and/or other materials provided with the distribution; 24 | * neither the name of the copyright holders nor the names of its 25 | * contributors may be used to endorse or promote products derived from 26 | * this software without specific prior written permission. 27 | * 28 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 29 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 30 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 31 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 32 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 33 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 34 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 35 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 36 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 37 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 38 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 39 | */ 40 | 41 | ; #include 42 | #include "m5ops_generic.h" 43 | 44 | .macro m5op_func, name, func 45 | .globl \name 46 | \name: 47 | // Load the value of m5_mem into x9... 48 | #if defined(M5OP_PIC) 49 | // using the global offset table. 50 | adrp x9, :got:m5_mem 51 | ldr x9, [ x9, #:got_lo12:m5_mem ] 52 | ldr x9, [ x9 ] 53 | #else 54 | // normally. 55 | adrp x9, m5_mem 56 | ldr x9, [ x9, #:lo12:m5_mem ] 57 | #endif 58 | movz x10, #(\func << 8) 59 | ldr x0, [ x9, x10 ] 60 | ret 61 | .endm 62 | 63 | .text 64 | #define M5OP(name, func) m5op_func M5OP_MERGE_TOKENS(name, _addr), func; 65 | M5OP_FOREACH 66 | #undef M5OP 67 | -------------------------------------------------------------------------------- /tools/client/m5/m5_mmap.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, 2017 ARM Limited 3 | * All rights reserved 4 | * 5 | * The license below extends only to copyright in the software and shall 6 | * not be construed as granting a license to any other intellectual 7 | * property including but not limited to intellectual property relating 8 | * to a hardware implementation of the functionality of the software 9 | * licensed hereunder. You may use the software subject to the license 10 | * terms below provided that you ensure that this notice is replicated 11 | * unmodified and in its entirety in all distributions of the software, 12 | * modified or unmodified, in source code or in binary form. 13 | * 14 | * Copyright (c) 2003-2005 The Regents of The University of Michigan 15 | * All rights reserved. 16 | * 17 | * Redistribution and use in source and binary forms, with or without 18 | * modification, are permitted provided that the following conditions are 19 | * met: redistributions of source code must retain the above copyright 20 | * notice, this list of conditions and the following disclaimer; 21 | * redistributions in binary form must reproduce the above copyright 22 | * notice, this list of conditions and the following disclaimer in the 23 | * documentation and/or other materials provided with the distribution; 24 | * neither the name of the copyright holders nor the names of its 25 | * contributors may be used to endorse or promote products derived from 26 | * this software without specific prior written permission. 27 | * 28 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 29 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 30 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 31 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 32 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 33 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 34 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 35 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 36 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 37 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 38 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 39 | */ 40 | 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | #include 49 | #include 50 | 51 | #include "m5_mmap.h" 52 | 53 | void *m5_mem = NULL; 54 | 55 | #ifndef M5OP_ADDR 56 | // #define M5OP_ADDR 0 57 | #define M5OP_ADDR __M5_ADDR__ 58 | #endif 59 | uint64_t m5op_addr = M5OP_ADDR; 60 | 61 | uint64_t m5_addr() { 62 | return m5op_addr; 63 | } 64 | 65 | const char *m5_mmap_dev = "/dev/mem"; 66 | 67 | void 68 | map_m5_mem() 69 | { 70 | int fd; 71 | 72 | if (m5_mem) { 73 | fprintf(stderr, "m5 mem already mapped.\n"); 74 | exit(1); 75 | } 76 | 77 | fd = open(m5_mmap_dev, O_RDWR | O_SYNC); 78 | if (fd == -1) { 79 | fprintf(stderr, "Can't open %s: %s\n", m5_mmap_dev, strerror(errno)); 80 | exit(1); 81 | } 82 | 83 | m5_mem = mmap(NULL, 0x10000, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 84 | m5op_addr); 85 | close(fd); 86 | 87 | if (!m5_mem) { 88 | fprintf(stderr, "Can't map %s: %s\n", m5_mmap_dev, strerror(errno)); 89 | exit(1); 90 | } 91 | } 92 | 93 | void 94 | unmap_m5_mem() 95 | { 96 | if (m5_mem) { 97 | munmap(m5_mem, 0x10000); 98 | m5_mem = NULL; 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /tools/client/m5/m5_mmap.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, 2017 ARM Limited 3 | * All rights reserved 4 | * 5 | * The license below extends only to copyright in the software and shall 6 | * not be construed as granting a license to any other intellectual 7 | * property including but not limited to intellectual property relating 8 | * to a hardware implementation of the functionality of the software 9 | * licensed hereunder. You may use the software subject to the license 10 | * terms below provided that you ensure that this notice is replicated 11 | * unmodified and in its entirety in all distributions of the software, 12 | * modified or unmodified, in source code or in binary form. 13 | * 14 | * Copyright (c) 2003-2005 The Regents of The University of Michigan 15 | * All rights reserved. 16 | * 17 | * Redistribution and use in source and binary forms, with or without 18 | * modification, are permitted provided that the following conditions are 19 | * met: redistributions of source code must retain the above copyright 20 | * notice, this list of conditions and the following disclaimer; 21 | * redistributions in binary form must reproduce the above copyright 22 | * notice, this list of conditions and the following disclaimer in the 23 | * documentation and/or other materials provided with the distribution; 24 | * neither the name of the copyright holders nor the names of its 25 | * contributors may be used to endorse or promote products derived from 26 | * this software without specific prior written permission. 27 | * 28 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 29 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 30 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 31 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 32 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 33 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 34 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 35 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 36 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 37 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 38 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 39 | */ 40 | 41 | #ifndef __UTIL_M5_MMAP_H__ 42 | #define __UTIL_M5_MMAP_H__ 43 | 44 | #include 45 | 46 | #ifdef __cplusplus 47 | extern "C" { 48 | #endif 49 | 50 | extern void *m5_mem; 51 | extern uint64_t m5op_addr; 52 | extern const char *m5_mmap_dev; 53 | void map_m5_mem(); 54 | void unmap_m5_mem(); 55 | uint64_t m5_addr(); 56 | 57 | #ifdef __cplusplus 58 | } 59 | #endif 60 | 61 | #endif // __UTIL_M5_MMAP_H__ 62 | -------------------------------------------------------------------------------- /tools/client/m5/m5ops.go: -------------------------------------------------------------------------------- 1 | package m5ops 2 | 3 | /* 4 | #cgo CFLAGS: -g 5 | #cgo amd64 LDFLAGS: -L. -lm5ops 6 | #cgo arm64 LDFLAGS: -L. -lm5ops 7 | #include "m5ops.h" 8 | #include "m5_mmap.h" 9 | */ 10 | import "C" 11 | import ( 12 | "os/user" 13 | 14 | log "github.com/sirupsen/logrus" 15 | ) 16 | 17 | type M5Ops struct { 18 | // init bool 19 | mapped bool 20 | } 21 | 22 | func NewM5Ops() M5Ops { 23 | op := M5Ops{} 24 | op.init() 25 | return op 26 | } 27 | 28 | func isRoot() bool { 29 | currentUser, err := user.Current() 30 | if err != nil { 31 | log.Warn("Cannot get current user") 32 | } 33 | return currentUser.Uid == "0" 34 | } 35 | 36 | func (op *M5Ops) init() { 37 | if !isRoot() { 38 | log.Warn("M5 magic instructions require root access") 39 | log.Warn("Will not be enabled!!!") 40 | return 41 | } 42 | log.Printf("Initialize M5 magic instructions. M5Ops Addr:%#x\n", C.m5op_addr) 43 | C.map_m5_mem() 44 | op.mapped = true 45 | } 46 | 47 | func (op *M5Ops) Close() { 48 | if op.mapped { 49 | C.unmap_m5_mem() 50 | op.mapped = false 51 | } 52 | } 53 | 54 | func (op M5Ops) Exit(delay int) { 55 | if !op.mapped { 56 | log.Warn("M5 disabled") 57 | return 58 | } 59 | C.m5_exit_addr(C.ulong(delay)) 60 | } 61 | 62 | func (op M5Ops) Fail(delay, code int) { 63 | if !op.mapped { 64 | log.Warn("M5 disabled") 65 | return 66 | } 67 | C.m5_fail_addr(C.ulong(delay), C.ulong(code)) 68 | } 69 | 70 | func (op M5Ops) DumpStats(delay, period int) { 71 | if !op.mapped { 72 | return 73 | } 74 | C.m5_dump_stats_addr(C.ulong(delay), C.ulong(period)) 75 | } 76 | 77 | func (op M5Ops) ResetStats(delay, period int) { 78 | if !op.mapped { 79 | return 80 | } 81 | C.m5_reset_stats_addr(C.ulong(delay), C.ulong(period)) 82 | } 83 | 84 | func (op M5Ops) DumpResetStats(delay, period int) { 85 | if !op.mapped { 86 | return 87 | } 88 | C.m5_dump_reset_stats_addr(C.ulong(delay), C.ulong(period)) 89 | } 90 | func (op M5Ops) WorkBegin(workid, threadid int) { 91 | if !op.mapped { 92 | return 93 | } 94 | C.m5_work_begin_addr(C.ulong(workid), C.ulong(threadid)) 95 | } 96 | 97 | func (op M5Ops) WorkEnd(workid, threadid int) { 98 | if !op.mapped { 99 | return 100 | } 101 | C.m5_work_end_addr(C.ulong(workid), C.ulong(threadid)) 102 | } 103 | 104 | func (op M5Ops) Workload() { 105 | if !op.mapped { 106 | return 107 | } 108 | C.m5_workload_addr() 109 | } 110 | -------------------------------------------------------------------------------- /tools/client/m5/m5ops.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2003-2006 The Regents of The University of Michigan 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are 7 | * met: redistributions of source code must retain the above copyright 8 | * notice, this list of conditions and the following disclaimer; 9 | * redistributions in binary form must reproduce the above copyright 10 | * notice, this list of conditions and the following disclaimer in the 11 | * documentation and/or other materials provided with the distribution; 12 | * neither the name of the copyright holders nor the names of its 13 | * contributors may be used to endorse or promote products derived from 14 | * this software without specific prior written permission. 15 | * 16 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | */ 28 | 29 | #ifndef __GEM5_M5OP_H__ 30 | #define __GEM5_M5OP_H__ 31 | 32 | #ifdef __cplusplus 33 | extern "C" { 34 | #endif 35 | 36 | #include 37 | 38 | #include "m5ops_generic.h" 39 | 40 | void m5_arm(uint64_t address); 41 | void m5_quiesce(void); 42 | void m5_quiesce_ns(uint64_t ns); 43 | void m5_quiesce_cycle(uint64_t cycles); 44 | uint64_t m5_quiesce_time(void); 45 | uint64_t m5_rpns(); 46 | void m5_wake_cpu(uint64_t cpuid); 47 | 48 | void m5_exit(uint64_t ns_delay); 49 | void m5_fail(uint64_t ns_delay, uint64_t code); 50 | // m5_sum is for sanity checking the gem5 op interface. 51 | unsigned m5_sum(unsigned a, unsigned b, unsigned c, 52 | unsigned d, unsigned e, unsigned f); 53 | uint64_t m5_init_param(uint64_t key_str1, uint64_t key_str2); 54 | void m5_checkpoint(uint64_t ns_delay, uint64_t ns_period); 55 | void m5_reset_stats(uint64_t ns_delay, uint64_t ns_period); 56 | void m5_dump_stats(uint64_t ns_delay, uint64_t ns_period); 57 | void m5_dump_reset_stats(uint64_t ns_delay, uint64_t ns_period); 58 | uint64_t m5_read_file(void *buffer, uint64_t len, uint64_t offset); 59 | uint64_t m5_write_file(void *buffer, uint64_t len, uint64_t offset, 60 | const char *filename); 61 | void m5_debug_break(void); 62 | void m5_switch_cpu(void); 63 | void m5_dist_toggle_sync(void); 64 | void m5_add_symbol(uint64_t addr, const char *symbol); 65 | void m5_load_symbol(); 66 | void m5_panic(void); 67 | void m5_work_begin(uint64_t workid, uint64_t threadid); 68 | void m5_work_end(uint64_t workid, uint64_t threadid); 69 | 70 | /* 71 | * Send a very generic poke to the workload so it can do something. It's up to 72 | * the workload to know what information to look for to interpret an event, 73 | * such as what PC it came from, what register values are, or the context of 74 | * the workload itself (is this SE mode? which OS is running?). 75 | */ 76 | void m5_workload(); 77 | 78 | /* 79 | * Create _addr and _semi versions all declarations, e.g. m5_exit_addr and 80 | * m5_exit_semi. These expose the the memory and semihosting variants of the 81 | * ops. 82 | * 83 | * Some of those declarations are not defined for certain ISAs, e.g. X86 84 | * does not have _semi, but we felt that ifdefing them out could cause more 85 | * trouble tham leaving them in. 86 | */ 87 | #define M5OP(name, func) __typeof__(name) M5OP_MERGE_TOKENS(name, _addr); \ 88 | __typeof__(name) M5OP_MERGE_TOKENS(name, _semi); 89 | M5OP_FOREACH 90 | #undef M5OP 91 | 92 | #ifdef __cplusplus 93 | } 94 | #endif 95 | #endif // __GEM5_M5OP_H__ 96 | -------------------------------------------------------------------------------- /tools/client/version.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | const version = "v0.1.3-LOCAL" 4 | --------------------------------------------------------------------------------