├── .clang-format ├── .flake8 ├── .github ├── linters │ ├── .arm-ttk.psd1 │ └── .jscpd.json └── workflows │ ├── auto-pr.yml │ ├── docs.yml │ └── main.yml ├── .gitignore ├── .gitmodules ├── .pre-commit-config.yaml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── README.md ├── client-docs ├── Makefile ├── README.md ├── cli │ ├── api.rst │ ├── index.rst │ └── usage.rst ├── conf.py ├── config │ ├── azure.rst │ ├── config.rst │ └── index.rst ├── img │ ├── logo.png │ └── mc2_workflow.jpeg ├── index.rst ├── install.rst ├── make.bat ├── opaquesql_usage.rst ├── python │ ├── api.rst │ ├── index.rst │ └── usage.rst ├── quickstart.rst ├── requirements.txt └── spelling_wordlist.txt ├── demo ├── azure.yaml ├── config.yaml ├── data │ ├── opaquesql.csv │ ├── opaquesql_schema.json │ ├── securexgb_test.csv │ └── securexgb_train.csv ├── keys │ ├── README.md │ ├── root.crt │ ├── root.pem │ ├── root.pub │ ├── user1.crt │ ├── user1.pem │ ├── user1.pub │ └── user1_sym.key ├── opaque_sql_demo.scala └── opaquesql │ ├── data │ ├── opaquesql.csv │ └── opaquesql_schema.json │ └── opaque_sql_demo.scala ├── mc2.py ├── mc2_client_env ├── pyproject.toml ├── python-package ├── mc2client │ ├── __init__.py │ ├── cache.py │ ├── core.py │ ├── exceptions.py │ ├── opaquesql │ │ ├── __init__.py │ │ └── opaquesql.py │ ├── rpc │ │ ├── README.md │ │ ├── __init__.py │ │ └── protos │ │ │ ├── attest.proto │ │ │ ├── ndarray.proto │ │ │ ├── opaquesql.proto │ │ │ └── remote.proto │ ├── toolchain │ │ ├── __init__.py │ │ ├── encrypt_and_upload.py │ │ ├── flatbuffers │ │ │ └── __init__.py │ │ ├── log_timer.py │ │ ├── mc2-schema.json │ │ ├── mc2_azure │ │ │ ├── __init__.py │ │ │ ├── azure-config-template.json │ │ │ ├── azure-vm-template.json │ │ │ ├── config.py │ │ │ ├── example-full.yaml │ │ │ └── node_provider.py │ │ ├── node_provider.py │ │ ├── scripts │ │ │ ├── build_opaque.sh │ │ │ ├── build_spark.sh │ │ │ ├── install_oe.sh │ │ │ ├── install_secure_xgboost.sh │ │ │ └── install_spark.sh │ │ ├── tags.py │ │ ├── toolchain.py │ │ ├── updater.py │ │ └── util.py │ └── xgb │ │ ├── __init__.py │ │ ├── rabit.py │ │ └── securexgboost.py ├── setup.py └── tests │ ├── __init__.py │ ├── azure │ ├── create_azure_resources.py │ ├── delete_azure_resources.py │ └── dummy.txt │ ├── data │ ├── agaricus.txt.test │ ├── agaricus.txt.train │ ├── expected_booster.dump │ ├── test_data.csv │ └── test_data.schema │ ├── keys │ ├── mc2_test_signing_key.pub │ ├── root.crt │ ├── root.pem │ ├── root.pub │ ├── user1.crt │ ├── user1.pem │ ├── user1.pub │ ├── user1_sym.key │ ├── user2.pub │ └── user3.pub │ ├── opaquesql │ ├── opaquesql_example.scala │ └── to_test_opaquesql.py │ ├── test.yaml │ ├── test_crypto.py │ └── to_test_securexgboost.py ├── quickstart ├── azure.yaml ├── config.yaml ├── data │ ├── opaquesql.csv │ └── opaquesql_schema.json ├── keys │ ├── root.crt │ ├── root.pem │ └── root.pub ├── opaque_sql_demo.py └── opaque_sql_demo.scala ├── requirements.txt └── src ├── CMakeLists.txt ├── c_api.cpp ├── context.h ├── include ├── base64.h ├── csv.hpp └── json.hpp ├── io.cpp ├── io.h └── utils.h /.clang-format: -------------------------------------------------------------------------------- 1 | MaxEmptyLinesToKeep: 2 2 | IndentWidth: 4 3 | NamespaceIndentation: All 4 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E203, E266, E501, W503, W605, F403, F401, C901 3 | max-line-length = 79 4 | max-complexity = 18 5 | select = B,C,E,F,W,T4,B9 6 | -------------------------------------------------------------------------------- /.github/linters/.arm-ttk.psd1: -------------------------------------------------------------------------------- 1 | # Documentation: 2 | # - Test Parameters: https://docs.microsoft.com/en-us/azure/azure-resource-manager/templates/test-toolkit#test-parameters 3 | # - Test Cases: https://docs.microsoft.com/en-us/azure/azure-resource-manager/templates/test-cases 4 | @{ 5 | Test = @( 6 | 'Parameters Property Must Exist', 7 | 'Parameters Must Be Referenced', 8 | 'Secure String Parameters Cannot Have Default', 9 | 'Resources Should Have Location', 10 | 'VM Size Should Be A Parameter', 11 | 'Min And Max Value Are Numbers', 12 | 'artifacts-parameter', 13 | 'Variables Must Be Referenced', 14 | 'Dynamic Variable References Should Not Use Concat', 15 | 'Providers apiVersions Is Not Permitted', 16 | 'Template Should Not Contain Blanks', 17 | 'DependsOn Must Not Be Conditional', 18 | 'Deployment Resources Must Not Be Debug', 19 | 'adminUsername Should Not Be A Literal', 20 | 'VM Images Should Use Latest Version', 21 | 'Virtual-Machines-Should-Not-Be-Preview', 22 | 'ManagedIdentityExtension must not be used', 23 | 'Outputs Must Not Contain Secrets' 24 | ) 25 | Skip = @( 26 | 'apiVersions Should Be Recent', 27 | 'IDs Should Be Derived From ResourceIDs', 28 | 'Location Should Not Be Hardcoded', 29 | 'ResourceIds should not contain' 30 | ) 31 | } 32 | -------------------------------------------------------------------------------- /.github/linters/.jscpd.json: -------------------------------------------------------------------------------- 1 | { 2 | "threshold": 4, 3 | "reporters": [ 4 | "consoleFull" 5 | ], 6 | "ignore": [ 7 | "**/__snapshots__/**" 8 | ], 9 | "absolute": true 10 | } 11 | -------------------------------------------------------------------------------- /.github/workflows/auto-pr.yml: -------------------------------------------------------------------------------- 1 | name: Pull Request on update- branch push 2 | 3 | # Trigger this workflow on a push to any branch in this repo 4 | on: push 5 | jobs: 6 | auto-pull-request: 7 | name: PullRequestAction 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: pull-request-action 11 | uses: vsoch/pull-request-action@master 12 | env: 13 | GITHUB_TOKEN: ${{ secrets.MC2_BOT_PAT }} 14 | BRANCH_PREFIX: "update-" 15 | PULL_REQUEST_BRANCH: "master" 16 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: docs 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-18.04 11 | steps: 12 | - name: Checkout code 13 | uses: actions/checkout@v2 14 | with: 15 | submodules: true 16 | path: main 17 | 18 | - name: Set up Python 3.6 19 | uses: actions/setup-python@v2 20 | with: 21 | python-version: 3.6 22 | 23 | - name: Upgrade Pip 24 | run: python -m pip install --upgrade pip setuptools wheel 25 | 26 | - name: Install apt package dependencies 27 | run: | 28 | # Install OpenEnclave 0.17.1 29 | echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bionic main' | sudo tee /etc/apt/sources.list.d/intel-sgx.list 30 | wget -qO - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | sudo apt-key add - 31 | echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-7 main" | sudo tee /etc/apt/sources.list.d/llvm-toolchain-bionic-7.list 32 | wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - 33 | echo "deb [arch=amd64] https://packages.microsoft.com/ubuntu/18.04/prod bionic main" | sudo tee /etc/apt/sources.list.d/msprod.list 34 | wget -qO - https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - 35 | 36 | sudo apt update 37 | sudo apt -y install clang-8 libssl-dev gdb libsgx-enclave-common libsgx-quote-ex libprotobuf10 libsgx-dcap-ql libsgx-dcap-ql-dev az-dcap-client open-enclave=0.17.1 38 | 39 | # CMake 40 | wget https://github.com/Kitware/CMake/releases/download/v3.15.6/cmake-3.15.6-Linux-x86_64.sh 41 | 42 | # Mbed TLS 43 | sudo apt-get install -y libmbedtls-dev 44 | 45 | - name: Build C++ 46 | run: | 47 | # Build C++ source 48 | cd main 49 | cd src 50 | mkdir build 51 | cd build 52 | cmake .. 53 | make -j4 54 | cd ../.. 55 | 56 | - name: Install Python dependencies 57 | run: | 58 | cd main 59 | pip install -r requirements.txt 60 | pip install -r client-docs/requirements.txt 61 | 62 | - name: Install mc2client Python package 63 | run: | 64 | # Install the Python package 65 | cd main/python-package 66 | python setup.py install 67 | 68 | - name: Checkout sequencefile Python package 69 | uses: actions/checkout@master 70 | with: 71 | path: sequencefile 72 | repository: opaque-systems/sequencefile 73 | 74 | - name: Install sequencefile Python package 75 | run: cd sequencefile; python setup.py install 76 | shell: bash 77 | 78 | - name: Little hack to make this repo's docs build properly 79 | run: | 80 | cd main/client-docs 81 | mkdir -p _build/tmp 82 | cp ../mc2.py _build/tmp 83 | 84 | - name: Build docs 85 | run: | 86 | cd main/client-docs 87 | make html 88 | 89 | - name: Clone gh-pages branch 90 | uses: actions/checkout@v2 91 | with: 92 | repository: mc2-project/mc2 93 | ref: gh-pages 94 | path: main/gh-pages 95 | 96 | - name: Commit documentation changes 97 | run: | 98 | cp -r main/client-docs/_build/html/* main/gh-pages/ 99 | cd main/gh-pages 100 | touch .nojekyll 101 | git config --local user.email "action@github.com" 102 | git config --local user.name "GitHub Action" 103 | git add . 104 | git commit -m "${GITHUB_ACTOR}'s changes in ${GITHUB_SHA} triggered this build" -a || true 105 | 106 | - name: Push changes 107 | uses: ad-m/github-push-action@master 108 | with: 109 | branch: gh-pages 110 | directory: main/gh-pages 111 | github_token: ${{ secrets.GITHUB_TOKEN }} 112 | 113 | - name: Update MC2 website 114 | uses: peter-evans/repository-dispatch@v1 115 | with: 116 | token: ${{ secrets.MC2_BOT_PAT }} 117 | repository: mc2-project/mc2-project.github.io 118 | event-type: client-docs-dispatch 119 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | # Run this workflow every time a new commit pushed to your repository 4 | on: 5 | push: 6 | branches: 7 | - master 8 | pull_request: 9 | branches: 10 | - master 11 | 12 | jobs: 13 | 14 | build: 15 | name: Build, install, and test MC2 Client 16 | # Define the OS to run on 17 | runs-on: ubuntu-18.04 18 | strategy: 19 | matrix: 20 | python-version: ['3.6', '3.7', '3.8'] 21 | 22 | # Steps represent a sequence of tasks that will be executed as part of the job 23 | steps: 24 | - name: Checkout code 25 | uses: actions/checkout@v2 26 | with: 27 | submodules: true 28 | path: main 29 | 30 | - name: Set up Python ${{ matrix.python-version }} 31 | uses: actions/setup-python@v2 32 | with: 33 | python-version: ${{ matrix.python-version }} 34 | 35 | - name: Upgrade Pip 36 | run: python -m pip install --upgrade pip setuptools wheel 37 | 38 | - name: Install apt package dependencies 39 | run: | 40 | # Install OpenEnclave 0.17.1 41 | echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bionic main' | sudo tee /etc/apt/sources.list.d/intel-sgx.list 42 | wget -qO - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | sudo apt-key add - 43 | echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-7 main" | sudo tee /etc/apt/sources.list.d/llvm-toolchain-bionic-7.list 44 | wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - 45 | echo "deb [arch=amd64] https://packages.microsoft.com/ubuntu/18.04/prod bionic main" | sudo tee /etc/apt/sources.list.d/msprod.list 46 | wget -qO - https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - 47 | 48 | sudo apt update 49 | sudo apt -y install clang-8 libssl-dev gdb libsgx-enclave-common libsgx-quote-ex libprotobuf10 libsgx-dcap-ql libsgx-dcap-ql-dev az-dcap-client open-enclave=0.17.1 50 | 51 | # CMake 52 | wget https://github.com/Kitware/CMake/releases/download/v3.15.6/cmake-3.15.6-Linux-x86_64.sh 53 | 54 | # Mbed TLS 55 | sudo apt-get install -y libmbedtls-dev 56 | 57 | - name: Build C++ 58 | run: | 59 | # Build C++ source 60 | cd main 61 | cd src 62 | mkdir build 63 | cd build 64 | cmake .. 65 | make -j4 66 | cd ../.. 67 | 68 | - name: Install Python dependencies 69 | run: | 70 | # Python packages 71 | # pip install setuptools wheel pytest 72 | 73 | cd main 74 | pip install -r requirements.txt 75 | 76 | - name: Install mc2client Python package 77 | run: | 78 | # Install the Python package 79 | cd main/python-package 80 | python setup.py install 81 | 82 | - name: Checkout sequencefile Python package 83 | uses: actions/checkout@master 84 | with: 85 | path: sequencefile 86 | repository: opaque-systems/sequencefile 87 | 88 | - name: Install sequencefile Python package 89 | run: cd sequencefile; python setup.py install 90 | shell: bash 91 | 92 | - name: Run tests 93 | run: cd main/python-package/tests; pytest 94 | shell: bash 95 | 96 | lint: 97 | # Name the Job 98 | name: Lint code base 99 | # Set the type of machine to run on 100 | runs-on: ubuntu-18.04 101 | 102 | steps: 103 | # Checks out a copy of your repository on the ubuntu-18.04 machine 104 | - name: Checkout code 105 | uses: actions/checkout@v2 106 | 107 | # Runs the Super-Linter action 108 | - name: Run Super-Linter 109 | uses: github/super-linter@v4.5.1 110 | env: 111 | DEFAULT_BRANCH: master 112 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 113 | FILTER_REGEX_EXCLUDE: .*src/include/(csv.hpp|json.hpp|base64.h) 114 | 115 | LINTER_RULES_PATH: / 116 | VALIDATE_CLANG_FORMAT: true 117 | VALIDATE_PYTHON_BLACK: true 118 | PYTHON_BLACK_CONFIG_FILE: pyproject.toml 119 | VALIDATE_PYTHON_FLAKE8: true 120 | PYTHON_FLAKE8_CONFIG_FILE: .flake8 121 | 122 | # Build documentation 123 | docs: 124 | name: Build and check documentation 125 | # Define the OS to run on 126 | runs-on: ubuntu-18.04 127 | 128 | # Steps represent a sequence of tasks that will be executed as part of the job 129 | steps: 130 | - name: Checkout code 131 | uses: actions/checkout@v2 132 | with: 133 | submodules: true 134 | path: main 135 | 136 | - name: Set up Python 3.6 137 | uses: actions/setup-python@v2 138 | with: 139 | python-version: 3.6 140 | 141 | - name: Upgrade Pip 142 | run: python -m pip install --upgrade pip setuptools wheel 143 | 144 | - name: Install apt package dependencies 145 | run: | 146 | # Install OpenEnclave 0.17.1 147 | echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bionic main' | sudo tee /etc/apt/sources.list.d/intel-sgx.list 148 | wget -qO - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | sudo apt-key add - 149 | echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-7 main" | sudo tee /etc/apt/sources.list.d/llvm-toolchain-bionic-7.list 150 | wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - 151 | echo "deb [arch=amd64] https://packages.microsoft.com/ubuntu/18.04/prod bionic main" | sudo tee /etc/apt/sources.list.d/msprod.list 152 | wget -qO - https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - 153 | 154 | sudo apt update 155 | sudo apt -y install clang-8 libssl-dev gdb libsgx-enclave-common libsgx-quote-ex libprotobuf10 libsgx-dcap-ql libsgx-dcap-ql-dev az-dcap-client open-enclave=0.17.1 enchant 156 | 157 | # CMake 158 | wget https://github.com/Kitware/CMake/releases/download/v3.15.6/cmake-3.15.6-Linux-x86_64.sh 159 | 160 | # Mbed TLS 161 | sudo apt-get install -y libmbedtls-dev 162 | 163 | - name: Build C++ 164 | run: | 165 | # Build C++ source 166 | cd main 167 | cd src 168 | mkdir build 169 | cd build 170 | cmake .. 171 | make -j4 172 | cd ../.. 173 | 174 | - name: Install Python dependencies 175 | run: | 176 | cd main 177 | pip install -r requirements.txt 178 | pip install -r client-docs/requirements.txt 179 | 180 | - name: Install mc2client Python package 181 | run: | 182 | # Install the Python package 183 | cd main/python-package 184 | python setup.py install 185 | 186 | - name: Checkout sequencefile Python package 187 | uses: actions/checkout@master 188 | with: 189 | path: sequencefile 190 | repository: opaque-systems/sequencefile 191 | 192 | - name: Install sequencefile Python package 193 | run: cd sequencefile; python setup.py install 194 | shell: bash 195 | 196 | - name: Little hack to make this repo's docs build properly 197 | run: | 198 | cd main/client-docs 199 | mkdir -p _build/tmp 200 | cp ../mc2.py _build/tmp 201 | 202 | - name: Build docs 203 | run: | 204 | cd main/client-docs 205 | make html 206 | 207 | - name: Run spellcheck 208 | run : | 209 | cd main/client-docs 210 | sphinx-build -b spelling . _build 211 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | python-package/mc2client/__pycache__/ 2 | python-package/mc2client/toolchain/__pycache__/ 3 | __pycache__/ 4 | python-package/build/ 5 | python-package/dist/ 6 | *.egg-info/ 7 | src/build/ 8 | 9 | # Auto-generated gRPC files 10 | python-package/mc2client/rpc/ndarray_pb2.py 11 | python-package/mc2client/rpc/ndarray_pb2_grpc.py 12 | python-package/mc2client/rpc/opaquesql_pb2.py 13 | python-package/mc2client/rpc/opaquesql_pb2_grpc.py 14 | python-package/mc2client/rpc/remote_pb2.py 15 | python-package/mc2client/rpc/remote_pb2_grpc.py 16 | python-package/mc2client/rpc/attest_pb2.py 17 | python-package/mc2client/rpc/attest_pb2_grpc.py 18 | 19 | # Auto-generated Flatbuffers files 20 | python-package/mc2client/toolchain/flatbuffers/tuix/* 21 | 22 | # Auto-generated documentation 23 | client-docs/_build/ 24 | 25 | # Keys 26 | *.pem 27 | *.key 28 | 29 | # Any sample encrypted files 30 | *.enc 31 | 32 | # Playground 33 | playground/ 34 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc2-project/mc2/38a6888a60ea6f9e3a513fadc29c2123334e6cdf/.gitmodules -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/ambv/black 3 | rev: 21.6b0 4 | hooks: 5 | - id: black 6 | language_version: python3 7 | - repo: https://gitlab.com/pycqa/flake8 8 | rev: 3.9.2 9 | hooks: 10 | - id: flake8 11 | - repo: https://github.com/pocc/pre-commit-hooks 12 | rev: v1.1.1 13 | hooks: 14 | - id: clang-format 15 | exclude: ^(src/include/(csv.hpp|json.hpp|base64.h)) 16 | args: 17 | - -i 18 | - --style=file 19 | - --fallback-style=Chromium 20 | 21 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | mc2-dev@googlegroups.com. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to MC2 2 | 3 | ## Reporting bugs and asking questions 4 | 5 | You can ask questions, bring up issues, or garner feedback through the following channels: 6 | 7 | 1. [Slack](https://join.slack.com/t/mc2-project/shared_invite/zt-rt3kxyy8-GS4KA0A351Ysv~GKwy8NEQ) 8 | 2. [GitHub Issues](https://github.com/mc2-project/mc2/issues) 9 | 3. Email: send an email to mc2-dev@googlegroups.com 10 | 11 | ## To contribute a patch 12 | 13 | 1. Break your work into small, single-purpose patches if possible. It's much harder to merge in a large change with a lot of disjoint features. 14 | 2. Submit the patch as a GitHub pull request against the master branch. 15 | 3. Make sure that your code passes the automated tests. 16 | 4. Run `pip3 install pre-commit; pre-commit install` to create a git hook that will run the linter and automatically format your code when you commit. If the hook reformats your code, you'll need to `git add` the changed files again. 17 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Using Ubuntu 18.04 image 2 | FROM ubuntu:18.04 3 | 4 | USER root 5 | 6 | RUN mkdir -p /mc2/data 7 | 8 | ###################### 9 | # Install MC2 Client # 10 | ###################### 11 | 12 | # Copy the current directory contents into the container 13 | RUN mkdir -p /mc2/client 14 | COPY . /mc2/client 15 | 16 | # Install wget 17 | RUN apt-get update 18 | RUN apt-get install -y wget sudo gnupg2 git vim 19 | RUN useradd -m docker && echo "docker:docker" | chpasswd && adduser docker sudo 20 | 21 | # Install CMake 22 | RUN cd /mc2 && \ 23 | wget https://github.com/Kitware/CMake/releases/download/v3.15.6/cmake-3.15.6-Linux-x86_64.sh && \ 24 | sudo bash cmake-3.15.6-Linux-x86_64.sh --skip-license --prefix=/usr/local 25 | 26 | # Configure Intel and Microsoft APT repos 27 | RUN echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bionic main' | sudo tee /etc/apt/sources.list.d/intel-sgx.list && \ 28 | wget -qO - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | sudo apt-key add - && \ 29 | echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-7 main" | sudo tee /etc/apt/sources.list.d/llvm-toolchain-bionic-7.list && \ 30 | wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - && \ 31 | echo "deb [arch=amd64] https://packages.microsoft.com/ubuntu/18.04/prod bionic main" | sudo tee /etc/apt/sources.list.d/msprod.list && \ 32 | wget -qO - https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - && \ 33 | sudo apt update 34 | 35 | # Install Intel and Open Enclave packages and dependencies 36 | RUN sudo apt -y install clang-8 libssl-dev gdb libsgx-enclave-common libsgx-quote-ex libprotobuf10 libsgx-dcap-ql libsgx-dcap-ql-dev az-dcap-client open-enclave=0.17.1 37 | 38 | # Install Azure CLI 39 | RUN curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash 40 | 41 | # Mbed TLS and pip 42 | RUN sudo apt-get install -y libmbedtls-dev python3-pip 43 | 44 | # Upgrade pip 45 | RUN pip3 install --upgrade pip 46 | 47 | # Install Python dependencies 48 | RUN cd /mc2/client && pip3 install -r requirements.txt 49 | 50 | # Install Opaque Systems' sequencefile 51 | RUN cd /mc2 && \ 52 | git clone https://github.com/opaque-systems/sequencefile.git && \ 53 | cd sequencefile && \ 54 | sudo python3 setup.py install 55 | 56 | # Build mc2client 57 | RUN cd /mc2/client && \ 58 | cd src && mkdir -p build && cd build && \ 59 | cmake .. && \ 60 | make -j4 61 | 62 | # Install mc2client Python package 63 | RUN cd /mc2/client/python-package/ && \ 64 | sudo python3 setup.py install 65 | 66 | RUN echo "alias mc2=\"python3 /mc2/client/mc2.py\"" >> ~/.bashrc 67 | 68 | ###################### 69 | # Install Opaque SQL # 70 | ###################### 71 | 72 | # Clone Opaque SQL 73 | RUN cd /mc2/ && git clone https://github.com/mc2-project/opaque-sql.git 74 | 75 | # Install SBT dependencies 76 | RUN sudo apt -y install build-essential openjdk-8-jdk python libssl-dev 77 | 78 | # Install Spark 3.1.1 79 | RUN wget https://archive.apache.org/dist/spark/spark-3.1.1/spark-3.1.1-bin-hadoop2.7.tgz && \ 80 | tar xvf spark-3.1.1* && \ 81 | sudo mkdir -p /opt/spark && \ 82 | sudo mv spark-3.1.1*/* /opt/spark && \ 83 | rm -rf spark-3.1.1* && \ 84 | sudo mkdir -p /opt/spark/work && \ 85 | sudo chmod -R a+wx /opt/spark/work 86 | 87 | # Set Spark environment variables in bashrc 88 | RUN echo "" >> ~/.bashrc && \ 89 | echo "# Spark settings" >> ~/.bashrc && \ 90 | echo "export SPARK_HOME=/opt/spark" >> ~/.bashrc && \ 91 | echo "export PATH=$PATH:/opt/spark/bin:/opt/spark/sbin" >> ~/.bashrc && \ 92 | echo "" >> ~/.bashrc 93 | 94 | # Source Open Enclave on every login 95 | RUN echo "source /opt/openenclave/share/openenclave/openenclaverc" >> ~/.bashrc 96 | 97 | # Set environment variables 98 | ENV OPAQUE_HOME="/mc2/opaque-sql" 99 | ENV OPAQUE_DATA_DIR=${OPAQUE_HOME}/data/ 100 | ENV PRIVATE_KEY_PATH=${OPAQUE_HOME}/src/test/keys/mc2_test_key.pem 101 | ENV MODE=SIMULATE 102 | ENV OE_SDK_PATH=/opt/openenclave/ 103 | 104 | # Spark settings 105 | ENV SPARK_SCALA_VERSION=2.12 106 | ENV SPARK_HOME=/opt/spark 107 | ENV PATH=$PATH:/opt/spark/bin:/opt/spark/sbin 108 | 109 | # Build Opaque SQL 110 | SHELL ["/bin/bash", "-c"] 111 | RUN cd /mc2/opaque-sql && source /opt/openenclave/share/openenclave/openenclaverc && build/sbt package 112 | 113 | # Set the working directory to /mc2 114 | WORKDIR /mc2/client 115 | -------------------------------------------------------------------------------- /client-docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /client-docs/README.md: -------------------------------------------------------------------------------- 1 | # Documentation Development 2 | 3 | To build the documentation locally, install the required `sphinx` Python packages, then build the documentation from this directory. 4 | 5 | ```sh 6 | # In mc2/docs/ directory 7 | sudo apt-get install -y enchant 8 | pip3 install furo numpydoc sphinx sphinx-argparse sphinx-copybutton sphinxcontrib-spelling 9 | make html 10 | ``` 11 | 12 | To view the documentation in a browser, open the generated `index.html` file. 13 | 14 | ```sh 15 | open _build/html/index.html 16 | ``` 17 | 18 | To run spellcheck: 19 | 20 | ```sh 21 | sphinx-build -b spelling . _build 22 | ``` 23 | To add correctly spelled words to the dictionary (and prevent spellcheck errors on these words), modify `docs/spelling_wordlist.txt`. 24 | -------------------------------------------------------------------------------- /client-docs/cli/api.rst: -------------------------------------------------------------------------------- 1 | CLI Reference 2 | ============= 3 | 4 | See the :doc:`Configuration <../config/config>` section on how to specify parameters for each command. 5 | 6 | .. argparse:: 7 | :filename: _build/tmp/mc2.py 8 | :func: parser 9 | :prog: mc2 10 | -------------------------------------------------------------------------------- /client-docs/cli/index.rst: -------------------------------------------------------------------------------- 1 | CLI Interface 2 | ============= 3 | |platform| Client's command line interface requires the :substitution-code:`|python-package|` Python package to be installed. Follow the instructions :doc:`here <../install>` to install the Python package if you haven't yet done so. Below are links to CLI usage examples as well as an API reference. 4 | 5 | .. toctree:: 6 | usage 7 | api 8 | -------------------------------------------------------------------------------- /client-docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import sys 15 | 16 | curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) 17 | libpath = os.path.join(curr_path, "../python-package/") 18 | sys.path.insert(0, libpath) 19 | sys.path.insert(0, curr_path) 20 | 21 | 22 | # -- Project information ----------------------------------------------------- 23 | 24 | project = "MC2 Client" 25 | copyright = "2021, MC² Team" 26 | author = "MC² Team" 27 | 28 | # The full version, including alpha/beta/rc tags 29 | # release = "0.0.1" 30 | 31 | 32 | # -- General configuration --------------------------------------------------- 33 | 34 | # Add any Sphinx extension module names here, as strings. They can be 35 | # extensions coming with Sphinx (named "sphinx.ext.*") or your custom 36 | # ones. 37 | extensions = [ 38 | "numpydoc", 39 | "sphinx.ext.autodoc", 40 | "sphinx.ext.autosectionlabel", 41 | "sphinxarg.ext", 42 | "sphinx_copybutton", 43 | "sphinxcontrib.spelling", 44 | "sphinx-prompt", 45 | "sphinx_substitution_extensions", 46 | ] 47 | 48 | # Add any paths that contain templates here, relative to this directory. 49 | templates_path = ["_templates"] 50 | 51 | # List of patterns, relative to source directory, that match files and 52 | # directories to ignore when looking for source files. 53 | # This pattern also affects html_static_path and html_extra_path. 54 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 55 | 56 | 57 | # -- Options for HTML output ------------------------------------------------- 58 | 59 | # The theme to use for HTML and HTML Help pages. See the documentation for 60 | # a list of builtin themes. 61 | # 62 | html_theme = "furo" 63 | 64 | # Theme options are theme-specific and customize the look and feel of a theme 65 | # further. For a list of options available for each theme, see the 66 | # documentation. 67 | # 68 | html_theme_options = { 69 | "light_css_variables": { 70 | "color-brand-primary": "#00B0FF", 71 | "color-brand-content": "#00B0FF", 72 | "color-admonition-background": "#3681da", 73 | }, 74 | } 75 | 76 | # Add any paths that contain custom static files (such as style sheets) here, 77 | # relative to this directory. They are copied after the builtin static files, 78 | # so a file named "default.css" will overwrite the builtin "default.css". 79 | html_static_path = ["_static"] 80 | 81 | # Correctly spelled words to be ignored during spellcheck 82 | spelling_word_list_filename = "spelling_wordlist.txt" 83 | 84 | # Emit misspelling as Sphinx warning 85 | spelling_warning = True 86 | 87 | # -------- Substitutions ---------------------------- 88 | rst_prolog = """ 89 | .. |platform| replace:: MC\ :sup:`2` 90 | .. |platform_uppercase| replace:: MC2 91 | .. |github-org| replace:: mc2-project 92 | .. |github-repo| replace:: mc2 93 | .. |cmd| replace:: mc2 94 | .. |python-package| replace:: mc2client 95 | .. |python-package-short| replace:: mc2 96 | .. |release_version| replace:: 0.1.3 97 | .. |docker-org| replace:: mc2project 98 | """ 99 | -------------------------------------------------------------------------------- /client-docs/config/azure.rst: -------------------------------------------------------------------------------- 1 | Azure Configuration 2 | ==================== 3 | 4 | MC\ :sup:`2` Client provides an interface to directly launch Azure resources -- you can launch VMs, blob storage accounts, or storage containers. To do so, you'll need to configure some parameters that will tell MC\ :sup:`2` Client how to set up your Azure resources. You should specify the path to this configuration in the ``["launch"]["azure_config"]`` section of your :doc:`Global Configuration `. 5 | 6 | Below is an example of the Azure configuration with comments about each field. 7 | 8 | .. code-block:: yaml 9 | 10 | 11 | # An unique identifier for the head node and workers of this cluster. 12 | cluster_name: default 13 | 14 | # The total number of workers nodes to launch in addition to the head 15 | # node. This number should be >= 0. 16 | num_workers: 0 17 | 18 | # Cloud-provider specific configuration. 19 | provider: 20 | type: azure 21 | # https://docs.microsoft.com/en-us/azure/confidential-computing/virtual-machine-solutions 22 | 23 | # Location of resources 24 | # For a full list of regions that have VMs with enclave support, 25 | # go to the following link and look for DCs-v2 series VMs 26 | # https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all 27 | location: eastus 28 | 29 | # The name of the resource group that will contain all resources 30 | resource_group: mc2-client-dev 31 | 32 | # The name of the storage account to create 33 | storage_name: mc2storage 34 | 35 | # The name of the storage container to create 36 | container_name: blob-container-1 37 | 38 | # If left blank or commented out, the default subscription ID 39 | # from the Azure CLI will be used 40 | # subscription_id: 41 | 42 | # How MC2 will authenticate with newly launched nodes. 43 | auth: 44 | # The username to use to SSH to launched nodes 45 | ssh_user: mc2 46 | 47 | # You must specify paths to matching private and public key pair files 48 | # Use `ssh-keygen -t rsa -b 4096` to generate a new SSH key pair 49 | ssh_private_key: ~/.ssh/id_rsa 50 | ssh_public_key: ~/.ssh/id_rsa.pub 51 | 52 | # Provider-specific config for the head node, e.g. instance type. 53 | head_node: 54 | azure_arm_parameters: 55 | # https://docs.microsoft.com/en-us/azure/confidential-computing/virtual-machine-solutions 56 | # The DCs_v2 VMs support Intel SGX 57 | vmSize: Standard_DC2s_v2 58 | 59 | # If launching a minimal Ubuntu machine 60 | # (and manually installing using setup commands) 61 | imagePublisher: Canonical 62 | imageOffer: UbuntuServer 63 | imageSku: 18_04-lts-gen2 64 | imageVersion: latest 65 | 66 | # Provider-specific config for worker nodes, e.g. instance type. 67 | worker_nodes: 68 | azure_arm_parameters: 69 | # https://docs.microsoft.com/en-us/azure/confidential-computing/virtual-machine-solutions 70 | # The DCs_v2 VMs support Intel SGX 71 | vmSize: Standard_DC2s_v2 72 | 73 | # If launching a minimal Ubuntu machine 74 | # (and manually installing using setup commands) 75 | imagePublisher: Canonical 76 | imageOffer: UbuntuServer 77 | imageSku: 18_04-lts-gen2 78 | imageVersion: latest 79 | 80 | ############################################################################## 81 | # Everything below this can be ignored - you likely won't have to # 82 | # modify it. # 83 | ############################################################################## 84 | 85 | # Files or directories to copy to the head and worker nodes. The format is a 86 | # dictionary from REMOTE_PATH: LOCAL_PATH, e.g. 87 | file_mounts: { 88 | # This script installs Open Enclave 89 | "~/install_oe.sh" : "scripts/install_oe.sh", 90 | 91 | # This script builds Spark 3.1.1 from source 92 | "~/build_spark.sh" : "scripts/build_spark.sh", 93 | 94 | # This script downloads a pre-built Spark 3.1.1 binary 95 | "~/install_spark.sh" : "scripts/install_spark.sh", 96 | 97 | # This script builds Opaque SQL from source 98 | "~/build_opaque.sh" : "scripts/build_opaque.sh", 99 | 100 | # This script installs Secure XGBoost from source 101 | "~/install_secure_xgboost.sh" : "scripts/install_secure_xgboost.sh" 102 | } 103 | 104 | # List of commands that will be run before `setup_commands`. If docker is 105 | # enabled, these commands will run outside the container and before docker 106 | # is setup. 107 | initialization_commands: 108 | # Get rid of annoying Ubuntu message 109 | - touch ~/.sudo_as_admin_successful 110 | 111 | # List of shell commands to run to set up nodes. 112 | # Note: Use empty list if using image 113 | setup_commands: 114 | # This script installs Open Enclave on the node 115 | - chmod +x ~/install_oe.sh 116 | - source ~/install_oe.sh 117 | # This script installs Apache Spark on the node 118 | - chmod +x ~/install_spark.sh 119 | - source ~/install_spark.sh 120 | # This script installs Opaque SQL on the node 121 | - chmod +x ~/build_opaque.sh 122 | - source ~/build_opaque.sh 123 | # This script installs Secure XGBoost on the node 124 | - chmod +x ~/install_secure_xgboost.sh 125 | - source ~/install_secure_xgboost.sh 126 | 127 | # Custom commands that will be run on the head node after common setup. 128 | # Set to empty list if using image 129 | head_setup_commands: [] 130 | 131 | # Custom commands that will be run on worker nodes after common setup. 132 | # Set to empty list if using image 133 | worker_setup_commands: [] 134 | 135 | # Command to start MC2 on the head node. 136 | # Set to empty list if using image 137 | head_start_mc2_commands: 138 | - cd $SPARK_HOME; ./sbin/start-master.sh 139 | 140 | # Command to start MC2 on worker nodes. 141 | # Set to empty list if using image 142 | worker_start_mc2_commands: 143 | - cd $SPARK_HOME; ./sbin/start-slave.sh $MC2_HEAD_IP:7077 144 | -------------------------------------------------------------------------------- /client-docs/config/index.rst: -------------------------------------------------------------------------------- 1 | Configuration 2 | ============= 3 | 4 | Before using MC\ :sup:`2` Client, you should configure it by filling out two YAML files. One YAML file is for MC\ :sup:`2` Client global configuration. The other is for Azure resource configuration -- specifying your keys for authentication, filling out names of resources you want to create, etc. 5 | 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | :caption: Contents: 10 | 11 | config 12 | azure 13 | -------------------------------------------------------------------------------- /client-docs/img/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc2-project/mc2/38a6888a60ea6f9e3a513fadc29c2123334e6cdf/client-docs/img/logo.png -------------------------------------------------------------------------------- /client-docs/img/mc2_workflow.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc2-project/mc2/38a6888a60ea6f9e3a513fadc29c2123334e6cdf/client-docs/img/mc2_workflow.jpeg -------------------------------------------------------------------------------- /client-docs/index.rst: -------------------------------------------------------------------------------- 1 | .. MC\ :sup:`2` Client documentation master file, created by 2 | sphinx-quickstart on Thu Mar 4 20:37:41 2021. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to |platform| Client's documentation! 7 | =============================================== 8 | |platform| Client is a local trusted client that enables users to securely interface with the |platform| ecosystem. In particular, |platform| Client allows users to initialize their identities, launch |platform|-loaded VMs and other resources on Azure Confidential Computing, start and stop |platform| compute services (Opaque SQL and Secure XGBoost), transfer their encrypted sensitive data to the cloud for processing, and remotely run secure computation on their data. 9 | 10 | |platform| Client provides two interfaces, a :doc:`Python interface ` and a :doc:`command line interface `. Both rely on a :doc:`configuration file ` to configure |platform| Client; the command line interface is simpler but less flexible, while the Python interface gives users finer grained control over what they want to do. 11 | 12 | While |platform| currently offers various compute services, the client is currently only compatible with Opaque SQL. We are currently in the midst of updating Secure XGBoost to be compatible with the client. 13 | 14 | .. toctree:: 15 | :maxdepth: 2 16 | :caption: Contents: 17 | 18 | install 19 | quickstart 20 | opaquesql_usage 21 | config/index 22 | python/index 23 | cli/index 24 | 25 | -------------------------------------------------------------------------------- /client-docs/install.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | You can install |platform| Client from source or choose to build a Docker image from a provided Dockerfile with |platform| Client, Opaque SQL, and all dependencies pre-installed. Building the Docker image is much faster than building from source (takes about 7 min), but should be used for development/testing purposes only. 4 | 5 | 6 | Building with Docker for a local deployment 7 | ------------------------------------------- 8 | To quickly play with |platform| Client and Opaque SQL locally, you can use the provided Dockerfile to build a container (takes ~7 min) with all |platform| Client and Opaque SQL dependencies. Alternatively, you can pull a pre-built Docker image instead of building one. To do either, you must have `Docker `_ installed. 9 | 10 | The container will have the contents of this ``opaque-client`` directory at ``/mc2/client``. Opaque SQL will be at ``/mc2/opaque-sql`` 11 | 12 | .. note:: 13 | 14 | If you have installed Docker on Linux, you will either need to follow the post-installation instructions provided [here](https://docs.docker.com/engine/install/linux-postinstall/) or run all docker commands as ``sudo``. 15 | 16 | 17 | For ease of use, we recommend that you create a directory within your host ``opaque-client`` directory that will serve as your playground, and then mount your ``playground`` directory to the Docker container. Mounting will ensure that changes you make in your ``playground`` directory outside the container will be reflected inside the container, and vice versa. If you're bringing your own data, you can either copy your data over to your playground directory, or separately mount your data directory to the container. 18 | 19 | .. code-block:: bash 20 | :substitutions: 21 | 22 | # From the `|github-org|/|github-repo|`, create a `playground/` directory 23 | mkdir playground 24 | 25 | # Clone the `|github-repo|` repo 26 | git clone https://github.com/|github-org|/|github-repo| 27 | 28 | # Build a Docker image called `|cmd|_img` 29 | docker build -t |cmd|_img . 30 | 31 | # Alternatively, pull a pre-built image (~3 GB) 32 | # docker pull |docker-org|/|cmd|_img:v|release_version| 33 | 34 | # Run the container, mounting your playground to the container, and open a shell into the container 35 | docker run -it -v $(pwd)/playground:/mc2/client/playground |cmd|_img /bin/bash 36 | 37 | Building from Source 38 | -------------------- 39 | To install |platform| Client, you'll need to be running Ubuntu 18.04. Ubuntu 16.04 should also work but is untested. 40 | 41 | |platform| Client is written in both C++ and Python. As a result, we'll have to first build the C++ source, and then build and install the Python package. 42 | 43 | 1. Install dependencies. 44 | 45 | .. code-block:: bash 46 | :substitutions: 47 | 48 | # CMake 49 | wget https://github.com/Kitware/CMake/releases/download/v3.15.6/cmake-3.15.6-Linux-x86_64.sh 50 | sudo bash cmake-3.15.6-Linux-x86_64.sh --skip-license --prefix=/usr/local 51 | 52 | # Open Enclave 53 | echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bionic main' | sudo tee /etc/apt/sources.list.d/intel-sgx.list && \ 54 | wget -qO - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | sudo apt-key add - && \ 55 | echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-7 main" | sudo tee /etc/apt/sources.list.d/llvm-toolchain-bionic-7.list && \ 56 | wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - && \ 57 | echo "deb [arch=amd64] https://packages.microsoft.com/ubuntu/18.04/prod bionic main" | sudo tee /etc/apt/sources.list.d/msprod.list && \ 58 | wget -qO - https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - && \ 59 | sudo apt update 60 | sudo apt -y install clang-8 libssl-dev gdb libsgx-enclave-common libsgx-quote-ex libprotobuf10 libsgx-dcap-ql libsgx-dcap-ql-dev az-dcap-client open-enclave=0.17.1 61 | 62 | # Azure CLI 63 | curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash 64 | 65 | # Mbed TLS and Pip 66 | sudo apt-get install -y libmbedtls-dev python3-pip 67 | pip3 install --upgrade pip 68 | 69 | # |platform| Client Python package dependencies 70 | git clone https://github.com/|github-org|/|github-repo|.git 71 | cd |github-repo| 72 | pip3 install -r requirements.txt 73 | cd .. 74 | 75 | # Opaque Systems `sequencefile` Python package 76 | git clone https://github.com/opaque-systems/sequencefile.git 77 | cd sequencefile 78 | sudo python3 setup.py install 79 | cd .. 80 | 81 | 2. Clone the |platform| Client GitHub repo and install the Python package. 82 | 83 | .. code-block:: bash 84 | :substitutions: 85 | 86 | cd |github-repo|/python-package 87 | sudo python3 setup.py install 88 | 89 | 90 | You're done! Try importing the :substitution-code:`|python-package|` Python package to check that your installation was successful. 91 | 92 | .. code-block:: 93 | :substitutions: 94 | 95 | $ python3 96 | Python 3.8.7 (default, Dec 30 2020, 10:13:08) 97 | [Clang 12.0.0 (clang-1200.0.32.28)] on darwin 98 | Type "help", "copyright", "credits" or "license" for more information. 99 | 100 | >>> import |python-package| as |python-package-short| 101 | 102 | Azure Login 103 | ----------- 104 | If you want to manage your Azure resources using |platform| Client, authenticate to Azure and set your subscription ID. Find your subscription ID by following `these instructions `_. 105 | 106 | .. code-block:: bash 107 | 108 | az login 109 | az account set -s 110 | -------------------------------------------------------------------------------- /client-docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /client-docs/python/api.rst: -------------------------------------------------------------------------------- 1 | API Reference 2 | ============== 3 | |platform| Client provides a Python interface through the Python package, :substitution-code:`|python-package|`. See the Installation section for installation instructions. To use the Python package, we'll need to import it. 4 | 5 | .. code-block:: python 6 | :substitutions: 7 | 8 | import |python-package| as |python-package-short| 9 | 10 | Global Configuration 11 | ~~~~~~~~~~~~~~~~~~~~ 12 | .. autofunction:: mc2client.set_config 13 | 14 | Cryptographic Utilities 15 | ~~~~~~~~~~~~~~~~~~~~~~~ 16 | .. autofunction:: mc2client.configure_job 17 | 18 | .. autofunction:: mc2client.decrypt_data 19 | 20 | .. autofunction:: mc2client.encrypt_data 21 | 22 | .. autofunction:: mc2client.generate_keypair 23 | 24 | .. autofunction:: mc2client.generate_symmetric_key 25 | 26 | Cloud Management 27 | ~~~~~~~~~~~~~~~~ 28 | .. autofunction:: mc2client.create_cluster 29 | 30 | .. autofunction:: mc2client.create_container 31 | 32 | .. autofunction:: mc2client.create_resource_group 33 | 34 | .. autofunction:: mc2client.create_storage 35 | 36 | .. autofunction:: mc2client.delete_cluster 37 | 38 | .. autofunction:: mc2client.delete_container 39 | 40 | .. autofunction:: mc2client.delete_resource_group 41 | 42 | .. autofunction:: mc2client.delete_storage 43 | 44 | .. autofunction:: mc2client.download_file 45 | 46 | .. autofunction:: mc2client.get_head_ip 47 | 48 | .. autofunction:: mc2client.get_worker_ips 49 | 50 | .. autofunction:: mc2client.upload_file 51 | 52 | -------------------------------------------------------------------------------- /client-docs/python/index.rst: -------------------------------------------------------------------------------- 1 | Python Interface 2 | ================ 3 | 4 | To install the Python package, check out the :doc:`Installation Guide <../install>`. Below, find links to an introduction to the |platform| Client Python package as well as an API reference. 5 | 6 | .. toctree:: 7 | usage 8 | api 9 | -------------------------------------------------------------------------------- /client-docs/quickstart.rst: -------------------------------------------------------------------------------- 1 | Quickstart 2 | ========== 3 | This quickstart will give you a flavor of using |platform| with Opaque SQL, and can be entirely done locally with Docker if desired. You will use |platform| Client to encrypt some data, transfer the encrypted data to a remote machine, run an Opaque SQL job on the encrypted data on the remote machine, and retrieve and decrypt the job's encrypted results. To run everything securely, you can choose to spin up Opaque SQL on Azure VMs with SGX-support. Alternatively, to get a flavor of |platform| without having to use Azure, you can use the deployment of Opaque SQL in the Docker container. 4 | 5 | |platform| Client provides a command line interface that enables you to remotely interact with |platform| compute services. The CLI relies on a :doc:`configuration file ` that you should modify before each step in this quickstart to tell |platform| Client what exactly you want to do. 6 | 7 | Docker Quickstart 8 | ----------------- 9 | If you'd like to try everything out locally, you can do so within the Docker container you built in the :doc:`installation ` section. 10 | 11 | 1. Decide whether you want to run Spark (Scala) or PySpark (Python). The former is the default, while the latter will require two modifications to ``/mc2/client/quickstart/config.yaml``: 12 | 13 | Change ``run --> script`` to point to the Python file. 14 | 15 | .. code-block:: yaml 16 | :substitutions: 17 | 18 | run: 19 | script: ${|platform_uppercase|_CLIENT_HOME}/demo/single-party/opaquesql/opaque_sql_demo.py 20 | 21 | Change ``start --> head`` to include instructions on starting the PySpark listener. For a local listener, this will be 22 | 23 | .. code-block:: yaml 24 | :substitutions: 25 | 26 | start: 27 | head: 28 | - cd /home/mc2/opaque-sql; build/sbt assembly 29 | - cd /home/mc2/opaque-sql; spark-submit --master local[1] --jars ${|platform_uppercase|_HOME}/target/scala-2.12/opaque-assembly-0.1.jar --py-files ${|platform_uppercase|_HOME}/target/python.zip ${|platform_uppercase|_HOME}/target/python/listener.py 30 | 31 | 32 | 2. In the container, copy the contents of the ``quickstart`` directory to your mounted ``playground`` directory to ensure that your changes inside the container get reflected on your host. Then, specify the path to your configuration file. 33 | 34 | .. code-block:: bash 35 | :substitutions: 36 | 37 | # From the /mc2/client directory 38 | cp -r quickstart/* playground 39 | |cmd| configure $(pwd)/playground/config.yaml 40 | 41 | 3. Generate a keypair and a symmetric key that |platform| Client will use to encrypt your data. Specify your username and output paths in the ``user`` section of the configuration file. Then, generate the keys. 42 | 43 | .. code-block:: bash 44 | :substitutions: 45 | 46 | |cmd| init 47 | 48 | 4. Start the Opaque SQL compute service. 49 | 50 | .. code-block:: bash 51 | :substitutions: 52 | 53 | |cmd| start 54 | 55 | 5. Prepare your data for computation by encrypting and uploading it. Note that "uploading" here means copying because we have a local deployment. 56 | 57 | .. code-block:: bash 58 | :substitutions: 59 | 60 | |cmd| upload 61 | 62 | 5. Run the provided Opaque SQL quickstart script, to be executed by |platform|. The Scala script can be found `here `_, while Python is found `here `_. 63 | 64 | .. code-block:: bash 65 | :substitutions: 66 | 67 | |cmd| run 68 | 69 | 6. Once computation has finished, you can retrieve your encrypted results and decrypt them. Specify the results' path and their encryption format in the ``download`` section of configuration. The decrypted results will be in the same directory. 70 | 71 | .. code-block:: bash 72 | :substitutions: 73 | 74 | |cmd| download 75 | 76 | Azure Quickstart 77 | ---------------- 78 | You can also choose to run this quickstart with enclave-enabled VMs on the cloud with Azure Confidential Computing. This guide will take you through launching such VMs and using them with |platform|. 79 | 80 | 1. Decide whether you want to run Spark (Scala) or PySpark (Python). The former is the default, while the latter will require two modifications to ``/mc2/client/quickstart/config.yaml``: 81 | 82 | Change ``run --> script`` to point to the Python file. 83 | 84 | .. code-block:: yaml 85 | :substitutions: 86 | 87 | run: 88 | script: ${|platform_uppercase|_CLIENT_HOME}/quickstart/opaque_sql_demo.py 89 | 90 | Change ``start --> head`` to include instructions on starting the PySpark listener. For a local listener, this will be 91 | 92 | .. code-block:: yaml 93 | :substitutions: 94 | 95 | start: 96 | head: 97 | - cd /home/mc2/opaque-sql; build/sbt assembly 98 | - cd /home/mc2/opaque-sql; spark-submit --master local[1] --jars ${|platform_uppercase|_HOME}/target/scala-2.12/opaque-assembly-0.1.jar --py-files ${|platform_uppercase|_HOME}/target/python.zip ${|platform_uppercase|_HOME}/target/python/listener.py 99 | 100 | 101 | 2. In the container, copy the contents of the ``quickstart`` directory to your mounted ``playground`` directory to ensure that your changes inside the container get reflected on your host. Then, set the path to your configuration file. 102 | 103 | .. code-block:: bash 104 | :substitutions: 105 | 106 | # From the /mc2/client directory 107 | cp -r quickstart/* playground 108 | |cmd| configure $(pwd)/playground/config.yaml 109 | 110 | 3. Generate a keypair and a symmetric key that |platform| Client will use to encrypt your data. Specify your username and output paths in the ``user`` section of the configuration file. Then, generate the keys. 111 | 112 | .. code-block:: bash 113 | :substitutions: 114 | 115 | |cmd| init 116 | 117 | 4. Next, launch the machines and resources you'll be using for computation. |platform| Client provides an interface to launch resources on Azure (and sets up the machines with necessary dependencies). Take a look at the ``launch`` section of the configuration file -- you'll need to specify the path to your :doc:`Azure configuration file `, which is a YAML file that details the names and types of various resources you will launch. 118 | 119 | Next, log in to Azure through the command line and set your subscription ID. `Here `_ are instructions on how to find your subscription ID. 120 | 121 | .. code-block:: bash 122 | 123 | az login 124 | az account set -s 125 | 126 | Once you've done that, launch the resources. 127 | 128 | .. code-block:: bash 129 | :substitutions: 130 | 131 | |cmd| launch 132 | 133 | 5. Start the Opaque SQL compute service. 134 | 135 | .. code-block:: bash 136 | :substitutions: 137 | 138 | |cmd| start 139 | 140 | 6. Prepare your data for computation by encrypting and uploading it. 141 | 142 | .. code-block:: bash 143 | :substitutions: 144 | 145 | |cmd| upload 146 | 147 | 7. Run the provided Opaque SQL demo script, to be executed by |platform|. The Scala script can be found `here `_, while Python is found `here `_. Both perform a filter operation over our data -- the results will contain records of all patients who are younger than 30 years old. Results are encrypted by |platform| before being saved, and can only be decrypted with the key you used to encrypt your data in the previous step. 148 | 149 | .. code-block:: bash 150 | :substitutions: 151 | 152 | |cmd| run 153 | 154 | 8. Once computation has finished, you can retrieve your encrypted results and decrypt them. 155 | 156 | .. code-block:: bash 157 | :substitutions: 158 | 159 | |cmd| download 160 | 161 | 9. Once you've finished using your Azure resources, you can use |platform| Client to terminate them. You can specify which resources to terminate in the ``teardown`` section of the configuration. 162 | 163 | .. code-block:: bash 164 | :substitutions: 165 | 166 | |cmd| teardown 167 | -------------------------------------------------------------------------------- /client-docs/requirements.txt: -------------------------------------------------------------------------------- 1 | setuptools 2 | azure-mgmt-resource==10.0.0 3 | azure-mgmt-compute==10.0.0 4 | azure-mgmt-storage==10.0.0 5 | azure-mgmt-network==10.0.0 6 | azure-cli-core 7 | azure-storage-blob 8 | azure-identity 9 | cryptography==3.3.2 10 | furo 11 | grpcio 12 | grpcio-tools 13 | jsonschema 14 | numproto 15 | numpy==1.19.5 16 | numpydoc 17 | pandas 18 | pytest 19 | pyyaml 20 | sphinx 21 | sphinx-argparse 22 | sphinx-copybutton 23 | sphinxcontrib-spelling 24 | Sphinx-Substitution-Extensions 25 | -------------------------------------------------------------------------------- /client-docs/spelling_wordlist.txt: -------------------------------------------------------------------------------- 1 | config 2 | crypto 3 | cryptographic 4 | cryptographically 5 | decrypt 6 | decrypted 7 | decrypting 8 | keypair 9 | orchestrator 10 | plaintext 11 | repo 12 | SGX 13 | -------------------------------------------------------------------------------- /demo/azure.yaml: -------------------------------------------------------------------------------- 1 | # An unique identifier for the head node and workers of this cluster. 2 | cluster_name: default 3 | 4 | # The total number of workers nodes to launch in addition to the head 5 | # node. This number should be >= 0. 6 | num_workers: 0 7 | 8 | # Cloud-provider specific configuration. 9 | provider: 10 | type: azure 11 | # https://docs.microsoft.com/en-us/azure/confidential-computing/virtual-machine-solutions 12 | location: eastus 13 | resource_group: mc2-client-dev 14 | storage_name: mc2storage 15 | container_name: blob-container-1 16 | # set subscription id otherwise the default from az cli will be used 17 | # subscription_id: 18 | 19 | # How MC2 will authenticate with newly launched nodes. 20 | auth: 21 | # TODO: remove this field and make it the same as the username specified in config.yaml 22 | ssh_user: mc2 23 | # you must specify paths to matching private and public key pair files 24 | # use `ssh-keygen -t rsa -b 4096` to generate a new ssh key pair 25 | ssh_private_key: ~/.ssh/id_rsa 26 | ssh_public_key: ~/.ssh/id_rsa.pub 27 | 28 | # More specific customization to node configurations can be made using the ARM template azure-vm-template.json file 29 | # See documentation here: https://docs.microsoft.com/en-us/azure/templates/microsoft.compute/2019-03-01/virtualmachines 30 | # Changes to the local file will be used during deployment of the head node, however worker nodes deployment occurs 31 | # on the head node, so changes to the template must be included in the wheel file used in setup_commands section below 32 | 33 | # Provider-specific config for the head node, e.g. instance type. 34 | head_node: 35 | azure_arm_parameters: 36 | # https://docs.microsoft.com/en-us/azure/confidential-computing/virtual-machine-solutions 37 | vmSize: Standard_DC2s_v2 38 | 39 | # If launching a minimal Ubuntu machine 40 | # (and manually installing using setup commands) 41 | imagePublisher: Canonical 42 | imageOffer: UbuntuServer 43 | imageSku: 18_04-lts-gen2 44 | imageVersion: latest 45 | 46 | # Provider-specific config for worker nodes, e.g. instance type. 47 | worker_nodes: 48 | azure_arm_parameters: 49 | # https://docs.microsoft.com/en-us/azure/confidential-computing/virtual-machine-solutions 50 | vmSize: Standard_DC2s_v2 51 | 52 | # If launching a minimal Ubuntu machine 53 | # (and manually installing using setup commands) 54 | imagePublisher: Canonical 55 | imageOffer: UbuntuServer 56 | imageSku: 18_04-lts-gen2 57 | imageVersion: latest 58 | 59 | ############################################################################## 60 | # Everything below this can be ignored - you likely won't have to # 61 | # modify it. # 62 | ############################################################################## 63 | 64 | # Files or directories to copy to the head and worker nodes. The format is a 65 | # dictionary from REMOTE_PATH: LOCAL_PATH, e.g. 66 | file_mounts: { 67 | # This script installs Open Enclave 68 | "~/install_oe.sh" : "scripts/install_oe.sh", 69 | # This script builds Spark 3.1.1 from source 70 | "~/build_spark.sh" : "scripts/build_spark.sh", 71 | # This script downloads a pre-built Spark 3.1.1 binary 72 | "~/install_spark.sh" : "scripts/install_spark.sh", 73 | # This script builds Opaque from source 74 | "~/build_opaque.sh" : "scripts/build_opaque.sh", 75 | # This script installs Secure XGBoost from source 76 | "~/install_secure_xgboost.sh" : "scripts/install_secure_xgboost.sh" 77 | } 78 | 79 | # List of commands that will be run before `setup_commands`. If docker is 80 | # enabled, these commands will run outside the container and before docker 81 | # is setup. 82 | initialization_commands: 83 | # get rid of annoying Ubuntu message 84 | - touch ~/.sudo_as_admin_successful 85 | 86 | # List of shell commands to run to set up nodes. 87 | # Note: Use empty list if using image 88 | setup_commands: 89 | # This script installs Open Enclave on the node 90 | - chmod +x ~/install_oe.sh 91 | - source ~/install_oe.sh 92 | # This script installs Apache Spark on the node 93 | - chmod +x ~/install_spark.sh 94 | - source ~/install_spark.sh 95 | # This script installs Opaque on the node 96 | - chmod +x ~/build_opaque.sh 97 | - source ~/build_opaque.sh 98 | # This script installs Secure XGBoost on the node 99 | - chmod +x ~/install_secure_xgboost.sh 100 | - source ~/install_secure_xgboost.sh 101 | 102 | # Custom commands that will be run on the head node after common setup. 103 | # Set to empty list if using image 104 | head_setup_commands: [] 105 | 106 | # Custom commands that will be run on worker nodes after common setup. 107 | # Set to empty list if using image 108 | worker_setup_commands: [] 109 | 110 | # Command to start MC2 on the head node. 111 | # Set to empty list if using image 112 | head_start_mc2_commands: 113 | - cd $SPARK_HOME; ./sbin/start-master.sh 114 | 115 | # Command to start MC2 on worker nodes. 116 | # Set to empty list if using image 117 | worker_start_mc2_commands: 118 | - cd $SPARK_HOME; ./sbin/start-slave.sh $MC2_HEAD_IP:7077 119 | -------------------------------------------------------------------------------- /demo/config.yaml: -------------------------------------------------------------------------------- 1 | # User configuration 2 | user: 3 | # Your username - username should be specified in certificate 4 | username: user1 5 | 6 | # Path to your symmetric key - will be used for encryption/decryption 7 | # If you don't have a symmetric key, specify a path here 8 | # and run `mc2 init` to generate a key 9 | # 10 | # `mc2 init` will not overwrite anything at this path 11 | symmetric_key: ${MC2_CLIENT_HOME}/demo/keys/user1_sym.key 12 | 13 | 14 | # Path to your keypair and certificate. 15 | # If you don't have a private key / certificate, specify paths here 16 | # and run `mc2 init` to generate a keypair 17 | # 18 | # `mc2 init` will not overwrite anything at this path 19 | private_key: ${MC2_CLIENT_HOME}/demo/keys/user1.pem 20 | public_key: ${MC2_CLIENT_HOME}/demo/keys/user1.pub 21 | certificate: ${MC2_CLIENT_HOME}/demo/keys/user1.crt 22 | 23 | # Path to CA certificate and private key 24 | # Needed if you want to generate a certificate signed by CA 25 | root_certificate: ${MC2_CLIENT_HOME}/demo/keys/root.crt 26 | root_private_key: ${MC2_CLIENT_HOME}/demo/keys/root.pem 27 | 28 | # Configuration for launching cloud resources 29 | launch: 30 | # The absolute path to your Azure configuraton 31 | # This needs to be an absolute path 32 | azure_config: ${MC2_CLIENT_HOME}/demo/azure.yaml 33 | 34 | # # Manually specify the IPs of and usernames/SSH private keys used to log 35 | # # in to the head and worker nodes. If these values exist, Opaque Client 36 | # # will not launch or try to use any Azure resources 37 | head: 38 | ip: 0.0.0.0 39 | # username: mc2 40 | # ssh_key: ~/.ssh/id_rsa 41 | # workers: 42 | # - ip: 98.171.139.77 43 | # username: mc2 44 | # ssh_key: ~/.ssh/id_rsa 45 | 46 | # Whether to launch a cluster of VMs 47 | cluster: true 48 | 49 | # Whether to launch Azure blob storage 50 | storage: true 51 | 52 | # Whether to launch a storage container 53 | container: true 54 | 55 | # Commands to start compute service 56 | start: 57 | # Commands to run on head node 58 | head: 59 | - cd /home/mc2/opaque-sql; build/sbt run 60 | 61 | # Commands to run on worker nodes 62 | workers: 63 | - echo "Hello from worker" 64 | 65 | # Configuration for `mc2 upload` 66 | upload: 67 | # Whether to upload data to Azure blob storage or disk 68 | # Allowed values are `blob` or `disk` 69 | # If `blob`, Azure CLI will be called to upload data 70 | # Else, `scp` will be used 71 | storage: disk 72 | 73 | # Encryption format to use 74 | # Options are `sql` if you want to use Opaque SQL 75 | # or `xgb` if you want to use Secure XGBoost 76 | format: sql 77 | 78 | # Files to encrypt and upload 79 | src: 80 | - ${MC2_CLIENT_HOME}/demo/opaquesql/data/opaquesql.csv 81 | 82 | # If you want to run Opaque SQL, you must also specify a schema, 83 | # one for each file you want to encrypt and upload 84 | schemas: 85 | - ${MC2_CLIENT_HOME}/demo/opaquesql/data/opaquesql_schema.json 86 | 87 | # Directory to upload data to 88 | # FIXME: If storage is `blob` this value must be a file 89 | # Need to investigate whether we can use directories in Azure blob storage 90 | dst: /tmp/ 91 | 92 | 93 | # Computation configuration 94 | run: 95 | # Script to run 96 | script: ${MC2_CLIENT_HOME}/demo/opaquesql/opaque_sql_demo.scala 97 | 98 | # Compute service you're using 99 | # Choices are `xgb` or `sql` 100 | compute: sql 101 | 102 | # Attestation configuration 103 | attestation: 104 | # Whether we are running in simulation mode 105 | # If 0 (False), we are _not_ running in simulation mode, 106 | # and should verify the attestation evidence 107 | simulation_mode: 0 108 | 109 | # MRENCLAVE value to check 110 | # MRENCLAVE is a hash of the enclave build log 111 | mrenclave: NULL 112 | 113 | # Path to MRSIGNER value to check 114 | # MRSIGNER is the key used to sign the built enclave 115 | mrsigner: ${MC2_CLIENT_HOME}/python-package/tests/keys/mc2_test_signing_key.pub 116 | 117 | # Configuration for downloading results 118 | download: 119 | # Whether to download data to Azure blob storage or disk 120 | # Allowed values are `blob` or `disk` 121 | # If `blob`, Azure CLI will be called to download data 122 | # Else, `scp` will be used 123 | storage: disk 124 | 125 | # Format this data is encrypted with 126 | format: sql 127 | 128 | # Directory/file to download 129 | # FIXME: If storage is `blob` this value must be a file 130 | # Need to investigate whether we can use directories in Azure blob storage 131 | src: 132 | - /tmp/opaque_sql_result 133 | 134 | # Local directory to download data to 135 | dst: results/ 136 | 137 | # Configuration for stopping services 138 | stop: 139 | 140 | # Configuration for deleting Azure resources 141 | teardown: 142 | 143 | # Whether to terminate launched VMs 144 | cluster: true 145 | 146 | # Whether to terminate created Azure blob storage 147 | storage: true 148 | 149 | # Whether to terminate created storage container 150 | container: true 151 | resource_group: true 152 | -------------------------------------------------------------------------------- /demo/data/opaquesql_schema.json: -------------------------------------------------------------------------------- 1 | Age:integer,BMI:float,Glucose:integer,Insulin:float,HOMA:float,Leptin:float,Adiponectin:float,Resistin:float,MCP.1:float,Classification:integer 2 | -------------------------------------------------------------------------------- /demo/keys/README.md: -------------------------------------------------------------------------------- 1 | TODO: Check certificate validity and expiration 2 | -------------------------------------------------------------------------------- /demo/keys/root.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID/TCCAmWgAwIBAgIUag4+JmUlpecz79+hB5nWdv8vsuswDQYJKoZIhvcNAQEL 3 | BQAwDzENMAsGA1UEAwwEcm9vdDAeFw0yMDA4MzExNzQ5NDNaFw0zMDA4MjkxNzQ5 4 | NDNaMA8xDTALBgNVBAMMBHJvb3QwggGgMA0GCSqGSIb3DQEBAQUAA4IBjQAwggGI 5 | AoIBgQDGJob5lAEOyq+QDi/6gGxnmkrwWGLApC/jIVTsTt5r7Vp73mn1QBbpovso 6 | kze9ahqU6x+gT8kI7VPavXwN+MjJMzbPKEErvl3XMoY6XPSIaQrEEPJiMEB2wx2y 7 | b5cLC9VqcYryPbrXd5LP5eXqXMoEpxFgOhnXKuMtBotAEMpBS/Uk5pZuzw6IC2BM 8 | +tKqWyma3ZtSy+CRDqOrSGsvtPHAExBxNbGEIg3rRUN7v3FtRcmydKwURMxFVBV+ 9 | /UrSDocz6Vq7IBeWQP2ytsX1pcDJyWJd1R78fE6ID8jF7IKnykzc8Bl6YxV+kxwt 10 | du9o13l3fiq+vSi8yb7eckJtkG3OIXQRCRGWyll2KEZvgDW032v30X9W/OdEZWZF 11 | ATLt3ufmS2QdPOxEFt/KeAb8A8Awc05uhYYIK2kvo1i7iikwBD5OevJZdB8108Dr 12 | 5UZ4rkhtiClrnIP26shZlzwtMJOQ+TanGB7pinKaf2K6Mq3ZF58OUQ8RsFLyeDL2 13 | 7TX+8GECAQOjUzBRMB0GA1UdDgQWBBRQm3Yp9L/SChF+/OJ1E0lZhB3SajAfBgNV 14 | HSMEGDAWgBRQm3Yp9L/SChF+/OJ1E0lZhB3SajAPBgNVHRMBAf8EBTADAQH/MA0G 15 | CSqGSIb3DQEBCwUAA4IBgQBx8ZxK0UuGAq67zQVPn/ddJoqFEvCPRWVGfzOQLaTP 16 | 5URHJLFRoSrluZgpxUAuyMfGnoVh5v2aT5mYgBqvx7X+SWJx90weU2rfGzFslvt2 17 | VtPDZcZH2VgN1l2upbor7RpT6K3Fd0Mftw7+YcWTgUo+xffbBuYfBlgcQo/PcfUr 18 | 4FbsmP50DGDTTU80JW/icRc0tCCb3MroZe36tZ5nTUrGghwo8c1ICFJNrkdrfGyf 19 | y6PytCbD9desjc24SzI7eu5Y0MmwmfGHUl/KwbZtLNGf3lNhgiI/tbFdo6nBGGrE 20 | ogfdkhe+A8r7o6QtQYzsaLRePeWpu1/yDrxgJySA0E+BhEDn7kNVSpqn3B2gVAHe 21 | Yxxy6HOfCTWMKTkj8pD8B6Swo81vBM1uH2hHdyEWZG80jPgxWVttniYkfv1jIcJW 22 | 5zgZ7/HT/3jRSNwARQMEs/vH38Cyntx5TCU4aDgP67fp+SfGf43xEZhxcqoCXzTN 23 | Voyw9vprJOhvJ05ewzhelqQ= 24 | -----END CERTIFICATE----- 25 | -------------------------------------------------------------------------------- /demo/keys/root.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIG5AIBAAKCAYEAxiaG+ZQBDsqvkA4v+oBsZ5pK8FhiwKQv4yFU7E7ea+1ae95p 3 | 9UAW6aL7KJM3vWoalOsfoE/JCO1T2r18DfjIyTM2zyhBK75d1zKGOlz0iGkKxBDy 4 | YjBAdsMdsm+XCwvVanGK8j2613eSz+Xl6lzKBKcRYDoZ1yrjLQaLQBDKQUv1JOaW 5 | bs8OiAtgTPrSqlspmt2bUsvgkQ6jq0hrL7TxwBMQcTWxhCIN60VDe79xbUXJsnSs 6 | FETMRVQVfv1K0g6HM+lauyAXlkD9srbF9aXAycliXdUe/HxOiA/IxeyCp8pM3PAZ 7 | emMVfpMcLXbvaNd5d34qvr0ovMm+3nJCbZBtziF0EQkRlspZdihGb4A1tN9r99F/ 8 | VvznRGVmRQEy7d7n5ktkHTzsRBbfyngG/APAMHNOboWGCCtpL6NYu4opMAQ+Tnry 9 | WXQfNdPA6+VGeK5IbYgpa5yD9urIWZc8LTCTkPk2pxge6Ypymn9iujKt2RefDlEP 10 | EbBS8ngy9u01/vBhAgEDAoIBgQCEGa9RDVYJ3HUKtB/8VZ2aZtygOuyAbXVCFjid 11 | iemdSOb9PvFOKrnxF1IbDM/TnBG4nL/ANTCwnjfnKP1epdswzM80xYDH1D6PdwQm 12 | 6KMFm1yCtfbsICr512khn7oHXTjxoQdMKSc6T7c1Q+6cPdwDGguVfBE6HJdzWbIq 13 | tdwrh/jDRGRJ318FXOrd/Ixxkhu8k7zh3UBgtG0c2vIfzfaADLWgznZYFrPyLiz9 14 | KkueLoZ2+HK4Ld2DjWOp/jHhXwOg8sfm6SYMoCQWKQEKYaz76HxnYrpYEm/86Ss/ 15 | VvCkEATZOJkqTr8R9RA6IyiQ5fCOCYMB6+Z2KoU8Jlml96ctcpe68RkeHWEvyo9M 16 | 2cXSFiyuzQq+xn1ZI8X+cddOtfIwgit/e8V7liERtow2uIXvWYKk2R9RYj8s9Eg8 17 | My4pDfo3BPP7lMBeQlbnQ0mGbIznr2kx7eY26T4Q9uckmE+2b/sDLvd0tzCiNKNe 18 | gDcHg9rJsezKOrQDWcM0QgWktGMCgcEA5Ctw7VAKOTK8Zum/a/t/PxI47m+E/Dvd 19 | s1YeTWwxQBWdrCWXnSnoehsy5ab389MiETe2IaybzUlS2HqZDD4P1h7FdQkT8By8 20 | tsJxUjiYir0L+W7ZbC9uIqfwJIVvKE+6w5qi78jTv9H0lCpyMyNURZPJxWBngO94 21 | 3OWltobqPhB/29c7jAjyDf5XFijl2TnNXN6RDesyv9uJLJ8gjlX4+HMjqZ2mWH7F 22 | kbbaPMXCyEfqpOnAZkDKUyNK0SIPMFDjAoHBAN5RvfNyVEoeCyqPhPoXvhDabtRR 23 | gnwkyNlb6Zl96HGcp+r1nB3DDmmIUPCbOpurbpE4MBousz5ApCu+Iuhe4zPWywOW 24 | V/mBive1/ioA9G8BHPgvFcyjvRwHzSLRAM9+Qdntf+46cErjuZu7wnbLowPZQLHf 25 | b40okY9PRqq2ebRexyAcSNQMDJpx53rXclXRp7UiepLMd+SxYhOFwOf2IwbeGni0 26 | BWH45BV5k2+smIWJ7Drca3wXeppOQ1doHleQ6wKBwQCYHPXzirF7dyhEm9Typ6oq 27 | DCX0SlioJ+kiOWmI8suADmkdbmUTcUWmvMyZGfqijMFgz87BHb0zhjc6/GYIKV/k 28 | FIOjW2KgEyh51vY20GWx011Q9JDyyklsb/Vtrkoa39HXvGyf2zfVNqMNcaF3bOLZ 29 | DTEuQEUAn6XomRkkWfF+taqSj30IBfaz/uS5cJk7e9496bYJR3cqkltzFMBe4/tQ 30 | TMJxE8Q6/y5hJJF92SyFhUcYm9WZgIbiF4c2FrTK4JcCgcEAlDZ+okw4MWlcxwpY 31 | prp+teb0jYusUsMwkOfxEP6a9mhv8fkSvoIJm7A19bzRvRz0YNAgEXR3ftXCx9QX 32 | RZSXd+SHV7mP+6ux+nlUHACi9KtopXS5MxfTaAUzbItV36mBO/OqntGgMe0mZ9KB 33 | pIfCApDVy+pKXhsLtN+Ecc77zZSEwBLbOAgIZvaaUeT24+EaeMGnDIhP7cuWt66A 34 | mqQXWelm+yKuQVCYDlEM9R27A7FIJz2c/WT8Zt7Xj5q+5QtHAoHBAKSRenWOHjHG 35 | eoVkz3UKR3Nwn1Ctn/cJmMHE53vR16MeN/FlqnPQdYlvdAPaAHMy91B/zbXAKHGB 36 | WxWlEEv6PbDa0xpHwOzKgkaiES3znEyq7cjlJ6HfURdaAbbbq+uYYdaE9/qQUddC 37 | xttQW+WqaKUz71cMRmAKzzXNBmeeueQ5V514k9r5smgfm/8+//xqltPDomNnoaqz 38 | zMubnimitg5M7OcDv/eR0Hfs+N9Rh3U4yo8DJBRfyvnVMrtw7ydwnQ== 39 | -----END RSA PRIVATE KEY----- 40 | -------------------------------------------------------------------------------- /demo/keys/root.pub: -------------------------------------------------------------------------------- 1 | -----BEGIN PUBLIC KEY----- 2 | MIIBoDANBgkqhkiG9w0BAQEFAAOCAY0AMIIBiAKCAYEAxiaG+ZQBDsqvkA4v+oBs 3 | Z5pK8FhiwKQv4yFU7E7ea+1ae95p9UAW6aL7KJM3vWoalOsfoE/JCO1T2r18DfjI 4 | yTM2zyhBK75d1zKGOlz0iGkKxBDyYjBAdsMdsm+XCwvVanGK8j2613eSz+Xl6lzK 5 | BKcRYDoZ1yrjLQaLQBDKQUv1JOaWbs8OiAtgTPrSqlspmt2bUsvgkQ6jq0hrL7Tx 6 | wBMQcTWxhCIN60VDe79xbUXJsnSsFETMRVQVfv1K0g6HM+lauyAXlkD9srbF9aXA 7 | ycliXdUe/HxOiA/IxeyCp8pM3PAZemMVfpMcLXbvaNd5d34qvr0ovMm+3nJCbZBt 8 | ziF0EQkRlspZdihGb4A1tN9r99F/VvznRGVmRQEy7d7n5ktkHTzsRBbfyngG/APA 9 | MHNOboWGCCtpL6NYu4opMAQ+TnryWXQfNdPA6+VGeK5IbYgpa5yD9urIWZc8LTCT 10 | kPk2pxge6Ypymn9iujKt2RefDlEPEbBS8ngy9u01/vBhAgED 11 | -----END PUBLIC KEY----- 12 | -------------------------------------------------------------------------------- /demo/keys/user1.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDkzCCAfsCAQAwDQYJKoZIhvcNAQELBQAwDzENMAsGA1UEAwwEcm9vdDAeFw0y 3 | MTA1MjcyMzA3NDBaFw0zMTA1MjUyMzA3NDBaMBAxDjAMBgNVBAMMBXVzZXIxMIIB 4 | ojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA1yVG8sLUWJhaQwAICOsPicM4 5 | RwkIFLoGFnXdHZch8EnqRpG5MGIzfbpNgvNq8UNp/WChkDWLypUZ5pIJt3XcFlJq 6 | JOyktQDcgU+tdoYynF+A2afKtJi8xmRXZ3BXQNJ3qyw3jXDiF4OKifLJj4Fa7j9a 7 | e+5Tr8RSd0jiFczazFB4AsxF9NB06vUc1YbTYtAZD3xHvA0do0/bAiwiuEvWljZ+ 8 | Bp4+dRvUS/Hh65aPPEO7Ro8J9k4xPasHxLPSDQxieOeabKzabe5I/OGiv8nEQ5uV 9 | FN30IR8ICyzhb1QdCp6vgiBkI9Z9ccgRAUx0Qzieo1j7jenNjUI9l5YIyY6BSUKq 10 | 4bha1PDO5wi02N3Ph2HbuLThSg5dzi68yB4KeDS1DsprLbGdbtvnXIp72r30JRdy 11 | vWRmVOdIf621fhGEUEibps8aV2iAEMEkp23Zz5LxQgiLIZFAUU1iCoXwZ+UnfRuh 12 | 8DLEceAolUgRwhnSIu2Z51Na6m6KzjwxGDJKPFmHAgMBAAEwDQYJKoZIhvcNAQEL 13 | BQADggGBAFUA6DwaC994PBCpnCK1KZRMYyT9Xn13KRpWRQ95AxkH+xuVKeqIWu7K 14 | U22e80EsR3tAqkXTLmO+0Vv6pF0pp3kXe1nEcV5l9QgRJscr9TAWw5qhmP4Dpk1k 15 | FZzU+2Bf4RleDFrfkT0cKdMkoY+LvDv8V5fRXpC3UDQuCWkcy8vu9GGIId2PAg6Q 16 | s9/E3BOXnFIRw18+kMYUy2Mi808jLDR6y1h0qMTt792/9+mp6hVjNtgJxlNhmOEZ 17 | d8hEgVk1zXk9kErIk1SgGjB0t76jkyIdt4MpX+xjaPOODr9+8FUbp83v/N1E6M9b 18 | 4TttHGM7N5dyOCK8p7LwY55c2jpBY+FZVN4ddID5zh2RLgV/vOIL8h9kOv/mUDzR 19 | qojuw/nHNj7++hvTieXF6crykWsAG4LZjQjSpymN2kG4MAMUulX7FyxAL4lYwi6n 20 | 2f6MQLISd0hXYU9IOIVkkrBZdCI4MQ1tUe/U9A++jZ1oIpgLzKtinYmzwbL2rjxn 21 | Q8gucYqc+Q== 22 | -----END CERTIFICATE----- 23 | -------------------------------------------------------------------------------- /demo/keys/user1.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpgIBAAKCAQEAy/xJzSmRFVX1QWbcN6tijae1nPOD0mfoKCmA1k/LVyo/vyOF 3 | lO0SZEWQ/GEpZQOii7oel6tdJicv8FLsVdWDxhUali02C7741EYrGRKmg6AjQzIS 4 | pcdayyX2fSDgveu6PZg/Iws922bkVXmChbasXy6a1YRdqJSRhqzysuQgKBNCBfKt 5 | xkk2ZH6kb/ORy59Y4FRsr9bOeN4KomHPsCledfGwHYrolHQdVxaSJFlpXj3r4SdY 6 | G3jtmLt2oJgNFlWlsVxA/LrpSA/1DKLAi7yndacyWQbUV4UHHzpYqWYa02irqAIw 7 | GgQCv14vqzFIOsxd6jjoZ1ZHpkV4D1FlwgQ2YwIDAQABAoIBAQCIkXdDsXGeDDew 8 | OtJzPv4zgrX3x51Y8BIhWXAVi8TCrdmR+ZFEnNg0Y9Lm9ZIGPEHGNKyotKUflxrk 9 | VlA+Qz3/D0AGjWxivs/PJs9R8QeklkElis+rR+YEnn0BV1LANKMy+8xBhRQ74Q48 10 | H6cAKMYMz6IAE6e05wrrVL0EgfD5goEt50AsVCLwak2QUYNax5x80h53l6qHrNf1 11 | Gm7yobNDVMkDdkXfArCr+KO6pdgOY++NOq/s+aZEYFG1zqj298ZUjf/8VE0G/OpU 12 | gZj2cKNR1ZwAMR0z6gnafx98sB0PQmaSC/d41sbc90K6OS+3wyJq2pKksNvZLKYo 13 | RHpeujGZAoGBAOcv6S2buL3q2l6j4Xl17rGUKiWbUU8Hs3sUZ8HdcVMh/bkptf1V 14 | 0KOTC0jlbH/iZMFBh7ezyQGscy+mEfZHk3tmEiJQcHfC2b2OkEQNumf1hWDNNeT/ 15 | QrWqcUo64WRCtlO0XceeytkoB3K7K6ueSBzyW253x9yqn1GwarNlUCeXAoGBAOHg 16 | /GbaVRpkCP/bAC3iuLoVAhaG9tmAU/JrnXvQiHVwjaqfy71EaVS5flaU7yX8cMgt 17 | XAV71g9rfQPFNj1UQkzy9ZAuXn6YAUjOiT6g/qZKMs8+IRS/4NM/RwLwWZAOLqX9 18 | xIwdHIpJpt7otLqB8/o4IFSy0PsAXEKLw8MMLaEVAoGBANGBsp+5UhcSAOjpPMj0 19 | neZhrjw8X9ft07IDUO/3/N3onOUzLpCxNw01kXFzL/tIsCQOfa51iAAzRoN0zSxR 20 | uw6F9oMQQIvXkbldu9FgW4AgmMmbzm6DAPJezqIFcAKLDm4WszHW7l8TDoTjp/Sz 21 | 0QgifcdDV2TbXtwJsvh2JMhrAoGBALcDf/4l8MZNPy1u2BpVlK7QzrxJ5kAcRegp 22 | YZQsRiRBacdRadaUU3OeR8sHKS1x+D944RJgX/RmdwhTBbtzRbTrNiP4LgAMUR4Y 23 | mgLwluNNQxW/lTYmsZb3siWjcC3UD9/WWSXdgH5bZqU9jxF+sZuPVkLKD6EPjbEt 24 | ZRPoDiBxAoGBAIO/ijeKISVoCnEBtawkQ1McGKehcBNF7yJuv+IC0z/z+ZltGj3A 25 | 9uIiMSH6Ij1tu0Y5VQUWhnwhd6PiSvNbni9kEftaYqLzQp+khdHrDCyek9rh7lys 26 | lrUp48X43OrorXRX4p6thdADY89QSSa8xRgsygLeMoGxPrTH4uLXUpIb 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /demo/keys/user1.pub: -------------------------------------------------------------------------------- 1 | -----BEGIN PUBLIC KEY----- 2 | MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAy/xJzSmRFVX1QWbcN6ti 3 | jae1nPOD0mfoKCmA1k/LVyo/vyOFlO0SZEWQ/GEpZQOii7oel6tdJicv8FLsVdWD 4 | xhUali02C7741EYrGRKmg6AjQzISpcdayyX2fSDgveu6PZg/Iws922bkVXmChbas 5 | Xy6a1YRdqJSRhqzysuQgKBNCBfKtxkk2ZH6kb/ORy59Y4FRsr9bOeN4KomHPsCle 6 | dfGwHYrolHQdVxaSJFlpXj3r4SdYG3jtmLt2oJgNFlWlsVxA/LrpSA/1DKLAi7yn 7 | dacyWQbUV4UHHzpYqWYa02irqAIwGgQCv14vqzFIOsxd6jjoZ1ZHpkV4D1FlwgQ2 8 | YwIDAQAB 9 | -----END PUBLIC KEY----- 10 | -------------------------------------------------------------------------------- /demo/keys/user1_sym.key: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc2-project/mc2/38a6888a60ea6f9e3a513fadc29c2123334e6cdf/demo/keys/user1_sym.key -------------------------------------------------------------------------------- /demo/opaque_sql_demo.scala: -------------------------------------------------------------------------------- 1 | import edu.berkeley.cs.rise.opaque.implicits._ 2 | import org.apache.spark.sql.types._ 3 | 4 | val df = spark.read.format("edu.berkeley.cs.rise.opaque.EncryptedSource").load("/root/data/opaquesql.csv.enc") 5 | 6 | val result = df.filter($"Age" < lit(30)) 7 | 8 | // This will save the result DataFrame to the result directory on the cloud 9 | result.write.format("edu.berkeley.cs.rise.opaque.EncryptedSource").save("/root/results/opaque_sql_result") 10 | -------------------------------------------------------------------------------- /demo/opaquesql/data/opaquesql_schema.json: -------------------------------------------------------------------------------- 1 | Age:integer,BMI:float,Glucose:integer,Insulin:float,HOMA:float,Leptin:float,Adiponectin:float,Resistin:float,MCP.1:float,Classification:integer 2 | -------------------------------------------------------------------------------- /demo/opaquesql/opaque_sql_demo.scala: -------------------------------------------------------------------------------- 1 | import edu.berkeley.cs.rise.opaque.implicits._ 2 | import org.apache.spark.sql.types._ 3 | 4 | val df = spark.read.format("edu.berkeley.cs.rise.opaque.EncryptedSource").load("/tmp/opaquesql.csv.enc") 5 | 6 | val result = df.filter($"Age" < lit(30)) 7 | 8 | // This will save the result DataFrame to the result directory on the cloud 9 | result.write.format("edu.berkeley.cs.rise.opaque.EncryptedSource").save("/tmp/opaque_sql_result") 10 | -------------------------------------------------------------------------------- /mc2_client_env: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | 3 | export MC2_CLIENT_HOME="`( cd \"$(dirname "${BASH_SOURCE[0]}")\" && pwd )`" 4 | alias mc2="python3 ${MC2_CLIENT_HOME}/mc2.py" 5 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 79 3 | include = '\.pyi?$' 4 | exclude = ''' 5 | /( 6 | \.git 7 | | \.hg 8 | | \.mypy_cache 9 | | \.tox 10 | | \.venv 11 | | _build 12 | | buck-out 13 | | build 14 | | dist 15 | )/ 16 | ''' 17 | -------------------------------------------------------------------------------- /python-package/mc2client/__init__.py: -------------------------------------------------------------------------------- 1 | from .core import ( 2 | configure_job, 3 | clear_cache, 4 | create_cluster, 5 | create_container, 6 | create_resource_group, 7 | create_storage, 8 | decrypt_data, 9 | delete_cluster, 10 | delete_container, 11 | delete_resource_group, 12 | delete_storage, 13 | download_file, 14 | encrypt_data, 15 | generate_keypair, 16 | generate_symmetric_key, 17 | get_head_ip, 18 | get_worker_ips, 19 | run_remote_cmds, 20 | stop_remote_cmds, 21 | set_config, 22 | upload_file, 23 | ) 24 | from .opaquesql import run 25 | from .xgb import Booster, DMatrix, rabit 26 | 27 | __all__ = [ 28 | "configure_job", 29 | "Booster", 30 | "clear_cache", 31 | "create_cluster", 32 | "create_container", 33 | "create_resource_group", 34 | "create_storage", 35 | "decrypt_data", 36 | "delete_cluster", 37 | "delete_container", 38 | "delete_resource_group", 39 | "delete_storage", 40 | "DMatrix", 41 | "download_file", 42 | "encrypt_data", 43 | "generate_keypair", 44 | "generate_symmetric_key", 45 | "get_head_ip", 46 | "get_worker_ips", 47 | "rabit", 48 | "run", 49 | "run_remote_cmds", 50 | "stop_remote_cmds", 51 | "set_config", 52 | "upload_file", 53 | ] 54 | -------------------------------------------------------------------------------- /python-package/mc2client/cache.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module provides caching functionality to MC2 Client so that state can 3 | be maintained across CLI calls. Currently, the following things are cached: 4 | * Whether the TMS has been attested 5 | * State information for each process spawned via `start()`. This state is 6 | used by `stop()` to terminate these processes. 7 | """ 8 | import json 9 | import os 10 | 11 | 12 | def cache_op(op): 13 | """ 14 | A function decorater for cache operations. 15 | 16 | This decorater will check for the existence of a cache, create one if needed, 17 | and handle the serialization/deserialization of the cache to disk before and 18 | after function calls. 19 | 20 | parameters 21 | ---------- 22 | f : function 23 | The function to be executed 24 | op : str 25 | "create" if adding an entry to the cache, "check" otherwise 26 | """ 27 | 28 | def prelude_wrapper(f): 29 | def prelude(*args, **kwargs): 30 | global cache 31 | 32 | # Load the cache from disk if it exists 33 | cache_path = os.path.expanduser("~/.cache/mc2") 34 | if not os.path.exists(cache_path): 35 | if op == "check": 36 | # If we're viewing or removing a cache entry and the cache 37 | # doesn't exist, return without doing anything 38 | return None 39 | elif op == "create": 40 | # Otherwise, create the cache before executing `f` 41 | try: 42 | os.makedirs(os.path.expanduser("~/.cache")) 43 | except FileExistsError: 44 | pass 45 | cache = dict() 46 | else: 47 | cache = json.load(open(cache_path)) 48 | 49 | # Execute `f` which will have access to `cache` 50 | ret_val = f(*args, cache=cache, **kwargs) 51 | 52 | # Serialize `cache` to disk 53 | with open(cache_path, "w") as cache_file: 54 | json.dump(cache, cache_file) 55 | 56 | return ret_val 57 | 58 | return prelude 59 | 60 | return prelude_wrapper 61 | 62 | 63 | @cache_op("create") 64 | def add_cache_entry(key, value, cache=dict()): 65 | """ 66 | Add `value` to the Opaque Client cache at index `key`. This will overwrite 67 | any existing values at `key`. 68 | 69 | parameters 70 | ---------- 71 | key : str 72 | Key in the cache 73 | value: (valid JSON type) 74 | The value to store in the cache 75 | cache: dict 76 | The cache dictionary. This should only be set by the decorator function. 77 | """ 78 | cache[key] = value 79 | 80 | 81 | @cache_op("check") 82 | def get_cache_entry(key, cache=dict()): 83 | """ 84 | Return the value at index `key` in the Opaque Client cache or None if the 85 | index `key` doesn't exist. 86 | 87 | parameters 88 | ---------- 89 | key : str 90 | Key in the cache 91 | cache: dict 92 | The cache dictionary. This should only be set by the decorator function. 93 | """ 94 | return cache.get(key) 95 | 96 | 97 | @cache_op("check") 98 | def remove_cache_entry(key, cache=dict()): 99 | """ 100 | Remove the value at index `key` from the Opaque Client cache and return it. 101 | Returns None if the index `key` doesn't exist. 102 | 103 | parameters 104 | ---------- 105 | key : str 106 | Key in the cache 107 | cache: dict 108 | The cache dictionary. This should only be set by the decorator function. 109 | """ 110 | return cache.pop(key, None) 111 | -------------------------------------------------------------------------------- /python-package/mc2client/exceptions.py: -------------------------------------------------------------------------------- 1 | class AttestationError(Exception): 2 | pass 3 | 4 | 5 | class CryptoError(Exception): 6 | pass 7 | 8 | 9 | class MC2ClientComputeError(Exception): 10 | """Error thrown by MC2 Client due to compute service error.""" 11 | 12 | 13 | class MC2ClientConfigError(Exception): 14 | """Error thrown by MC2 Client due to error in configuration files.""" 15 | -------------------------------------------------------------------------------- /python-package/mc2client/opaquesql/__init__.py: -------------------------------------------------------------------------------- 1 | from .opaquesql import run 2 | 3 | __all__ = ["run"] 4 | -------------------------------------------------------------------------------- /python-package/mc2client/opaquesql/opaquesql.py: -------------------------------------------------------------------------------- 1 | import grpc 2 | import sys 3 | 4 | from ..core import _CONF, _check_remote_call, get_head_ip, logger 5 | from ..rpc import ( # pylint: disable=no-name-in-module 6 | opaquesql_pb2, 7 | opaquesql_pb2_grpc, 8 | ) 9 | 10 | 11 | def run(script): 12 | """ 13 | Run a Opaque SQL Scala script 14 | 15 | Parameters 16 | ---------- 17 | script : str 18 | path to script 19 | """ 20 | with open(script, "r") as f: 21 | code = f.read() 22 | 23 | # Job requires a TMS 24 | if _CONF["use_azure"]: 25 | head_address = get_head_ip() + ":50052" 26 | else: 27 | head_address = _CONF["head"]["ip"] + ":50052" 28 | 29 | try: 30 | with grpc.insecure_channel(head_address) as channel: 31 | stub = opaquesql_pb2_grpc.ListenerStub(channel) 32 | response = stub.ReceiveQuery( 33 | opaquesql_pb2.QueryRequest(request=code) 34 | ) 35 | except grpc.RpcError as rpc_error: 36 | logger.error( 37 | "When submitting a query to Opaque SQL, " + rpc_error.details() 38 | ) 39 | sys.exit(1) 40 | 41 | if response.status.status != 0: 42 | logger.error( 43 | "Error executing Opaque SQL query. Traceback pasted below. \n{}".format( 44 | response.status.exception 45 | ) 46 | ) 47 | sys.exit(1) 48 | -------------------------------------------------------------------------------- /python-package/mc2client/rpc/README.md: -------------------------------------------------------------------------------- 1 | # Building RPC 2 | 3 | If making changes to any of the files, rebuild using the following command within the rpc/ directory 4 | 5 | python3 -m grpc_tools.protoc -Iprotos --python_out=. --grpc_python_out=. protos/remote.proto protos/ndarray.proto 6 | -------------------------------------------------------------------------------- /python-package/mc2client/rpc/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from pathlib import Path 3 | 4 | sys.path.append(str(Path(__file__).parent)) 5 | -------------------------------------------------------------------------------- /python-package/mc2client/rpc/protos/attest.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package opaque.protos; 4 | 5 | // The Client <-> Enclave service 6 | service ClientToEnclave { 7 | // Client sends AttestationStatus to the Enclave, receives 8 | // an evidence report for each enclave 9 | rpc GetRemoteEvidence(AttestationStatus) returns (Evidences) {} 10 | 11 | // After processing Evidence, Client sends encrypted shared key 12 | // as Encrypted Data and receives AttestationStatus 13 | rpc GetFinalAttestationResult(EncryptedKeys) returns (AttestationStatus) {} 14 | } 15 | 16 | message Evidences { 17 | // A collection of `oe_evidence_msg_t` objects. 18 | // includes attestation evidence, enclave public key, and nonce 19 | repeated bytes evidences = 1; 20 | } 21 | 22 | message AttestationStatus { 23 | // This is 0 if not attested or attestation failed, 24 | // 1 if whichever node is sending it has completed 25 | // attestation successfully 26 | int32 status = 1; 27 | } 28 | 29 | message EncryptedKeys { 30 | // A collection of encrypted `SignedKey` flatbuffer objects 31 | repeated bytes keys = 1; 32 | } 33 | -------------------------------------------------------------------------------- /python-package/mc2client/rpc/protos/ndarray.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package numproto.protobuf; 4 | 5 | message NDArray { 6 | bytes ndarray = 1; 7 | } 8 | -------------------------------------------------------------------------------- /python-package/mc2client/rpc/protos/opaquesql.proto: -------------------------------------------------------------------------------- 1 | /* This file should be kept in sync with Opaque's listener.proto */ 2 | 3 | syntax = "proto3"; 4 | 5 | package opaque.protos; 6 | 7 | service Listener { 8 | rpc ReceiveQuery (QueryRequest) returns (QueryResult) {} 9 | } 10 | 11 | message QueryRequest { 12 | string request = 1; 13 | } 14 | 15 | message QueryResult { 16 | string result = 1; 17 | Status status = 2; 18 | } 19 | 20 | message Status { 21 | // Status 22 | int32 status = 1; 23 | 24 | // Exception message 25 | string exception = 2; 26 | } 27 | -------------------------------------------------------------------------------- /python-package/mc2client/toolchain/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc2-project/mc2/38a6888a60ea6f9e3a513fadc29c2123334e6cdf/python-package/mc2client/toolchain/__init__.py -------------------------------------------------------------------------------- /python-package/mc2client/toolchain/encrypt_and_upload.py: -------------------------------------------------------------------------------- 1 | # Import the needed management objects from the libraries. The azure.common library 2 | # is installed automatically with the other libraries. 3 | import os 4 | import logging 5 | 6 | from azure.common.client_factory import get_client_from_cli_profile 7 | from azure.mgmt.storage import StorageManagementClient 8 | from azure.storage.blob import BlobServiceClient 9 | from azure.core.exceptions import ResourceExistsError 10 | from cryptography.hazmat.backends import default_backend 11 | from cryptography.hazmat.primitives.ciphers.aead import AESGCM 12 | 13 | container_client = None 14 | 15 | # Get Opaque Client loger 16 | logger = logging.getLogger(__name__) 17 | 18 | # Acquire the Azure logger and set the log level to WARNING 19 | azure_logger = logging.getLogger("azure") 20 | azure_logger.setLevel(logging.WARNING) 21 | 22 | 23 | class CryptoUtil(object): 24 | KEY_SIZE = 16 25 | NONCE_SIZE = 12 26 | backend = default_backend() 27 | 28 | # Generate a private key given a password 29 | @classmethod 30 | def generate_priv_key(cls, priv_key_path): 31 | # 128 bit security, like in Secure XGBoost 32 | key = AESGCM.generate_key(bit_length=cls.KEY_SIZE * 8) 33 | with open(priv_key_path, "wb") as priv_key: 34 | priv_key.write(key) 35 | 36 | @classmethod 37 | def encrypt_data(cls, priv_key_path, input_filename): 38 | # FIXME: if file is larger than memory, we should read in chunks 39 | with open(priv_key_path, "rb") as priv_key, open( 40 | input_filename, "rb" 41 | ) as input_file: 42 | 43 | key = priv_key.read() 44 | data = input_file.read() 45 | cipher = AESGCM(key) 46 | 47 | nonce = os.urandom(cls.NONCE_SIZE) 48 | enc_data = cipher.encrypt(nonce, data, b"") 49 | 50 | data = nonce + enc_data 51 | 52 | return data 53 | 54 | @classmethod 55 | def decrypt_data(cls, priv_key_filename, enc_data, output_filename): 56 | with open(priv_key_filename, "rb") as priv_key, open( 57 | output_filename, "wb" 58 | ) as output_file: 59 | 60 | key = priv_key.read() 61 | cipher = AESGCM(key) 62 | 63 | nonce = enc_data[: cls.NONCE_SIZE] 64 | data = enc_data[cls.NONCE_SIZE :] # noqa: E203 65 | 66 | aad = b"" 67 | dec_data = cipher.decrypt(nonce, data, aad) 68 | output_file.write(dec_data) 69 | 70 | 71 | def create_storage(config): 72 | rg_name = config["resource_group"] 73 | location = config["location"] 74 | storage_name = config["storage_name"] 75 | 76 | storage_client = get_client_from_cli_profile(StorageManagementClient) 77 | availability_result = ( 78 | storage_client.storage_accounts.check_name_availability(storage_name) 79 | ) 80 | 81 | if not availability_result.name_available: 82 | logger.warning( 83 | "Storage account {} already exists, skipping storage account creation".format( 84 | storage_name 85 | ) 86 | ) 87 | return 88 | 89 | # The name is available, so provision the account 90 | poller = storage_client.storage_accounts.create( 91 | rg_name, 92 | storage_name, 93 | { 94 | "location": location, 95 | "kind": "StorageV2", 96 | "sku": {"name": "Standard_ZRS"}, 97 | }, 98 | ) 99 | 100 | # Long-running operations return a poller object; calling poller.result() 101 | # waits for completion. 102 | account_result = poller.result() 103 | logger.info(f"Provisioned storage account {account_result.name}") 104 | 105 | 106 | def terminate_storage(config): 107 | rg_name = config["resource_group"] 108 | location = config["location"] 109 | storage_name = config["storage_name"] 110 | 111 | storage_client = get_client_from_cli_profile(StorageManagementClient) 112 | storage_client.storage_accounts.delete( 113 | rg_name, storage_name, {"location": location, "kind": "StorageV2"} 114 | ) 115 | 116 | logger.info("Terminated storage account {}".format(storage_name)) 117 | 118 | 119 | def create_container(config): 120 | container_name = "" 121 | try: 122 | blob_service_client = get_blob_service_client(config) 123 | container_name = config["container_name"] 124 | blob_service_client.create_container(container_name) 125 | logger.info(f"Provisioned storage container {container_name}") 126 | except ResourceExistsError: 127 | logger.warning( 128 | "The specified container {} already exists, skipping storage container creation".format( 129 | container_name 130 | ) 131 | ) 132 | 133 | 134 | def terminate_container(config): 135 | blob_service_client = get_blob_service_client(config) 136 | container_name = config["container_name"] 137 | container_client = blob_service_client.get_container_client(container_name) 138 | container_client.delete_container() 139 | logger.info("Terminated storage container {}".format(container_name)) 140 | 141 | 142 | # Obtain the management object for resources, using the credentials from the CLI login. 143 | def get_blob_service_client(config): 144 | storage_client = get_client_from_cli_profile(StorageManagementClient) 145 | rg_name = config["resource_group"] 146 | storage_name = config["storage_name"] 147 | keys = storage_client.storage_accounts.list_keys(rg_name, storage_name) 148 | conn_string = ( 149 | "DefaultEndpointsProtocol=https;AccountName={};" 150 | "AccountKey={};EndpointSuffix=core.windows.net".format( 151 | storage_name, keys.keys[0].value 152 | ) 153 | ) 154 | blob_service_client = BlobServiceClient.from_connection_string( 155 | conn_str=conn_string 156 | ) 157 | return blob_service_client 158 | 159 | 160 | # `blob_name` is the blob that we want to write to/read from 161 | def get_blob_client(config, blob_name): 162 | blob_service_client = get_blob_service_client(config) 163 | blob_client = blob_service_client.get_blob_client( 164 | container=config["container_name"], blob=blob_name 165 | ) 166 | return blob_client 167 | 168 | 169 | def upload_data(config, data, blob_name, overwrite=True): 170 | blob_client = get_blob_client(config, blob_name) 171 | blob_client.upload_blob(data, overwrite=overwrite) 172 | 173 | 174 | def upload_data_from_file(config, input_filename, blob_name, overwrite=True): 175 | with open(input_filename, "rb") as input_file: 176 | data = input_file.read() 177 | upload_data(config, data, blob_name, overwrite=overwrite) 178 | 179 | 180 | def download_data(config, blob_name, output_filename=None): 181 | blob_client = get_blob_client(config, blob_name) 182 | data = blob_client.download_blob().readall() 183 | 184 | if output_filename: 185 | with open(output_filename, "wb") as output_file: 186 | output_file.write(data) 187 | else: 188 | return data 189 | 190 | 191 | # The below two functions are currently not used and do not work 192 | def encrypt_and_upload_data(config, input_file_path, blob_name): 193 | priv_key_path = ["priv_key_path"] 194 | enc_data = CryptoUtil.encrypt_data(priv_key_path, input_file_path) 195 | upload_data(config, enc_data, blob_name) 196 | 197 | 198 | def download_and_decrypt_data(config, blob_name, output_file_path): 199 | priv_key_path = config["priv_key_path"] 200 | enc_data = download_data(config, blob_name) 201 | CryptoUtil.decrypt_data(priv_key_path, enc_data, output_file_path) 202 | -------------------------------------------------------------------------------- /python-package/mc2client/toolchain/flatbuffers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc2-project/mc2/38a6888a60ea6f9e3a513fadc29c2123334e6cdf/python-package/mc2client/toolchain/flatbuffers/__init__.py -------------------------------------------------------------------------------- /python-package/mc2client/toolchain/log_timer.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import logging 3 | 4 | logger = logging.getLogger(__name__) 5 | 6 | 7 | class LogTimer: 8 | def __init__(self, message, show_status=False): 9 | self._message = message 10 | self._show_status = show_status 11 | 12 | def __enter__(self): 13 | self._start_time = datetime.datetime.utcnow() 14 | 15 | def __exit__(self, *error_vals): 16 | td = datetime.datetime.utcnow() - self._start_time 17 | status = "" 18 | if self._show_status: 19 | status = "failed" if any(error_vals) else "succeeded" 20 | logger.info( 21 | " ".join( 22 | [ 23 | self._message, 24 | status, 25 | "[LogTimer={:.0f}ms]".format(td.total_seconds() * 1000), 26 | ] 27 | ) 28 | ) 29 | -------------------------------------------------------------------------------- /python-package/mc2client/toolchain/mc2-schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "$id": "http://github.com/mc2-project/mc2-scripts/mc2-schema.json", 4 | "title": "MC2 cluster management scripts", 5 | "description": "MC2 cluster management schema", 6 | "type": "object", 7 | "definitions": { 8 | "commands": { 9 | "type": "array", 10 | "items": { 11 | "type": "string", 12 | "description": "shell command" 13 | } 14 | } 15 | }, 16 | "required": [ 17 | "cluster_name", 18 | "provider" 19 | ], 20 | "additionalProperties": false, 21 | "properties": { 22 | "cluster_name": { 23 | "description": "An unique identifier for the head node and workers of this cluster.", 24 | "type": "string" 25 | }, 26 | "num_workers": { 27 | "description": "The total number of workers nodes to launch in addition to the head node. This number should be >= 0", 28 | "type": "integer", 29 | "minimum": 0 30 | }, 31 | "provider": { 32 | "type": "object", 33 | "description": "Cloud-provider specific configuration.", 34 | "required": [ "type" ], 35 | "additionalProperties": true, 36 | "properties": { 37 | "type": { 38 | "type": "string", 39 | "description": "e.g. aws, azure, gcp,..." 40 | }, 41 | "region": { 42 | "type": "string", 43 | "description": "e.g. us-east-1" 44 | }, 45 | "module": { 46 | "type": "string", 47 | "description": "module, if using external node provider" 48 | }, 49 | "head_ip": { 50 | "type": "string", 51 | "description": "gcp project id, if using gcp" 52 | }, 53 | "worker_ips": { 54 | "type": "array", 55 | "description": "local cluster head node" 56 | }, 57 | "use_internal_ips": { 58 | "type": "boolean", 59 | "description": "don't require public ips" 60 | }, 61 | "namespace": { 62 | "type": "string", 63 | "description": "k8s namespace, if using k8s" 64 | }, 65 | "location": { 66 | "type": "string", 67 | "description": "Azure location" 68 | }, 69 | "resource_group": { 70 | "type": "string", 71 | "description": "Azure resource group" 72 | }, 73 | "tags": { 74 | "type": "object", 75 | "description": "Azure user-defined tags" 76 | }, 77 | "subscription_id": { 78 | "type": "string", 79 | "description": "Azure subscription id" 80 | }, 81 | "msi_identity_id": { 82 | "type": "string", 83 | "description": "User-defined managed identity (generated by config)" 84 | }, 85 | "msi_identity_principal_id": { 86 | "type": "string", 87 | "description": "User-defined managed identity principal id (generated by config)" 88 | }, 89 | "subnet_id": { 90 | "type": "string", 91 | "description": "Network subnet id" 92 | }, 93 | "autoscaler_service_account": { 94 | "type": "object", 95 | "description": "k8s autoscaler permissions, if using k8s" 96 | }, 97 | "autoscaler_role": { 98 | "type": "object", 99 | "description": "k8s autoscaler permissions, if using k8s" 100 | }, 101 | "autoscaler_role_binding": { 102 | "type": "object", 103 | "description": "k8s autoscaler permissions, if using k8s" 104 | }, 105 | "cache_stopped_nodes": { 106 | "type": "boolean", 107 | "description": " Whether to try to reuse previously stopped nodes instead of launching nodes. This will also cause the autoscaler to stop nodes instead of terminating them. Only implemented for AWS." 108 | }, 109 | "availability_zone": { 110 | "type": "string", 111 | "description": "GCP availability zone" 112 | }, 113 | "project_id": { 114 | "type": ["string", "null"], 115 | "description": "GCP globally unique project id" 116 | }, 117 | "gcp_credentials": { 118 | "type": "string", 119 | "description": "JSON string constituting GCP credentials" 120 | } 121 | } 122 | }, 123 | "auth": { 124 | "type": "object", 125 | "description": "How MC2 will authenticate with newly launched nodes.", 126 | "additionalProperties": false, 127 | "properties": { 128 | "ssh_user": { 129 | "type": "string", 130 | "default": "ubuntu" 131 | }, 132 | "ssh_public_key": { 133 | "type": "string" 134 | }, 135 | "ssh_private_key": { 136 | "type": "string" 137 | } 138 | } 139 | }, 140 | "head_node": { 141 | "type": "object", 142 | "description": "Provider-specific config for the head node, e.g. instance type." 143 | }, 144 | "worker_nodes": { 145 | "type": "object", 146 | "description": "Provider-specific config for worker nodes. e.g. instance type." 147 | }, 148 | "file_mounts": { 149 | "type": "object", 150 | "description": "Map of remote paths to local paths, e.g. {\"/tmp/data\": \"/my/local/data\"}" 151 | }, 152 | "initialization_commands": { 153 | "$ref": "#/definitions/commands", 154 | "description": "List of commands that will be run before `setup_commands`. If docker is enabled, these commands will run outside the container and before docker is setup." 155 | }, 156 | "setup_commands": { 157 | "$ref": "#/definitions/commands", 158 | "description": "List of common shell commands to run to setup nodes." 159 | }, 160 | "head_setup_commands": { 161 | "$ref": "#/definitions/commands", 162 | "description": "Commands that will be run on the head node after common setup." 163 | }, 164 | "worker_setup_commands": { 165 | "$ref": "#/definitions/commands", 166 | "description": "Commands that will be run on worker nodes after common setup." 167 | }, 168 | "head_start_mc2_commands": { 169 | "$ref": "#/definitions/commands", 170 | "description": "Command to start MC2 on the head node. You shouldn't need to modify this." 171 | }, 172 | "worker_start_mc2_commands": { 173 | "$ref": "#/definitions/commands", 174 | "description": "Command to start MC2 on worker nodes. You shouldn't need to modify this." 175 | }, 176 | "available_instance_types": { 177 | "type": "object", 178 | "description": "A list of instance types available for launching with 'auto' worker type.", 179 | "patternProperties": { 180 | ".*": { 181 | "type": "object", 182 | "properties": { 183 | "max_workers": {"type": "integer"}, 184 | "resources": { 185 | "type": "object", 186 | ".*": {"type": "number"} 187 | } 188 | }, 189 | "additionalProperties": false 190 | } 191 | }, 192 | "additionalProperties": false 193 | } 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /python-package/mc2client/toolchain/mc2_azure/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc2-project/mc2/38a6888a60ea6f9e3a513fadc29c2123334e6cdf/python-package/mc2client/toolchain/mc2_azure/__init__.py -------------------------------------------------------------------------------- /python-package/mc2client/toolchain/mc2_azure/azure-config-template.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "subnet": { 6 | "type": "string", 7 | "metadata": { 8 | "description": "The subnet to be used" 9 | } 10 | } 11 | }, 12 | "resources": [ 13 | { 14 | "type": "Microsoft.ManagedIdentity/userAssignedIdentities", 15 | "apiVersion": "2018-11-30", 16 | "location": "[resourceGroup().location]", 17 | "name": "mc2-msi-user-identity" 18 | }, 19 | { 20 | "type": "Microsoft.Network/networkSecurityGroups", 21 | "apiVersion": "2019-02-01", 22 | "name": "mc2-nsg", 23 | "location": "[resourceGroup().location]", 24 | "properties": { 25 | "securityRules": [ 26 | { 27 | "name": "SSH", 28 | "properties": { 29 | "priority": 1000, 30 | "protocol": "TCP", 31 | "access": "Allow", 32 | "direction": "Inbound", 33 | "sourceAddressPrefix": "*", 34 | "sourcePortRange": "*", 35 | "destinationAddressPrefix": "*", 36 | "destinationPortRange": "22" 37 | } 38 | }, 39 | { 40 | "name": "gRPC", 41 | "properties": { 42 | "priority": 1010, 43 | "protocol": "TCP", 44 | "access": "Allow", 45 | "direction": "Inbound", 46 | "sourceAddressPrefix": "*", 47 | "sourcePortRange": "*", 48 | "destinationAddressPrefix": "*", 49 | "destinationPortRange": "50051-50055" 50 | } 51 | } 52 | ] 53 | } 54 | }, 55 | { 56 | "type": "Microsoft.Network/virtualNetworks", 57 | "apiVersion": "2019-11-01", 58 | "name": "mc2-vnet", 59 | "location": "[resourceGroup().location]", 60 | "properties": { 61 | "addressSpace": { 62 | "addressPrefixes": [ 63 | "[parameters('subnet')]" 64 | ] 65 | }, 66 | "subnets": [ 67 | { 68 | "name": "mc2-subnet", 69 | "properties": { 70 | "addressPrefix": "[parameters('subnet')]", 71 | "networkSecurityGroup": { 72 | "id": "[resourceId('Microsoft.Network/networkSecurityGroups','mc2-nsg')]" 73 | } 74 | } 75 | } 76 | ] 77 | }, 78 | "dependsOn": [ 79 | "[resourceId('Microsoft.Network/networkSecurityGroups', 'mc2-nsg')]" 80 | ] 81 | } 82 | ] 83 | } 84 | -------------------------------------------------------------------------------- /python-package/mc2client/toolchain/mc2_azure/config.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | import random 5 | 6 | from azure.common.client_factory import get_client_from_cli_profile 7 | from azure.mgmt.resource import ResourceManagementClient 8 | from azure.mgmt.resource.resources.models import DeploymentMode 9 | 10 | RETRIES = 30 11 | MSI_NAME = "mc2-msi-user-identity" 12 | NSG_NAME = "mc2-nsg" 13 | SUBNET_NAME = "mc2-subnet" 14 | VNET_NAME = "mc2-vnet" 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | 19 | def bootstrap_azure(config): 20 | config = _configure_key_pair(config) 21 | config = _configure_resource_group(config) 22 | return config 23 | 24 | 25 | def _get_client(client_class, config): 26 | kwargs = {} 27 | if "subscription_id" in config["provider"]: 28 | kwargs["subscription_id"] = config["provider"]["subscription_id"] 29 | 30 | return get_client_from_cli_profile(client_class=client_class, **kwargs) 31 | 32 | 33 | def create_or_delete_resource_group(config, create): 34 | # TODO: look at availability sets 35 | # https://docs.microsoft.com/en-us/azure/virtual-machines/windows/tutorial-availability-sets 36 | resource_client = _get_client(ResourceManagementClient, config) 37 | 38 | subscription_id = resource_client.config.subscription_id 39 | logger.info("Using subscription id: %s", subscription_id) 40 | config["provider"]["subscription_id"] = subscription_id 41 | 42 | assert ( 43 | "resource_group" in config["provider"] 44 | ), "Provider config must include resource_group field" 45 | resource_group = config["provider"]["resource_group"] 46 | 47 | assert ( 48 | "location" in config["provider"] 49 | ), "Provider config must include location field" 50 | params = {"location": config["provider"]["location"]} 51 | 52 | if "tags" in config["provider"]: 53 | params["tags"] = config["provider"]["tags"] 54 | 55 | if create: 56 | logger.info("Creating/Updating Resource Group: %s", resource_group) 57 | resource_client.resource_groups.create_or_update( 58 | resource_group_name=resource_group, parameters=params 59 | ) 60 | else: 61 | logger.info("Deleting Resource Group: %s", resource_group) 62 | delete_async_op = resource_client.resource_groups.delete( 63 | resource_group 64 | ) 65 | delete_async_op.wait() 66 | 67 | 68 | def _configure_resource_group(config): 69 | # TODO: look at availability sets 70 | # https://docs.microsoft.com/en-us/azure/virtual-machines/windows/tutorial-availability-sets 71 | resource_client = _get_client(ResourceManagementClient, config) 72 | 73 | subscription_id = resource_client.config.subscription_id 74 | logger.info("Using subscription id: %s", subscription_id) 75 | config["provider"]["subscription_id"] = subscription_id 76 | 77 | assert ( 78 | "resource_group" in config["provider"] 79 | ), "Provider config must include resource_group field" 80 | resource_group = config["provider"]["resource_group"] 81 | 82 | assert ( 83 | "location" in config["provider"] 84 | ), "Provider config must include location field" 85 | params = {"location": config["provider"]["location"]} 86 | 87 | if "tags" in config["provider"]: 88 | params["tags"] = config["provider"]["tags"] 89 | 90 | logger.info("Creating/Updating Resource Group: %s", resource_group) 91 | resource_client.resource_groups.create_or_update( 92 | resource_group_name=resource_group, parameters=params 93 | ) 94 | 95 | # load the template file 96 | current_path = os.path.dirname(os.path.abspath(__file__)) 97 | template_path = os.path.join(current_path, "azure-config-template.json") 98 | 99 | with open(template_path, "r") as template_fp: 100 | template = json.load(template_fp) 101 | 102 | # choose a random subnet, skipping most common value of 0 103 | random.seed(resource_group) 104 | subnet_mask = "10.{}.0.0/16".format(random.randint(1, 254)) 105 | 106 | parameters = { 107 | "properties": { 108 | "mode": DeploymentMode.incremental, 109 | "template": template, 110 | "parameters": {"subnet": {"value": subnet_mask}}, 111 | } 112 | } 113 | 114 | resource_client.deployments.create_or_update( 115 | resource_group_name=resource_group, 116 | deployment_name="mc2-config", 117 | parameters=parameters, 118 | ).wait() 119 | 120 | return config 121 | 122 | 123 | def _configure_key_pair(config): 124 | ssh_user = config["auth"]["ssh_user"] 125 | # search if the keys exist 126 | for key_type in ["ssh_private_key", "ssh_public_key"]: 127 | try: 128 | key_path = os.path.expanduser(config["auth"][key_type]) 129 | except KeyError: 130 | raise Exception("Config must define {}".format(key_type)) 131 | except TypeError: 132 | raise Exception("Invalid config value for {}".format(key_type)) 133 | 134 | assert os.path.exists(key_path), "Could not find ssh key: {}".format( 135 | key_path 136 | ) 137 | 138 | if key_type == "ssh_public_key": 139 | with open(key_path, "r") as f: 140 | public_key = f.read() 141 | 142 | for node_type in ["head_node", "worker_nodes"]: 143 | config[node_type]["azure_arm_parameters"]["adminUsername"] = ssh_user 144 | config[node_type]["azure_arm_parameters"]["publicKey"] = public_key 145 | 146 | return config 147 | -------------------------------------------------------------------------------- /python-package/mc2client/toolchain/mc2_azure/example-full.yaml: -------------------------------------------------------------------------------- 1 | # An unique identifier for the head node and workers of this cluster. 2 | cluster_name: default 3 | 4 | # The total number of workers nodes to launch in addition to the head 5 | # node. This number should be >= 0. 6 | num_workers: 0 7 | 8 | # Cloud-provider specific configuration. 9 | provider: 10 | type: azure 11 | # https://docs.microsoft.com/en-us/azure/confidential-computing/virtual-machine-solutions 12 | location: eastus 13 | resource_group: mc2-client-dev 14 | storage_name: mc2storage 15 | container_name: blob-container-1 16 | # set subscription id otherwise the default from az cli will be used 17 | # subscription_id: 18 | 19 | # How MC2 will authenticate with newly launched nodes. 20 | auth: 21 | # TODO: remove this field and make it the same as the username specified in config.yaml 22 | ssh_user: mc2 23 | # you must specify paths to matching private and public key pair files 24 | # use `ssh-keygen -t rsa -b 4096` to generate a new ssh key pair 25 | ssh_private_key: ~/.ssh/id_rsa 26 | ssh_public_key: ~/.ssh/id_rsa.pub 27 | 28 | # More specific customization to node configurations can be made using the ARM template azure-vm-template.json file 29 | # See documentation here: https://docs.microsoft.com/en-us/azure/templates/microsoft.compute/2019-03-01/virtualmachines 30 | # Changes to the local file will be used during deployment of the head node, however worker nodes deployment occurs 31 | # on the head node, so changes to the template must be included in the wheel file used in setup_commands section below 32 | 33 | # Provider-specific config for the head node, e.g. instance type. 34 | head_node: 35 | azure_arm_parameters: 36 | # https://docs.microsoft.com/en-us/azure/confidential-computing/virtual-machine-solutions 37 | vmSize: Standard_DC2s_v2 38 | 39 | # If launching a minimal Ubuntu machine 40 | # (and manually installing using setup commands) 41 | imagePublisher: Canonical 42 | imageOffer: UbuntuServer 43 | imageSku: 18_04-lts-gen2 44 | imageVersion: latest 45 | 46 | # Provider-specific config for worker nodes, e.g. instance type. 47 | worker_nodes: 48 | azure_arm_parameters: 49 | # https://docs.microsoft.com/en-us/azure/confidential-computing/virtual-machine-solutions 50 | vmSize: Standard_DC2s_v2 51 | 52 | # If launching a minimal Ubuntu machine 53 | # (and manually installing using setup commands) 54 | imagePublisher: Canonical 55 | imageOffer: UbuntuServer 56 | imageSku: 18_04-lts-gen2 57 | imageVersion: latest 58 | 59 | ############################################################################## 60 | # Everything below this can be ignored - you likely won't have to # 61 | # modify it. # 62 | ############################################################################## 63 | 64 | # Files or directories to copy to the head and worker nodes. The format is a 65 | # dictionary from REMOTE_PATH: LOCAL_PATH, e.g. 66 | file_mounts: { 67 | # # This script installs Open Enclave 68 | # "~/install_oe.sh" : "scripts/install_oe.sh", 69 | # # This script builds Spark 3.1.1 from source 70 | # "~/build_spark.sh" : "scripts/build_spark.sh", 71 | # # This script downloads a pre-built Spark 3.1.1 binary 72 | # "~/install_spark.sh" : "scripts/install_spark.sh", 73 | # # This script builds Opaque from source 74 | # "~/build_opaque.sh" : "scripts/build_opaque.sh", 75 | # # This script installs Secure XGBoost from source 76 | # "~/install_secure_xgboost.sh" : "scripts/install_secure_xgboost.sh" 77 | } 78 | 79 | # List of commands that will be run before `setup_commands`. If docker is 80 | # enabled, these commands will run outside the container and before docker 81 | # is setup. 82 | initialization_commands: 83 | # get rid of annoying Ubuntu message 84 | - touch ~/.sudo_as_admin_successful 85 | 86 | # List of shell commands to run to set up nodes. 87 | # Note: Use empty list if using image 88 | setup_commands: 89 | # # This script installs Open Enclave on the node 90 | # - chmod +x ~/install_oe.sh 91 | # - source ~/install_oe.sh 92 | # # This script installs Apache Spark on the node 93 | # - chmod +x ~/install_spark.sh 94 | # - source ~/install_spark.sh 95 | # # This script installs Opaque on the node 96 | # - chmod +x ~/build_opaque.sh 97 | # - source ~/build_opaque.sh 98 | # # This script installs Secure XGBoost on the node 99 | # - chmod +x ~/install_secure_xgboost.sh 100 | # - source ~/install_secure_xgboost.sh 101 | 102 | # Custom commands that will be run on the head node after common setup. 103 | # Set to empty list if using image 104 | head_setup_commands: [] 105 | 106 | # Custom commands that will be run on worker nodes after common setup. 107 | # Set to empty list if using image 108 | worker_setup_commands: [] 109 | 110 | # Command to start MC2 on the head node. 111 | # Set to empty list if using image 112 | head_start_mc2_commands: 113 | # - cd $SPARK_HOME; ./sbin/start-master.sh 114 | 115 | # Command to start MC2 on worker nodes. 116 | # Set to empty list if using image 117 | worker_start_mc2_commands: 118 | # - cd $SPARK_HOME; ./sbin/start-slave.sh $MC2_HEAD_IP:7077 119 | -------------------------------------------------------------------------------- /python-package/mc2client/toolchain/node_provider.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import logging 3 | import os 4 | import yaml 5 | 6 | from .updater import SSHCommandRunner 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | 11 | def import_azure(): 12 | from .mc2_azure.config import bootstrap_azure 13 | from .mc2_azure.node_provider import AzureNodeProvider 14 | 15 | return bootstrap_azure, AzureNodeProvider 16 | 17 | 18 | def load_azure_example_config(): 19 | from . import mc2_azure 20 | 21 | return os.path.join( 22 | os.path.dirname(mc2_azure.__file__), "example-full.yaml" 23 | ) 24 | 25 | 26 | NODE_PROVIDERS = { 27 | "azure": import_azure, 28 | } 29 | 30 | DEFAULT_CONFIGS = { 31 | "azure": load_azure_example_config, 32 | } 33 | 34 | 35 | def load_class(path): 36 | """ 37 | Load a class at runtime given a full path. 38 | 39 | Example of the path: mypkg.mysubpkg.myclass 40 | """ 41 | class_data = path.split(".") 42 | if len(class_data) < 2: 43 | raise ValueError( 44 | "You need to pass a valid path like mymodule.provider_class" 45 | ) 46 | module_path = ".".join(class_data[:-1]) 47 | class_str = class_data[-1] 48 | module = importlib.import_module(module_path) 49 | return getattr(module, class_str) 50 | 51 | 52 | def get_node_provider(provider_config, cluster_name): 53 | if provider_config["type"] == "external": 54 | provider_cls = load_class(path=provider_config["module"]) 55 | return provider_cls(provider_config, cluster_name) 56 | 57 | importer = NODE_PROVIDERS.get(provider_config["type"]) 58 | 59 | if importer is None: 60 | raise NotImplementedError( 61 | "Unsupported node provider: {}".format(provider_config["type"]) 62 | ) 63 | _, provider_cls = importer() 64 | return provider_cls(provider_config, cluster_name) 65 | 66 | 67 | def get_default_config(provider_config): 68 | if provider_config["type"] == "external": 69 | return {} 70 | load_config = DEFAULT_CONFIGS.get(provider_config["type"]) 71 | if load_config is None: 72 | raise NotImplementedError( 73 | "Unsupported node provider: {}".format(provider_config["type"]) 74 | ) 75 | path_to_default = load_config() 76 | defaults = yaml.safe_load(open(path_to_default).read()) 77 | return dict(defaults) 78 | 79 | 80 | class NodeProvider: 81 | """Interface for getting and returning nodes from a Cloud. 82 | 83 | NodeProviders are namespaced by the `cluster_name` parameter; they only 84 | operate on nodes within that namespace. 85 | 86 | Nodes may be in one of three states: {pending, running, terminated}. Nodes 87 | appear immediately once started by `create_node`, and transition 88 | immediately to terminated when `terminate_node` is called. 89 | """ 90 | 91 | def __init__(self, provider_config, cluster_name): 92 | self.provider_config = provider_config 93 | self.cluster_name = cluster_name 94 | 95 | def non_terminated_nodes(self, tag_filters): 96 | """Return a list of node ids filtered by the specified tags dict. 97 | 98 | This list must not include terminated nodes. For performance reasons, 99 | providers are allowed to cache the result of a call to nodes() to 100 | serve single-node queries (e.g. is_running(node_id)). This means that 101 | nodes() must be called again to refresh results. 102 | 103 | Examples: 104 | >>> provider.non_terminated_nodes({TAG_MC2_NODE_TYPE: "worker"}) 105 | ["node-1", "node-2"] 106 | """ 107 | raise NotImplementedError 108 | 109 | def get_head_node(self): 110 | """Return the head node id.""" 111 | raise NotImplementedError 112 | 113 | def get_worker_nodes(self): 114 | """Return the worker node ids.""" 115 | raise NotImplementedError 116 | 117 | def is_running(self, node_id): 118 | """Return whether the specified node is running.""" 119 | raise NotImplementedError 120 | 121 | def is_terminated(self, node_id): 122 | """Return whether the specified node is terminated.""" 123 | raise NotImplementedError 124 | 125 | def node_tags(self, node_id): 126 | """Returns the tags of the given node (string dict).""" 127 | raise NotImplementedError 128 | 129 | def external_ip(self, node_id): 130 | """Returns the external ip of the given node.""" 131 | raise NotImplementedError 132 | 133 | def internal_ip(self, node_id): 134 | """Returns the internal ip (Ray ip) of the given node.""" 135 | raise NotImplementedError 136 | 137 | def create_node(self, node_config, tags, count): 138 | """Creates a number of nodes within the namespace.""" 139 | raise NotImplementedError 140 | 141 | def set_node_tags(self, node_id, tags): 142 | """Sets the tag values (string dict) for the specified node.""" 143 | raise NotImplementedError 144 | 145 | def terminate_node(self, node_id): 146 | """Terminates the specified node.""" 147 | raise NotImplementedError 148 | 149 | def terminate_nodes(self, node_ids): 150 | """Terminates a set of nodes. May be overridden with a batch method.""" 151 | for node_id in node_ids: 152 | logger.info("NodeProvider: {}: Terminating node".format(node_id)) 153 | self.terminate_node(node_id) 154 | 155 | def cleanup(self): 156 | """Clean-up when a Provider is no longer required.""" 157 | pass 158 | 159 | def create_node_of_type(self, node_config, tags, instance_type, count): 160 | """Creates a number of nodes with a given instance type. 161 | 162 | This is an optional method only required if using the resource 163 | demand scheduler. 164 | """ 165 | assert instance_type is not None 166 | raise NotImplementedError 167 | 168 | def get_instance_type(self, node_config): 169 | """Returns the instance type of this node config. 170 | 171 | This is an optional method only required if using the resource 172 | demand scheduler.""" 173 | return None 174 | 175 | def get_command_runner( 176 | self, 177 | log_prefix, 178 | node_id, 179 | auth_config, 180 | cluster_name, 181 | process_runner, 182 | use_internal_ip, 183 | docker_config=None, 184 | ): 185 | """Returns the CommandRunner class used to perform SSH commands. 186 | 187 | Args: 188 | log_prefix(str): stores "NodeUpdater: {}: ".format(). Used 189 | to print progress in the CommandRunner. 190 | node_id(str): the node ID. 191 | auth_config(dict): the authentication configs from the autoscaler 192 | yaml file. 193 | cluster_name(str): the name of the cluster. 194 | process_runner(module): the module to use to run the commands 195 | in the CommandRunner. E.g., subprocess. 196 | use_internal_ip(bool): whether the node_id belongs to an internal ip 197 | or external ip. 198 | docker_config(dict): If set, the docker information of the docker 199 | container that commands should be run on. 200 | """ 201 | common_args = { 202 | "log_prefix": log_prefix, 203 | "node_id": node_id, 204 | "provider": self, 205 | "auth_config": auth_config, 206 | "cluster_name": cluster_name, 207 | "process_runner": process_runner, 208 | "use_internal_ip": use_internal_ip, 209 | } 210 | if docker_config and docker_config["container_name"] != "": 211 | raise Exception("DockerCommandRunner not currently supported") 212 | # return DockerCommandRunner(docker_config, **common_args) 213 | else: 214 | return SSHCommandRunner(**common_args) 215 | -------------------------------------------------------------------------------- /python-package/mc2client/toolchain/scripts/build_opaque.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Clone and build Opaque 4 | git clone https://github.com/mc2-project/opaque-sql.git 5 | 6 | # Set environment variables 7 | export OPAQUE_HOME=~/opaque-sql 8 | export OPAQUE_DATA_DIR=${OPAQUE_HOME}/data/ 9 | export SPARK_SCALA_VERSION=2.12 10 | export PRIVATE_KEY_PATH=${OPAQUE_HOME}/src/test/keys/mc2_test_key.pem 11 | export MODE=HARDWARE 12 | export OE_SDK_PATH=/opt/openenclave/ 13 | 14 | # Generate keys 15 | cd opaque-sql 16 | build/sbt keys 17 | 18 | # Set environment variables permanently 19 | echo "export OPAQUE_HOME=~/opaque-sql" >> ~/.bashrc 20 | echo "export OPAQUE_DATA_DIR=${OPAQUE_HOME}/data" >> ~/.bashrc 21 | echo "export SPARK_SCALA_VERSION=2.12" >> ~/.bashrc 22 | echo "export PRIVATE_KEY_PATH=${OPAQUE_HOME}/src/test/keys/mc2_test_key.pem" >> ~/.bashrc 23 | echo "export MODE=HARDWARE" >> ~/.bashrc 24 | echo "export OE_SDK_PATH=/opt/openenclave/" >> ~/.bashrc 25 | 26 | # Build Opaque SQL 27 | build/sbt package 28 | 29 | -------------------------------------------------------------------------------- /python-package/mc2client/toolchain/scripts/build_spark.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Install Java 8 4 | sudo apt-get update 5 | sudo apt-get -y install openjdk-8-jdk openjdk-8-jre 6 | 7 | # Build Spark 3.1.1 8 | wget https://archive.apache.org/dist/spark/spark-3.1.1/spark-3.1.1.tgz 9 | tar -xvzf spark-3.1.1.tgz 10 | cd spark-3.1.1 11 | ./build/mvn -DskipTests clean package 12 | 13 | # Set environment variables 14 | echo "" >> ~/.bashrc 15 | echo "export SPARK_HOME=$PWD" >> ~/.bashrc 16 | echo "export SPARK_SCALA_VERSION=2.12" >> ~/.bashrc 17 | source ~/.bashrc 18 | 19 | # Opaque needs these configs to be set 20 | touch $SPARK_HOME/conf/spark-defaults.conf 21 | echo "" >> $SPARK_HOME/conf/spark-defaults.conf 22 | echo "spark.executor.instances 1" >> $SPARK_HOME/conf/spark-defaults.conf 23 | echo "spark.task.maxFailures 10" >> $SPARK_HOME/conf/spark-defaults.conf 24 | -------------------------------------------------------------------------------- /python-package/mc2client/toolchain/scripts/install_oe.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | if [[ $(lsb_release -is) != "Ubuntu" ]]; then 6 | echo "Not installing Open Enclave: unsupported OS distribution." 7 | exit 1 8 | 9 | elif [[ $(lsb_release -rs) == "18.04" ]]; then 10 | # Install Open Enclave on Ubuntu 18.04 11 | 12 | # Configure the Intel and Microsoft APT Repositories 13 | echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bionic main' | sudo tee /etc/apt/sources.list.d/intel-sgx.list 14 | wget -qO - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | sudo apt-key add - 15 | 16 | echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-7 main" | sudo tee /etc/apt/sources.list.d/llvm-toolchain-bionic-7.list 17 | wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - 18 | 19 | echo "deb [arch=amd64] https://packages.microsoft.com/ubuntu/18.04/prod bionic main" | sudo tee /etc/apt/sources.list.d/msprod.list 20 | wget -qO - https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - 21 | 22 | # Install the Intel SGX DCAP Driver 23 | sudo apt update 24 | sudo apt -y install dkms 25 | wget https://download.01.org/intel-sgx/sgx-dcap/1.9/linux/distro/ubuntu18.04-server/sgx_linux_x64_driver_1.36.2.bin 26 | chmod +x sgx_linux_x64_driver_1.36.2.bin 27 | sudo ./sgx_linux_x64_driver_1.36.2.bin 28 | 29 | # Install the Intel and Open Enclave packages and dependencies 30 | sudo apt -y install clang-8 libssl-dev gdb libsgx-enclave-common libsgx-quote-ex libprotobuf10 libsgx-dcap-ql libsgx-dcap-ql-dev az-dcap-client open-enclave=0.17.1 31 | 32 | # Configure OE environment variables 33 | echo "source /opt/openenclave/share/openenclave/openenclaverc" >> ~/.bashrc 34 | source /opt/openenclave/share/openenclave/openenclaverc 35 | 36 | elif [[ $(lsb_release -rs) == "16.04" ]]; then 37 | # Install Open Enclave on Ubuntu 16.04 38 | 39 | # Configure the Intel and Microsoft APT Repositories 40 | echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu xenial main' | sudo tee /etc/apt/sources.list.d/intel-sgx.list 41 | wget -qO - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | sudo apt-key add - 42 | 43 | echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main" | sudo tee /etc/apt/sources.list.d/llvm-toolchain-xenial-7.list 44 | wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - 45 | 46 | echo "deb [arch=amd64] https://packages.microsoft.com/ubuntu/16.04/prod xenial main" | sudo tee /etc/apt/sources.list.d/msprod.list 47 | wget -qO - https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - 48 | 49 | # Install the Intel SGX DCAP Driver 50 | sudo apt update 51 | sudo apt -y install dkms 52 | wget https://download.01.org/intel-sgx/sgx-dcap/1.9/linux/distro/ubuntu16.04-server/sgx_linux_x64_driver_1.36.2.bin 53 | chmod +x sgx_linux_x64_driver_1.36.2.bin 54 | sudo ./sgx_linux_x64_driver_1.36.2.bin 55 | 56 | # Install the Intel and Open Enclave packages and dependencies 57 | sudo apt -y install clang-8 libssl-dev gdb libsgx-enclave-common libsgx-quote-ex libprotobuf10 libsgx-dcap-ql libsgx-dcap-ql-dev az-dcap-client open-enclave=0.17.1 58 | 59 | # Configure OE environment variables 60 | echo "source /opt/openenclave/share/openenclave/openenclaverc" >> ~/.bashrc 61 | source /opt/openenclave/share/openenclave/openenclaverc 62 | 63 | else 64 | echo "Not installing Open Enclave: unsupported Ubuntu version." 65 | exit 1 66 | fi 67 | 68 | # CMake 69 | wget https://github.com/Kitware/CMake/releases/download/v3.15.6/cmake-3.15.6-Linux-x86_64.sh 70 | sudo bash cmake-3.15.6-Linux-x86_64.sh --skip-license --prefix=/usr/local 71 | 72 | # Make 73 | sudo apt-get install make 74 | 75 | # Mbed TLS 76 | sudo apt-get install -y libmbedtls-dev 77 | -------------------------------------------------------------------------------- /python-package/mc2client/toolchain/scripts/install_secure_xgboost.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sudo apt-get install -y libmbedtls-dev python3-pip 4 | pip3 install numpy pandas sklearn numproto grpcio grpcio-tools 5 | 6 | git clone https://github.com/mc2-project/secure-xgboost.git 7 | 8 | cd secure-xgboost 9 | mkdir build 10 | 11 | cd build 12 | cmake .. 13 | make -j4 14 | 15 | cd ../python-package 16 | sudo python3 setup.py install 17 | -------------------------------------------------------------------------------- /python-package/mc2client/toolchain/scripts/install_spark.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Install Java 8 4 | sudo apt-get update 5 | sudo apt-get -y install openjdk-8-jdk openjdk-8-jre 6 | 7 | # Download pre-built Spark 8 | wget https://archive.apache.org/dist/spark/spark-3.1.1/spark-3.1.1-bin-hadoop2.7.tgz 9 | tar -xzvf spark-3.1.1-bin-hadoop2.7.tgz 10 | 11 | # Set environment variables 12 | echo "" >> ~/.bashrc 13 | echo "export SPARK_HOME=$PWD/spark-3.1.1-bin-hadoop2.7" >> ~/.bashrc 14 | echo "export SPARK_SCALA_VERSION=2.12" >> ~/.bashrc 15 | source ~/.bashrc 16 | 17 | # Opaque needs these configs to be set 18 | touch $SPARK_HOME/conf/spark-defaults.conf 19 | echo "" >> $SPARK_HOME/conf/spark-defaults.conf 20 | echo "spark.executor.instances 1" >> $SPARK_HOME/conf/spark-defaults.conf 21 | echo "spark.task.maxFailures 10" >> $SPARK_HOME/conf/spark-defaults.conf 22 | -------------------------------------------------------------------------------- /python-package/mc2client/toolchain/tags.py: -------------------------------------------------------------------------------- 1 | """Tags/labels are used to associate metadata with instances.""" 2 | 3 | # Tag for the name of the node 4 | TAG_MC2_NODE_NAME = "mc2-node-name" 5 | 6 | # Tag for the type of node (e.g. Head, Worker) 7 | TAG_MC2_NODE_TYPE = "mc2-node-type" 8 | NODE_TYPE_HEAD = "head" 9 | NODE_TYPE_WORKER = "worker" 10 | 11 | # Tag for the provider-specific instance type (e.g., m4.4xlarge). This is used 12 | # for automatic worker instance type selection. 13 | TAG_MC2_INSTANCE_TYPE = "mc2-instance-type" 14 | 15 | # Tag that reports the current state of the node (e.g. Updating, Up-to-date) 16 | TAG_MC2_NODE_STATUS = "mc2-node-status" 17 | STATUS_UNINITIALIZED = "uninitialized" 18 | STATUS_WAITING_FOR_SSH = "waiting-for-ssh" 19 | STATUS_SYNCING_FILES = "syncing-files" 20 | STATUS_SETTING_UP = "setting-up" 21 | STATUS_UPDATE_FAILED = "update-failed" 22 | STATUS_UP_TO_DATE = "up-to-date" 23 | 24 | # Tag uniquely identifying all nodes of a cluster 25 | TAG_MC2_CLUSTER_NAME = "mc2-cluster-name" 26 | 27 | # Hash of the node launch config, used to identify out-of-date nodes 28 | TAG_MC2_LAUNCH_CONFIG = "mc2-launch-config" 29 | 30 | # Hash of the node runtime config, used to determine if updates are needed 31 | TAG_MC2_RUNTIME_CONFIG = "mc2-runtime-config" 32 | -------------------------------------------------------------------------------- /python-package/mc2client/toolchain/util.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import json 3 | import os 4 | 5 | import jsonschema 6 | 7 | from .node_provider import get_default_config 8 | 9 | MC2_SCHEMA_PATH = os.path.join(os.path.dirname(__file__), "mc2-schema.json") 10 | 11 | 12 | def validate_config(config): 13 | """Required Dicts indicate that no extra fields can be introduced.""" 14 | if not isinstance(config, dict): 15 | raise ValueError("Config {} is not a dictionary".format(config)) 16 | 17 | with open(MC2_SCHEMA_PATH) as f: 18 | schema = json.load(f) 19 | try: 20 | jsonschema.validate(config, schema) 21 | except jsonschema.ValidationError as e: 22 | raise jsonschema.ValidationError(message=e.message) from None 23 | 24 | 25 | def prepare_config(config): 26 | with_defaults = fillout_defaults(config) 27 | merge_setup_commands(with_defaults) 28 | return with_defaults 29 | 30 | 31 | def fillout_defaults(config): 32 | defaults = get_default_config(config["provider"]) 33 | defaults.update(config) 34 | defaults["auth"] = defaults.get("auth", {}) 35 | return defaults 36 | 37 | 38 | def merge_setup_commands(config): 39 | config["head_setup_commands"] = ( 40 | config["setup_commands"] + config["head_setup_commands"] 41 | ) 42 | config["worker_setup_commands"] = ( 43 | config["setup_commands"] + config["worker_setup_commands"] 44 | ) 45 | return config 46 | 47 | 48 | def hash_launch_conf(node_conf, auth): 49 | hasher = hashlib.sha1() 50 | hasher.update( 51 | json.dumps([node_conf, auth], sort_keys=True).encode("utf-8") 52 | ) 53 | return hasher.hexdigest() 54 | 55 | 56 | # Cache the file hashes to avoid rescanning it each time. Also, this avoids 57 | # inadvertently restarting workers if the file mount content is mutated on the 58 | # head node. 59 | _hash_cache = {} 60 | 61 | 62 | def hash_runtime_conf(file_mounts, extra_objs): 63 | hasher = hashlib.sha1() 64 | 65 | def add_content_hashes(path): 66 | def add_hash_of_file(fpath): 67 | opaque_client_toolchain_path = os.path.dirname(__file__) 68 | fpath = os.path.join(opaque_client_toolchain_path, fpath) 69 | with open(fpath, "rb") as f: 70 | for chunk in iter(lambda: f.read(2 ** 20), b""): 71 | hasher.update(chunk) 72 | 73 | path = os.path.expanduser(path) 74 | if os.path.isdir(path): 75 | dirs = [] 76 | for dirpath, _, filenames in os.walk(path): 77 | dirs.append((dirpath, sorted(filenames))) 78 | for dirpath, filenames in sorted(dirs): 79 | hasher.update(dirpath.encode("utf-8")) 80 | for name in filenames: 81 | hasher.update(name.encode("utf-8")) 82 | fpath = os.path.join(dirpath, name) 83 | add_hash_of_file(fpath) 84 | else: 85 | add_hash_of_file(path) 86 | 87 | conf_str = json.dumps(file_mounts, sort_keys=True).encode( 88 | "utf-8" 89 | ) + json.dumps(extra_objs, sort_keys=True).encode("utf-8") 90 | 91 | # Important: only hash the files once. Otherwise, we can end up restarting 92 | # workers if the files were changed and we re-hashed them. 93 | if conf_str not in _hash_cache: 94 | hasher.update(conf_str) 95 | for local_path in sorted(file_mounts.values()): 96 | add_content_hashes(local_path) 97 | _hash_cache[conf_str] = hasher.hexdigest() 98 | 99 | return _hash_cache[conf_str] 100 | -------------------------------------------------------------------------------- /python-package/mc2client/xgb/__init__.py: -------------------------------------------------------------------------------- 1 | from . import rabit # noqa 2 | from .securexgboost import Booster, DMatrix 3 | 4 | __all__ = ["DMatrix", "Booster"] 5 | -------------------------------------------------------------------------------- /python-package/mc2client/xgb/rabit.py: -------------------------------------------------------------------------------- 1 | import grpc 2 | 3 | from ..core import _CONF 4 | from ..exceptions import MC2ClientConfigError 5 | from ..rpc import ( # pylint: disable=no-name-in-module 6 | remote_pb2, 7 | remote_pb2_grpc, 8 | ) 9 | from .securexgboost import _check_remote_call 10 | 11 | 12 | def init(args=None): 13 | """Initialize the rabit library with arguments""" 14 | 15 | channel_addr = _CONF.get("remote_addr") 16 | current_user = _CONF.get("current_user") 17 | 18 | if channel_addr is None: 19 | raise MC2ClientConfigError( 20 | "Remote orchestrator IP not set. Run oc.create_cluster() \ 21 | to launch VMs and configure IPs automatically or explicitly set it in the user YAML." 22 | ) 23 | 24 | if current_user is None: 25 | raise MC2ClientConfigError("Username not set") 26 | 27 | # FIXME: add signature to rabit init 28 | with grpc.insecure_channel(channel_addr) as channel: 29 | stub = remote_pb2_grpc.RemoteStub(channel) 30 | _check_remote_call( 31 | stub.rpc_RabitInit( 32 | remote_pb2.RabitParams( 33 | params=remote_pb2.Status(status=1), username=current_user 34 | ) 35 | ) 36 | ) 37 | 38 | 39 | def finalize(): 40 | """Finalize the process, notify tracker everything is done.""" 41 | channel_addr = _CONF.get("remote_addr") 42 | current_user = _CONF.get("current_user") 43 | 44 | if channel_addr is None: 45 | raise MC2ClientConfigError( 46 | "Remote orchestrator IP not set. Run oc.create_cluster() \ 47 | to launch VMs and configure IPs automatically or explicitly set it in the user YAML." 48 | ) 49 | 50 | if current_user is None: 51 | raise MC2ClientConfigError("Username not set") 52 | 53 | # FIXME: add signature to rabit finalize 54 | with grpc.insecure_channel(channel_addr) as channel: 55 | stub = remote_pb2_grpc.RemoteStub(channel) 56 | _check_remote_call( 57 | stub.rpc_RabitFinalize( 58 | remote_pb2.RabitParams( 59 | params=remote_pb2.Status(status=1), username=current_user 60 | ) 61 | ) 62 | ) 63 | -------------------------------------------------------------------------------- /python-package/setup.py: -------------------------------------------------------------------------------- 1 | import multiprocessing 2 | import os 3 | import shutil 4 | import subprocess 5 | import sys 6 | 7 | from setuptools import find_packages, setup 8 | 9 | # Get the absolute path of setup.py 10 | setup_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) 11 | 12 | # ---- Generate gRPC files ---- 13 | rpc_path = os.path.join(setup_path, "mc2client/rpc") 14 | subprocess.Popen( 15 | [ 16 | "python3", 17 | "-m", 18 | "grpc_tools.protoc", 19 | "-I", 20 | os.path.join(rpc_path, "protos"), 21 | f"--python_out={rpc_path}", 22 | f"--grpc_python_out={rpc_path}", 23 | os.path.join(rpc_path, "protos/ndarray.proto"), 24 | os.path.join(rpc_path, "protos/opaquesql.proto"), 25 | os.path.join(rpc_path, "protos/remote.proto"), 26 | os.path.join(rpc_path, "protos/attest.proto"), 27 | ] 28 | ) 29 | 30 | # ---- Run CMake build ---- 31 | 32 | # Create the build directory 33 | build_path = os.path.join(setup_path, "../src/build") 34 | if not os.path.exists(build_path): 35 | os.makedirs(build_path) 36 | 37 | # Call CMake and then Make (number of threads is equal to the number of 38 | # physical CPU cores) 39 | subprocess.check_call( 40 | ["cmake", os.path.join(build_path, "../")], cwd=build_path 41 | ) 42 | subprocess.check_call( 43 | ["make", "-j", str(multiprocessing.cpu_count())], cwd=build_path 44 | ) 45 | 46 | # ---- Generate flatbuffers files ---- 47 | 48 | config_fb_path = os.path.join(setup_path, "mc2client/toolchain/flatbuffers") 49 | flatc_path = os.path.join( 50 | setup_path, 51 | "../src/build/_deps/mc2_serialization-build/flatbuffers/bin/flatc", 52 | ) 53 | schemas_path = os.path.join( 54 | setup_path, "../src/build/_deps/mc2_serialization-src/src/flatbuffers/" 55 | ) 56 | schemas = [ 57 | "SignedKey.fbs", 58 | "sql/EncryptedBlock.fbs", 59 | "sql/Rows.fbs", 60 | ] 61 | for schema in schemas: 62 | schema_path = schemas_path + schema 63 | subprocess.Popen( 64 | [flatc_path, "--python", "-o", config_fb_path, schema_path] 65 | ) 66 | subprocess.Popen(["chmod", "a+rx", config_fb_path + "/tuix"]) 67 | 68 | # ---- Install Opaque Client---- 69 | 70 | lib_path = [ 71 | os.path.relpath(os.path.join(setup_path, "../src/build/libmc2client.so")) 72 | ] 73 | 74 | # Get Python package dependencies 75 | # https://stackoverflow.com/a/14399775 76 | with open("../requirements.txt") as f: 77 | required = f.read().splitlines() 78 | 79 | setup( 80 | name="mc2client", 81 | version="0.0.1", 82 | description="MC2 Client Python Package", 83 | install_requires=required, 84 | zip_safe=False, 85 | packages=find_packages(), 86 | package_data={ 87 | "mc2client": [ 88 | os.path.join(setup_path, "mc2client/toolchain/scripts/*.sh"), 89 | os.path.join(setup_path, "mc2client/toolchain/mc2-schema.json"), 90 | os.path.join(setup_path, "mc2client/toolchain/mc2_azure/*.json"), 91 | os.path.join(setup_path, "mc2client/toolchain/mc2_azure/*.yaml"), 92 | ] 93 | }, 94 | data_files=[("mc2client", lib_path)], 95 | include_package_data=True, 96 | python_requires=">=3.6", 97 | ) 98 | 99 | # ---- Configure environment variables ---- 100 | # Configure the shell to set the environment variable MC2_CLIENT_HOME equal 101 | # to the root directory of the mc2 repository for easier paths in 102 | # the configuration yamls. 103 | 104 | print("\n####################\n") 105 | print("Configuring shell for MC2 Client...\n") 106 | 107 | home_path = os.path.abspath(os.path.join(setup_path, "../")) 108 | # Terminal shell initialization scripts to support 109 | shell_paths = [ 110 | os.path.expanduser("~/.profile"), 111 | os.path.expanduser("~/.bashrc"), 112 | ] 113 | 114 | 115 | def set_path(path): 116 | try: 117 | with open(path, "r+") as f: 118 | # Only write to the file if the environment variable isn't 119 | # already present 120 | if "export MC2_CLIENT_HOME" not in f.read(): 121 | # At this point the file pointer is at the end of the file 122 | # so writing will append 123 | f.write(f"export MC2_CLIENT_HOME={home_path}\n") 124 | 125 | # Reset the file pointer 126 | f.seek(0) 127 | 128 | # Add an alias to a bashrc or bash_profile if it exists 129 | if ( 130 | "bashrc" in path or "bash_profile" in path 131 | ) and "alias mc2" not in f.read(): 132 | # At this point the file pointer is at the end of the file 133 | # so writing will append 134 | alias_str = f'alias mc2="python3 {home_path}/mc2.py"' 135 | f.write(alias_str + "\n") 136 | subprocess.check_call([alias_str]) 137 | return True 138 | except FileNotFoundError: 139 | return False 140 | 141 | 142 | # Attempt to set the environment variable for each shell script 143 | shell_paths_set = [set_path(path) for path in shell_paths] 144 | 145 | if not any(shell_paths_set): 146 | print("ERROR: Failed to write to any of the following:\n") 147 | for path in shell_paths: 148 | print(f"\t{path}") 149 | print( 150 | "\nPlease add the following line to your shell initialization\n" 151 | "file to ensure that MC2 Client is configured automatically:\n\n" 152 | f"\texport MC2_CLIENT_HOME={home_path}" 153 | ) 154 | else: 155 | print("Successfully modified:\n") 156 | for (success, path) in zip(shell_paths_set, shell_paths): 157 | if success: 158 | print(f"\t{path}") 159 | print("\nTo run MC2 Client you may need to restart your current shell.") 160 | 161 | env_path = os.path.join(home_path, "mc2_client_env") 162 | print(f"\nTo configure your current shell, run:\n\n\tsource {env_path}\n") 163 | -------------------------------------------------------------------------------- /python-package/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc2-project/mc2/38a6888a60ea6f9e3a513fadc29c2123334e6cdf/python-package/tests/__init__.py -------------------------------------------------------------------------------- /python-package/tests/azure/create_azure_resources.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pathlib 3 | 4 | import mc2client as mc2 5 | 6 | tests_dir = pathlib.Path(__file__).parent.absolute() 7 | config = os.path.join(tests_dir, "../test.yaml") 8 | mc2.set_config(config) 9 | 10 | dummy_file = os.path.join(tests_dir, "dummy.txt") 11 | 12 | print("Creating resource group") 13 | mc2.create_resource_group() 14 | 15 | print("Creating storage") 16 | mc2.create_storage() 17 | 18 | print("Creating container") 19 | mc2.create_container() 20 | 21 | print("Uploading file") 22 | mc2.upload_file(dummy_file, "dummy.txt") 23 | 24 | print("Creating cluster") 25 | mc2.create_cluster() 26 | -------------------------------------------------------------------------------- /python-package/tests/azure/delete_azure_resources.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pathlib 3 | 4 | import mc2client as mc2 5 | 6 | print("Setting config file path") 7 | tests_dir = pathlib.Path(__file__).parent.absolute() 8 | config = os.path.join(tests_dir, "../test.yaml") 9 | mc2.set_config(config) 10 | 11 | download_path = os.path.join(tests_dir, "dummy_downloaded.txt") 12 | 13 | print("Downloading file") 14 | mc2.download_file("dummy.txt", download_path) 15 | 16 | if not os.path.isfile(download_path): 17 | print("Error: Couldn't download file from Azure") 18 | else: 19 | os.remove(download_path) 20 | 21 | print("Deleting container") 22 | mc2.delete_container() 23 | 24 | print("Deleting storage") 25 | mc2.delete_storage() 26 | 27 | print("Deleting cluster") 28 | mc2.delete_cluster() 29 | 30 | print("Deleting resource group") 31 | mc2.delete_resource_group() 32 | -------------------------------------------------------------------------------- /python-package/tests/azure/dummy.txt: -------------------------------------------------------------------------------- 1 | Hello world! 2 | -------------------------------------------------------------------------------- /python-package/tests/data/expected_booster.dump: -------------------------------------------------------------------------------- 1 | booster[0]: 2 | 0:[f29<2.00001001] yes=1,no=2,missing=2 3 | 1:[f109<2.00001001] yes=3,no=4,missing=4 4 | 3:leaf=0.557894766 5 | 4:[f67<2.00001001] yes=7,no=8,missing=8 6 | 7:leaf=0.242553204 7 | 8:leaf=-0.595593095 8 | 2:[f56<2.00001001] yes=5,no=6,missing=6 9 | 5:[f21<2.00001001] yes=9,no=10,missing=10 10 | 9:leaf=-0.594312787 11 | 10:leaf=0.533333361 12 | 6:[f60<2.00001001] yes=11,no=12,missing=12 13 | 11:leaf=-0.58518523 14 | 12:leaf=0.57052362 15 | booster[1]: 16 | 0:[f29<2.00001001] yes=1,no=2,missing=2 17 | 1:[f109<2.00001001] yes=3,no=4,missing=4 18 | 3:leaf=0.4361763 19 | 4:[f67<2.00001001] yes=7,no=8,missing=8 20 | 7:leaf=0.178685576 21 | 8:leaf=-0.4607777 22 | 2:[f21<2.00001001] yes=5,no=6,missing=6 23 | 5:[f27<2.00001001] yes=9,no=10,missing=10 24 | 9:leaf=0.460993111 25 | 10:leaf=-0.276318461 26 | 6:leaf=0.468871444 27 | booster[2]: 28 | 0:[f29<2.00001001] yes=1,no=2,missing=2 29 | 1:[f109<2.00001001] yes=3,no=4,missing=4 30 | 3:leaf=0.375119895 31 | 4:[f87<2.00001001] yes=7,no=8,missing=8 32 | 7:leaf=0.398808211 33 | 8:leaf=-0.395200908 34 | 2:[f56<2.00001001] yes=5,no=6,missing=6 35 | 5:[f21<2.00001001] yes=9,no=10,missing=10 36 | 9:leaf=-0.420767218 37 | 10:leaf=0.35382548 38 | 6:[f60<2.00001001] yes=11,no=12,missing=12 39 | 11:leaf=-0.41420272 40 | 12:leaf=0.382028222 41 | booster[3]: 42 | 0:[f29<2.00001001] yes=1,no=2,missing=2 43 | 1:[f109<2.00001001] yes=3,no=4,missing=4 44 | 3:leaf=0.336961627 45 | 4:[f67<2.00001001] yes=7,no=8,missing=8 46 | 7:leaf=0.162825078 47 | 8:leaf=-0.363649756 48 | 2:[f23<2.00001001] yes=5,no=6,missing=6 49 | 5:[f36<2.00001001] yes=9,no=10,missing=10 50 | 9:leaf=-0.375322014 51 | 10:leaf=-0.796177089 52 | 6:[f24<2.00001001] yes=11,no=12,missing=12 53 | 11:leaf=-0.448040783 54 | 12:leaf=0.378991574 55 | booster[4]: 56 | 0:[f29<2.00001001] yes=1,no=2,missing=2 57 | 1:[f109<2.00001001] yes=3,no=4,missing=4 58 | 3:leaf=0.309557945 59 | 4:[f39<2.00001001] yes=7,no=8,missing=8 60 | 7:leaf=-0.350704819 61 | 8:leaf=-0.0279991869 62 | 2:[f23<2.00001001] yes=5,no=6,missing=6 63 | 5:[f36<2.00001001] yes=9,no=10,missing=10 64 | 9:leaf=-0.348102778 65 | 10:leaf=-0.51200211 66 | 6:[f24<2.00001001] yes=11,no=12,missing=12 67 | 11:leaf=-0.408176273 68 | 12:leaf=0.354143769 69 | -------------------------------------------------------------------------------- /python-package/tests/data/test_data.schema: -------------------------------------------------------------------------------- 1 | id:integer,firstname:string,lastname:string,age:integer,balance:float,join_date:date 2 | -------------------------------------------------------------------------------- /python-package/tests/keys/mc2_test_signing_key.pub: -------------------------------------------------------------------------------- 1 | -----BEGIN PUBLIC KEY----- 2 | MIIBoDANBgkqhkiG9w0BAQEFAAOCAY0AMIIBiAKCAYEAqGYTZsYklPViPIS5M9eq 3 | 4kbC3ZVs4YHcGBZBlZ0lJ/GHE4XTPxwH+y0K9H5acRdx2zetRTSoNpaEQpKyGRvi 4 | XWVVHHfRmzS1I1rOoA7GIJUSWySb0dsba25cgFljQzgNvfXYSz+tjt8dLIcjo1cx 5 | KDMfTPh83zxW42UYvC4gc+XznM/bRbAx12WKdWELpxX7JHyj4mNl6JEp6tknHvFd 6 | x0kFSatbpZIYAJ4Ybkd0IoQqZVKqIF4EeTZZAxQSc5wdA2sFe1fuGZ9/ck+VxbFC 7 | /kofUDwM+xz2Xe5U2SoH9MJ2YIqbaClA+vPGT6H8Wio3KSKwndAw6lWcieCeSsPX 8 | NydZc2dc1dUb7SwBwWgBE05nxTJp34xAnsVVIrcTJ8LtzPZROe81A+Ek4tTtdLkB 9 | 5rUuc5xL6VzaOddwLzBusydSob2xVcZ/d40zWCsu+Fmk80b52MWZ8H72DqaT9MSN 10 | F+0spA0/m8ov+J1o+8mo7j83Xtu8uNP0RZ/Tx1/N/iLRAgED 11 | -----END PUBLIC KEY----- 12 | -------------------------------------------------------------------------------- /python-package/tests/keys/root.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID/TCCAmWgAwIBAgIUag4+JmUlpecz79+hB5nWdv8vsuswDQYJKoZIhvcNAQEL 3 | BQAwDzENMAsGA1UEAwwEcm9vdDAeFw0yMDA4MzExNzQ5NDNaFw0zMDA4MjkxNzQ5 4 | NDNaMA8xDTALBgNVBAMMBHJvb3QwggGgMA0GCSqGSIb3DQEBAQUAA4IBjQAwggGI 5 | AoIBgQDGJob5lAEOyq+QDi/6gGxnmkrwWGLApC/jIVTsTt5r7Vp73mn1QBbpovso 6 | kze9ahqU6x+gT8kI7VPavXwN+MjJMzbPKEErvl3XMoY6XPSIaQrEEPJiMEB2wx2y 7 | b5cLC9VqcYryPbrXd5LP5eXqXMoEpxFgOhnXKuMtBotAEMpBS/Uk5pZuzw6IC2BM 8 | +tKqWyma3ZtSy+CRDqOrSGsvtPHAExBxNbGEIg3rRUN7v3FtRcmydKwURMxFVBV+ 9 | /UrSDocz6Vq7IBeWQP2ytsX1pcDJyWJd1R78fE6ID8jF7IKnykzc8Bl6YxV+kxwt 10 | du9o13l3fiq+vSi8yb7eckJtkG3OIXQRCRGWyll2KEZvgDW032v30X9W/OdEZWZF 11 | ATLt3ufmS2QdPOxEFt/KeAb8A8Awc05uhYYIK2kvo1i7iikwBD5OevJZdB8108Dr 12 | 5UZ4rkhtiClrnIP26shZlzwtMJOQ+TanGB7pinKaf2K6Mq3ZF58OUQ8RsFLyeDL2 13 | 7TX+8GECAQOjUzBRMB0GA1UdDgQWBBRQm3Yp9L/SChF+/OJ1E0lZhB3SajAfBgNV 14 | HSMEGDAWgBRQm3Yp9L/SChF+/OJ1E0lZhB3SajAPBgNVHRMBAf8EBTADAQH/MA0G 15 | CSqGSIb3DQEBCwUAA4IBgQBx8ZxK0UuGAq67zQVPn/ddJoqFEvCPRWVGfzOQLaTP 16 | 5URHJLFRoSrluZgpxUAuyMfGnoVh5v2aT5mYgBqvx7X+SWJx90weU2rfGzFslvt2 17 | VtPDZcZH2VgN1l2upbor7RpT6K3Fd0Mftw7+YcWTgUo+xffbBuYfBlgcQo/PcfUr 18 | 4FbsmP50DGDTTU80JW/icRc0tCCb3MroZe36tZ5nTUrGghwo8c1ICFJNrkdrfGyf 19 | y6PytCbD9desjc24SzI7eu5Y0MmwmfGHUl/KwbZtLNGf3lNhgiI/tbFdo6nBGGrE 20 | ogfdkhe+A8r7o6QtQYzsaLRePeWpu1/yDrxgJySA0E+BhEDn7kNVSpqn3B2gVAHe 21 | Yxxy6HOfCTWMKTkj8pD8B6Swo81vBM1uH2hHdyEWZG80jPgxWVttniYkfv1jIcJW 22 | 5zgZ7/HT/3jRSNwARQMEs/vH38Cyntx5TCU4aDgP67fp+SfGf43xEZhxcqoCXzTN 23 | Voyw9vprJOhvJ05ewzhelqQ= 24 | -----END CERTIFICATE----- 25 | -------------------------------------------------------------------------------- /python-package/tests/keys/root.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIG5AIBAAKCAYEAxiaG+ZQBDsqvkA4v+oBsZ5pK8FhiwKQv4yFU7E7ea+1ae95p 3 | 9UAW6aL7KJM3vWoalOsfoE/JCO1T2r18DfjIyTM2zyhBK75d1zKGOlz0iGkKxBDy 4 | YjBAdsMdsm+XCwvVanGK8j2613eSz+Xl6lzKBKcRYDoZ1yrjLQaLQBDKQUv1JOaW 5 | bs8OiAtgTPrSqlspmt2bUsvgkQ6jq0hrL7TxwBMQcTWxhCIN60VDe79xbUXJsnSs 6 | FETMRVQVfv1K0g6HM+lauyAXlkD9srbF9aXAycliXdUe/HxOiA/IxeyCp8pM3PAZ 7 | emMVfpMcLXbvaNd5d34qvr0ovMm+3nJCbZBtziF0EQkRlspZdihGb4A1tN9r99F/ 8 | VvznRGVmRQEy7d7n5ktkHTzsRBbfyngG/APAMHNOboWGCCtpL6NYu4opMAQ+Tnry 9 | WXQfNdPA6+VGeK5IbYgpa5yD9urIWZc8LTCTkPk2pxge6Ypymn9iujKt2RefDlEP 10 | EbBS8ngy9u01/vBhAgEDAoIBgQCEGa9RDVYJ3HUKtB/8VZ2aZtygOuyAbXVCFjid 11 | iemdSOb9PvFOKrnxF1IbDM/TnBG4nL/ANTCwnjfnKP1epdswzM80xYDH1D6PdwQm 12 | 6KMFm1yCtfbsICr512khn7oHXTjxoQdMKSc6T7c1Q+6cPdwDGguVfBE6HJdzWbIq 13 | tdwrh/jDRGRJ318FXOrd/Ixxkhu8k7zh3UBgtG0c2vIfzfaADLWgznZYFrPyLiz9 14 | KkueLoZ2+HK4Ld2DjWOp/jHhXwOg8sfm6SYMoCQWKQEKYaz76HxnYrpYEm/86Ss/ 15 | VvCkEATZOJkqTr8R9RA6IyiQ5fCOCYMB6+Z2KoU8Jlml96ctcpe68RkeHWEvyo9M 16 | 2cXSFiyuzQq+xn1ZI8X+cddOtfIwgit/e8V7liERtow2uIXvWYKk2R9RYj8s9Eg8 17 | My4pDfo3BPP7lMBeQlbnQ0mGbIznr2kx7eY26T4Q9uckmE+2b/sDLvd0tzCiNKNe 18 | gDcHg9rJsezKOrQDWcM0QgWktGMCgcEA5Ctw7VAKOTK8Zum/a/t/PxI47m+E/Dvd 19 | s1YeTWwxQBWdrCWXnSnoehsy5ab389MiETe2IaybzUlS2HqZDD4P1h7FdQkT8By8 20 | tsJxUjiYir0L+W7ZbC9uIqfwJIVvKE+6w5qi78jTv9H0lCpyMyNURZPJxWBngO94 21 | 3OWltobqPhB/29c7jAjyDf5XFijl2TnNXN6RDesyv9uJLJ8gjlX4+HMjqZ2mWH7F 22 | kbbaPMXCyEfqpOnAZkDKUyNK0SIPMFDjAoHBAN5RvfNyVEoeCyqPhPoXvhDabtRR 23 | gnwkyNlb6Zl96HGcp+r1nB3DDmmIUPCbOpurbpE4MBousz5ApCu+Iuhe4zPWywOW 24 | V/mBive1/ioA9G8BHPgvFcyjvRwHzSLRAM9+Qdntf+46cErjuZu7wnbLowPZQLHf 25 | b40okY9PRqq2ebRexyAcSNQMDJpx53rXclXRp7UiepLMd+SxYhOFwOf2IwbeGni0 26 | BWH45BV5k2+smIWJ7Drca3wXeppOQ1doHleQ6wKBwQCYHPXzirF7dyhEm9Typ6oq 27 | DCX0SlioJ+kiOWmI8suADmkdbmUTcUWmvMyZGfqijMFgz87BHb0zhjc6/GYIKV/k 28 | FIOjW2KgEyh51vY20GWx011Q9JDyyklsb/Vtrkoa39HXvGyf2zfVNqMNcaF3bOLZ 29 | DTEuQEUAn6XomRkkWfF+taqSj30IBfaz/uS5cJk7e9496bYJR3cqkltzFMBe4/tQ 30 | TMJxE8Q6/y5hJJF92SyFhUcYm9WZgIbiF4c2FrTK4JcCgcEAlDZ+okw4MWlcxwpY 31 | prp+teb0jYusUsMwkOfxEP6a9mhv8fkSvoIJm7A19bzRvRz0YNAgEXR3ftXCx9QX 32 | RZSXd+SHV7mP+6ux+nlUHACi9KtopXS5MxfTaAUzbItV36mBO/OqntGgMe0mZ9KB 33 | pIfCApDVy+pKXhsLtN+Ecc77zZSEwBLbOAgIZvaaUeT24+EaeMGnDIhP7cuWt66A 34 | mqQXWelm+yKuQVCYDlEM9R27A7FIJz2c/WT8Zt7Xj5q+5QtHAoHBAKSRenWOHjHG 35 | eoVkz3UKR3Nwn1Ctn/cJmMHE53vR16MeN/FlqnPQdYlvdAPaAHMy91B/zbXAKHGB 36 | WxWlEEv6PbDa0xpHwOzKgkaiES3znEyq7cjlJ6HfURdaAbbbq+uYYdaE9/qQUddC 37 | xttQW+WqaKUz71cMRmAKzzXNBmeeueQ5V514k9r5smgfm/8+//xqltPDomNnoaqz 38 | zMubnimitg5M7OcDv/eR0Hfs+N9Rh3U4yo8DJBRfyvnVMrtw7ydwnQ== 39 | -----END RSA PRIVATE KEY----- 40 | -------------------------------------------------------------------------------- /python-package/tests/keys/root.pub: -------------------------------------------------------------------------------- 1 | -----BEGIN PUBLIC KEY----- 2 | MIIBoDANBgkqhkiG9w0BAQEFAAOCAY0AMIIBiAKCAYEAxiaG+ZQBDsqvkA4v+oBs 3 | Z5pK8FhiwKQv4yFU7E7ea+1ae95p9UAW6aL7KJM3vWoalOsfoE/JCO1T2r18DfjI 4 | yTM2zyhBK75d1zKGOlz0iGkKxBDyYjBAdsMdsm+XCwvVanGK8j2613eSz+Xl6lzK 5 | BKcRYDoZ1yrjLQaLQBDKQUv1JOaWbs8OiAtgTPrSqlspmt2bUsvgkQ6jq0hrL7Tx 6 | wBMQcTWxhCIN60VDe79xbUXJsnSsFETMRVQVfv1K0g6HM+lauyAXlkD9srbF9aXA 7 | ycliXdUe/HxOiA/IxeyCp8pM3PAZemMVfpMcLXbvaNd5d34qvr0ovMm+3nJCbZBt 8 | ziF0EQkRlspZdihGb4A1tN9r99F/VvznRGVmRQEy7d7n5ktkHTzsRBbfyngG/APA 9 | MHNOboWGCCtpL6NYu4opMAQ+TnryWXQfNdPA6+VGeK5IbYgpa5yD9urIWZc8LTCT 10 | kPk2pxge6Ypymn9iujKt2RefDlEPEbBS8ngy9u01/vBhAgED 11 | -----END PUBLIC KEY----- 12 | -------------------------------------------------------------------------------- /python-package/tests/keys/user1.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDpDCCAgwCFB4OVnAtHaqaM1NhhnidzrHganchMA0GCSqGSIb3DQEBCwUAMA8x 3 | DTALBgNVBAMMBHJvb3QwHhcNMjAwODMxMTc1MzQ2WhcNMzAwODI5MTc1MzQ2WjAQ 4 | MQ4wDAYDVQQDDAV1c2VyMTCCAaAwDQYJKoZIhvcNAQEBBQADggGNADCCAYgCggGB 5 | ANTPOwS4GD1YFA3jizOnRBxQEPkn50BwmG5GDDa+SMg1T+hdzLCKK3n9cOF5c2Ji 6 | DH/4h5VJUCcV2YcgVXI8ZjL8AirANsnZJSXyqKSTIM6ydH61t5PB8Ajunn+CF2lw 7 | 9bFdZnHCZmOZvaT3iyPK08gGduclsIe8Rkl0pO2ixDQV3mqOviHdfxeG4Hj2rrIw 8 | ujv2n2dDOW1+Gmqd7VMVYnkiqNO7b9iHEG6gLD8BokCuZkKcLBZLvDRrtsclFuL0 9 | 3JsGo4EzycSGNDX6p2DCKnJ1+l1KHBBSzNhLM4Y4AGgRG9W0RMp5dfNNUpx5Y8VN 10 | yEMjSgeMA4yCyja+UGjR4+JCqNeTz4VsZpOoGbPu7dgsdQbYaCbkg9ZtXnnMgQ/D 11 | V7zSAnsHXpaI3CZos7bBe4cpV8y0sj9ddhZYj4wYFrD9800HZJTnp2qTyMFnl7YD 12 | 8CVgHAXZeVaXo1gv2ZcHP77R+sg+vxohErRAfSmyZZt4mHbSYkpIajBKp1Rj1YO9 13 | kwIBAzANBgkqhkiG9w0BAQsFAAOCAYEAOQIUMKWiZyiy8csb0Yf7owqiAOAZ+ZVn 14 | eX0slFZlBKDmNNxejfKIZgfpvh4W29PfwWRGrlmmoGOTvzyKkFE05SXkj+AlsMtM 15 | 5tjBVaJ4/bwiIkKvcsXtVT5n/EinKUEutRwLDLFb8nWB9Ffl0neKPuRd9BbjGJPe 16 | gJquPmr9LFYIz+HzNYykStuyZkX8aKeaCWqPfHBkOD8eshczJGi9Y9GR2gffKgAE 17 | fV+p3yISI8HImfjV/Lxat96WEjPEHFn4nVzAz3i4I/8ABg0EGDoh0pbK5cXDiv6F 18 | RDfqXFO/LlgSZ4HAs23vN+1KIU/5dQSM4GxpTPnfRytP3caF3M/GM9uWhKDw1o7J 19 | /v4Xgqv6ly7h2HCfM2z0MzYVdJOQDgSt47UW0gEZ0tVIWmxasP/v0lb2+wrC1/ms 20 | 9/7TMwcae24jZe9jin12WQWP4qVcycaBETf5AxJGhieygVpsBPy4EZ2N4trIERIp 21 | bdb91L+ElO1b44zu7HuJlvEqQGztC7zO 22 | -----END CERTIFICATE----- 23 | -------------------------------------------------------------------------------- /python-package/tests/keys/user1.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpQIBAAKCAQEA7G/D/7lheSMmiHk97lPbw0A4UtjyVq2ZpuazFmFUph0xazJb 3 | UwEurKa7LyYUbF+GHk0GZytLKzeHWLHc5iCKRByaQZJ2MJCeGa9q2oLyAFQdWg3o 4 | R5ie3W+Hs/yGxpYVYmTDrVI48Vb0ChE+9FmZtdy+EtdiOaHs7PcLNExCYjMTW396 5 | oFMOCf5P1KZE762uUomDqUbZxzvnVKQQSDRVY9UAaoMcYu3aa3L5vl0b+G5anXr3 6 | 5k+x+Mq/MXerJQZIVYZKTckJb9bQ5j89uEIZXVNTh11fIKZjC/hJ3SYhFPPSdlWE 7 | VenU9hZaE8BsO1ITttyiMl6IFXnLHKL8F2aZBQIDAQABAoIBAQCGl30QhrwZFExs 8 | JfB+ShhxBo7Jgpw5gWtVWkCaPd/NDBNEvY7FKogiWmt2BIxdwOb9OsWpVzjcKikO 9 | 8XkZO27aJRoet7w2Gy0U3AnMx/vw+fEXgZE3qIbO8R1j9R3X0DnuIARQFt05bVFQ 10 | pc6blKHup/E/koJk4muX3W3wYHTtpqNhiZ+aB8kMWO8z39Vc6xVc1y/u9Tta4QlD 11 | uCU/QubDW1NJRQ4ghM7fXeQxqvDZwqKShbg5cUYmq5X3t86AQBXTWhPkQikKCwqK 12 | 6h843PkfbBnLLE9P/Pmoc/AWi+7QAyj0AOzTr1oP9giiDb05cN4d10HWhc8v8ul3 13 | 2IUmuNDhAoGBAP3hVhx+xXObLdaszYVMV6hdeQzOzrS2TSPCd26+V3tMSOD7aakf 14 | 9oaUvMjjupai4SbUk5zoqpUWr+hvTlOUa0gQON4LSqXXj0zXSg8OWIXoErmyC7Dh 15 | 77Fcg+QTQE8VKKeg5xOQ8WsMYz89wsxdiddW6HX+2LgkqCMHpr/9op35AoGBAO5p 16 | JNL48PbXn9hxRQdvySbVZHoh5sPBbxUA+pR2sjffeHq4wq2AjYibLcHNgAWnwvv2 17 | Isuw3qO9/fWUlDICQ2oIDZPL0nzKZ9TvHtVlcvW5y4+7BfLO0DWY4avFTNDzHqaD 18 | Ul6KMHwWkAd2w7GW8DzEmI9+4+miNRj6G2NggYZtAoGBAL6+iWsZWBJ2mab75+Hp 19 | rMZjjCoySx19BlICrqb2vVV2yB645fbae+c7YudwKeU5dP4uosU2Dcu74ug4kFm9 20 | XikjfwZc53XYkeLCsfLD7YCWD1OTULNR3Tudbb5zNFL2a7gd7N9HfArYoMyIA452 21 | DLVMp5TXp04axHSlMAR5dK9JAoGBANCBHDd2iCcZhS5iQaCzXxSbY/h8Vbm2HlQc 22 | OwpElLDQvCl4FKpw11c1f9sSwngvtBNvvBawZMaHjueMPd9Oo27EBDvR8hA5ZH2R 23 | c0HmK5hEGYdmZVlpDicRwavcLcZAGfo+t3b/HFAp22TrtVJHU2uR9Grq6qCVwCJL 24 | +k/7QswxAoGAFsz0er6hBo6VNAIiha271G2fGYlPe84RT+MGKDFnn6iKNKJ4LnYc 25 | xKM3tjwHvfQEd06UYamgrrrAugV971nQmeFgGJDuSRsHguUFlnBQ7XyIrXYfsAHE 26 | nzcJ803vfZr+6Sts0FsjE9On7oBlsdJtrrxpv/2Urx9LRvzbHVdTGNg= 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /python-package/tests/keys/user1.pub: -------------------------------------------------------------------------------- 1 | -----BEGIN PUBLIC KEY----- 2 | MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA7G/D/7lheSMmiHk97lPb 3 | w0A4UtjyVq2ZpuazFmFUph0xazJbUwEurKa7LyYUbF+GHk0GZytLKzeHWLHc5iCK 4 | RByaQZJ2MJCeGa9q2oLyAFQdWg3oR5ie3W+Hs/yGxpYVYmTDrVI48Vb0ChE+9FmZ 5 | tdy+EtdiOaHs7PcLNExCYjMTW396oFMOCf5P1KZE762uUomDqUbZxzvnVKQQSDRV 6 | Y9UAaoMcYu3aa3L5vl0b+G5anXr35k+x+Mq/MXerJQZIVYZKTckJb9bQ5j89uEIZ 7 | XVNTh11fIKZjC/hJ3SYhFPPSdlWEVenU9hZaE8BsO1ITttyiMl6IFXnLHKL8F2aZ 8 | BQIDAQAB 9 | -----END PUBLIC KEY----- 10 | -------------------------------------------------------------------------------- /python-package/tests/keys/user1_sym.key: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc2-project/mc2/38a6888a60ea6f9e3a513fadc29c2123334e6cdf/python-package/tests/keys/user1_sym.key -------------------------------------------------------------------------------- /python-package/tests/keys/user2.pub: -------------------------------------------------------------------------------- 1 | -----BEGIN PUBLIC KEY----- 2 | MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtJIijcVBjaxRr72A4zWC 3 | C+RAUWHH2eWrGbkXDAi3nZhyaCVfFmZhRvtHF/xHieZkGpoyC7meY6WBeAFaMQTk 4 | uxhBKNauhw2FK5lu5dQx1yv0RdhEn2xxw/xMcq18Ue1bKB36wnhvyNVa1PAmb6qT 5 | MBu7d8E6fV7NU7C1tHRPt/MZpTMvzBhQcaCc/KFhSOPPKdHshf+eHL+1mCKgTYnJ 6 | edRHWZSBzjUj8Zr3QYAZxBR1pTtqDFIeGn30bBblClttKQOQmrwAq5i4W/zv4ITG 7 | wRzkTNc34HhxWElxPe7M462IYic58tDlOq001UHB9qqI1QCHnwQFFRvehgg9wZ1S 8 | swIDAQAB 9 | -----END PUBLIC KEY----- 10 | -------------------------------------------------------------------------------- /python-package/tests/keys/user3.pub: -------------------------------------------------------------------------------- 1 | -----BEGIN PUBLIC KEY----- 2 | MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAy3tYUpfyIehcncZ0NmgQ 3 | 7M/JMk8BYw1Bv3e6j6CsTL/klewJ6uKu05smvJRJqpfWxz/KZhFX8TwsvlYKoOr/ 4 | LSp3M2HwTFgxQyP7OEKjHlDsrZCebzLpDKIJ55f6Vnu8gdN05ZQe1Z+td7M+QJpK 5 | aOg7LQKQ8QbNCNZWAHyi1PKYG7OS1OSDUrIJA9/9ElKsc8N/aLNGdrxJhOm5uoFA 6 | JdqoyN0tU/zsmtUew9MjFXbgs0h/PuqgwA94arC7yCQyLx6gCQinzogT1+YdE0wy 7 | j2NjiN7lTPfM9jSLRB7NDye65Xvlw9l+d9z3s61Z7uTTV+Yaof4WMWX/xmA3GyX2 8 | FQIDAQAB 9 | -----END PUBLIC KEY----- 10 | -------------------------------------------------------------------------------- /python-package/tests/opaquesql/opaquesql_example.scala: -------------------------------------------------------------------------------- 1 | import edu.berkeley.cs.rise.opaque.implicits._ 2 | edu.berkeley.cs.rise.opaque.Utils.initSQLContext(spark.sqlContext) 3 | 4 | val data = Seq(("foo", 4), ("bar", 1), ("baz", 5)) 5 | val df = spark.createDataFrame(data).toDF("word", "count") 6 | val dfEncrypted = df.encrypted 7 | val result = dfEncrypted.filter($"count" > lit(3)) 8 | // This will save the result DataFrame to the result directory locally 9 | result.write.format("edu.berkeley.cs.rise.opaque.EncryptedSource").save("result") 10 | 11 | 12 | -------------------------------------------------------------------------------- /python-package/tests/opaquesql/to_test_opaquesql.py: -------------------------------------------------------------------------------- 1 | import mc2client as mc2 2 | import mc2client.opaquesql as osql 3 | 4 | # TODO: modify the `orchestrator` value in tests/config.yaml to reflect 5 | # Opaque SQL driver IP address 6 | mc2.set_config("config.yaml") 7 | osql.run("opaquesql_example.scala") 8 | -------------------------------------------------------------------------------- /python-package/tests/test.yaml: -------------------------------------------------------------------------------- 1 | # User configuration 2 | user: 3 | # Your username - username should be specified in certificate 4 | username: user1 5 | 6 | # Path to your symmetric key - will be used for encryption/decryption 7 | # If you don't have a symmetric key, specify a path here 8 | # and run `mc2 init` to generate a key 9 | # 10 | # `mc2 init` will not overwrite anything at this path 11 | symmetric_key: keys/user1_sym.key 12 | 13 | # Path to your private key and certificate 14 | # If you don't have a private key / certificate, specify paths here 15 | # and run `mc2 init` to generate a keypair 16 | # 17 | # `mc2 init` will not overwrite anything at this path 18 | private_key: keys/user1.pem 19 | public_key: keys/user1.pub 20 | certificate: keys/user1.crt 21 | 22 | # Path to CA certificate and private key 23 | # Needed if you want to generate a certificate signed by CA 24 | root_certificate: keys/root.crt 25 | root_private_key: keys/root.pem 26 | 27 | # Configuration for launching cloud resources 28 | launch: 29 | # The absolute path to your Azure configuraton 30 | # This needs to be an absolute path 31 | #azure_config: /dummy/azure.yaml 32 | 33 | # # Manually specify the IP/uname/ssh_key of the head node or workers. 34 | # # If these values exist, they will override any values in `azure_config`. 35 | # # Consequently, the `launch` and `stop` commands will do nothing. 36 | head: 37 | ip: 0.0.0.0 38 | # username: 39 | # ssh_key: 40 | # workers: 41 | # - ip: 42 | # username: 43 | # ssh_key: 44 | 45 | # Whether to launch a cluster of VMs 46 | cluster: true 47 | 48 | # Whether to launch Azure blob storage 49 | storage: true 50 | 51 | # Whether to launch a storage container 52 | container: true 53 | 54 | # Commands to start compute service 55 | start: 56 | # Commands to run on head node 57 | head: 58 | - build/sbt run 59 | 60 | # Commands to run on worker nodes 61 | workers: 62 | - echo "Hello from worker" 63 | 64 | # Configuration for `mc2 upload` 65 | upload: 66 | # Whether to upload data to Azure blob storage or disk 67 | # Allowed values are `blob` or `disk` 68 | # If `blob`, Azure CLI will be called to upload data 69 | # Else, `scp` will be used 70 | storage: disk 71 | 72 | # Encryption format to use 73 | # Options are `sql` if you want to use Opaque SQL 74 | # or `xgb` if you want to use Secure XGBoost 75 | format: sql 76 | 77 | # Files to encrypt and upload 78 | src: 79 | - data/test_data.csv 80 | 81 | # If you want to run Opaque SQL, you must also specify a schema, 82 | # one for each file you want to encrypt and upload 83 | schemas: 84 | - data/test_data.schema 85 | 86 | # Directory to upload data to 87 | # FIXME: ignored if using Azure blob storage. Need to investigate if 88 | # we can use directories in Azure blob storage 89 | dst: 90 | 91 | 92 | # Computation configuration 93 | run: 94 | # Script to run 95 | script: opaquesql/opaquesql_example.scala 96 | 97 | # Compute service you're using 98 | # Choices are `xgb` or `sql` 99 | compute: sql 100 | 101 | # Attestation configuration 102 | attestation: 103 | # Whether we are running in simulation mode 104 | # If 0 (False), we are _not_ running in simulation mode, 105 | # and should verify the attestation evidence 106 | simulation_mode: 0 107 | 108 | # MRENCLAVE value to check 109 | # MRENCLAVE is a hash of the enclave build log 110 | mrenclave: NULL 111 | 112 | # Path to MRSIGNER value to check 113 | # MRSIGNER is the key used to sign the built enclave 114 | mrsigner: keys/mc2_test_key.pub 115 | 116 | # Configuration for downloading results 117 | download: 118 | # Whether to upload data to Azure blob storage or disk 119 | # Allowed values are `blob` or `disk` 120 | # If `blob`, Azure CLI will be called to upload data 121 | # Else, `scp` will be used 122 | storage: disk 123 | 124 | # Format this data is encrypted with 125 | format: sql 126 | 127 | # Directory/file to download 128 | # FIXME: If storage is `blob` this value must be a file 129 | # Need to investigate whether we can use directories in Azure blob storage 130 | src: 131 | - /root/results/opaque_sql_result 132 | 133 | # Local directory to download data to 134 | dst: results/ 135 | 136 | # Configuration for stopping services 137 | stop: 138 | 139 | # Configuration for deleting Azure resources 140 | teardown: 141 | cluster: true 142 | storage: true 143 | container: true 144 | resource_group: true 145 | -------------------------------------------------------------------------------- /python-package/tests/test_crypto.py: -------------------------------------------------------------------------------- 1 | import filecmp 2 | import os 3 | import pathlib 4 | 5 | import mc2client as mc2 6 | import pytest 7 | 8 | import yaml 9 | from envyaml import EnvYAML 10 | 11 | 12 | @pytest.fixture(autouse=True) 13 | def config(tmp_path): 14 | tests_dir = pathlib.Path(__file__).parent.absolute() 15 | original_config_path = os.path.join(tests_dir, "test.yaml") 16 | 17 | test_cert = os.path.join(tmp_path, "test.crt") 18 | test_priv_key = os.path.join(tmp_path, "test.pem") 19 | test_pub_key = os.path.join(tmp_path, "test.pub") 20 | test_symm_key = os.path.join(tmp_path, "test_sym.key") 21 | 22 | # Rewrite config YAML with test paths 23 | config = EnvYAML(original_config_path) 24 | config["user"]["certificate"] = test_cert 25 | config["user"]["private_key"] = test_priv_key 26 | config["user"]["public_key"] = test_pub_key 27 | config["user"]["symmetric_key"] = test_symm_key 28 | 29 | # Point to root certificate 30 | config["user"]["root_private_key"] = os.path.join( 31 | tests_dir, "keys/root.pem" 32 | ) 33 | config["user"]["root_certificate"] = os.path.join( 34 | tests_dir, "keys/root.crt" 35 | ) 36 | 37 | test_config_path = os.path.join(tmp_path, "config.yaml") 38 | 39 | with open(test_config_path, "w") as out: 40 | yaml.dump(dict(config), out, default_flow_style=False) 41 | 42 | mc2.set_config(test_config_path) 43 | return tests_dir 44 | 45 | 46 | @pytest.fixture() 47 | def keys(config, tmp_path): 48 | mc2.generate_symmetric_key() 49 | mc2.generate_keypair() 50 | 51 | 52 | @pytest.fixture 53 | def data_paths(config, tmp_path): 54 | plaintext = os.path.join(config, "data/test_data.csv") 55 | encrypted = os.path.join(tmp_path, "enc_data") 56 | decrypted = os.path.join(tmp_path, "test_data.csv.copy") 57 | return plaintext, encrypted, decrypted 58 | 59 | 60 | @pytest.fixture 61 | def schema(config): 62 | return os.path.join(config, "data/test_data.schema") 63 | 64 | 65 | def test_key_generation(keys, tmp_path): 66 | test_cert = os.path.join(tmp_path, "test.crt") 67 | test_priv_key = os.path.join(tmp_path, "test.pem") 68 | test_pub_key = os.path.join(tmp_path, "test.pub") 69 | test_symm_key = os.path.join(tmp_path, "test_sym.key") 70 | 71 | assert os.path.exists(test_cert) 72 | assert os.path.exists(test_priv_key) 73 | assert os.path.exists(test_pub_key) 74 | assert os.path.exists(test_symm_key) 75 | 76 | 77 | def test_opaque_encryption(keys, data_paths, schema): 78 | plaintext, encrypted, decrypted = data_paths 79 | 80 | mc2.encrypt_data( 81 | plaintext, 82 | encrypted, 83 | schema_file=schema, 84 | enc_format="sql", 85 | ) 86 | 87 | mc2.decrypt_data(encrypted, decrypted, enc_format="sql") 88 | 89 | # Remove first line (header) from original file, 90 | # as the decrypted copy doesn't have it 91 | with open(plaintext, "r") as f: 92 | original = f.readlines()[1:] 93 | 94 | with open(decrypted, "r") as f: 95 | copy = f.readlines() 96 | 97 | assert original == copy 98 | 99 | 100 | def test_securexgboost_encryption(keys, data_paths): 101 | plaintext, encrypted, decrypted = data_paths 102 | 103 | mc2.encrypt_data( 104 | plaintext, 105 | encrypted, 106 | enc_format="xgb", 107 | ) 108 | 109 | mc2.decrypt_data(encrypted, decrypted, enc_format="xgb") 110 | 111 | assert filecmp.cmp(plaintext, decrypted) 112 | -------------------------------------------------------------------------------- /python-package/tests/to_test_securexgboost.py: -------------------------------------------------------------------------------- 1 | import filecmp 2 | import os 3 | import pathlib 4 | import shutil 5 | import yaml 6 | 7 | import numpy as np 8 | import mc2client as mc2 9 | import pytest 10 | 11 | from envyaml import EnvYAML 12 | 13 | # Note: to run this test, you'll need to start a gRPC orchestrator and an enclave running Secure XGBoost 14 | # Follow the demo here to do so: https://secure-xgboost.readthedocs.io/en/latest/tutorials/outsourced.html 15 | # Then, in the test config.yaml, modify the `orchestrator` variable to hold the IP address of the remote VM 16 | # Lastly, run `pytest -s to_test_securexgboost.py` to run this test. 17 | # You'll need to restart the enclave every time you run this test, otherwise you'll get an issue with the nonce 18 | # counter not resetting. 19 | 20 | 21 | @pytest.fixture(autouse=True) 22 | def config(tmp_path): 23 | tests_dir = pathlib.Path(__file__).parent.absolute() 24 | original_config_path = os.path.join(tests_dir, "config.yaml") 25 | 26 | tmp_keys_dir = os.path.join(tmp_path, "keys") 27 | shutil.copytree(os.path.join(tests_dir, "keys"), tmp_keys_dir) 28 | 29 | test_cert = os.path.join(tmp_path, "keys", "user1.crt") 30 | test_priv_key = os.path.join(tmp_path, "keys", "user1.pem") 31 | test_symm_key = os.path.join(tmp_path, "keys", "user1_sym.key") 32 | 33 | # Rewrite config YAML with test paths 34 | config = EnvYAML(original_config_path) 35 | config["user"]["certificate"] = test_cert 36 | config["user"]["private_key"] = test_priv_key 37 | config["user"]["symmetric_key"] = test_symm_key 38 | 39 | # Point to root certificate 40 | config["user"]["root_private_key"] = os.path.join( 41 | tmp_path, "keys/root.pem" 42 | ) 43 | config["user"]["root_certificate"] = os.path.join( 44 | tmp_path, "keys/root.crt" 45 | ) 46 | 47 | test_config_path = os.path.join(tmp_path, "config.yaml") 48 | 49 | with open(test_config_path, "w") as out: 50 | yaml.dump(dict(config), out, default_flow_style=False) 51 | 52 | mc2.set_config(test_config_path) 53 | return tests_dir 54 | 55 | 56 | @pytest.fixture() 57 | def attest(config): 58 | mc2.attest() 59 | 60 | 61 | # TODO: Ideally, we'd separate all the function calls in `test_securexgboost` into 62 | # their own individual tests, but some functions rely on the results of previous functions 63 | # Additionally, we would ideally attest only once at the beginning of the test suite. 64 | def test_securexgboost(attest, tmp_path): 65 | # Load our training data 66 | dtrain = create_dtrain() 67 | 68 | # Load our test data 69 | dtest = create_dtest() 70 | 71 | # Train a model for 5 rounds 72 | booster = learn(dtrain) 73 | 74 | # Check if the trained model produces the expected predictions 75 | predict(booster, dtest) 76 | 77 | # Save original model, load it, and test to see 78 | # if it produces the same predictions 79 | save_and_load_model(booster, dtest) 80 | 81 | # Get feature importance 82 | get_feature_importance_by_weight(booster) 83 | get_feature_importance_by_gain(booster) 84 | get_feature_importance_by_cover(booster) 85 | get_feature_importance_by_total_gain(booster) 86 | get_feature_importance_by_total_cover(booster) 87 | 88 | # Get model dump 89 | get_dump(tmp_path, booster) 90 | 91 | 92 | def create_dtrain(): 93 | dtrain = mc2.xgb.DMatrix({"user1": "/home/chester/agaricus_train.enc"}) 94 | 95 | num_col = dtrain.num_col() 96 | assert num_col == 127 97 | 98 | return dtrain 99 | 100 | 101 | def create_dtest(): 102 | dtest = mc2.xgb.DMatrix({"user1": "/home/chester/agaricus_test.enc"}) 103 | 104 | num_col = dtest.num_col() 105 | assert num_col == 127 106 | 107 | return dtest 108 | 109 | 110 | def learn(dtrain): 111 | params = { 112 | "tree_method": "hist", 113 | "n_gpus": "0", 114 | "objective": "binary:logistic", 115 | "min_child_weight": "1", 116 | "gamma": "0.1", 117 | "max_depth": "3", 118 | "verbosity": "0", 119 | } 120 | 121 | bst = mc2.xgb.Booster(params, [dtrain]) 122 | 123 | for i in range(5): 124 | bst.update(dtrain, i, None) 125 | 126 | return bst 127 | 128 | 129 | def predict(bst, dtest): 130 | predictions = bst.predict(dtest)[0] 131 | predictions = [float(i) for i in predictions[:10]] 132 | predictions = np.round(predictions, 7).tolist() 133 | 134 | expected_predictions = [ 135 | 0.1045543, 136 | 0.8036663, 137 | 0.1045543, 138 | 0.1045543, 139 | 0.1366708, 140 | 0.3470695, 141 | 0.8036663, 142 | 0.1176554, 143 | 0.8036663, 144 | 0.1060325, 145 | ] 146 | 147 | # Check that predictions are as expected for this model and test data 148 | assert predictions == expected_predictions 149 | 150 | 151 | def save_and_load_model(bst, dtest): 152 | bst.save_model("/home/chester/test_model.model") 153 | 154 | new_booster = mc2.xgb.Booster() 155 | new_booster.load_model("/home/chester/test_model.model") 156 | 157 | predict(new_booster, dtest) 158 | 159 | 160 | def get_feature_importance_by_weight(bst): 161 | features = bst.get_fscore() 162 | 163 | # Check that feature importance is as expected 164 | assert features == { 165 | "f29": 5, 166 | "f109": 5, 167 | "f67": 3, 168 | "f56": 2, 169 | "f21": 3, 170 | "f60": 2, 171 | "f27": 1, 172 | "f87": 1, 173 | "f23": 2, 174 | "f36": 2, 175 | "f24": 2, 176 | "f39": 1, 177 | } 178 | 179 | 180 | def get_feature_importance_by_gain(bst): 181 | features = bst.get_score(importance_type="gain") 182 | 183 | # Check that feature importance is as expected 184 | assert features == { 185 | "f29": 1802.9560316, 186 | "f109": 92.41320182000001, 187 | "f67": 55.9419556, 188 | "f56": 806.4257524999999, 189 | "f21": 276.0743410333333, 190 | "f60": 396.88085950000004, 191 | "f27": 258.393555, 192 | "f87": 33.4832764, 193 | "f23": 273.617882, 194 | "f36": 7.1899185345, 195 | "f24": 324.178024, 196 | "f39": 26.8505859, 197 | } 198 | 199 | 200 | def get_feature_importance_by_cover(bst): 201 | features = bst.get_score(importance_type="cover") 202 | 203 | # Check that feature importance is as expected 204 | assert features == { 205 | "f29": 1253.9055662, 206 | "f109": 534.3081298, 207 | "f67": 584.0368756666667, 208 | "f56": 830.696289, 209 | "f21": 352.7288766333333, 210 | "f60": 727.8263855, 211 | "f27": 248.831985, 212 | "f87": 530.806152, 213 | "f23": 542.0738525, 214 | "f36": 53.75369265, 215 | "f24": 488.320175, 216 | "f39": 337.194916, 217 | } 218 | 219 | 220 | def get_feature_importance_by_total_gain(bst): 221 | features = bst.get_score(importance_type="total_gain") 222 | 223 | # Check that feature importance is as expected 224 | assert features == { 225 | "f29": 9014.780158, 226 | "f109": 462.06600910000003, 227 | "f67": 167.8258668, 228 | "f56": 1612.8515049999999, 229 | "f21": 828.2230231, 230 | "f60": 793.7617190000001, 231 | "f27": 258.393555, 232 | "f87": 33.4832764, 233 | "f23": 547.235764, 234 | "f36": 14.379837069, 235 | "f24": 648.356048, 236 | "f39": 26.8505859, 237 | } 238 | 239 | 240 | def get_feature_importance_by_total_cover(bst): 241 | features = bst.get_score(importance_type="total_cover") 242 | 243 | # Check that feature importance is as expected 244 | assert features == { 245 | "f29": 6269.527831, 246 | "f109": 2671.5406489999996, 247 | "f67": 1752.110627, 248 | "f56": 1661.392578, 249 | "f21": 1058.1866298999998, 250 | "f60": 1455.652771, 251 | "f27": 248.831985, 252 | "f87": 530.806152, 253 | "f23": 1084.147705, 254 | "f36": 107.5073853, 255 | "f24": 976.64035, 256 | "f39": 337.194916, 257 | } 258 | 259 | 260 | def get_dump(tmp_path, bst): 261 | tests_dir = pathlib.Path(__file__).parent.absolute() 262 | expected_output = os.path.join(tests_dir, "data/expected_booster.dump") 263 | 264 | output = os.path.join(tmp_path, "booster.dump") 265 | bst.dump_model(output) 266 | 267 | # Check that dumped model is the same as expected 268 | assert filecmp.cmp(expected_output, output) 269 | -------------------------------------------------------------------------------- /quickstart/azure.yaml: -------------------------------------------------------------------------------- 1 | # An unique identifier for the head node and workers of this cluster. 2 | cluster_name: default 3 | 4 | # The total number of workers nodes to launch in addition to the head 5 | # node. This number should be >= 0. 6 | num_workers: 0 7 | 8 | # Cloud-provider specific configuration. 9 | provider: 10 | type: azure 11 | # https://docs.microsoft.com/en-us/azure/confidential-computing/virtual-machine-solutions 12 | location: eastus 13 | resource_group: mc2-client-dev 14 | storage_name: mc2storage 15 | container_name: blob-container-1 16 | 17 | # If left blank, the subscription ID from the Azure CLI will be used 18 | # subscription_id: 19 | 20 | # How MC2 will authenticate with newly launched nodes. 21 | auth: 22 | # TODO: remoe this field and make it the same as the username specified in config.yaml 23 | ssh_user: mc2 24 | # you must specify paths to matching private and public key pair files 25 | # use `ssh-keygen -t rsa -b 4096` to generate a new ssh key pair 26 | ssh_private_key: ~/.ssh/id_rsa 27 | ssh_public_key: ~/.ssh/id_rsa.pub 28 | 29 | # More specific customization to node configurations can be made using the ARM template azure-vm-template.json file 30 | # See documentation here: https://docs.microsoft.com/en-us/azure/templates/microsoft.compute/2019-03-01/virtualmachines 31 | # Changes to the local file will be used during deployment of the head node, however worker nodes deployment occurs 32 | # on the head node, so changes to the template must be included in the wheel file used in setup_commands section below 33 | 34 | # Provider-specific config for the head node, e.g. instance type. 35 | head_node: 36 | azure_arm_parameters: 37 | # https://docs.microsoft.com/en-us/azure/confidential-computing/virtual-machine-solutions 38 | vmSize: Standard_DC2s_v2 39 | 40 | # If launching a minimal Ubuntu machine 41 | # (and manually installing using setup commands) 42 | imagePublisher: Canonical 43 | imageOffer: UbuntuServer 44 | imageSku: 18_04-lts-gen2 45 | imageVersion: latest 46 | 47 | # Provider-specific config for worker nodes, e.g. instance type. 48 | worker_nodes: 49 | azure_arm_parameters: 50 | # https://docs.microsoft.com/en-us/azure/confidential-computing/virtual-machine-solutions 51 | vmSize: Standard_DC2s_v2 52 | 53 | # If launching a minimal Ubuntu machine 54 | # (and manually installing using setup commands) 55 | imagePublisher: Canonical 56 | imageOffer: UbuntuServer 57 | imageSku: 18_04-lts-gen2 58 | imageVersion: latest 59 | 60 | ############################################################################## 61 | # Everything below this can be ignored - you likely won't have to # 62 | # modify it. # 63 | ############################################################################## 64 | 65 | # Files or directories to copy to the head and worker nodes. The format is a 66 | # dictionary from REMOTE_PATH: LOCAL_PATH, e.g. 67 | file_mounts: { 68 | # This script installs Open Enclave 69 | "~/install_oe.sh" : "scripts/install_oe.sh", 70 | # This script builds Spark 3.1.1 from source 71 | "~/build_spark.sh" : "scripts/build_spark.sh", 72 | # This script downloads a pre-built Spark 3.1.1 binary 73 | "~/install_spark.sh" : "scripts/install_spark.sh", 74 | # This script builds Opaque from source 75 | "~/build_opaque.sh" : "scripts/build_opaque.sh", 76 | # This script installs Secure XGBoost from source 77 | "~/install_secure_xgboost.sh" : "scripts/install_secure_xgboost.sh" 78 | } 79 | 80 | # List of commands that will be run before `setup_commands`. If docker is 81 | # enabled, these commands will run outside the container and before docker 82 | # is setup. 83 | initialization_commands: 84 | # get rid of annoying Ubuntu message 85 | - touch ~/.sudo_as_admin_successful 86 | 87 | # List of shell commands to run to set up nodes. 88 | # Note: Use empty list if using image 89 | setup_commands: 90 | # This script installs Open Enclave on the node 91 | - chmod +x ~/install_oe.sh 92 | - source ~/install_oe.sh 93 | # This script installs Apache Spark on the node 94 | - chmod +x ~/install_spark.sh 95 | - source ~/install_spark.sh 96 | # This script installs Opaque on the node 97 | - chmod +x ~/build_opaque.sh 98 | - source ~/build_opaque.sh 99 | # This script installs Secure XGBoost on the node 100 | - chmod +x ~/install_secure_xgboost.sh 101 | - source ~/install_secure_xgboost.sh 102 | 103 | # Custom commands that will be run on the head node after common setup. 104 | # Set to empty list if using image 105 | head_setup_commands: [] 106 | 107 | # Custom commands that will be run on worker nodes after common setup. 108 | # Set to empty list if using image 109 | worker_setup_commands: [] 110 | 111 | # Command to start MC2 on the head node. 112 | # Set to empty list if using image 113 | head_start_mc2_commands: 114 | - cd $SPARK_HOME; ./sbin/start-master.sh 115 | 116 | # Command to start MC2 on worker nodes. 117 | # Set to empty list if using image 118 | worker_start_mc2_commands: 119 | - cd $SPARK_HOME; ./sbin/start-slave.sh $MC2_HEAD_IP:7077 120 | -------------------------------------------------------------------------------- /quickstart/config.yaml: -------------------------------------------------------------------------------- 1 | # User configuration 2 | user: 3 | # Your username - username should be specified in certificate 4 | username: user1 5 | 6 | # Path to your symmetric key - will be used for encryption/decryption 7 | # If you don't have a symmetric key, specify a path here 8 | # and run `mc2 init` to generate a key 9 | # 10 | # `mc2 init` will not overwrite anything at this path 11 | symmetric_key: ${MC2_CLIENT_HOME}/playground/keys/user1_sym.key 12 | 13 | # Path to your keypair and certificate. 14 | # If you don't have a private key / certificate, specify paths here 15 | # and run `mc2 init` to generate a keypair 16 | # 17 | # `mc2 init` will not overwrite anything at this path 18 | private_key: ${MC2_CLIENT_HOME}/playground/keys/user1.pem 19 | public_key: ${MC2_CLIENT_HOME}/playground/keys/user1.pub 20 | certificate: ${MC2_CLIENT_HOME}/playground/keys/user1.crt 21 | 22 | # Path to CA certificate and private key 23 | # Needed if you want to generate a certificate signed by CA 24 | root_certificate: ${MC2_CLIENT_HOME}/playground/keys/root.crt 25 | root_private_key: ${MC2_CLIENT_HOME}/playground/keys/root.pem 26 | 27 | # Configuration for launching cloud resources 28 | launch: 29 | # The absolute path to your Azure configuraton 30 | # This needs to be an absolute path 31 | azure_config: ${MC2_CLIENT_HOME}/playground/azure.yaml 32 | 33 | # # Manually specify the IP/uname/ssh_key of the head node or workers. 34 | # # If these values exist, they will override any values in `azure_config`. 35 | # Consequently, the `launch` and `stop` commands will do nothing. 36 | head: 37 | ip: 0.0.0.0 38 | # username: 39 | # ssh_key: 40 | workers: 41 | - ip: 0.0.0.0 42 | # username: 43 | # ssh_key: 44 | 45 | # Whether to launch a cluster of VMs 46 | cluster: false 47 | 48 | # Whether to launch Azure blob storage 49 | storage: false 50 | 51 | # Whether to launch a storage container 52 | container: false 53 | 54 | # Commands to start compute service 55 | start: 56 | # Commands to run on head node 57 | # This command is used to start the Opaque SQL service 58 | head: 59 | - cd /mc2/opaque-sql; build/sbt run 60 | 61 | # Commands to run on worker nodes 62 | # For this quickstart there is only one node - no worker nodes 63 | workers: [] 64 | 65 | # Configuration for `mc2 upload` 66 | upload: 67 | # Whether to upload data to Azure blob storage or disk 68 | # Allowed values are `blob` or `disk` 69 | # If `blob`, Azure CLI will be called to upload data 70 | # Else, `scp` will be used 71 | storage: disk 72 | 73 | # Encryption format to use 74 | # Options are `sql` if you want to use Opaque SQL 75 | # or `xgb` if you want to use Secure XGBoost 76 | format: sql 77 | 78 | # Files to encrypt and upload 79 | src: 80 | - ${MC2_CLIENT_HOME}/playground/data/opaquesql.csv 81 | 82 | # If you want to run Opaque SQL, you must also specify a schema, 83 | # one for each file you want to encrypt and upload 84 | schemas: 85 | - ${MC2_CLIENT_HOME}/playground/data/opaquesql_schema.json 86 | 87 | # Directory to upload data to 88 | dst: /mc2/data 89 | 90 | 91 | # Computation configuration 92 | run: 93 | # Script to run 94 | script: ${MC2_CLIENT_HOME}/quickstart/opaque_sql_demo.scala 95 | 96 | # Compute service you're using 97 | # Choices are `xgb` or `sql` 98 | compute: sql 99 | 100 | # Attestation configuration 101 | attestation: 102 | # Whether we are running in simulation mode 103 | # If 0 (False), we are _not_ running in simulation mode, 104 | # and should verify the attestation evidence 105 | simulation_mode: 1 106 | 107 | # MRENCLAVE value to check 108 | # MRENCLAVE is a hash of the enclave build log 109 | mrenclave: NULL 110 | 111 | # Path to MRSIGNER value to check 112 | # MRSIGNER is the key used to sign the built enclave 113 | mrsigner: ${MC2_CLIENT_HOME}/../opaque-sql/public_key.pub 114 | 115 | # Configuration for downloading results 116 | download: 117 | # Whether to download data from Azure blob storage or disk 118 | # Allowed values are `blob` or `disk` 119 | # If `blob`, Azure CLI will be called to download data 120 | # Else, `scp` will be used 121 | storage: disk 122 | 123 | # Format this data is encrypted with 124 | format: sql 125 | 126 | # Directory/file to download 127 | # FIXME: If storage is `blob` this value must be a file 128 | # Need to investigate whether we can use directories in Azure blob storage 129 | src: 130 | - /mc2/opaque-sql/opaque_sql_result 131 | 132 | # Local directory to download data to 133 | dst: results/ 134 | 135 | # Configuration for stopping services 136 | stop: 137 | 138 | # Configuration for deleting Azure resources 139 | teardown: 140 | # Whether to terminate launched VMs 141 | cluster: false 142 | 143 | # Whether to terminate created Azure blob storage 144 | storage: false 145 | 146 | # Whether to terminate created storage container 147 | container: false 148 | resource_group: false 149 | -------------------------------------------------------------------------------- /quickstart/data/opaquesql_schema.json: -------------------------------------------------------------------------------- 1 | Age:integer,BMI:float,Glucose:integer,Insulin:float,HOMA:float,Leptin:float,Adiponectin:float,Resistin:float,MCP.1:float,Classification:integer 2 | -------------------------------------------------------------------------------- /quickstart/keys/root.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID/TCCAmWgAwIBAgIUag4+JmUlpecz79+hB5nWdv8vsuswDQYJKoZIhvcNAQEL 3 | BQAwDzENMAsGA1UEAwwEcm9vdDAeFw0yMDA4MzExNzQ5NDNaFw0zMDA4MjkxNzQ5 4 | NDNaMA8xDTALBgNVBAMMBHJvb3QwggGgMA0GCSqGSIb3DQEBAQUAA4IBjQAwggGI 5 | AoIBgQDGJob5lAEOyq+QDi/6gGxnmkrwWGLApC/jIVTsTt5r7Vp73mn1QBbpovso 6 | kze9ahqU6x+gT8kI7VPavXwN+MjJMzbPKEErvl3XMoY6XPSIaQrEEPJiMEB2wx2y 7 | b5cLC9VqcYryPbrXd5LP5eXqXMoEpxFgOhnXKuMtBotAEMpBS/Uk5pZuzw6IC2BM 8 | +tKqWyma3ZtSy+CRDqOrSGsvtPHAExBxNbGEIg3rRUN7v3FtRcmydKwURMxFVBV+ 9 | /UrSDocz6Vq7IBeWQP2ytsX1pcDJyWJd1R78fE6ID8jF7IKnykzc8Bl6YxV+kxwt 10 | du9o13l3fiq+vSi8yb7eckJtkG3OIXQRCRGWyll2KEZvgDW032v30X9W/OdEZWZF 11 | ATLt3ufmS2QdPOxEFt/KeAb8A8Awc05uhYYIK2kvo1i7iikwBD5OevJZdB8108Dr 12 | 5UZ4rkhtiClrnIP26shZlzwtMJOQ+TanGB7pinKaf2K6Mq3ZF58OUQ8RsFLyeDL2 13 | 7TX+8GECAQOjUzBRMB0GA1UdDgQWBBRQm3Yp9L/SChF+/OJ1E0lZhB3SajAfBgNV 14 | HSMEGDAWgBRQm3Yp9L/SChF+/OJ1E0lZhB3SajAPBgNVHRMBAf8EBTADAQH/MA0G 15 | CSqGSIb3DQEBCwUAA4IBgQBx8ZxK0UuGAq67zQVPn/ddJoqFEvCPRWVGfzOQLaTP 16 | 5URHJLFRoSrluZgpxUAuyMfGnoVh5v2aT5mYgBqvx7X+SWJx90weU2rfGzFslvt2 17 | VtPDZcZH2VgN1l2upbor7RpT6K3Fd0Mftw7+YcWTgUo+xffbBuYfBlgcQo/PcfUr 18 | 4FbsmP50DGDTTU80JW/icRc0tCCb3MroZe36tZ5nTUrGghwo8c1ICFJNrkdrfGyf 19 | y6PytCbD9desjc24SzI7eu5Y0MmwmfGHUl/KwbZtLNGf3lNhgiI/tbFdo6nBGGrE 20 | ogfdkhe+A8r7o6QtQYzsaLRePeWpu1/yDrxgJySA0E+BhEDn7kNVSpqn3B2gVAHe 21 | Yxxy6HOfCTWMKTkj8pD8B6Swo81vBM1uH2hHdyEWZG80jPgxWVttniYkfv1jIcJW 22 | 5zgZ7/HT/3jRSNwARQMEs/vH38Cyntx5TCU4aDgP67fp+SfGf43xEZhxcqoCXzTN 23 | Voyw9vprJOhvJ05ewzhelqQ= 24 | -----END CERTIFICATE----- 25 | -------------------------------------------------------------------------------- /quickstart/keys/root.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIG5AIBAAKCAYEAxiaG+ZQBDsqvkA4v+oBsZ5pK8FhiwKQv4yFU7E7ea+1ae95p 3 | 9UAW6aL7KJM3vWoalOsfoE/JCO1T2r18DfjIyTM2zyhBK75d1zKGOlz0iGkKxBDy 4 | YjBAdsMdsm+XCwvVanGK8j2613eSz+Xl6lzKBKcRYDoZ1yrjLQaLQBDKQUv1JOaW 5 | bs8OiAtgTPrSqlspmt2bUsvgkQ6jq0hrL7TxwBMQcTWxhCIN60VDe79xbUXJsnSs 6 | FETMRVQVfv1K0g6HM+lauyAXlkD9srbF9aXAycliXdUe/HxOiA/IxeyCp8pM3PAZ 7 | emMVfpMcLXbvaNd5d34qvr0ovMm+3nJCbZBtziF0EQkRlspZdihGb4A1tN9r99F/ 8 | VvznRGVmRQEy7d7n5ktkHTzsRBbfyngG/APAMHNOboWGCCtpL6NYu4opMAQ+Tnry 9 | WXQfNdPA6+VGeK5IbYgpa5yD9urIWZc8LTCTkPk2pxge6Ypymn9iujKt2RefDlEP 10 | EbBS8ngy9u01/vBhAgEDAoIBgQCEGa9RDVYJ3HUKtB/8VZ2aZtygOuyAbXVCFjid 11 | iemdSOb9PvFOKrnxF1IbDM/TnBG4nL/ANTCwnjfnKP1epdswzM80xYDH1D6PdwQm 12 | 6KMFm1yCtfbsICr512khn7oHXTjxoQdMKSc6T7c1Q+6cPdwDGguVfBE6HJdzWbIq 13 | tdwrh/jDRGRJ318FXOrd/Ixxkhu8k7zh3UBgtG0c2vIfzfaADLWgznZYFrPyLiz9 14 | KkueLoZ2+HK4Ld2DjWOp/jHhXwOg8sfm6SYMoCQWKQEKYaz76HxnYrpYEm/86Ss/ 15 | VvCkEATZOJkqTr8R9RA6IyiQ5fCOCYMB6+Z2KoU8Jlml96ctcpe68RkeHWEvyo9M 16 | 2cXSFiyuzQq+xn1ZI8X+cddOtfIwgit/e8V7liERtow2uIXvWYKk2R9RYj8s9Eg8 17 | My4pDfo3BPP7lMBeQlbnQ0mGbIznr2kx7eY26T4Q9uckmE+2b/sDLvd0tzCiNKNe 18 | gDcHg9rJsezKOrQDWcM0QgWktGMCgcEA5Ctw7VAKOTK8Zum/a/t/PxI47m+E/Dvd 19 | s1YeTWwxQBWdrCWXnSnoehsy5ab389MiETe2IaybzUlS2HqZDD4P1h7FdQkT8By8 20 | tsJxUjiYir0L+W7ZbC9uIqfwJIVvKE+6w5qi78jTv9H0lCpyMyNURZPJxWBngO94 21 | 3OWltobqPhB/29c7jAjyDf5XFijl2TnNXN6RDesyv9uJLJ8gjlX4+HMjqZ2mWH7F 22 | kbbaPMXCyEfqpOnAZkDKUyNK0SIPMFDjAoHBAN5RvfNyVEoeCyqPhPoXvhDabtRR 23 | gnwkyNlb6Zl96HGcp+r1nB3DDmmIUPCbOpurbpE4MBousz5ApCu+Iuhe4zPWywOW 24 | V/mBive1/ioA9G8BHPgvFcyjvRwHzSLRAM9+Qdntf+46cErjuZu7wnbLowPZQLHf 25 | b40okY9PRqq2ebRexyAcSNQMDJpx53rXclXRp7UiepLMd+SxYhOFwOf2IwbeGni0 26 | BWH45BV5k2+smIWJ7Drca3wXeppOQ1doHleQ6wKBwQCYHPXzirF7dyhEm9Typ6oq 27 | DCX0SlioJ+kiOWmI8suADmkdbmUTcUWmvMyZGfqijMFgz87BHb0zhjc6/GYIKV/k 28 | FIOjW2KgEyh51vY20GWx011Q9JDyyklsb/Vtrkoa39HXvGyf2zfVNqMNcaF3bOLZ 29 | DTEuQEUAn6XomRkkWfF+taqSj30IBfaz/uS5cJk7e9496bYJR3cqkltzFMBe4/tQ 30 | TMJxE8Q6/y5hJJF92SyFhUcYm9WZgIbiF4c2FrTK4JcCgcEAlDZ+okw4MWlcxwpY 31 | prp+teb0jYusUsMwkOfxEP6a9mhv8fkSvoIJm7A19bzRvRz0YNAgEXR3ftXCx9QX 32 | RZSXd+SHV7mP+6ux+nlUHACi9KtopXS5MxfTaAUzbItV36mBO/OqntGgMe0mZ9KB 33 | pIfCApDVy+pKXhsLtN+Ecc77zZSEwBLbOAgIZvaaUeT24+EaeMGnDIhP7cuWt66A 34 | mqQXWelm+yKuQVCYDlEM9R27A7FIJz2c/WT8Zt7Xj5q+5QtHAoHBAKSRenWOHjHG 35 | eoVkz3UKR3Nwn1Ctn/cJmMHE53vR16MeN/FlqnPQdYlvdAPaAHMy91B/zbXAKHGB 36 | WxWlEEv6PbDa0xpHwOzKgkaiES3znEyq7cjlJ6HfURdaAbbbq+uYYdaE9/qQUddC 37 | xttQW+WqaKUz71cMRmAKzzXNBmeeueQ5V514k9r5smgfm/8+//xqltPDomNnoaqz 38 | zMubnimitg5M7OcDv/eR0Hfs+N9Rh3U4yo8DJBRfyvnVMrtw7ydwnQ== 39 | -----END RSA PRIVATE KEY----- 40 | -------------------------------------------------------------------------------- /quickstart/keys/root.pub: -------------------------------------------------------------------------------- 1 | -----BEGIN PUBLIC KEY----- 2 | MIIBoDANBgkqhkiG9w0BAQEFAAOCAY0AMIIBiAKCAYEAxiaG+ZQBDsqvkA4v+oBs 3 | Z5pK8FhiwKQv4yFU7E7ea+1ae95p9UAW6aL7KJM3vWoalOsfoE/JCO1T2r18DfjI 4 | yTM2zyhBK75d1zKGOlz0iGkKxBDyYjBAdsMdsm+XCwvVanGK8j2613eSz+Xl6lzK 5 | BKcRYDoZ1yrjLQaLQBDKQUv1JOaWbs8OiAtgTPrSqlspmt2bUsvgkQ6jq0hrL7Tx 6 | wBMQcTWxhCIN60VDe79xbUXJsnSsFETMRVQVfv1K0g6HM+lauyAXlkD9srbF9aXA 7 | ycliXdUe/HxOiA/IxeyCp8pM3PAZemMVfpMcLXbvaNd5d34qvr0ovMm+3nJCbZBt 8 | ziF0EQkRlspZdihGb4A1tN9r99F/VvznRGVmRQEy7d7n5ktkHTzsRBbfyngG/APA 9 | MHNOboWGCCtpL6NYu4opMAQ+TnryWXQfNdPA6+VGeK5IbYgpa5yD9urIWZc8LTCT 10 | kPk2pxge6Ypymn9iujKt2RefDlEPEbBS8ngy9u01/vBhAgED 11 | -----END PUBLIC KEY----- 12 | -------------------------------------------------------------------------------- /quickstart/opaque_sql_demo.py: -------------------------------------------------------------------------------- 1 | # This file performs the equivalent operations of opaque_sql_demo.scala, but in Python 2 | # To use, replace run --> script in config.yaml with the path to this script 3 | # and start --> head with the commands to start a PySpark cluster 4 | 5 | # Load in the encrypted data 6 | df = spark.read.format( # noqa: F821 7 | "edu.berkeley.cs.rise.opaque.EncryptedSource" 8 | ).load("/tmp/opaquesql.csv.enc") 9 | 10 | # Filter out all patients older than 30 11 | result = df.filter(df["Age"] < 30) 12 | 13 | # This will save the result DataFrame to the result directory on the cloud 14 | result.write.format("edu.berkeley.cs.rise.opaque.EncryptedSource").save( 15 | "/tmp/opaque_sql_result" 16 | ) 17 | -------------------------------------------------------------------------------- /quickstart/opaque_sql_demo.scala: -------------------------------------------------------------------------------- 1 | import edu.berkeley.cs.rise.opaque.implicits._ 2 | import org.apache.spark.sql.types._ 3 | 4 | val df = spark.read.format("edu.berkeley.cs.rise.opaque.EncryptedSource").load("/mc2/data/opaquesql.csv.enc") 5 | 6 | val result = df.filter($"Age" < lit(30)) 7 | 8 | // This will save the result DataFrame to the result directory on the cloud 9 | result.write.format("edu.berkeley.cs.rise.opaque.EncryptedSource").save("/mc2/opaque-sql/opaque_sql_result") 10 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | setuptools 2 | azure-mgmt-resource==10.0.0 3 | azure-mgmt-compute==10.0.0 4 | azure-mgmt-storage==10.0.0 5 | azure-mgmt-network==10.0.0 6 | azure-cli-core 7 | azure-storage-blob 8 | azure-identity 9 | cryptography==3.3.1 10 | flatbuffers 11 | grpcio 12 | grpcio-tools 13 | jsonschema 14 | numproto 15 | numpy==1.19.5 16 | pandas 17 | paramiko 18 | pytest 19 | pyyaml 20 | envyaml 21 | scp 22 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.11) 2 | 3 | project("MC2 Client" LANGUAGES C CXX) 4 | 5 | # Currently the `OpenEnclave` package depends on `project()`. 6 | find_package(OpenEnclave CONFIG REQUIRED) 7 | 8 | set(CMAKE_CXX_STANDARD 11) 9 | 10 | # ---- Fetch external packages ---- 11 | include(FetchContent) 12 | 13 | # Fetch mc2-utils 14 | FetchContent_Declare( 15 | mc2_utils_h 16 | GIT_REPOSITORY https://github.com/mc2-project/mc2-utils.git 17 | ) 18 | set(FETCHCONTENT_QUIET OFF) 19 | set(HOST ON CACHE BOOL "") 20 | FetchContent_MakeAvailable(mc2_utils_h) 21 | 22 | # Add necessary compiler flags and headers for spdlog dependency 23 | add_definitions(-DSPDLOG_NO_THREAD_ID -DFMT_USE_INT128=0) 24 | include_directories(${spdlog_SOURCE_DIR}/include) 25 | 26 | # Fetch mc2-serialization 27 | include(FetchContent) 28 | FetchContent_Declare( 29 | mc2_serialization 30 | GIT_REPOSITORY https://github.com/mc2-project/mc2-serialization.git 31 | ) 32 | set(FETCHCONTENT_QUIET OFF) 33 | FetchContent_MakeAvailable(mc2_serialization) 34 | 35 | # Add necessary headers for flatbuffers dependency 36 | message(${mc2_serialization_BINARY_DIR}) 37 | include_directories(${mc2_serialization_BINARY_DIR}/flatbuffers/include/) 38 | 39 | # Include external headers 40 | include_directories(${CMAKE_SOURCE_DIR}/include/) 41 | 42 | add_library( 43 | mc2client 44 | SHARED 45 | ${CMAKE_SOURCE_DIR}/c_api.cpp 46 | ${CMAKE_SOURCE_DIR}/io.cpp) 47 | add_dependencies(mc2client spdlog mc2_serialization mc2_utils_h) 48 | target_link_libraries(mc2client openenclave::oehost mc2_utils_h) 49 | -------------------------------------------------------------------------------- /src/c_api.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2020-22 by Contributors 3 | */ 4 | 5 | #include 6 | 7 | #include "base64.h" 8 | #include "io.h" 9 | 10 | #include "context.h" 11 | #include "crypto.h" 12 | 13 | typedef struct oe_evidence_msg_t { 14 | uint8_t public_key[CIPHER_PK_SIZE]; 15 | uint8_t nonce[CIPHER_IV_SIZE]; 16 | size_t evidence_size; 17 | uint8_t evidence[]; 18 | } oe_evidence_msg_t; 19 | 20 | extern "C" size_t cipher_iv_size() { return CIPHER_IV_SIZE; } 21 | 22 | extern "C" size_t rsa_mod_size() { return RSA_MOD_SIZE; } 23 | 24 | extern "C" int get_format_settings(uint8_t **format_settings, 25 | size_t *format_settings_size) { 26 | return Context::getInstance().m_attestation->GetFormatSettings( 27 | &(Context::getInstance().format_uuid), format_settings, 28 | format_settings_size); 29 | } 30 | 31 | extern "C" size_t cipher_pk_size() { return CIPHER_PK_SIZE; } 32 | 33 | extern "C" void get_public_key(uint8_t *evidence_msg, uint8_t *enclave_pem) { 34 | oe_evidence_msg_t *evidence_msg_t = 35 | reinterpret_cast(evidence_msg); 36 | memcpy(enclave_pem, evidence_msg_t->public_key, CIPHER_PK_SIZE); 37 | } 38 | 39 | extern "C" int attest_evidence(unsigned char *enclave_signer_pem, 40 | size_t enclave_signer_pem_size, 41 | uint8_t *evidence_msg, 42 | size_t evidence_msg_size) { 43 | oe_evidence_msg_t *evidence_msg_t = 44 | reinterpret_cast(evidence_msg); 45 | return Context::getInstance().m_attestation->AttestEvidence( 46 | &(Context::getInstance().format_uuid), 47 | reinterpret_cast(enclave_signer_pem), 48 | evidence_msg_t->evidence, evidence_msg_t->public_key, 49 | evidence_msg_t->nonce, enclave_signer_pem_size, 50 | evidence_msg_t->evidence_size, CIPHER_PK_SIZE); 51 | } 52 | 53 | extern "C" size_t sym_enc_size(size_t data_size) { 54 | return Context::getInstance().m_crypto->SymEncSize(data_size); 55 | } 56 | 57 | extern "C" int sym_enc(unsigned char *data, size_t data_size, uint8_t *sym_key, 58 | size_t key_size, uint8_t *encrypted_data) { 59 | return Context::getInstance().m_crypto->SymEnc( 60 | sym_key, reinterpret_cast(data), nullptr, encrypted_data, 61 | data_size, 0); 62 | } 63 | 64 | extern "C" size_t asym_enc_size(size_t data_size) { 65 | return Context::getInstance().m_crypto->AsymEncSize(data_size); 66 | } 67 | 68 | extern "C" int asym_enc(unsigned char *data, size_t data_size, uint8_t *pem_key, 69 | size_t key_size, uint8_t *encrypted_data) { 70 | return Context::getInstance().m_crypto->AsymEnc( 71 | pem_key, reinterpret_cast(data), encrypted_data, data_size); 72 | } 73 | 74 | extern "C" size_t asym_sign_size() { 75 | return Context::getInstance().m_crypto->AsymSignSize(); 76 | } 77 | 78 | extern "C" int sign_using_keyfile(char *keyfile, uint8_t *data, 79 | size_t data_size, uint8_t *signature) { 80 | return Context::getInstance().m_crypto->SignUsingKeyfile( 81 | keyfile, reinterpret_cast(data), signature, data_size); 82 | } 83 | 84 | extern "C" int verify(uint8_t *pem_key, uint8_t *data, size_t data_size, 85 | uint8_t *signature) { 86 | return Context::getInstance().m_crypto->Verify(pem_key, data, signature, 87 | data_size); 88 | } 89 | 90 | extern "C" int decrypt_dump(char *key, char **models, uint64_t length) { 91 | const char *total_encrypted; 92 | int out_len; 93 | for (int i = 0; i < length; i++) { 94 | total_encrypted = models[i]; 95 | 96 | // Allocate memory to deserialize the ciphertext. Base64 is a wasteful 97 | // encoding so this buffer will always be large enough. 98 | uint8_t *ct = new uint8_t[strlen(total_encrypted)]; 99 | auto ct_size = 100 | data::base64_decode(total_encrypted, strlen(total_encrypted), 101 | reinterpret_cast(ct)); 102 | 103 | // Allocate memory for the plaintext 104 | size_t pt_size = Context::getInstance().m_crypto->SymDecSize(ct_size); 105 | uint8_t *pt = new uint8_t[ct_size + 1]; 106 | 107 | auto ret = Context::getInstance().m_crypto->SymDec( 108 | reinterpret_cast(key), ct, nullptr, pt, ct_size, 0); 109 | pt[pt_size] = '\0'; 110 | if (ret != 0) 111 | return ret; 112 | 113 | delete[] ct; 114 | models[i] = (char *)pt; 115 | } 116 | return 0; 117 | } 118 | 119 | extern "C" void sxgb_encrypt_data(char *plaintext_file, char *encrypted_file, 120 | char *key_file, int *result) { 121 | int status = sxgb_encrypt_file(plaintext_file, encrypted_file, key_file); 122 | *result = status; 123 | } 124 | 125 | extern "C" void sxgb_decrypt_data(char *encrypted_file, char *plaintext_file, 126 | char *key_file, int *result) { 127 | int status = sxgb_decrypt_file(encrypted_file, plaintext_file, key_file); 128 | *result = status; 129 | } 130 | 131 | extern "C" void opaque_encrypt_data(char *plaintext_file, char *schema_file, 132 | char *encrypted_file, char *key_file, 133 | int *result) { 134 | OpaqueFileProcessor ofp; 135 | int status = ofp.opaque_encrypt_file(plaintext_file, schema_file, 136 | encrypted_file, key_file); 137 | *result = status; 138 | } 139 | 140 | extern "C" void opaque_decrypt_data(char **encrypted_files, 141 | size_t num_encrypted_files, 142 | char *plaintext_file, char *key_file, 143 | int *result) { 144 | OpaqueFileProcessor ofp; 145 | int status = ofp.opaque_decrypt_data(encrypted_files, num_encrypted_files, 146 | plaintext_file, key_file); 147 | *result = status; 148 | } 149 | 150 | extern "C" size_t cipher_key_size() { return CIPHER_KEY_SIZE; } 151 | -------------------------------------------------------------------------------- /src/context.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2020-22 by Contributors 3 | */ 4 | 5 | #include "attestation.h" 6 | #include "crypto.h" 7 | #include 8 | 9 | /* 10 | * This class generates a singleton containing Crypto and Attestation 11 | * instances 12 | */ 13 | class Context { 14 | private: 15 | Context() { 16 | m_crypto = new Crypto(); 17 | m_attestation = new Attestation(m_crypto); 18 | } 19 | 20 | public: 21 | Attestation *m_attestation; 22 | Crypto *m_crypto; 23 | // The format_uuid to use for attestation 24 | oe_uuid_t format_uuid = {OE_FORMAT_UUID_SGX_ECDSA}; 25 | 26 | // Don't forget to declare these two. You want to make sure they 27 | // are unacceptable otherwise you may accidentally get copies of 28 | // your singleton appearing. 29 | Context(Context const &) = delete; 30 | void operator=(Context const &) = delete; 31 | 32 | static Context &getInstance() { 33 | static Context instance; 34 | return instance; 35 | } 36 | }; 37 | -------------------------------------------------------------------------------- /src/include/base64.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef BASE64_H 3 | #define BASE64_H 4 | 5 | #include 6 | #include 7 | 8 | namespace data { 9 | 10 | static int base64_decode(const char* encoded_string, size_t in_len, char* decoded_string); 11 | static std::string base64_decode(std::string const& encoded_string); 12 | static std::string base64_encode(unsigned char const* bytes_to_encode, unsigned int in_len); 13 | 14 | static const std::string base64_chars = 15 | "ABCDEFGHIJKLMNOPQRSTUVWXYZ" 16 | "abcdefghijklmnopqrstuvwxyz" 17 | "0123456789+/"; 18 | 19 | 20 | static inline bool is_base64(unsigned char c) { 21 | return (isalnum(c) || (c == '+') || (c == '/')); 22 | } 23 | 24 | std::string base64_encode(unsigned char const* bytes_to_encode, unsigned int in_len) { 25 | std::string ret; 26 | int i = 0; 27 | int j = 0; 28 | unsigned char char_array_3[3]; 29 | unsigned char char_array_4[4]; 30 | 31 | while (in_len--) { 32 | char_array_3[i++] = *(bytes_to_encode++); 33 | if (i == 3) { 34 | char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; 35 | char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); 36 | char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); 37 | char_array_4[3] = char_array_3[2] & 0x3f; 38 | 39 | for(i = 0; (i <4) ; i++) 40 | ret += base64_chars[char_array_4[i]]; 41 | i = 0; 42 | } 43 | } 44 | 45 | if (i) { 46 | for(j = i; j < 3; j++) 47 | char_array_3[j] = '\0'; 48 | 49 | char_array_4[0] = ( char_array_3[0] & 0xfc) >> 2; 50 | char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); 51 | char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); 52 | 53 | for (j = 0; (j < i + 1); j++) 54 | ret += base64_chars[char_array_4[j]]; 55 | 56 | while((i++ < 3)) 57 | ret += '='; 58 | } 59 | 60 | return ret; 61 | 62 | } 63 | 64 | int base64_decode(const char* encoded_string, size_t in_len, char* decoded_string) { 65 | int i = 0; 66 | int j = 0; 67 | int in_ = 0; 68 | unsigned char char_array_4[4], char_array_3[3]; 69 | size_t pos = 0; 70 | 71 | while (in_len-- && ( encoded_string[in_] != '=') && is_base64(encoded_string[in_])) { 72 | char_array_4[i++] = encoded_string[in_]; in_++; 73 | if (i ==4) { 74 | for (i = 0; i <4; i++) 75 | char_array_4[i] = base64_chars.find(char_array_4[i]) & 0xff; 76 | 77 | char_array_3[0] = ( char_array_4[0] << 2 ) + ((char_array_4[1] & 0x30) >> 4); 78 | char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); 79 | char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; 80 | 81 | for (i = 0; (i < 3); i++) { 82 | decoded_string[pos++] = char_array_3[i]; 83 | } 84 | i = 0; 85 | } 86 | } 87 | if (i) { 88 | for (j = 0; j < i; j++) 89 | char_array_4[j] = base64_chars.find(char_array_4[j]) & 0xff; 90 | 91 | char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); 92 | char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); 93 | 94 | for (j = 0; (j < i - 1); j++) { 95 | decoded_string[pos++] = char_array_3[j]; 96 | } 97 | } 98 | return pos; 99 | } 100 | 101 | std::string base64_decode(std::string const& encoded_string) { 102 | size_t in_len = encoded_string.size(); 103 | int i = 0; 104 | int j = 0; 105 | int in_ = 0; 106 | unsigned char char_array_4[4], char_array_3[3]; 107 | std::string ret; 108 | 109 | while (in_len-- && ( encoded_string[in_] != '=') && is_base64(encoded_string[in_])) { 110 | char_array_4[i++] = encoded_string[in_]; in_++; 111 | if (i ==4) { 112 | for (i = 0; i <4; i++) 113 | char_array_4[i] = base64_chars.find(char_array_4[i]) & 0xff; 114 | 115 | char_array_3[0] = ( char_array_4[0] << 2 ) + ((char_array_4[1] & 0x30) >> 4); 116 | char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); 117 | char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; 118 | 119 | for (i = 0; (i < 3); i++) 120 | ret += char_array_3[i]; 121 | i = 0; 122 | } 123 | } 124 | 125 | if (i) { 126 | for (j = 0; j < i; j++) 127 | char_array_4[j] = base64_chars.find(char_array_4[j]) & 0xff; 128 | 129 | char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); 130 | char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); 131 | 132 | for (j = 0; (j < i - 1); j++) ret += char_array_3[j]; 133 | } 134 | return ret; 135 | } 136 | 137 | } // namespace data 138 | 139 | #endif // BASE64_H 140 | -------------------------------------------------------------------------------- /src/io.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2020-22 by Contributors 3 | */ 4 | 5 | #include "crypto.h" 6 | #include "flatbuffers/EncryptedBlock_generated.h" 7 | #include "flatbuffers/Rows_generated.h" 8 | 9 | // Encrypt a file in Secure XGBoost encryption format 10 | int sxgb_encrypt_file(char *fname, char *e_fname, char *k_fname); 11 | 12 | // Decrypt a file encrypted using Secure XGBoost encryption format 13 | int sxgb_decrypt_file(char *fname, char *d_fname, char *k_fname); 14 | 15 | class OpaqueFileProcessor { 16 | public: 17 | // Encrypt a file in Opaque encryption format 18 | int opaque_encrypt_file(char *fname, char *schema_file, char *e_fname, 19 | char *k_fname); 20 | 21 | // Decrypt a file encrypted using Opaque encryption format 22 | int opaque_decrypt_data(char **e_fnames, size_t num_encrypted_files, 23 | char *d_fname, char *k_fname); 24 | 25 | private: 26 | uint8_t symm_key[CIPHER_KEY_SIZE]; 27 | int num_partitions_outputted = 0; 28 | 29 | std::string output_dir; 30 | 31 | // For encrypted blocks 32 | // A Flatbuffers builder containing one or more serialized Encrypted Block's 33 | flatbuffers::FlatBufferBuilder enc_blocks_builder; 34 | // A vector holding offsets to each built Encrypted Block 35 | std::vector> enc_block_offsets; 36 | 37 | // For rows 38 | // A Flatbuffers builder containing one or more serialized Row's 39 | flatbuffers::FlatBufferBuilder rows_builder; 40 | // A vector holding the offsets to each built row 41 | std::vector> row_offsets; 42 | 43 | void finish_block(); 44 | void finish_encrypted_blocks(); 45 | void write_schema(std::vector column_names, 46 | std::vector column_types, 47 | const char *schema_path); 48 | }; 49 | -------------------------------------------------------------------------------- /src/utils.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2020-22 by Contributors 3 | */ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "crypto.h" 15 | 16 | // Print out a map> 17 | void print_map(std::map> dict) { 18 | for (const auto &pair : dict) { 19 | std::cout << pair.first << ": "; 20 | for (float x : pair.second) { 21 | std::cout << x << ", "; 22 | } 23 | std::cout << std::endl; 24 | } 25 | } 26 | 27 | void print_map(std::map dict) { 28 | for (const auto &pair : dict) { 29 | std::cout << pair.first << ": "; 30 | std::cout << std::boolalpha << pair.second << std::endl; 31 | std::cout << std::endl; 32 | } 33 | } 34 | 35 | void print_map_keys(std::map> dict) { 36 | for (const auto &pair : dict) { 37 | if (pair.first.length() > 20) 38 | continue; 39 | std::cout << pair.first << std::endl; 40 | } 41 | } 42 | 43 | // Print integers instead of bytes for encryption debugging. 44 | int print_bytes(uint8_t *data, size_t len) { 45 | for (int i = 0; i < len; i++) { 46 | std::cout << (int)data[i] << " "; 47 | } 48 | std::cout << std::endl; 49 | } 50 | 51 | // Delete a double pointer. 52 | void delete_double_ptr(unsigned char **src, size_t num) { 53 | for (int i = 0; i < num; i++) { 54 | delete src[i]; 55 | } 56 | delete src; 57 | } 58 | 59 | // Split a string by delimiter 60 | std::vector split(const std::string &s, char delim) { 61 | std::vector result; 62 | std::stringstream ss(s); 63 | std::string item; 64 | 65 | while (std::getline(ss, item, delim)) { 66 | result.push_back(item); 67 | } 68 | 69 | return result; 70 | } 71 | 72 | void load_key(char *k_fname, char key[CIPHER_KEY_SIZE]) { 73 | std::ifstream keyfile; 74 | keyfile.open(k_fname); 75 | keyfile.read(key, CIPHER_KEY_SIZE); 76 | keyfile.close(); 77 | } 78 | 79 | // Convert a date to its integer representation 80 | int date_to_int(std::string &&date) { 81 | struct tm tm = {0}; 82 | std::istringstream iss(date); 83 | iss >> std::get_time(&tm, "%Y-%m-%d"); 84 | // mktime returns seconds since the epoch, but opaque expects days since 85 | // the epoch 86 | auto days_since_epoch = mktime(&tm) / (24 * 3600); 87 | return days_since_epoch; 88 | } 89 | 90 | // Convert an integer to its date representation 91 | std::string int_to_date(int days_since_epoch) { 92 | // `time_t` is represented as seconds since the epoch 93 | time_t secs_since_epoch = days_since_epoch * 24 * 3600; 94 | struct tm *tm = gmtime(&secs_since_epoch); 95 | std::ostringstream oss; 96 | oss << std::put_time(tm, "%Y-%m-%d"); 97 | return oss.str(); 98 | } 99 | 100 | // This function ensures that decrypted floating point values have their types 101 | // correctly inferred in Spark. By default, if a floating point value is 102 | // equivalent to an integer, C++ will display it without a decimal + trailing 103 | // zero. However, this is incorrect behaivor in Spark. 104 | template std::string fmt_floating(F value) { 105 | static_assert( 106 | std::is_floating_point::value, 107 | "Attempted to call fmt_decimal on value which isn't a floating point"); 108 | // Extract the integer part of the value. This is necessary since a simple 109 | // cast can fail if the value is too large. 110 | double int_part; 111 | modf(value, &int_part); 112 | 113 | std::ostringstream oss; 114 | if (value == int_part) { 115 | oss << value << ".0"; 116 | } else { 117 | oss << value; 118 | } 119 | return oss.str(); 120 | } 121 | --------------------------------------------------------------------------------