├── images ├── occlum_maa.png └── spark_sgx_azure.png ├── run_azure_nytaxi.sh ├── docker ├── README.md ├── mount.sh ├── occlum-build.sh ├── add_conf.sh ├── build-docker-image.sh ├── Dockerfile ├── entrypoint.sh └── run_spark_on_occlum_glibc.sh ├── driver.yaml ├── run_spark_pi.sh ├── executor.yaml ├── run_nytaxi_k8s.sh ├── secretProviderClass.yaml ├── nytaxi ├── src │ └── main │ │ └── scala │ │ └── AzureNytaxi.scala └── pom.xml └── README.md /images/occlum_maa.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/BigDL-PPML-Azure-Occlum-Example/main/images/occlum_maa.png -------------------------------------------------------------------------------- /images/spark_sgx_azure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/BigDL-PPML-Azure-Occlum-Example/main/images/spark_sgx_azure.png -------------------------------------------------------------------------------- /run_azure_nytaxi.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | $SPARK_HOME/bin/spark-submit \ 4 | --class AzureNytaxi \ 5 | --master local[8] \ 6 | spark-azure-nytaxi-1.0-SNAPSHOT.jar 7 | -------------------------------------------------------------------------------- /docker/README.md: -------------------------------------------------------------------------------- 1 | # Build a runnable Occlum image 2 | 3 | 1. To build a runnable image, you need to change `$image_name` and `$final_name` in `build-docker-image.sh`. 4 | `image_name` means temp image, `final_name` means the final runnable image you will get. 5 | 6 | 2. Configure four SGX parameters in `occlum-build.sh`. These configures can not be changed by `spark-submit` commands. 7 | ```bash 8 | -e SGX_MEM_SIZE=8GB \ 9 | -e SGX_THREAD=2048 \ 10 | -e SGX_HEAP=1GB \ 11 | -e SGX_KERNEL_HEAP=1GB 12 | ``` 13 | 14 | 3. Run `build-docker-image.sh` to build, and the image named `$final_name` will be built. 15 | ```bash 16 | bash build-docker-image.sh 17 | ``` -------------------------------------------------------------------------------- /docker/mount.sh: -------------------------------------------------------------------------------- 1 | 2 | #To avoid soft-link error, need to mkdir copy dir 3 | [[ -d /opt/spark/conf/..data ]] || mkdir /opt/spark/conf/..data 4 | [[ -d /opt/spark/conf-copy ]] || mkdir /opt/spark/conf-copy 5 | cp /opt/spark/conf/..data/* /opt/spark/conf-copy 6 | 7 | [[ -d /opt/spark/pod-template/..data ]] || mkdir /opt/spark/pod-template/..data 8 | [[ -d /opt/spark/pod-template-copy ]] || mkdir /opt/spark/pod-template-copy 9 | cp /opt/spark/pod-template/..data/* /opt/spark/pod-template-copy 10 | 11 | [[ -d /var/run/secrets/kubernetes.io/serviceaccount/..data ]] || mkdir -p /var/run/secrets/kubernetes.io/serviceaccount/..data 12 | [[ -d /var/run/secrets/kubernetes.io/serviceaccount-copy ]] || mkdir -p /var/run/secrets/kubernetes.io/serviceaccount-copy 13 | cp /var/run/secrets/kubernetes.io/serviceaccount/..data/* /var/run/secrets/kubernetes.io/serviceaccount-copy 14 | -------------------------------------------------------------------------------- /driver.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: spark-deployment 5 | namespace: default 6 | spec: 7 | containers: 8 | - name: spark-example 9 | imagePullPolicy: Never 10 | volumeMounts: 11 | - name: device-plugin 12 | mountPath: /var/lib/kubelet/device-plugins 13 | - name: aesm 14 | mountPath: /var/run/aesmd 15 | securityContext: 16 | privileged: true 17 | env: 18 | - name: DRIVER_MEMORY 19 | value: "1GB" 20 | - name: SGX_MEM_SIZE 21 | value: "8GB" 22 | - name: SGX_THREAD 23 | value: "256" 24 | - name: SGX_HEAP 25 | value: "1GB" 26 | - name: SGX_KERNEL_HEAP 27 | value: "1GB" 28 | - name: AZDCAP_DEBUG_LOG_LEVEL 29 | value: "fatal" 30 | - name: FS_TYPE 31 | value: "hostfs" 32 | volumes: 33 | - name: device-plugin 34 | hostPath: 35 | path: /var/lib/kubelet/device-plugins 36 | - name: aesm 37 | hostPath: 38 | path: /var/run/aesmd 39 | -------------------------------------------------------------------------------- /run_spark_pi.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | IMAGE=intelanalytics/bigdl-ppml-azure-occlum:2.1.0 3 | ${SPARK_HOME}/bin/spark-submit \ 4 | --master k8s://https://${kubernetes_master_url}:${k8s_apiserver_port} \ 5 | --deploy-mode cluster \ 6 | --name spark-pi \ 7 | --class org.apache.spark.examples.SparkPi \ 8 | --conf spark.executor.instances=1 \ 9 | --conf spark.rpc.netty.dispatcher.numThreads=32 \ 10 | --conf spark.kubernetes.container.image=${IMAGE} \ 11 | --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark \ 12 | --conf spark.kubernetes.executor.deleteOnTermination=false \ 13 | --conf spark.kubernetes.driver.podTemplateFile=./driver.yaml \ 14 | --conf spark.kubernetes.executor.podTemplateFile=./executor.yaml \ 15 | --conf spark.kubernetes.sgx.log.level=off \ 16 | --conf spark.executorEnv.SGX_EXECUTOR_JVM_MEM_SIZE="1g" \ 17 | --conf spark.kubernetes.driverEnv.SGX_DRIVER_JVM_MEM_SIZE="1g" \ 18 | local:/opt/spark/examples/jars/spark-examples_2.12-3.1.2.jar 19 | -------------------------------------------------------------------------------- /executor.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: spark-deployment 5 | namespace: default 6 | spec: 7 | containers: 8 | - name: spark-example 9 | imagePullPolicy: Never 10 | volumeMounts: 11 | - name: device-plugin 12 | mountPath: /var/lib/kubelet/device-plugins 13 | - name: aesm 14 | mountPath: /var/run/aesmd 15 | securityContext: 16 | privileged: true 17 | env: 18 | - name: EXECUTOR_MEMORY 19 | value: "1GB" 20 | - name: NETTY_THREAD 21 | value: "32" 22 | - name: SGX_MEM_SIZE 23 | value: "8GB" 24 | - name: SGX_THREAD 25 | value: "256" 26 | - name: SGX_HEAP 27 | value: "1GB" 28 | - name: SGX_KERNEL_HEAP 29 | value: "1GB" 30 | - name: AZDCAP_DEBUG_LOG_LEVEL 31 | value: "fatal" 32 | - name: FS_TYPE 33 | value: "hostfs" 34 | volumes: 35 | - name: device-plugin 36 | hostPath: 37 | path: /var/lib/kubelet/device-plugins 38 | - name: aesm 39 | hostPath: 40 | path: /var/run/aesmd 41 | -------------------------------------------------------------------------------- /run_nytaxi_k8s.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | IMAGE=intelanalytics/bigdl-ppml-azure-occlum:2.1.0 3 | ${SPARK_HOME}/bin/spark-submit \ 4 | --master k8s://https://${kubernetes_master_url}:${k8s_apiserver_port} \ 5 | --deploy-mode cluster \ 6 | --name spark-nytaxi \ 7 | --class AzureNytaxi \ 8 | --driver-cores 4 \ 9 | --executor-cores 4 \ 10 | --conf spark.executor.instances=1 \ 11 | --conf spark.rpc.netty.dispatcher.numThreads=128 \ 12 | --conf spark.kubernetes.container.image=${IMAGE} \ 13 | --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark \ 14 | --conf spark.kubernetes.executor.deleteOnTermination=false \ 15 | --conf spark.kubernetes.driver.podTemplateFile=./driver.yaml \ 16 | --conf spark.kubernetes.executor.podTemplateFile=./executor.yaml \ 17 | --conf spark.kubernetes.sgx.log.level=off \ 18 | --conf spark.executorEnv.SGX_EXECUTOR_JVM_MEM_SIZE="1g" \ 19 | --conf spark.kubernetes.driverEnv.SGX_DRIVER_JVM_MEM_SIZE="1g" \ 20 | local:/bin/jars/spark-azure-nytaxi-1.0-SNAPSHOT.jar 21 | -------------------------------------------------------------------------------- /secretProviderClass.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: secrets-store.csi.x-k8s.io/v1 2 | kind: SecretProviderClass 3 | metadata: 4 | name: azure-kv-user-msi 5 | spec: 6 | provider: azure 7 | parameters: 8 | usePodIdentity: "false" 9 | useVMManagedIdentity: "true" # Set to true for using managed identity 10 | userAssignedIdentityID: # Set the clientID of the user-assigned managed identity to use 11 | keyvaultName: # Set to the name of your key vault 12 | cloudName: "" # [OPTIONAL for Azure] if not provided, the Azure environment defaults to AzurePublicCloud 13 | objects: | 14 | array: 15 | - | 16 | objectName: enclave-key-pem 17 | objectType: secret # object types: secret, key, or cert 18 | objectVersion: "" # [OPTIONAL] object versions, default to latest if empty 19 | - | 20 | objectName: key-pass 21 | objectType: secret 22 | objectVersion: "" 23 | tenantId: -------------------------------------------------------------------------------- /docker/occlum-build.sh: -------------------------------------------------------------------------------- 1 | # default 2 | export container_name=bigdl-ppml-trusted-big-data-ml-scala-occlum-build 3 | export image_name=intelanalytics/bigdl-ppml-azure-occlum:2.1.0-SNAPSHOT 4 | export final_name=intelanalytics/bigdl-ppml-azure-occlum:2.1.0 5 | while getopts ":c:i:f:" opt 6 | do 7 | case $opt in 8 | c) 9 | export container_name=$OPTARG 10 | echo "container_name:" $container_name 11 | ;; 12 | i) 13 | export image_name=$OPTARG 14 | echo "image_name:" $image_name 15 | ;; 16 | f) 17 | export final_name=$OPTARG 18 | echo "final_name:" $final_name 19 | ;; 20 | ?) 21 | echo $OPTARG 22 | echo "unknow parameter" 23 | exit 1;; 24 | esac 25 | done 26 | #Clean up old 27 | sudo docker rm -f $container_name 28 | 29 | # Run new command in container 30 | sudo docker run -it \ 31 | --net=host \ 32 | --name=$container_name \ 33 | --cpuset-cpus 3-5 \ 34 | -e LOCAL_IP=$LOCAL_IP \ 35 | -e SGX_MEM_SIZE=12GB \ 36 | -e SGX_THREAD=2048 \ 37 | -e SGX_HEAP=1GB \ 38 | -e SGX_KERNEL_HEAP=1GB \ 39 | -e PCCS_URL=$PCCS_URL \ 40 | $image_name \ 41 | bash /opt/run_spark_on_occlum_glibc.sh init 42 | echo "build finish" 43 | docker commit $container_name $final_name 44 | -------------------------------------------------------------------------------- /docker/add_conf.sh: -------------------------------------------------------------------------------- 1 | #To add occlum mount conf and mkdir target dir 2 | edit_json="$(cat Occlum.json | jq '.mount+=[{"target": "/opt/spark/pod-template","type": "hostfs","source": "/opt/spark/pod-template-copy"}]')" && \ 3 | echo "${edit_json}" > Occlum.json 4 | edit_json="$(cat Occlum.json | jq '.mount+=[{"target": "/var/run/secrets/kubernetes.io/serviceaccount","type": "hostfs","source": "/var/run/secrets/kubernetes.io/serviceaccount-copy"}]')" && \ 5 | echo "${edit_json}" > Occlum.json 6 | edit_json="$(cat Occlum.json | jq '.mount+=[{"target": "/opt/spark/conf","type": "hostfs","source": "/opt/spark/conf-copy"}]')" && \ 7 | echo "${edit_json}" > Occlum.json 8 | 9 | #mkdir target (should exist in occlum build) 10 | mkdir -p /opt/spark/conf 11 | mkdir -p /opt/spark/pod-template 12 | mkdir -p /var/run/secrets/kubernetes.io/serviceaccount 13 | 14 | #mkdir source ..data 15 | mkdir -p /opt/spark/conf/..data 16 | mkdir -p /opt/spark/pod-template/..data 17 | mkdir -p /var/run/secrets/kubernetes.io/serviceaccount/..data 18 | 19 | #make ..data not empty 20 | 21 | #create test.txt 22 | [[ -c ./test.txt ]] || touch ./test.txt 23 | echo "test" > test.txt 24 | 25 | # cp test.txt to ..data 26 | cp ./test.txt /opt/spark/conf/..data/ 27 | cp ./test.txt /opt/spark/pod-template/..data/ 28 | cp ./test.txt /var/run/secrets/kubernetes.io/serviceaccount/..data/ 29 | rm ./test.txt 30 | 31 | -------------------------------------------------------------------------------- /nytaxi/src/main/scala/AzureNytaxi.scala: -------------------------------------------------------------------------------- 1 | import java.util.Date 2 | 3 | import org.apache.spark.sql.SparkSession 4 | 5 | object AzureNytaxi { 6 | val spark: SparkSession = SparkSession.builder() 7 | .appName("azure-nytaxi") 8 | .getOrCreate() 9 | 10 | def main(args: Array[String]): Unit = { 11 | val blobAccountName = sys.env.getOrElse("AZURE_BLOB_ACCOUNT_NAME", "") 12 | val blobContainerName = sys.env.getOrElse("AZURE_BLOB_CONTAINER_NAME", "") 13 | val blobRelativePath = sys.env.getOrElse("AZURE_BLOB_RELATIVE_PATH", "") 14 | val blobSasToken = sys.env.getOrElse("AZURE_BLOB_SAS_TOKEN", "") 15 | val azureSqlAeJdbc = sys.env.getOrElse("AZURE_SQL_AE_JDBC", "") 16 | 17 | println("########################################") 18 | println("############### COUNT * ################") 19 | println("########################################") 20 | val startTime = new Date().getTime 21 | val path = "wasbs://%s@%s.blob.core.windows.net/%s".format(blobContainerName, blobAccountName, blobRelativePath) 22 | spark.conf.set("fs.azure.sas.%s.%s.blob.core.windows.net".format(blobContainerName, blobAccountName), blobSasToken) 23 | val parquetDF = spark.read.parquet(path) 24 | println("Input DataFrame Count: " + parquetDF.count()) 25 | val endTime = new Date().getTime 26 | println("Aggregation duration: " + (endTime - startTime)) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /docker/build-docker-image.sh: -------------------------------------------------------------------------------- 1 | export HTTP_PROXY_HOST=your_http_proxy_host 2 | export HTTP_PROXY_PORT=your_http_proxy_port 3 | export HTTPS_PROXY_HOST=your_https_proxy_host 4 | export HTTPS_PROXY_PORT=your_https_proxy_port 5 | export AZURE_BLOB_ACCOUNT_NAME=azureopendatastorage 6 | export AZURE_BLOB_CONTAINER_NAME=nyctlc 7 | export AZURE_BLOB_RELATIVE_PATH=yellow 8 | export AZURE_BLOB_SAS_TOKEN=r 9 | 10 | cd ../nytaxi 11 | mvn clean package 12 | cd ../docker 13 | cp ../nytaxi/target/spark-azure-nytaxi-1.0-SNAPSHOT-jar-with-dependencies.jar ./ 14 | 15 | export container_name=bigdl-ppml-trusted-big-data-ml-scala-occlum-build 16 | export image_name=intelanalytics/bigdl-ppml-azure-occlum:2.1.0-SNAPSHOT 17 | export final_name=intelanalytics/bigdl-ppml-azure-occlum:2.1.0 18 | 19 | sudo docker build \ 20 | --build-arg http_proxy=http://$HTTP_PROXY_HOST:$HTTP_PROXY_PORT \ 21 | --build-arg https_proxy=http://$HTTPS_PROXY_HOST:$HTTPS_PROXY_PORT \ 22 | --build-arg HTTP_PROXY_HOST=$HTTP_PROXY_HOST \ 23 | --build-arg HTTP_PROXY_PORT=$HTTP_PROXY_PORT \ 24 | --build-arg HTTPS_PROXY_HOST=$HTTPS_PROXY_HOST \ 25 | --build-arg HTTPS_PROXY_PORT=$HTTPS_PROXY_PORT \ 26 | --build-arg AZURE_BLOB_ACCOUNT_NAME=$AZURE_BLOB_ACCOUNT_NAME \ 27 | --build-arg AZURE_BLOB_CONTAINER_NAME=$AZURE_BLOB_CONTAINER_NAME \ 28 | --build-arg AZURE_BLOB_RELATIVE_PATH=$AZURE_BLOB_RELATIVE_PATH \ 29 | --build-arg AZURE_BLOB_SAS_TOKEN=$AZURE_BLOB_SAS_TOKEN \ 30 | --build-arg AZURE_SQL_AE_JDBC=$AZURE_SQL_AE_JDBC \ 31 | -t $image_name -f ./Dockerfile . 32 | 33 | bash occlum-build.sh -c $container_name -i $image_name -f $final_name 34 | -------------------------------------------------------------------------------- /nytaxi/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.example 8 | spark-azure-nytaxi 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 1.8 13 | 3.4.0 14 | 2.12 15 | 16 | 17 | 18 | 19 | org.apache.spark 20 | spark-core_${scala.version} 21 | ${spark.version} 22 | 23 | 24 | 25 | org.apache.spark 26 | spark-sql_${scala.version} 27 | ${spark.version} 28 | 29 | 30 | 31 | 32 | 33 | 34 | org.scala-tools 35 | maven-scala-plugin 36 | 2.12.2 37 | 38 | 39 | 40 | compile 41 | testCompile 42 | 43 | 44 | 45 | 46 | 47 | org.apache.maven.plugins 48 | maven-assembly-plugin 49 | 3.0.0 50 | 51 | 52 | jar-with-dependencies 53 | 54 | 55 | 56 | 57 | make-assembly 58 | package 59 | 60 | single 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM intelanalytics/bigdl-ppml-trusted-big-data-ml-scala-occlum:2.1.0 as ppml 2 | 3 | ARG BIGDL_VERSION=2.1.0 4 | ARG HTTP_PROXY_HOST 5 | ARG HTTP_PROXY_PORT 6 | ARG HTTPS_PROXY_HOST 7 | ARG HTTPS_PROXY_PORT 8 | ARG AZURE_BLOB_ACCOUNT_NAME 9 | ARG AZURE_BLOB_CONTAINER_NAME 10 | ARG AZURE_BLOB_RELATIVE_PATH 11 | ARG AZURE_BLOB_SAS_TOKEN 12 | ARG AZURE_SQL_AE_JDBC 13 | 14 | ENV BIGDL_VERSION=${BIGDL_VERSION} 15 | ENV HTTP_PROXY=http://$HTTP_PROXY_HOST:$HTTP_PROXY_PORT 16 | ENV HTTPS_PROXY=http://$HTTPS_PROXY_HOST:$HTTPS_PROXY_PORT 17 | ENV AZURE_BLOB_ACCOUNT_NAME=$AZURE_BLOB_ACCOUNT_NAME 18 | ENV AZURE_BLOB_CONTAINER_NAME=$AZURE_BLOB_CONTAINER_NAME 19 | ENV AZURE_BLOB_RELATIVE_PATH=$AZURE_BLOB_RELATIVE_PATH 20 | ENV AZURE_BLOB_SAS_TOKEN=$AZURE_BLOB_SAS_TOKEN 21 | ENV AZURE_SQL_AE_JDBC=$AZURE_SQL_AE_JDBC 22 | 23 | ENV AZDCAP_DEBUG_LOG_LEVEL=fatal 24 | 25 | RUN mkdir -p /opt/src && \ 26 | cd /opt/src && \ 27 | git clone https://github.com/occlum/occlum.git && \ 28 | cd occlum && \ 29 | apt purge libsgx-dcap-default-qpl -y 30 | 31 | RUN echo "deb [arch=amd64] https://packages.microsoft.com/ubuntu/20.04/prod focal main" | sudo tee /etc/apt/sources.list.d/msprod.list && \ 32 | wget -qO - https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - && \ 33 | apt update && \ 34 | apt install az-dcap-client tree 35 | 36 | RUN cd /opt/src/occlum && \ 37 | git submodule update --init 38 | 39 | RUN wget -P /opt/spark/jars/ https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-azure/3.2.0/hadoop-azure-3.2.0.jar && \ 40 | wget -P /opt/spark/jars/ https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-azure-datalake/3.2.0/hadoop-azure-datalake-3.2.0.jar && \ 41 | wget -P /opt/spark/jars/ https://repo1.maven.org/maven2/com/microsoft/azure/azure-storage/7.0.0/azure-storage-7.0.0.jar && \ 42 | wget -P /opt/spark/jars/ https://repo1.maven.org/maven2/com/microsoft/azure/azure-data-lake-store-sdk/2.2.9/azure-data-lake-store-sdk-2.2.9.jar && \ 43 | wget -P /opt/spark/jars/ https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util-ajax/9.3.24.v20180605/jetty-util-ajax-9.3.24.v20180605.jar && \ 44 | wget -P /opt/spark/jars/ https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util/9.3.24.v20180605/jetty-util-9.3.24.v20180605.jar && \ 45 | wget -P /opt/spark/jars/ https://repo1.maven.org/maven2/org/wildfly/openssl/wildfly-openssl/1.0.7.Final/wildfly-openssl-1.0.7.Final.jar 46 | 47 | RUN rm /opt/run_spark_on_occlum_glibc.sh && \ 48 | rm /opt/entrypoint.sh && \ 49 | rm -rf /opt/spark-source 50 | 51 | ADD ./run_spark_on_occlum_glibc.sh /opt/run_spark_on_occlum_glibc.sh 52 | ADD ./entrypoint.sh /opt/entrypoint.sh 53 | ADD ./mount.sh /opt/mount.sh 54 | ADD ./add_conf.sh /opt/occlum_spark/add_conf.sh 55 | 56 | RUN cp /opt/run_spark_on_occlum_glibc.sh /root/run_spark_on_occlum_glibc.sh 57 | 58 | RUN cd /opt/src/occlum/demos/remote_attestation/azure_attestation/maa_init/init && \ 59 | cargo clean && \ 60 | cargo build --release 61 | 62 | RUN chmod a+x /opt/entrypoint.sh && \ 63 | chmod a+x /opt/run_spark_on_occlum_glibc.sh && \ 64 | chmod a+x /root/run_spark_on_occlum_glibc.sh && \ 65 | chmod a+x /opt/mount.sh && \ 66 | chmod a+x /opt/occlum_spark/add_conf.sh 67 | 68 | ADD ./spark-azure-nytaxi-1.0-SNAPSHOT-jar-with-dependencies.jar /root/spark-azure-nytaxi-1.0-SNAPSHOT.jar 69 | RUN cp /root/spark-azure-nytaxi-1.0-SNAPSHOT.jar /opt/bigdl-${BIGDL_VERSION}/jars/ 70 | ADD ./run_azure_nytaxi.sh /root/run_azure_nytaxi.sh 71 | 72 | ENTRYPOINT [ "/opt/entrypoint.sh" ] 73 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # BigDL-PPML-Azure-Occlum-Example 2 | 3 | ## Overview 4 | 5 | This repository demonstrates how to run standard Apache Spark applications with BigDL PPML and Occlum on Azure Intel SGX enabled Confidential Virtual machines ([DCsv3](https://docs.microsoft.com/en-us/azure/virtual-machines/dcv3-series) or [Azure Kubernetes Service (AKS)](https://azure.microsoft.com/en-us/services/kubernetes-service/)). These Azure Virtual Machines include the Intel SGX extensions. 6 | 7 | Key points: 8 | 9 | * Azure DC Series: We run distributed Spark 3.1.2 examples, on an Azure DCsv3 machine running Docker. These machines are backed by the 3rd generation Intel Xeon Scalabe Processor with large Encrypted Page Cache (EPC) memory. 10 | * Occlum: To run Spark inside an Intel SGX enclave - we leverage Occlum, who have essentially taken the Open Source Spark code, and wrapped it with their enclave runtime so that Spark can run inside SGX enclaves (a task that requires deep knowledge of the SGX ecosystem - something Occlum is an expert at). 11 | 12 | ![Distributed Spark in SGX on Azure](images/spark_sgx_azure.png) 13 | 14 |

15 | 16 | * For Azure attestation details in Occlum init process please refer to [`maa_init`](https://github.com/occlum/occlum/tree/master/demos/remote_attestation/azure_attestation/maa_init). 17 | 18 | ## Prerequisites 19 | 20 | * Set up Azure VM on Azure 21 | * [DCsv3](https://docs.microsoft.com/en-us/azure/virtual-machines/dcv3-series) for [single node spark example](#single-node-spark-examples-on-azure). 22 | * [Azure Kubernetes Service (AKS)](https://azure.microsoft.com/en-us/services/kubernetes-service/) for [distributed Spark examples](#distributed-spark-example-on-aks). 23 | * Prepare image of Spark 24 | * (Required for distributed Spark examples only) Download [Spark 3.1.2](https://archive.apache.org/dist/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz) and extract Spark binary. Install OpenJDK-8, and `export SPARK_HOME=${Spark_Binary_dir}`. 25 | 26 | Pull the image from [Dockerhub](https://hub.docker.com/r/intelanalytics/bigdl-ppml-azure-occlum): 27 | 28 | ```bash 29 | docker pull intelanalytics/bigdl-ppml-azure-occlum:2.1.0 30 | ``` 31 | 32 | Or you can clone this repository and build image with `build-docker-image.sh`. Configure environment variables in `build-docker-image.sh`. 33 | 34 | Build the docker image: 35 | 36 | ```bash 37 | bash build-docker-image.sh 38 | ``` 39 | 40 | ## Single Node Spark Examples on Azure 41 | 42 | Single node Spark Examples require 1 Azure VM with SGX. All examples are running in SGX. You can apply it to your application with a few changes in dockerfile or scripts. 43 | 44 | ### SparkPi example 45 | 46 | Run the SparkPi example with `run_spark_on_occlum_glibc.sh`. 47 | 48 | ```bash 49 | docker run --rm -it \ 50 | --name=azure-ppml-example-with-occlum \ 51 | --device=/dev/sgx/enclave \ 52 | --device=/dev/sgx/provision \ 53 | intelanalytics/bigdl-ppml-azure-occlum:2.1.0 bash 54 | 55 | cd /opt 56 | bash run_spark_on_occlum_glibc.sh pi 57 | ``` 58 | 59 | ### Nytaxi example with Azure NYTaxi 60 | 61 | Run the Nytaxi example with `run_azure_nytaxi.sh`. 62 | 63 | ```bash 64 | docker run --rm -it \ 65 | --name=azure-ppml-example-with-occlum \ 66 | --device=/dev/sgx/enclave \ 67 | --device=/dev/sgx/provision \ 68 | intelanalytics/bigdl-ppml-azure-occlum:2.1.0 bash 69 | 70 | bash run_azure_nytaxi.sh 71 | ``` 72 | 73 | You should get Nytaxi dataframe count and aggregation duration when succeed. 74 | 75 | ## Distributed Spark Examples on AKS 76 | 77 | ### SparkPi on AKS 78 | 79 | Configure environment variables in `run_spark_pi.sh`, `driver.yaml` and `executor.yaml`. Then you can submit SparkPi task with `run_spark_pi.sh`. 80 | 81 | ```bash 82 | bash run_spark_pi.sh 83 | ``` 84 | 85 | ### Nytaxi on AKS 86 | 87 | Configure environment variables in `run_nytaxi_k8s.sh`, `driver.yaml` and `executor.yaml`. Then you can submit Nytaxi query task with `run_nytaxi_k8s.sh`. 88 | ```bash 89 | bash run_nytaxi_k8s.sh 90 | ``` 91 | 92 | ## Known issues 93 | 94 | 1. If you meet the following error when running the docker image: 95 | 96 | ```bash 97 | aesm_service[10]: Failed to set logging callback for the quote provider library. 98 | aesm_service[10]: The server sock is 0x5624fe742330 99 | ``` 100 | 101 | This may be associated with [SGX DCAP](https://github.com/intel/linux-sgx/issues/812). And it's expected error message if not all interfaces in quote provider library are valid, and will not cause a failure. 102 | 103 | 2. If you meet the following error when running MAA example: 104 | 105 | ```bash 106 | [get_platform_quote_cert_data ../qe_logic.cpp:352] p_sgx_get_quote_config returned NULL for p_pck_cert_config. 107 | thread 'main' panicked at 'IOCTRL IOCTL_GET_DCAP_QUOTE_SIZE failed', /opt/src/occlum/tools/toolchains/dcap_lib/src/occlum_dcap.rs:70:13 108 | note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace 109 | [ERROR] occlum-pal: The init process exit with code: 101 (line 62, file src/pal_api.c) 110 | [ERROR] occlum-pal: Failed to run the init process: EINVAL (line 150, file src/pal_api.c) 111 | [ERROR] occlum-pal: Failed to do ECall: occlum_ecall_broadcast_interrupts with error code 0x2002: Invalid enclave identification. (line 26, file src/pal_interrupt_thread.c) 112 | /opt/occlum/build/bin/occlum: line 337: 3004 Segmentation fault (core dumped) RUST_BACKTRACE=1 "$instance_dir/build/bin/occlum-run" "$@" 113 | ``` 114 | 115 | This may be associated with [[RFC] IOCTRL IOCTL_GET_DCAP_QUOTE_SIZE failed](https://github.com/occlum/occlum/issues/899). 116 | 117 | ## Reference 118 | 119 | 1. https://www.intel.com/content/www/us/en/developer/tools/software-guard-extensions/overview.html 120 | 2. https://github.com/intel-analytics/BigDL 121 | 3. https://github.com/occlum/occlum 122 | 4. [Confidential Data Analytics with Apache Spark on Intel SGX Confidential Containers](https://github.com/Azure-Samples/confidential-container-samples/blob/main/confidential-big-data-spark/README.md) 123 | -------------------------------------------------------------------------------- /docker/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | 4 | # Check whether there is a passwd entry for the container UID 5 | myuid=$(id -u) 6 | mygid=$(id -g) 7 | # turn off -e for getent because it will return error code in anonymous uid case 8 | set +e 9 | uidentry=$(getent passwd $myuid) 10 | set -e 11 | 12 | # If there is no passwd entry for the container UID, attempt to create one 13 | if [ -z "$uidentry" ] ; then 14 | if [ -w /etc/passwd ] ; then 15 | echo "$myuid:x:$myuid:$mygid:anonymous uid:$SPARK_HOME:/bin/false" >> /etc/passwd 16 | else 17 | echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID" 18 | fi 19 | fi 20 | 21 | # check occlum log level for k8s 22 | if [[ -z "$ENABLE_SGX_DEBUG" ]]; then 23 | echo "No ENABLE_SGX_DEBUG specified, set to off." 24 | export ENABLE_SGX_DEBUG=false 25 | fi 26 | export OCCLUM_LOG_LEVEL=off 27 | if [[ -z "$SGX_LOG_LEVEL" ]]; then 28 | echo "No SGX_LOG_LEVEL specified, set to off." 29 | else 30 | echo "Set SGX_LOG_LEVEL to $SGX_LOG_LEVEL" 31 | if [[ $SGX_LOG_LEVEL == "debug" ]] || [[ $SGX_LOG_LEVEL == "trace" ]]; then 32 | export ENABLE_SGX_DEBUG=true 33 | export OCCLUM_LOG_LEVEL=$SGX_LOG_LEVEL 34 | fi 35 | fi 36 | 37 | # check the NETTY_THREAD 38 | if [[ -z "$NETTY_THREAD" ]]; then 39 | echo "NETTY_THREAD not set, using default value 16" 40 | NETTY_THREAD=16 41 | fi 42 | 43 | SPARK_K8S_CMD="$1" 44 | case "$SPARK_K8S_CMD" in 45 | driver | executor) 46 | shift 1 47 | ;; 48 | "") 49 | ;; 50 | *) 51 | echo "Non-spark-on-k8s command provided, proceeding in pass-through mode..." 52 | exec /sbin/tini -s -- "$@" 53 | ;; 54 | esac 55 | 56 | SPARK_CLASSPATH="$SPARK_CLASSPATH:/opt/spark/jars/*:/bin/jars/*" 57 | env | grep SPARK_JAVA_OPT_ | sort -t_ -k4 -n | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt 58 | readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt 59 | 60 | if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then 61 | SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH" 62 | fi 63 | 64 | if [[ -z "$META_SPACE" ]]; then 65 | echo "META_SPACE not set, using default value 256m" 66 | META_SPACE=256m 67 | else 68 | echo "META_SPACE=$META_SPACE" 69 | fi 70 | 71 | echo "SGX_LOG_LEVEL $SGX_LOG_LEVEL" && \ 72 | echo "SGX_DRIVER_JVM_MEM_SIZE $SGX_DRIVER_JVM_MEM_SIZE" && \ 73 | echo "SGX_EXECUTOR_JVM_MEM_SIZE $SGX_EXECUTOR_JVM_MEM_SIZE" && \ 74 | echo "SPARK_DRIVER_MEMORY $DRIVER_MEMORY" && \ 75 | echo "SPARK_EXECUTOR_MEMORY $SPARK_EXECUTOR_MEMORY" && \ 76 | 77 | cd /opt 78 | bash mount.sh 79 | 80 | case "$SPARK_K8S_CMD" in 81 | driver) 82 | echo "SGX Mem $SGX_MEM_SIZE" 83 | if [[ -z "$DRIVER_MEMORY" ]]; then 84 | echo "DRIVER_MEMORY not set, using default value 10g" 85 | DRIVER_MEMORY=10g 86 | else 87 | echo "DRIVER_MEMORY=$DRIVER_MEMORY" 88 | fi 89 | #/opt/run_spark_on_occlum_glibc.sh init 90 | cd /opt/occlum_spark 91 | DMLC_TRACKER_URI=$SPARK_DRIVER_BIND_ADDRESS 92 | 93 | if [[ -z "$SGX_DRIVER_JVM_MEM_SIZE" ]]; then 94 | echo "SGX_DRIVER_JVM_MEM_SIZE not set, using default DRIVER_MEMORY" 95 | CMD=( 96 | /usr/lib/jvm/java-8-openjdk-amd64/bin/java \ 97 | -Divy.home="/tmp/.ivy" \ 98 | -Dos.name="Linux" \ 99 | -XX:-UseCompressedOops \ 100 | -XX:MaxMetaspaceSize=$META_SPACE \ 101 | -Djdk.lang.Process.launchMechanism=posix_spawn \ 102 | -cp "$SPARK_CLASSPATH" \ 103 | -Xmx$DRIVER_MEMORY \ 104 | -XX:ActiveProcessorCount=4 \ 105 | -Dio.netty.availableProcessors=$NETTY_THREAD \ 106 | org.apache.spark.deploy.SparkSubmit \ 107 | --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS" \ 108 | --deploy-mode client \ 109 | "$@" 110 | ) 111 | else 112 | echo "use SGX_DRIVER_JVM_MEM_SIZE=$SGX_DRIVER_JVM_MEM_SIZE" 113 | CMD=( 114 | /usr/lib/jvm/java-8-openjdk-amd64/bin/java \ 115 | -Divy.home="/tmp/.ivy" \ 116 | -Dos.name="Linux" \ 117 | -XX:-UseCompressedOops \ 118 | -XX:MaxMetaspaceSize=$META_SPACE \ 119 | -Djdk.lang.Process.launchMechanism=posix_spawn \ 120 | -cp "$SPARK_CLASSPATH" \ 121 | -Xmx$SGX_DRIVER_JVM_MEM_SIZE \ 122 | -XX:ActiveProcessorCount=4 \ 123 | -Dio.netty.availableProcessors=$NETTY_THREAD \ 124 | org.apache.spark.deploy.SparkSubmit \ 125 | --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS" \ 126 | --deploy-mode client \ 127 | "$@" 128 | ) 129 | fi 130 | ;; 131 | executor) 132 | echo "SGX Mem $SGX_MEM_SIZE" 133 | #/opt/run_spark_on_occlum_glibc.sh init 134 | cd /opt/occlum_spark 135 | DMLC_TRACKER_URI=$SPARK_DRIVER_BIND_ADDRESS 136 | 137 | if [[ -z "$SGX_EXECUTOR_JVM_MEM_SIZE" ]]; then 138 | echo "SGX_EXECUTOR_JVM_MEM_SIZE not set, using default EXCUTOR_MEMORY" 139 | CMD=( 140 | /usr/lib/jvm/java-8-openjdk-amd64/bin/java \ 141 | "${SPARK_EXECUTOR_JAVA_OPTS[@]}" \ 142 | -XX:-UseCompressedOops \ 143 | -XX:MaxMetaspaceSize=$META_SPACE \ 144 | -XX:ActiveProcessorCount=$SPARK_EXECUTOR_CORES \ 145 | -Divy.home=/tmp/.ivy \ 146 | -Xms$SPARK_EXECUTOR_MEMORY \ 147 | -Xmx$SPARK_EXECUTOR_MEMORY \ 148 | -Dos.name=Linux \ 149 | -Dio.netty.availableProcessors=$NETTY_THREAD \ 150 | -Djdk.lang.Process.launchMechanism=posix_spawn \ 151 | -cp "$SPARK_CLASSPATH" \ 152 | org.apache.spark.executor.CoarseGrainedExecutorBackend \ 153 | --driver-url $SPARK_DRIVER_URL \ 154 | --executor-id $SPARK_EXECUTOR_ID \ 155 | --cores $SPARK_EXECUTOR_CORES \ 156 | --app-id $SPARK_APPLICATION_ID \ 157 | --hostname $SPARK_EXECUTOR_POD_IP 158 | ) 159 | else 160 | echo "use SGX_EXECUTOR_JVM_MEM_SIZE=$SGX_EXECUTOR_JVM_MEM_SIZE" 161 | CMD=( 162 | /usr/lib/jvm/java-8-openjdk-amd64/bin/java \ 163 | "${SPARK_EXECUTOR_JAVA_OPTS[@]}" \ 164 | -XX:-UseCompressedOops \ 165 | -XX:MaxMetaspaceSize=$META_SPACE \ 166 | -XX:ActiveProcessorCount=$SPARK_EXECUTOR_CORES \ 167 | -Divy.home=/tmp/.ivy \ 168 | -Xms$SGX_EXECUTOR_JVM_MEM_SIZE \ 169 | -Xmx$SGX_EXECUTOR_JVM_MEM_SIZE \ 170 | -Dos.name=Linux \ 171 | -Dio.netty.availableProcessors=$NETTY_THREAD \ 172 | -Djdk.lang.Process.launchMechanism=posix_spawn \ 173 | -cp "$SPARK_CLASSPATH" \ 174 | org.apache.spark.executor.CoarseGrainedExecutorBackend \ 175 | --driver-url $SPARK_DRIVER_URL \ 176 | --executor-id $SPARK_EXECUTOR_ID \ 177 | --cores $SPARK_EXECUTOR_CORES \ 178 | --app-id $SPARK_APPLICATION_ID \ 179 | --hostname $SPARK_EXECUTOR_POD_IP 180 | ) 181 | fi 182 | ;; 183 | *) 184 | echo "Unknown command: $SPARK_K8S_CMD" 1>&2 185 | exit 1 186 | esac 187 | 188 | /sbin/tini -s -- occlum run "${CMD[@]}" 189 | -------------------------------------------------------------------------------- /docker/run_spark_on_occlum_glibc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -x 3 | 4 | BLUE='\033[1;34m' 5 | NC='\033[0m' 6 | occlum_glibc=/opt/occlum/glibc/lib 7 | # occlum-node IP 8 | HOST_IP=`cat /etc/hosts | grep $HOSTNAME | awk '{print $1}'` 9 | INSTANCE_DIR="/opt/occlum_spark" 10 | # INSTANCE_DIR="/opt/src/occlum/demos/remote_attestation/azure_attestation/maa_init/occlum_instance" 11 | INIT_DIR="/opt/src/occlum/demos/remote_attestation/azure_attestation/maa_init/init" 12 | IMG_BOM="/opt/src/occlum/demos/remote_attestation/azure_attestation/maa_init/bom.yaml" 13 | INIT_BOM="/opt/src/occlum/demos/remote_attestation/azure_attestation/maa_init/init_maa.yaml" 14 | 15 | check_sgx_dev() { 16 | if [ -c "/dev/sgx/enclave" ]; then 17 | echo "/dev/sgx/enclave is ready" 18 | elif [ -c "/dev/sgx_enclave" ]; then 19 | echo "/dev/sgx/enclave not ready, try to link to /dev/sgx_enclave" 20 | mkdir -p /dev/sgx 21 | ln -s /dev/sgx_enclave /dev/sgx/enclave 22 | else 23 | echo "both /dev/sgx/enclave /dev/sgx_enclave are not ready, please check the kernel and driver" 24 | fi 25 | 26 | if [ -c "/dev/sgx/provision" ]; then 27 | echo "/dev/sgx/provision is ready" 28 | elif [ -c "/dev/sgx_provision" ]; then 29 | echo "/dev/sgx/provision not ready, try to link to /dev/sgx_provision" 30 | mkdir -p /dev/sgx 31 | ln -s /dev/sgx_provision /dev/sgx/provision 32 | else 33 | echo "both /dev/sgx/provision /dev/sgx_provision are not ready, please check the kernel and driver" 34 | fi 35 | 36 | ls -al /dev/sgx 37 | } 38 | 39 | init_instance() { 40 | # check and fix sgx device 41 | check_sgx_dev 42 | # Init Occlum instance 43 | cd /opt 44 | # check if occlum_spark exists 45 | [[ -d occlum_spark ]] || mkdir occlum_spark 46 | cd ${INSTANCE_DIR} 47 | occlum init 48 | new_json="$(jq '.resource_limits.user_space_size = "SGX_MEM_SIZE" | 49 | .resource_limits.max_num_of_threads = "SGX_THREAD" | 50 | .process.default_heap_size = "SGX_HEAP" | 51 | .metadata.debuggable = "ENABLE_SGX_DEBUG" | 52 | .resource_limits.kernel_space_heap_size="SGX_KERNEL_HEAP" | 53 | .entry_points = [ "/bin", "/usr/lib/jvm/java-8-openjdk-amd64/bin" ] | 54 | .env.untrusted = [ "DMLC_TRACKER_URI", "SPARK_DRIVER_URL", "SPARK_TESTING" ] | 55 | .env.default = [ "LD_LIBRARY_PATH=/usr/lib/jvm/java-8-openjdk-amd64/lib/server:/usr/lib/jvm/java-8-openjdk-amd64/lib:/usr/lib/jvm/java-8-openjdk-amd64/../lib:/lib","SPARK_CONF_DIR=/opt/spark/conf","SPARK_ENV_LOADED=1","PYTHONHASHSEED=0","SPARK_HOME=/opt/spark","SPARK_SCALA_VERSION=2.12","SPARK_JARS_DIR=/opt/spark/jars","LAUNCH_CLASSPATH=/bin/jars/*","MAA_REPORT_DATA=BASE64_STRING","MAA_PROVIDER_URL=https://shareduks.uks.attest.azure.net","MAA_TOKEN_PATH=/root","OCCLUM=yes","AZURE_BLOB_ACCOUNT_NAME=azureopendatastorage","AZURE_BLOB_CONTAINER_NAME=nyctlc","AZURE_BLOB_SAS_TOKEN=r","AZURE_BLOB_RELATIVE_PATH=yellow","AZURE_SQL_AE_JDBC="]' Occlum.json)" && \ 56 | echo "${new_json}" > Occlum.json 57 | echo "SGX_MEM_SIZE ${SGX_MEM_SIZE}" 58 | 59 | bash add_conf.sh 60 | cat Occlum.json 61 | 62 | if [[ -z "$META_SPACE" ]]; then 63 | echo "META_SPACE not set, using default value 256m" 64 | META_SPACE=256m 65 | else 66 | echo "META_SPACE=$META_SPACE" 67 | fi 68 | 69 | if [[ -z "$SGX_MEM_SIZE" ]]; then 70 | sed -i "s/SGX_MEM_SIZE/20GB/g" Occlum.json 71 | else 72 | sed -i "s/SGX_MEM_SIZE/${SGX_MEM_SIZE}/g" Occlum.json 73 | fi 74 | 75 | if [[ -z "$SGX_THREAD" ]]; then 76 | sed -i "s/\"SGX_THREAD\"/512/g" Occlum.json 77 | else 78 | sed -i "s/\"SGX_THREAD\"/${SGX_THREAD}/g" Occlum.json 79 | fi 80 | 81 | if [[ -z "$SGX_HEAP" ]]; then 82 | sed -i "s/SGX_HEAP/512MB/g" Occlum.json 83 | else 84 | sed -i "s/SGX_HEAP/${SGX_HEAP}/g" Occlum.json 85 | fi 86 | 87 | if [[ -z "$SGX_KERNEL_HEAP" ]]; then 88 | sed -i "s/SGX_KERNEL_HEAP/1GB/g" Occlum.json 89 | else 90 | sed -i "s/SGX_KERNEL_HEAP/${SGX_KERNEL_HEAP}/g" Occlum.json 91 | fi 92 | 93 | sed -i "s/\"ENABLE_SGX_DEBUG\"/$ENABLE_SGX_DEBUG/g" Occlum.json 94 | sed -i "s/#USE_SECURE_CERT=FALSE/USE_SECURE_CERT=FALSE/g" /etc/sgx_default_qcnl.conf 95 | 96 | openssl genrsa -out key.pem 2048 97 | report_data=$(base64 -w 0 key.pem) 98 | sed -i "s/BASE64_STRING/$report_data/g" Occlum.json 99 | 100 | if [[ $FS_TYPE == "ramfs" ]]; then 101 | edit_json="$(cat Occlum.json | jq '.mount+=[{"target": "/tmp","type": "ramfs"}]')" && \ 102 | echo "${edit_json}" > Occlum.json 103 | elif [[ $FS_TYPE == "hostfs" ]]; then 104 | mkdir ./shuffle 105 | edit_json="$(cat Occlum.json | jq '.mount+=[{"target": "/tmp","type": "hostfs","source":"./shuffle"}]')" && \ 106 | echo "${edit_json}" > Occlum.json 107 | fi 108 | } 109 | 110 | build_spark() { 111 | cd ${INSTANCE_DIR} 112 | 113 | copy_bom -f ${IMG_BOM} --root image --include-dir /opt/occlum/etc/template 114 | copy_bom -f ${INIT_BOM} --root initfs --include-dir /opt/occlum/etc/template 115 | # Copy JVM and class file into Occlum instance and build 116 | mkdir -p image/usr/lib/jvm 117 | cp -r /usr/lib/jvm/java-8-openjdk-amd64 image/usr/lib/jvm 118 | cp -rf /etc/java-8-openjdk image/etc/ 119 | # Copy K8s secret 120 | mkdir -p image/var/run/secrets/ 121 | cp -r /var/run/secrets/* image/var/run/secrets/ 122 | ls image/var/run/secrets/kubernetes.io/serviceaccount/ 123 | # Copy libs 124 | cp /lib/x86_64-linux-gnu/libz.so.1 image/lib 125 | cp /lib/x86_64-linux-gnu/libz.so.1 image/$occlum_glibc 126 | cp /lib/x86_64-linux-gnu/libtinfo.so.5 image/$occlum_glibc 127 | cp /lib/x86_64-linux-gnu/libnss*.so.2 image/$occlum_glibc 128 | cp /lib/x86_64-linux-gnu/libresolv.so.2 image/$occlum_glibc 129 | cp $occlum_glibc/libdl.so.2 image/$occlum_glibc 130 | cp $occlum_glibc/librt.so.1 image/$occlum_glibc 131 | cp $occlum_glibc/libm.so.6 image/$occlum_glibc 132 | # Copy libhadoop 133 | cp /opt/libhadoop.so image/lib 134 | # Prepare Spark 135 | mkdir -p image/opt/spark 136 | cp -rf $SPARK_HOME/* image/opt/spark/ 137 | # Remove spark's jackson libraries and use jackson libraries in BigDL instead 138 | rm -f image/opt/spark/jars/jackson-annotations* \ 139 | image/opt/spark/jars/jackson-core-2.* \ 140 | image/opt/spark/jars/jackson-databind* \ 141 | image/opt/spark/jars/jackson-module-scala* \ 142 | image/opt/spark/jars/jackson-datatype-jsr310* 143 | # Copy etc files 144 | cp -rf /etc/hosts image/etc/ 145 | echo "$HOST_IP occlum-node" >> image/etc/hosts 146 | # cat image/etc/hosts 147 | 148 | cp -rf /etc/hostname image/etc/ 149 | cp -rf /etc/ssl image/etc/ 150 | cp -rf /etc/passwd image/etc/ 151 | cp -rf /etc/group image/etc/ 152 | cp -rf /etc/nsswitch.conf image/etc/ 153 | 154 | # Prepare BigDL 155 | mkdir -p image/bin/jars 156 | cp -f $BIGDL_HOME/jars/* image/bin/jars 157 | 158 | cd ${INSTANCE_DIR} 159 | cp -f $INIT_DIR/target/release/init initfs/bin 160 | cp -rf image/lib64 initfs/lib64 161 | cp -rf image/$occlum_glibc initfs/opt/occlum/glibc 162 | cp -f /lib/x86_64-linux-gnu/libcrypto.so.1.1 initfs/$occlum_glibc/libcrypto.so.1.1 163 | cp -f /lib/x86_64-linux-gnu/libgcc_s.so.1 initfs/$occlum_glibc/libgcc_s.so.1 164 | cp -f /lib/x86_64-linux-gnu/libssl.so.1.1 initfs/$occlum_glibc/libssl.so.1.1 165 | 166 | # Build 167 | occlum build 168 | 169 | } 170 | 171 | run_maa_example() { 172 | init_instance spark 173 | build_spark 174 | 175 | echo -e "${BLUE}occlum run MAA example${NC}" 176 | occlum run /bin/busybox cat /root/token 177 | } 178 | 179 | run_spark_pi() { 180 | # init_instance spark 181 | ##build_spark 182 | echo -e "${BLUE}occlum run spark Pi${NC}" 183 | 184 | if [[ -z "$META_SPACE" ]]; then 185 | echo "META_SPACE not set, using default value 256m" 186 | META_SPACE=256m 187 | else 188 | echo "META_SPACE=$META_SPACE" 189 | fi 190 | 191 | cd /opt/occlum_spark 192 | bash /opt/mount.sh 193 | occlum run /usr/lib/jvm/java-8-openjdk-amd64/bin/java \ 194 | -XX:-UseCompressedOops -XX:MaxMetaspaceSize=$META_SPACE \ 195 | -XX:ActiveProcessorCount=4 \ 196 | -Divy.home="/tmp/.ivy" \ 197 | -Dos.name="Linux" \ 198 | -cp "$SPARK_HOME/conf/:$SPARK_HOME/jars/*:/bin/jars/*" \ 199 | -Xmx512m org.apache.spark.deploy.SparkSubmit \ 200 | --jars $SPARK_HOME/examples/jars/spark-examples_2.12-3.1.2.jar,$SPARK_HOME/examples/jars/scopt_2.12-3.7.1.jar \ 201 | --class org.apache.spark.examples.SparkPi spark-internal 202 | } 203 | 204 | run_spark_unittest() { 205 | init_instance spark 206 | build_spark 207 | echo -e "${BLUE}occlum run spark unit test ${NC}" 208 | run_spark_unittest_only 209 | } 210 | 211 | run_spark_unittest_only() { 212 | export SPARK_TESTING=1 213 | cd ${INSTANCE_DIR} 214 | mkdir -p data/olog 215 | echo -e "${BLUE}occlum run spark unit test only ${NC}" 216 | occlum start 217 | for suite in `cat /opt/sqlSuites` 218 | do occlum exec /usr/lib/jvm/java-8-openjdk-amd64/bin/java -Xmx24g \ 219 | -Divy.home="/tmp/.ivy" \ 220 | -Dos.name="Linux" \ 221 | -Djdk.lang.Process.launchMechanism=posix_spawn \ 222 | -XX:MaxMetaspaceSize=$META_SPACE \ 223 | -Dspark.testing=true \ 224 | -Dspark.test.home=/opt/spark-source \ 225 | -Dspark.python.use.daemon=false \ 226 | -Dspark.python.worker.reuse=false \ 227 | -Dspark.driver.host=127.0.0.1 \ 228 | -cp "$SPARK_HOME/conf/:$SPARK_HOME/jars/*:$SPARK_HOME/test-jars/*:$SPARK_HOME/test-classes/" \ 229 | org.scalatest.tools.Runner \ 230 | -s ${suite} \ 231 | -fF /host/data/olog/${suite}.txt 232 | done 233 | #-Dspark.sql.warehouse.dir=hdfs://localhost:9000/111-spark-warehouse \ 234 | occlum stop 235 | } 236 | 237 | run_spark_lenet_mnist(){ 238 | init_instance spark 239 | build_spark 240 | echo -e "${BLUE}occlum run BigDL lenet mnist{NC}" 241 | echo -e "${BLUE}logfile=$log${NC}" 242 | occlum run /usr/lib/jvm/java-8-openjdk-amd64/bin/java \ 243 | -XX:-UseCompressedOops -XX:MaxMetaspaceSize=256m \ 244 | -XX:ActiveProcessorCount=4 \ 245 | -Divy.home="/tmp/.ivy" \ 246 | -Dos.name="Linux" \ 247 | -cp "$SPARK_HOME/conf/:$SPARK_HOME/jars/*:/bin/jars/*" \ 248 | -Xmx10g org.apache.spark.deploy.SparkSubmit \ 249 | --master 'local[4]' \ 250 | --conf spark.driver.port=10027 \ 251 | --conf spark.scheduler.maxRegisteredResourcesWaitingTime=5000000 \ 252 | --conf spark.worker.timeout=600 \ 253 | --conf spark.starvation.timeout=250000 \ 254 | --conf spark.rpc.askTimeout=600 \ 255 | --conf spark.blockManager.port=10025 \ 256 | --conf spark.driver.host=127.0.0.1 \ 257 | --conf spark.driver.blockManager.port=10026 \ 258 | --conf spark.io.compression.codec=lz4 \ 259 | --class com.intel.analytics.bigdl.dllib.models.lenet.Train \ 260 | --driver-memory 10G \ 261 | /bin/jars/bigdl-dllib-spark_${SPARK_VERSION}-${BIGDL_VERSION}.jar \ 262 | -f /host/data \ 263 | $* | tee spark.local.sgx.log 264 | } 265 | 266 | run_spark_resnet_cifar(){ 267 | init_instance spark 268 | build_spark 269 | echo -e "${BLUE}occlum run BigDL Resnet Cifar10${NC}" 270 | occlum run /usr/lib/jvm/java-8-openjdk-amd64/bin/java \ 271 | -XX:-UseCompressedOops -XX:MaxMetaspaceSize=$META_SPACE \ 272 | -XX:ActiveProcessorCount=4 \ 273 | -Divy.home="/tmp/.ivy" \ 274 | -Dos.name="Linux" \ 275 | -cp "$SPARK_HOME/conf/:$SPARK_HOME/jars/*:/bin/jars/*" \ 276 | -Xmx10g org.apache.spark.deploy.SparkSubmit \ 277 | --master 'local[4]' \ 278 | --conf spark.driver.port=10027 \ 279 | --conf spark.scheduler.maxRegisteredResourcesWaitingTime=5000000 \ 280 | --conf spark.worker.timeout=600 \ 281 | --conf spark.starvation.timeout=250000 \ 282 | --conf spark.rpc.askTimeout=600 \ 283 | --conf spark.blockManager.port=10025 \ 284 | --conf spark.driver.host=127.0.0.1 \ 285 | --conf spark.driver.blockManager.port=10026 \ 286 | --conf spark.io.compression.codec=lz4 \ 287 | --class com.intel.analytics.bigdl.dllib.models.resnet.TrainCIFAR10 \ 288 | --driver-memory 10G \ 289 | /bin/jars/bigdl-dllib-spark_${SPARK_VERSION}-${BIGDL_VERSION}.jar \ 290 | -f /host/data \ 291 | $* | tee spark.local.sgx.log 292 | } 293 | 294 | run_spark_tpch(){ 295 | init_instance spark 296 | build_spark 297 | echo -e "${BLUE}occlum run BigDL spark tpch${NC}" 298 | occlum run /usr/lib/jvm/java-8-openjdk-amd64/bin/java \ 299 | -XX:-UseCompressedOops -XX:MaxMetaspaceSize=$META_SPACE \ 300 | -XX:ActiveProcessorCount=4 \ 301 | -Divy.home="/tmp/.ivy" \ 302 | -Dos.name="Linux" \ 303 | -cp "$SPARK_HOME/conf/:$SPARK_HOME/jars/*:/bin/jars/*" \ 304 | -Xmx78g -Xms78g \ 305 | org.apache.spark.deploy.SparkSubmit \ 306 | --master 'local[4]' \ 307 | --conf spark.driver.port=54321 \ 308 | --conf spark.driver.memory=12g \ 309 | --conf spark.driver.blockManager.port=10026 \ 310 | --conf spark.blockManager.port=10025 \ 311 | --conf spark.scheduler.maxRegisteredResourcesWaitingTime=5000000 \ 312 | --conf spark.worker.timeout=600 \ 313 | --conf spark.python.use.daemon=false \ 314 | --conf spark.python.worker.reuse=false \ 315 | --conf spark.network.timeout=10000000 \ 316 | --conf spark.starvation.timeout=250000 \ 317 | --conf spark.rpc.askTimeout=600 \ 318 | --conf spark.sql.autoBroadcastJoinThreshold=-1 \ 319 | --conf spark.io.compression.codec=lz4 \ 320 | --conf spark.sql.shuffle.partitions=8 \ 321 | --conf spark.speculation=false \ 322 | --conf spark.executor.heartbeatInterval=10000000 \ 323 | --conf spark.executor.instances=8 \ 324 | --executor-cores 2 \ 325 | --total-executor-cores 16 \ 326 | --executor-memory 8G \ 327 | --class main.scala.TpchQuery \ 328 | --verbose \ 329 | /bin/jars/spark-tpc-h-queries_2.12-1.0.jar \ 330 | /host/data /host/data/output 331 | } 332 | 333 | run_spark_xgboost() { 334 | init_instance spark 335 | build_spark 336 | echo -e "${BLUE}occlum run BigDL Spark XGBoost${NC}" 337 | occlum run /usr/lib/jvm/java-8-openjdk-amd64/bin/java \ 338 | -XX:-UseCompressedOops -XX:MaxMetaspaceSize=$META_SPACE \ 339 | -XX:ActiveProcessorCount=8 \ 340 | -Divy.home="/tmp/.ivy" \ 341 | -Dos.name="Linux" \ 342 | -cp "$SPARK_HOME/conf/:$SPARK_HOME/jars/*:/bin/jars/*" \ 343 | -Xmx18g -Xms18g org.apache.spark.deploy.SparkSubmit \ 344 | --master local[16] \ 345 | --conf spark.task.cpus=8 \ 346 | --class com.intel.analytics.bigdl.dllib.examples.nnframes.xgboost.xgbClassifierTrainingExampleOnCriteoClickLogsDataset \ 347 | --num-executors 2 \ 348 | --executor-cores 2 \ 349 | --executor-memory 9G \ 350 | --driver-memory 2G \ 351 | /bin/jars/bigdl-dllib-spark_3.1.2-2.1.0-SNAPSHOT.jar \ 352 | /host/data /host/data/model 2 100 2 353 | } 354 | 355 | 356 | id=$([ -f "$pid" ] && echo $(wc -l < "$pid") || echo "0") 357 | 358 | arg=$1 359 | case "$arg" in 360 | init) 361 | init_instance 362 | build_spark 363 | ;; 364 | pi) 365 | run_spark_pi 366 | cd ../ 367 | ;; 368 | lenet) 369 | run_spark_lenet_mnist 370 | cd ../ 371 | ;; 372 | ut) 373 | run_spark_unittest 374 | cd ../ 375 | ;; 376 | ut_Only) 377 | run_spark_unittest_only 378 | cd ../ 379 | ;; 380 | resnet) 381 | run_spark_resnet_cifar 382 | cd ../ 383 | ;; 384 | tpch) 385 | run_spark_tpch 386 | cd ../ 387 | ;; 388 | xgboost) 389 | run_spark_xgboost 390 | cd ../ 391 | ;; 392 | maa) 393 | run_maa_example 394 | cd ../ 395 | ;; 396 | esac 397 | --------------------------------------------------------------------------------