├── .gitignore ├── app ├── dcr_tee │ ├── .gitignore │ ├── attestation │ │ ├── go.mod │ │ ├── go.sum │ │ └── main.go │ ├── README.md │ ├── Dockerfile │ ├── build.sh │ └── encryption │ │ ├── main.go │ │ └── go.mod ├── jupyterlab_manatee │ ├── ui-tests │ │ ├── yarn.lock │ │ ├── playwright.config.js │ │ ├── package.json │ │ ├── jupyter_server_test_config.py │ │ ├── tests │ │ │ └── jupyterlab_manatee.spec.ts │ │ └── README.md │ ├── style │ │ ├── index.js │ │ ├── base.css │ │ └── index.css │ ├── babel.config.js │ ├── .yarnrc.yml │ ├── tsconfig.test.json │ ├── jupyter-config │ │ └── jupyter_server_config.d │ │ │ └── jupyterlab_manatee.json │ ├── install.json │ ├── setup.py │ ├── tsconfig.json │ ├── Dockerfile │ ├── jest.config.js │ ├── src │ │ ├── __tests__ │ │ │ └── jupyterlab_manatee.spec.ts │ │ ├── sidebar.ts │ │ ├── index.ts │ │ └── sources.ts │ ├── build.sh │ ├── LICENSE │ ├── jupyterlab_manatee │ │ └── __init__.py │ ├── .dockerignore │ ├── .gitignore │ ├── pyproject.toml │ ├── README.md │ ├── RELEASE.md │ └── package.json ├── dcr_monitor │ ├── .gitignore │ ├── Dockerfile │ ├── main.go │ ├── client │ │ ├── hertz.go │ │ └── k8s.go │ ├── build.sh │ ├── monitor │ │ └── instance.go │ └── go.mod ├── dcr_api │ ├── .hz │ ├── script │ │ └── bootstrap.sh │ ├── router_gen.go │ ├── router.go │ ├── biz │ │ ├── router │ │ │ ├── register.go │ │ │ └── job │ │ │ │ ├── job.go │ │ │ │ └── middleware.go │ │ ├── service │ │ │ ├── build_service.go │ │ │ └── job_service.go │ │ ├── dal │ │ │ ├── init.go │ │ │ └── db │ │ │ │ ├── init.go │ │ │ │ └── job.go │ │ └── handler │ │ │ └── health.go │ ├── .gitignore │ ├── main.go │ ├── Dockerfile │ ├── build.sh │ ├── idl │ │ └── job.thrift │ └── go.mod ├── conf │ ├── config.yaml │ └── Dockerfile └── build.sh ├── two-stage.png ├── resources ├── .gitignore ├── gcp │ ├── providers.tf │ ├── keys.tf │ ├── buckets.tf │ ├── identitypool.tf │ ├── .terraform.lock.hcl │ ├── repositories.tf │ ├── service_accounts.tf │ ├── variables.tf │ ├── database.tf │ ├── network.tf │ ├── cluster.tf │ └── iam.tf ├── kubernetes │ ├── providers.tf │ ├── secret.tf │ ├── namespace.tf │ ├── role.tf │ ├── variables.tf │ ├── .terraform.lock.hcl │ └── service_accounts.tf └── apply.sh ├── .env.singleuser ├── .env.multiusers ├── deployment ├── data-clean-room │ ├── config.yaml │ ├── .helmignore │ ├── templates │ │ ├── service.yaml │ │ ├── tests │ │ │ └── test-connection.yaml │ │ ├── NOTES.txt │ │ ├── _helpers.tpl │ │ ├── ingress.yaml │ │ ├── cronjob.yaml │ │ └── deployment.yaml │ ├── Chart.yaml │ ├── deploy.sh │ └── values.yaml ├── deploy.sh └── jupyterhub │ ├── config.yaml │ └── deploy.sh ├── pkg ├── utils │ ├── k8s.go │ ├── constant.go │ ├── resp.go │ └── file.go ├── errno │ └── errno.go ├── cloud │ └── provider.go └── go.mod ├── CONTRIBUTING.md └── CODE_OF_CONDUCT.md /.gitignore: -------------------------------------------------------------------------------- 1 | .env -------------------------------------------------------------------------------- /app/dcr_tee/.gitignore: -------------------------------------------------------------------------------- 1 | conf 2 | github.com -------------------------------------------------------------------------------- /app/jupyterlab_manatee/ui-tests/yarn.lock: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/dcr_monitor/.gitignore: -------------------------------------------------------------------------------- 1 | conf 2 | github.com 3 | dcr_monitor -------------------------------------------------------------------------------- /app/jupyterlab_manatee/style/index.js: -------------------------------------------------------------------------------- 1 | import './base.css'; 2 | -------------------------------------------------------------------------------- /app/jupyterlab_manatee/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = require('@jupyterlab/testutils/lib/babel.config'); 2 | -------------------------------------------------------------------------------- /two-stage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/HEAD/two-stage.png -------------------------------------------------------------------------------- /resources/.gitignore: -------------------------------------------------------------------------------- 1 | .terraform/ 2 | terraform.tfstate 3 | terraform.tfstate.backup 4 | terraform.tfvars 5 | backend.tf -------------------------------------------------------------------------------- /.env.singleuser: -------------------------------------------------------------------------------- 1 | env="" 2 | mysql_username="" 3 | mysql_password="" 4 | project_id="" 5 | project_number="" 6 | region="" -------------------------------------------------------------------------------- /app/jupyterlab_manatee/.yarnrc.yml: -------------------------------------------------------------------------------- 1 | nodeLinker: node-modules 2 | 3 | npmRegistryServer: "https://registry.yarnpkg.com/" 4 | -------------------------------------------------------------------------------- /.env.multiusers: -------------------------------------------------------------------------------- 1 | env="" 2 | username="" 3 | mysql_username="" 4 | mysql_password="" 5 | project_id="" 6 | project_number="" 7 | region="" -------------------------------------------------------------------------------- /app/dcr_api/.hz: -------------------------------------------------------------------------------- 1 | // Code generated by hz. DO NOT EDIT. 2 | 3 | hz version: v0.9.0 4 | handlerDir: "" 5 | modelDir: biz/model 6 | routerDir: "" 7 | -------------------------------------------------------------------------------- /app/jupyterlab_manatee/tsconfig.test.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig", 3 | "compilerOptions": { 4 | "types": ["jest"] 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /app/dcr_api/script/bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | CURDIR=$(cd $(dirname $0); pwd) 3 | BinaryName=hertz_service 4 | echo "$CURDIR/bin/${BinaryName}" 5 | exec $CURDIR/bin/${BinaryName} -------------------------------------------------------------------------------- /app/jupyterlab_manatee/style/base.css: -------------------------------------------------------------------------------- 1 | /* 2 | See the JupyterLab Developer Guide for useful CSS Patterns: 3 | 4 | https://jupyterlab.readthedocs.io/en/stable/developer/css.html 5 | */ 6 | -------------------------------------------------------------------------------- /app/dcr_tee/attestation/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tiktok-privacy-innovation/PrivacyGoDataCleanRoom/app/dcr_tee/attestation 2 | 3 | go 1.22.0 4 | 5 | require github.com/pkg/errors v0.9.1 // indirect 6 | -------------------------------------------------------------------------------- /app/dcr_tee/attestation/go.sum: -------------------------------------------------------------------------------- 1 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 2 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 3 | -------------------------------------------------------------------------------- /app/jupyterlab_manatee/jupyter-config/jupyter_server_config.d/jupyterlab_manatee.json: -------------------------------------------------------------------------------- 1 | { 2 | "ServerApp": { 3 | "jpserver_extensions": { 4 | "jupyterlab_manatee": true 5 | } 6 | } 7 | } -------------------------------------------------------------------------------- /app/jupyterlab_manatee/install.json: -------------------------------------------------------------------------------- 1 | { 2 | "packageManager": "python", 3 | "packageName": "jupyterlab_manatee", 4 | "uninstallInstructions": "Use your Python package manager (pip, conda, etc.) to uninstall the package jupyterlab_manatee" 5 | } 6 | -------------------------------------------------------------------------------- /app/dcr_api/router_gen.go: -------------------------------------------------------------------------------- 1 | // Code generated by hertz generator. DO NOT EDIT. 2 | 3 | package main 4 | 5 | import ( 6 | "github.com/cloudwego/hertz/pkg/app/server" 7 | router "github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/app/dcr_api/biz/router" 8 | ) 9 | 10 | // register registers all routers. 11 | func register(r *server.Hertz) { 12 | 13 | router.GeneratedRegister(r) 14 | 15 | customizedRegister(r) 16 | } 17 | -------------------------------------------------------------------------------- /app/dcr_api/router.go: -------------------------------------------------------------------------------- 1 | // Code generated by hertz generator. 2 | 3 | package main 4 | 5 | import ( 6 | "github.com/cloudwego/hertz/pkg/app/server" 7 | handler "github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/app/dcr_api/biz/handler" 8 | ) 9 | 10 | // customizeRegister registers customize routers. 11 | func customizedRegister(r *server.Hertz) { 12 | r.GET("/health", handler.Health) 13 | 14 | // your code ... 15 | } 16 | -------------------------------------------------------------------------------- /app/dcr_tee/README.md: -------------------------------------------------------------------------------- 1 | # Data Clean Room TEE Base Image 2 | The base image is used to build user's custom image that is going to run in google cloud confidential space. This folder contains the tools that required by the base image including an encryption tool to encrypt the output `ipynb` file and an attestation tool to generate custom attestation report. 3 | 4 | ## Build 5 | Just run the `build.sh` to build the base image and push to gcp docker repository. -------------------------------------------------------------------------------- /app/jupyterlab_manatee/ui-tests/playwright.config.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Configuration for Playwright using default from @jupyterlab/galata 3 | */ 4 | const baseConfig = require('@jupyterlab/galata/lib/playwright-config'); 5 | 6 | module.exports = { 7 | ...baseConfig, 8 | webServer: { 9 | command: 'jlpm start', 10 | url: 'http://localhost:8888/lab', 11 | timeout: 120 * 1000, 12 | reuseExistingServer: !process.env.CI 13 | } 14 | }; 15 | -------------------------------------------------------------------------------- /deployment/data-clean-room/config.yaml: -------------------------------------------------------------------------------- 1 | cloudSql: 2 | connection_name: $connection_name 3 | serviceAccount: 4 | create: false 5 | name: $service_account 6 | apiImage: 7 | repository: $api_reference 8 | tag: $tag 9 | monitorImage: 10 | repository: $monitor_reference 11 | tag: $tag 12 | nodeSelector: { iam.gke.io/gke-metadata-server-enabled: "true"} 13 | mysql: 14 | host: localhost 15 | port: 9910 16 | namespace: "" -------------------------------------------------------------------------------- /app/dcr_api/biz/router/register.go: -------------------------------------------------------------------------------- 1 | // Code generated by hertz generator. DO NOT EDIT. 2 | 3 | package router 4 | 5 | import ( 6 | "github.com/cloudwego/hertz/pkg/app/server" 7 | job "github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/app/dcr_api/biz/router/job" 8 | ) 9 | 10 | // GeneratedRegister registers routers generated by IDL. 11 | func GeneratedRegister(r *server.Hertz) { 12 | //INSERT_POINT: DO NOT DELETE THIS LINE! 13 | job.Register(r) 14 | } 15 | -------------------------------------------------------------------------------- /app/jupyterlab_manatee/style/index.css: -------------------------------------------------------------------------------- 1 | @import url('base.css'); 2 | 3 | .jp-DCRSources { 4 | min-height: 50px; 5 | margin-top: 3px; 6 | } 7 | 8 | [data-jp-debugger='true'].jp-Editor .jp-mod-readOnly { 9 | background: var(--jp-layout-color2); 10 | height: 100%; 11 | } 12 | 13 | .jp-DCRSources-body [data-jp-debugger='true'].jp-Editor { 14 | height: 100%; 15 | } 16 | 17 | .jp-DCRSources-body { 18 | height: 100%; 19 | overflow-y: auto; 20 | } 21 | -------------------------------------------------------------------------------- /deployment/data-clean-room/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /deployment/data-clean-room/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "data-clean-room-chart.fullname" . }} 5 | labels: 6 | {{- include "data-clean-room-chart.labels" . | nindent 4 }} 7 | spec: 8 | type: {{ .Values.service.type }} 9 | ports: 10 | - port: {{ .Values.service.port }} 11 | targetPort: http 12 | protocol: TCP 13 | selector: 14 | {{- include "data-clean-room-chart.selectorLabels" . | nindent 4 }} 15 | -------------------------------------------------------------------------------- /app/jupyterlab_manatee/ui-tests/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "jupyterlab_manatee-ui-tests", 3 | "version": "1.0.0", 4 | "description": "JupyterLab jupyterlab-manatee Integration Tests", 5 | "private": true, 6 | "scripts": { 7 | "start": "jupyter lab --config jupyter_server_test_config.py", 8 | "test": "jlpm playwright test", 9 | "test:update": "jlpm playwright test --update-snapshots" 10 | }, 11 | "devDependencies": { 12 | "@jupyterlab/galata": "^5.0.5", 13 | "@playwright/test": "^1.37.0" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /deployment/data-clean-room/templates/tests/test-connection.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: "{{ include "data-clean-room-chart.fullname" . }}-test-connection" 5 | labels: 6 | {{- include "data-clean-room-chart.labels" . | nindent 4 }} 7 | annotations: 8 | "helm.sh/hook": test 9 | spec: 10 | containers: 11 | - name: wget 12 | image: busybox 13 | command: ['wget'] 14 | args: ['{{ include "data-clean-room-chart.fullname" . }}:{{ .Values.service.port }}'] 15 | restartPolicy: Never 16 | -------------------------------------------------------------------------------- /app/dcr_api/.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.a 3 | *.so 4 | _obj 5 | _test 6 | *.[568vq] 7 | [568vq].out 8 | *.cgo1.go 9 | *.cgo2.c 10 | _cgo_defun.c 11 | _cgo_gotypes.go 12 | _cgo_export.* 13 | _testmain.go 14 | *.exe 15 | *.exe~ 16 | *.test 17 | *.prof 18 | *.rar 19 | *.zip 20 | *.gz 21 | *.psd 22 | *.bmd 23 | *.cfg 24 | *.pptx 25 | *.log 26 | *nohup.out 27 | *settings.pyc 28 | *.sublime-project 29 | *.sublime-workspace 30 | !.gitkeep 31 | .DS_Store 32 | /.idea 33 | /.vscode 34 | /output 35 | *.local.yml 36 | dumped_hertz_remote_config.json 37 | conf 38 | github.com 39 | dcr_api 40 | -------------------------------------------------------------------------------- /pkg/utils/k8s.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | 7 | "github.com/cloudwego/hertz/pkg/common/hlog" 8 | ) 9 | 10 | func RunningInsideKubernetes() bool { 11 | if _, err := os.Stat("/var/run/secrets/kubernetes.io/serviceaccount"); os.IsNotExist(err) { 12 | hlog.Info("Not running inside Kubernetes") 13 | return false 14 | } 15 | hlog.Info("Running inside Kubernetes") 16 | return true 17 | } 18 | 19 | func GetNamespace() string { 20 | ns, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") 21 | if err != nil { 22 | hlog.Errorf("Failed to get namespace %s", err.Error()) 23 | } 24 | return strings.TrimSpace(string(ns)) 25 | } 26 | -------------------------------------------------------------------------------- /app/dcr_api/main.go: -------------------------------------------------------------------------------- 1 | // Code generated by hertz generator. 2 | 3 | package main 4 | 5 | import ( 6 | "github.com/cloudwego/hertz/pkg/app/server" 7 | "github.com/cloudwego/hertz/pkg/common/hlog" 8 | 9 | "github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/app/dcr_api/biz/dal" 10 | "github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pkg/config" 11 | ) 12 | 13 | func Init() { 14 | err := config.InitConfig() 15 | if err != nil { 16 | hlog.Errorf("failed to init config %+v", err) 17 | panic(err) 18 | } 19 | dal.Init() 20 | } 21 | 22 | func main() { 23 | Init() 24 | h := server.Default(server.WithHostPorts(":8080")) 25 | 26 | register(h) 27 | h.Spin() 28 | } 29 | -------------------------------------------------------------------------------- /app/jupyterlab_manatee/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 TikTok Pte. Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __import__("setuptools").setup() 16 | -------------------------------------------------------------------------------- /app/jupyterlab_manatee/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "allowSyntheticDefaultImports": true, 4 | "composite": true, 5 | "declaration": true, 6 | "esModuleInterop": true, 7 | "incremental": true, 8 | "jsx": "react", 9 | "module": "esnext", 10 | "moduleResolution": "node", 11 | "noEmitOnError": true, 12 | "noImplicitAny": true, 13 | "noUnusedLocals": true, 14 | "preserveWatchOutput": true, 15 | "resolveJsonModule": true, 16 | "outDir": "lib", 17 | "rootDir": "src", 18 | "strict": true, 19 | "strictNullChecks": true, 20 | "target": "ES2018", 21 | "skipLibCheck": true, 22 | }, 23 | "include": ["src/*"] 24 | } 25 | -------------------------------------------------------------------------------- /app/dcr_monitor/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.22 as builder 2 | 3 | WORKDIR /app 4 | 5 | COPY ./github.com /go/pkg/mod/github.com 6 | COPY ./client ./client 7 | COPY ./monitor ./monitor 8 | COPY ./go.mod ./go.mod 9 | COPY ./main.go ./main.go 10 | COPY ./go.sum ./go.sum 11 | RUN go build -o /app/dcr_monitor && chmod +x /app/dcr_monitor 12 | 13 | FROM debian:12.4 14 | RUN useradd dcrmonitor 15 | RUN apt-get update && apt-get install -y ca-certificates 16 | COPY ./conf /usr/local/dcr_conf 17 | 18 | WORKDIR /app 19 | RUN chown -R dcrmonitor:dcrmonitor /app 20 | 21 | COPY --from=builder --chown=dcrmonitor:dcrmonitor /app/dcr_monitor /usr/local/bin/dcr_monitor 22 | COPY ./conf /usr/local/dcr_conf 23 | USER dcrmonitor 24 | ENTRYPOINT dcr_monitor -------------------------------------------------------------------------------- /app/dcr_api/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.22 as builder 2 | WORKDIR /app 3 | COPY ./biz ./biz 4 | COPY ./main.go ./main.go 5 | COPY ./router_gen.go ./router_gen.go 6 | COPY ./router.go ./router.go 7 | # TODO DELETE 8 | COPY ./github.com /go/pkg/mod/github.com 9 | COPY ./go.mod ./go.mod 10 | COPY ./go.sum ./go.sum 11 | RUN go build -o /app/dcr_api && chmod +x /app/dcr_api 12 | 13 | 14 | FROM debian:12.4 15 | RUN useradd dcrapi 16 | RUN apt-get update && apt-get install -y ca-certificates 17 | COPY ./conf /usr/local/dcr_conf 18 | 19 | WORKDIR /app 20 | 21 | RUN mkdir /app/data 22 | RUN chown -R dcrapi:dcrapi /app/data 23 | COPY --from=builder --chown=dcrapi:dcrapi /app/dcr_api /usr/local/bin/dcr_api 24 | USER dcrapi 25 | ENTRYPOINT dcr_api 26 | -------------------------------------------------------------------------------- /app/jupyterlab_manatee/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM quay.io/jupyter/scipy-notebook as builder 2 | WORKDIR /app 3 | USER root 4 | RUN python3 -m pip install --upgrade pip 5 | RUN pip install build twine hatch jupyterhub==4.0.2 jupyterlab 6 | COPY ./ ./ 7 | 8 | RUN jlpm install 9 | RUN jlpm clean:all 10 | RUN jlpm build 11 | RUN python -m build 12 | 13 | FROM quay.io/jupyter/scipy-notebook 14 | USER root 15 | COPY --from=builder /app/dist/jupyterlab_manatee-0.1.0-py3-none-any.whl ./ 16 | RUN pip install jupyterlab_manatee-0.1.0-py3-none-any.whl 17 | RUN rm jupyterlab_manatee-0.1.0-py3-none-any.whl 18 | RUN jupyter labextension disable @jupyterlab/docmanager-extension:download 19 | RUN jupyter labextension disable @jupyterlab/filebrowser-extension:download -------------------------------------------------------------------------------- /pkg/utils/constant.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 TikTok Pte. Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package utils 16 | 17 | const ( 18 | Layout = "2006-01-02 15:04:05" 19 | ) 20 | -------------------------------------------------------------------------------- /app/jupyterlab_manatee/jest.config.js: -------------------------------------------------------------------------------- 1 | const jestJupyterLab = require('@jupyterlab/testutils/lib/jest-config'); 2 | 3 | const esModules = [ 4 | '@codemirror', 5 | '@jupyter/ydoc', 6 | '@jupyterlab/', 7 | 'lib0', 8 | 'nanoid', 9 | 'vscode-ws-jsonrpc', 10 | 'y-protocols', 11 | 'y-websocket', 12 | 'yjs' 13 | ].join('|'); 14 | 15 | const baseConfig = jestJupyterLab(__dirname); 16 | 17 | module.exports = { 18 | ...baseConfig, 19 | automock: false, 20 | collectCoverageFrom: [ 21 | 'src/**/*.{ts,tsx}', 22 | '!src/**/*.d.ts', 23 | '!src/**/.ipynb_checkpoints/*' 24 | ], 25 | coverageReporters: ['lcov', 'text'], 26 | testRegex: 'src/.*/.*.spec.ts[x]?$', 27 | transformIgnorePatterns: [`/node_modules/(?!${esModules}).+`] 28 | }; 29 | -------------------------------------------------------------------------------- /app/dcr_api/biz/service/build_service.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/cloudwego/hertz/pkg/common/hlog" 7 | "github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/app/dcr_api/biz/dal/db" 8 | "github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pkg/utils" 9 | ) 10 | 11 | func BuildImage(c context.Context, j db.Job, token string) error { 12 | if utils.RunningInsideKubernetes() { 13 | // use kaniko 14 | kanikoService := NewKanikoService(c) 15 | err := kanikoService.BuildImage(&j, token) 16 | if err != nil { 17 | hlog.Errorf("failed to run task %+v", err) 18 | return err 19 | } 20 | } else { 21 | hlog.Error("[BuildService] Not on kubernetes, can't build image") 22 | } 23 | return nil 24 | } 25 | -------------------------------------------------------------------------------- /resources/gcp/providers.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 TikTok Pte. Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | required_providers { 19 | google = { 20 | source = "hashicorp/google" 21 | } 22 | } 23 | } -------------------------------------------------------------------------------- /app/dcr_api/biz/dal/init.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 TikTok Pte. Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package dal 16 | 17 | import "github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/app/dcr_api/biz/dal/db" 18 | 19 | func Init() { 20 | db.Init() 21 | } 22 | -------------------------------------------------------------------------------- /resources/gcp/keys.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 TikTok Pte. Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | resource "google_kms_key_ring" "dcr_key_ring" { 18 | name = "dcr-${var.env}-keyring" 19 | location = var.region 20 | project = var.project_id 21 | lifecycle { 22 | prevent_destroy = false 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /app/conf/config.yaml: -------------------------------------------------------------------------------- 1 | CloudProvider: 2 | GCP: 3 | Project: "$PROJECTID" 4 | ProjectNumber: $PROJECTNUMBER 5 | Repository: "dcr-ENV-user-images" 6 | HubBucket: "dcr-ENV-hub" 7 | InputBucket: "dcr-ENV-input" 8 | CvmServiceAccount: "dcr-ENV-cvm-sa" 9 | Zone: "$ZONE" 10 | Region: "$REGION" 11 | CPUs: 2 12 | DiskSize: 50 13 | DebugInstanceImageSource: "projects/confidential-space-images/global/images/confidential-space-debug-240200" 14 | ReleaseInstanceImageSource: "projects/confidential-space-images/global/images/confidential-space-240200" 15 | Debug: false 16 | KeyRing: "dcr-ENV-keyring" 17 | WorkloadIdentityPool: "dcr-ENV-pool" 18 | IssuerUri: "https://confidentialcomputing.googleapis.com/" 19 | AllowedAudiences: ["https://sts.googleapis.com"] 20 | Network: "dcr-ENV-network" 21 | Subnetwork: "dcr-ENV-subnetwork" 22 | Env: ENV 23 | Cluster: 24 | PodServiceAccount: "dcr-k8s-pod-sa" -------------------------------------------------------------------------------- /app/jupyterlab_manatee/src/__tests__/jupyterlab_manatee.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 TikTok Pte. Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /** 18 | * Example of [Jest](https://jestjs.io/docs/getting-started) unit tests 19 | */ 20 | 21 | describe('jupyterlab_manatee', () => { 22 | it('should be tested', () => { 23 | expect(1 + 1).toEqual(2); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /app/dcr_monitor/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/cloudwego/hertz/pkg/common/hlog" 8 | "github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/app/dcr_monitor/client" 9 | "github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/app/dcr_monitor/monitor" 10 | "github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pkg/config" 11 | ) 12 | 13 | func main() { 14 | err := config.InitConfig() 15 | if err != nil { 16 | fmt.Printf("ERROR: failed to init config %+v \n", err) 17 | panic(err) 18 | } 19 | client.InitK8sClient() 20 | client.InitHTTPClient() 21 | ctx := context.Background() 22 | err = monitor.CheckKanikoJobs(ctx, client.K8sClientSet) 23 | if err != nil { 24 | hlog.Errorf("[CronJob]failed to check kaniko jobs %+v", err) 25 | } 26 | err = monitor.CheckTeeInstance(ctx) 27 | if err != nil { 28 | hlog.Errorf("[CronJob]failed to check TEE instances %+v", err) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /resources/gcp/buckets.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 TikTok Pte. Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | resource "google_storage_bucket" "data_clean_room_hub" { 18 | name = "dcr-${var.env}-hub" 19 | location = "us" 20 | project = var.project_id 21 | public_access_prevention = "enforced" 22 | uniform_bucket_level_access = true 23 | } 24 | -------------------------------------------------------------------------------- /deployment/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2024 TikTok Pte. Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -e 16 | deploy_service() { 17 | app=$1 18 | pushd $app 19 | ./deploy.sh 20 | popd 21 | } 22 | 23 | if [[ $# == 0 ]]; then 24 | deploy_service data-clean-room 25 | deploy_service jupyterhub 26 | elif [[ $# == 1 ]]; then 27 | deploy_service $1 28 | else 29 | echo "ERROR: unkown parameters" 30 | fi 31 | -------------------------------------------------------------------------------- /resources/gcp/identitypool.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 TikTok Pte. Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | resource "google_iam_workload_identity_pool" "dcr-workload-identity-pool" { 18 | workload_identity_pool_id = "dcr-${var.env}-pool" 19 | display_name = "DCR Workload Identity Pool" 20 | description = "Data Clean Room workload identity pool" 21 | project = var.project_id 22 | } -------------------------------------------------------------------------------- /app/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2024 TikTok Pte. Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -e 16 | build_apps() { 17 | app=$1 18 | pushd $app 19 | ./build.sh 20 | popd 21 | } 22 | 23 | if [[ $# == 0 ]]; then 24 | build_apps dcr_tee 25 | build_apps dcr_api 26 | build_apps dcr_monitor 27 | build_apps jupyterlab_manatee 28 | elif [[ $# == 1 ]]; then 29 | build_apps $1 30 | else 31 | echo "ERROR: unkown parameters" 32 | fi 33 | -------------------------------------------------------------------------------- /resources/kubernetes/providers.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 TikTok Pte. Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | required_providers { 19 | google = { 20 | source = "hashicorp/google" 21 | } 22 | kubernetes = { 23 | source = "hashicorp/kubernetes" 24 | } 25 | } 26 | 27 | } 28 | 29 | provider "kubernetes" { 30 | config_path = "~/.kube/config" 31 | } 32 | 33 | data "google_client_openid_userinfo" "me" {} -------------------------------------------------------------------------------- /resources/kubernetes/secret.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 TikTok Pte. Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | resource "kubernetes_secret" "secret" { 18 | metadata { 19 | name = "mysql-secret" 20 | namespace = kubernetes_namespace.data_clean_room_k8s_namespace.metadata[0].name 21 | } 22 | data = { 23 | mysql-username = var.mysql_username, 24 | mysql-password = var.mysql_password, 25 | mysql-database = local.database 26 | } 27 | } -------------------------------------------------------------------------------- /app/dcr_tee/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.22 as builder 2 | WORKDIR /app 3 | COPY ./attestation ./attestation 4 | COPY ./encryption ./encryption 5 | 6 | COPY ./github.com /go/pkg/mod/github.com 7 | WORKDIR /app/attestation 8 | RUN go build -o /app/gen_custom_token && chmod +x /app/gen_custom_token 9 | WORKDIR /app/encryption 10 | RUN go build -o /app/encrypt_tool && chmod +x /app/encrypt_tool 11 | 12 | FROM quay.io/jupyter/scipy-notebook 13 | RUN pip install google-cloud-storage 14 | USER root 15 | RUN apt-get update && apt-get -y install gnupg 16 | RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg && apt-get update -y && apt-get install google-cloud-sdk -y 17 | 18 | COPY ./conf /usr/local/dcr_conf 19 | COPY --from=builder /app/gen_custom_token /usr/local/bin/gen_custom_token 20 | COPY --from=builder /app/encrypt_tool /usr/local/bin/encrypt_tool -------------------------------------------------------------------------------- /app/dcr_monitor/client/hertz.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 TikTok Pte. Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package client 16 | 17 | import ( 18 | "time" 19 | 20 | "github.com/cloudwego/hertz/pkg/app/client" 21 | ) 22 | 23 | var ( 24 | HTTPClient *client.Client 25 | ) 26 | 27 | const ( 28 | UpdatePath = "/v1/job/update/" 29 | ) 30 | 31 | func InitHTTPClient() { 32 | var err error 33 | HTTPClient, err = client.NewClient(client.WithDialTimeout(1 * time.Second)) 34 | if err != nil { 35 | panic(err) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /app/dcr_api/biz/handler/health.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 TikTok Pte. Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Code generated by hertz generator. 16 | 17 | package handler 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/cloudwego/hertz/pkg/app" 23 | "github.com/cloudwego/hertz/pkg/common/utils" 24 | "github.com/cloudwego/hertz/pkg/protocol/consts" 25 | ) 26 | 27 | // Health . 28 | func Health(ctx context.Context, c *app.RequestContext) { 29 | c.JSON(consts.StatusOK, utils.H{ 30 | "message": "pong", 31 | }) 32 | } 33 | -------------------------------------------------------------------------------- /app/jupyterlab_manatee/ui-tests/jupyter_server_test_config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 TikTok Pte. Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Server configuration for integration tests. 16 | 17 | !! Never use this configuration in production because it 18 | opens the server to the world and provide access to JupyterLab 19 | JavaScript objects through the global window variable. 20 | """ 21 | from jupyterlab.galata import configure_jupyter_server 22 | 23 | configure_jupyter_server(c) 24 | 25 | # Uncomment to set server log level to debug level 26 | # c.ServerApp.log_level = "DEBUG" 27 | -------------------------------------------------------------------------------- /app/jupyterlab_manatee/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2024 TikTok Pte. Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -e 16 | VAR_FILE="../../.env" 17 | if [ ! -f "$VAR_FILE" ]; then 18 | echo "Error: Variables file does not exist." 19 | exit 1 20 | fi 21 | 22 | VAR_FILE=$(realpath $VAR_FILE) 23 | source $VAR_FILE 24 | 25 | docker_repo="dcr-$env-images" 26 | image_url="us-docker.pkg.dev/${project_id}/${docker_repo}/scipy-notebook-with-dcr" 27 | if [ -z "$username" ]; then 28 | tag=$env 29 | else 30 | tag=$username 31 | fi 32 | 33 | docker build --platform linux/amd64 --tag "$image_url:$tag" . 34 | docker push "$image_url:$tag" -------------------------------------------------------------------------------- /resources/gcp/.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/hashicorp/google" { 5 | version = "5.30.0" 6 | hashes = [ 7 | "h1:tFWLBCrtm/kDPbpm1HnXpzJ4dNpEnGGXfHDZkhYzdqc=", 8 | "zh:13169e55958e932aefa03551b63a3c951f18942be048205dc463c33089db232f", 9 | "zh:5093845d3150ccc3d99c4f4958679aa1a2cc53e5871620b1ac3dc59c6f72e1d8", 10 | "zh:8098f3253d72b5e591d55685234584861738b20fe57ab3124156b2233a71ee48", 11 | "zh:816fdb667d6dca429270e09531895c5c912ac5071c7a315ba011488cbf32a38e", 12 | "zh:880c38683c0b5f6c9d5314e55c29ddbacede4f1b7e18bebd2bed563756362c42", 13 | "zh:881ae2de266958118745a51331ca5308b4f47648e318d679dddf35c6422b292d", 14 | "zh:8d1f46aa7cf70fd0de222874ecdbef8587a10c9231aab0e319bf8095863b58d0", 15 | "zh:a4c9280a6126ea7a5350a7e3d46d7dcdff440d1bcf401f72c6392b7976400d73", 16 | "zh:bcae1f498b2c6d43aba5f6a755969d6e1108c82a77648550b5080d25ea56aabe", 17 | "zh:ce4e0f905fe19249ad7cb01123de9a3be82b0310937f8e872e85ba4d70aef031", 18 | "zh:d528cdd349ed7142d2288a4b9e03a5d3c1d1b18c9213bc21eb4cc03eca5c7ab2", 19 | "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /deployment/data-clean-room/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: data-clean-room-chart 3 | description: A Helm chart for Kubernetes 4 | 5 | # A chart can be either an 'application' or a 'library' chart. 6 | # 7 | # Application charts are a collection of templates that can be packaged into versioned archives 8 | # to be deployed. 9 | # 10 | # Library charts provide useful utilities or functions for the chart developer. They're included as 11 | # a dependency of application charts to inject those utilities and functions into the rendering 12 | # pipeline. Library charts do not define any templates and therefore cannot be deployed. 13 | type: application 14 | 15 | # This is the chart version. This version number should be incremented each time you make changes 16 | # to the chart and its templates, including the app version. 17 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 18 | version: 0.1.0 19 | 20 | # This is the version number of the application being deployed. This version number should be 21 | # incremented each time you make changes to the application. Versions are not expected to 22 | # follow Semantic Versioning. They should reflect the version the application is using. 23 | # It is recommended to use it with quotes. 24 | appVersion: "1.16.0" 25 | -------------------------------------------------------------------------------- /resources/kubernetes/namespace.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 TikTok Pte. Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | locals { 18 | dcr_k8s_namespace = var.username != "" ? "data-clean-room-${var.username}" : "data-clean-room-${var.env}" 19 | jupyter_k8s_namespace = var.username != "" ? "jupyterhub-${var.username}" : "jupyterhub-${var.env}" 20 | } 21 | 22 | resource "kubernetes_namespace" "data_clean_room_k8s_namespace" { 23 | metadata { 24 | name = local.dcr_k8s_namespace 25 | } 26 | 27 | depends_on = [ kubernetes_cluster_role_binding.cluster_admin_binding ] 28 | } 29 | 30 | resource "kubernetes_namespace" "jupyterhub_k8s_namespace" { 31 | metadata { 32 | name = local.jupyter_k8s_namespace 33 | } 34 | } -------------------------------------------------------------------------------- /resources/gcp/repositories.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 TikTok Pte. Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | resource "google_artifact_registry_repository" "dcr_user_images" { 18 | project = var.project_id 19 | location = "us" 20 | repository_id = "dcr-${var.env}-user-images" 21 | description = "The repository stores the images that are built by data clean room API and running in the confidential space." 22 | format = "DOCKER" 23 | } 24 | 25 | resource "google_artifact_registry_repository" "data_clean_room_images" { 26 | project = var.project_id 27 | location = "us" 28 | repository_id = "dcr-${var.env}-images" 29 | description = "Data Clean Room Images" 30 | format = "DOCKER" 31 | } -------------------------------------------------------------------------------- /app/jupyterlab_manatee/ui-tests/tests/jupyterlab_manatee.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 TikTok Pte. Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { expect, test } from '@jupyterlab/galata'; 18 | 19 | /** 20 | * Don't load JupyterLab webpage before running the tests. 21 | * This is required to ensure we capture all log messages. 22 | */ 23 | test.use({ autoGoto: false }); 24 | 25 | test('should emit an activation console message', async ({ page }) => { 26 | const logs: string[] = []; 27 | 28 | page.on('console', message => { 29 | logs.push(message.text()); 30 | }); 31 | 32 | await page.goto(); 33 | 34 | expect( 35 | logs.filter(s => s === 'JupyterLab extension jupyterlab-manatee is activated!') 36 | ).toHaveLength(1); 37 | }); 38 | -------------------------------------------------------------------------------- /app/dcr_monitor/client/k8s.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 TikTok Pte. Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package client 16 | 17 | import ( 18 | "os" 19 | 20 | "k8s.io/client-go/kubernetes" 21 | "k8s.io/client-go/rest" 22 | ) 23 | 24 | var ( 25 | K8sClientSet *kubernetes.Clientset 26 | RunningNameSpace string 27 | ) 28 | 29 | func InitK8sClient() { 30 | var err error 31 | clusterConfig, err := rest.InClusterConfig() 32 | if err != nil { 33 | panic(err) 34 | } 35 | 36 | K8sClientSet, err = kubernetes.NewForConfig(clusterConfig) 37 | if err != nil { 38 | panic(err) 39 | } 40 | 41 | RunningNameSpaceByte, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") 42 | if err != nil { 43 | panic(err) 44 | } 45 | RunningNameSpace = string(RunningNameSpaceByte) 46 | } 47 | -------------------------------------------------------------------------------- /app/conf/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE_IMAGE 2 | FROM $BASE_IMAGE 3 | ARG OUTPUTPATH 4 | ARG ENCRYPTED_FILENAME 5 | ARG ENCRYPTED_CLOUDSTORAGE_PATH 6 | ARG CREATOR 7 | ARG IMPERSONATION_SERVICE_ACCOUNT 8 | ARG JUPYTER_FILENAME 9 | ARG USER_WORKSPACE 10 | ARG CUSTOMTOKEN_CLOUDSTORAGE_PATH 11 | 12 | ENV OUTPUTPATH=$OUTPUTPATH 13 | ENV ENCRYPTED_FILENAME=$ENCRYPTED_FILENAME 14 | ENV ENCRYPTED_CLOUDSTORAGE_PATH=$ENCRYPTED_CLOUDSTORAGE_PATH 15 | ENV IMPERSONATION_SERVICE_ACCOUNT=$IMPERSONATION_SERVICE_ACCOUNT 16 | ENV CREATOR=$CREATOR 17 | ENV JUPYTER_FILENAME=$JUPYTER_FILENAME 18 | ENV CUSTOMTOKEN_CLOUDSTORAGE_PATH=$CUSTOMTOKEN_CLOUDSTORAGE_PATH 19 | 20 | WORKDIR /home/jovyan 21 | COPY $USER_WORKSAPCE/* ./ 22 | 23 | LABEL "tee.launch_policy.allow_env_override"="USER_TOKEN,EXECUTION_STAGE,DEPLOYMENT_ENV,PROJECT_ID,KEY_LOCATION" 24 | 25 | ENTRYPOINT jupyter nbconvert --execute --to notebook --inplace $JUPYTER_FILENAME --ExecutePreprocessor.timeout=-1 --allow-errors \ 26 | && hash=$(md5sum $JUPYTER_FILENAME | awk '{ print $1 }') \ 27 | && gsutil cp $JUPYTER_FILENAME $OUTPUTPATH \ 28 | && encrypt_tool --user=$CREATOR --input=$JUPYTER_FILENAME --output=$ENCRYPTED_FILENAME --impersonation $IMPERSONATION_SERVICE_ACCOUNT \ 29 | && gsutil cp $ENCRYPTED_FILENAME $ENCRYPTED_CLOUDSTORAGE_PATH \ 30 | && gen_custom_token --nonce $hash \ 31 | && gsutil cp custom_token $CUSTOMTOKEN_CLOUDSTORAGE_PATH -------------------------------------------------------------------------------- /deployment/jupyterhub/config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 TikTok Pte. Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | singleuser: 16 | image: 17 | name: us-docker.pkg.dev/${project_id}/${artifact_repo_docker}/datascience-notebook-with-dcr 18 | tag: ${tag} 19 | pullPolicy: Always 20 | cmd: null 21 | nodeSelector: { iam.gke.io/gke-metadata-server-enabled: "true"} 22 | serviceAccountName: $single_user_pod_sa 23 | extraEnv: 24 | NOTEBOOK_ARGS: '--NotebookApp.terminals_enabled=False --NotebookApp.allow_root=False' 25 | JUPYTERHUB_SINGLEUSER_APP: "jupyter_server.serverapp.ServerApp" 26 | DATA_CLEAN_ROOM_HOST: "$api" 27 | DEPLOYMENT_ENV: "$env" 28 | PROJECT_ID: "$project_id" 29 | KEY_LOCALTION: "$region" 30 | networkPolicy: 31 | egressAllowRules: 32 | cloudMetadataServer: true 33 | cloudMetadata: 34 | blockWithIptables: false -------------------------------------------------------------------------------- /app/dcr_api/biz/dal/db/init.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 TikTok Pte. Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package db 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | 21 | "gorm.io/driver/mysql" 22 | "gorm.io/gorm" 23 | "gorm.io/gorm/logger" 24 | ) 25 | 26 | var DB *gorm.DB 27 | 28 | func Init() { 29 | var err error 30 | mysqlDsn := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8&parseTime=True&loc=Local", os.Getenv("MYSQL_USERNAME"), os.Getenv("MYSQL_PASSWORD"), os.Getenv("MYSQL_HOST"), os.Getenv("MYSQL_PORT"), os.Getenv("MYSQL_DATABASE")) 31 | DB, err = gorm.Open(mysql.Open(mysqlDsn), &gorm.Config{ 32 | SkipDefaultTransaction: true, 33 | PrepareStmt: true, 34 | Logger: logger.Default.LogMode(logger.Info), 35 | }) 36 | if err != nil { 37 | panic(err) 38 | } 39 | 40 | // Auto database schema migration 41 | // This has caveat: see https://gorm.io/docs/migration.html 42 | err = DB.AutoMigrate(&Job{}) 43 | if err != nil { 44 | panic(err) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /resources/gcp/service_accounts.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 TikTok Pte. Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | # A service account used for data clean room cluster 18 | resource "google_service_account" "gcp_dcr_cluster_sa" { 19 | account_id = "dcr-${var.env}-cluster-sa" 20 | display_name = "A Service account for data clean room cluster" 21 | project = var.project_id 22 | } 23 | 24 | resource "google_service_account" "gcp_cvm_sa" { 25 | account_id = "dcr-${var.env}-cvm-sa" 26 | display_name = "A Service account for confidential vm" 27 | project = var.project_id 28 | } 29 | 30 | resource "google_service_account" "gcp_dcr_pod_sa" { 31 | account_id = "dcr-${var.env}-pod-sa" 32 | display_name = "A Service account for data clean room api pod" 33 | project = var.project_id 34 | } 35 | 36 | resource "google_service_account" "gcp_jupyter_pod_sa" { 37 | account_id = "jupyter-${var.env}-pod-sa" 38 | display_name = "A Service account for jupyterhub single user pod" 39 | project = var.project_id 40 | } -------------------------------------------------------------------------------- /pkg/errno/errno.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 TikTok Pte. Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package errno 16 | 17 | import ( 18 | "fmt" 19 | ) 20 | 21 | const ( 22 | SuccessCode = 0 23 | ServiceErrCode = iota + 10000 24 | ReachJobLimitErrCode 25 | ) 26 | 27 | const ( 28 | SuccessMsg = "Success" 29 | ServiceErrMsg = "Service internal error" 30 | ReachJobLimitErrMsg = "The number of in progress jobs has reached the limit" 31 | ) 32 | 33 | type ErrNo struct { 34 | ErrCode int32 35 | ErrMsg string 36 | } 37 | 38 | func (e ErrNo) Error() string { 39 | return fmt.Sprintf("err_code=%d, err_msg=%s", e.ErrCode, e.ErrMsg) 40 | } 41 | 42 | func NewErrNo(code int32, msg string) ErrNo { 43 | return ErrNo{code, msg} 44 | } 45 | 46 | func (e ErrNo) WithMessage(msg string) ErrNo { 47 | e.ErrMsg = msg 48 | return e 49 | } 50 | 51 | var ( 52 | Success = NewErrNo(SuccessCode, SuccessMsg) 53 | ServiceErr = NewErrNo(ServiceErrCode, ServiceErrMsg) 54 | ReachJobLimitErr = NewErrNo(ReachJobLimitErrCode, ReachJobLimitErrMsg) 55 | ) 56 | -------------------------------------------------------------------------------- /app/jupyterlab_manatee/LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2023, Dayeol Lee 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /resources/kubernetes/role.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 TikTok Pte. Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | resource "kubernetes_role" "role" { 18 | metadata { 19 | name = "dcr-pod-role" 20 | namespace = local.dcr_k8s_namespace 21 | } 22 | 23 | rule { 24 | api_groups = ["batch", ""] 25 | resources = ["jobs", "pods", "pods/log"] 26 | verbs = ["get", "list", "watch", "create", "update", "patch", "delete"] 27 | } 28 | 29 | depends_on = [kubernetes_cluster_role_binding.cluster_admin_binding] 30 | } 31 | 32 | resource "kubernetes_role_binding" "role_binding" { 33 | metadata { 34 | name = "dcr-pod-role-binding" 35 | namespace = local.dcr_k8s_namespace 36 | } 37 | role_ref { 38 | api_group = "rbac.authorization.k8s.io" 39 | kind = "Role" 40 | name = kubernetes_role.role.metadata[0].name 41 | } 42 | subject { 43 | kind = "ServiceAccount" 44 | name = kubernetes_service_account.k8s_dcr_pod_service_account.metadata[0].name 45 | namespace = local.dcr_k8s_namespace 46 | } 47 | } 48 | 49 | -------------------------------------------------------------------------------- /app/jupyterlab_manatee/jupyterlab_manatee/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 TikTok Pte. Ltd. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import jupyter_server 16 | from jupyter_server.utils import url_path_join 17 | from ._version import __version__ 18 | from .handlers import * 19 | 20 | def _jupyter_server_extension_points(): 21 | return [{ 22 | 'module': 'jupyterlab_manatee' 23 | }] 24 | 25 | def _jupyter_labextension_paths(): 26 | return [{ 27 | "src": "labextension", 28 | "dest": "jupyterlab_manatee" 29 | }] 30 | 31 | 32 | def _load_jupyter_server_extension(serverapp: jupyter_server.serverapp.ServerApp): 33 | """ 34 | Called when the extension is loaded. 35 | """ 36 | 37 | web_app = serverapp.web_app 38 | base_url = web_app.settings['base_url'] 39 | handlers = [ 40 | (url_path_join(base_url, 'manatee', 'jobs'), DataCleanRoomJobHandler), 41 | (url_path_join(base_url, 'manatee', 'output'), DataCleanRoomOutputHandler), (url_path_join(base_url, 'manatee', 'attestation'), DataCleanRoomAttestationHandler), 42 | ] 43 | web_app.add_handlers('.*$', handlers) -------------------------------------------------------------------------------- /resources/kubernetes/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 TikTok Pte. Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "username" { 18 | type = string 19 | description = "Username to postfix resources with" 20 | default = "" 21 | } 22 | 23 | variable "env" { 24 | type = string 25 | description = "Deployment environment, e.g., dev, prod, oss" 26 | } 27 | 28 | variable "project_id" { 29 | type = string 30 | description = "The GCP project ID" 31 | } 32 | 33 | variable "project_number" { 34 | type = string 35 | description = "The GCP project number" 36 | } 37 | 38 | variable "mysql_username" { 39 | type = string 40 | description = "Mysql username" 41 | } 42 | 43 | variable "mysql_password" { 44 | type = string 45 | description = "Mysql password" 46 | } 47 | 48 | variable "region" { 49 | type = string 50 | description = "Region to create the gcp resources" 51 | } 52 | 53 | locals { 54 | gcp_dcr_pod_sa = "dcr-${var.env}-pod-sa" 55 | gcp_jupyter_pod_sa = "jupyter-${var.env}-pod-sa" 56 | database = "dcr-${var.env}-database" 57 | } -------------------------------------------------------------------------------- /pkg/utils/resp.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 TikTok Pte. Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package utils 16 | 17 | import ( 18 | "errors" 19 | "net/http" 20 | 21 | "github.com/cloudwego/hertz/pkg/app" 22 | "github.com/gin-gonic/gin" 23 | 24 | "github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pkg/errno" 25 | ) 26 | 27 | type BaseResp struct { 28 | StatusCode int32 29 | StatusMsg string 30 | } 31 | 32 | // BuildBaseResp convert error and build BaseResp 33 | func BuildBaseResp(err error) *BaseResp { 34 | if err == nil { 35 | return baseResp(errno.Success) 36 | } 37 | 38 | e := errno.ErrNo{} 39 | if errors.As(err, &e) { 40 | return baseResp(e) 41 | } 42 | 43 | s := errno.ServiceErr.WithMessage(err.Error()) 44 | return baseResp(s) 45 | } 46 | 47 | // baseResp build BaseResp from error 48 | func baseResp(err errno.ErrNo) *BaseResp { 49 | return &BaseResp{ 50 | StatusCode: err.ErrCode, 51 | StatusMsg: err.ErrMsg, 52 | } 53 | } 54 | 55 | func ReturnsJSONError(c *app.RequestContext, err error) { 56 | resp := BuildBaseResp(err) 57 | c.JSON(http.StatusOK, gin.H{"code": resp.StatusCode, "msg": resp.StatusMsg}) 58 | c.Abort() 59 | } 60 | -------------------------------------------------------------------------------- /app/jupyterlab_manatee/src/sidebar.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 TikTok Pte. Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { SidePanel, trustedIcon } from '@jupyterlab/ui-components'; 18 | import { ITranslator, nullTranslator } from '@jupyterlab/translation'; 19 | import { IDocumentManager } from '@jupyterlab/docmanager'; 20 | import { DataCleanRoomSources } from './sources'; 21 | // import { DataCleanRoomInputs } from './inputs'; 22 | import { DataCleanRoomJobs } from './jobs'; 23 | 24 | export class DataCleanRoomSidebar extends SidePanel { 25 | constructor(options: DataCleanRoomSidebar.IOptions) { 26 | const { manager } = options; 27 | const translator = options.translator || nullTranslator; 28 | super({ translator }); 29 | 30 | const sourcesPanel = new DataCleanRoomSources({ manager, translator }); 31 | const jobsPanel = new DataCleanRoomJobs({ translator }); 32 | 33 | this.title.icon = trustedIcon; 34 | this.id = "jp-DCRSource-sidebar" 35 | this.addWidget(sourcesPanel); 36 | this.addWidget(jobsPanel); 37 | } 38 | } 39 | 40 | export namespace DataCleanRoomSidebar { 41 | export interface IOptions { 42 | manager: IDocumentManager; 43 | translator?: ITranslator; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /app/jupyterlab_manatee/src/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 TikTok Pte. Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { 18 | JupyterFrontEnd, 19 | JupyterFrontEndPlugin, 20 | ILayoutRestorer, 21 | } from '@jupyterlab/application'; 22 | import { IDocumentManager } from '@jupyterlab/docmanager'; 23 | import { ITranslator } from '@jupyterlab/translation'; 24 | import { DataCleanRoomSidebar } from './sidebar'; 25 | 26 | 27 | async function activate(app: JupyterFrontEnd, docManager: IDocumentManager, translator: ITranslator, restorer: ILayoutRestorer | null) { 28 | console.log("JupyterLab extension jupyterlab_manatee is activated!"); 29 | 30 | const sidebar = new DataCleanRoomSidebar({manager: docManager}); 31 | 32 | app.shell.add(sidebar, 'right', {rank: 850}); 33 | 34 | if (restorer) { 35 | restorer.add(sidebar, "data-clean-room-side-bar"); 36 | } 37 | } 38 | 39 | /** 40 | * Initialization data for the jupyterlab-manatee extension. 41 | */ 42 | const plugin: JupyterFrontEndPlugin = { 43 | id: 'jupyterlab_manatee:plugin', 44 | description: 'This is an open-source JupyterLab extension for data clean room', 45 | autoStart: true, 46 | requires: [IDocumentManager, ITranslator], 47 | optional: [ILayoutRestorer], 48 | activate: activate 49 | }; 50 | 51 | export default plugin; 52 | -------------------------------------------------------------------------------- /resources/apply.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2024 TikTok Pte. Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Check if gcloud is installed 17 | if ! [ -x "$(command -v gcloud)" ]; then 18 | echo "Error: gcloud is not installed." >&2 19 | exit 1 20 | fi 21 | 22 | # Check if gcloud logged in 23 | if ! gcloud auth list | grep -q 'ACTIVE'; then 24 | echo "Error: No active gcloud account found." >&2 25 | exit 1 26 | fi 27 | 28 | # check whether variables has been set 29 | VAR_FILE="../.env" 30 | if [ ! -f "$VAR_FILE" ]; then 31 | echo "Error: Variables file does not exist." 32 | exit 1 33 | fi 34 | VAR_FILE=$(realpath $VAR_FILE) 35 | source $VAR_FILE 36 | 37 | if ! gsutil ls gs://dcr-tf-state-$env > /dev/null 2>&1; then 38 | gsutil mb -l us gs://dcr-tf-state-$env 39 | fi 40 | 41 | pushd gcp 42 | 43 | cat << EOF > backend.tf 44 | terraform { 45 | backend "gcs" { 46 | bucket = "dcr-tf-state-$env" 47 | prefix = "cloud" 48 | } 49 | } 50 | EOF 51 | 52 | cp $VAR_FILE terraform.tfvars 53 | terraform init -reconfigure 54 | terraform apply 55 | popd 56 | 57 | zone=$region-a 58 | # get kubernete cluster credentials 59 | gcloud container clusters get-credentials dcr-$env-cluster --zone $zone --project $project_id 60 | 61 | pushd kubernetes 62 | cp $VAR_FILE terraform.tfvars 63 | terraform init -reconfigure 64 | terraform apply 65 | popd -------------------------------------------------------------------------------- /deployment/data-clean-room/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | 1. Get the application URL by running these commands: 2 | {{- if .Values.ingress.enabled }} 3 | {{- range $host := .Values.ingress.hosts }} 4 | {{- range .paths }} 5 | http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} 6 | {{- end }} 7 | {{- end }} 8 | {{- else if contains "NodePort" .Values.service.type }} 9 | export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "data-clean-room-chart.fullname" . }}) 10 | export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") 11 | echo http://$NODE_IP:$NODE_PORT 12 | {{- else if contains "LoadBalancer" .Values.service.type }} 13 | NOTE: It may take a few minutes for the LoadBalancer IP to be available. 14 | You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "data-clean-room-chart.fullname" . }}' 15 | export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "data-clean-room-chart.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") 16 | echo http://$SERVICE_IP:{{ .Values.service.port }} 17 | {{- else if contains "ClusterIP" .Values.service.type }} 18 | export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "data-clean-room-chart.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") 19 | export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") 20 | echo "Visit http://127.0.0.1:8080 to use your application" 21 | kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT 22 | {{- end }} 23 | -------------------------------------------------------------------------------- /resources/gcp/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 TikTok Pte. Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "username" { 18 | type = string 19 | description = "Username to postfix resources with" 20 | default = "" 21 | } 22 | 23 | variable "env" { 24 | type = string 25 | description = "Deployment environment, e.g., dev, prod, oss" 26 | } 27 | 28 | variable "region" { 29 | type = string 30 | description = "Region to create the gcp resources" 31 | } 32 | 33 | variable "project_id" { 34 | type = string 35 | description = "The GCP project ID" 36 | } 37 | 38 | variable "project_number" { 39 | type = string 40 | description = "The GCP project number" 41 | } 42 | 43 | variable "mysql_username" { 44 | type = string 45 | description = "Mysql username" 46 | } 47 | 48 | variable "mysql_password" { 49 | type = string 50 | description = "Mysql password" 51 | } 52 | 53 | variable "type" { 54 | type = string 55 | description = "Instance type for the GKE instances" 56 | default = "n2-standard-4" 57 | } 58 | 59 | variable "num_nodes" { 60 | type = number 61 | description = "Number of nodes to create in the GKE cluster" 62 | default = 2 63 | } 64 | 65 | locals { 66 | zone = "${var.region}-a" 67 | } 68 | 69 | locals { 70 | 71 | } 72 | -------------------------------------------------------------------------------- /resources/gcp/database.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 TikTok Pte. Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | resource "google_sql_database_instance" "dcr_database_instance" { 18 | name = "dcr-${var.env}-db-instance" 19 | database_version = "MYSQL_8_0" 20 | project = var.project_id 21 | region = var.region 22 | settings { 23 | tier = "db-f1-micro" 24 | ip_configuration { 25 | ipv4_enabled = false 26 | private_network = google_compute_network.data_clean_room_network.id 27 | enable_private_path_for_google_cloud_services = true 28 | } 29 | } 30 | lifecycle { 31 | prevent_destroy = false 32 | } 33 | depends_on = [ google_compute_subnetwork.data_clean_room_subnetwork, google_compute_global_address.dcr_private_address, google_service_networking_connection.private_vpc_connection ] 34 | } 35 | 36 | resource "google_sql_database" "database" { 37 | name = "dcr-${var.env}-database" 38 | project = var.project_id 39 | instance = google_sql_database_instance.dcr_database_instance.name 40 | } 41 | 42 | resource "google_sql_user" "dcr_db_user" { 43 | name = var.mysql_username 44 | instance = google_sql_database_instance.dcr_database_instance.name 45 | password = var.mysql_password 46 | project = var.project_id 47 | } -------------------------------------------------------------------------------- /deployment/data-clean-room/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2024 TikTok Pte. Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | VAR_FILE="../../.env" 17 | if [ ! -f "$VAR_FILE" ]; then 18 | echo "Error: Variables file does not exist." 19 | exit 1 20 | fi 21 | 22 | VAR_FILE=$(realpath $VAR_FILE) 23 | source $VAR_FILE 24 | 25 | if [ -z "$username" ]; then 26 | helm_name="data-clean-room-helm-$env" 27 | k8s_namespace="data-clean-room-$env" 28 | tag=$env 29 | else 30 | helm_name="data-clean-room-helm-$username" 31 | k8s_namespace="data-clean-room-$username" 32 | tag=$username 33 | fi 34 | 35 | connection_name="${project_id}:${region}:dcr-${env}-db-instance" 36 | service_account="dcr-k8s-pod-sa" 37 | docker_repo="dcr-${env}-images" 38 | api_docker_reference="us-docker.pkg.dev/${project_id}/${docker_repo}/data-clean-room-api" 39 | monitor_docker_reference="us-docker.pkg.dev/${project_id}/${docker_repo}/data-clean-room-monitor" 40 | 41 | helm upgrade --cleanup-on-fail \ 42 | --set apiImage.repository=${api_docker_reference} \ 43 | --set apiImage.tag=${tag} \ 44 | --set monitorImage.repository=${monitor_docker_reference} \ 45 | --set monitorImage.tag=${tag} \ 46 | --set serviceAccount.name=${service_account} \ 47 | --set cloudSql.connection_name=${connection_name} \ 48 | --set namespace=${k8s_namespace} \ 49 | --install $helm_name ./ \ 50 | --namespace $k8s_namespace \ 51 | --values config.yaml -------------------------------------------------------------------------------- /app/dcr_api/biz/router/job/job.go: -------------------------------------------------------------------------------- 1 | // Code generated by hertz generator. DO NOT EDIT. 2 | 3 | package job 4 | 5 | import ( 6 | "github.com/cloudwego/hertz/pkg/app/server" 7 | job "github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/app/dcr_api/biz/handler/job" 8 | ) 9 | 10 | /* 11 | This file will register all the routes of the services in the master idl. 12 | And it will update automatically when you use the "update" command for the idl. 13 | So don't modify the contents of the file, or your code will be deleted when it is updated. 14 | */ 15 | 16 | // Register register routes based on the IDL 'api.${HTTP Method}' annotation. 17 | func Register(r *server.Hertz) { 18 | 19 | root := r.Group("/", rootMw()...) 20 | { 21 | _v1 := root.Group("/v1", _v1Mw()...) 22 | { 23 | _job := _v1.Group("/job", _jobMw()...) 24 | { 25 | _attestation := _job.Group("/attestation", _attestationMw()...) 26 | _attestation.POST("/", append(_queryjobattestationreportMw(), job.QueryJobAttestationReport)...) 27 | } 28 | { 29 | _delete := _job.Group("/delete", _deleteMw()...) 30 | _delete.POST("/", append(_deletejobMw(), job.DeleteJob)...) 31 | } 32 | { 33 | _output := _job.Group("/output", _outputMw()...) 34 | { 35 | _attrs := _output.Group("/attrs", _attrsMw()...) 36 | _attrs.POST("/", append(_queryjoboutputattrMw(), job.QueryJobOutputAttr)...) 37 | } 38 | { 39 | _download := _output.Group("/download", _downloadMw()...) 40 | _download.POST("/", append(_downloadjoboutputMw(), job.DownloadJobOutput)...) 41 | } 42 | } 43 | { 44 | _query := _job.Group("/query", _queryMw()...) 45 | _query.POST("/", append(_queryjobMw(), job.QueryJob)...) 46 | } 47 | { 48 | _submit := _job.Group("/submit", _submitMw()...) 49 | _submit.POST("/", append(_submitjobMw(), job.SubmitJob)...) 50 | } 51 | { 52 | _update := _job.Group("/update", _updateMw()...) 53 | _update.POST("/", append(_updatejobstatusMw(), job.UpdateJobStatus)...) 54 | } 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /resources/gcp/network.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 TikTok Pte. Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | resource "google_compute_network" "data_clean_room_network" { 18 | name = "dcr-${var.env}-network" 19 | auto_create_subnetworks = false 20 | project = var.project_id 21 | } 22 | 23 | resource "google_compute_global_address" "dcr_private_address" { 24 | name = "dcr-${var.env}-private-address" 25 | project = var.project_id 26 | purpose = "VPC_PEERING" 27 | address_type = "INTERNAL" 28 | prefix_length = 16 29 | network = google_compute_network.data_clean_room_network.self_link 30 | } 31 | 32 | resource "google_service_networking_connection" "private_vpc_connection" { 33 | network = google_compute_network.data_clean_room_network.self_link 34 | service = "servicenetworking.googleapis.com" 35 | reserved_peering_ranges = [google_compute_global_address.dcr_private_address.name] 36 | depends_on = [ google_compute_global_address.dcr_private_address ] 37 | } 38 | 39 | resource "google_compute_subnetwork" "data_clean_room_subnetwork" { 40 | name = "dcr-${var.env}-subnetwork" 41 | project = var.project_id 42 | ip_cidr_range = "10.0.0.0/22" 43 | region = var.region 44 | 45 | stack_type = "IPV4_IPV6" 46 | ipv6_access_type = "EXTERNAL" 47 | 48 | network = google_compute_network.data_clean_room_network.id 49 | depends_on = [ google_service_networking_connection.private_vpc_connection ] 50 | } 51 | -------------------------------------------------------------------------------- /app/dcr_api/biz/router/job/middleware.go: -------------------------------------------------------------------------------- 1 | // Code generated by hertz generator. 2 | 3 | package job 4 | 5 | import ( 6 | "github.com/cloudwego/hertz/pkg/app" 7 | ) 8 | 9 | func rootMw() []app.HandlerFunc { 10 | // your code... 11 | return nil 12 | } 13 | 14 | func _v1Mw() []app.HandlerFunc { 15 | // your code... 16 | return nil 17 | } 18 | 19 | func _jobMw() []app.HandlerFunc { 20 | // your code... 21 | return nil 22 | } 23 | 24 | func _deleteMw() []app.HandlerFunc { 25 | // your code... 26 | return nil 27 | } 28 | 29 | func _deletejobMw() []app.HandlerFunc { 30 | // your code... 31 | return nil 32 | } 33 | 34 | func _queryMw() []app.HandlerFunc { 35 | // your code... 36 | return nil 37 | } 38 | 39 | func _queryjobMw() []app.HandlerFunc { 40 | // your code... 41 | return nil 42 | } 43 | 44 | func _submitMw() []app.HandlerFunc { 45 | // your code... 46 | return nil 47 | } 48 | 49 | func _createjobMw() []app.HandlerFunc { 50 | // your code... 51 | return nil 52 | } 53 | 54 | func _updateMw() []app.HandlerFunc { 55 | // your code... 56 | return nil 57 | } 58 | 59 | func _updatejobstatusMw() []app.HandlerFunc { 60 | // your code... 61 | return nil 62 | } 63 | 64 | func _fileMw() []app.HandlerFunc { 65 | // your code... 66 | return nil 67 | } 68 | 69 | func _attrsMw() []app.HandlerFunc { 70 | // your code... 71 | return nil 72 | } 73 | 74 | func _queryjoboutputattrMw() []app.HandlerFunc { 75 | // your code... 76 | return nil 77 | } 78 | 79 | func _downloadMw() []app.HandlerFunc { 80 | // your code... 81 | return nil 82 | } 83 | 84 | func _downloadjoboutputMw() []app.HandlerFunc { 85 | // your code... 86 | return nil 87 | } 88 | 89 | func _attestationMw() []app.HandlerFunc { 90 | // your code... 91 | return nil 92 | } 93 | 94 | func _queryjobattestationreportMw() []app.HandlerFunc { 95 | // your code... 96 | return nil 97 | } 98 | 99 | func _outputMw() []app.HandlerFunc { 100 | // your code... 101 | return nil 102 | } 103 | 104 | func _submitjobMw() []app.HandlerFunc { 105 | // your code... 106 | return nil 107 | } 108 | -------------------------------------------------------------------------------- /deployment/data-clean-room/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "data-clean-room-chart.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "data-clean-room-chart.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "data-clean-room-chart.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "data-clean-room-chart.labels" -}} 37 | helm.sh/chart: {{ include "data-clean-room-chart.chart" . }} 38 | {{ include "data-clean-room-chart.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "data-clean-room-chart.selectorLabels" -}} 49 | app.kubernetes.io/name: {{ include "data-clean-room-chart.name" . }} 50 | app.kubernetes.io/instance: {{ .Release.Name }} 51 | {{- end }} 52 | 53 | {{/* 54 | Create the name of the service account to use 55 | */}} 56 | {{- define "data-clean-room-chart.serviceAccountName" -}} 57 | {{- if .Values.serviceAccount.create }} 58 | {{- default (include "data-clean-room-chart.fullname" .) .Values.serviceAccount.name }} 59 | {{- else }} 60 | {{- default "default" .Values.serviceAccount.name }} 61 | {{- end }} 62 | {{- end }} 63 | -------------------------------------------------------------------------------- /deployment/jupyterhub/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2024 TikTok Pte. Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | VAR_FILE="../../.env" 17 | if [ ! -f "$VAR_FILE" ]; then 18 | echo "Error: Variables file does not exist." 19 | exit 1 20 | fi 21 | 22 | VAR_FILE=$(realpath $VAR_FILE) 23 | source $VAR_FILE 24 | 25 | if [ -z "$username" ]; then 26 | helm_name="jupyterhub-helm-$env" 27 | k8s_namespace="jupyterhub-$env" 28 | tag=$env 29 | api="http://data-clean-room.data-clean-room-$env.svc.cluster.local" 30 | else 31 | helm_name="jupyterhub-helm-$username" 32 | k8s_namespace="jupyterhub-$username" 33 | tag=$username 34 | api="http://data-clean-room.data-clean-room-$username.svc.cluster.local" 35 | fi 36 | 37 | service_account="jupyter-k8s-pod-sa" 38 | docker_repo="dcr-${env}-images" 39 | docker_reference="us-docker.pkg.dev/${project_id}/${docker_repo}/scipy-notebook-with-dcr" 40 | 41 | helm repo add jupyterhub https://hub.jupyter.org/helm-chart/ 42 | helm repo update 43 | 44 | helm upgrade --cleanup-on-fail \ 45 | --set singleuser.image.name=${docker_reference} \ 46 | --set singleuser.image.tag=${tag} \ 47 | --set singleuser.serviceAccountName=${service_account} \ 48 | --set singleuser.extraEnv.DATA_CLEAN_ROOM_HOST=${api} \ 49 | --set singleuser.extraEnv.DEPLOYMENT_ENV=${env} \ 50 | --set singleuser.extraEnv.PROJECT_ID=${project_id} \ 51 | --set singleuser.extraEnv.KEY_LOCALTION=${region} \ 52 | --set singleuser.networkPolicy.enabled=false \ 53 | --install $helm_name jupyterhub/jupyterhub \ 54 | --namespace ${k8s_namespace} \ 55 | --version=3.0.3 \ 56 | --values config.yaml 57 | 58 | echo "Deployment Completed." 59 | echo "Try 'kubectl --namespace=$k8s_namespace get service proxy-public' to obtain external IP" -------------------------------------------------------------------------------- /resources/gcp/cluster.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 TikTok Pte. Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | locals { 18 | cluster_name = "dcr-${var.env}-cluster" 19 | node_pool_name = "dcr-${var.env}-node-pool" 20 | } 21 | 22 | # GKE Cluster 23 | resource "google_container_cluster" "dcr_cluster" { 24 | project = var.project_id 25 | name = local.cluster_name 26 | # if use region, each zone will create a node 27 | location = local.zone 28 | # We can't create a cluster with no node pool defined, but we want to only use 29 | # separately managed node pools. So we create the smallest possible default 30 | # node pool and immediately delete it. 31 | deletion_protection = false 32 | remove_default_node_pool = true 33 | enable_l4_ilb_subsetting = true 34 | initial_node_count = 1 35 | workload_identity_config { 36 | workload_pool = "${var.project_id}.svc.id.goog" 37 | } 38 | ip_allocation_policy { 39 | stack_type = "IPV4_IPV6" 40 | } 41 | datapath_provider = "ADVANCED_DATAPATH" 42 | network = google_compute_network.data_clean_room_network.self_link 43 | subnetwork = google_compute_subnetwork.data_clean_room_subnetwork.self_link 44 | } 45 | 46 | 47 | # Note pool for GKE cluster 48 | resource "google_container_node_pool" "dcr_node_pool" { 49 | project = var.project_id 50 | name = local.node_pool_name 51 | location = local.zone 52 | cluster = google_container_cluster.dcr_cluster.name 53 | node_count = var.num_nodes 54 | 55 | node_config { 56 | service_account = google_service_account.gcp_dcr_cluster_sa.email 57 | preemptible = true 58 | machine_type = var.type 59 | } 60 | 61 | depends_on = [ 62 | google_service_account.gcp_dcr_cluster_sa, 63 | ] 64 | } 65 | -------------------------------------------------------------------------------- /resources/kubernetes/.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/hashicorp/google" { 5 | version = "5.30.0" 6 | hashes = [ 7 | "h1:tFWLBCrtm/kDPbpm1HnXpzJ4dNpEnGGXfHDZkhYzdqc=", 8 | "zh:13169e55958e932aefa03551b63a3c951f18942be048205dc463c33089db232f", 9 | "zh:5093845d3150ccc3d99c4f4958679aa1a2cc53e5871620b1ac3dc59c6f72e1d8", 10 | "zh:8098f3253d72b5e591d55685234584861738b20fe57ab3124156b2233a71ee48", 11 | "zh:816fdb667d6dca429270e09531895c5c912ac5071c7a315ba011488cbf32a38e", 12 | "zh:880c38683c0b5f6c9d5314e55c29ddbacede4f1b7e18bebd2bed563756362c42", 13 | "zh:881ae2de266958118745a51331ca5308b4f47648e318d679dddf35c6422b292d", 14 | "zh:8d1f46aa7cf70fd0de222874ecdbef8587a10c9231aab0e319bf8095863b58d0", 15 | "zh:a4c9280a6126ea7a5350a7e3d46d7dcdff440d1bcf401f72c6392b7976400d73", 16 | "zh:bcae1f498b2c6d43aba5f6a755969d6e1108c82a77648550b5080d25ea56aabe", 17 | "zh:ce4e0f905fe19249ad7cb01123de9a3be82b0310937f8e872e85ba4d70aef031", 18 | "zh:d528cdd349ed7142d2288a4b9e03a5d3c1d1b18c9213bc21eb4cc03eca5c7ab2", 19 | "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", 20 | ] 21 | } 22 | 23 | provider "registry.terraform.io/hashicorp/kubernetes" { 24 | version = "2.30.0" 25 | hashes = [ 26 | "h1:wRVWY3sK32BNInDOlQnoGSmL638f3jjLFypCAotwpc8=", 27 | "zh:06531333a72fe6d2829f37a328e08a3fc4ed66226344a003b62418a834ac6c69", 28 | "zh:34480263939ef5007ce65c9f4945df5cab363f91e5260ae552bcd9f2ffeed444", 29 | "zh:59e71f9177da570c33507c44828288264c082d512138c5755800f2cd706c62bc", 30 | "zh:6e979b0c07326f9c8d1999096a920322d22261ca61d346b3a9775283d00a2fa5", 31 | "zh:73e3f228de0077b5c0a84ec5b1ada507fbb3456cba35a6b5758723f77715b7af", 32 | "zh:79e0de985159c056f001cc47a654620d51f5d55f554bcbcde1fe7d52f667db40", 33 | "zh:8accb9100f609377db42e3ced42cc9d5c36065a06644dfb21d3893bb8d4797fd", 34 | "zh:9f99aa0bf5caa4223a7dbf5d22d71c16083e782c4eea4b0130abfd6e6f1cec18", 35 | "zh:bcb2ad76ad05ec23f8da62231a2360d1f70bbcd28abd06b8458a9e2f17da7873", 36 | "zh:bce317d7790c2d3c4e724726dc78070db28daf7d861faa646fc891fe28842a29", 37 | "zh:ed0a8e7fa8a1c419a19840b421d18200c3a63cf16ccbcbc400cb375d5397f615", 38 | "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", 39 | ] 40 | } 41 | -------------------------------------------------------------------------------- /deployment/data-clean-room/templates/ingress.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.ingress.enabled -}} 2 | {{- $fullName := include "data-clean-room-chart.fullname" . -}} 3 | {{- $svcPort := .Values.service.port -}} 4 | {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} 5 | {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} 6 | {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} 7 | {{- end }} 8 | {{- end }} 9 | {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} 10 | apiVersion: networking.k8s.io/v1 11 | {{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} 12 | apiVersion: networking.k8s.io/v1beta1 13 | {{- else -}} 14 | apiVersion: extensions/v1beta1 15 | {{- end }} 16 | kind: Ingress 17 | metadata: 18 | name: {{ $fullName }} 19 | labels: 20 | {{- include "data-clean-room-chart.labels" . | nindent 4 }} 21 | {{- with .Values.ingress.annotations }} 22 | annotations: 23 | {{- toYaml . | nindent 4 }} 24 | {{- end }} 25 | spec: 26 | {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} 27 | ingressClassName: {{ .Values.ingress.className }} 28 | {{- end }} 29 | {{- if .Values.ingress.tls }} 30 | tls: 31 | {{- range .Values.ingress.tls }} 32 | - hosts: 33 | {{- range .hosts }} 34 | - {{ . | quote }} 35 | {{- end }} 36 | secretName: {{ .secretName }} 37 | {{- end }} 38 | {{- end }} 39 | rules: 40 | {{- range .Values.ingress.hosts }} 41 | - host: {{ .host | quote }} 42 | http: 43 | paths: 44 | {{- range .paths }} 45 | - path: {{ .path }} 46 | {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} 47 | pathType: {{ .pathType }} 48 | {{- end }} 49 | backend: 50 | {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} 51 | service: 52 | name: {{ $fullName }} 53 | port: 54 | number: {{ $svcPort }} 55 | {{- else }} 56 | serviceName: {{ $fullName }} 57 | servicePort: {{ $svcPort }} 58 | {{- end }} 59 | {{- end }} 60 | {{- end }} 61 | {{- end }} 62 | -------------------------------------------------------------------------------- /app/jupyterlab_manatee/.dockerignore: -------------------------------------------------------------------------------- 1 | *.bundle.* 2 | lib/ 3 | node_modules/ 4 | *.log 5 | .eslintcache 6 | .stylelintcache 7 | *.egg-info/ 8 | .ipynb_checkpoints 9 | *.tsbuildinfo 10 | labextension 11 | # Version file is handled by hatchling 12 | jupyterlab_manatee/_version.py 13 | 14 | # Integration tests 15 | ui-tests/test-results/ 16 | ui-tests/playwright-report/ 17 | 18 | # Created by https://www.gitignore.io/api/python 19 | # Edit at https://www.gitignore.io/?templates=python 20 | 21 | ### Python ### 22 | # Byte-compiled / optimized / DLL files 23 | __pycache__/ 24 | *.py[cod] 25 | *$py.class 26 | 27 | # C extensions 28 | *.so 29 | 30 | # Distribution / packaging 31 | .Python 32 | build/ 33 | develop-eggs/ 34 | dist/ 35 | downloads/ 36 | eggs/ 37 | .eggs/ 38 | lib/ 39 | lib64/ 40 | parts/ 41 | sdist/ 42 | var/ 43 | wheels/ 44 | pip-wheel-metadata/ 45 | share/python-wheels/ 46 | .installed.cfg 47 | *.egg 48 | MANIFEST 49 | 50 | # PyInstaller 51 | # Usually these files are written by a python script from a template 52 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 53 | *.manifest 54 | *.spec 55 | 56 | # Installer logs 57 | pip-log.txt 58 | pip-delete-this-directory.txt 59 | 60 | # Unit test / coverage reports 61 | htmlcov/ 62 | .tox/ 63 | .nox/ 64 | .coverage 65 | .coverage.* 66 | .cache 67 | nosetests.xml 68 | coverage/ 69 | coverage.xml 70 | *.cover 71 | .hypothesis/ 72 | .pytest_cache/ 73 | 74 | # Translations 75 | *.mo 76 | *.pot 77 | 78 | # Scrapy stuff: 79 | .scrapy 80 | 81 | # Sphinx documentation 82 | docs/_build/ 83 | 84 | # PyBuilder 85 | target/ 86 | 87 | # pyenv 88 | .python-version 89 | 90 | # celery beat schedule file 91 | celerybeat-schedule 92 | 93 | # SageMath parsed files 94 | *.sage.py 95 | 96 | # Spyder project settings 97 | .spyderproject 98 | .spyproject 99 | 100 | # Rope project settings 101 | .ropeproject 102 | 103 | # Mr Developer 104 | .mr.developer.cfg 105 | .project 106 | .pydevproject 107 | 108 | # mkdocs documentation 109 | /site 110 | 111 | # mypy 112 | .mypy_cache/ 113 | .dmypy.json 114 | dmypy.json 115 | 116 | # Pyre type checker 117 | .pyre/ 118 | 119 | # End of https://www.gitignore.io/api/python 120 | 121 | # OSX files 122 | .DS_Store 123 | 124 | # Yarn cache 125 | .yarn/ 126 | 127 | cheat-sheet.md 128 | *.yml 129 | !.yarnrc.yml 130 | build_pkg.sh -------------------------------------------------------------------------------- /app/jupyterlab_manatee/.gitignore: -------------------------------------------------------------------------------- 1 | *.bundle.* 2 | lib/ 3 | node_modules/ 4 | *.log 5 | .eslintcache 6 | .stylelintcache 7 | *.egg-info/ 8 | .ipynb_checkpoints 9 | *.tsbuildinfo 10 | jupyterlab_manatee/labextension 11 | # Version file is handled by hatchling 12 | jupyterlab_manatee/_version.py 13 | 14 | # Integration tests 15 | ui-tests/test-results/ 16 | ui-tests/playwright-report/ 17 | 18 | # Created by https://www.gitignore.io/api/python 19 | # Edit at https://www.gitignore.io/?templates=python 20 | 21 | ### Python ### 22 | # Byte-compiled / optimized / DLL files 23 | __pycache__/ 24 | *.py[cod] 25 | *$py.class 26 | 27 | # C extensions 28 | *.so 29 | 30 | # Distribution / packaging 31 | .Python 32 | build/ 33 | develop-eggs/ 34 | dist/ 35 | downloads/ 36 | eggs/ 37 | .eggs/ 38 | lib/ 39 | lib64/ 40 | parts/ 41 | sdist/ 42 | var/ 43 | wheels/ 44 | pip-wheel-metadata/ 45 | share/python-wheels/ 46 | .installed.cfg 47 | *.egg 48 | MANIFEST 49 | 50 | # PyInstaller 51 | # Usually these files are written by a python script from a template 52 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 53 | *.manifest 54 | *.spec 55 | 56 | # Installer logs 57 | pip-log.txt 58 | pip-delete-this-directory.txt 59 | 60 | # Unit test / coverage reports 61 | htmlcov/ 62 | .tox/ 63 | .nox/ 64 | .coverage 65 | .coverage.* 66 | .cache 67 | nosetests.xml 68 | coverage/ 69 | coverage.xml 70 | *.cover 71 | .hypothesis/ 72 | .pytest_cache/ 73 | 74 | # Translations 75 | *.mo 76 | *.pot 77 | 78 | # Scrapy stuff: 79 | .scrapy 80 | 81 | # Sphinx documentation 82 | docs/_build/ 83 | 84 | # PyBuilder 85 | target/ 86 | 87 | # pyenv 88 | .python-version 89 | 90 | # celery beat schedule file 91 | celerybeat-schedule 92 | 93 | # SageMath parsed files 94 | *.sage.py 95 | 96 | # Spyder project settings 97 | .spyderproject 98 | .spyproject 99 | 100 | # Rope project settings 101 | .ropeproject 102 | 103 | # Mr Developer 104 | .mr.developer.cfg 105 | .project 106 | .pydevproject 107 | 108 | # mkdocs documentation 109 | /site 110 | 111 | # mypy 112 | .mypy_cache/ 113 | .dmypy.json 114 | dmypy.json 115 | 116 | # Pyre type checker 117 | .pyre/ 118 | 119 | # End of https://www.gitignore.io/api/python 120 | 121 | # OSX files 122 | .DS_Store 123 | 124 | # Yarn cache 125 | .yarn/ 126 | 127 | cheat-sheet.md 128 | *.yml 129 | !.yarnrc.yml 130 | build_pkg.sh -------------------------------------------------------------------------------- /app/dcr_tee/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2024 TikTok Pte. Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -e 16 | VAR_FILE="../../.env" 17 | if [ ! -f "$VAR_FILE" ]; then 18 | echo "Error: Variables file does not exist." 19 | exit 1 20 | fi 21 | 22 | VAR_FILE=$(realpath $VAR_FILE) 23 | source $VAR_FILE 24 | 25 | zone=$region-a 26 | 27 | PKG_SOURCE_CODE="../../pkg" 28 | if [ ! -d "$PKG_SOURCE_CODE" ]; then 29 | echo "Error: Pkg soruce codes don't exist." 30 | exit 1 31 | fi 32 | 33 | rm -rf github.com 34 | mkdir -p github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/ 35 | cp -r $PKG_SOURCE_CODE github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pkg@v0.0.1 36 | 37 | cp -r ../conf ./ 38 | if [[ "$OSTYPE" == "darwin"* ]]; then 39 | # macOS 40 | sed -i '' "s/ENV/$env/g" conf/config.yaml 41 | sed -i '' "s/Project: \"\$PROJECTID\"/Project: \"$project_id\"/g" conf/config.yaml 42 | sed -i '' "s/ProjectNumber: \$PROJECTNUMBER/ProjectNumber: $project_number/g" conf/config.yaml 43 | sed -i '' "s/Region: \"\$REGION\"/Region: \"$region\"/g" conf/config.yaml 44 | sed -i '' "s/Zone: \"\$ZONE\"/Zone: \"$zone\"/g" conf/config.yaml 45 | else 46 | sed -i "s/ENV/$env/g" conf/config.yaml 47 | sed -i "s/Project: \"\$PROJECTID\"/Project: \"$project_id\"/g" conf/config.yaml 48 | sed -i "s/ProjectNumber: \$PROJECTNUMBER/ProjectNumber: $project_number/g" conf/config.yaml 49 | sed -i "s/Region: \"\$REGION\"/Region: \"$region\"/g" conf/config.yaml 50 | sed -i "s/Zone: \"\$ZONE\"/Zone: \"$zone\"/g" conf/config.yaml 51 | fi 52 | 53 | # base tee image will be pushed to "dcr-${var.env}-user-images" 54 | user_docker_repo="dcr-$env-user-images" 55 | IMAGE_URL="us-docker.pkg.dev/${project_id}/${user_docker_repo}/data-clean-room-base" 56 | TAG="latest" 57 | docker build --platform linux/amd64 --tag "$IMAGE_URL:$TAG" . 58 | docker push "$IMAGE_URL:$TAG" 59 | -------------------------------------------------------------------------------- /pkg/cloud/provider.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 TikTok Pte. Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cloud 16 | 17 | import ( 18 | "context" 19 | "io" 20 | ) 21 | 22 | const ( 23 | INSTANCE_RUNNING = 1 24 | INSTANCE_TERMINATED = 2 25 | INSTANCE_OTHER = 3 26 | ) 27 | 28 | type Instance struct { 29 | UUID string 30 | Name string 31 | Token string // Token is reserved for later user authentication 32 | Status int 33 | CreationTime string 34 | } 35 | 36 | type CloudProvider interface { 37 | // cloud storage 38 | DownloadFile(remoteSrcPath string, localDestPath string) error 39 | ListFiles(remoteDir string) ([]string, error) 40 | GetFileSize(remotePath string) (int64, error) 41 | GetFilebyChunk(remotePath string, offset int64, chunkSize int64) ([]byte, error) 42 | DeleteFile(remotePath string) error 43 | UploadFile(fileReader io.Reader, remotePath string, compress bool) error 44 | // KMS 45 | CreateSymmetricKeys(keyId string) error 46 | CheckIfKeyExists(keyId string) (bool, error) 47 | EncryptWithKMS(keyId string, plaintext string) (string, error) 48 | DecryptWithKMS(keyId string, ciphertextB64 string) (string, error) 49 | GrantServiceAccountKeyRole(serviceAccount string, keyId string, role string) error 50 | // workload identity pool 51 | CreateWorkloadIdentityPoolProvider(wipName string) error 52 | UpdateWorkloadIdentityPoolProvider(wipName string, imageDigest string) error 53 | // compute engine 54 | GetServiceAccountEmail() (string, error) 55 | // instance 56 | ListAllInstances() ([]*Instance, error) 57 | DeleteInstance(instanceName string) error 58 | // confidential space 59 | CreateConfidentialSpace(instanceName string, dockerImage string, stage1Token string, uuid string) error 60 | PrepareResourcesForUser(userName string) error 61 | } 62 | 63 | func GetCloudProvider(ctx context.Context) CloudProvider { 64 | return NewGcpService(ctx) 65 | } 66 | -------------------------------------------------------------------------------- /app/dcr_api/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2024 TikTok Pte. Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -e 16 | 17 | VAR_FILE="../../.env" 18 | if [ ! -f "$VAR_FILE" ]; then 19 | echo "Error: Variables file does not exist." 20 | exit 1 21 | fi 22 | 23 | VAR_FILE=$(realpath $VAR_FILE) 24 | source $VAR_FILE 25 | 26 | zone=$region-a 27 | 28 | PKG_SOURCE_CODE="../../pkg" 29 | if [ ! -d "$PKG_SOURCE_CODE" ]; then 30 | echo "Error: Pkg soruce codes don't exist." 31 | exit 1 32 | fi 33 | 34 | rm -rf github.com 35 | mkdir -p github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/ 36 | cp -r $PKG_SOURCE_CODE github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pkg@v0.0.1 37 | 38 | cp -r ../conf ./ 39 | if [[ "$OSTYPE" == "darwin"* ]]; then 40 | # macOS 41 | sed -i '' "s/ENV/$env/g" conf/config.yaml 42 | sed -i '' "s/Project: \"\$PROJECTID\"/Project: \"$project_id\"/g" conf/config.yaml 43 | sed -i '' "s/ProjectNumber: \$PROJECTNUMBER/ProjectNumber: $project_number/g" conf/config.yaml 44 | sed -i '' "s/Region: \"\$REGION\"/Region: \"$region\"/g" conf/config.yaml 45 | sed -i '' "s/Zone: \"\$ZONE\"/Zone: \"$zone\"/g" conf/config.yaml 46 | else 47 | sed -i "s/ENV/$env/g" conf/config.yaml 48 | sed -i "s/Project: \"\$PROJECTID\"/Project: \"$project_id\"/g" conf/config.yaml 49 | sed -i "s/ProjectNumber: \$PROJECTNUMBER/ProjectNumber: $project_number/g" conf/config.yaml 50 | sed -i "s/Region: \"\$REGION\"/Region: \"$region\"/g" conf/config.yaml 51 | sed -i "s/Zone: \"\$ZONE\"/Zone: \"$zone\"/g" conf/config.yaml 52 | fi 53 | 54 | hz update --model_dir biz/model -idl idl/job.thrift 55 | 56 | docker_repo="dcr-${env}-images" 57 | image_url="us-docker.pkg.dev/${project_id}/${docker_repo}/data-clean-room-api" 58 | if [ -z "$username" ]; then 59 | tag=$env 60 | else 61 | tag=$username 62 | fi 63 | docker build --platform linux/amd64 --tag "$image_url:$tag" . 64 | docker push "$image_url:$tag" 65 | -------------------------------------------------------------------------------- /deployment/data-clean-room/templates/cronjob.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1 2 | kind: CronJob 3 | metadata: 4 | name: {{ printf "%s-monitor" (include "data-clean-room-chart.fullname" .) | quote }} 5 | labels: 6 | {{- include "data-clean-room-chart.labels" . | nindent 4 }} 7 | namespace: {{ .Values.namespace }} 8 | spec: 9 | schedule: "{{ .Values.schedule }}" 10 | successfulJobsHistoryLimit: 3 11 | failedJobsHistoryLimit: 3 12 | jobTemplate: 13 | spec: 14 | template: 15 | metadata: 16 | {{- with .Values.podAnnotations }} 17 | annotations: 18 | {{- toYaml . | nindent 12 }} 19 | {{- end }} 20 | labels: 21 | {{- include "data-clean-room-chart.labels" . | nindent 12 }} 22 | {{- with .Values.podLabels }} 23 | {{- toYaml . | nindent 12 }} 24 | {{- end }} 25 | spec: 26 | {{- with .Values.imagePullSecrets }} 27 | imagePullSecrets: 28 | {{- toYaml . | nindent 12 }} 29 | {{- end }} 30 | serviceAccountName: {{ include "data-clean-room-chart.serviceAccountName" . }} 31 | securityContext: 32 | {{- toYaml .Values.podSecurityContext | nindent 12 }} 33 | restartPolicy: "Never" 34 | containers: 35 | - name: {{ .Chart.Name }} 36 | securityContext: 37 | {{- toYaml .Values.securityContext | nindent 16 }} 38 | image: "{{ .Values.monitorImage.repository }}:{{ .Values.monitorImage.tag | default .Chart.AppVersion }}" 39 | imagePullPolicy: {{ .Values.monitorImage.pullPolicy }} 40 | resources: 41 | {{- toYaml .Values.resources | nindent 16 }} 42 | env: 43 | - name: DATA_CLEAN_ROOM_HOST 44 | value: {{ printf "http://%s.%s.svc.cluster.local" (include "data-clean-room-chart.fullname" .) .Values.namespace | quote }} 45 | {{- with .Values.volumeMounts }} 46 | volumeMounts: 47 | {{- toYaml . | nindent 16 }} 48 | {{- end }} 49 | {{- with .Values.volumes }} 50 | volumes: 51 | {{- toYaml . | nindent 12 }} 52 | {{- end }} 53 | {{- with .Values.nodeSelector }} 54 | nodeSelector: 55 | {{- toYaml . | nindent 12 }} 56 | {{- end }} 57 | {{- with .Values.affinity }} 58 | affinity: 59 | {{- toYaml . | nindent 12 }} 60 | {{- end }} 61 | {{- with .Values.tolerations }} 62 | tolerations: 63 | {{- toYaml . | nindent 12 }} 64 | {{- end }} -------------------------------------------------------------------------------- /app/jupyterlab_manatee/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling>=1.5.0", "jupyterlab>=4.0.0,<5", "hatch-nodejs-version>=0.3.2", "aiohttp"] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "jupyterlab_manatee" 7 | readme = "README.md" 8 | license = { file = "LICENSE" } 9 | requires-python = ">=3.8" 10 | classifiers = [ 11 | "Framework :: Jupyter", 12 | "Framework :: Jupyter :: JupyterLab", 13 | "Framework :: Jupyter :: JupyterLab :: 4", 14 | "Framework :: Jupyter :: JupyterLab :: Extensions", 15 | "Framework :: Jupyter :: JupyterLab :: Extensions :: Prebuilt", 16 | "License :: OSI Approved :: BSD License", 17 | "Programming Language :: Python", 18 | "Programming Language :: Python :: 3", 19 | "Programming Language :: Python :: 3.8", 20 | "Programming Language :: Python :: 3.9", 21 | "Programming Language :: Python :: 3.10", 22 | "Programming Language :: Python :: 3.11", 23 | "Programming Language :: Python :: 3.12", 24 | ] 25 | dependencies = [ 26 | "aiohttp", 27 | "tornado>=6.3", 28 | ] 29 | dynamic = ["version", "description", "authors", "urls", "keywords"] 30 | 31 | [tool.hatch.version] 32 | source = "nodejs" 33 | 34 | [tool.hatch.metadata.hooks.nodejs] 35 | fields = ["description", "authors", "urls"] 36 | 37 | [tool.hatch.build.targets.sdist] 38 | artifacts = ["jupyterlab_manatee/labextension"] 39 | exclude = [".github", "binder"] 40 | 41 | [tool.hatch.build.targets.wheel.shared-data] 42 | "jupyter-config/jupyter_server_config.d" = "etc/jupyter/jupyter_server_config.d" 43 | "jupyterlab_manatee/labextension" = "share/jupyter/labextensions/jupyterlab_manatee" 44 | "install.json" = "share/jupyter/labextensions/jupyterlab_manatee/install.json" 45 | 46 | [tool.hatch.build.hooks.version] 47 | path = "jupyterlab_manatee/_version.py" 48 | 49 | [tool.hatch.build.hooks.jupyter-builder] 50 | dependencies = ["hatch-jupyter-builder>=0.5"] 51 | build-function = "hatch_jupyter_builder.npm_builder" 52 | ensured-targets = [ 53 | "jupyterlab_manatee/labextension/static/style.js", 54 | "jupyterlab_manatee/labextension/package.json", 55 | ] 56 | skip-if-exists = ["jupyterlab_manatee/labextension/static/style.js"] 57 | 58 | [tool.hatch.build.hooks.jupyter-builder.build-kwargs] 59 | build_cmd = "build:prod" 60 | npm = ["jlpm"] 61 | 62 | [tool.hatch.build.hooks.jupyter-builder.editable-build-kwargs] 63 | build_cmd = "install:extension" 64 | npm = ["jlpm"] 65 | source_dir = "src" 66 | build_dir = "jupyterlab_manatee/labextension" 67 | 68 | [tool.jupyter-releaser.options] 69 | version_cmd = "hatch version" 70 | 71 | [tool.jupyter-releaser.hooks] 72 | before-build-npm = [ 73 | "python -m pip install 'jupyterlab>=4.0.0,<5'", 74 | "jlpm", 75 | "jlpm build:prod" 76 | ] 77 | before-build-python = ["jlpm clean:all"] 78 | 79 | [tool.check-wheel-contents] 80 | ignore = ["W002"] 81 | -------------------------------------------------------------------------------- /app/dcr_monitor/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2024 TikTok Pte. Ltd. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -e 16 | 17 | VAR_FILE="../../.env" 18 | if [ ! -f "$VAR_FILE" ]; then 19 | echo "Error: Variables file does not exist." 20 | exit 1 21 | fi 22 | 23 | VAR_FILE=$(realpath $VAR_FILE) 24 | source $VAR_FILE 25 | 26 | zone=$region-a 27 | 28 | PKG_SOURCE_CODE="../../pkg" 29 | PKG_SOURCE_CODE=$(realpath $PKG_SOURCE_CODE) 30 | if [ ! -d "$PKG_SOURCE_CODE" ]; then 31 | echo "Error: Pkg soruce codes don't exist." 32 | exit 1 33 | fi 34 | 35 | rm -rf github.com 36 | mkdir -p github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/ 37 | cp -r $PKG_SOURCE_CODE github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pkg@v0.0.1 38 | 39 | API_SOURCE_CODE="../dcr_api" 40 | if [ ! -d "$API_SOURCE_CODE" ]; then 41 | echo "Error: Api soruce codes don't exist." 42 | exit 1 43 | fi 44 | mkdir -p github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/app 45 | cp -r $API_SOURCE_CODE github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/app/dcr_api@v0.0.1 46 | 47 | cp -r ../conf ./ 48 | if [[ "$OSTYPE" == "darwin"* ]]; then 49 | # macOS 50 | sed -i '' "s/ENV/$env/g" conf/config.yaml 51 | sed -i '' "s/Project: \"\$PROJECTID\"/Project: \"$project_id\"/g" conf/config.yaml 52 | sed -i '' "s/ProjectNumber: \$PROJECTNUMBER/ProjectNumber: $project_number/g" conf/config.yaml 53 | sed -i '' "s/Region: \"\$REGION\"/Region: \"$region\"/g" conf/config.yaml 54 | sed -i '' "s/Zone: \"\$ZONE\"/Zone: \"$zone\"/g" conf/config.yaml 55 | else 56 | sed -i "s/ENV/$env/g" conf/config.yaml 57 | sed -i "s/Project: \"\$PROJECTID\"/Project: \"$project_id\"/g" conf/config.yaml 58 | sed -i "s/ProjectNumber: \$PROJECTNUMBER/ProjectNumber: $project_number/g" conf/config.yaml 59 | sed -i "s/Region: \"\$REGION\"/Region: \"$region\"/g" conf/config.yaml 60 | sed -i "s/Zone: \"\$ZONE\"/Zone: \"$zone\"/g" conf/config.yaml 61 | fi 62 | 63 | docker_repo="dcr-${env}-images" 64 | echo $docker_repo 65 | image_url="us-docker.pkg.dev/${project_id}/${docker_repo}/data-clean-room-monitor" 66 | if [ -z "$username" ]; then 67 | tag=$env 68 | else 69 | tag=$username 70 | fi 71 | docker build --platform linux/amd64 --tag "$image_url:$tag" . 72 | docker push "$image_url:$tag" 73 | -------------------------------------------------------------------------------- /app/dcr_tee/attestation/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 TikTok Pte. Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "encoding/json" 20 | "flag" 21 | "fmt" 22 | "io" 23 | "log" 24 | "net" 25 | "net/http" 26 | "os" 27 | "strings" 28 | 29 | "github.com/pkg/errors" 30 | ) 31 | 32 | const TikTokAudience = "https://research.tiktok.com/" 33 | const TokenFilename = "custom_token" 34 | 35 | type CustomToken struct { 36 | Audience string `json:"audience"` 37 | Nonces []string `json:"nonces"` // each nonce must be min 64bits 38 | TokenType string `json:"token_type"` 39 | } 40 | 41 | func generateCustomAttestationToken(nonce string) ([]byte, error) { 42 | request := CustomToken{ 43 | Audience: TikTokAudience, 44 | Nonces: []string{nonce}, 45 | TokenType: "OIDC", 46 | } 47 | httpClient := http.Client{ 48 | Transport: &http.Transport{ 49 | DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { 50 | return net.Dial("unix", "/run/container_launcher/teeserver.sock") 51 | }, 52 | }, 53 | } 54 | customJSON, err := json.Marshal(request) 55 | if err != nil { 56 | return nil, errors.Wrap(err, "failed to marshal request") 57 | } 58 | url := "http://localhost/v1/token" 59 | resp, err := httpClient.Post(url, "application/json", strings.NewReader(string(customJSON))) 60 | if err != nil { 61 | return nil, errors.Wrap(err, "faile to get custom token") 62 | } 63 | defer resp.Body.Close() 64 | tokenbytes, err := io.ReadAll(resp.Body) 65 | if err != nil { 66 | return nil, errors.Wrap(err, "faile to read from response") 67 | } 68 | 69 | return tokenbytes, nil 70 | } 71 | 72 | func requireParameter(name string, para string) { 73 | if para == "" { 74 | fmt.Printf("ERROR: %s parameter is required \n", name) 75 | flag.PrintDefaults() 76 | os.Exit(1) 77 | } 78 | } 79 | 80 | func main() { 81 | nonce := flag.String("nonce", "", "The nonce to generate custom token") 82 | flag.Parse() 83 | requireParameter("nonce", *nonce) 84 | customToken, err := generateCustomAttestationToken(*nonce) 85 | if err != nil { 86 | fmt.Printf("ERROR: failed to generate custom token %+v \n", err) 87 | panic(err) 88 | } 89 | 90 | err = os.WriteFile(TokenFilename, customToken, 0644) 91 | if err != nil { 92 | fmt.Printf("ERROR: failed to write custom token to file %+v \n", err) 93 | log.Fatal(err) 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /app/dcr_tee/encryption/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "os" 8 | 9 | kms "cloud.google.com/go/kms/apiv1" 10 | "cloud.google.com/go/kms/apiv1/kmspb" 11 | "google.golang.org/api/option" 12 | 13 | "github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pkg/config" 14 | ) 15 | 16 | const credentialConfig = `{ 17 | "type": "external_account", 18 | "audience": "//iam.googleapis.com/%s", 19 | "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", 20 | "token_url": "https://sts.googleapis.com/v1/token", 21 | "credential_source": { 22 | "file": "/run/container_launcher/attestation_verifier_claims_token" 23 | }, 24 | "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/%s:generateAccessToken" 25 | }` 26 | 27 | func requireParameter(name string, para string) { 28 | if para == "" { 29 | fmt.Printf("ERROR: %s parameter is required \n", name) 30 | flag.PrintDefaults() 31 | os.Exit(1) 32 | } 33 | } 34 | 35 | func encryptBytes(ctx context.Context, keyName string, trustedServiceAccountEmail string, wippro string, sourceData []byte) ([]byte, error) { 36 | cc := fmt.Sprintf(credentialConfig, wippro, trustedServiceAccountEmail) 37 | kmsClient, err := kms.NewKeyManagementClient(ctx, option.WithCredentialsJSON([]byte(cc))) 38 | if err != nil { 39 | return nil, fmt.Errorf("creating a new KMS client with federated credentials: %w", err) 40 | } 41 | 42 | encryptRequest := &kmspb.EncryptRequest{ 43 | Name: keyName, 44 | Plaintext: sourceData, 45 | } 46 | encryptResponse, err := kmsClient.Encrypt(ctx, encryptRequest) 47 | if err != nil { 48 | return nil, fmt.Errorf("could not encrypt source data: %w", err) 49 | } 50 | 51 | return encryptResponse.Ciphertext, nil 52 | } 53 | 54 | func main() { 55 | err := config.InitConfig() 56 | if err != nil { 57 | fmt.Printf("ERROR: failed to init config %+v \n", err) 58 | return 59 | } 60 | 61 | user := flag.String("user", "", "The user who submits the job") 62 | inputFileName := flag.String("input", "", "The input file to be encrypted") 63 | outputFileName := flag.String("output", "", "The encrypted file") 64 | impersonationServiceAccount := flag.String("impersonation", "", "The impersonation service account it used") 65 | flag.Parse() 66 | requireParameter("user", *user) 67 | requireParameter("input", *inputFileName) 68 | requireParameter("output", *outputFileName) 69 | requireParameter("impersonation", *outputFileName) 70 | keyName := config.GetKeyFullName(config.GetUserKey(*user)) 71 | wipProvider := config.GetWipProviderFullName(config.GetUserWipProvider(*user)) 72 | 73 | sourceData, err := os.ReadFile(*inputFileName) 74 | if err != nil { 75 | fmt.Printf("ERROR: failed to read from input file %+v \n", err) 76 | return 77 | } 78 | 79 | encryptedData, err := encryptBytes(context.Background(), keyName, *impersonationServiceAccount, wipProvider, sourceData) 80 | if err != nil { 81 | fmt.Printf("ERROR: encrypt output %+v \n", err) 82 | return 83 | } 84 | err = os.WriteFile(*outputFileName, encryptedData, 0644) 85 | if err != nil { 86 | fmt.Printf("ERROR: failed to write output to file %+v", err) 87 | return 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /deployment/data-clean-room/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for data-clean-room-chart. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | 5 | replicaCount: 1 6 | 7 | apiImage: 8 | repository: "" 9 | pullPolicy: Always 10 | # Overrides the image tag whose default is the chart appVersion. 11 | tag: "" 12 | 13 | monitorImage: 14 | repository: "" 15 | pullPolicy: Always 16 | # Overrides the image tag whose default is the chart appVersion. 17 | tag: "" 18 | 19 | 20 | imagePullSecrets: [] 21 | nameOverride: "data-clean-room" 22 | fullnameOverride: "data-clean-room" 23 | 24 | serviceAccount: 25 | # Specifies whether a service account should be created 26 | create: false 27 | # Automatically mount a ServiceAccount's API credentials? 28 | automount: true 29 | # Annotations to add to the service account 30 | annotations: {} 31 | # The name of the service account to use. 32 | # If not set and create is true, a name is generated using the fullname template 33 | name: "" 34 | 35 | podAnnotations: {} 36 | podLabels: {} 37 | 38 | podSecurityContext: {} 39 | # fsGroup: 2000 40 | 41 | securityContext: {} 42 | # capabilities: 43 | # drop: 44 | # - ALL 45 | # readOnlyRootFilesystem: true 46 | # runAsNonRoot: true 47 | # runAsUser: 1000 48 | 49 | service: 50 | type: ClusterIP 51 | port: 80 52 | 53 | api: 54 | port: 8080 55 | 56 | ingress: 57 | enabled: false 58 | className: "" 59 | annotations: {} 60 | # kubernetes.io/ingress.class: nginx 61 | # kubernetes.io/tls-acme: "true" 62 | hosts: 63 | - host: chart-example.local 64 | paths: 65 | - path: / 66 | pathType: ImplementationSpecific 67 | tls: [] 68 | # - secretName: chart-example-tls 69 | # hosts: 70 | # - chart-example.local 71 | 72 | resources: {} 73 | # We usually recommend not to specify default resources and to leave this as a conscious 74 | # choice for the user. This also increases chances charts run on environments with little 75 | # resources, such as Minikube. If you do want to specify resources, uncomment the following 76 | # lines, adjust them as necessary, and remove the curly braces after 'resources:'. 77 | # limits: 78 | # cpu: 100m 79 | # memory: 128Mi 80 | # requests: 81 | # cpu: 100m 82 | # memory: 128Mi 83 | 84 | autoscaling: 85 | enabled: false 86 | minReplicas: 1 87 | maxReplicas: 100 88 | targetCPUUtilizationPercentage: 80 89 | # targetMemoryUtilizationPercentage: 80 90 | 91 | # Additional volumes on the output Deployment definition. 92 | volumes: [] 93 | # - name: foo 94 | # secret: 95 | # secretName: mysecret 96 | # optional: false 97 | 98 | # Additional volumeMounts on the output Deployment definition. 99 | volumeMounts: [] 100 | # - name: foo 101 | # mountPath: "/etc/foo" 102 | # readOnly: true 103 | 104 | nodeSelector: {} 105 | 106 | tolerations: [] 107 | 108 | affinity: {} 109 | 110 | cloudSql: 111 | connection_name: "" 112 | 113 | mysql: 114 | host: "localhost" 115 | port: "9910" 116 | 117 | useMinikube: false 118 | 119 | # every minute 120 | schedule: "*/1 * * * *" 121 | 122 | namespace: "" -------------------------------------------------------------------------------- /app/jupyterlab_manatee/README.md: -------------------------------------------------------------------------------- 1 | # jupyterlab_manatee 2 | 3 | This is an open-source JupyterLab extension for data clean room 4 | 5 | ## Building 6 | 7 | Run the `./build.sh` script to build the jupyterhub single user docker image. When the script finished, the docker image `scipy-notebook-with-dcr` will be pushed to the gcp docker repository. Then you can turn to deploy the jupyterhub in the `deployment` directory. 8 | 9 | ## Requirements 10 | 11 | - JupyterLab >= 4.0.0 12 | 13 | ## Contributing 14 | 15 | ### Development install 16 | 17 | Note: You will need NodeJS to build the extension package. 18 | 19 | The `jlpm` command is JupyterLab's pinned version of 20 | [yarn](https://yarnpkg.com/) that is installed with JupyterLab. You may use 21 | `yarn` or `npm` in lieu of `jlpm` below. 22 | 23 | ```bash 24 | # Clone the repo to your local environment 25 | # Change directory to the jupyterlab_manatee directory 26 | # Install package in development mode 27 | pip install -e "." 28 | # Link your development version of the extension with JupyterLab 29 | jupyter labextension develop . --overwrite 30 | # Rebuild extension Typescript source after making changes 31 | jlpm build 32 | ``` 33 | 34 | You can watch the source directory and run JupyterLab at the same time in different terminals to watch for changes in the extension's source and automatically rebuild the extension. 35 | 36 | ```bash 37 | # Watch the source directory in one terminal, automatically rebuilding when needed 38 | jlpm watch 39 | # Run JupyterLab in another terminal 40 | jupyter lab 41 | ``` 42 | 43 | With the watch command running, every saved change will immediately be built locally and available in your running JupyterLab. Refresh JupyterLab to load the change in your browser (you may need to wait several seconds for the extension to be rebuilt). 44 | 45 | By default, the `jlpm build` command generates the source maps for this extension to make it easier to debug using the browser dev tools. To also generate source maps for the JupyterLab core extensions, you can run the following command: 46 | 47 | ```bash 48 | jupyter lab build --minimize=False 49 | ``` 50 | 51 | ### Development uninstall 52 | 53 | ```bash 54 | pip uninstall jupyterlab_manatee 55 | ``` 56 | 57 | In development mode, you will also need to remove the symlink created by `jupyter labextension develop` 58 | command. To find its location, you can run `jupyter labextension list` to figure out where the `labextensions` 59 | folder is located. Then you can remove the symlink named `jupyterlab_manatee` within that folder. 60 | 61 | ### Testing the extension 62 | 63 | #### Frontend tests 64 | 65 | This extension is using [Jest](https://jestjs.io/) for JavaScript code testing. 66 | 67 | To execute them, execute: 68 | 69 | ```sh 70 | jlpm 71 | jlpm test 72 | ``` 73 | 74 | #### Integration tests 75 | 76 | This extension uses [Playwright](https://playwright.dev/docs/intro) for the integration tests (aka user level tests). 77 | More precisely, the JupyterLab helper [Galata](https://github.com/jupyterlab/jupyterlab/tree/master/galata) is used to handle testing the extension in JupyterLab. 78 | 79 | More information are provided within the [ui-tests](./ui-tests/README.md) README. 80 | 81 | ### Packaging the extension 82 | 83 | See [RELEASE](RELEASE.md) -------------------------------------------------------------------------------- /app/dcr_monitor/monitor/instance.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 TikTok Pte. Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package monitor 16 | 17 | import ( 18 | "context" 19 | "strings" 20 | "time" 21 | 22 | "github.com/cloudwego/hertz/pkg/common/hlog" 23 | 24 | "github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/app/dcr_api/biz/model/job" 25 | "github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pkg/cloud" 26 | ) 27 | 28 | func getCreator(token string) (string, error) { 29 | parts := strings.Split(token, "-") 30 | return parts[0], nil 31 | } 32 | 33 | func CheckTeeInstance(ctx context.Context) error { 34 | hlog.Info("[InstancesMonitor] start to monitor instances' status") 35 | provider := cloud.GetCloudProvider(ctx) 36 | list, err := provider.ListAllInstances() 37 | if err != nil { 38 | hlog.Errorf("[InstancesMonitor] failed to query instances' status %+v", err) 39 | return err 40 | } 41 | for _, instance := range list { 42 | hlog.Debugf("[InstancesMonitor] instance status: %v", instance) 43 | token := instance.Token 44 | status := instance.Status 45 | UUID := instance.UUID 46 | if UUID == "" { 47 | continue 48 | } 49 | creator, err := getCreator(instance.Name) 50 | if err != nil { 51 | hlog.Errorf("[InstancesMonitor] failed to get creator info %+v", err) 52 | continue 53 | } 54 | timeCreation := instance.CreationTime 55 | layout := "2006-01-02T15:04:05.999999999Z07:00" 56 | formattedTimeCreation, err := time.Parse(layout, timeCreation) 57 | jobStuck := time.Since(formattedTimeCreation) > 6*time.Hour 58 | 59 | if status == cloud.INSTANCE_TERMINATED { 60 | err = updateTeeInstanceStatus(creator, UUID, token, int64(job.JobStatus_VMFinished)) 61 | if err != nil { 62 | return err 63 | } 64 | hlog.Info("[InstancesMonitor]Successfully updated the instance status to finished") 65 | err := provider.DeleteInstance(instance.Name) 66 | if err != nil { 67 | return err 68 | } 69 | 70 | } 71 | if jobStuck { 72 | hlog.Infof("[InstancesMonitor] job %v has benn stucked more than 6 hours", UUID) 73 | err = updateTeeInstanceStatus(creator, UUID, token, int64(job.JobStatus_VMFinished)) 74 | if err != nil { 75 | return err 76 | } 77 | return provider.DeleteInstance(instance.Name) 78 | } 79 | } 80 | return nil 81 | } 82 | 83 | func convertTokenToStage1(stage2Token string) (string, error) { 84 | // TODO add authentication 85 | return stage2Token, nil 86 | } 87 | 88 | func updateTeeInstanceStatus(creator, UUID, token string, status int64) error { 89 | stage1Token, err := convertTokenToStage1(token) 90 | if err != nil { 91 | return err 92 | } 93 | err = updateJobStatus(creator, UUID, stage1Token, "", status) 94 | return err 95 | } 96 | -------------------------------------------------------------------------------- /resources/kubernetes/service_accounts.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 TikTok Pte. Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | locals { 18 | gcp_dcr_pod_sa_email = "${local.gcp_dcr_pod_sa}@${var.project_id}.iam.gserviceaccount.com" 19 | gcp_jupyter_pod_sa_email = "${local.gcp_jupyter_pod_sa}@${var.project_id}.iam.gserviceaccount.com" 20 | admin_binding = var.username != "" ? "cluster-admin-binding-${var.env}-${var.username}" : "cluster-admin-binding--${var.env}" 21 | } 22 | 23 | resource "kubernetes_service_account" "k8s_dcr_pod_service_account" { 24 | metadata { 25 | name = "dcr-k8s-pod-sa" 26 | namespace = local.dcr_k8s_namespace 27 | annotations = { 28 | "iam.gke.io/gcp-service-account" = local.gcp_dcr_pod_sa_email 29 | } 30 | } 31 | automount_service_account_token = true 32 | depends_on = [kubernetes_namespace.data_clean_room_k8s_namespace] 33 | } 34 | 35 | resource "kubernetes_service_account" "k8s_jupyter_pod_service_account" { 36 | metadata { 37 | name = "jupyter-k8s-pod-sa" 38 | namespace = local.jupyter_k8s_namespace 39 | 40 | annotations = { 41 | "iam.gke.io/gcp-service-account" = local.gcp_jupyter_pod_sa_email 42 | } 43 | } 44 | automount_service_account_token = true 45 | depends_on = [kubernetes_namespace.jupyterhub_k8s_namespace] 46 | } 47 | 48 | 49 | resource "google_service_account_iam_member" "dcr_pod_sa_iam_member" { 50 | service_account_id = "projects/${var.project_id}/serviceAccounts/${local.gcp_dcr_pod_sa}@${var.project_id}.iam.gserviceaccount.com" 51 | role = "roles/iam.workloadIdentityUser" 52 | member = "serviceAccount:${var.project_id}.svc.id.goog[${local.dcr_k8s_namespace}/${kubernetes_service_account.k8s_dcr_pod_service_account.metadata[0].name}]" 53 | depends_on = [kubernetes_namespace.data_clean_room_k8s_namespace] 54 | } 55 | 56 | resource "google_service_account_iam_member" "jupyter_pod_sa_iam_member" { 57 | service_account_id = "projects/${var.project_id}/serviceAccounts/${local.gcp_jupyter_pod_sa}@${var.project_id}.iam.gserviceaccount.com" 58 | role = "roles/iam.workloadIdentityUser" 59 | member = "serviceAccount:${var.project_id}.svc.id.goog[${local.jupyter_k8s_namespace}/${kubernetes_service_account.k8s_jupyter_pod_service_account.metadata[0].name}]" 60 | depends_on = [kubernetes_namespace.jupyterhub_k8s_namespace] 61 | } 62 | 63 | resource "kubernetes_cluster_role_binding" "cluster_admin_binding" { 64 | metadata { 65 | name = local.admin_binding 66 | } 67 | role_ref { 68 | api_group = "rbac.authorization.k8s.io" 69 | kind = "ClusterRole" 70 | name = "cluster-admin" 71 | } 72 | subject { 73 | kind = "User" 74 | name = data.google_client_openid_userinfo.me.email 75 | api_group = "rbac.authorization.k8s.io" 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to PrivacyGo Data Clean Room 2 | 3 | We happily welcome contributions to the PGDCR. We use [GitHub Issues](https://github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/issues) to track community reported issues and [GitHub Pull Requests](https://github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pulls) for accepting changes. 4 | 5 | 6 | Read our [Code of Coduct](./CODE_OF_CONDUCT.md) to keep our community approachable and respectable. 7 | 8 | This guide details how to use issues and pull requests to improve the project. 9 | 10 | ## General Guidelines 11 | 12 | ### Pull Requests 13 | 14 | Make sure to keep Pull Requests small and functional to make them easier to review, understand, and look up in commit history. 15 | 16 | Adding the appropriate documentation, unit tests and e2e tests as part of a feature is the responsibility of the feature owner, whether it is done in the same Pull Request or not. 17 | 18 | Pull Requests should follow the "Title: Description" format, where the Description describes what part of the code is being modified. 19 | 20 | ### Design Docs 21 | 22 | A contributor proposes a design with a PR on the repository to allow for revisions and discussions. If a design needs to be discussed before formulating a document for it, make use of Google doc and [GitHub issue](https://github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/issues) to involve the community on the discussion. 23 | 24 | ### GitHub Issues 25 | 26 | GitHub Issues are used to file bugs, work items, and feature requests with actionable items/issues (Please refer to the "Reporting Bugs/Feature Requests" section below for more information). 27 | 28 | ### Reporting Bugs/Feature Requests 29 | 30 | We welcome you to use the GitHub issue tracker to report bugs or suggest features that have actionable items/issues (as opposed to introducing a feature request on GitHub Discussions). 31 | 32 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 33 | 34 | - A reproducible test case or series of steps 35 | - The version of the code being used 36 | - Any modifications you've made relevant to the bug 37 | - Anything unusual about your environment or deployment 38 | 39 | ## Contributing via Pull Requests 40 | 41 | ### Find interesting issue 42 | 43 | If you spot a problem, [search if an issue already exists](https://github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/issues). If a related issue doesn't exist, you can open a new issue by clicking the [New issue](https://github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/issues/new). 44 | 45 | 46 | ### Open a Pull request. 47 | 48 | When you're done making the changes, open a [Pull Requests](https://github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pulls) and fill PR template so we can better review your PR. The [template](https://github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/issues/new) helps reviewers understand your changes and the purpose of your pull request. 49 | 50 | Don't forget to link PR to issue if you are solving one. 51 | 52 | If you run into any merge issues, checkout this [github tutorial](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/addressing-merge-conflicts) to help you resolve merge conflicts and other issues. 53 | 54 | 55 | ## Finding contributions to work on 56 | 57 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, uses the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' and 'good first issue' issues are a great place to start. 58 | -------------------------------------------------------------------------------- /app/dcr_api/biz/dal/db/job.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 TikTok Pte. Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package db 16 | 17 | import ( 18 | "time" 19 | 20 | "github.com/pkg/errors" 21 | "gorm.io/gorm" 22 | ) 23 | 24 | type Job struct { 25 | gorm.Model 26 | ID uint64 `gorm:"id" json:"id""` 27 | UUID string `gorm:"uuid" json:"uuid"` 28 | Creator string `gorm:"creator" json:"creator"` 29 | JupyterFileName string `gorm:"jupyter_file_name" json:"jupyter_file_name"` 30 | DockerImage string `gorm:"docker_image" json:"docker_image"` 31 | DockerImageDigest string `gorm:"docker_image_digest" json:"docker_image_digest"` 32 | AttestationReport string `gorm:"attestation_report" json:"attestation_report"` 33 | JobStatus int `gorm:"job_status" json:"job_status"` 34 | InstanceName string `gorm:"instance_name" json:"instance_name"` 35 | } 36 | 37 | func (Job) TableName() string { 38 | return "jobs" 39 | } 40 | 41 | func CreateJob(job *Job) error { 42 | timestamp := time.Now() 43 | job.UpdatedAt = timestamp 44 | job.CreatedAt = timestamp 45 | err := DB.Create(job).Error 46 | if err != nil { 47 | return errors.Wrap(err, "failed to insert job into job table ") 48 | } 49 | return nil 50 | } 51 | 52 | func UpdateJob(j *Job) error { 53 | result := DB.Model(j).Updates(Job{JobStatus: j.JobStatus, DockerImageDigest: j.DockerImageDigest, DockerImage: j.DockerImage, AttestationReport: j.AttestationReport, InstanceName: j.InstanceName}) 54 | if result.Error != nil { 55 | return errors.Wrap(result.Error, "failed to update job %v") 56 | } 57 | return nil 58 | } 59 | 60 | func QueryJobsByCreator(creator string, page, pageSize int64) ([]*Job, int64, error) { 61 | db := DB.Model(Job{}) 62 | if len(creator) != 0 { 63 | db = db.Where("creator = ?", creator) 64 | } 65 | var total int64 66 | if err := db.Count(&total).Error; err != nil { 67 | return nil, 0, errors.Wrap(err, "failed to count jobs ") 68 | } 69 | var res []*Job 70 | if err := db.Limit(int(pageSize)).Offset(int(pageSize * (page - 1))).Find(&res).Order("id DESC").Error; err != nil { 71 | return nil, 0, errors.Wrap(err, "failed to query jobs ") 72 | } 73 | return res, total, nil 74 | } 75 | 76 | func QueryJobByIdAndCreator(jobId int64, creator string) (*Job, error) { 77 | db := DB.Model(Job{}) 78 | var res Job 79 | db = db.Where("id = ?", jobId).Where("creator = ?", creator) 80 | if err := db.First(&res).Error; err != nil { 81 | return nil, errors.Wrap(err, "failed to query jobs or it doesn't exist") 82 | } 83 | return &res, nil 84 | } 85 | 86 | func DeleteJob(creator string, uuid string) { 87 | DB.Model(Job{}).Where("creator = ? AND uuid = ?", creator, uuid).Delete(&Job{}) 88 | } 89 | 90 | func QueryJobByUUIDAndCreator(creator string, uuid string) (*Job, error) { 91 | db := DB.Model(Job{}) 92 | var res Job 93 | if err := db.Where("creator = ? AND uuid = ?", creator, uuid).First(&res).Error; err != nil { 94 | return nil, errors.Wrap(err, "failed to query jobs ") 95 | } 96 | return &res, nil 97 | } 98 | 99 | func GetInProgressJobs(creator string) ([]*Job, error) { 100 | var res []*Job 101 | if err := DB.Model(Job{}).Where("creator = ? AND job_status in (1, 3, 4)", creator).Find(&res).Error; err != nil { 102 | return nil, errors.Wrap(err, "failed to find finished job status") 103 | } 104 | return res, nil 105 | } 106 | -------------------------------------------------------------------------------- /deployment/data-clean-room/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ printf "%s-api" (include "data-clean-room-chart.fullname" .) | quote }} 5 | labels: 6 | {{- include "data-clean-room-chart.labels" . | nindent 4 }} 7 | namespace: {{ .Values.namespace }} 8 | spec: 9 | {{- if not .Values.autoscaling.enabled }} 10 | replicas: {{ .Values.replicaCount }} 11 | {{- end }} 12 | selector: 13 | matchLabels: 14 | {{- include "data-clean-room-chart.selectorLabels" . | nindent 6 }} 15 | template: 16 | metadata: 17 | {{- with .Values.podAnnotations }} 18 | annotations: 19 | {{- toYaml . | nindent 8 }} 20 | {{- end }} 21 | labels: 22 | {{- include "data-clean-room-chart.labels" . | nindent 8 }} 23 | {{- with .Values.podLabels }} 24 | {{- toYaml . | nindent 8 }} 25 | {{- end }} 26 | spec: 27 | {{- with .Values.imagePullSecrets }} 28 | imagePullSecrets: 29 | {{- toYaml . | nindent 8 }} 30 | {{- end }} 31 | serviceAccountName: {{ include "data-clean-room-chart.serviceAccountName" . }} 32 | securityContext: 33 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 34 | containers: 35 | - name: {{ .Chart.Name }} 36 | securityContext: 37 | {{- toYaml .Values.securityContext | nindent 12 }} 38 | image: "{{ .Values.apiImage.repository }}:{{ .Values.apiImage.tag | default .Chart.AppVersion }}" 39 | imagePullPolicy: {{ .Values.apiImage.pullPolicy }} 40 | env: 41 | - name: MYSQL_USERNAME 42 | valueFrom: 43 | secretKeyRef: 44 | name: mysql-secret 45 | key: mysql-username 46 | - name: MYSQL_PASSWORD 47 | valueFrom: 48 | secretKeyRef: 49 | name: mysql-secret 50 | key: mysql-password 51 | - name: MYSQL_DATABASE 52 | valueFrom: 53 | secretKeyRef: 54 | name: mysql-secret 55 | key: mysql-database 56 | - name: UNUSED_TIMESTAMP 57 | value: {{ now | quote }} 58 | - name: MYSQL_HOST 59 | value: {{ .Values.mysql.host }} 60 | - name: MYSQL_PORT 61 | value: {{ .Values.mysql.port | quote }} 62 | ports: 63 | - name: http 64 | containerPort: {{ .Values.api.port }} 65 | protocol: TCP 66 | livenessProbe: 67 | httpGet: 68 | path: /health 69 | port: http 70 | readinessProbe: 71 | httpGet: 72 | path: /health 73 | port: http 74 | resources: 75 | {{- toYaml .Values.resources | nindent 12 }} 76 | {{- with .Values.volumeMounts }} 77 | volumeMounts: 78 | {{- toYaml . | nindent 12 }} 79 | {{- end }} 80 | {{- if not .Values.useMinikube }} 81 | - name: cloud-sql-proxy 82 | image: gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.8.0 83 | args: 84 | - "--structured-logs" 85 | - "--private-ip" 86 | - {{ print "--port=" .Values.mysql.port }} 87 | - {{ .Values.cloudSql.connection_name }} 88 | securityContext: 89 | runAsNonRoot: true 90 | resources: 91 | requests: 92 | memory: "2Gi" 93 | cpu: "0.5" 94 | {{- end }} 95 | {{- with .Values.volumes }} 96 | volumes: 97 | {{- toYaml . | nindent 8 }} 98 | {{- end }} 99 | {{- with .Values.nodeSelector }} 100 | nodeSelector: 101 | {{- toYaml . | nindent 8 }} 102 | {{- end }} 103 | {{- with .Values.affinity }} 104 | affinity: 105 | {{- toYaml . | nindent 8 }} 106 | {{- end }} 107 | {{- with .Values.tolerations }} 108 | tolerations: 109 | {{- toYaml . | nindent 8 }} 110 | {{- end }} 111 | -------------------------------------------------------------------------------- /app/jupyterlab_manatee/RELEASE.md: -------------------------------------------------------------------------------- 1 | # Making a new release of jupyterlab_manatee 2 | 3 | The extension can be published to `PyPI` and `npm` manually or using the [Jupyter Releaser](https://github.com/jupyter-server/jupyter_releaser). 4 | 5 | ## Manual release 6 | 7 | ### Python package 8 | 9 | This extension can be distributed as Python packages. All of the Python 10 | packaging instructions are in the `pyproject.toml` file to wrap your extension in a 11 | Python package. Before generating a package, you first need to install some tools: 12 | 13 | ```bash 14 | pip install build twine hatch 15 | ``` 16 | 17 | Bump the version using `hatch`. By default this will create a tag. 18 | See the docs on [hatch-nodejs-version](https://github.com/agoose77/hatch-nodejs-version#semver) for details. 19 | 20 | ```bash 21 | hatch version 22 | ``` 23 | 24 | Make sure to clean up all the development files before building the package: 25 | 26 | ```bash 27 | jlpm clean:all 28 | ``` 29 | 30 | You could also clean up the local git repository: 31 | 32 | ```bash 33 | git clean -dfX 34 | ``` 35 | 36 | To create a Python source package (`.tar.gz`) and the binary package (`.whl`) in the `dist/` directory, do: 37 | 38 | ```bash 39 | python -m build 40 | ``` 41 | 42 | > `python setup.py sdist bdist_wheel` is deprecated and will not work for this package. 43 | 44 | Then to upload the package to PyPI, do: 45 | 46 | ```bash 47 | twine upload dist/* 48 | ``` 49 | 50 | ### NPM package 51 | 52 | To publish the frontend part of the extension as a NPM package, do: 53 | 54 | ```bash 55 | npm login 56 | npm publish --access public 57 | ``` 58 | 59 | ## Automated releases with the Jupyter Releaser 60 | 61 | The extension repository should already be compatible with the Jupyter Releaser. 62 | 63 | Check out the [workflow documentation](https://jupyter-releaser.readthedocs.io/en/latest/get_started/making_release_from_repo.html) for more information. 64 | 65 | Here is a summary of the steps to cut a new release: 66 | 67 | - Add tokens to the [Github Secrets](https://docs.github.com/en/actions/security-guides/encrypted-secrets) in the repository: 68 | - `ADMIN_GITHUB_TOKEN` (with "public_repo" and "repo:status" permissions); see the [documentation](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) 69 | - `NPM_TOKEN` (with "automation" permission); see the [documentation](https://docs.npmjs.com/creating-and-viewing-access-tokens) 70 | - Set up PyPI 71 | 72 |
Using PyPI trusted publisher (modern way) 73 | 74 | - Set up your PyPI project by [adding a trusted publisher](https://docs.pypi.org/trusted-publishers/adding-a-publisher/) 75 | - The _workflow name_ is `publish-release.yml` and the _environment_ should be left blank. 76 | - Ensure the publish release job as `permissions`: `id-token : write` (see the [documentation](https://docs.pypi.org/trusted-publishers/using-a-publisher/)) 77 | 78 |
79 | 80 |
Using PyPI token (legacy way) 81 | 82 | - If the repo generates PyPI release(s), create a scoped PyPI [token](https://packaging.python.org/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/#saving-credentials-on-github). We recommend using a scoped token for security reasons. 83 | 84 | - You can store the token as `PYPI_TOKEN` in your fork's `Secrets`. 85 | 86 | - Advanced usage: if you are releasing multiple repos, you can create a secret named `PYPI_TOKEN_MAP` instead of `PYPI_TOKEN` that is formatted as follows: 87 | 88 | ```text 89 | owner1/repo1,token1 90 | owner2/repo2,token2 91 | ``` 92 | 93 | If you have multiple Python packages in the same repository, you can point to them as follows: 94 | 95 | ```text 96 | owner1/repo1/path/to/package1,token1 97 | owner1/repo1/path/to/package2,token2 98 | ``` 99 | 100 |
101 | 102 | - Go to the Actions panel 103 | - Run the "Step 1: Prep Release" workflow 104 | - Check the draft changelog 105 | - Run the "Step 2: Publish Release" workflow 106 | 107 | ## Publishing to `conda-forge` 108 | 109 | If the package is not on conda forge yet, check the documentation to learn how to add it: https://conda-forge.org/docs/maintainer/adding_pkgs.html 110 | 111 | Otherwise a bot should pick up the new version publish to PyPI, and open a new PR on the feedstock repository automatically. 112 | -------------------------------------------------------------------------------- /app/jupyterlab_manatee/ui-tests/README.md: -------------------------------------------------------------------------------- 1 | # Integration Testing 2 | 3 | This folder contains the integration tests of the extension. 4 | 5 | They are defined using [Playwright](https://playwright.dev/docs/intro) test runner 6 | and [Galata](https://github.com/jupyterlab/jupyterlab/tree/main/galata) helper. 7 | 8 | The Playwright configuration is defined in [playwright.config.js](./playwright.config.js). 9 | 10 | The JupyterLab server configuration to use for the integration test is defined 11 | in [jupyter_server_test_config.py](./jupyter_server_test_config.py). 12 | 13 | The default configuration will produce video for failing tests and an HTML report. 14 | 15 | > There is a new experimental UI mode that you may fall in love with; see [that video](https://www.youtube.com/watch?v=jF0yA-JLQW0). 16 | 17 | ## Run the tests 18 | 19 | > All commands are assumed to be executed from the root directory 20 | 21 | To run the tests, you need to: 22 | 23 | 1. Compile the extension: 24 | 25 | ```sh 26 | jlpm install 27 | jlpm build:prod 28 | ``` 29 | 30 | > Check the extension is installed in JupyterLab. 31 | 32 | 2. Install test dependencies (needed only once): 33 | 34 | ```sh 35 | cd ./ui-tests 36 | jlpm install 37 | jlpm playwright install 38 | cd .. 39 | ``` 40 | 41 | 3. Execute the [Playwright](https://playwright.dev/docs/intro) tests: 42 | 43 | ```sh 44 | cd ./ui-tests 45 | jlpm playwright test 46 | ``` 47 | 48 | Test results will be shown in the terminal. In case of any test failures, the test report 49 | will be opened in your browser at the end of the tests execution; see 50 | [Playwright documentation](https://playwright.dev/docs/test-reporters#html-reporter) 51 | for configuring that behavior. 52 | 53 | ## Update the tests snapshots 54 | 55 | > All commands are assumed to be executed from the root directory 56 | 57 | If you are comparing snapshots to validate your tests, you may need to update 58 | the reference snapshots stored in the repository. To do that, you need to: 59 | 60 | 1. Compile the extension: 61 | 62 | ```sh 63 | jlpm install 64 | jlpm build:prod 65 | ``` 66 | 67 | > Check the extension is installed in JupyterLab. 68 | 69 | 2. Install test dependencies (needed only once): 70 | 71 | ```sh 72 | cd ./ui-tests 73 | jlpm install 74 | jlpm playwright install 75 | cd .. 76 | ``` 77 | 78 | 3. Execute the [Playwright](https://playwright.dev/docs/intro) command: 79 | 80 | ```sh 81 | cd ./ui-tests 82 | jlpm playwright test -u 83 | ``` 84 | 85 | > Some discrepancy may occurs between the snapshots generated on your computer and 86 | > the one generated on the CI. To ease updating the snapshots on a PR, you can 87 | > type `please update playwright snapshots` to trigger the update by a bot on the CI. 88 | > Once the bot has computed new snapshots, it will commit them to the PR branch. 89 | 90 | ## Create tests 91 | 92 | > All commands are assumed to be executed from the root directory 93 | 94 | To create tests, the easiest way is to use the code generator tool of playwright: 95 | 96 | 1. Compile the extension: 97 | 98 | ```sh 99 | jlpm install 100 | jlpm build:prod 101 | ``` 102 | 103 | > Check the extension is installed in JupyterLab. 104 | 105 | 2. Install test dependencies (needed only once): 106 | 107 | ```sh 108 | cd ./ui-tests 109 | jlpm install 110 | jlpm playwright install 111 | cd .. 112 | ``` 113 | 114 | 3. Start the server: 115 | 116 | ```sh 117 | cd ./ui-tests 118 | jlpm start 119 | ``` 120 | 121 | 4. Execute the [Playwright code generator](https://playwright.dev/docs/codegen) in **another terminal**: 122 | 123 | ```sh 124 | cd ./ui-tests 125 | jlpm playwright codegen localhost:8888 126 | ``` 127 | 128 | ## Debug tests 129 | 130 | > All commands are assumed to be executed from the root directory 131 | 132 | To debug tests, a good way is to use the inspector tool of playwright: 133 | 134 | 1. Compile the extension: 135 | 136 | ```sh 137 | jlpm install 138 | jlpm build:prod 139 | ``` 140 | 141 | > Check the extension is installed in JupyterLab. 142 | 143 | 2. Install test dependencies (needed only once): 144 | 145 | ```sh 146 | cd ./ui-tests 147 | jlpm install 148 | jlpm playwright install 149 | cd .. 150 | ``` 151 | 152 | 3. Execute the Playwright tests in [debug mode](https://playwright.dev/docs/debug): 153 | 154 | ```sh 155 | cd ./ui-tests 156 | jlpm playwright test --debug 157 | ``` 158 | 159 | ## Upgrade Playwright and the browsers 160 | 161 | To update the web browser versions, you must update the package `@playwright/test`: 162 | 163 | ```sh 164 | cd ./ui-tests 165 | jlpm up "@playwright/test" 166 | jlpm playwright install 167 | ``` 168 | -------------------------------------------------------------------------------- /app/dcr_api/idl/job.thrift: -------------------------------------------------------------------------------- 1 | namespace go job 2 | 3 | enum JobStatus { 4 | ImageBuilding = 1 5 | ImageBuildingFailed = 2 6 | VMWaiting = 3 7 | VMRunning = 4 8 | VMFinished = 5 9 | VMKilled = 6 10 | VMFailed = 7 11 | VMOther = 8 12 | } 13 | 14 | struct Job { 15 | 1: i64 id 16 | 2: string uuid 17 | 3: string creator 18 | 4: JobStatus job_status 19 | 5: string jupyter_file_name 20 | 6: string created_at 21 | 7: string updated_at 22 | } 23 | 24 | struct SubmitJobRequest{ 25 | 1: string jupyter_file_name (api.body="filename", api.vd="len($) > 0 && len($) < 128 && regexp('^.*\\.ipynb$') && !regexp('.*\\.\\..*')") 26 | 2: string creator (api.body="creator", api.vd="len($) > 0 && len($) < 32 && !regexp('.*\\.\\..*')") 27 | 255: required string access_token (api.header="Authorization") 28 | } 29 | 30 | struct SubmitJobResponse{ 31 | 1: i32 code 32 | 2: string msg 33 | 3: string uuid 34 | } 35 | 36 | struct QueryJobRequest { 37 | 1: i64 page (api.body="page", api.query="page",api.vd="$>0") 38 | 2: i64 page_size (api.body="page_size", api.query="page_size", api.vd="$ > 0 || $ <= 100") 39 | 3: string creator (api.body="creator", api.vd="len($) > 0 && len($) < 32 && !regexp('.*\\.\\..*')") 40 | 255: required string access_token (api.header="Authorization") 41 | } 42 | 43 | struct QueryJobResponse { 44 | 1: i32 code 45 | 2: string msg 46 | 3: list jobs 47 | 4: i64 total 48 | } 49 | 50 | struct DeleteJobRequest { 51 | 1: string uuid (api.body="uuid", api.query="uuid") 52 | 2: string creator (api.body="creator", api.vd="len($) > 0 && len($) < 32 && !regexp('.*\\.\\..*')") 53 | 255: required string access_token (api.header="Authorization") 54 | } 55 | 56 | struct DeleteJobResponse { 57 | 1: i32 code 58 | 2: string msg 59 | } 60 | 61 | struct UpdateJobStatusRequest { 62 | 1: string uuid (api.body="uuid", api.query="uuid") 63 | 2: JobStatus status (api.body="status", api.query="status") 64 | 3: string docker_image (api.body="image", api.query="image") 65 | 4: string docker_image_digest (api.body="digest", api.query="digest") 66 | 5: string creator (api.body="creator", api.vd="len($) > 0 && len($) < 32 && !regexp('.*\\.\\..*')") 67 | 6: string attestation_token (api.body="token", api.query="token") 68 | 255: required string access_token (api.header="Authorization") 69 | } 70 | 71 | struct UpdateJobStatusResponse { 72 | 1: i32 code 73 | 2: string msg 74 | } 75 | 76 | struct QueryJobOutputRequest { 77 | 1: i64 id (api.body="id", api.query="id") 78 | 2: string creator (api.body="creator", api.vd="len($) > 0 && len($) < 32 && !regexp('.*\\.\\..*')") 79 | 255: required string access_token (api.header="Authorization") 80 | } 81 | 82 | struct QueryJobOutputResponse { 83 | 1: i32 code 84 | 2: string msg 85 | 3: i64 size 86 | 4: string filename 87 | } 88 | 89 | struct DownloadJobOutputRequest { 90 | 1: i64 id (api.body="id", api.query="id", api.vd="$>0") 91 | 2: string creator (api.body="creator", api.vd="len($) > 0 && len($) < 32 && !regexp('.*\\.\\..*')") 92 | 3: i64 offset (api.body="offset", api.query="offset") 93 | 4: i64 chunk (api.body="chunk", api.query="chunk", api.vd="$>0 && $ < 5242880") 94 | 255: required string access_token (api.header="Authorization") 95 | } 96 | 97 | struct DownloadJobOutputResponse { 98 | 1: i32 code 99 | 2: string msg 100 | 3: string content 101 | } 102 | 103 | struct QueryJobAttestationRequest { 104 | 1: i64 id (api.body="id", api.query="id", api.vd="$>0") 105 | 2: string creator (api.body="creator", api.vd="len($) > 0 && len($) < 32 && !regexp('.*\\.\\..*')") 106 | } 107 | 108 | struct QueryJobAttestationResponse { 109 | 1: i32 code 110 | 2: string msg 111 | 3: string token 112 | } 113 | 114 | service JobHandler { 115 | SubmitJobResponse SubmitJob(1:SubmitJobRequest req)(api.post="/v1/job/submit/") 116 | QueryJobResponse QueryJob(1:QueryJobRequest req)(api.post="/v1/job/query/") 117 | DeleteJobResponse DeleteJob(1:DeleteJobRequest req)(api.post="/v1/job/delete/") 118 | UpdateJobStatusResponse UpdateJobStatus(1:UpdateJobStatusRequest req)(api.post="/v1/job/update/") 119 | QueryJobOutputResponse QueryJobOutputAttr(1:QueryJobOutputRequest req) (api.post="/v1/job/output/attrs/") 120 | DownloadJobOutputResponse DownloadJobOutput(1:DownloadJobOutputRequest req) (api.post="/v1/job/output/download/") 121 | QueryJobAttestationResponse QueryJobAttestationReport(1:QueryJobAttestationRequest req) (api.post="/v1/job/attestation/") 122 | } -------------------------------------------------------------------------------- /app/jupyterlab_manatee/src/sources.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 TikTok Pte. Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { Contents } from '@jupyterlab/services'; 18 | import { IDocumentManager } from '@jupyterlab/docmanager'; 19 | import { PanelWithToolbar, ToolbarButton, fileUploadIcon, } from '@jupyterlab/ui-components'; 20 | import { filter } from '@lumino/algorithm'; 21 | import { ITranslator, nullTranslator } from '@jupyterlab/translation'; 22 | import { FileBrowser, FilterFileBrowserModel } from '@jupyterlab/filebrowser'; 23 | import { showDialog, Dialog } from '@jupyterlab/apputils'; 24 | import { ServerConnection } from '@jupyterlab/services'; 25 | 26 | /* 27 | This class overrides items() to make the filebrowser list only ipynb files. 28 | We're doing this soley for the demo purpose, and the actual product may have additional files 29 | (e.g., local python modules) 30 | */ 31 | class NotebookOnlyFilterFileBrowserModel extends FilterFileBrowserModel { 32 | override items(): IterableIterator { 33 | return filter(super.items(), value => { 34 | if (value.type === 'notebook') { 35 | return true; 36 | } else { 37 | return false; 38 | } 39 | }); 40 | } 41 | } 42 | 43 | export class DataCleanRoomSources extends PanelWithToolbar { 44 | constructor(options: DataCleanRoomSources.IOptions) { 45 | super(); 46 | const { manager } = options; 47 | this._manager = manager; 48 | const trans = (options.translator ?? nullTranslator).load('jupyterlab'); 49 | this.title.label = trans.__('Sources'); 50 | 51 | const fbModel = new NotebookOnlyFilterFileBrowserModel({ 52 | manager: manager, 53 | }); 54 | this._browser = new FileBrowser({ 55 | id: 'jupyterlab_manatee:plugin:sources', 56 | model: fbModel 57 | }); 58 | this.toolbar.addItem( 59 | 'submit', 60 | new ToolbarButton({ 61 | icon: fileUploadIcon, 62 | onClick: () => this.sendSelectedFilesToAPI(), 63 | tooltip: trans.__('Submit Job to Data Clean Room') 64 | }) 65 | ); 66 | 67 | this.addWidget(this._browser); 68 | }; 69 | 70 | async sendSelectedFilesToAPI() { 71 | for (const item of this._browser.selectedItems()) { 72 | const result = await showDialog({ 73 | title: "Submitting a Job to Data Clean Room?", 74 | body: 'Path: ' + item.path, 75 | buttons: [Dialog.okButton(), Dialog.cancelButton()] 76 | }); 77 | 78 | if (result.button.accept) { 79 | const file = await this._manager.services.contents.get(item.path); 80 | // Prepare data 81 | const data = JSON.stringify({ 82 | path: item.path, 83 | filename: file.name 84 | }); 85 | 86 | const settings = ServerConnection.makeSettings(); 87 | 88 | console.log("Sending... %s", settings.baseUrl); 89 | ServerConnection.makeRequest(settings.baseUrl + "manatee/jobs", { 90 | body: data, method: "POST" 91 | }, settings).then(response => { 92 | if (response.status !== 200) { 93 | console.log("Error has occured!"); 94 | } 95 | response.body?.getReader().read().then(({done, value}) => { 96 | if (done) { 97 | console.log("stream is closed"); 98 | return; 99 | } 100 | let decoder = new TextDecoder('utf-8'); 101 | console.log("value:", decoder.decode(value)); 102 | }); 103 | }); 104 | } 105 | } 106 | } 107 | 108 | protected _manager : IDocumentManager; 109 | protected _browser : FileBrowser; 110 | } 111 | 112 | export namespace DataCleanRoomSources { 113 | export interface IOptions { 114 | manager: IDocumentManager; 115 | translator?: ITranslator; 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /app/dcr_tee/encryption/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/app/dcr_tee/encryption 2 | 3 | go 1.22.0 4 | 5 | require ( 6 | cloud.google.com/go/kms v1.17.1 7 | github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pkg v0.0.1 8 | google.golang.org/api v0.180.0 9 | ) 10 | 11 | replace github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pkg => /go/pkg/mod/github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pkg@v0.0.1 12 | 13 | require ( 14 | cloud.google.com/go v0.113.0 // indirect 15 | cloud.google.com/go/auth v0.4.1 // indirect 16 | cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect 17 | cloud.google.com/go/compute v1.27.0 // indirect 18 | cloud.google.com/go/compute/metadata v0.3.0 // indirect 19 | cloud.google.com/go/iam v1.1.8 // indirect 20 | cloud.google.com/go/longrunning v0.5.7 // indirect 21 | cloud.google.com/go/storage v1.41.0 // indirect 22 | github.com/bytedance/go-tagexpr/v2 v2.9.2 // indirect 23 | github.com/bytedance/gopkg v0.0.0-20220413063733-65bf48ffb3a7 // indirect 24 | github.com/bytedance/sonic v1.11.6 // indirect 25 | github.com/bytedance/sonic/loader v0.1.1 // indirect 26 | github.com/cloudwego/base64x v0.1.4 // indirect 27 | github.com/cloudwego/hertz v0.9.0 // indirect 28 | github.com/cloudwego/iasm v0.2.0 // indirect 29 | github.com/felixge/httpsnoop v1.0.4 // indirect 30 | github.com/fsnotify/fsnotify v1.7.0 // indirect 31 | github.com/gabriel-vasile/mimetype v1.4.3 // indirect 32 | github.com/gin-contrib/sse v0.1.0 // indirect 33 | github.com/gin-gonic/gin v1.10.0 // indirect 34 | github.com/go-logr/logr v1.4.1 // indirect 35 | github.com/go-logr/stdr v1.2.2 // indirect 36 | github.com/go-playground/locales v0.14.1 // indirect 37 | github.com/go-playground/universal-translator v0.18.1 // indirect 38 | github.com/go-playground/validator/v10 v10.20.0 // indirect 39 | github.com/goccy/go-json v0.10.2 // indirect 40 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 41 | github.com/golang/protobuf v1.5.4 // indirect 42 | github.com/google/s2a-go v0.1.7 // indirect 43 | github.com/google/uuid v1.6.0 // indirect 44 | github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect 45 | github.com/googleapis/gax-go/v2 v2.12.4 // indirect 46 | github.com/hashicorp/hcl v1.0.0 // indirect 47 | github.com/henrylee2cn/ameda v1.4.10 // indirect 48 | github.com/henrylee2cn/goutil v0.0.0-20210127050712-89660552f6f8 // indirect 49 | github.com/json-iterator/go v1.1.12 // indirect 50 | github.com/klauspost/cpuid/v2 v2.2.7 // indirect 51 | github.com/leodido/go-urn v1.4.0 // indirect 52 | github.com/magiconair/properties v1.8.7 // indirect 53 | github.com/mattn/go-isatty v0.0.20 // indirect 54 | github.com/mitchellh/mapstructure v1.5.0 // indirect 55 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 56 | github.com/modern-go/reflect2 v1.0.2 // indirect 57 | github.com/nyaruka/phonenumbers v1.0.55 // indirect 58 | github.com/pelletier/go-toml/v2 v2.2.2 // indirect 59 | github.com/pkg/errors v0.9.1 // indirect 60 | github.com/sagikazarmark/locafero v0.4.0 // indirect 61 | github.com/sagikazarmark/slog-shim v0.1.0 // indirect 62 | github.com/sourcegraph/conc v0.3.0 // indirect 63 | github.com/spf13/afero v1.11.0 // indirect 64 | github.com/spf13/cast v1.6.0 // indirect 65 | github.com/spf13/pflag v1.0.5 // indirect 66 | github.com/spf13/viper v1.18.2 // indirect 67 | github.com/subosito/gotenv v1.6.0 // indirect 68 | github.com/tidwall/gjson v1.14.4 // indirect 69 | github.com/tidwall/match v1.1.1 // indirect 70 | github.com/tidwall/pretty v1.2.0 // indirect 71 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 72 | github.com/ugorji/go/codec v1.2.12 // indirect 73 | go.opencensus.io v0.24.0 // indirect 74 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect 75 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect 76 | go.opentelemetry.io/otel v1.24.0 // indirect 77 | go.opentelemetry.io/otel/metric v1.24.0 // indirect 78 | go.opentelemetry.io/otel/trace v1.24.0 // indirect 79 | go.uber.org/atomic v1.9.0 // indirect 80 | go.uber.org/multierr v1.9.0 // indirect 81 | golang.org/x/arch v0.8.0 // indirect 82 | golang.org/x/crypto v0.23.0 // indirect 83 | golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect 84 | golang.org/x/net v0.25.0 // indirect 85 | golang.org/x/oauth2 v0.20.0 // indirect 86 | golang.org/x/sync v0.7.0 // indirect 87 | golang.org/x/sys v0.20.0 // indirect 88 | golang.org/x/text v0.15.0 // indirect 89 | golang.org/x/time v0.5.0 // indirect 90 | google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda // indirect 91 | google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8 // indirect 92 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240509183442-62759503f434 // indirect 93 | google.golang.org/grpc v1.63.2 // indirect 94 | google.golang.org/protobuf v1.34.1 // indirect 95 | gopkg.in/ini.v1 v1.67.0 // indirect 96 | gopkg.in/yaml.v3 v3.0.1 // indirect 97 | ) 98 | -------------------------------------------------------------------------------- /pkg/utils/file.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 TikTok Pte. Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package utils 16 | 17 | import ( 18 | "archive/tar" 19 | "compress/gzip" 20 | stdErrors "errors" 21 | "fmt" 22 | "io" 23 | "os" 24 | "path/filepath" 25 | "strings" 26 | 27 | "github.com/pkg/errors" 28 | ) 29 | 30 | func GetDcrConfDir() string { 31 | return "/usr/local/dcr_conf" 32 | } 33 | 34 | func GetConfigFile() string { 35 | return fmt.Sprintf("%s/%s", GetDcrConfDir(), "config.yaml") 36 | } 37 | 38 | func GetDockerFile() string { 39 | return fmt.Sprintf("%s/%s", GetDcrConfDir(), "Dockerfile") 40 | } 41 | 42 | func GetWorkDirectory() (string, error) { 43 | return "/app/data", nil 44 | } 45 | 46 | // GetCurrentAbPathByExecutable Get current executable ab path 47 | func GetCurrentAbPathByExecutable() (string, error) { 48 | exePath, err := os.Executable() 49 | if err != nil { 50 | return "", errors.Wrap(err, "failed to get executable path") 51 | } 52 | res, err := filepath.EvalSymlinks(filepath.Dir(exePath)) 53 | if err != nil { 54 | return "", errors.Wrap(err, "failed to get executable path") 55 | } 56 | return res, nil 57 | } 58 | 59 | func CopyFile(src string, dest string) error { 60 | src = filepath.Clean(src) 61 | srcFile, err := os.Open(src) 62 | if err != nil { 63 | return errors.Wrap(err, "failed to open source file") 64 | } 65 | defer func(srcFile *os.File) { 66 | err := srcFile.Close() 67 | if err != nil { 68 | return 69 | } 70 | }(srcFile) 71 | 72 | dest = filepath.Clean(dest) 73 | dstFile, err := os.Create(dest) 74 | if err != nil { 75 | return errors.Wrap(err, "failed to open dest file") 76 | } 77 | defer func(dstFile *os.File) { 78 | err := dstFile.Close() 79 | if err != nil { 80 | return 81 | } 82 | }(dstFile) 83 | // Copy file content 84 | _, err = io.Copy(dstFile, srcFile) 85 | if err != nil { 86 | return errors.Wrap(err, "failed to copy file") 87 | } 88 | return nil 89 | } 90 | 91 | func safeJoin(base, path string) (string, error) { 92 | absPath := filepath.Join(base, path) 93 | if !strings.HasPrefix(absPath, filepath.Clean(base)+string(os.PathSeparator)) { 94 | return "", fmt.Errorf("%s is an invalid path", path) 95 | } 96 | return absPath, nil 97 | } 98 | 99 | func UnTarGz(compressedFile string, destDirectory string) error { 100 | destDirectory, err := filepath.Abs(destDirectory) 101 | if err != nil { 102 | return errors.Wrap(err, fmt.Sprintf("failed to get abs path of %s", destDirectory)) 103 | } 104 | if !strings.HasSuffix(compressedFile, "tar.gz") { 105 | return errors.Wrap(stdErrors.New("only support file ending with tar.gz"), "") 106 | } 107 | compressedFile = filepath.Clean(compressedFile) 108 | file, err := os.Open(compressedFile) 109 | if err != nil { 110 | return errors.Wrap(err, fmt.Sprintf("failed to open compressed file: %s", compressedFile)) 111 | } 112 | defer func(file *os.File) { 113 | _ = file.Close() 114 | }(file) 115 | // create gzip reader 116 | gr, err := gzip.NewReader(file) 117 | if err != nil { 118 | return errors.Wrap(err, "failed to open gzip reader") 119 | } 120 | defer func(r *gzip.Reader) { 121 | _ = r.Close() 122 | }(gr) 123 | // create tar reader 124 | tr := tar.NewReader(gr) 125 | 126 | for { 127 | hdr, err := tr.Next() 128 | if err == io.EOF { 129 | break 130 | } 131 | if err != nil { 132 | return errors.Wrap(err, "failed to traverse directory") 133 | } 134 | 135 | target, err := safeJoin(destDirectory, hdr.Name) 136 | if err != nil { 137 | return err 138 | } 139 | switch hdr.Typeflag { 140 | case tar.TypeDir: 141 | target = filepath.Clean(target) 142 | if err := os.Mkdir(target, hdr.FileInfo().Mode()); err != nil { 143 | return errors.Wrap(err, "failed to mkdir") 144 | } 145 | case tar.TypeReg: 146 | f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, hdr.FileInfo().Mode()) 147 | if err != nil { 148 | return errors.Wrap(err, "failed to create file") 149 | } 150 | if _, err := io.Copy(f, tr); err != nil { 151 | return errors.Wrap(err, "failed to write file") 152 | } 153 | case tar.TypeLink: 154 | linkname, err := safeJoin(destDirectory, hdr.Linkname) 155 | if err != nil { 156 | return err 157 | } 158 | if err := os.Link(linkname, target); err != nil { 159 | return errors.Wrap(err, "failed to create link") 160 | } 161 | case tar.TypeSymlink: 162 | if err := os.Symlink(hdr.Linkname, target); err != nil { 163 | return errors.Wrap(err, "failed to create symlink") 164 | } 165 | default: 166 | return errors.Wrap(fmt.Errorf("%s unkown type: %c", hdr.Name, hdr.Typeflag), "") 167 | } 168 | } 169 | return nil 170 | } 171 | -------------------------------------------------------------------------------- /pkg/go.mod: -------------------------------------------------------------------------------- 1 | // Copyright 2024 TikTok Pte. Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | module github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pkg 16 | 17 | go 1.22.0 18 | 19 | require ( 20 | cloud.google.com/go v0.113.0 // indirect 21 | cloud.google.com/go/auth v0.4.1 // indirect 22 | cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect 23 | cloud.google.com/go/compute v1.27.0 // indirect 24 | cloud.google.com/go/compute/metadata v0.3.0 // indirect 25 | cloud.google.com/go/iam v1.1.8 // indirect 26 | cloud.google.com/go/kms v1.17.1 // indirect 27 | cloud.google.com/go/longrunning v0.5.7 // indirect 28 | cloud.google.com/go/storage v1.41.0 // indirect 29 | github.com/bytedance/go-tagexpr/v2 v2.9.2 // indirect 30 | github.com/bytedance/gopkg v0.0.0-20220413063733-65bf48ffb3a7 // indirect 31 | github.com/bytedance/sonic v1.11.6 // indirect 32 | github.com/bytedance/sonic/loader v0.1.1 // indirect 33 | github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect 34 | github.com/cloudwego/base64x v0.1.4 // indirect 35 | github.com/cloudwego/hertz v0.9.0 // indirect 36 | github.com/cloudwego/iasm v0.2.0 // indirect 37 | github.com/felixge/httpsnoop v1.0.4 // indirect 38 | github.com/fsnotify/fsnotify v1.7.0 // indirect 39 | github.com/gabriel-vasile/mimetype v1.4.3 // indirect 40 | github.com/gin-contrib/sse v0.1.0 // indirect 41 | github.com/gin-gonic/gin v1.10.0 // indirect 42 | github.com/go-logr/logr v1.4.1 // indirect 43 | github.com/go-logr/stdr v1.2.2 // indirect 44 | github.com/go-playground/locales v0.14.1 // indirect 45 | github.com/go-playground/universal-translator v0.18.1 // indirect 46 | github.com/go-playground/validator/v10 v10.20.0 // indirect 47 | github.com/goccy/go-json v0.10.2 // indirect 48 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 49 | github.com/golang/protobuf v1.5.4 // indirect 50 | github.com/google/s2a-go v0.1.7 // indirect 51 | github.com/google/uuid v1.6.0 // indirect 52 | github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect 53 | github.com/googleapis/gax-go/v2 v2.12.4 // indirect 54 | github.com/hashicorp/hcl v1.0.0 // indirect 55 | github.com/henrylee2cn/ameda v1.4.10 // indirect 56 | github.com/henrylee2cn/goutil v0.0.0-20210127050712-89660552f6f8 // indirect 57 | github.com/json-iterator/go v1.1.12 // indirect 58 | github.com/klauspost/cpuid/v2 v2.2.7 // indirect 59 | github.com/leodido/go-urn v1.4.0 // indirect 60 | github.com/magiconair/properties v1.8.7 // indirect 61 | github.com/mattn/go-isatty v0.0.20 // indirect 62 | github.com/mitchellh/mapstructure v1.5.0 // indirect 63 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 64 | github.com/modern-go/reflect2 v1.0.2 // indirect 65 | github.com/nyaruka/phonenumbers v1.0.55 // indirect 66 | github.com/pelletier/go-toml/v2 v2.2.2 // indirect 67 | github.com/pkg/errors v0.9.1 // indirect 68 | github.com/sagikazarmark/locafero v0.4.0 // indirect 69 | github.com/sagikazarmark/slog-shim v0.1.0 // indirect 70 | github.com/sourcegraph/conc v0.3.0 // indirect 71 | github.com/spf13/afero v1.11.0 // indirect 72 | github.com/spf13/cast v1.6.0 // indirect 73 | github.com/spf13/pflag v1.0.5 // indirect 74 | github.com/spf13/viper v1.18.2 // indirect 75 | github.com/subosito/gotenv v1.6.0 // indirect 76 | github.com/tidwall/gjson v1.14.4 // indirect 77 | github.com/tidwall/match v1.1.1 // indirect 78 | github.com/tidwall/pretty v1.2.0 // indirect 79 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 80 | github.com/ugorji/go/codec v1.2.12 // indirect 81 | go.opencensus.io v0.24.0 // indirect 82 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect 83 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect 84 | go.opentelemetry.io/otel v1.24.0 // indirect 85 | go.opentelemetry.io/otel/metric v1.24.0 // indirect 86 | go.opentelemetry.io/otel/trace v1.24.0 // indirect 87 | go.uber.org/atomic v1.9.0 // indirect 88 | go.uber.org/multierr v1.9.0 // indirect 89 | golang.org/x/arch v0.8.0 // indirect 90 | golang.org/x/crypto v0.23.0 // indirect 91 | golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect 92 | golang.org/x/net v0.25.0 // indirect 93 | golang.org/x/oauth2 v0.20.0 // indirect 94 | golang.org/x/sync v0.7.0 // indirect 95 | golang.org/x/sys v0.20.0 // indirect 96 | golang.org/x/text v0.15.0 // indirect 97 | golang.org/x/time v0.5.0 // indirect 98 | google.golang.org/api v0.180.0 // indirect 99 | google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda // indirect 100 | google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8 // indirect 101 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240509183442-62759503f434 // indirect 102 | google.golang.org/grpc v1.63.2 // indirect 103 | google.golang.org/protobuf v1.34.1 // indirect 104 | gopkg.in/ini.v1 v1.67.0 // indirect 105 | gopkg.in/yaml.v3 v3.0.1 // indirect 106 | ) 107 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement. 63 | All complaints will be reviewed and investigated promptly and fairly. 64 | 65 | All community leaders are obligated to respect the privacy and security of the 66 | reporter of any incident. 67 | 68 | ## Enforcement Guidelines 69 | 70 | Community leaders will follow these Community Impact Guidelines in determining 71 | the consequences for any action they deem in violation of this Code of Conduct: 72 | 73 | ### 1. Correction 74 | 75 | **Community Impact**: Use of inappropriate language or other behavior deemed 76 | unprofessional or unwelcome in the community. 77 | 78 | **Consequence**: A private, written warning from community leaders, providing 79 | clarity around the nature of the violation and an explanation of why the 80 | behavior was inappropriate. A public apology may be requested. 81 | 82 | ### 2. Warning 83 | 84 | **Community Impact**: A violation through a single incident or series 85 | of actions. 86 | 87 | **Consequence**: A warning with consequences for continued behavior. No 88 | interaction with the people involved, including unsolicited interaction with 89 | those enforcing the Code of Conduct, for a specified period of time. This 90 | includes avoiding interactions in community spaces as well as external channels 91 | like social media. Violating these terms may lead to a temporary or 92 | permanent ban. 93 | 94 | ### 3. Temporary Ban 95 | 96 | **Community Impact**: A serious violation of community standards, including 97 | sustained inappropriate behavior. 98 | 99 | **Consequence**: A temporary ban from any sort of interaction or public 100 | communication with the community for a specified period of time. No public or 101 | private interaction with the people involved, including unsolicited interaction 102 | with those enforcing the Code of Conduct, is allowed during this period. 103 | Violating these terms may lead to a permanent ban. 104 | 105 | ### 4. Permanent Ban 106 | 107 | **Community Impact**: Demonstrating a pattern of violation of community 108 | standards, including sustained inappropriate behavior, harassment of an 109 | individual, or aggression toward or disparagement of classes of individuals. 110 | 111 | **Consequence**: A permanent ban from any sort of public interaction within 112 | the community. 113 | 114 | ## Attribution 115 | 116 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 117 | version 2.0, available at 118 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 119 | 120 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 121 | enforcement ladder](https://github.com/mozilla/diversity). 122 | 123 | [homepage]: https://www.contributor-covenant.org 124 | 125 | For answers to common questions about this code of conduct, see the FAQ at 126 | https://www.contributor-covenant.org/faq. Translations are available at 127 | https://www.contributor-covenant.org/translations. 128 | -------------------------------------------------------------------------------- /resources/gcp/iam.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 TikTok Pte. Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | ################################################################################# 18 | # IAM for dcr-cluster-sa 19 | ################################################################################# 20 | resource "google_project_iam_member" "dcr_cluster_sa_registry_reader" { 21 | project = var.project_id 22 | role = "roles/artifactregistry.reader" 23 | member = "serviceAccount:${google_service_account.gcp_dcr_cluster_sa.email}" 24 | } 25 | 26 | resource "google_project_iam_member" "dcr_cluster_sa_logger_writer" { 27 | project = var.project_id 28 | role = "roles/logging.logWriter" 29 | member = "serviceAccount:${google_service_account.gcp_dcr_cluster_sa.email}" 30 | } 31 | 32 | resource "google_project_iam_member" "dcr_cluster_sa_metric_writer" { 33 | project = var.project_id 34 | role = "roles/monitoring.metricWriter" 35 | member = "serviceAccount:${google_service_account.gcp_dcr_cluster_sa.email}" 36 | } 37 | 38 | resource "google_project_iam_member" "dcr_cluster_sa_metric_viewer" { 39 | project = var.project_id 40 | role = "roles/monitoring.viewer" 41 | member = "serviceAccount:${google_service_account.gcp_dcr_cluster_sa.email}" 42 | } 43 | 44 | ################################################################################# 45 | # IAM for dcr-cvm-sa 46 | ################################################################################# 47 | resource "google_project_iam_member" "cvm_sa_log_writter" { 48 | project = var.project_id 49 | role = "roles/logging.logWriter" 50 | member = "serviceAccount:${google_service_account.gcp_cvm_sa.email}" 51 | } 52 | 53 | resource "google_project_iam_member" "cvm_sa_cc_workload_user" { 54 | project = var.project_id 55 | role = "roles/confidentialcomputing.workloadUser" 56 | member = "serviceAccount:${google_service_account.gcp_cvm_sa.email}" 57 | } 58 | 59 | resource "google_project_iam_member" "cvm_sa_storage_admin" { 60 | project = var.project_id 61 | role = "roles/storage.objectAdmin" 62 | member = "serviceAccount:${google_service_account.gcp_cvm_sa.email}" 63 | } 64 | 65 | resource "google_project_iam_member" "cvm_sa_registry_reader" { 66 | project = var.project_id 67 | role = "roles/artifactregistry.reader" 68 | member = "serviceAccount:${google_service_account.gcp_cvm_sa.email}" 69 | } 70 | 71 | ################################################################################# 72 | # IAM for dcr-pod-sa 73 | ################################################################################# 74 | resource "google_project_iam_member" "dcr_pod_sa_storage_admin" { 75 | project = var.project_id 76 | role = "roles/storage.objectAdmin" 77 | member = "serviceAccount:${google_service_account.gcp_dcr_pod_sa.email}" 78 | } 79 | 80 | resource "google_project_iam_member" "dcr_pod_sa_sql_client" { 81 | project = var.project_id 82 | role = "roles/cloudsql.client" 83 | member = "serviceAccount:${google_service_account.gcp_dcr_pod_sa.email}" 84 | } 85 | 86 | resource "google_project_iam_member" "dcr_pod_sa_log_writer" { 87 | project = var.project_id 88 | role = "roles/logging.logWriter" 89 | member = "serviceAccount:${google_service_account.gcp_dcr_pod_sa.email}" 90 | } 91 | 92 | resource "google_project_iam_member" "dcr_pod_sa_instance_admin" { 93 | project = var.project_id 94 | role = "roles/compute.instanceAdmin.v1" 95 | member = "serviceAccount:${google_service_account.gcp_dcr_pod_sa.email}" 96 | } 97 | 98 | resource "google_project_iam_member" "dcr_pod_sa_sauser" { 99 | project = var.project_id 100 | role = "roles/iam.serviceAccountUser" 101 | member = "serviceAccount:${google_service_account.gcp_dcr_pod_sa.email}" 102 | } 103 | 104 | resource "google_kms_key_ring_iam_member" "dcr_pod_sa_keyring_admin" { 105 | key_ring_id = google_kms_key_ring.dcr_key_ring.id 106 | role = "roles/cloudkms.admin" 107 | member = "serviceAccount:${google_service_account.gcp_dcr_pod_sa.email}" 108 | } 109 | 110 | resource "google_project_iam_member" "dcr_pod_sa_wip_admin" { 111 | project = var.project_id 112 | role = "roles/iam.workloadIdentityPoolAdmin" 113 | member = "serviceAccount:${google_service_account.gcp_dcr_pod_sa.email}" 114 | } 115 | 116 | resource "google_service_account_iam_member" "dcr_pod_sa_iam_member" { 117 | service_account_id = google_service_account.gcp_dcr_pod_sa.id 118 | role = "roles/iam.workloadIdentityUser" 119 | member = "principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.dcr-workload-identity-pool.name}/*" 120 | } 121 | 122 | resource "google_project_iam_member" "dcr_pod_sa_repo_writer" { 123 | project = var.project_id 124 | role = "roles/artifactregistry.writer" 125 | member = "serviceAccount:${google_service_account.gcp_dcr_pod_sa.email}" 126 | } 127 | 128 | ################################################################################# 129 | # IAM for jupyter-pod-sa 130 | ################################################################################# 131 | 132 | resource "google_project_iam_member" "jupyter_pod_sa_logger_writer" { 133 | project = var.project_id 134 | role = "roles/logging.logWriter" 135 | member = "serviceAccount:${google_service_account.gcp_jupyter_pod_sa.email}" 136 | } 137 | 138 | resource "google_project_iam_member" "jupyter_pod_sa_metric_writer" { 139 | project = var.project_id 140 | role = "roles/monitoring.metricWriter" 141 | member = "serviceAccount:${google_service_account.gcp_jupyter_pod_sa.email}" 142 | } 143 | 144 | resource "google_project_iam_member" "jupyter_pod_sa_metric_viewer" { 145 | project = var.project_id 146 | role = "roles/monitoring.viewer" 147 | member = "serviceAccount:${google_service_account.gcp_jupyter_pod_sa.email}" 148 | } -------------------------------------------------------------------------------- /app/dcr_api/biz/service/job_service.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 TikTok Pte. Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package service 15 | 16 | import ( 17 | "context" 18 | "encoding/base64" 19 | "fmt" 20 | "io" 21 | 22 | "github.com/cloudwego/hertz/pkg/common/hlog" 23 | "github.com/google/uuid" 24 | "github.com/pkg/errors" 25 | "github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/app/dcr_api/biz/dal/db" 26 | "github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/app/dcr_api/biz/model/job" 27 | "github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pkg/cloud" 28 | "github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pkg/config" 29 | "github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pkg/errno" 30 | "github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pkg/utils" 31 | ) 32 | 33 | type JobService struct { 34 | ctx context.Context 35 | } 36 | 37 | // NewJobService create job service 38 | func NewJobService(ctx context.Context) *JobService { 39 | return &JobService{ctx: ctx} 40 | } 41 | 42 | func (js *JobService) SubmitJob(req *job.SubmitJobRequest, userWorkspace io.Reader) (string, error) { 43 | creator := req.Creator 44 | 45 | jobInList, err := db.GetInProgressJobs(creator) 46 | if err != nil { 47 | return "", err 48 | } else if len(jobInList) > 2 { 49 | return "", errors.Wrap(fmt.Errorf(errno.ReachJobLimitErrMsg), "") 50 | } 51 | 52 | provider := cloud.GetCloudProvider(js.ctx) 53 | err = provider.UploadFile(userWorkspace, config.GetUserWorkSpacePath(creator), false) 54 | if err != nil { 55 | return "", err 56 | } 57 | err = provider.PrepareResourcesForUser(creator) 58 | if err != nil { 59 | return "", err 60 | } 61 | 62 | uuidStr, err := uuid.NewUUID() 63 | if err != nil { 64 | return "", errors.Wrap(err, "failed to generate uuid") 65 | } 66 | t := db.Job{ 67 | UUID: uuidStr.String(), 68 | Creator: req.Creator, 69 | JupyterFileName: req.JupyterFileName, 70 | JobStatus: int(job.JobStatus_ImageBuilding), 71 | } 72 | err = BuildImage(js.ctx, t, req.AccessToken) 73 | if err != nil { 74 | return "", err 75 | } 76 | err = db.CreateJob(&t) 77 | 78 | if err != nil { 79 | return "", err 80 | } 81 | hlog.Infof("[JobService] inserted job. Job Status %+v", job.JobStatus_ImageBuilding) 82 | return uuidStr.String(), nil 83 | } 84 | 85 | func convertEntityToModel(j *db.Job) *job.Job { 86 | return &job.Job{ 87 | ID: int64(j.ID), 88 | UUID: j.UUID, 89 | Creator: j.Creator, 90 | JobStatus: job.JobStatus(j.JobStatus), 91 | JupyterFileName: j.JupyterFileName, 92 | CreatedAt: j.CreatedAt.Format(utils.Layout), 93 | UpdatedAt: j.UpdatedAt.Format(utils.Layout), 94 | } 95 | } 96 | 97 | func (js *JobService) QueryUsersJobs(req *job.QueryJobRequest) ([]*job.Job, int64, error) { 98 | jobs, total, err := db.QueryJobsByCreator(req.Creator, req.Page, req.PageSize) 99 | if err != nil { 100 | return nil, 0, err 101 | } 102 | res := []*job.Job{} 103 | for _, j := range jobs { 104 | res = append(res, convertEntityToModel(j)) 105 | } 106 | return res, total, nil 107 | } 108 | 109 | func (js *JobService) GetJobOutputAttrs(req *job.QueryJobOutputRequest) (string, int64, error) { 110 | j, err := db.QueryJobByIdAndCreator(req.ID, req.Creator) 111 | if err != nil { 112 | return "", 0, err 113 | } 114 | outputPath := config.GetJobOutputPath(j.Creator, j.UUID, j.JupyterFileName) 115 | 116 | provider := cloud.GetCloudProvider(js.ctx) 117 | size, err := provider.GetFileSize(outputPath) 118 | if err != nil { 119 | return "", 0, err 120 | } 121 | return config.GetJobOutputFilename(fmt.Sprintf("%v", j.ID), j.JupyterFileName), size, nil 122 | } 123 | 124 | func (js *JobService) DownloadJobOutput(req *job.DownloadJobOutputRequest) (string, error) { 125 | j, err := db.QueryJobByIdAndCreator(req.ID, req.Creator) 126 | if err != nil { 127 | return "", err 128 | } 129 | outputPath := config.GetJobOutputPath(j.Creator, j.UUID, j.JupyterFileName) 130 | provider := cloud.GetCloudProvider(js.ctx) 131 | datg, err := provider.GetFilebyChunk(outputPath, req.Offset, req.Chunk) 132 | if err != nil { 133 | return "", err 134 | } 135 | encoded := base64.StdEncoding.EncodeToString(datg) 136 | return encoded, nil 137 | } 138 | 139 | func (js *JobService) DeleteJob(req *job.DeleteJobRequest) { 140 | db.DeleteJob(req.Creator, req.UUID) 141 | } 142 | 143 | func (js *JobService) UpdateJob(req *job.UpdateJobStatusRequest) error { 144 | creator := req.Creator 145 | j, err := db.QueryJobByUUIDAndCreator(creator, req.UUID) 146 | if err != nil { 147 | return err 148 | } 149 | j.JobStatus = int(req.Status) 150 | if j.JobStatus == int(job.JobStatus_VMWaiting) { 151 | j.DockerImage = req.DockerImage 152 | j.DockerImageDigest = req.DockerImageDigest 153 | j.InstanceName = config.GetInstanceName(j.Creator, j.UUID) 154 | err := js.RunJob(js.ctx, j, req.AccessToken) 155 | if err != nil { 156 | return err 157 | } 158 | j.JobStatus = int(job.JobStatus_VMRunning) 159 | } 160 | if j.JobStatus == int(job.JobStatus_VMFinished) { 161 | j.AttestationReport = req.AttestationToken 162 | j.JobStatus = int(job.JobStatus_VMFinished) 163 | } 164 | err = db.UpdateJob(j) 165 | if err != nil { 166 | return err 167 | } 168 | return nil 169 | } 170 | 171 | func (js *JobService) RunJob(c context.Context, j *db.Job, token string) error { 172 | hlog.Infof("[JobSerive] docker image stored in DB, run the job. Job status: %+v", job.JobStatus_VMWaiting) 173 | provider := cloud.GetCloudProvider(js.ctx) 174 | provider.UpdateWorkloadIdentityPoolProvider(config.GetUserWipProvider(j.Creator), j.DockerImageDigest) 175 | err := provider.CreateConfidentialSpace(j.InstanceName, j.DockerImage, token, j.UUID) 176 | if err != nil { 177 | return err 178 | } 179 | return nil 180 | } 181 | 182 | func (js *JobService) GetJobAttestationReport(req *job.QueryJobAttestationRequest) (string, error) { 183 | j, err := db.QueryJobByIdAndCreator(req.ID, req.Creator) 184 | if err != nil { 185 | return "", err 186 | } 187 | if j.AttestationReport == "" { 188 | return "", errors.Wrap(fmt.Errorf("failed to query attestation for job %v", req.ID), "") 189 | } 190 | return j.AttestationReport, nil 191 | } 192 | -------------------------------------------------------------------------------- /app/dcr_monitor/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/app/dcr_monitor 2 | 3 | go 1.22.0 4 | 5 | require ( 6 | github.com/cloudwego/hertz v0.9.0 7 | github.com/pkg/errors v0.9.1 8 | github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pkg v0.0.1 9 | k8s.io/api v0.30.1 10 | k8s.io/apimachinery v0.30.1 11 | k8s.io/client-go v0.30.1 12 | ) 13 | 14 | require ( 15 | cloud.google.com/go v0.113.0 // indirect 16 | cloud.google.com/go/auth v0.4.1 // indirect 17 | cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect 18 | cloud.google.com/go/compute v1.27.0 // indirect 19 | cloud.google.com/go/compute/metadata v0.3.0 // indirect 20 | cloud.google.com/go/iam v1.1.8 // indirect 21 | cloud.google.com/go/kms v1.17.1 // indirect 22 | cloud.google.com/go/longrunning v0.5.7 // indirect 23 | cloud.google.com/go/storage v1.41.0 // indirect 24 | github.com/apache/thrift v0.13.0 // indirect 25 | github.com/bytedance/go-tagexpr/v2 v2.9.2 // indirect 26 | github.com/bytedance/gopkg v0.0.0-20220413063733-65bf48ffb3a7 // indirect 27 | github.com/bytedance/sonic v1.11.6 // indirect 28 | github.com/bytedance/sonic/loader v0.1.1 // indirect 29 | github.com/cloudwego/base64x v0.1.4 // indirect 30 | github.com/cloudwego/iasm v0.2.0 // indirect 31 | github.com/cloudwego/netpoll v0.5.0 // indirect 32 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 33 | github.com/emicklei/go-restful/v3 v3.11.0 // indirect 34 | github.com/felixge/httpsnoop v1.0.4 // indirect 35 | github.com/fsnotify/fsnotify v1.7.0 // indirect 36 | github.com/gabriel-vasile/mimetype v1.4.3 // indirect 37 | github.com/gin-contrib/sse v0.1.0 // indirect 38 | github.com/gin-gonic/gin v1.10.0 // indirect 39 | github.com/go-logr/logr v1.4.1 // indirect 40 | github.com/go-logr/stdr v1.2.2 // indirect 41 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 42 | github.com/go-openapi/jsonreference v0.20.2 // indirect 43 | github.com/go-openapi/swag v0.22.3 // indirect 44 | github.com/go-playground/locales v0.14.1 // indirect 45 | github.com/go-playground/universal-translator v0.18.1 // indirect 46 | github.com/go-playground/validator/v10 v10.20.0 // indirect 47 | github.com/go-sql-driver/mysql v1.7.0 // indirect 48 | github.com/goccy/go-json v0.10.2 // indirect 49 | github.com/gogo/protobuf v1.3.2 // indirect 50 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 51 | github.com/golang/protobuf v1.5.4 // indirect 52 | github.com/google/gnostic-models v0.6.8 // indirect 53 | github.com/google/gofuzz v1.2.0 // indirect 54 | github.com/google/s2a-go v0.1.7 // indirect 55 | github.com/google/uuid v1.6.0 // indirect 56 | github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect 57 | github.com/googleapis/gax-go/v2 v2.12.4 // indirect 58 | github.com/hashicorp/hcl v1.0.0 // indirect 59 | github.com/henrylee2cn/ameda v1.4.10 // indirect 60 | github.com/henrylee2cn/goutil v0.0.0-20210127050712-89660552f6f8 // indirect 61 | github.com/jinzhu/inflection v1.0.0 // indirect 62 | github.com/jinzhu/now v1.1.5 // indirect 63 | github.com/josharian/intern v1.0.0 // indirect 64 | github.com/json-iterator/go v1.1.12 // indirect 65 | github.com/klauspost/cpuid/v2 v2.2.7 // indirect 66 | github.com/leodido/go-urn v1.4.0 // indirect 67 | github.com/magiconair/properties v1.8.7 // indirect 68 | github.com/mailru/easyjson v0.7.7 // indirect 69 | github.com/mattn/go-isatty v0.0.20 // indirect 70 | github.com/mitchellh/mapstructure v1.5.0 // indirect 71 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 72 | github.com/modern-go/reflect2 v1.0.2 // indirect 73 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 74 | github.com/nyaruka/phonenumbers v1.0.55 // indirect 75 | github.com/pelletier/go-toml/v2 v2.2.2 // indirect 76 | github.com/sagikazarmark/locafero v0.4.0 // indirect 77 | github.com/sagikazarmark/slog-shim v0.1.0 // indirect 78 | github.com/sourcegraph/conc v0.3.0 // indirect 79 | github.com/spf13/afero v1.11.0 // indirect 80 | github.com/spf13/cast v1.6.0 // indirect 81 | github.com/spf13/pflag v1.0.5 // indirect 82 | github.com/spf13/viper v1.18.2 // indirect 83 | github.com/subosito/gotenv v1.6.0 // indirect 84 | github.com/tidwall/gjson v1.14.4 // indirect 85 | github.com/tidwall/match v1.1.1 // indirect 86 | github.com/tidwall/pretty v1.2.0 // indirect 87 | github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/app/dcr_api v0.0.0-00010101000000-000000000000 // indirect 88 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 89 | github.com/ugorji/go/codec v1.2.12 // indirect 90 | go.opencensus.io v0.24.0 // indirect 91 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect 92 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect 93 | go.opentelemetry.io/otel v1.24.0 // indirect 94 | go.opentelemetry.io/otel/metric v1.24.0 // indirect 95 | go.opentelemetry.io/otel/trace v1.24.0 // indirect 96 | go.uber.org/atomic v1.9.0 // indirect 97 | go.uber.org/multierr v1.9.0 // indirect 98 | golang.org/x/arch v0.8.0 // indirect 99 | golang.org/x/crypto v0.23.0 // indirect 100 | golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect 101 | golang.org/x/net v0.25.0 // indirect 102 | golang.org/x/oauth2 v0.20.0 // indirect 103 | golang.org/x/sync v0.7.0 // indirect 104 | golang.org/x/sys v0.20.0 // indirect 105 | golang.org/x/term v0.20.0 // indirect 106 | golang.org/x/text v0.15.0 // indirect 107 | golang.org/x/time v0.5.0 // indirect 108 | google.golang.org/api v0.180.0 // indirect 109 | google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda // indirect 110 | google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8 // indirect 111 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240509183442-62759503f434 // indirect 112 | google.golang.org/grpc v1.63.2 // indirect 113 | google.golang.org/protobuf v1.34.1 // indirect 114 | gopkg.in/inf.v0 v0.9.1 // indirect 115 | gopkg.in/ini.v1 v1.67.0 // indirect 116 | gopkg.in/yaml.v2 v2.4.0 // indirect 117 | gopkg.in/yaml.v3 v3.0.1 // indirect 118 | gorm.io/driver/mysql v1.5.6 // indirect 119 | gorm.io/gorm v1.25.10 // indirect 120 | k8s.io/klog/v2 v2.120.1 // indirect 121 | k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect 122 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect 123 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 124 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect 125 | sigs.k8s.io/yaml v1.3.0 // indirect 126 | ) 127 | 128 | replace github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pkg => /go/pkg/mod/github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pkg@v0.0.1 129 | 130 | replace github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/app/dcr_api => /go/pkg/mod/github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/app/dcr_api@v0.0.1 -------------------------------------------------------------------------------- /app/jupyterlab_manatee/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "jupyterlab_manatee", 3 | "version": "0.1.0", 4 | "description": "This is an open-source JupyterLab extension for data clean room", 5 | "keywords": [ 6 | "jupyter", 7 | "jupyterlab", 8 | "jupyterlab-extension" 9 | ], 10 | "license": "BSD-3-Clause", 11 | "author": { 12 | "name": "Dayeol Lee", 13 | "email": "dayeol.lee@tiktok.com" 14 | }, 15 | "files": [ 16 | "lib/**/*.{d.ts,eot,gif,html,jpg,js,js.map,json,png,svg,woff2,ttf}", 17 | "style/**/*.{css,js,eot,gif,html,jpg,json,png,svg,woff2,ttf}" 18 | ], 19 | "main": "lib/index.js", 20 | "types": "lib/index.d.ts", 21 | "style": "style/index.css", 22 | "scripts": { 23 | "build": "jlpm build:lib && jlpm build:labextension:dev", 24 | "build:prod": "jlpm clean && jlpm build:lib:prod && jlpm build:labextension", 25 | "build:labextension": "jupyter labextension build .", 26 | "build:labextension:dev": "jupyter labextension build --development True .", 27 | "build:lib": "tsc --sourceMap", 28 | "build:lib:prod": "tsc", 29 | "clean": "jlpm clean:lib", 30 | "clean:lib": "rimraf lib tsconfig.tsbuildinfo", 31 | "clean:lintcache": "rimraf .eslintcache .stylelintcache", 32 | "clean:labextension": "rimraf jupyterlab_manatee/labextension jupyterlab_manatee/_version.py", 33 | "clean:all": "jlpm clean:lib && jlpm clean:labextension && jlpm clean:lintcache", 34 | "eslint": "jlpm eslint:check --fix", 35 | "eslint:check": "eslint . --cache --ext .ts,.tsx", 36 | "install:extension": "jlpm build", 37 | "lint": "jlpm stylelint && jlpm prettier && jlpm eslint", 38 | "lint:check": "jlpm stylelint:check && jlpm prettier:check && jlpm eslint:check", 39 | "prettier": "jlpm prettier:base --write --list-different", 40 | "prettier:base": "prettier \"**/*{.ts,.tsx,.js,.jsx,.css,.json,.md}\"", 41 | "prettier:check": "jlpm prettier:base --check", 42 | "stylelint": "jlpm stylelint:check --fix", 43 | "stylelint:check": "stylelint --cache \"style/**/*.css\"", 44 | "test": "jest --coverage", 45 | "watch": "run-p watch:src watch:labextension", 46 | "watch:src": "tsc -w --sourceMap", 47 | "watch:labextension": "jupyter labextension watch ." 48 | }, 49 | "dependencies": { 50 | "@arco-design/web-react": "^2.57.1", 51 | "@jupyterlab/application": "^4.0.0" 52 | }, 53 | "devDependencies": { 54 | "@jupyterlab/builder": "^4.0.0", 55 | "@jupyterlab/testutils": "^4.0.0", 56 | "@types/jest": "^29.2.0", 57 | "@types/json-schema": "^7.0.11", 58 | "@types/react": "^18.0.26", 59 | "@types/react-addons-linked-state-mixin": "^0.14.22", 60 | "@typescript-eslint/eslint-plugin": "^6.1.0", 61 | "@typescript-eslint/parser": "^6.1.0", 62 | "css-loader": "^6.7.1", 63 | "eslint": "^8.36.0", 64 | "eslint-config-prettier": "^8.8.0", 65 | "eslint-plugin-prettier": "^5.0.0", 66 | "jest": "^29.2.0", 67 | "npm-run-all": "^4.1.5", 68 | "prettier": "^3.0.0", 69 | "rimraf": "^5.0.1", 70 | "source-map-loader": "^1.0.2", 71 | "style-loader": "^3.3.1", 72 | "stylelint": "^15.10.1", 73 | "stylelint-config-recommended": "^13.0.0", 74 | "stylelint-config-standard": "^34.0.0", 75 | "stylelint-csstree-validator": "^3.0.0", 76 | "stylelint-prettier": "^4.0.0", 77 | "typescript": "~5.0.2", 78 | "yjs": "^13.5.0" 79 | }, 80 | "sideEffects": [ 81 | "style/*.css", 82 | "style/index.js" 83 | ], 84 | "styleModule": "style/index.js", 85 | "publishConfig": { 86 | "access": "public" 87 | }, 88 | "jupyterlab": { 89 | "extension": true, 90 | "outputDir": "jupyterlab_manatee/labextension" 91 | }, 92 | "eslintIgnore": [ 93 | "node_modules", 94 | "dist", 95 | "coverage", 96 | "**/*.d.ts", 97 | "tests", 98 | "**/__tests__", 99 | "ui-tests" 100 | ], 101 | "eslintConfig": { 102 | "extends": [ 103 | "eslint:recommended", 104 | "plugin:@typescript-eslint/eslint-recommended", 105 | "plugin:@typescript-eslint/recommended", 106 | "plugin:prettier/recommended" 107 | ], 108 | "parser": "@typescript-eslint/parser", 109 | "parserOptions": { 110 | "project": "tsconfig.json", 111 | "sourceType": "module" 112 | }, 113 | "plugins": [ 114 | "@typescript-eslint" 115 | ], 116 | "rules": { 117 | "@typescript-eslint/naming-convention": [ 118 | "error", 119 | { 120 | "selector": "interface", 121 | "format": [ 122 | "PascalCase" 123 | ], 124 | "custom": { 125 | "regex": "^I[A-Z]", 126 | "match": true 127 | } 128 | } 129 | ], 130 | "@typescript-eslint/no-unused-vars": [ 131 | "warn", 132 | { 133 | "args": "none" 134 | } 135 | ], 136 | "@typescript-eslint/no-explicit-any": "off", 137 | "@typescript-eslint/no-namespace": "off", 138 | "@typescript-eslint/no-use-before-define": "off", 139 | "@typescript-eslint/quotes": [ 140 | "error", 141 | "single", 142 | { 143 | "avoidEscape": true, 144 | "allowTemplateLiterals": false 145 | } 146 | ], 147 | "curly": [ 148 | "error", 149 | "all" 150 | ], 151 | "eqeqeq": "error", 152 | "prefer-arrow-callback": "error" 153 | } 154 | }, 155 | "prettier": { 156 | "singleQuote": true, 157 | "trailingComma": "none", 158 | "arrowParens": "avoid", 159 | "endOfLine": "auto", 160 | "overrides": [ 161 | { 162 | "files": "package.json", 163 | "options": { 164 | "tabWidth": 4 165 | } 166 | } 167 | ] 168 | }, 169 | "stylelint": { 170 | "extends": [ 171 | "stylelint-config-recommended", 172 | "stylelint-config-standard", 173 | "stylelint-prettier/recommended" 174 | ], 175 | "plugins": [ 176 | "stylelint-csstree-validator" 177 | ], 178 | "rules": { 179 | "csstree/validator": true, 180 | "property-no-vendor-prefix": null, 181 | "selector-class-pattern": "^([a-z][A-z\\d]*)(-[A-z\\d]+)*$", 182 | "selector-no-vendor-prefix": null, 183 | "value-no-vendor-prefix": null 184 | } 185 | } 186 | } 187 | -------------------------------------------------------------------------------- /app/dcr_api/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/app/dcr_api 2 | 3 | go 1.22.0 4 | 5 | require ( 6 | github.com/apache/thrift v0.13.0 7 | github.com/cloudwego/hertz v0.9.0 8 | github.com/docker/docker v26.1.3+incompatible 9 | github.com/google/uuid v1.6.0 10 | github.com/pkg/errors v0.9.1 11 | github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pkg v0.0.1 12 | gorm.io/driver/mysql v1.5.6 13 | gorm.io/gorm v1.25.10 14 | k8s.io/api v0.30.1 15 | k8s.io/apimachinery v0.30.1 16 | k8s.io/client-go v0.30.1 17 | ) 18 | 19 | require ( 20 | cloud.google.com/go v0.113.0 // indirect 21 | cloud.google.com/go/auth v0.4.1 // indirect 22 | cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect 23 | cloud.google.com/go/compute v1.27.0 // indirect 24 | cloud.google.com/go/compute/metadata v0.3.0 // indirect 25 | cloud.google.com/go/iam v1.1.8 // indirect 26 | cloud.google.com/go/kms v1.17.1 // indirect 27 | cloud.google.com/go/longrunning v0.5.7 // indirect 28 | cloud.google.com/go/storage v1.41.0 // indirect 29 | github.com/Microsoft/hcsshim v0.12.3 // indirect 30 | github.com/bytedance/go-tagexpr/v2 v2.9.2 // indirect 31 | github.com/bytedance/gopkg v0.0.0-20220413063733-65bf48ffb3a7 // indirect 32 | github.com/bytedance/sonic v1.11.6 // indirect 33 | github.com/bytedance/sonic/loader v0.1.1 // indirect 34 | github.com/cloudwego/base64x v0.1.4 // indirect 35 | github.com/cloudwego/iasm v0.2.0 // indirect 36 | github.com/cloudwego/netpoll v0.5.0 // indirect 37 | github.com/containerd/containerd v1.7.17 // indirect 38 | github.com/containerd/log v0.1.0 // indirect 39 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 40 | github.com/emicklei/go-restful/v3 v3.11.0 // indirect 41 | github.com/felixge/httpsnoop v1.0.4 // indirect 42 | github.com/fsnotify/fsnotify v1.7.0 // indirect 43 | github.com/gabriel-vasile/mimetype v1.4.3 // indirect 44 | github.com/gin-contrib/sse v0.1.0 // indirect 45 | github.com/gin-gonic/gin v1.10.0 // indirect 46 | github.com/go-logr/logr v1.4.1 // indirect 47 | github.com/go-logr/stdr v1.2.2 // indirect 48 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 49 | github.com/go-openapi/jsonreference v0.20.2 // indirect 50 | github.com/go-openapi/swag v0.22.3 // indirect 51 | github.com/go-playground/locales v0.14.1 // indirect 52 | github.com/go-playground/universal-translator v0.18.1 // indirect 53 | github.com/go-playground/validator/v10 v10.20.0 // indirect 54 | github.com/go-sql-driver/mysql v1.7.0 // indirect 55 | github.com/goccy/go-json v0.10.2 // indirect 56 | github.com/gogo/protobuf v1.3.2 // indirect 57 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 58 | github.com/golang/protobuf v1.5.4 // indirect 59 | github.com/google/gnostic-models v0.6.8 // indirect 60 | github.com/google/gofuzz v1.2.0 // indirect 61 | github.com/google/s2a-go v0.1.7 // indirect 62 | github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect 63 | github.com/googleapis/gax-go/v2 v2.12.4 // indirect 64 | github.com/hashicorp/hcl v1.0.0 // indirect 65 | github.com/henrylee2cn/ameda v1.4.10 // indirect 66 | github.com/henrylee2cn/goutil v0.0.0-20210127050712-89660552f6f8 // indirect 67 | github.com/jinzhu/inflection v1.0.0 // indirect 68 | github.com/jinzhu/now v1.1.5 // indirect 69 | github.com/josharian/intern v1.0.0 // indirect 70 | github.com/json-iterator/go v1.1.12 // indirect 71 | github.com/klauspost/compress v1.17.0 // indirect 72 | github.com/klauspost/cpuid/v2 v2.2.7 // indirect 73 | github.com/leodido/go-urn v1.4.0 // indirect 74 | github.com/magiconair/properties v1.8.7 // indirect 75 | github.com/mailru/easyjson v0.7.7 // indirect 76 | github.com/mattn/go-isatty v0.0.20 // indirect 77 | github.com/mitchellh/mapstructure v1.5.0 // indirect 78 | github.com/moby/patternmatcher v0.6.0 // indirect 79 | github.com/moby/sys/sequential v0.5.0 // indirect 80 | github.com/moby/sys/user v0.1.0 // indirect 81 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 82 | github.com/modern-go/reflect2 v1.0.2 // indirect 83 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 84 | github.com/nyaruka/phonenumbers v1.0.55 // indirect 85 | github.com/pelletier/go-toml/v2 v2.2.2 // indirect 86 | github.com/sagikazarmark/locafero v0.4.0 // indirect 87 | github.com/sagikazarmark/slog-shim v0.1.0 // indirect 88 | github.com/sirupsen/logrus v1.9.3 // indirect 89 | github.com/sourcegraph/conc v0.3.0 // indirect 90 | github.com/spf13/afero v1.11.0 // indirect 91 | github.com/spf13/cast v1.6.0 // indirect 92 | github.com/spf13/pflag v1.0.5 // indirect 93 | github.com/spf13/viper v1.18.2 // indirect 94 | github.com/subosito/gotenv v1.6.0 // indirect 95 | github.com/tidwall/gjson v1.14.4 // indirect 96 | github.com/tidwall/match v1.1.1 // indirect 97 | github.com/tidwall/pretty v1.2.0 // indirect 98 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 99 | github.com/ugorji/go/codec v1.2.12 // indirect 100 | go.opencensus.io v0.24.0 // indirect 101 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect 102 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect 103 | go.opentelemetry.io/otel v1.24.0 // indirect 104 | go.opentelemetry.io/otel/metric v1.24.0 // indirect 105 | go.opentelemetry.io/otel/trace v1.24.0 // indirect 106 | go.uber.org/atomic v1.9.0 // indirect 107 | go.uber.org/multierr v1.9.0 // indirect 108 | golang.org/x/arch v0.8.0 // indirect 109 | golang.org/x/crypto v0.23.0 // indirect 110 | golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect 111 | golang.org/x/net v0.25.0 // indirect 112 | golang.org/x/oauth2 v0.20.0 // indirect 113 | golang.org/x/sync v0.7.0 // indirect 114 | golang.org/x/sys v0.20.0 // indirect 115 | golang.org/x/term v0.20.0 // indirect 116 | golang.org/x/text v0.15.0 // indirect 117 | golang.org/x/time v0.5.0 // indirect 118 | google.golang.org/api v0.180.0 // indirect 119 | google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda // indirect 120 | google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8 // indirect 121 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240509183442-62759503f434 // indirect 122 | google.golang.org/grpc v1.63.2 // indirect 123 | google.golang.org/protobuf v1.34.1 // indirect 124 | gopkg.in/inf.v0 v0.9.1 // indirect 125 | gopkg.in/ini.v1 v1.67.0 // indirect 126 | gopkg.in/yaml.v2 v2.4.0 // indirect 127 | gopkg.in/yaml.v3 v3.0.1 // indirect 128 | gotest.tools/v3 v3.5.1 // indirect 129 | k8s.io/klog/v2 v2.120.1 // indirect 130 | k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect 131 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect 132 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 133 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect 134 | sigs.k8s.io/yaml v1.3.0 // indirect 135 | ) 136 | 137 | replace github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pkg => /go/pkg/mod/github.com/tiktok-privacy-innovation/PrivacyGo-DataCleanRoom/pkg@v0.0.1 138 | 139 | replace github.com/apache/thrift => github.com/apache/thrift v0.13.0 140 | --------------------------------------------------------------------------------